Files
16f0d307dd fix(realtime): announce comments folded into a queued run (#9702)
A reply posted while the agent already has a queued run in the same thread is
folded into that run on the server: the reply becomes the run's trigger and the
earlier comment joins its covered set. The run's status does not change, so no
task event fired and open issue pages kept the run under the earlier comment,
leaving the new reply with no visible queue row.

Publish task:coalesced after the merge (and after a delegated-failure merge).
Web and desktop already refresh the issue's runs on any task: event, which moves
the queued block under the newest comment it answers; mobile now listens for it
too.

Co-authored-by: Cursor <[email protected]>
Co-authored-by: multica-agent <[email protected]>
2026-10-30 17:45:54 +08:00

8364 lines
364 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package service
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"unicode/utf8"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgtype"
"github.com/multica-ai/multica/server/internal/analytics"
"github.com/multica-ai/multica/server/internal/attribution"
"github.com/multica-ai/multica/server/internal/chattitle"
"github.com/multica-ai/multica/server/internal/entitlement"
"github.com/multica-ai/multica/server/internal/events"
"github.com/multica-ai/multica/server/internal/featureflags"
"github.com/multica-ai/multica/server/internal/inbox"
"github.com/multica-ai/multica/server/internal/issuestatus"
obsmetrics "github.com/multica-ai/multica/server/internal/metrics"
"github.com/multica-ai/multica/server/internal/realtime"
"github.com/multica-ai/multica/server/internal/runtimeapps"
"github.com/multica-ai/multica/server/internal/util"
db "github.com/multica-ai/multica/server/pkg/db/generated"
"github.com/multica-ai/multica/server/pkg/dbid"
"github.com/multica-ai/multica/server/pkg/featureflag"
"github.com/multica-ai/multica/server/pkg/protocol"
"github.com/multica-ai/multica/server/pkg/redact"
"github.com/multica-ai/multica/server/pkg/skillbundle"
"github.com/multica-ai/multica/server/pkg/taskfailure"
)
type TaskService struct {
// RecordMemberActivity enables the member_delegation_day ledger, which
// only feeds self-host telemetry. Off unless the server enables telemetry
// at startup, so DO_NOT_TRACK stops collection, not just delivery.
RecordMemberActivity bool
Queries *db.Queries
TxStarter TxStarter
Hub *realtime.Hub
Bus *events.Bus
Analytics analytics.Client
Metrics *obsmetrics.BusinessMetrics
Wakeup TaskWakeupNotifier
// PRSnapshotsEnabled reports whether the server can refresh GitHub check snapshots.
PRSnapshotsEnabled bool
// Entitlements supplies Cloud's workspace-scoped issue-count instruction.
// Nil keeps self-hosted and isolated test services unlimited.
Entitlements entitlement.Provider
// SourceContextStorage is used only by the bounded 30-day cleanup pass for
// terminal quick-create captures. Nil disables it where storage is absent.
SourceContextStorage SourceContextObjectStore
// FeatureFlags is the server-side toggle router. Nil is valid and returns
// each call site's default.
FeatureFlags *featureflag.Service
// EmptyClaim caches "this runtime has no queued task" so the daemon
// poll path can skip a Postgres scan on the steady-state empty case.
// Optional — a nil cache disables the fast path and every claim
// goes through the DB. Wired in router.go from the shared Redis
// client.
EmptyClaim *EmptyClaimCache
// ReclaimCheck schedules the next time a runtime can plausibly contain a
// stale dispatched task. It removes the unconditional reclaim UPDATE from
// idle claim polls while missing/error states preserve the DB fallback.
ReclaimCheck *ReclaimCheckCache
// Composio computes the per-task MCP overlay (Stage 3 of the Composio
// epic, MUL-3721) — the integration's "current user's connected apps
// → MCP session URL" hook called from each Enqueue* path. Optional: a
// nil ComposioOverlayBuilder turns the overlay step into a no-op so
// every Multica deployment that hasn't enabled Composio behaves
// exactly as before. Wired in router.go after composiointeg.NewService
// succeeds; the concrete type is *composio.Service.
Composio ComposioOverlayBuilder
// QuickActions generates chat follow-up suggestions through the
// server-internal LLM layer. Optional: nil (or a disabled client) turns the
// whole feature off — no pending marker, no pills — which is the expected
// state for a self-hosted deployment with no MULTICA_LLM_* configuration.
// Wired in router.go from the same *llm.Client that backs chat auto-titling.
QuickActions ChatQuickActionsLLM
// quickActionsInFlight (chat session id -> struct{}{}) and
// quickActionsRunning admit suggestion passes: one per session, and a
// process-wide ceiling. Both zero values are usable, so a TaskService built
// without NewTaskService still gates correctly rather than deadlocking or
// shedding everything.
quickActionsInFlight sync.Map
quickActionsRunning atomic.Int64
analyticsContextMu sync.Mutex
analyticsContextCache map[string]analytics.TaskContext
analyticsContextOrder []string
}
type SourceContextObjectStore interface {
DeleteObject(ctx context.Context, key string) error
KeyFromURL(rawURL string) string
}
// ComposioOverlayBuilder is the seam TaskService uses to build the per-task
// MCP overlay at enqueue time. Implemented by
// internal/integrations/composio.Service.BuildTaskOverlay; tests provide an
// inline fake so they don't have to spin a fake Composio SDK.
//
// Contract: a zero MCPOverlayResult means "no overlay for this run" — covers
// all gates the implementation enforces (no owner / empty allowlist / empty
// intersection with active connections / empty session URL). Any non-empty
// MCPOverlay is the exact value to store in agent_task_queue.runtime_mcp_overlay;
// ConnectedApps is non-secret metadata to store alongside it for daemon brief
// injection. A non-nil error is surfaced to the caller but treated as
// best-effort — failed overlay computation must not fail the enqueue.
//
// agent is passed by value so the builder can inspect OwnerID and
// ComposioToolkitAllowlist without re-querying the DB; every enqueue path
// already loaded the agent for runtime/archive checks, so passing it is
// free and avoids a second GetAgent round-trip in the hot path.
type ComposioOverlayBuilder interface {
BuildTaskOverlay(ctx context.Context, originatorUserID pgtype.UUID, agent db.Agent) (runtimeapps.MCPOverlayResult, error)
}
type TaskWakeupNotifier interface {
NotifyTaskAvailable(runtimeID, taskID string)
}
// triggerSummaryMaxLen caps the snapshot length so the row stays cheap to
// transmit (it ends up in every task list response). 200 is enough for a
// recognisable preview of a one-paragraph comment.
const triggerSummaryMaxLen = 200
// truncateForSummary returns s shortened to maxRunes, with a trailing
// `…` when truncated. Operates on runes (not bytes) so multibyte characters
// — Chinese / emoji — count as one each. Strips surrounding whitespace
// first so a leading newline doesn't waste budget.
func truncateForSummary(s string, maxRunes int) string {
// strings.Builder + Grow avoids the O(N²) realloc cycle of `+=` in
// a loop. Grow uses byte length, which is an upper bound for the
// rune-equivalent output (replacing \n/\r/\t with space is byte-equal
// for ASCII whitespace), so we never reallocate.
var b strings.Builder
b.Grow(len(s))
for _, r := range s {
switch r {
case '\n', '\r', '\t':
b.WriteByte(' ')
default:
b.WriteRune(r)
}
}
rs := []rune(strings.TrimSpace(b.String()))
if len(rs) <= maxRunes {
return string(rs)
}
return string(rs[:maxRunes]) + "…"
}
// maxSynthesizedFallbackCommentRunes bounds the completion-fallback comment that
// CompleteTask synthesizes from a task's final output when the agent left no
// comment of its own during the run. A real final assistant message is at most
// a few thousand words; anything larger is a runaway raw-stream dump — every
// streamed text delta concatenated together plus a literal `tool call` line per
// tool_use event — which some runtimes/providers emit as the task's Output on
// long, tool-heavy runs. Such a dump (observed at 190–264 KB) must never be
// posted, even partially, to the issue thread (GH #5455).
const maxSynthesizedFallbackCommentRunes = 8000
const oversizedFallbackCommentNotice = "This task completed, but its output was too large to post safely. The raw output was not posted. Review the task in this issue's Execution log."
// truncateFallbackCommentBody bounds a synthesized completion-fallback comment
// body. Unlike truncateForSummary (which flattens newlines for a one-line row
// snapshot), it preserves genuine final messages below the cap verbatim. Output
// above the cap is untrusted: the reported failure mode puts process narration
// and tool traces at the head, so retaining any excerpt can expose execution
// details and still discard the final answer. Replace the entire body with a
// fixed notice instead. Callers pass the already-redacted body.
func truncateFallbackCommentBody(body string, maxRunes int) string {
if utf8.RuneCountInString(body) <= maxRunes {
return body
}
return oversizedFallbackCommentNotice
}
const (
taskAnalyticsContextCacheMax = 4096
// RuntimeClaimFreshnessSeconds is the maximum DB heartbeat age accepted by
// every task release path (deferred promotion, stale-dispatch reclaim, and
// fresh claim). It must exceed the 60s DB heartbeat flush interval, one ~15s
// daemon heartbeat, and the ~30s batch scheduler tick. 150s leaves a 45s
// buffer above that 105s worst-case age. Keep the runtime sweeper threshold
// aligned: a runtime must not start work after it is stale enough to be
// marked offline.
RuntimeClaimFreshnessSeconds = 150.0
// claimResponseRecoveryWindow must exceed daemon client.Timeout for
// /tasks/claim (30s) plus /tasks/{id}/start (30s) plus scheduling slack.
// Longer pre-start work is protected by prepareLeaseDuration instead of
// stretching this global crash-recovery window.
claimResponseRecoveryWindow = 90 * time.Second
prepareLeaseDuration = 45 * time.Second
)
func (s *TaskService) trackTaskForReclaim(task db.AgentTaskQueue, checkAfter time.Time) {
if !task.RuntimeID.Valid || !task.ID.Valid || task.Status != "dispatched" {
return
}
// The Redis hint is advisory and intentionally uses an application-clock
// elapsed deadline captured beside the DB command. Comparing PostgreSQL's
// absolute dispatched_at/lease timestamps with time.Now would mix clocks;
// PostgreSQL remains authoritative when the reclaim query actually runs.
// A bounded background context lets this best-effort write outlive a request
// cancellation without allowing Redis to stall the hot path indefinitely.
s.ReclaimCheck.Track(
context.Background(),
util.UUIDToString(task.RuntimeID),
util.UUIDToString(task.ID),
checkAfter,
)
}
func (s *TaskService) extendTaskReclaimHint(task db.AgentTaskQueue, checkAfter time.Time) {
if !task.RuntimeID.Valid || !task.ID.Valid || task.Status != "dispatched" {
return
}
// Preserve a later initial recovery deadline; a short prepare lease should
// only move the hint when repeated extensions actually protect the task for
// longer. See trackTaskForReclaim for the clock/context rationale.
s.ReclaimCheck.TrackLater(
context.Background(),
util.UUIDToString(task.RuntimeID),
util.UUIDToString(task.ID),
checkAfter,
)
}
func (s *TaskService) forgetTaskReclaim(task db.AgentTaskQueue) {
if !task.RuntimeID.Valid || !task.ID.Valid {
return
}
// This cleanup is best-effort and uses the cache's bounded timeout; request
// cancellation must not make a committed task transition leave a stale hint.
s.ReclaimCheck.Forget(
context.Background(),
util.UUIDToString(task.RuntimeID),
util.UUIDToString(task.ID),
)
}
// buildCommentTriggerSummary fetches the comment content and truncates
// it for storage on the task row. Returns an invalid pgtype.Text when
// the comment is missing (deleted / wrong workspace / etc) so the column
// stays NULL — front-end falls back to a structural label in that case.
//
// workspaceID scopes the fetch to the task's own workspace: the summary is
// later returned in claim / task-history responses, so a foreign comment UUID
// reaching an enqueue/merge path must NOT leak another workspace's text even in
// truncated form (MUL-4252).
func (s *TaskService) buildCommentTriggerSummary(ctx context.Context, workspaceID, commentID pgtype.UUID) pgtype.Text {
if !commentID.Valid {
return pgtype.Text{}
}
comment, err := s.Queries.GetCommentInWorkspace(ctx, db.GetCommentInWorkspaceParams{
ID: commentID,
WorkspaceID: workspaceID,
})
if err != nil {
return pgtype.Text{}
}
summary := truncateForSummary(comment.Content, triggerSummaryMaxLen)
if summary == "" {
return pgtype.Text{}
}
return pgtype.Text{String: summary, Valid: true}
}
// ResolveOriginatorFromTriggerComment is the exported wrapper used by the
// comment-merge path (MUL-4195) to compute the top-of-chain human originator
// for a newly-arrived comment, so a merge can be gated on the originator being
// unchanged. workspaceID scopes the comment lookup to the task's workspace
// (MUL-4252). See resolveOriginatorFromTriggerComment for the chain rules.
func (s *TaskService) ResolveOriginatorFromTriggerComment(ctx context.Context, workspaceID, commentID pgtype.UUID) pgtype.UUID {
return s.resolveOriginatorFromTriggerComment(ctx, workspaceID, commentID)
}
// AttributionForMergedComment resolves the FULL attribution snapshot for a comment
// being coalesced into an already-queued task (MUL-4302). A merge re-attributes the
// run to the newly-arrived comment's human, so the whole snapshot — source, evidence,
// delegation lineage, and both person columns — must move together as one
// attribution.Result; re-stamping only the person columns would leave a run showing
// B accountable while still pointing at A's stale source / evidence / level. isMention
// picks the agent-authored label (delegation for a mention / thread-parent, otherwise
// comment_source), matching the fresh-enqueue routing.
//
// The merge re-opens the same fail-closed decision the original enqueue faced: a merge
// swaps the effective trigger, responsible human, and evidence to the NEW comment, so
// "the enqueue already checked" does not carry over. It runs the comment through
// applyAttributionFallback — the identical fail-closed gate the fresh-enqueue path uses
// — and returns ErrAttributionFailClosed when the new comment cannot be attributed
// precisely and the workspace forbids the owner_fallback degrade. The caller must then
// REFUSE the merge and keep the original (precisely-attributed) task snapshot rather
// than re-stamp a queued run to a degraded owner_fallback (Elon must-fix).
func (s *TaskService) AttributionForMergedComment(ctx context.Context, workspaceID, commentID pgtype.UUID, isMention bool, agent db.Agent) (attribution.Result, error) {
agentAuthoredSource := attribution.SourceCommentSource
if isMention {
agentAuthoredSource = attribution.SourceDelegation
}
attr := s.attributionFromTriggerComment(ctx, workspaceID, commentID, agentAuthoredSource)
return s.applyAttributionFallback(ctx, attr, agent)
}
// BuildCommentTriggerSummary is the exported wrapper used by the comment-merge
// path (MUL-4195) to refresh a coalesced task's trigger_summary to the newest
// trigger comment's snapshot. workspaceID scopes the lookup (MUL-4252).
func (s *TaskService) BuildCommentTriggerSummary(ctx context.Context, workspaceID, commentID pgtype.UUID) pgtype.Text {
return s.buildCommentTriggerSummary(ctx, workspaceID, commentID)
}
// BuildRuntimeMCPOverlayForMerge recomputes the Composio MCP overlay +
// connected-app metadata for (originatorUserID, agent), used when a merge
// re-stamps a coalesced task's originator (MUL-4195 review must-fix #1). The
// overlay is a pure function of (originator, agent); re-stamping it alongside
// originator_user_id keeps the coalescing run's connected-app capabilities and
// audit attribution consistent with the latest trigger comment's originator
// instead of the task's original one. Fails soft to empty (same as the enqueue
// path) so a transient Composio hiccup never blocks the merge.
func (s *TaskService) BuildRuntimeMCPOverlayForMerge(ctx context.Context, originatorUserID pgtype.UUID, agent db.Agent) (overlay, connectedApps []byte) {
data := s.buildRuntimeMCPOverlay(ctx, originatorUserID, agent)
return data.Overlay, data.ConnectedApps
}
func NewTaskService(q *db.Queries, tx TxStarter, hub *realtime.Hub, bus *events.Bus, wakeups ...TaskWakeupNotifier) *TaskService {
var wakeup TaskWakeupNotifier
if len(wakeups) > 0 {
wakeup = wakeups[0]
}
return &TaskService{Queries: q, TxStarter: tx, Hub: hub, Bus: bus, Wakeup: wakeup}
}
var trivialDoneMarkers = []string{
"done",
"готово",
"готова",
"сделано",
"完成",
"完了",
}
func isTrivialDoneOutput(output string) bool {
normalized := strings.TrimSpace(strings.ToLower(output))
normalized = strings.Trim(normalized, ".!!。… ")
for _, marker := range trivialDoneMarkers {
if normalized == marker {
return true
}
}
return false
}
func (s *TaskService) captureTaskQueued(ctx context.Context, task db.AgentTaskQueue) {
if s.Metrics != nil {
source, runtimeMode, _ := s.taskMetricsContext(ctx, task)
s.Metrics.RecordTaskEnqueued(source, runtimeMode)
}
}
type runtimeMCPOverlayData struct {
Overlay json.RawMessage
ConnectedApps json.RawMessage
}
// buildRuntimeMCPOverlay computes the optional per-task Composio MCP overlay.
// Enqueue paths call this BEFORE inserting the queued row so the daemon cannot
// claim a task during the network round-trip to Composio and miss the overlay.
func (s *TaskService) buildRuntimeMCPOverlay(ctx context.Context, originatorUserID pgtype.UUID, agent db.Agent) runtimeMCPOverlayData {
if s == nil || s.Composio == nil {
return runtimeMCPOverlayData{}
}
if !featureflags.ComposioMCPAppsEnabled(ctx, s.FeatureFlags) {
return runtimeMCPOverlayData{}
}
result, err := s.Composio.BuildTaskOverlay(ctx, originatorUserID, agent)
if err != nil {
slog.Warn("runtime mcp overlay: BuildTaskOverlay failed; task will run without composio overlay",
"originator_user_id", util.UUIDToString(originatorUserID),
"agent_id", util.UUIDToString(agent.ID),
"error", err,
)
return runtimeMCPOverlayData{}
}
if len(result.MCPOverlay) == 0 {
slog.Debug("runtime mcp overlay: no composio overlay for task",
"originator_user_id", util.UUIDToString(originatorUserID),
"agent_id", util.UUIDToString(agent.ID),
)
return runtimeMCPOverlayData{}
}
data := runtimeMCPOverlayData{Overlay: result.MCPOverlay}
if len(result.ConnectedApps) > 0 {
raw, err := json.Marshal(result.ConnectedApps)
if err != nil {
slog.Warn("runtime mcp overlay: marshal connected app metadata failed",
"originator_user_id", util.UUIDToString(originatorUserID),
"agent_id", util.UUIDToString(agent.ID),
"error", err,
)
return data
}
data.ConnectedApps = raw
}
return data
}
// resolveOriginatorFromTriggerComment returns the top-of-chain HUMAN user
// id for a comment that triggered an Enqueue* path. The chain rules
// (MUL-3869):
//
// - trigger comment authored by a member → originator = author_id (that
// member IS the top-of-chain human).
// - trigger comment authored by an agent → read the parent task via
// comment.source_task_id and inherit its originator_user_id. This is
// the load-bearing case for agent fan-out: agent A @-mentions agent B,
// comment author is A, but we MUST surface the human who originally
// told A to run, not lose the originator at the first agent hop.
// - missing comment / unknown source task / NULL parent originator →
// invalid pgtype.UUID. BuildTaskOverlay treats that as "no overlay"
// (gate 1).
//
// A nil receiver / nil Queries falls through to invalid so unit-test
// setups that don't wire a DB stay safe. workspaceID scopes the comment lookup
// to the task's workspace so a foreign comment UUID cannot resolve an
// originator from another tenant (MUL-4252).
func (s *TaskService) resolveOriginatorFromTriggerComment(ctx context.Context, workspaceID, commentID pgtype.UUID) pgtype.UUID {
// The originator VALUE is independent of the agent-authored source label, so
// any label works here; comment_source is passed only as a placeholder.
return s.attributionFromTriggerComment(ctx, workspaceID, commentID, attribution.SourceCommentSource).UserID
}
// attributionFromTriggerComment resolves the full attribution (accountable
// human + provenance label + delegation lineage + evidence) for a
// comment-triggered run. It performs the DB reads and hands the gathered facts
// to the pure attribution.ClassifyComment rules so the classification stays
// side-effect-free and unit-tested. The returned UserID is byte-identical to
// the pre-MUL-4302 originator resolution, so authorization behavior (Composio
// overlay, canInvokeAgent A2A gate) is unchanged. workspaceID scopes the comment
// lookup to the task's workspace (MUL-4252).
//
// agentAuthoredSource selects the label for an agent-authored trigger comment:
// attribution.SourceCommentSource for the issue-assignee-reacting path,
// attribution.SourceDelegation for an explicit mention / thread-parent /
// squad-leader path.
func (s *TaskService) attributionFromTriggerComment(ctx context.Context, workspaceID, commentID pgtype.UUID, agentAuthoredSource attribution.Source) attribution.Result {
if s == nil || s.Queries == nil || !commentID.Valid {
return attribution.Result{Source: attribution.SourceUnattributed}
}
comment, err := s.Queries.GetCommentInWorkspace(ctx, db.GetCommentInWorkspaceParams{
ID: commentID,
WorkspaceID: workspaceID,
})
if err != nil {
return attribution.Result{Source: attribution.SourceUnattributed}
}
return s.attributionFromComment(ctx, comment, agentAuthoredSource)
}
// attributionFromComment classifies a run from an already-loaded trigger comment,
// so a caller that already has the row (e.g. to inspect author_type) does not
// re-read it. Kept byte-identical to the inline logic attributionFromTriggerComment
// used before, so authorization behavior is unchanged.
func (s *TaskService) attributionFromComment(ctx context.Context, comment db.Comment, agentAuthoredSource attribution.Source) attribution.Result {
facts := attribution.CommentFacts{
CommentID: comment.ID,
AuthorType: comment.AuthorType,
AuthorID: comment.AuthorID,
CreatedAt: comment.CreatedAt,
}
// For an agent-authored comment, walk comment.source_task_id → parent task →
// parent.originator_user_id (set by every agent comment-write path since
// migration 120). A NULL/missing source task leaves ParentOriginator
// invalid, which ClassifyComment maps to unattributed.
if comment.AuthorType == "agent" && comment.SourceTaskID.Valid {
facts.SourceTaskID = comment.SourceTaskID
if parent, err := s.Queries.GetAgentTask(ctx, comment.SourceTaskID); err == nil {
facts.ParentOriginator = parent.OriginatorUserID
facts.ParentAccountable = parent.AccountableUserID
}
}
return attribution.ClassifyComment(facts, agentAuthoredSource)
}
// resolveOriginatorForIssueTask returns the top-of-chain human for issue-backed
// dispatches. Comment-triggered runs keep the existing comment-chain semantics;
// direct issue assignment/creation falls back to the issue's member creator.
// Agent-created issues that carry an explicit task-origin link — quick_create
// (daemon quick-create flow) or agent_create (an agent's ordinary `issue
// create`, MUL-4305) — inherit that origin task's originator, since origin_id
// points at the agent_task_queue row that created the issue. Other
// agent/system origins, including autopilot, deliberately remain unattributed.
func (s *TaskService) resolveOriginatorForIssueTask(ctx context.Context, issue db.Issue, triggerCommentID pgtype.UUID) pgtype.UUID {
return s.attributionForIssueTask(ctx, issue, triggerCommentID, attribution.SourceCommentSource, pgtype.UUID{}).UserID
}
// attributionForIssueTask resolves the full attribution for an issue-backed
// enqueue. Comment-triggered runs keep the comment-chain semantics; direct
// assignment/creation falls back to the issue's member creator; agent-created
// quick-create issues inherit the origin task's human as a delegation. The
// accountable-human value is byte-identical to resolveOriginatorForIssueTask,
// which now delegates here — so there is a single source of truth and
// authorization is unaffected. agentAuthoredSource labels the agent-authored
// trigger comment case (see attributionFromTriggerComment).
func (s *TaskService) attributionForIssueTask(ctx context.Context, issue db.Issue, triggerCommentID pgtype.UUID, agentAuthoredSource attribution.Source, actorUserID pgtype.UUID) attribution.Result {
// A direct member action is the accountable human AND originator, ahead of any
// trigger comment, origin, or rule (MUL-4302 §4/§5). This covers assign/promote,
// a manual autopilot trigger, and a manual rerun — the last of which may INHERIT
// a triggerCommentID for the daemon's prompt context, but must still attribute to
// the member who clicked rerun, not the original comment's human. So the actor is
// checked before the trigger-comment / origin branches.
if actorUserID.Valid {
return attribution.ClassifyDirect(attribution.DirectFacts{IssueID: issue.ID, ActorUserID: actorUserID})
}
if triggerCommentID.Valid {
if s == nil || s.Queries == nil {
return attribution.Result{Source: attribution.SourceUnattributed}
}
// workspace-scoped so a foreign comment UUID cannot resolve a human from
// another tenant (MUL-4252).
comment, err := s.Queries.GetCommentInWorkspace(ctx, db.GetCommentInWorkspaceParams{
ID: triggerCommentID,
WorkspaceID: issue.WorkspaceID,
})
if err != nil {
return attribution.Result{Source: attribution.SourceUnattributed}
}
// A member/agent trigger comment resolves the human (direct_human / delegation
// / comment_source). A SYSTEM-authored comment — historically the
// Stage-completion child-done comment, which woke the parent assignee
// and threaded no actor — carries no human and is not part of any delegation
// chain. Classifying it would degrade straight to owner_fallback (the agent's
// own owner), which is wrong for a Stage cascade: the woken run should be
// accountable to whoever caused the PARENT issue to exist. So for a system
// comment we skip the comment branch and fall through to the parent issue's
// own provenance below — the same creator / agent_create-origin /
// autopilot-origin chain a direct enqueue resolves — reaching owner_fallback
// only if that provenance itself has no human (MUL-4302; raised by Bohan on
// the stage-cascade fallback).
if comment.AuthorType != "system" {
return s.attributionFromComment(ctx, comment, agentAuthoredSource)
}
}
// Autopilot-origin issues (origin_id is the autopilot id) from a schedule /
// webhook trigger attribute to the firing trigger's persisted created_by
// principal — trigger_owner (MUL-4302; MUL-6951; legacy semantics in
// ResolveAutopilotTriggerPrincipal) — degrading to the audit-only rule publisher
// when the trigger has none. That human is the originator as well as the
// accountable, so a create_issue-mode run carries the same authorization a
// manual "run now" by that member would; an edit of the trigger does not move
// it. Resolved the same way
// run_only dispatch resolves it, so both autopilot execution modes attribute
// identically. (A manual trigger carries an actor and is already handled above.)
// The issue only stores the autopilot id, so bridge issue → latest run →
// trigger_id, including terminal runs so later handoffs retain the principal.
if s != nil && s.Queries != nil && issue.OriginType.Valid &&
issue.OriginType.String == "autopilot" && issue.OriginID.Valid {
var triggerID pgtype.UUID
if run, err := s.Queries.GetLatestAutopilotRunForIssueAttribution(ctx, db.GetLatestAutopilotRunForIssueAttributionParams{
IssueID: issue.ID,
AutopilotID: issue.OriginID,
WorkspaceID: issue.WorkspaceID,
}); err == nil {
triggerID = run.TriggerID
}
return triggerOwnerAttribution(ctx, s.Queries, triggerID, issue.WorkspaceID, issue.OriginID, attribution.EvidenceIssueAssignment, issue.ID)
}
facts := attribution.DirectFacts{
IssueID: issue.ID,
CreatorType: issue.CreatorType,
CreatorID: issue.CreatorID,
}
// Member-created issues resolve without a DB read. Only origin-linked
// agent-created issues (quick_create, agent_create) need to load the origin
// task to inherit its human, and only when the DB is wired (nil Queries keeps
// unit-test setups safe and yields unattributed). Both origin types stamp
// origin_id with the agent_task_queue row that created the issue, so the
// top-of-chain human is that task's originator_user_id (MUL-4305).
if !(issue.CreatorType == "member" && issue.CreatorID.Valid) &&
s != nil && s.Queries != nil && issue.OriginType.Valid && issue.OriginID.Valid &&
(issue.OriginType.String == "quick_create" || issue.OriginType.String == "agent_create") {
facts.OriginType = issue.OriginType.String
facts.OriginTaskID = issue.OriginID
if task, err := s.Queries.GetAgentTask(ctx, issue.OriginID); err == nil {
facts.OriginOriginator = task.OriginatorUserID
facts.OriginAccountable = task.AccountableUserID
}
}
return attribution.ClassifyDirect(facts)
}
// ruleOwnerAttribution resolves the rule_owner attribution for an autopilot run
// from its active (latest) rule version snapshot (MUL-4302 §3.4). Shared by both
// autopilot execution modes — run_only dispatch and the create_issue enqueue path —
// so they attribute identically. originator stays NULL: an autopilot DOES carry a
// human's authority since MUL-6951, but it comes from the trigger's created_by
// principal (see ResolveAutopilotTriggerPrincipal), and this is the fallback for a
// trigger that has none. Only the
// audit-accountable side is set, to the version's member publisher. A missing version (autopilot published before this feature, or
// none yet) or a non-member/absent publisher degrades to unattributed rather than
// fabricating a human. Never returns an error: attribution must not fail an
// enqueue, and a degraded label is the honest fallback.
func ruleOwnerAttribution(ctx context.Context, q *db.Queries, workspaceID, autopilotID pgtype.UUID, evidenceKind attribution.EvidenceKind, evidenceRefID pgtype.UUID) attribution.Result {
if q == nil || !autopilotID.Valid {
return attribution.RuleOwner(pgtype.UUID{}, pgtype.UUID{}, evidenceKind, evidenceRefID)
}
ver, err := q.GetActiveAutopilotRuleVersion(ctx, db.GetActiveAutopilotRuleVersionParams{
WorkspaceID: workspaceID,
AutopilotID: autopilotID,
})
if err != nil {
return attribution.RuleOwner(pgtype.UUID{}, pgtype.UUID{}, evidenceKind, evidenceRefID)
}
var publisher pgtype.UUID
if ver.PublishedByType == "member" {
publisher = ver.PublishedByID
}
return attribution.RuleOwner(publisher, ver.ID, evidenceKind, evidenceRefID)
}
// triggerOwnerAttribution resolves an autopilot schedule/webhook run to the firing
// trigger's dispatch principal, created_by (MUL-4302; MUL-6951) — see
// ResolveAutopilotTriggerPrincipal for what that column does and does not prove.
// triggerID is the autopilot_run's trigger_id.
//
// No edit rewrites created_by — a substantive edit re-stamps published_by — so it
// cannot re-authorize the automation as the editor (MUL-6951, Bohan's ruling).
// Because the DB invariant forces accountable == originator once the originator is
// set, BOTH columns on the task name that principal; the editor's responsibility
// for the config lives on autopilot_trigger.published_by.
//
// A trigger with no principal degrades to ruleOwnerAttribution, which is
// audit-only — the run then carries no originator and the invoke gate fails closed.
// Never errors: attribution must not fail an enqueue.
func triggerOwnerAttribution(ctx context.Context, q *db.Queries, triggerID, workspaceID, autopilotID pgtype.UUID, evidenceKind attribution.EvidenceKind, evidenceRefID pgtype.UUID) attribution.Result {
if principal := ResolveAutopilotTriggerPrincipal(ctx, q, triggerID, autopilotID, workspaceID); principal.Valid {
return attribution.TriggerOwner(principal, evidenceKind, evidenceRefID)
}
// No principal: degrade to the rule publisher, which is AUDIT-ONLY.
// rule_owner must never become an authorization identity — it is a guess at
// "who probably owns this rule", and promoting it would hand a legacy trigger
// somebody's invoke rights without that person ever arming anything. The run
// then carries no originator and the invoke gate fails closed (MUL-6951).
return ruleOwnerAttribution(ctx, q, workspaceID, autopilotID, evidenceKind, evidenceRefID)
}
// ResolveAutopilotTriggerPrincipal returns the human a schedule/webhook dispatch
// ACTS AS — the trigger's created_by — or an invalid UUID when there is none, in
// which case every caller must fail closed rather than substitute a different
// human.
//
// This is the single source of that answer (MUL-6951). Admission
// (autopilotAdmitInvoke), the originator stamped on the task, and every run
// delegated from it all resolve through here, so one dispatch can never admit as
// person A and then run with person B's rights — a combination neither of them
// could produce by hand, and the exact fork Elon's review found.
//
// The principal is the trigger's created_by, not published_by: published_by
// transfers to whoever last substantively edits the trigger, so using it would let
// a collaborator adjusting a cron expression silently hand the automation their
// own rights (MUL-6951, Bohan's ruling: the run always acts as the trigger's
// creator).
//
// What created_by records depends on the trigger's age. For a trigger created
// since MUL-6951 it is the member who created it, written at creation. For a legacy
// trigger it is a principal inferred once by backfill and frozen — the last
// publisher (migration 449), else the autopilot's creator (migration 467) — so it
// is NOT proof of who created that trigger; treat it only as the dispatch
// principal. Nothing here infers a principal at dispatch time, and no edit
// rewrites it.
//
// Three conditions, all required, all fail-closed:
//
// - the trigger row is fetched BOUND to this autopilot AND its workspace, so a
// trigger id from another autopilot, or from an autopilot in another tenant,
// cannot select the principal. The membership check below is not a substitute:
// it proves the resolved human is in the workspace passed in, which a member of
// two workspaces satisfies even when the trigger came from the other one;
// - created_by names a member — a legacy trigger that neither backfill could
// fill resolves nobody;
// - that member is STILL in the autopilot's workspace, re-checked on every
// dispatch, so removing someone actually revokes what their triggers can do.
func ResolveAutopilotTriggerPrincipal(ctx context.Context, q *db.Queries, triggerID, autopilotID, workspaceID pgtype.UUID) pgtype.UUID {
if q == nil || !triggerID.Valid || !autopilotID.Valid || !workspaceID.Valid {
return pgtype.UUID{}
}
trig, err := q.GetAutopilotTriggerForAutopilot(ctx, db.GetAutopilotTriggerForAutopilotParams{
ID: triggerID,
AutopilotID: autopilotID,
WorkspaceID: workspaceID,
})
if err != nil {
return pgtype.UUID{}
}
if !trig.CreatedByType.Valid || trig.CreatedByType.String != "member" || !trig.CreatedByID.Valid {
return pgtype.UUID{}
}
if _, err := q.GetMemberByUserAndWorkspace(ctx, db.GetMemberByUserAndWorkspaceParams{
UserID: trig.CreatedByID,
WorkspaceID: workspaceID,
}); err != nil {
return pgtype.UUID{}
}
return trig.CreatedByID
}
// ErrAttributionFailClosed signals that a run resolved to no precise accountable
// human and the enqueue is REFUSED rather than started. It covers three cases, all
// of which mean "we cannot guarantee an accountable human for this run" (MUL-4302
// §1/§3.5): the workspace opted into fail-closed; the workspace policy could not be
// read (so we cannot confirm fallback is allowed — fail closed, don't run); or
// owner_fallback has no agent owner to fall back to. Enqueue paths surface it so the
// run never starts.
var ErrAttributionFailClosed = errors.New("attribution: no precise accountable human and enqueue refused (fail-closed policy, policy read failed, or no agent owner)")
// ErrDuplicatePendingTask means a fresh enqueue lost the race to a concurrent
// one: a queued/dispatched task for the same (issue, agent) already exists, so
// the pending-task unique index rejected the insert (#5914). This is a benign
// outcome — a sibling run already covers this target
// — not a server fault. Enqueue paths return it so callers can report a
// success-shaped coalesced outcome / structured 409 instead of surfacing the
// raw Postgres constraint as a 500. It is returned BARE (the raw driver text,
// including the index name, is logged once at debug and never wrapped in) so no
// upper-layer log or response can leak the constraint name (#5914, Elon review).
var ErrDuplicatePendingTask = errors.New("a pending task for this issue and agent already exists")
// isDuplicatePendingTaskErr reports whether err is the pending-task unique-index
// violation (a concurrent enqueue won the race). Accept both names while v1 and
// v2 can coexist during a rolling deploy.
func isDuplicatePendingTaskErr(err error) bool {
var pgErr *pgconn.PgError
if !errors.As(err, &pgErr) || pgErr.Code != "23505" {
return false
}
switch pgErr.ConstraintName {
case "idx_one_pending_task_per_issue_agent", "idx_one_pending_task_per_issue_agent_v2", "idx_one_pending_task_per_issue_agent_thread":
return true
default:
return false
}
}
// pendingSlotTakenErr reports whether err means "the (issue, agent) pending slot
// was already occupied when we tried to enqueue".
//
// Both paths behind RerunIssue now normalize the unique violation into the bare
// ErrDuplicatePendingTask sentinel: enqueueMentionTaskWithCommentPlan since
// #5958, and enqueueIssueTaskWithCommentPlan as of #5914 above. The sentinel
// half is therefore what matches in practice, and it has to be here — the
// mention path is the one taken by EVERY rerun whose target is not the issue's
// current agent assignee (a squad leader, a displaced agent re-fired by
// task_id, a mentioned agent), and matching only the raw pgconn error meant the
// reclaim never ran for those, so a system retry winning the slot surfaced as a
// hard error instead.
//
// The raw-violation half is kept deliberately, as a cheap defensive net: one
// errors.As, and it keeps this predicate honest for any caller that reaches it
// without passing through one of those two normalizing paths.
func pendingSlotTakenErr(err error) bool {
return isDuplicatePendingTaskErr(err) || errors.Is(err, ErrDuplicatePendingTask)
}
// applyAttributionFallback applies the workspace's degraded-attribution policy to a
// resolved attribution whose source came back unattributed (no precise human). A
// PRECISE attribution passes through untouched (no policy read at all). For an
// unattributed run the accountable-never-null guarantee is enforced fail-closed —
// we never silently enqueue a task that could run with a NULL accountable_user_id:
//
// - policy read fails (or no workspace) → REFUSE. We cannot confirm the workspace
// permits fallback, so we do not run an unattributable task on a transient DB
// hiccup. (Only the rare unattributed path pays this; precise runs never read.)
// - fail-closed workspace → REFUSE.
// - otherwise → owner_fallback (accountable = agent owner, audit-only, originator
// untouched). If there is no valid agent owner, owner_fallback stays
// unattributed → REFUSE rather than enqueue a NULL-accountable task.
//
// Keeping this at the enqueue boundary (not inside the pure classifiers) means
// owner_fallback needs the agent owner, which every enqueue path has in hand.
func (s *TaskService) applyAttributionFallback(ctx context.Context, attr attribution.Result, agent db.Agent) (attribution.Result, error) {
if attr.Source != attribution.SourceUnattributed {
return attr, nil
}
if s == nil || s.Queries == nil || !agent.WorkspaceID.Valid {
return attr, fmt.Errorf("%w: workspace policy unavailable", ErrAttributionFailClosed)
}
failClosed, err := s.Queries.GetWorkspaceAttributionFailClosed(ctx, agent.WorkspaceID)
if err != nil {
// Cannot confirm the workspace allows fallback → fail closed rather than
// silently run an unattributable task.
return attr, fmt.Errorf("%w: policy read failed: %v", ErrAttributionFailClosed, err)
}
if failClosed {
return attr, ErrAttributionFailClosed
}
fallback := attribution.OwnerFallback(attr, agent.OwnerID)
if fallback.Source == attribution.SourceUnattributed {
// owner_fallback could not resolve an accountable human (no valid agent
// owner): refuse rather than enqueue a NULL-accountable task.
return attr, fmt.Errorf("%w: no agent owner to attribute", ErrAttributionFailClosed)
}
return fallback, nil
}
// attributionCreateParams maps a resolved attribution onto the CreateAgentTask
// provenance columns. originator_source is always stamped (never NULL for a new
// row); delegation lineage and evidence are stamped only when present.
func attributionCreateParams(attr attribution.Result) (source pgtype.Text, delegatedFrom pgtype.UUID, evidenceKind pgtype.Text, evidenceRef pgtype.UUID) {
source = pgtype.Text{String: attr.Source.String(), Valid: true}
delegatedFrom = attr.DelegatedFromTaskID
evidenceKind = pgtype.Text{String: string(attr.EvidenceKind), Valid: attr.EvidenceKind != ""}
evidenceRef = attr.EvidenceRefID
return
}
// RecordHumanDelegation notes that attr's member personally started an agent
// run, for the self-host telemetry "delegating member" counts. Call it once the
// run row (or the fold into a queued row) is committed, with the attribution
// that row was written with. Runs that only inherit a human (retries, system
// wakeups, delegation, autopilot) carry HumanAction=false and are skipped.
//
// Best-effort and outside any caller transaction: a failed write costs one
// activity mark, never the run.
func (s *TaskService) RecordHumanDelegation(ctx context.Context, attr attribution.Result) {
if s == nil || !s.RecordMemberActivity || s.Queries == nil || !attr.HumanAction || !attr.UserID.Valid {
return
}
if err := s.Queries.RecordMemberDelegationDay(ctx, db.RecordMemberDelegationDayParams{
UserID: attr.UserID,
ActedAt: attr.HumanActedAt,
}); err != nil {
slog.Warn("record member delegation day failed", "error", err)
}
}
// OriginatorForIssueTask exposes resolveOriginatorForIssueTask to callers
// outside the service package (the squad-leader access gate in the handler
// layer) so the gate judges the top-of-chain human with the exact same
// resolution the enqueue path persists on the task row. Without a shared entry
// point the gate saw an empty originator for agent-triggered assigns and denied
// private leaders that the write path would have attributed correctly
// (MUL-4305).
func (s *TaskService) OriginatorForIssueTask(ctx context.Context, issue db.Issue, triggerCommentID pgtype.UUID) pgtype.UUID {
return s.resolveOriginatorForIssueTask(ctx, issue, triggerCommentID)
}
func (s *TaskService) captureTaskDispatched(ctx context.Context, task db.AgentTaskQueue) {
if s.Metrics != nil {
source, runtimeMode, _ := s.taskMetricsContext(ctx, task)
s.Metrics.RecordTaskDispatched(util.UUIDToString(task.ID), source, runtimeMode, taskQueueWaitSeconds(task), taskClaimableWaitSeconds(task))
}
}
func (s *TaskService) AnalyticsContextForTask(ctx context.Context, task db.AgentTaskQueue) analytics.TaskContext {
return s.taskAnalyticsContext(ctx, task)
}
func (s *TaskService) captureTaskStarted(ctx context.Context, task db.AgentTaskQueue) {
if s.Metrics != nil {
source, runtimeMode, provider := s.taskMetricsContext(ctx, task)
s.Metrics.RecordTaskStarted(source, runtimeMode, provider)
}
}
func (s *TaskService) captureTaskCompleted(ctx context.Context, task db.AgentTaskQueue) {
if s.Metrics != nil {
source, runtimeMode, _ := s.taskMetricsContext(ctx, task)
s.Metrics.RecordTaskTerminal(util.UUIDToString(task.ID), source, runtimeMode, task.Status, taskRunSeconds(task), taskTotalSeconds(task), task.Attempt)
}
}
func (s *TaskService) captureTaskFailed(ctx context.Context, task db.AgentTaskQueue) {
failureReason := taskFailureReason(task)
if s.Metrics != nil {
source, runtimeMode, _ := s.taskMetricsContext(ctx, task)
s.Metrics.RecordTaskTerminal(util.UUIDToString(task.ID), source, runtimeMode, task.Status, taskRunSeconds(task), taskTotalSeconds(task), task.Attempt)
s.Metrics.RecordTaskFailed(source, runtimeMode, failureReason)
}
}
func (s *TaskService) captureTaskCancelled(ctx context.Context, task db.AgentTaskQueue) {
if s.Metrics != nil {
source, runtimeMode, _ := s.taskMetricsContext(ctx, task)
s.Metrics.RecordTaskTerminal(util.UUIDToString(task.ID), source, runtimeMode, task.Status, taskRunSeconds(task), taskTotalSeconds(task), task.Attempt)
}
// Revoke any mat_ task tokens minted for this task. Cancellation is
// a terminal transition, so the running agent process no longer
// needs to call back; eagerly deleting the token closes the
// window where a compromised process could keep authenticating
// against the API until the 24h expiry. Failure is non-fatal — the
// expiry / FK cascade are the durable guards. MUL-2600.
if err := s.Queries.DeleteTaskTokensByTask(ctx, task.ID); err != nil {
slog.Warn("cancel task: failed to revoke task tokens",
"task_id", util.UUIDToString(task.ID), "error", err)
}
}
// costUSDTicks is the provider's own price for this usage in 1e-10 USD, or 0
// when it reported none — the metrics layer prefers it over its rate table.
func (s *TaskService) CaptureTaskUsage(ctx context.Context, task db.AgentTaskQueue, provider, model string, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, costUSDTicks int64) {
if s.Metrics == nil {
return
}
source, runtimeMode, _ := s.taskMetricsContext(ctx, task)
s.Metrics.RecordLLMUsage(source, runtimeMode, provider, model, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, costUSDTicks)
}
func (s *TaskService) CaptureQueuedExpiredTasks(ctx context.Context, tasks []db.AgentTaskQueue) {
if s.Metrics == nil {
return
}
for _, task := range tasks {
source, runtimeMode, _ := s.taskMetricsContext(ctx, task)
s.Metrics.RecordTaskQueuedExpired(source, runtimeMode)
}
}
func (s *TaskService) CaptureLeaseExpiredTasks(ctx context.Context, tasks []db.AgentTaskQueue) {
if s.Metrics == nil {
return
}
for _, task := range tasks {
source, _, _ := s.taskMetricsContext(ctx, task)
s.Metrics.RecordTaskLeaseExpired(source)
}
}
func (s *TaskService) cachedTaskAnalyticsContext(task db.AgentTaskQueue) (analytics.TaskContext, bool) {
key := taskAnalyticsContextKey(task)
if key == "" {
return analytics.TaskContext{}, false
}
s.analyticsContextMu.Lock()
defer s.analyticsContextMu.Unlock()
if s.analyticsContextCache == nil {
return analytics.TaskContext{}, false
}
tc, ok := s.analyticsContextCache[key]
return tc, ok
}
func (s *TaskService) storeTaskAnalyticsContext(task db.AgentTaskQueue, tc analytics.TaskContext) {
if tc.WorkspaceID == "" {
return
}
key := taskAnalyticsContextKey(task)
if key == "" {
return
}
s.analyticsContextMu.Lock()
defer s.analyticsContextMu.Unlock()
if s.analyticsContextCache == nil {
s.analyticsContextCache = make(map[string]analytics.TaskContext)
}
if _, ok := s.analyticsContextCache[key]; !ok {
s.analyticsContextOrder = append(s.analyticsContextOrder, key)
if len(s.analyticsContextOrder) > taskAnalyticsContextCacheMax {
oldest := s.analyticsContextOrder[0]
s.analyticsContextOrder = s.analyticsContextOrder[1:]
delete(s.analyticsContextCache, oldest)
}
}
s.analyticsContextCache[key] = tc
}
func taskAnalyticsContextKey(task db.AgentTaskQueue) string {
taskID := util.UUIDToString(task.ID)
if taskID == "" {
return ""
}
return strings.Join([]string{
taskID,
util.UUIDToString(task.RuntimeID),
util.UUIDToString(task.IssueID),
util.UUIDToString(task.ChatSessionID),
util.UUIDToString(task.AutopilotRunID),
}, "|")
}
func (s *TaskService) taskMetricsContext(ctx context.Context, task db.AgentTaskQueue) (source, runtimeMode, provider string) {
tc := s.taskAnalyticsContext(ctx, task)
source = "other"
switch {
case task.ChatSessionID.Valid:
source = "chat"
case task.IssueID.Valid:
if tc.Source == analytics.SourceAutopilot {
source = "autopilot_issue"
} else {
source = "issue"
}
case task.AutopilotRunID.Valid:
source = "autopilot"
default:
if _, ok := s.parseQuickCreateContext(task); ok {
source = "quick_create"
} else if tc.Source != "" {
source = tc.Source
}
}
return source, tc.RuntimeMode, tc.Provider
}
func (s *TaskService) taskAnalyticsContext(ctx context.Context, task db.AgentTaskQueue) analytics.TaskContext {
if tc, ok := s.cachedTaskAnalyticsContext(task); ok {
return tc
}
tc := analytics.TaskContext{
AgentID: util.UUIDToString(task.AgentID),
TaskID: util.UUIDToString(task.ID),
Source: analytics.SourceManual,
}
if task.IssueID.Valid {
tc.IssueID = util.UUIDToString(task.IssueID)
}
if task.ChatSessionID.Valid {
tc.ChatSessionID = util.UUIDToString(task.ChatSessionID)
tc.Source = analytics.SourceChat
}
if task.AutopilotRunID.Valid {
tc.AutopilotRunID = util.UUIDToString(task.AutopilotRunID)
tc.Source = analytics.SourceAutopilot
}
if task.RuntimeID.Valid {
if rt, err := s.runtimeLookup().Get(ctx, task.RuntimeID); err == nil {
tc.WorkspaceID = util.UUIDToString(rt.WorkspaceID)
tc.RuntimeMode = rt.RuntimeMode
tc.Provider = rt.Provider
}
}
if tc.WorkspaceID == "" || tc.RuntimeMode == "" {
if agent, err := s.Queries.GetAgent(ctx, task.AgentID); err == nil {
if tc.WorkspaceID == "" {
tc.WorkspaceID = util.UUIDToString(agent.WorkspaceID)
}
if tc.RuntimeMode == "" {
tc.RuntimeMode = agent.RuntimeMode
}
}
}
if task.IssueID.Valid {
if issue, err := s.Queries.GetIssue(ctx, task.IssueID); err == nil {
tc.WorkspaceID = util.UUIDToString(issue.WorkspaceID)
if issue.CreatorType == "member" {
tc.UserID = util.UUIDToString(issue.CreatorID)
}
if issue.OriginType.Valid {
switch issue.OriginType.String {
case "autopilot":
tc.Source = analytics.SourceAutopilot
if ap, err := s.Queries.GetAutopilot(ctx, issue.OriginID); err == nil {
if ap.CreatedByType == "member" {
tc.UserID = util.UUIDToString(ap.CreatedByID)
}
}
case "quick_create":
tc.Source = analytics.SourceManual
}
}
}
}
if task.ChatSessionID.Valid {
if cs, err := s.Queries.GetChatSession(ctx, task.ChatSessionID); err == nil {
tc.WorkspaceID = util.UUIDToString(cs.WorkspaceID)
tc.UserID = util.UUIDToString(cs.CreatorID)
}
}
if task.AutopilotRunID.Valid {
if run, err := s.Queries.GetAutopilotRun(ctx, task.AutopilotRunID); err == nil {
if ap, err := s.Queries.GetAutopilot(ctx, run.AutopilotID); err == nil {
tc.WorkspaceID = util.UUIDToString(ap.WorkspaceID)
if ap.CreatedByType == "member" {
tc.UserID = util.UUIDToString(ap.CreatedByID)
}
}
}
}
if qc, ok := s.parseQuickCreateContext(task); ok {
tc.WorkspaceID = qc.WorkspaceID
tc.UserID = qc.RequesterID
tc.Source = analytics.SourceManual
}
s.storeTaskAnalyticsContext(task, tc)
return tc
}
func taskQueueWaitSeconds(task db.AgentTaskQueue) float64 {
return durationSeconds(task.CreatedAt, task.DispatchedAt)
}
// taskClaimableWaitSeconds excludes an intentional deferred delay when fire_at
// is still available on the claimed row. Immediate tasks start at creation.
func taskClaimableWaitSeconds(task db.AgentTaskQueue) float64 {
claimableAt := task.CreatedAt
if task.FireAt.Valid && (!claimableAt.Valid || task.FireAt.Time.After(claimableAt.Time)) {
claimableAt = task.FireAt
}
return durationSeconds(claimableAt, task.DispatchedAt)
}
func taskRunSeconds(task db.AgentTaskQueue) float64 {
return durationSeconds(task.StartedAt, task.CompletedAt)
}
func taskTotalSeconds(task db.AgentTaskQueue) float64 {
return durationSeconds(task.CreatedAt, task.CompletedAt)
}
func durationSeconds(start, end pgtype.Timestamptz) float64 {
if !start.Valid || !end.Valid {
return -1
}
seconds := end.Time.Sub(start.Time).Seconds()
if seconds < 0 {
return 0
}
return seconds
}
func taskFailureReason(task db.AgentTaskQueue) string {
if task.FailureReason.Valid && task.FailureReason.String != "" {
return task.FailureReason.String
}
return "agent_error"
}
func taskErrorType(reason string) string {
switch reason {
case "runtime_offline", "runtime_recovery":
return "runtime"
case "timeout", "codex_semantic_inactivity":
return "timeout"
case "iteration_limit", "agent_fallback_message":
return "agent_output"
case "cancelled", "user_cancelled":
return "cancelled"
default:
return "agent_error"
}
}
// EnqueueTaskForIssue creates a queued task for an agent-assigned issue.
// No context snapshot is stored — the agent fetches all data it needs at
// runtime via the multica CLI.
func (s *TaskService) EnqueueTaskForIssue(ctx context.Context, issue db.Issue, triggerCommentID ...pgtype.UUID) (db.AgentTaskQueue, error) {
var commentID pgtype.UUID
if len(triggerCommentID) > 0 {
commentID = triggerCommentID[0]
}
return s.enqueueIssueTask(ctx, issue, commentID, false, "", pgtype.UUID{}, pgtype.UUID{}, pgtype.Timestamptz{}, OriginDerived)
}
// EnqueueDeferredChannelIssueTask persists the assigned task for a media-backed
// channel /issue turn without making it claimable yet. The fireAt deadline is a
// crash-safe fallback; the channel router promotes the task as soon as the
// detached attachment transaction settles.
func (s *TaskService) EnqueueDeferredChannelIssueTask(ctx context.Context, issue db.Issue, fireAt time.Time) (db.AgentTaskQueue, error) {
task, err := s.enqueueIssueTask(ctx, issue, pgtype.UUID{}, false, "", pgtype.UUID{}, pgtype.UUID{}, pgtype.Timestamptz{Time: fireAt, Valid: true}, OriginDerived)
if err != nil {
return db.AgentTaskQueue{}, err
}
// The task is durable but not claimable yet. Wake once without a task ID so
// the daemon refreshes its deferred schedule without treating it as ready.
s.notifyRuntimeMayHaveWork(task.RuntimeID, "")
return task, nil
}
// createDeferredChannelIssueTaskWithQueries inserts the inert media-gated task
// through the caller's query handle. IssueService passes its transaction-bound
// Queries so the issue and task become visible atomically. Composio is
// intentionally absent from the transaction-scoped service: the task cannot be
// claimed while deferred, so the optional external overlay is hydrated after
// commit without holding database locks across a network call.
func (s *TaskService) createDeferredChannelIssueTaskWithQueries(ctx context.Context, q *db.Queries, issue db.Issue, fireAt time.Time) (db.AgentTaskQueue, error) {
txService := &TaskService{Queries: q}
return txService.enqueueIssueTask(ctx, issue, pgtype.UUID{}, false, "", pgtype.UUID{}, pgtype.UUID{}, pgtype.Timestamptz{Time: fireAt, Valid: true}, OriginDerived)
}
// hydrateDeferredChannelIssueTaskOverlay fills the optional Composio overlay
// after the issue+task transaction commits. The conditional update refuses to
// overwrite a comment merge that won the post-commit race and already
// re-attributed the task (with its own matching overlay).
func (s *TaskService) hydrateDeferredChannelIssueTaskOverlay(ctx context.Context, task db.AgentTaskQueue) error {
if s == nil || s.Queries == nil || s.Composio == nil || !featureflags.ComposioMCPAppsEnabled(ctx, s.FeatureFlags) {
return nil
}
agent, err := s.Queries.GetAgent(ctx, task.AgentID)
if err != nil {
return fmt.Errorf("load agent for deferred channel issue task overlay: %w", err)
}
overlay := s.buildRuntimeMCPOverlay(ctx, task.OriginatorUserID, agent)
if len(overlay.Overlay) == 0 {
return nil
}
updated, err := s.Queries.SetDeferredChannelIssueTaskRuntimeOverlay(ctx, db.SetDeferredChannelIssueTaskRuntimeOverlayParams{
ID: task.ID,
RuntimeMcpOverlay: overlay.Overlay,
RuntimeConnectedApps: overlay.ConnectedApps,
ExpectedOriginatorUserID: task.OriginatorUserID,
})
if err != nil {
return fmt.Errorf("set deferred channel issue task overlay: %w", err)
}
if updated == 0 {
slog.Debug("deferred channel issue task overlay skipped: task plan changed",
"task_id", util.UUIDToString(task.ID))
}
return nil
}
// EnqueueTaskForIssueByActor is the assign/promote variant of
// EnqueueTaskForIssue. actorUserID is the member who performed the
// assign/promote and becomes the accountable human for the run (MUL-4302 §4);
// invalid when the caller has no member actor.
func (s *TaskService) EnqueueTaskForIssueByActor(ctx context.Context, issue db.Issue, actorUserID pgtype.UUID) (db.AgentTaskQueue, error) {
return s.enqueueIssueTask(ctx, issue, pgtype.UUID{}, false, "", actorUserID, pgtype.UUID{}, pgtype.Timestamptz{}, OriginDerived)
}
// EnqueueTaskForIssueWithHandoff is the backward-compatible assign/promote
// variant used when an installed client still sends handoff_note. The note is
// persisted on the task so both old and current daemons can render it in the
// run's opening prompt. Empty text behaves like EnqueueTaskForIssueByActor.
func (s *TaskService) EnqueueTaskForIssueWithHandoff(ctx context.Context, issue db.Issue, handoffNote string, actorUserID pgtype.UUID) (db.AgentTaskQueue, error) {
return s.enqueueIssueTask(ctx, issue, pgtype.UUID{}, false, handoffNote, actorUserID, pgtype.UUID{}, pgtype.Timestamptz{}, OriginDerived)
}
// enqueueIssueTask is the shared implementation behind EnqueueTaskForIssue
// and the manual rerun path. forceFreshSession=true marks the task so the
// daemon claim handler skips the (agent_id, issue_id) resume lookup — the
// user already judged the prior output bad, a fresh agent session is the
// expected behavior.
// ResolveIssueReviewSHA returns the head SHA of the commit currently under
// review for an issue (the head_sha of its most-relevant linked PR), or the
// empty string when the issue has no linked PR. Callers thread this into both
// the reviewer-loop dedup check and the enqueue path so a pending review task
// pinned to an old head does not satisfy a request after HEAD advanced
// (TEN-356). Empty string is the safe default: it makes dedup fall back to the
// pre-TEN-356 (issue_id, agent_id) key and leaves the task's context NULL.
//
// The lookup fails soft — any DB error (including "no linked PR") returns "" so
// a transient github-table hiccup can never over-dedup a review out of
// existence; the worst case is the pre-TEN-356 coalescing behavior.
func (s *TaskService) ResolveIssueReviewSHA(ctx context.Context, issueID pgtype.UUID) string {
if !issueID.Valid {
return ""
}
sha, err := s.Queries.GetIssueReviewHeadSha(ctx, issueID)
if err != nil {
if !errors.Is(err, pgx.ErrNoRows) {
slog.Warn("resolve issue review sha failed",
"issue_id", util.UUIDToString(issueID), "error", err)
}
return ""
}
return sha
}
// headShaText wraps a resolved review SHA into the pgtype.Text the dedup/enqueue
// queries expect. Empty SHA marshals to an invalid (NULL) Text so the queries
// take their fall-back branch.
func headShaText(sha string) pgtype.Text {
return pgtype.Text{String: sha, Valid: sha != ""}
}
// ResolveIssueReviewSHAParam is ResolveIssueReviewSHA wrapped as the pgtype.Text
// the dedup queries take, so both service- and handler-package call sites can
// key dedup on the reviewed head with a single call (TEN-356).
func (s *TaskService) ResolveIssueReviewSHAParam(ctx context.Context, issueID pgtype.UUID) pgtype.Text {
return headShaText(s.ResolveIssueReviewSHA(ctx, issueID))
}
func (s *TaskService) enqueueIssueTask(ctx context.Context, issue db.Issue, triggerCommentID pgtype.UUID, forceFreshSession bool, handoffNote string, actorUserID pgtype.UUID, rerunOfTaskID pgtype.UUID, fireAt pgtype.Timestamptz, origin RunOrigin) (db.AgentTaskQueue, error) {
return s.enqueueIssueTaskWithCommentPlan(ctx, issue, triggerCommentID, nil, forceFreshSession, handoffNote, actorUserID, rerunOfTaskID, fireAt, origin, false)
}
func (s *TaskService) enqueueIssueTaskWithCommentPlan(ctx context.Context, issue db.Issue, triggerCommentID pgtype.UUID, coalescedCommentIDs []pgtype.UUID, forceFreshSession bool, handoffNote string, actorUserID pgtype.UUID, rerunOfTaskID pgtype.UUID, fireAt pgtype.Timestamptz, origin RunOrigin, freshRerun bool) (db.AgentTaskQueue, error) {
if !issue.AssigneeID.Valid {
slog.Error("task enqueue failed", "issue_id", util.UUIDToString(issue.ID), "error", "issue has no assignee")
return db.AgentTaskQueue{}, fmt.Errorf("issue has no assignee")
}
if err := guardIssueNotInTriage(ctx, s.Queries, issue.ID, origin); err != nil {
return db.AgentTaskQueue{}, err
}
agent, err := s.Queries.GetAgent(ctx, issue.AssigneeID)
if err != nil {
slog.Error("task enqueue failed", "issue_id", util.UUIDToString(issue.ID), "error", err)
return db.AgentTaskQueue{}, fmt.Errorf("load agent: %w", err)
}
if agent.ArchivedAt.Valid {
slog.Debug("task enqueue skipped: agent is archived", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agent.ID))
return db.AgentTaskQueue{}, fmt.Errorf("agent is archived")
}
if !agent.RuntimeID.Valid {
slog.Error("task enqueue failed", "issue_id", util.UUIDToString(issue.ID), "error", "agent has no runtime")
return db.AgentTaskQueue{}, fmt.Errorf("agent has no runtime")
}
// The issue assignee reacting to an agent-authored comment is a
// comment_source attribution (a special case of delegation); a member
// comment or direct member assignment is direct_human. attr.UserID is the
// same value the pre-MUL-4302 resolver produced, so overlay/authorization
// are unchanged; the extra fields are audit provenance.
attr := s.attributionForIssueTask(ctx, issue, triggerCommentID, attribution.SourceCommentSource, actorUserID)
// No precise human resolved → owner_fallback (accountable = agent owner), or
// refuse the enqueue if the workspace is fail-closed (MUL-4302 §3.5).
attr, err = s.applyAttributionFallback(ctx, attr, agent)
if err != nil {
slog.Warn("task enqueue refused: attribution fail-closed", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(issue.AssigneeID))
return db.AgentTaskQueue{}, err
}
originatorUserID := attr.UserID
runtimeMCPOverlay := s.buildRuntimeMCPOverlay(ctx, originatorUserID, agent)
attrSource, attrDelegatedFrom, attrEvidenceKind, attrEvidenceRef := attributionCreateParams(attr)
createParams := db.CreateAgentTaskParams{
ID: dbid.NewV7(),
AgentID: issue.AssigneeID,
RuntimeID: agent.RuntimeID,
IssueID: issue.ID,
Priority: priorityToInt(issue.Priority),
TriggerCommentID: triggerCommentID,
CoalescedCommentIds: coalescedCommentIDs,
TriggerSummary: s.buildCommentTriggerSummary(ctx, issue.WorkspaceID, triggerCommentID),
ForceFreshSession: pgtype.Bool{Bool: forceFreshSession, Valid: forceFreshSession},
HandoffNote: pgtype.Text{String: handoffNote, Valid: handoffNote != ""},
OriginatorUserID: originatorUserID,
AccountableUserID: attr.AccountableUserID,
RuleVersionID: attr.RuleVersionID,
RerunOfTaskID: rerunOfTaskID,
FreshRerun: pgtype.Bool{Bool: freshRerun, Valid: true},
RuntimeMcpOverlay: runtimeMCPOverlay.Overlay,
RuntimeConnectedApps: runtimeMCPOverlay.ConnectedApps,
OriginatorSource: attrSource,
DelegatedFromTaskID: attrDelegatedFrom,
TriggerEvidenceKind: attrEvidenceKind,
TriggerEvidenceRefID: attrEvidenceRef,
// Stamp the reviewed head so dedup can distinguish this run's target
// from a later request against a new HEAD (TEN-356).
HeadSha: headShaText(s.ResolveIssueReviewSHA(ctx, issue.ID)),
}
var task db.AgentTaskQueue
if fireAt.Valid {
task, err = s.Queries.CreateDeferredChannelIssueTask(ctx, db.CreateDeferredChannelIssueTaskParams{
ID: dbid.NewV7(),
AgentID: createParams.AgentID,
RuntimeID: createParams.RuntimeID,
IssueID: createParams.IssueID,
Priority: createParams.Priority,
TriggerCommentID: createParams.TriggerCommentID,
CoalescedCommentIds: createParams.CoalescedCommentIds,
TriggerSummary: createParams.TriggerSummary,
ForceFreshSession: createParams.ForceFreshSession,
IsLeaderTask: createParams.IsLeaderTask,
HandoffNote: createParams.HandoffNote,
SquadID: createParams.SquadID,
HeadSha: createParams.HeadSha,
OriginatorUserID: createParams.OriginatorUserID,
AccountableUserID: createParams.AccountableUserID,
RuntimeMcpOverlay: createParams.RuntimeMcpOverlay,
RuntimeConnectedApps: createParams.RuntimeConnectedApps,
OriginatorSource: createParams.OriginatorSource,
DelegatedFromTaskID: createParams.DelegatedFromTaskID,
RuleVersionID: createParams.RuleVersionID,
RerunOfTaskID: createParams.RerunOfTaskID,
TriggerEvidenceKind: createParams.TriggerEvidenceKind,
TriggerEvidenceRefID: createParams.TriggerEvidenceRefID,
FireAt: fireAt,
})
} else {
task, err = s.Queries.CreateAgentTask(ctx, createParams)
}
if err != nil {
// A concurrent enqueue for the same (issue, agent) won the race and the
// unique index rejected this insert. That is benign — a sibling run
// already covers this target — so log it at debug and return a typed
// sentinel the caller maps to a coalesced outcome / 409 rather than a
// 500 that leaks the raw constraint name (#5914). Mirrors the mention
// path in enqueueMentionTaskWithCommentPlan.
if isDuplicatePendingTaskErr(err) {
slog.Debug("task enqueue coalesced: pending task already exists", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(issue.AssigneeID))
return db.AgentTaskQueue{}, ErrDuplicatePendingTask
}
slog.Error("task enqueue failed", "issue_id", util.UUIDToString(issue.ID), "error", err)
return db.AgentTaskQueue{}, fmt.Errorf("create task: %w", err)
}
slog.Info("task enqueued",
"task_id", util.UUIDToString(task.ID),
"issue_id", util.UUIDToString(issue.ID),
"agent_id", util.UUIDToString(issue.AssigneeID),
"force_fresh_session", forceFreshSession,
)
s.RecordHumanDelegation(ctx, attr)
if fireAt.Valid {
return task, nil
}
// Order matters: broadcast first, notify daemon second. notifyTaskAvailable
// kicks an in-process channel that the daemon picks up over HTTP and
// claims; the claim path then emits its own task:dispatch. Doing the
// queued broadcast afterwards risks the dispatch event reaching clients
// before the queued one (rare but unsafe-by-construction). Publishing
// in the desired observe-order makes correctness independent of timing.
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, task)
s.NotifyTaskEnqueued(ctx, task)
return task, nil
}
// EnqueueTaskForMention creates a queued task for a mentioned agent on an issue.
// Unlike EnqueueTaskForIssue, this takes an explicit agent ID rather than
// deriving it from the issue assignee.
func (s *TaskService) EnqueueTaskForMention(ctx context.Context, issue db.Issue, agentID pgtype.UUID, triggerCommentID pgtype.UUID, origin RunOrigin) (db.AgentTaskQueue, error) {
return s.enqueueMentionTask(ctx, issue, agentID, triggerCommentID, false, pgtype.UUID{}, false, "", pgtype.UUID{}, pgtype.UUID{}, origin)
}
// EnqueueTaskForThreadParent creates a queued task for the agent who authored
// the direct parent comment a member replied to.
func (s *TaskService) EnqueueTaskForThreadParent(ctx context.Context, issue db.Issue, agentID pgtype.UUID, triggerCommentID pgtype.UUID) (db.AgentTaskQueue, error) {
// Always named: the only caller is a member replying to what this agent said.
return s.enqueueMentionTask(ctx, issue, agentID, triggerCommentID, false, pgtype.UUID{}, false, "", pgtype.UUID{}, pgtype.UUID{}, OriginNamed)
}
// EnqueueTaskForSquadLeader is the leader-role variant of EnqueueTaskForMention.
// The resulting task carries is_leader_task=true so that downstream
// self-trigger guards can distinguish a comment posted while the agent was
// acting as the squad's leader (skip) from one posted while it was acting
// as a worker (do not skip). This matters for agents that are simultaneously
// the leader and a worker of the same squad — see migration 090.
//
// squadID is stamped onto the task's squad_id column so the daemon claim
// handler can locate the squad and inject its briefing regardless of how the
// leader task was triggered (comment @squad, issue assign, autopilot,
// sub-issue done callback). See migration 127.
func (s *TaskService) EnqueueTaskForSquadLeader(ctx context.Context, issue db.Issue, leaderID pgtype.UUID, squadID pgtype.UUID, triggerCommentID pgtype.UUID, origin RunOrigin) (db.AgentTaskQueue, error) {
return s.enqueueMentionTask(ctx, issue, leaderID, triggerCommentID, true, squadID, false, "", pgtype.UUID{}, pgtype.UUID{}, origin)
}
// EnqueueTaskForSquadLeaderByActor is the assign/promote variant of
// EnqueueTaskForSquadLeader. actorUserID is the member who performed the
// assign/promote and becomes the accountable human (MUL-4302 §4); invalid when
// the caller has no member actor.
func (s *TaskService) EnqueueTaskForSquadLeaderByActor(ctx context.Context, issue db.Issue, leaderID pgtype.UUID, squadID pgtype.UUID, actorUserID pgtype.UUID) (db.AgentTaskQueue, error) {
return s.enqueueMentionTask(ctx, issue, leaderID, pgtype.UUID{}, true, squadID, false, "", actorUserID, pgtype.UUID{}, OriginDerived)
}
// EnqueueTaskForSquadLeaderWithHandoff is the squad equivalent of
// EnqueueTaskForIssueWithHandoff.
func (s *TaskService) EnqueueTaskForSquadLeaderWithHandoff(ctx context.Context, issue db.Issue, leaderID pgtype.UUID, squadID pgtype.UUID, handoffNote string, actorUserID pgtype.UUID) (db.AgentTaskQueue, error) {
return s.enqueueMentionTask(ctx, issue, leaderID, pgtype.UUID{}, true, squadID, false, handoffNote, actorUserID, pgtype.UUID{}, OriginDerived)
}
func (s *TaskService) enqueueMentionTask(ctx context.Context, issue db.Issue, agentID pgtype.UUID, triggerCommentID pgtype.UUID, isLeader bool, squadID pgtype.UUID, forceFreshSession bool, handoffNote string, actorUserID pgtype.UUID, rerunOfTaskID pgtype.UUID, origin RunOrigin) (db.AgentTaskQueue, error) {
return s.enqueueMentionTaskWithCommentPlan(ctx, issue, agentID, triggerCommentID, nil, isLeader, squadID, forceFreshSession, handoffNote, actorUserID, rerunOfTaskID, origin, false)
}
func (s *TaskService) enqueueMentionTaskWithCommentPlan(ctx context.Context, issue db.Issue, agentID pgtype.UUID, triggerCommentID pgtype.UUID, coalescedCommentIDs []pgtype.UUID, isLeader bool, squadID pgtype.UUID, forceFreshSession bool, handoffNote string, actorUserID pgtype.UUID, rerunOfTaskID pgtype.UUID, origin RunOrigin, freshRerun bool) (db.AgentTaskQueue, error) {
if err := guardIssueNotInTriage(ctx, s.Queries, issue.ID, origin); err != nil {
return db.AgentTaskQueue{}, err
}
agent, err := s.Queries.GetAgent(ctx, agentID)
if err != nil {
slog.Error("mention task enqueue failed: agent not found", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agentID), "error", err)
return db.AgentTaskQueue{}, fmt.Errorf("load agent: %w", err)
}
if agent.ArchivedAt.Valid {
slog.Debug("mention task enqueue skipped: agent is archived", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agentID))
return db.AgentTaskQueue{}, fmt.Errorf("agent is archived")
}
if !agent.RuntimeID.Valid {
slog.Error("mention task enqueue failed: agent has no runtime", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agentID))
return db.AgentTaskQueue{}, fmt.Errorf("agent has no runtime")
}
// An explicit mention / thread-parent / squad-leader hop from an
// agent-authored comment is a delegation (the parent task's human is
// copied); a member mention is direct_human. attr.UserID matches the
// pre-MUL-4302 value, so authorization is unchanged.
attr := s.attributionForIssueTask(ctx, issue, triggerCommentID, attribution.SourceDelegation, actorUserID)
// No precise human resolved → owner_fallback (accountable = agent owner), or
// refuse the enqueue if the workspace is fail-closed (MUL-4302 §3.5).
attr, err = s.applyAttributionFallback(ctx, attr, agent)
if err != nil {
slog.Warn("mention task enqueue refused: attribution fail-closed", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agentID))
return db.AgentTaskQueue{}, err
}
originatorUserID := attr.UserID
runtimeMCPOverlay := s.buildRuntimeMCPOverlay(ctx, originatorUserID, agent)
attrSource, attrDelegatedFrom, attrEvidenceKind, attrEvidenceRef := attributionCreateParams(attr)
task, err := s.Queries.CreateAgentTask(ctx, db.CreateAgentTaskParams{
ID: dbid.NewV7(),
AgentID: agentID,
RuntimeID: agent.RuntimeID,
IssueID: issue.ID,
Priority: priorityToInt(issue.Priority),
TriggerCommentID: triggerCommentID,
CoalescedCommentIds: coalescedCommentIDs,
TriggerSummary: s.buildCommentTriggerSummary(ctx, issue.WorkspaceID, triggerCommentID),
IsLeaderTask: pgtype.Bool{Bool: isLeader, Valid: isLeader},
ForceFreshSession: pgtype.Bool{Bool: forceFreshSession, Valid: forceFreshSession},
HandoffNote: pgtype.Text{String: handoffNote, Valid: handoffNote != ""},
SquadID: squadID,
OriginatorUserID: originatorUserID,
AccountableUserID: attr.AccountableUserID,
RuleVersionID: attr.RuleVersionID,
RerunOfTaskID: rerunOfTaskID,
FreshRerun: pgtype.Bool{Bool: freshRerun, Valid: true},
RuntimeMcpOverlay: runtimeMCPOverlay.Overlay,
RuntimeConnectedApps: runtimeMCPOverlay.ConnectedApps,
OriginatorSource: attrSource,
DelegatedFromTaskID: attrDelegatedFrom,
TriggerEvidenceKind: attrEvidenceKind,
TriggerEvidenceRefID: attrEvidenceRef,
// Stamp the reviewed head so dedup can distinguish this run's target
// from a later request against a new HEAD (TEN-356).
HeadSha: headShaText(s.ResolveIssueReviewSHA(ctx, issue.ID)),
})
if err != nil {
// A concurrent enqueue for the same (issue, agent) won the race and the
// unique index rejected this insert. That is benign — a sibling run
// already covers this target — so log it at debug and return a typed
// sentinel the caller maps to a coalesced outcome / 409 rather than a
// 500 that leaks the raw constraint name (#5914).
if isDuplicatePendingTaskErr(err) {
slog.Debug("mention task enqueue coalesced: pending task already exists", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agentID))
return db.AgentTaskQueue{}, ErrDuplicatePendingTask
}
slog.Error("mention task enqueue failed", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agentID), "error", err)
return db.AgentTaskQueue{}, fmt.Errorf("create task: %w", err)
}
slog.Info("mention task enqueued", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agentID), "is_leader_task", isLeader)
s.RecordHumanDelegation(ctx, attr)
// See EnqueueTaskForIssue for ordering rationale.
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, task)
s.NotifyTaskEnqueued(ctx, task)
return task, nil
}
// QuickCreateContext is the JSON payload stored on a quick-create task's
// context column. The daemon detects this variant via Type == "quick_create"
// and switches to the quick-create prompt template; the completion path
// uses RequesterID + WorkspaceID to write the inbox notification.
//
// ProjectID is the optional project the user picked in the modal. When
// non-empty the daemon claim handler resolves the project's title +
// resources, and the prompt template instructs the agent to pass
// `--project <uuid>` so the new issue lands in that project.
//
// SquadID is non-empty when the user picked a squad (rather than an agent)
// in the modal. The task is still enqueued against the squad's leader
// agent (Queries.CreateQuickCreateTask is agent-scoped); SquadID is the
// hint the daemon claim handler uses to layer the squad-leader briefing
// onto the agent's Instructions, matching the behavior of issue-bound
// tasks assigned to the squad.
type QuickCreateContext struct {
Type string `json:"type"`
Prompt string `json:"prompt"`
RequesterID string `json:"requester_id"`
WorkspaceID string `json:"workspace_id"`
Priority string `json:"priority,omitempty"`
DueDate string `json:"due_date,omitempty"`
ProjectID string `json:"project_id,omitempty"`
SquadID string `json:"squad_id,omitempty"`
AttachmentIDs []string `json:"attachment_ids,omitempty"`
// ParentIssueID is the optional UUID of the parent issue the new issue
// should be filed under. Set when the user opens the modal from "Add
// sub issue" on an existing issue; the daemon claim handler resolves the
// parent's identifier and the prompt template instructs the agent to
// pass `--parent <uuid>` so the sub-issue relationship is preserved
// across the manual→agent mode flip.
ParentIssueID string `json:"parent_issue_id,omitempty"`
// SourceContextID identifies the immutable pending capture that must attach
// to the one issue this quick-create chain produces.
SourceContextID string `json:"source_context_id,omitempty"`
}
// QuickCreateContextType marks a task as a quick-create job.
const QuickCreateContextType = "quick_create"
// EnqueueQuickCreateTask creates a queued task that has no issue / chat /
// autopilot link — the user's natural-language prompt is stored in the
// task's context JSONB and the agent is expected to translate it into a
// `multica issue create` call. Pre-validates that the agent is reachable
// (not archived, has a runtime) so the API can reject up-front rather than
// queue a task no one will ever claim.
//
// projectID is optional (zero-valued pgtype.UUID when the user didn't pick
// one). The handler is responsible for validating it belongs to the same
// workspace before passing it in.
//
// squadID is non-empty (Valid) when the user picked a squad as the actor.
// The handler has already resolved it to the squad's leader agent for
// agentID; the squadID hint is stamped into the task context so the daemon
// claim handler can inject the squad-leader briefing on dispatch.
//
// parentIssueID is optional (zero-valued pgtype.UUID when the user didn't
// open the modal from "Add sub issue"). The handler is responsible for
// validating it belongs to the same workspace before passing it in.
func (s *TaskService) EnqueueQuickCreateTask(ctx context.Context, workspaceID, requesterID pgtype.UUID, agentID, squadID pgtype.UUID, prompt, priority, dueDate string, projectID, parentIssueID pgtype.UUID, attachmentIDs []pgtype.UUID) (db.AgentTaskQueue, error) {
return s.enqueueQuickCreateTask(ctx, workspaceID, requesterID, agentID, squadID, prompt, priority, dueDate, projectID, parentIssueID, attachmentIDs, nil)
}
func (s *TaskService) EnqueueQuickCreateTaskWithSourceContext(ctx context.Context, workspaceID, requesterID pgtype.UUID, agentID, squadID pgtype.UUID, prompt, priority, dueDate string, projectID, parentIssueID pgtype.UUID, attachmentIDs []pgtype.UUID, capture SourceContextCapture) (db.AgentTaskQueue, error) {
return s.enqueueQuickCreateTask(ctx, workspaceID, requesterID, agentID, squadID, prompt, priority, dueDate, projectID, parentIssueID, attachmentIDs, &capture)
}
func (s *TaskService) enqueueQuickCreateTask(ctx context.Context, workspaceID, requesterID pgtype.UUID, agentID, squadID pgtype.UUID, prompt, priority, dueDate string, projectID, parentIssueID pgtype.UUID, attachmentIDs []pgtype.UUID, capture *SourceContextCapture) (db.AgentTaskQueue, error) {
if err := CheckIssueCreateCapacity(ctx, s.Queries, s.Entitlements, workspaceID); err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("preflight quick-create issue capacity: %w", err)
}
agent, err := s.Queries.GetAgent(ctx, agentID)
if err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("load agent: %w", err)
}
if agent.ArchivedAt.Valid {
return db.AgentTaskQueue{}, fmt.Errorf("agent is archived")
}
if !agent.RuntimeID.Valid {
return db.AgentTaskQueue{}, fmt.Errorf("agent has no runtime")
}
payload := QuickCreateContext{
Type: QuickCreateContextType,
Prompt: prompt,
RequesterID: util.UUIDToString(requesterID),
WorkspaceID: util.UUIDToString(workspaceID),
Priority: priority,
DueDate: dueDate,
}
if projectID.Valid {
payload.ProjectID = util.UUIDToString(projectID)
}
if squadID.Valid {
payload.SquadID = util.UUIDToString(squadID)
}
if parentIssueID.Valid {
payload.ParentIssueID = util.UUIDToString(parentIssueID)
}
if capture != nil {
payload.SourceContextID = util.UUIDToString(capture.ID)
}
if len(attachmentIDs) > 0 {
payload.AttachmentIDs = make([]string, 0, len(attachmentIDs))
for _, id := range attachmentIDs {
if id.Valid {
payload.AttachmentIDs = append(payload.AttachmentIDs, util.UUIDToString(id))
}
}
}
contextJSON, err := json.Marshal(payload)
if err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("marshal quick-create context: %w", err)
}
// The requester who submitted the quick-create modal is the direct_human
// originator and accountable. Quick-create is the ONE enqueue path with no
// antecedent row to point the uniform evidence pair at: the run's whole job is
// to CREATE the issue, so at enqueue time there is no comment / issue / session
// / run to reference (the issue is linked back later via LinkTaskToIssue).
// Evidence is therefore intentionally NULL; the accountable human is captured on
// originator/accountable_user_id, so this is not a NULL-source bypass — source
// is still stamped direct_human (MUL-4302 §2).
attr := attribution.DirectHumanRun(requesterID, "", pgtype.UUID{})
// An unresolved requester degrades to owner_fallback (accountable = agent
// owner), or is refused if the workspace is fail-closed (MUL-4302 §3.5).
attr, err = s.applyAttributionFallback(ctx, attr, agent)
if err != nil {
return db.AgentTaskQueue{}, err
}
attrSource, _, attrEvidenceKind, attrEvidenceRef := attributionCreateParams(attr)
runtimeMCPOverlay := s.buildRuntimeMCPOverlay(ctx, requesterID, agent)
taskID := dbid.NewV7()
createParams := db.CreateQuickCreateTaskParams{
ID: taskID,
AgentID: agentID,
RuntimeID: agent.RuntimeID,
Priority: priorityToInt("high"),
Context: contextJSON,
OriginatorUserID: requesterID,
AccountableUserID: attr.AccountableUserID,
RuntimeMcpOverlay: runtimeMCPOverlay.Overlay,
RuntimeConnectedApps: runtimeMCPOverlay.ConnectedApps,
OriginatorSource: attrSource,
TriggerEvidenceKind: attrEvidenceKind,
TriggerEvidenceRefID: attrEvidenceRef,
}
var task db.AgentTaskQueue
if capture == nil {
task, err = s.Queries.CreateQuickCreateTask(ctx, createParams)
} else {
tx, beginErr := s.TxStarter.Begin(ctx)
if beginErr != nil {
return db.AgentTaskQueue{}, fmt.Errorf("begin quick-create source context tx: %w", beginErr)
}
defer tx.Rollback(ctx)
qtx := s.Queries.WithTx(tx)
if _, err = qtx.LockIssueForDescriptionUpdate(ctx, db.LockIssueForDescriptionUpdateParams{
ID: capture.SourceIssueID, WorkspaceID: workspaceID,
}); err == nil {
var locked []pgtype.UUID
locked, err = qtx.LockCommentAncestorPath(ctx, db.LockCommentAncestorPathParams{
CommentID: capture.AnchorCommentID, WorkspaceID: workspaceID, IssueID: capture.SourceIssueID,
})
if err == nil && len(locked) == 0 {
err = ErrAnchorCommentDeleted
}
}
if errors.Is(err, pgx.ErrNoRows) {
err = ErrSourceIssueDeleted
}
if err == nil {
var current SourceContextBuild
current, err = BuildSourceContext(ctx, qtx, workspaceID, capture.AnchorCommentID)
if err == nil && current.Digest != capture.Digest {
err = ErrSourceContextChanged
}
}
if err == nil {
task, err = qtx.CreateQuickCreateTask(ctx, createParams)
}
if err == nil {
_, err = PersistSourceContext(ctx, qtx, *capture, pgtype.UUID{}, task.ID)
}
if err == nil {
err = tx.Commit(ctx)
}
}
if err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("create quick-create task with source context: %w", err)
}
slog.Info("quick-create task enqueued",
"task_id", util.UUIDToString(task.ID),
"agent_id", util.UUIDToString(agentID),
"squad_id", payload.SquadID,
"requester_id", util.UUIDToString(requesterID),
"workspace_id", util.UUIDToString(workspaceID),
"project_id", payload.ProjectID,
"parent_issue_id", payload.ParentIssueID,
)
// Match every other Enqueue* path: kick the daemon WS so the task
// gets claimed promptly instead of waiting for the next 30 s poll
// cycle. Without this the user perceives "quick create never
// triggered" because the modal closes immediately and the task
// sits in 'queued' until the next sleepWithContextOrWakeup tick.
s.RecordHumanDelegation(ctx, attr)
s.NotifyTaskEnqueued(ctx, task)
return task, nil
}
// RetrySourceContextQuickCreate manually retries a failed issue-less
// quick-create while retaining the exact pending source context and cloned
// attachments. The context row is the serialization point: only the task that
// currently owns origin_task_id may create and receive a successor, so a
// double-click or a race with automatic retry cannot mint two live attempts.
//
// Only the original requester may retry this recovery item. That keeps the
// direct_human attribution truthful and prevents another workspace member from
// replaying a private prompt merely by learning the task ID.
func (s *TaskService) RetrySourceContextQuickCreate(ctx context.Context, workspaceID, requesterID, sourceTaskID pgtype.UUID, canInvoke func(agent db.Agent) bool) (*db.AgentTaskQueue, error) {
parent, err := s.Queries.GetAgentTaskInWorkspace(ctx, db.GetAgentTaskInWorkspaceParams{
ID: sourceTaskID, WorkspaceID: workspaceID,
})
if err != nil || parent.Status != "failed" {
return nil, ErrSourceContextRetryUnavailable
}
quickCreate, ok := s.parseQuickCreateContext(parent)
if !ok || quickCreate.SourceContextID == "" || quickCreate.RequesterID != util.UUIDToString(requesterID) {
return nil, ErrSourceContextRetryUnavailable
}
contextID, err := util.ParseUUID(quickCreate.SourceContextID)
if err != nil {
return nil, ErrSourceContextRetryUnavailable
}
agent, err := s.Queries.GetAgent(ctx, parent.AgentID)
if err != nil || agent.ArchivedAt.Valid || !agent.RuntimeID.Valid {
return nil, ErrSourceContextRetryUnavailable
}
if canInvoke != nil && !canInvoke(agent) {
return nil, ErrRerunInvokeNotAllowed
}
if err := CheckIssueCreateCapacity(ctx, s.Queries, s.Entitlements, workspaceID); err != nil {
return nil, fmt.Errorf("preflight quick-create issue capacity: %w", err)
}
overlay := s.buildRuntimeMCPOverlay(ctx, requesterID, agent)
tx, err := s.TxStarter.Begin(ctx)
if err != nil {
return nil, fmt.Errorf("begin source context manual retry: %w", err)
}
defer tx.Rollback(ctx)
qtx := s.Queries.WithTx(tx)
locked, err := qtx.GetPendingIssueSourceContextByOriginTask(ctx, db.GetPendingIssueSourceContextByOriginTaskParams{
WorkspaceID: workspaceID, OriginTaskID: sourceTaskID,
})
if err != nil || locked.ID != contextID {
return nil, ErrSourceContextRetryUnavailable
}
child, err := qtx.CreateManualQuickCreateRetryTask(ctx, db.CreateManualQuickCreateRetryTaskParams{
ActorUserID: requesterID,
RuntimeMcpOverlay: overlay.Overlay,
RuntimeConnectedApps: overlay.ConnectedApps,
NewTaskID: dbid.NewV7(),
SourceTaskID: sourceTaskID,
})
if err != nil {
return nil, fmt.Errorf("create source context manual retry: %w", err)
}
if _, err := qtx.TransferPendingIssueSourceContextTask(ctx, db.TransferPendingIssueSourceContextTaskParams{
NewTaskID: child.ID, WorkspaceID: workspaceID, ID: contextID, OldTaskID: sourceTaskID,
}); err != nil {
return nil, fmt.Errorf("transfer source context manual retry: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return nil, fmt.Errorf("commit source context manual retry: %w", err)
}
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, child)
s.NotifyTaskEnqueued(ctx, child)
return &child, nil
}
// ErrChatTaskAgentArchived signals that EnqueueChatTask refused to
// queue work because the destination agent has been archived. This
// is a productizable state — surface it to the user as "this agent
// has been archived" rather than retrying.
var ErrChatTaskAgentArchived = errors.New("chat task: agent archived")
// ErrChatTaskAgentNoRuntime signals that EnqueueChatTask refused to
// queue work because the agent has never been associated with a
// runtime (agent.runtime_id IS NULL). This is the "agent has no
// daemon configured" case — productizable as "agent offline".
//
// IMPORTANT: this is NOT the same as "the daemon is currently
// disconnected". When agent.runtime_id IS set, EnqueueChatTask
// enqueues the task and the daemon claims it on next online; that
// path returns a task row, not this error.
var ErrChatTaskAgentNoRuntime = errors.New("chat task: agent has no runtime")
// ErrChatQuickActionsNoTurn signals that a quick-actions regeneration was asked
// for a session with no eligible assistant turn to resume from (empty session,
// or the latest turn is a no_response / failure with nothing to suggest from).
var ErrChatQuickActionsNoTurn = errors.New("chat quick actions: no assistant turn to regenerate")
// ErrChatQuickActionsUnavailable signals that the deployment has no LLM layer
// configured (no MULTICA_LLM_API_KEY / MULTICA_LLM_BASE_URL), so suggestions
// cannot be generated at all. Automatic generation degrades silently in that
// case; an explicit refresh gets this error so the client can say why nothing
// happened.
var ErrChatQuickActionsUnavailable = errors.New("chat quick actions: llm layer not configured")
// ErrChatQuickActionsStale signals the turn the client asked to refresh is no
// longer the session's latest assistant turn (a newer reply landed since the
// refresh button was rendered). The regeneration is refused so the client's
// optimistic pending marker never points at a turn the resulting
// chat:quick_actions event will not match (MUL-5149).
var ErrChatQuickActionsStale = errors.New("chat quick actions: refresh target is stale")
// ErrChatQuickActionsBusy signals the session already has work in flight — a
// running turn about to change the latest reply, or another regenerate pass —
// so a new refresh is refused to avoid a stale-target supplement or a duplicate
// quota-spending pass (MUL-5149).
var ErrChatQuickActionsBusy = errors.New("chat quick actions: session busy")
// ErrChatSessionArchived signals that a send or a debounced channel flush lost
// a race with archiving the session and therefore must not persist a new turn.
var ErrChatSessionArchived = errors.New("chat task: session archived")
// PreparedChatTaskEnqueue is an opaque, side-effect-free input snapshot built
// before a caller opens a task-enqueue transaction.
type PreparedChatTaskEnqueue struct {
accountableUser pgtype.UUID
attrSource pgtype.Text
attrEvidenceKind pgtype.Text
runtimeOverlay runtimeMCPOverlayData
}
// MemberMayInvokeAgent reports whether userID may trigger runs for agentID.
//
// CanMemberInvokeAgent keyed by id rather than by row: channel inbound has the
// installation's agent id and no reason to load the agent itself. It asks this
// before storing a sender's message, so a member the web chat would refuse
// cannot reach the agent through a bot either.
//
// An agent that no longer exists admits nobody — that is a verdict, not a
// failure. Every other query failure comes back as an error, so an unreachable
// database is never read as a denial: the caller releases its dedup claim and
// the platform's redelivery is still the message's chance. CanMemberInvokeAgent
// is the fail-closed wrapper the scheduled triggers use instead.
func (s *TaskService) MemberMayInvokeAgent(ctx context.Context, agentID, userID pgtype.UUID) (bool, error) {
agent, err := s.Queries.GetAgent(ctx, agentID)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("load agent: %w", err)
}
return memberMayInvokeAgent(ctx, s.Queries, agent, userID, agent.WorkspaceID)
}
// PrepareChatTaskEnqueue performs reads and optional external integration work
// before the caller opens a transaction. BuildTaskOverlay may perform network
// I/O and must never run while /new holds route-rotation locks.
func (s *TaskService) PrepareChatTaskEnqueue(
ctx context.Context,
agentID, initiatorUserID pgtype.UUID,
) (PreparedChatTaskEnqueue, error) {
agent, err := s.Queries.GetAgent(ctx, agentID)
if err != nil {
slog.Error("chat task preparation failed", "agent_id", util.UUIDToString(agentID), "error", err)
return PreparedChatTaskEnqueue{}, fmt.Errorf("load agent: %w", err)
}
if agent.ArchivedAt.Valid {
return PreparedChatTaskEnqueue{}, ErrChatTaskAgentArchived
}
if !agent.RuntimeID.Valid {
return PreparedChatTaskEnqueue{}, ErrChatTaskAgentNoRuntime
}
attr := attribution.DirectHumanRun(
initiatorUserID, attribution.EvidenceChat, pgtype.UUID{},
)
attr, err = s.applyAttributionFallback(ctx, attr, agent)
if err != nil {
slog.Warn("chat task enqueue refused: attribution fail-closed",
"agent_id", util.UUIDToString(agentID))
return PreparedChatTaskEnqueue{}, err
}
attrSource, _, attrEvidenceKind, _ := attributionCreateParams(attr)
return PreparedChatTaskEnqueue{
accountableUser: attr.AccountableUserID,
attrSource: attrSource, attrEvidenceKind: attrEvidenceKind,
runtimeOverlay: s.buildRuntimeMCPOverlay(ctx, initiatorUserID, agent),
}, nil
}
// EnqueueChatTask creates a Direct Chat task. If the Chat also has a channel
// binding, its context generation is still snapshotted, but no external delivery
// snapshot is created: first-party sends reply only to first-party clients.
func (s *TaskService) EnqueueChatTask(
ctx context.Context,
chatSession db.ChatSession,
initiatorUserID pgtype.UUID,
forceFreshSession bool,
) (db.AgentTaskQueue, error) {
return s.enqueueChatTask(
ctx, chatSession, initiatorUserID, forceFreshSession, 0, false,
pgtype.UUID{}, 0,
)
}
// EnqueueChannelChatTask creates a channel-owned task for one durable context
// generation and freezes its external delivery route in the same transaction.
func (s *TaskService) EnqueueChannelChatTask(
ctx context.Context,
chatSession db.ChatSession,
initiatorUserID pgtype.UUID,
forceFreshSession bool,
contextRevision int64,
bindingID pgtype.UUID,
routeRevision int64,
) (db.AgentTaskQueue, error) {
if contextRevision <= 0 {
return db.AgentTaskQueue{}, errors.New("channel chat task requires a context revision")
}
return s.enqueueChatTask(
ctx, chatSession, initiatorUserID, forceFreshSession,
contextRevision, true, bindingID, routeRevision,
)
}
func (s *TaskService) enqueueChatTask(
ctx context.Context,
chatSession db.ChatSession,
initiatorUserID pgtype.UUID,
forceFreshSession bool,
contextRevision int64,
requireDelivery bool,
expectedBindingID pgtype.UUID,
expectedRouteRevision int64,
) (db.AgentTaskQueue, error) {
prepared, err := s.PrepareChatTaskEnqueue(ctx, chatSession.AgentID, initiatorUserID)
if err != nil {
return db.AgentTaskQueue{}, err
}
if s.TxStarter == nil {
return db.AgentTaskQueue{}, errors.New("chat task enqueue: transaction starter is required")
}
tx, err := s.TxStarter.Begin(ctx)
if err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("begin chat task enqueue: %w", err)
}
defer tx.Rollback(ctx)
task, err := s.enqueueChatTaskTx(
ctx, s.Queries.WithTx(tx), chatSession, initiatorUserID,
forceFreshSession, contextRevision, requireDelivery,
expectedBindingID, expectedRouteRevision, prepared,
)
if err != nil {
return db.AgentTaskQueue{}, err
}
if err := tx.Commit(ctx); err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("commit chat task enqueue: %w", err)
}
s.FinalizeChatTaskEnqueue(ctx, task)
return task, nil
}
// EnqueuePreparedChannelChatTaskInTx adds the first /new turn to a caller-owned
// route-rotation transaction. Preparation must happen before that transaction;
// post-commit effects belong to FinalizeChatTaskEnqueue.
func (s *TaskService) EnqueuePreparedChannelChatTaskInTx(
ctx context.Context,
tx pgx.Tx,
chatSession db.ChatSession,
initiatorUserID pgtype.UUID,
forceFreshSession bool,
contextRevision int64,
prepared PreparedChatTaskEnqueue,
) (db.AgentTaskQueue, error) {
if contextRevision <= 0 {
return db.AgentTaskQueue{}, errors.New("prepared channel chat task requires a context revision")
}
return s.enqueueChatTaskTx(
ctx, s.Queries.WithTx(tx), chatSession, initiatorUserID,
forceFreshSession, contextRevision, true, pgtype.UUID{}, 0, prepared,
)
}
func (s *TaskService) enqueueChatTaskTx(
ctx context.Context,
qtx *db.Queries,
chatSession db.ChatSession,
initiatorUserID pgtype.UUID,
forceFreshSession bool,
contextRevision int64,
requireDelivery bool,
expectedBindingID pgtype.UUID,
expectedRouteRevision int64,
prepared PreparedChatTaskEnqueue,
) (db.AgentTaskQueue, error) {
currentSession, err := qtx.LockChatSessionForEnqueue(ctx, chatSession.ID)
if err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("lock chat session: %w", err)
}
if currentSession.Status != "active" {
return db.AgentTaskQueue{}, ErrChatSessionArchived
}
agent, err := qtx.GetAgentForClaimUpdate(ctx, chatSession.AgentID)
if err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("reload chat agent: %w", err)
}
if agent.ArchivedAt.Valid {
return db.AgentTaskQueue{}, ErrChatTaskAgentArchived
}
if !agent.RuntimeID.Valid {
return db.AgentTaskQueue{}, ErrChatTaskAgentNoRuntime
}
binding, bindingErr := qtx.LockChannelChatSessionBindingForContext(ctx, chatSession.ID)
if bindingErr != nil && !errors.Is(bindingErr, pgx.ErrNoRows) {
return db.AgentTaskQueue{}, fmt.Errorf("lock channel chat binding: %w", bindingErr)
}
if requireDelivery {
if errors.Is(bindingErr, pgx.ErrNoRows) {
return db.AgentTaskQueue{}, fmt.Errorf("lock channel chat binding: %w", pgx.ErrNoRows)
}
routeMatches := expectedBindingID.Valid &&
binding.ID == expectedBindingID &&
binding.RouteRevision == expectedRouteRevision
if binding.RetiredAt.Valid && !routeMatches {
return db.AgentTaskQueue{}, fmt.Errorf("lock retired channel chat binding: %w", pgx.ErrNoRows)
}
if expectedBindingID.Valid && !routeMatches {
return db.AgentTaskQueue{}, fmt.Errorf("lock expected channel chat route: %w", pgx.ErrNoRows)
}
}
pendingFresh := false
if bindingErr == nil {
if contextRevision <= 0 {
contextRevision = binding.ContextRevision
}
generation, err := qtx.LockChannelChatContextGenerationByRevision(
ctx, db.LockChannelChatContextGenerationByRevisionParams{
ChatSessionID: chatSession.ID, Revision: contextRevision,
},
)
if err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("lock channel context generation: %w", err)
}
pendingFresh = generation.PendingFresh
}
if pendingFresh {
forceFreshSession = true
}
mediaPendingUntil, err := qtx.GetChannelMediaPendingUntil(
ctx, db.GetChannelMediaPendingUntilParams{
ChatSessionID: chatSession.ID,
ChannelContextRevision: pgtype.Int8{
Int64: contextRevision, Valid: contextRevision > 0,
},
},
)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return db.AgentTaskQueue{}, fmt.Errorf("load channel media pending deadline: %w", err)
}
if errors.Is(err, pgx.ErrNoRows) {
mediaPendingUntil = pgtype.Timestamptz{}
}
task, err := qtx.CreateChatTask(ctx, db.CreateChatTaskParams{
ID: dbid.NewV7(),
AgentID: chatSession.AgentID,
RuntimeID: agent.RuntimeID,
Priority: 2,
ChatSessionID: chatSession.ID,
InitiatorUserID: initiatorUserID,
FireAt: mediaPendingUntil,
OriginatorUserID: initiatorUserID,
AccountableUserID: prepared.accountableUser,
ForceFreshSession: pgtype.Bool{Bool: forceFreshSession, Valid: true},
RuntimeMcpOverlay: prepared.runtimeOverlay.Overlay,
RuntimeConnectedApps: prepared.runtimeOverlay.ConnectedApps,
OriginatorSource: prepared.attrSource,
TriggerEvidenceKind: prepared.attrEvidenceKind,
TriggerEvidenceRefID: chatSession.ID,
ChannelContextRevision: pgtype.Int8{
Int64: contextRevision, Valid: contextRevision > 0,
},
})
if err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("create chat task: %w", err)
}
if requireDelivery {
if _, err := qtx.CreateChannelTaskDeliveryFromSession(
ctx, db.CreateChannelTaskDeliveryFromSessionParams{
TaskID: task.ID, ChatSessionID: chatSession.ID,
ContextRevision: contextRevision,
},
); err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("snapshot channel task delivery: %w", err)
}
}
task, err = qtx.SetChatTaskInputOwnerSelf(ctx, task.ID)
if err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("set channel chat task input owner: %w", err)
}
if err := qtx.LinkUnownedChannelChatMessagesToTask(
ctx, db.LinkUnownedChannelChatMessagesToTaskParams{
TaskID: task.ID, ChatSessionID: chatSession.ID,
},
); err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("seal channel chat task input: %w", err)
}
switch corrected, err := qtx.DeferChatTaskForSealedPendingMedia(ctx, task.ID); {
case err == nil:
task = corrected
case !errors.Is(err, pgx.ErrNoRows):
return db.AgentTaskQueue{}, fmt.Errorf("defer chat task for sealed pending media: %w", err)
}
if pendingFresh {
if err := qtx.ClearChannelChatContextPendingFresh(
ctx, db.ClearChannelChatContextPendingFreshParams{
ChatSessionID: chatSession.ID, Revision: contextRevision,
},
); err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("clear channel context pending fresh: %w", err)
}
if err := qtx.ClearChannelChatSessionPendingFreshForRevision(
ctx, db.ClearChannelChatSessionPendingFreshForRevisionParams{
ChatSessionID: chatSession.ID, Revision: contextRevision,
},
); err != nil {
return db.AgentTaskQueue{}, fmt.Errorf("clear channel pending fresh: %w", err)
}
}
return task, nil
}
// FinalizeChatTaskEnqueue performs only post-commit effects.
func (s *TaskService) FinalizeChatTaskEnqueue(ctx context.Context, task db.AgentTaskQueue) {
// Every chat enqueue path stamps the message sender as a direct_human
// originator (DirectHumanRun); an unresolved sender is unattributed or
// owner_fallback and is skipped here.
if task.OriginatorSource.String == string(attribution.SourceDirectHuman) && !task.RetryOfTaskID.Valid {
s.RecordHumanDelegation(ctx, attribution.Result{UserID: task.OriginatorUserID, HumanAction: true})
}
if task.Status == "deferred" {
slog.Info("chat task deferred for channel media",
"task_id", util.UUIDToString(task.ID),
"chat_session_id", util.UUIDToString(task.ChatSessionID),
"agent_id", util.UUIDToString(task.AgentID),
"fire_at", task.FireAt.Time,
)
if err := s.PromoteChannelChatTasksIfMediaReady(ctx, task.ChatSessionID); err != nil {
slog.Warn("chat task media-ready fence failed; deferred task falls back to its deadline",
"task_id", util.UUIDToString(task.ID),
"chat_session_id", util.UUIDToString(task.ChatSessionID),
"error", err,
)
}
return
}
slog.Info("chat task enqueued",
"task_id", util.UUIDToString(task.ID),
"chat_session_id", util.UUIDToString(task.ChatSessionID),
"agent_id", util.UUIDToString(task.AgentID),
)
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, task)
s.NotifyTaskEnqueued(ctx, task)
}
// RegenerateChatQuickActions runs a fresh suggestion pass for the session's
// latest assistant turn, letting the user refresh the quick-action pills
// without sending a new message (MUL-5149). Generation happens server-side
// through the same LLM path as the automatic pass, so this enqueues no agent
// task and resumes no provider session — it just validates the target. It
// returns the target assistant message id (so the client can anchor its pending
// placeholder there) and the turn's task row, which the caller hands to
// GenerateChatQuickActionsAsync.
//
// This is an explicit user action, so it ignores the per-device quick-actions
// toggle (which only gates automatic generation at send time).
func (s *TaskService) RegenerateChatQuickActions(ctx context.Context, chatSession db.ChatSession, expectedMessageID pgtype.UUID) (targetMessageID pgtype.UUID, targetTask db.AgentTaskQueue, err error) {
if s.QuickActions == nil || !s.QuickActions.Enabled() {
return pgtype.UUID{}, db.AgentTaskQueue{}, ErrChatQuickActionsUnavailable
}
// The target is the latest assistant turn. Only an ordinary message turn
// can seed suggestions — a no_response / failure turn has nothing to build
// on, and the generator keys its write off the turn's task id.
target, err := s.Queries.GetLatestAssistantChatMessageForSession(ctx, chatSession.ID)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return pgtype.UUID{}, db.AgentTaskQueue{}, ErrChatQuickActionsNoTurn
}
return pgtype.UUID{}, db.AgentTaskQueue{}, fmt.Errorf("load latest assistant turn: %w", err)
}
if target.MessageKind != protocol.ChatMessageKindMessage || !target.TaskID.Valid {
return pgtype.UUID{}, db.AgentTaskQueue{}, ErrChatQuickActionsNoTurn
}
// Confirm the client is refreshing the turn that is STILL the latest. Under
// a multi-client race a newer reply can land between the button rendering
// and this request; regenerating then would build suggestions from the
// newer context yet attach them to the stale turn, and the client's pending
// marker would wait on a chat:quick_actions that never matches. Refuse so
// the client rolls back and re-offers refresh on the new turn (MUL-5149).
if target.ID != expectedMessageID {
return pgtype.UUID{}, db.AgentTaskQueue{}, ErrChatQuickActionsStale
}
// Refuse while a turn is already running on this session: it is about to
// replace the latest reply, so suggestions built now would be attached to a
// turn the user is seconds away from scrolling past.
busy, err := s.Queries.HasActiveChatTaskForSession(ctx, chatSession.ID)
if err != nil {
return pgtype.UUID{}, db.AgentTaskQueue{}, fmt.Errorf("check active chat task: %w", err)
}
if busy {
return pgtype.UUID{}, db.AgentTaskQueue{}, ErrChatQuickActionsBusy
}
// A refresh no longer creates a task row, so the check above cannot see a
// generation already running for this session — a second refresh (or one
// racing the automatic pass) would otherwise be accepted, spend a second
// upstream call, and race the first to write the same row. Refuse it here
// so the client rolls its optimistic marker back instead.
if s.chatQuickActionsInFlight(chatSession.ID) {
return pgtype.UUID{}, db.AgentTaskQueue{}, ErrChatQuickActionsBusy
}
task, err := s.Queries.GetAgentTask(ctx, target.TaskID)
if err != nil {
return pgtype.UUID{}, db.AgentTaskQueue{}, fmt.Errorf("load target turn task: %w", err)
}
slog.Info("chat quick-actions regenerate accepted",
"chat_session_id", util.UUIDToString(chatSession.ID),
"target_task_id", util.UUIDToString(target.TaskID),
"target_message_id", util.UUIDToString(target.ID),
)
// Validation only: the caller starts the pass. Keeping dispatch out of here
// means the refresh gates can be tested without a background goroutine
// writing to the same rows mid-assertion.
return target.ID, task, nil
}
// PromoteChannelChatTasksIfMediaReady queues channel tasks as soon as every
// unexpired media marker in the session has been cleared. If the process dies
// first, the normal deferred-task promoter queues them at their persisted
// fire_at deadline, preserving the placeholder fallback across restarts.
func (s *TaskService) PromoteChannelChatTasksIfMediaReady(ctx context.Context, sessionID pgtype.UUID) error {
tasks, err := s.Queries.PromoteChannelChatTasksIfMediaReady(ctx, sessionID)
if err != nil {
return fmt.Errorf("promote channel chat tasks after media: %w", err)
}
for _, task := range tasks {
slog.Info("channel media-ready chat task promoted",
"task_id", util.UUIDToString(task.ID),
"chat_session_id", util.UUIDToString(sessionID),
"agent_id", util.UUIDToString(task.AgentID),
)
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, task)
s.NotifyTaskEnqueued(ctx, task)
}
return nil
}
// PromoteDeferredChannelIssueTask makes a media-gated /issue task claimable.
// ErrNoRows means the deadline sweeper already promoted it (or the task was
// cancelled), so this is idempotent across the two promotion paths.
func (s *TaskService) PromoteDeferredChannelIssueTask(ctx context.Context, taskID pgtype.UUID) error {
task, err := s.Queries.PromoteDeferredChannelIssueTask(ctx, taskID)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return fmt.Errorf("promote deferred channel issue task: %w", err)
}
slog.Info("channel media-ready issue task promoted",
"task_id", util.UUIDToString(task.ID),
"issue_id", util.UUIDToString(task.IssueID),
"agent_id", util.UUIDToString(task.AgentID),
)
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, task)
s.NotifyTaskEnqueued(ctx, task)
return nil
}
// DirectChatSendResult carries the rows a transactional direct-chat send
// persisted, so the handler can broadcast the user message and shape its
// response without re-reading them.
type DirectChatSendResult struct {
Task db.AgentTaskQueue
Message db.ChatMessage
BoundAttachmentIDs []pgtype.UUID
Queued bool
InitialTitle string
}
var ErrChatSessionAlreadyStarted = errors.New("chat session already has a user message")
// SendDirectChatMessage atomically persists one web/mobile direct-chat turn:
// the owning task (which claims its own input batch via chat_input_task_id), the
// user message bound to that task, any attachment bindings, and the session
// touch all commit together (MUL-4351). The daemon is only notified after the
// commit, so it can never observe a message without a task or a task without its
// input owner — and a later claim reads exactly this task's user messages
// instead of scanning trailing history.
//
// The caller must have already gated the session and preflighted the agent
// (archived / no-runtime), passing the loaded agent in. Those checks are repeated
// under the transaction locks below because either row may change before enqueue.
func (s *TaskService) SendDirectChatMessage(
ctx context.Context,
session db.ChatSession,
agent db.Agent,
initiatorUserID pgtype.UUID,
content string,
attachmentIDs []pgtype.UUID,
uploaderType string,
uploaderID pgtype.UUID,
) (*DirectChatSendResult, error) {
// Build the per-task Composio overlay before the transaction — it can do
// network I/O and must not run with a DB transaction open.
overlay := s.buildRuntimeMCPOverlay(ctx, initiatorUserID, agent)
// Full attribution for the chat sender, resolved before the tx (the policy read
// + fallback must not run with a transaction open) — the same direct_human stamp
// EnqueueChatTask writes. Without this the direct-chat path was a bypass: it set
// originator_user_id but left accountable_user_id / source / evidence NULL,
// violating the one-way invariant and dropping the audit source (MUL-4302 §2).
attr := attribution.DirectHumanRun(initiatorUserID, attribution.EvidenceChat, session.ID)
attr, err := s.applyAttributionFallback(ctx, attr, agent)
if err != nil {
return nil, err
}
attrSource, _, attrEvidenceKind, attrEvidenceRef := attributionCreateParams(attr)
var out DirectChatSendResult
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
// Serialise this send against a concurrent runtime rebind of the same
// session (MUL-5163). The lock must be taken first and the agent re-read
// under it: the runtime_id the caller loaded can already be stale by the
// time we get here, and a send blocked behind a rebind would otherwise
// resume and stamp its task with the runtime the switch just moved away
// from — leaving the user with a "switched" confirmation and a reply
// running on the old runtime. Locking chat_session first also matches the
// delete path's lock order, so the two cannot deadlock.
if _, err := qtx.LockChatSessionForRuntimeBind(ctx, session.ID); err != nil {
return fmt.Errorf("lock chat session: %w", err)
}
currentSession, err := qtx.GetChatSession(ctx, session.ID)
if err != nil {
return fmt.Errorf("reload chat session: %w", err)
}
if currentSession.Status != "active" {
return ErrChatSessionArchived
}
carrier, err := qtx.GetAgentForClaimUpdate(ctx, session.AgentID)
if err != nil {
return fmt.Errorf("reload chat agent: %w", err)
}
if carrier.ArchivedAt.Valid {
return ErrChatTaskAgentArchived
}
if !carrier.RuntimeID.Valid {
return ErrChatTaskAgentNoRuntime
}
// The database status of every newly-created task is "queued" until a
// daemon claims it. Product queue semantics are positional instead: this
// send is a follow-up only when another visible task in the same session
// is already ahead of it. Deferred retries count because they resume an
// older turn before this one. The session + agent locks serialize sibling
// sends, retries, terminal writes, and claims around this read.
queued, err := qtx.HasPendingChatTurnForSession(ctx, session.ID)
if err != nil {
return fmt.Errorf("check direct chat queue position: %w", err)
}
out.Queued = queued
task, err := qtx.CreateChatTask(ctx, db.CreateChatTaskParams{
ID: dbid.NewV7(),
AgentID: session.AgentID,
RuntimeID: carrier.RuntimeID,
Priority: 2, // medium priority for chat; matches EnqueueChatTask
ChatSessionID: session.ID,
InitiatorUserID: initiatorUserID,
OriginatorUserID: attr.UserID,
AccountableUserID: attr.AccountableUserID,
ForceFreshSession: pgtype.Bool{Bool: false, Valid: true},
RuntimeMcpOverlay: overlay.Overlay,
RuntimeConnectedApps: overlay.ConnectedApps,
OriginatorSource: attrSource,
TriggerEvidenceKind: attrEvidenceKind,
TriggerEvidenceRefID: attrEvidenceRef,
})
if err != nil {
return fmt.Errorf("create direct chat task: %w", err)
}
// Claim this task's own input batch (chat_input_task_id = id) in the same
// transaction, before the user message is written with task_id = task.id.
task, err = qtx.SetChatTaskInputOwnerSelf(ctx, task.ID)
if err != nil {
return fmt.Errorf("stamp direct chat input owner: %w", err)
}
out.Task = task
// Adopt the onboarding kickoff, if this session still has an unowned one.
// It is written by OpenMikaOnboardingChat with no task, so this is the
// only thing that ever delivers it to a runtime — and it must happen
// before the member's own row is written, so the batch reads as
// "context, then their message" once ordered by created_at.
//
// A no-op for every other session: only Mika onboarding writes that kind,
// and only the first send of one finds it unowned.
if err := qtx.AdoptOrphanOnboardingKickoff(ctx, db.AdoptOrphanOnboardingKickoffParams{
ChatSessionID: session.ID,
TaskID: task.ID,
}); err != nil {
return fmt.Errorf("adopt onboarding kickoff: %w", err)
}
// Initialize an explicitly empty Chat before inserting its first public
// user message. The CAS query also protects a manual rename and competing
// first send; keeping it in this transaction prevents a title-only commit.
if title := chattitle.Derive(content); title != "" {
if _, err := qtx.InitializeChatSessionTitle(ctx, db.InitializeChatSessionTitleParams{ID: session.ID, Title: title}); err == nil {
out.InitialTitle = title
} else if !errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("initialize direct chat title: %w", err)
}
}
// Create the user message already owned by this task (task_id = task.id),
// so it belongs to this immutable input batch the instant it exists.
msg, err := qtx.CreateChatMessage(ctx, db.CreateChatMessageParams{
ID: dbid.NewV7(),
ChatSessionID: session.ID,
Role: "user",
Content: content,
TaskID: task.ID,
MessageKind: pgtype.Text{String: protocol.ChatMessageKindMessage, Valid: true},
})
if err != nil {
return fmt.Errorf("create user chat message: %w", err)
}
out.Message = msg
if len(attachmentIDs) > 0 {
bound, err := qtx.LinkAttachmentsToChatMessage(ctx, db.LinkAttachmentsToChatMessageParams{
ChatMessageID: msg.ID,
ChatSessionID: session.ID,
WorkspaceID: session.WorkspaceID,
UploaderType: uploaderType,
UploaderID: uploaderID,
AttachmentIds: attachmentIDs,
})
if err != nil {
return fmt.Errorf("link chat attachments: %w", err)
}
out.BoundAttachmentIDs = bound
// An explicitly empty /new Chat can receive its first turn from a
// first-party client. For an attachment-only turn, initialize the
// title from the first attachment that was actually bound. The media
// title CAS runs after the message insert by design and still refuses
// to overwrite a manual rename or a competing first message.
if out.InitialTitle == "" && strings.TrimSpace(content) == "" && len(bound) > 0 {
attachments, err := qtx.ListAttachmentsByChatMessage(ctx, db.ListAttachmentsByChatMessageParams{
ChatMessageID: msg.ID,
WorkspaceID: session.WorkspaceID,
})
if err != nil {
return fmt.Errorf("list bound chat attachments for title: %w", err)
}
if len(attachments) > 0 {
title := chattitle.Derive(attachments[0].Filename)
if title != "" {
if _, err := qtx.InitializeChatSessionMediaTitle(ctx, db.InitializeChatSessionMediaTitleParams{
ID: session.ID, MessageID: msg.ID, Title: title,
}); err == nil {
out.InitialTitle = title
} else if !errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("initialize direct attachment chat title: %w", err)
}
}
}
}
}
if err := qtx.TouchChatSession(ctx, session.ID); err != nil {
return fmt.Errorf("touch chat session: %w", err)
}
return nil
}); err != nil {
slog.Error("direct chat send failed",
"chat_session_id", util.UUIDToString(session.ID),
"agent_id", util.UUIDToString(session.AgentID),
"error", err)
return nil, err
}
slog.Info("direct chat task enqueued",
"task_id", util.UUIDToString(out.Task.ID),
"chat_session_id", util.UUIDToString(session.ID),
"agent_id", util.UUIDToString(session.AgentID))
s.RecordHumanDelegation(ctx, attr)
// Notify only after commit. See EnqueueTaskForIssue for ordering rationale.
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, out.Task)
s.NotifyTaskEnqueued(ctx, out.Task)
return &out, nil
}
// MikaOnboardingOpenResult carries the two rows that open a Mika conversation.
type MikaOnboardingOpenResult struct {
// Kickoff is the hidden product context. It is written WITHOUT a task —
// the member's first real send adopts it (AdoptOrphanOnboardingKickoff).
Kickoff db.ChatMessage
// Opening is what the member reads, already final. No agent produced it,
// so it carries no task id, no elapsed time, and nothing to regenerate.
Opening db.ChatMessage
}
// OpenMikaOnboardingChat writes a Mika conversation's first two rows in one
// transaction: the hidden kickoff and the product-authored opening the member
// sees (MUL-5827). Nothing is enqueued — this used to be a full chat task, and
// the member waited out a runtime cold start to read a reply the product had
// already decided word for word.
//
// "Session is still empty" is enforced under the chat-session lock, so retries,
// React double-submits, and two clients racing the same session still produce
// at most one opening. The kickoff row is what makes that check work: it is a
// role='user' row, so ChatSessionHasUserMessage sees it exactly as it saw the
// old kickoff turn.
func (s *TaskService) OpenMikaOnboardingChat(ctx context.Context, session db.ChatSession, kickoff, opening string) (*MikaOnboardingOpenResult, error) {
var out MikaOnboardingOpenResult
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
// Same lock and lock ORDER as the send path, so an opening racing a
// first send or a runtime rebind serializes instead of deadlocking.
if _, err := qtx.LockChatSessionForRuntimeBind(ctx, session.ID); err != nil {
return fmt.Errorf("lock chat session: %w", err)
}
current, err := qtx.GetChatSession(ctx, session.ID)
if err != nil {
return fmt.Errorf("reload chat session: %w", err)
}
if current.Status != "active" {
return ErrChatSessionArchived
}
hasUserMessage, err := qtx.ChatSessionHasUserMessage(ctx, session.ID)
if err != nil {
return fmt.Errorf("check chat session input: %w", err)
}
if hasUserMessage {
return ErrChatSessionAlreadyStarted
}
kickoffRow, err := qtx.CreateChatMessage(ctx, db.CreateChatMessageParams{
ID: dbid.NewV7(),
ChatSessionID: session.ID,
Role: "user",
Content: kickoff,
MessageKind: pgtype.Text{String: protocol.ChatMessageKindOnboardingKickoff, Valid: true},
})
if err != nil {
return fmt.Errorf("create onboarding kickoff: %w", err)
}
out.Kickoff = kickoffRow
// Ordered one microsecond after the kickoff — see the query comment for
// why a shared transaction timestamp is not good enough here.
openingRow, err := qtx.CreateMikaOnboardingOpening(ctx, db.CreateMikaOnboardingOpeningParams{
ID: dbid.NewV7(),
ChatSessionID: session.ID,
KickoffCreatedAt: kickoffRow.CreatedAt,
Content: opening,
})
if err != nil {
return fmt.Errorf("create onboarding opening: %w", err)
}
out.Opening = openingRow
if err := qtx.TouchChatSession(ctx, session.ID); err != nil {
return fmt.Errorf("touch chat session: %w", err)
}
return nil
}); err != nil {
if !errors.Is(err, ErrChatSessionAlreadyStarted) {
slog.Error("mika onboarding open failed",
"chat_session_id", util.UUIDToString(session.ID),
"agent_id", util.UUIDToString(session.AgentID),
"error", err)
}
return nil, err
}
return &out, nil
}
// CancelTasksForIssue cancels every active task on the issue, reconciles each
// affected agent's status, and broadcasts task:cancelled events so frontends
// clear their live cards.
//
// Callers are explicit issue-lifecycle cleanup paths only — DeleteIssue and
// BatchDeleteIssues, where the owning issue row is going away so its tasks
// must not be left orphaned. A plain status flip, `cancelled` included, no
// longer routes here (MUL-4465): cancelling an issue is not an implicit "stop
// all runs" switch. Do not re-add a status-driven caller.
//
// Before #1587 this path was "cancel rows and return", which left each affected
// agent stuck at status="working" indefinitely, requiring a manual
// `multica agent update <id> --status idle` to unwedge. It now reconciles agent
// status and broadcasts task:cancelled, matching CancelTask and RerunIssue.
func (s *TaskService) CancelTasksForIssue(ctx context.Context, issueID pgtype.UUID) error {
var cancelled []db.AgentTaskQueue
// The cancel and its settlement commit together: a settlement that failed
// after the cancel committed could never be repaired, because the outbox
// scan excludes a comment whose covering task is terminal and already holds
// the receipt. It would neither replay nor settle — it would sit in the
// partial index forever.
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
var err error
cancelled, err = qtx.CancelAgentTasksByIssue(ctx, issueID)
if err != nil {
return err
}
return SettleTerminalTaskState(ctx, qtx, cancelled...)
}); err != nil {
return err
}
for _, t := range cancelled {
s.captureTaskCancelled(ctx, t)
s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, t)
}
// Reconcile once per distinct agent instead of once per cancelled row:
// cancelling an issue often stops several tasks owned by the same agent,
// and each reconcile is a DB write plus a status broadcast. Matches
// CancelTasksForAgent's single-reconcile shape (D#3319).
for _, agentID := range distinctAgentIDs(cancelled) {
s.ReconcileAgentStatus(ctx, agentID)
}
s.notifyTasksFinished(cancelled)
return nil
}
// distinctAgentIDs returns each agent id appearing in the cancelled rows once,
// preserving first-seen order. Bulk cancellations frequently stop several tasks
// owned by the same agent; reconciling per distinct agent (rather than per row)
// collapses the redundant RefreshAgentStatusFromTasks writes and status
// broadcasts down to one per agent without changing the final agent status.
func distinctAgentIDs(cancelled []db.AgentTaskQueue) []pgtype.UUID {
seen := make(map[pgtype.UUID]struct{}, len(cancelled))
ids := make([]pgtype.UUID, 0, len(cancelled))
for _, t := range cancelled {
if _, dup := seen[t.AgentID]; dup {
continue
}
seen[t.AgentID] = struct{}{}
ids = append(ids, t.AgentID)
}
return ids
}
// CancelTasksForAgent cancels every active task belonging to an agent
// (queued + dispatched + running), reconciles the agent's status, and
// broadcasts task:cancelled events. Used by the agent-level "Cancel all
// tasks" action — same shape as CancelTasksForIssue but scoped on agent_id.
//
// Returns the cancelled rows so callers can report counts / log them.
func (s *TaskService) CancelTasksForAgent(ctx context.Context, agentID pgtype.UUID) ([]db.AgentTaskQueue, error) {
var cancelled []db.AgentTaskQueue
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
var err error
cancelled, err = qtx.CancelAgentTasksByAgent(ctx, agentID)
if err != nil {
return err
}
return SettleTerminalTaskState(ctx, qtx, cancelled...)
}); err != nil {
return nil, err
}
for _, t := range cancelled {
s.captureTaskCancelled(ctx, t)
s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, t)
}
// Reconcile once after the loop — agent transitions from
// working→available based on remaining task counts, no need to call
// per row (the rows we just cancelled all belong to the same agent).
s.ReconcileAgentStatus(ctx, agentID)
s.notifyTasksFinished(cancelled)
return cancelled, nil
}
// CancelTasksByTriggerComment cancels active tasks whose planned comment batch
// contains the given edited/deleted comment. The historical method name is
// retained for call-site stability. It must run before deletion clears the
// trigger FK; the returned rows let the handler re-route every surviving input.
func (s *TaskService) CancelTasksByTriggerComment(ctx context.Context, commentID pgtype.UUID) ([]db.AgentTaskQueue, error) {
var cancelled []db.AgentTaskQueue
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
var err error
cancelled, err = qtx.CancelAgentTasksByTriggerComment(ctx, commentID)
if err != nil {
return err
}
return SettleTerminalTaskState(ctx, qtx, cancelled...)
}); err != nil {
return nil, err
}
for _, t := range cancelled {
s.captureTaskCancelled(ctx, t)
s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, t)
}
// Reconcile once per distinct agent instead of once per cancelled row: an
// edited/deleted trigger comment can cancel several tasks owned by the same
// agent, and each reconcile is a DB write plus a status broadcast (D#3319).
for _, agentID := range distinctAgentIDs(cancelled) {
s.ReconcileAgentStatus(ctx, agentID)
}
s.notifyTasksFinished(cancelled)
return cancelled, nil
}
// BroadcastCancelledTasks reconciles each affected agent's status and emits
// task:cancelled for every row. Callers must invoke this AFTER committing the
// cancellation so subscribers don't observe a "cancelled" event for a row
// that the tx might still roll back.
//
// workspaceID comes from the caller instead of being resolved per task, because
// the transaction these callers have just committed can delete the row the
// resolution would read. A chat task's workspace is reached through its
// chat_session, and both DeleteChatSession and the runtime teardown remove that
// session — the teardown by deleting the system agent it hangs off. Afterwards
// ResolveTaskWorkspaceID finds nothing and returns "", and publishTaskEvent
// drops an event with no workspace before it reaches the bus: the rows are
// cancelled, nobody is told, and every queue view and channel indicator keeps
// showing a run that no longer exists. Each caller already knows the workspace
// — it is the one whose session, member or runtime is being torn down — so the
// lookup is not needed and cannot fail.
//
// reactionTargets optionally carries anchors captured before deleting the
// channel delivery rows. They stay on the internal event, outside its payload.
func (s *TaskService) BroadcastCancelledTasks(ctx context.Context, workspaceID string, cancelled []db.AgentTaskQueue, reactionTargets ...map[string]*events.ChannelReactionTarget) {
for _, t := range cancelled {
s.captureTaskCancelled(ctx, t)
s.ReconcileAgentStatus(ctx, t.AgentID)
if workspaceID != "" {
e := taskEvent(protocol.EventTaskCancelled, workspaceID, t)
if len(reactionTargets) > 0 {
e.ChannelReactionTarget = reactionTargets[0][util.UUIDToString(t.ID)]
}
s.Bus.Publish(e)
}
}
s.notifyTasksFinished(cancelled)
}
// BroadcastTaskQueued emits a post-commit queue invalidation for clients.
func (s *TaskService) BroadcastTaskQueued(ctx context.Context, task db.AgentTaskQueue) {
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, task)
}
// BroadcastTaskCoalesced tells clients a pending run took in another comment.
// The run keeps its status, but its trigger and covered comments changed, and
// clients place the run under the newest comment it answers.
func (s *TaskService) BroadcastTaskCoalesced(ctx context.Context, task db.AgentTaskQueue) {
s.broadcastTaskEvent(ctx, protocol.EventTaskCoalesced, task)
}
// CaptureCancelledTasks records analytics for an already-committed bulk
// cancellation. It deliberately does NOT settle delegated-failure recoveries:
// by the time it runs the cancel has committed, so a failure here could not be
// rolled back and the stranded receipt could never be repaired. Callers settle
// inside the transaction that performs the cancel.
func (s *TaskService) CaptureCancelledTasks(ctx context.Context, cancelled []db.AgentTaskQueue) {
for _, t := range cancelled {
s.captureTaskCancelled(ctx, t)
}
}
type CancelledChatMessageResult struct {
ChatSessionID string
MessageID string
Content string
RestoreToInput bool
// Attachments are the rows detached from the deleted user message so they
// survive the ON DELETE CASCADE and can re-bind when the restored draft is
// re-sent.
Attachments []db.Attachment
}
type CancelTaskResult struct {
Task db.AgentTaskQueue
CancelledChatMessage *CancelledChatMessageResult
}
var ErrTaskNoLongerQueued = errors.New("task is no longer queued")
// TaskCancellationActor is the point-in-time identity written onto a task when
// an authenticated member or agent explicitly stops it. Automatic paths leave
// this empty and the SQL transition records the system actor instead.
type TaskCancellationActor struct {
Type string
ID pgtype.UUID
Name string
}
// CancelTaskOptions carries what the caller knows about the client that asked
// for the cancellation.
type CancelTaskOptions struct {
// ClientSupportsDraftRestore is true when the caller can recover a prompt
// through the durable draft-restore path (#5219). Only such a client may be
// handed a deferred outcome; for anyone else the empty-transcript judgment
// stays synchronous, because the cancel response is their only chance to get
// the prompt back. See protocol.AppCapabilityChatDraftRestoreV1.
ClientSupportsDraftRestore bool
// QueuedOnly turns queue edit/remove into a session-scoped compare-and-set.
QueuedOnly bool
ExpectedChatSession pgtype.UUID
QueueAction string
// ErrorMessage / FailureReason, when set, are persisted onto the cancelled
// row. Only for cancellations the USER did not ask for (e.g. the worktree
// claim gate refusing a too-old daemon): a user-initiated cancel needs no
// explanation, but a server-initiated one without a persisted reason
// surfaces as an unexplained "cancelled" whose only trace is a 4xx in a
// daemon log the user never sees.
ErrorMessage string
FailureReason string
CancelledBy TaskCancellationActor
// UserInitiated distinguishes the issue UI/API cancel action from automatic
// server repairs. An explicit user cancellation terminally acknowledges any
// delegated-failure recovery signal planned into the task; automatic
// cancellations must leave that signal replayable.
UserInitiated bool
}
// CancelTask cancels a single task by ID for an automatic server path. It does
// not acknowledge delegated-failure recovery inputs; use CancelTaskByUser for
// an explicit issue-task cancellation. It broadcasts a task:cancelled event so
// frontends can update immediately.
func (s *TaskService) CancelTask(ctx context.Context, taskID pgtype.UUID) (*db.AgentTaskQueue, error) {
// Callers of this wrapper are automatic non-chat repair/daemon paths, so
// finalizeCancelledChatMessage returns before the gate is even read.
// Should a chat task ever reach here, there is no client waiting on a
// synchronous restore anyway, and the durable path is the only one that can
// hand the prompt back at all.
result, err := s.CancelTaskWithResult(ctx, taskID, CancelTaskOptions{ClientSupportsDraftRestore: true})
if err != nil {
return nil, err
}
return &result.Task, nil
}
// CancelTaskByUser is the explicit issue-task cancellation path. Unlike an
// automatic server cancellation, it terminally acknowledges any delegated-
// failure recovery signal carried by the task so the sweeper respects the
// user's decision instead of recreating the task.
func (s *TaskService) CancelTaskByUser(ctx context.Context, taskID pgtype.UUID, actor TaskCancellationActor) (*db.AgentTaskQueue, error) {
result, err := s.CancelTaskWithResult(ctx, taskID, CancelTaskOptions{
ClientSupportsDraftRestore: true,
CancelledBy: actor,
UserInitiated: true,
})
if err != nil {
return nil, err
}
return &result.Task, nil
}
// CancelTaskWithReason cancels a task the SERVER decided to stop, persisting an
// actionable reason onto the row. It runs the same full cancellation flow as
// CancelTask — audit capture, chat settle, agent status reconcile,
// task:cancelled broadcast, NotifyTaskFinished — because a raw
// CancelAgentTaskWithReason query bypass leaves the agent pill running, the
// live card stale, and any capacity/serial waiter unwoken.
func (s *TaskService) CancelTaskWithReason(ctx context.Context, taskID pgtype.UUID, errorMessage, failureReason string) (*db.AgentTaskQueue, error) {
result, err := s.CancelTaskWithResult(ctx, taskID, CancelTaskOptions{
ClientSupportsDraftRestore: true,
ErrorMessage: errorMessage,
FailureReason: failureReason,
})
if err != nil {
return nil, err
}
return &result.Task, nil
}
// CancelTaskWithResult cancels a single task and returns any chat-specific
// cleanup result needed by user-facing callers.
func (s *TaskService) CancelTaskWithResult(ctx context.Context, taskID pgtype.UUID, opts CancelTaskOptions) (*CancelTaskResult, error) {
// Both fields are persisted onto the cancelled row's TEXT columns below, and
// at least one caller interpolates an externally-supplied path into
// ErrorMessage. A NUL in either rolls the cancellation back and leaves the
// task running — the same wedge as GH #7098 on the fail/complete paths.
opts.ErrorMessage = util.SanitizeTextForPostgres(opts.ErrorMessage)
opts.FailureReason = util.SanitizeTextForPostgres(opts.FailureReason)
opts.CancelledBy.Name = util.SanitizeTextForPostgres(opts.CancelledBy.Name)
if opts.UserInitiated && (opts.ErrorMessage != "" || opts.FailureReason != "") {
return nil, errors.New("user-initiated cancellation cannot carry a server failure reason")
}
if opts.UserInitiated &&
(opts.CancelledBy.Type != "member" && opts.CancelledBy.Type != "agent") {
return nil, errors.New("user-initiated cancellation requires a member or agent actor")
}
if opts.UserInitiated && !opts.CancelledBy.ID.Valid {
return nil, errors.New("user-initiated cancellation requires an actor id")
}
var (
task db.AgentTaskQueue
cancelledChatMessage *CancelledChatMessageResult
err error
)
if opts.QueuedOnly {
if opts.QueueAction != "edit" && opts.QueueAction != "remove" {
return nil, errors.New("queue action must be edit or remove")
}
err = s.runInTx(ctx, func(qtx *db.Queries) error {
if _, err := qtx.LockChatSessionForTask(ctx, taskID); err != nil {
return fmt.Errorf("lock queued chat session: %w", err)
}
task, err = qtx.CancelQueuedAgentTask(ctx, db.CancelQueuedAgentTaskParams{
ID: taskID,
ChatSessionID: opts.ExpectedChatSession,
CancelledByType: pgtype.Text{String: opts.CancelledBy.Type, Valid: opts.CancelledBy.Type != ""},
CancelledByID: opts.CancelledBy.ID,
CancelledByName: pgtype.Text{String: opts.CancelledBy.Name, Valid: opts.CancelledBy.Name != ""},
})
if errors.Is(err, pgx.ErrNoRows) {
return ErrTaskNoLongerQueued
}
if err != nil {
return fmt.Errorf("cancel queued task: %w", err)
}
cancelledChatMessage, err = s.settleQueuedChatInput(ctx, qtx, task, opts.QueueAction)
return err
})
} else {
// The status flip and the chat resume-pointer advance commit together. Split
// across two statements, the `cancelled` status becomes visible to every
// other connection while the pointer still names the previous turn's
// session, and a queued follow-up can resume that older session.
err = s.runInTx(ctx, func(qtx *db.Queries) error {
if err := lockChatSessionForTaskWrite(ctx, qtx, taskID); err != nil {
return err
}
var (
cancelled db.AgentTaskQueue
err error
)
if opts.UserInitiated {
cancelled, err = qtx.CancelAgentTaskByUser(ctx, db.CancelAgentTaskByUserParams{
ID: taskID,
CancelledByType: pgtype.Text{String: opts.CancelledBy.Type, Valid: true},
CancelledByID: opts.CancelledBy.ID,
CancelledByName: pgtype.Text{String: opts.CancelledBy.Name, Valid: opts.CancelledBy.Name != ""},
})
} else if opts.ErrorMessage != "" || opts.FailureReason != "" {
cancelled, err = qtx.CancelAgentTaskWithReason(ctx, db.CancelAgentTaskWithReasonParams{
ID: taskID,
Error: pgtype.Text{String: opts.ErrorMessage, Valid: opts.ErrorMessage != ""},
FailureReason: pgtype.Text{String: opts.FailureReason, Valid: opts.FailureReason != ""},
})
} else {
cancelled, err = qtx.CancelAgentTask(ctx, taskID)
}
if err != nil {
return err
}
task = cancelled
if err := qtx.MarkPiTaskProcessCleanupPending(ctx, task.ID); err != nil {
return fmt.Errorf("mark cancelled runtime cleanup pending: %w", err)
}
// CancelAgentTaskByUser appends the recovery receipt in the same
// statement, so the returned row already carries it.
if err := SettleTerminalTaskState(ctx, qtx, cancelled); err != nil {
return err
}
if !cancelled.ChatSessionID.Valid {
return nil
}
return qtx.AdvanceCancelledChatSessionPointer(ctx, cancelled.ID)
})
}
if errors.Is(err, ErrTaskNoLongerQueued) {
return nil, err
}
if errors.Is(err, pgx.ErrNoRows) {
existing, err := s.Queries.GetAgentTask(ctx, taskID)
if err != nil {
return nil, fmt.Errorf("cancel task: %w", err)
}
return &CancelTaskResult{Task: existing}, nil
}
if err != nil {
return nil, fmt.Errorf("cancel task: %w", err)
}
slog.Info("task cancelled", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(task.IssueID))
s.captureTaskCancelled(ctx, task)
if !opts.QueuedOnly {
cancelledChatMessage = s.finalizeCancelledChatMessage(ctx, task, opts)
}
// Reconcile agent status
s.ReconcileAgentStatus(ctx, task.AgentID)
// Broadcast cancellation as a task:failed event so frontends clear the live card
s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, task)
s.NotifyTaskFinished(task)
return &CancelTaskResult{
Task: task,
CancelledChatMessage: cancelledChatMessage,
}, nil
}
// CancelQueuedChatTasks atomically cancels every queued follow-up in a chat
// session. The session lock preserves the delete path's session -> agent -> task
// order; the agent lock then prevents ClaimTask from promoting a row mid-update.
func (s *TaskService) CancelQueuedChatTasks(ctx context.Context, sessionID, agentID pgtype.UUID) error {
var tasks []db.AgentTaskQueue
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
if _, err := qtx.LockChatSessionForDelete(ctx, sessionID); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
return fmt.Errorf("lock chat session: %w", err)
}
if _, err := qtx.GetAgentForClaimUpdate(ctx, agentID); err != nil {
return fmt.Errorf("lock chat agent: %w", err)
}
var err error
tasks, err = qtx.CancelQueuedAgentTasksForSession(ctx, sessionID)
if err != nil {
return fmt.Errorf("cancel queued chat tasks: %w", err)
}
if err := SettleTerminalTaskState(ctx, qtx, tasks...); err != nil {
return err
}
for _, task := range tasks {
if _, err := s.settleQueuedChatInput(ctx, qtx, task, "remove"); err != nil {
return err
}
}
return nil
}); err != nil {
return err
}
for _, task := range tasks {
slog.Info("task cancelled", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(task.IssueID))
s.captureTaskCancelled(ctx, task)
}
if len(tasks) > 0 {
s.ReconcileAgentStatus(ctx, agentID)
}
for _, task := range tasks {
s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, task)
}
s.notifyTasksFinished(tasks)
return nil
}
// lockChatSessionForTaskWrite takes the chat_session row a task belongs to. It
// must be the FIRST statement of any transaction that ends up holding both that
// session row and the task's own row, which is every terminal-state path a chat
// task has: complete, fail, cancel, the cancelled-turn finalize, and the
// daemon's mid-flight pin.
//
// chat_session -> agent_task_queue is the repo-wide order (see
// LockChatSessionForTask in chat.sql). DeleteChatSession and
// FinalizeDeferredCancelledChat already took it; the terminal reports did not,
// because they only ever wrote the task row first and the session second and
// nothing else contended for both. Once the cancel and pin paths started
// holding the session row too, "task first" and "session first" existed side by
// side and any crossing pair could deadlock — PostgreSQL aborts one with 40P01
// and runInTx has no retry. There is no per-path fix for that: either every
// writer agrees or none of them are safe.
//
// ErrNoRows means there is nothing to lock — a non-chat task, or one whose
// session was already deleted (the FK NULLs the column) — and the caller then
// has no session write to protect either.
func lockChatSessionForTaskWrite(ctx context.Context, qtx *db.Queries, taskID pgtype.UUID) error {
if _, err := qtx.LockChatSessionForTask(ctx, taskID); err != nil && !errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("lock chat session for task write: %w", err)
}
return nil
}
// chatInputOwnerID resolves the id the task's user-message input batch is
// keyed on: chat_input_task_id when set (auto-retry clones inherit their
// parent's, so provenance checks reach the parent's sealed messages), falling
// back to the task's own id for legacy rows.
func chatInputOwnerID(task db.AgentTaskQueue) pgtype.UUID {
if task.ChatInputTaskID.Valid {
return task.ChatInputTaskID
}
return task.ID
}
// createAssistantChatMessage writes an assistant outcome and reanchors the
// newly-visible queued direct head in the caller's transaction. Keeping the two
// statements together is the transcript-order boundary: readers must not see
// the reply commit while the next head still carries its older enqueue time.
// The caller MUST observe the settling task outside the visible-head status set
// before invoking this helper; completion and failure settle it earlier in the
// same transaction, while cancellation commits that status in its prior
// transaction (#5219). Otherwise the head query still selects the settling task
// and reanchoring its successor is a no-op.
func createAssistantChatMessage(ctx context.Context, qtx *db.Queries, params db.CreateChatMessageParams) (db.ChatMessage, error) {
if params.Role != "assistant" {
return db.ChatMessage{}, fmt.Errorf("create assistant chat message: invalid role %q", params.Role)
}
row, err := qtx.CreateChatMessage(ctx, params)
if err != nil {
return db.ChatMessage{}, err
}
if err := qtx.ReanchorNextQueuedDirectChatInput(ctx, db.ReanchorNextQueuedDirectChatInputParams{
AssistantCreatedAt: row.CreatedAt,
ChatSessionID: row.ChatSessionID,
}); err != nil {
return db.ChatMessage{}, fmt.Errorf("reanchor next queued direct chat input: %w", err)
}
return row, nil
}
func (s *TaskService) settleQueuedChatInput(
ctx context.Context,
qtx *db.Queries,
task db.AgentTaskQueue,
action string,
) (*CancelledChatMessageResult, error) {
if !task.ChatSessionID.Valid {
return nil, nil
}
inputOwnerID := chatInputOwnerID(task)
channelIngested, err := qtx.TaskHasChannelIngestedMessages(ctx, inputOwnerID)
if err != nil {
return nil, fmt.Errorf("check queued chat channel provenance: %w", err)
}
if channelIngested {
if _, err := createAssistantChatMessage(ctx, qtx, db.CreateChatMessageParams{
ID: dbid.NewV7(),
ChatSessionID: task.ChatSessionID,
Role: "assistant",
Content: "Stopped.",
TaskID: task.ID,
ElapsedMs: computeChatElapsedMs(task),
}); err != nil {
return nil, fmt.Errorf("create cancelled queued chat message: %w", err)
}
return nil, nil
}
var detached []db.Attachment
if action == "edit" {
detached, err = qtx.DetachAttachmentsFromUserChatMessageByTask(ctx, inputOwnerID)
if err != nil {
return nil, fmt.Errorf("detach edited queued chat attachments: %w", err)
}
}
deleted, err := deleteUserChatInput(ctx, qtx, inputOwnerID)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("delete queued chat input: %w", err)
}
cancelled := &CancelledChatMessageResult{
ChatSessionID: util.UUIDToString(deleted.ChatSessionID),
MessageID: util.UUIDToString(deleted.ID),
Content: deleted.Content,
}
if action == "remove" {
return cancelled, nil
}
attachmentIDs := make([]pgtype.UUID, 0, len(detached))
for _, attachment := range detached {
attachmentIDs = append(attachmentIDs, attachment.ID)
}
if _, err := qtx.CreateChatDraftRestore(ctx, db.CreateChatDraftRestoreParams{
ID: deleted.ID,
ChatSessionID: task.ChatSessionID,
TaskID: task.ID,
Content: deleted.Content,
AttachmentIds: attachmentIDs,
}); err != nil {
return nil, fmt.Errorf("create queued chat draft restore: %w", err)
}
cancelled.RestoreToInput = true
cancelled.Attachments = detached
return cancelled, nil
}
// deleteUserChatInput removes a cancelled/edited turn's member-typed input and
// releases any onboarding kickoff the turn had adopted — onto the session's
// next queued turn when there is one, otherwise back to unowned (MUL-5827).
// The two must happen together: deleting the input while leaving the kickoff
// bound to the dead task would strand the onboarding context on a run that
// will never happen.
func deleteUserChatInput(ctx context.Context, qtx *db.Queries, inputOwnerID pgtype.UUID) (db.ChatMessage, error) {
if err := qtx.ReleaseOnboardingKickoffFromTask(ctx, inputOwnerID); err != nil {
return db.ChatMessage{}, fmt.Errorf("release onboarding kickoff: %w", err)
}
return qtx.DeleteUserChatMessageByTask(ctx, inputOwnerID)
}
func (s *TaskService) finalizeCancelledChatMessage(ctx context.Context, task db.AgentTaskQueue, opts CancelTaskOptions) *CancelledChatMessageResult {
if !task.ChatSessionID.Valid {
return nil
}
var cancelled *CancelledChatMessageResult
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
// Same protocol as every other terminal path: this transaction marks the
// task row and then writes chat_message rows, whose FK takes a KEY SHARE
// lock on the session — two rows again, so it takes the session first.
if err := lockChatSessionForTaskWrite(ctx, qtx, task.ID); err != nil {
return err
}
hasMessages, err := qtx.HasTaskMessages(ctx, task.ID)
if err != nil {
return fmt.Errorf("list cancelled chat task messages: %w", err)
}
restorable := !hasMessages
if restorable {
// Channel-ingested user messages are the durable record of what
// the platform sender wrote — the sender has no Multica composer
// to restore a draft into. The gate is the immutable per-message
// channel_ingested stamp, NOT the channel_chat_session_binding
// row: archiving a session or rebinding an installation deletes
// the binding while the messages (and a still-cancellable task)
// remain. Keyed by the input-batch owner id so an auto-retry
// clone (which inherits chat_input_task_id) reaches the same
// verdict as its parent. A channel task settles as "Stopped."
// below instead of deleting its sealed input batch.
channelIngested, err := qtx.TaskHasChannelIngestedMessages(ctx, chatInputOwnerID(task))
if err != nil {
return fmt.Errorf("check cancelled chat channel provenance: %w", err)
}
restorable = !channelIngested
}
if restorable && task.StartedAt.Valid && opts.ClientSupportsDraftRestore {
// A started task's daemon learns of the cancellation by polling
// and may still be flushing its transcript tail, so "empty" is
// not trustworthy yet. Defer the judgment until the daemon acks
// its flush (cancel-ack) or the sweeper grace period expires
// (#5219). "Non-empty" needs no deferral: late rows only append.
//
// Deferring is gated on the client: clients and server do not
// upgrade together, and a client that cannot read the durable
// restore would take an empty cancel response as "nothing to put
// back" and lose the prompt. Such a client falls through to the
// legacy synchronous branch below — it keeps the pre-#5219 race
// (an in-flight transcript tail can still be misjudged as empty),
// which is exactly the behaviour it has against an old server, and
// strictly better than dropping the input.
if _, err := qtx.MarkChatFinalizeDeferred(ctx, task.ID); err != nil {
return fmt.Errorf("mark chat finalize deferred: %w", err)
}
return nil
}
if restorable {
inputOwnerID := chatInputOwnerID(task)
// Detach attachments BEFORE deleting the user message — the
// attachment FK is ON DELETE CASCADE, so deleting first would
// destroy rows the restored draft needs to re-bind.
detached, err := qtx.DetachAttachmentsFromUserChatMessageByTask(ctx, inputOwnerID)
if err != nil {
return fmt.Errorf("detach cancelled chat message attachments: %w", err)
}
deleted, err := deleteUserChatInput(ctx, qtx, inputOwnerID)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return fmt.Errorf("delete empty cancelled chat user message: %w", err)
}
// Always restorable now: the delete cannot return a kickoff row, so
// what comes back is always what the member typed (MUL-5827).
cancelled = &CancelledChatMessageResult{
ChatSessionID: util.UUIDToString(deleted.ChatSessionID),
MessageID: util.UUIDToString(deleted.ID),
Content: deleted.Content,
RestoreToInput: true,
Attachments: detached,
}
return nil
}
if _, err := createAssistantChatMessage(ctx, qtx, db.CreateChatMessageParams{
ID: dbid.NewV7(),
ChatSessionID: task.ChatSessionID,
Role: "assistant",
Content: "Stopped.",
TaskID: task.ID,
ElapsedMs: computeChatElapsedMs(task),
}); err != nil {
return fmt.Errorf("create cancelled chat message: %w", err)
}
return nil
}); err != nil {
slog.Error("failed to finalize cancelled chat message",
"task_id", util.UUIDToString(task.ID),
"chat_session_id", util.UUIDToString(task.ChatSessionID),
"error", err,
)
return nil
}
return cancelled
}
// FinalizeDeferredCancelledChat settles the empty/non-empty judgment that
// finalizeCancelledChatMessage deferred for a started-but-empty cancelled
// chat task (#5219). Called from the daemon's cancel-ack (transcript flush
// complete) and from the sweeper grace-period fallback; the marker claim is
// atomic, so concurrent callers cannot finalize the same task twice and a
// call with no pending marker is a no-op. The settled outcome is broadcast
// as chat:cancel_finalized since the cancel HTTP response has long returned.
// RebroadcastCancelledTask re-announces an already-cancelled task after a
// post-terminal delivery landed on its row (the cancel-ack's branch name or
// preserved-worktree error). The original task:cancelled broadcast fired at
// cancel time, BEFORE the daemon's ack — clients may have refetched a row
// without the delivery and will not refetch again on their own. Consumers
// treat task:cancelled as idempotent cache invalidation, so a replay is safe.
func (s *TaskService) RebroadcastCancelledTask(ctx context.Context, taskID pgtype.UUID) {
task, err := s.Queries.GetAgentTask(ctx, taskID)
if err != nil {
slog.Warn("rebroadcast cancelled task: load failed",
"task_id", util.UUIDToString(taskID), "error", err)
return
}
if task.Status != "cancelled" {
// A complete/fail callback already announced its own terminal event
// carrying the row's final fields; nothing stale to refresh.
return
}
s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, task)
}
// RetireCancelledTaskSession records, on a CANCELLED task, a session the daemon
// judged undrivable: the run resumed it and produced no output at all before
// it was cancelled (MUL-8188). It is the cancel-ack's counterpart of the
// retired_session_id the complete/fail callbacks carry, and it has the same
// two effects: the (agent, issue) resume lookups stop selecting the session,
// and a chat pointer still naming it is cleared so the next turn starts fresh.
//
// The status CAS in SetAgentTaskRetiredSessionIDIfCancelled makes a late or
// replayed ack against a completed/failed row a no-op; the chat pointer is
// only touched when that write applied, so a stale ack can never clear a
// healthy pointer.
func (s *TaskService) RetireCancelledTaskSession(ctx context.Context, taskID pgtype.UUID, sessionID string) error {
if sessionID == "" {
return nil
}
return s.runInTx(ctx, func(qtx *db.Queries) error {
if err := lockChatSessionForTaskWrite(ctx, qtx, taskID); err != nil {
return err
}
rows, err := qtx.SetAgentTaskRetiredSessionIDIfCancelled(ctx, db.SetAgentTaskRetiredSessionIDIfCancelledParams{
ID: taskID,
RetiredSessionID: pgtype.Text{String: sessionID, Valid: true},
})
if err != nil {
return fmt.Errorf("record retired session on cancelled task: %w", err)
}
if rows == 0 {
return nil
}
t, err := qtx.GetAgentTask(ctx, taskID)
if err != nil {
return fmt.Errorf("load cancelled task: %w", err)
}
if !t.ChatSessionID.Valid {
return nil
}
if err := qtx.ClearChatSessionSessionIfMatches(ctx, db.ClearChatSessionSessionIfMatchesParams{
ID: t.ChatSessionID,
SessionID: pgtype.Text{String: sessionID, Valid: true},
RuntimeID: t.RuntimeID,
}); err != nil {
return fmt.Errorf("clear retired chat session resume pointer: %w", err)
}
return nil
})
}
func (s *TaskService) FinalizeDeferredCancelledChat(ctx context.Context, taskID pgtype.UUID) bool {
var (
task db.AgentTaskQueue
payload protocol.ChatCancelFinalizedPayload
changed bool
settled bool
)
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
// Lock the task's chat_session first. chat_draft_restore has no FK
// (MUL-3515), so the insert below takes no lock of its own on the
// session — without this, a workspace/agent/session delete that swept
// the table just before we commit would leave our restore row (holding
// the user's prompt) orphaned forever. The deleters take the same lock
// before their sweep, so one of us blocks: either they wait and their
// sweep sees our row, or we wait and find no session left to restore
// into. Locking the session BEFORE the task claim also fixes the global
// lock order (chat_session -> agent_task_queue) that keeps this from
// deadlocking against the deleters' cascade.
_, err := qtx.LockChatSessionForTask(ctx, taskID)
sessionGone := errors.Is(err, pgx.ErrNoRows)
if err != nil && !sessionGone {
return fmt.Errorf("lock chat session for deferred finalize: %w", err)
}
// Claim the marker inside the settlement tx: a failed settlement then
// rolls the claim back so the sweeper can retry, instead of leaving the
// task with a cleared marker and no finalized outcome. The row lock
// still serializes the daemon ack and the sweeper — the loser's UPDATE
// blocks until the winner commits, then matches no row (ErrNoRows) — so
// the same task is never finalized twice.
claimed, err := qtx.ClaimChatFinalizeDeferred(ctx, taskID)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return fmt.Errorf("claim deferred chat finalize: %w", err)
}
changed = true
task = claimed
if sessionGone {
// The session cascaded away (its FK NULLs the column below anyway):
// there is no transcript to settle and nowhere to put a restore. The
// claim above still cleared the marker, so the sweeper stops retrying.
return nil
}
if !claimed.ChatSessionID.Valid {
return nil
}
settled = true
payload.ChatSessionID = util.UUIDToString(claimed.ChatSessionID)
payload.TaskID = util.UUIDToString(claimed.ID)
payload.InitiatorUserID = util.UUIDToString(claimed.InitiatorUserID)
hasMessages, err := qtx.HasTaskMessages(ctx, claimed.ID)
if err != nil {
return fmt.Errorf("list cancelled chat task messages: %w", err)
}
restorable := !hasMessages
if restorable {
// Same immutable-provenance guard as finalizeCancelledChatMessage:
// channel tasks never restore-delete their sealed input. The sync
// path no longer defers such tasks; this covers markers created by
// an older replica during a rolling deploy.
channelIngested, err := qtx.TaskHasChannelIngestedMessages(ctx, chatInputOwnerID(claimed))
if err != nil {
return fmt.Errorf("check cancelled chat channel provenance: %w", err)
}
restorable = !channelIngested
}
if restorable {
inputOwnerID := chatInputOwnerID(claimed)
// The transcript stayed empty through the daemon flush: same
// outcome as the synchronous empty branch, but the cancel HTTP
// response is long gone and the broadcast is best-effort. The
// restore is persisted in this same tx and served by the
// creator-authorized draft-restores endpoint, so a client that
// misses the event recovers it on the next session open; the
// event itself carries no content and is only an invalidation
// hint.
detached, err := qtx.DetachAttachmentsFromUserChatMessageByTask(ctx, inputOwnerID)
if err != nil {
return fmt.Errorf("detach cancelled chat message attachments: %w", err)
}
deleted, err := deleteUserChatInput(ctx, qtx, inputOwnerID)
if errors.Is(err, pgx.ErrNoRows) {
payload.Outcome = ""
return nil
}
if err != nil {
return fmt.Errorf("delete empty cancelled chat user message: %w", err)
}
attachmentIDs := make([]pgtype.UUID, 0, len(detached))
for _, a := range detached {
attachmentIDs = append(attachmentIDs, a.ID)
}
if _, err := qtx.CreateChatDraftRestore(ctx, db.CreateChatDraftRestoreParams{
ID: deleted.ID,
ChatSessionID: claimed.ChatSessionID,
TaskID: claimed.ID,
Content: deleted.Content,
AttachmentIds: attachmentIDs,
}); err != nil {
return fmt.Errorf("create chat draft restore: %w", err)
}
payload.Outcome = protocol.ChatCancelOutcomeRestored
payload.MessageID = util.UUIDToString(deleted.ID)
return nil
}
row, err := createAssistantChatMessage(ctx, qtx, db.CreateChatMessageParams{
ID: dbid.NewV7(),
ChatSessionID: claimed.ChatSessionID,
Role: "assistant",
Content: "Stopped.",
TaskID: claimed.ID,
ElapsedMs: computeChatElapsedMs(claimed),
FailureReason: claimed.FailureReason,
})
if err != nil {
return fmt.Errorf("create cancelled chat message: %w", err)
}
payload.Outcome = protocol.ChatCancelOutcomeStopped
payload.MessageID = util.UUIDToString(row.ID)
payload.Content = row.Content
payload.MessageKind = row.MessageKind
if row.CreatedAt.Valid {
payload.CreatedAt = row.CreatedAt.Time.UTC().Format(time.RFC3339Nano)
}
if row.ElapsedMs.Valid {
payload.ElapsedMs = row.ElapsedMs.Int64
}
return nil
}); err != nil {
slog.Error("failed to finalize deferred cancelled chat",
"task_id", util.UUIDToString(taskID),
"error", err,
)
return false
}
if !settled || payload.Outcome == "" {
return changed
}
s.broadcastChatCancelFinalized(ctx, task, payload)
return changed
}
func (s *TaskService) broadcastChatCancelFinalized(ctx context.Context, task db.AgentTaskQueue, payload protocol.ChatCancelFinalizedPayload) {
scope := s.ResolveTaskScope(ctx, task)
workspaceID := scope.WorkspaceID
if workspaceID == "" {
return
}
s.Bus.Publish(events.Event{
Type: protocol.EventChatCancelFinalized,
WorkspaceID: workspaceID,
ActorType: "system",
ActorID: "",
ChatSessionID: util.UUIDToString(task.ChatSessionID),
ChatSessionOwnerID: scope.ChatSessionOwnerID,
Payload: payload,
})
}
// ClaimTask atomically claims the next queued task for an agent on its current
// runtime, respecting max_concurrent_tasks.
func (s *TaskService) ClaimTask(ctx context.Context, agentID pgtype.UUID) (*db.AgentTaskQueue, error) {
return s.claimTask(ctx, agentID, pgtype.UUID{}, claimScope{claimPath: claimPathAgent})
}
// claimTask is the runtime-scoped claim primitive used by daemon poll paths.
// The exported ClaimTask wrapper omits runtimeID and therefore resolves the
// agent's currently bound runtime. Scoping the SQL claim itself prevents an
// offline candidate on runtime A from causing the same agent's task on runtime
// B to be dispatched and then dropped by the caller's runtime guard.
func (s *TaskService) claimTask(ctx context.Context, agentID, runtimeID pgtype.UUID, scope claimScope) (*db.AgentTaskQueue, error) {
start := time.Now()
outcome := "unknown"
var getAgentMs, countRunningMs, claimAgentMs, reanchorMs, updateStatusMs, dispatchMs int64
var claimed *db.AgentTaskQueue
var reclaimCheckAfter time.Time
var decisionDetail claimDecisionDetail
defer func() {
s.maybeLogClaimSlow(agentID, outcome, start, getAgentMs, countRunningMs, claimAgentMs, reanchorMs, updateStatusMs, dispatchMs)
s.recordClaimDecision(agentID, runtimeID, outcome, scope, decisionDetail)
}()
err := s.runInTx(ctx, func(qtx *db.Queries) error {
t0 := time.Now()
agent, err := qtx.GetAgentForClaimUpdate(ctx, agentID)
getAgentMs = time.Since(t0).Milliseconds()
if err != nil {
outcome = "error_get_agent"
return fmt.Errorf("agent not found: %w", err)
}
claimRuntimeID := runtimeID
if !claimRuntimeID.Valid {
claimRuntimeID = agent.RuntimeID
}
if !claimRuntimeID.Valid {
outcome = "no_runtime"
return nil
}
// A daemon may still hold a stale candidate after the agent is rebound.
// Reject it before doing capacity work. ClaimAgentTask repeats the fence
// before its state transition; the claim handler then rechecks the freshly
// loaded Agent before returning any payload. Runtime mutation teardown is
// responsible for serializing and settling the remaining queued rows.
if runtimeID.Valid && agent.RuntimeID != runtimeID {
outcome = "runtime_mismatch"
return nil
}
t0 = time.Now()
running, err := qtx.CountRunningTasks(ctx, agentID)
countRunningMs = time.Since(t0).Milliseconds()
if err != nil {
outcome = "error_count_running"
return fmt.Errorf("count running tasks: %w", err)
}
if running >= int64(agent.MaxConcurrentTasks) {
decisionDetail = claimDecisionDetail{running: running, maxConcurrent: int64(agent.MaxConcurrentTasks)}
outcome = "no_capacity"
return nil
}
t0 = time.Now()
reclaimCheckAfter = t0.Add(claimResponseRecoveryWindow + ReclaimCheckHintSafetyMargin)
task, err := qtx.ClaimAgentTask(ctx, db.ClaimAgentTaskParams{
AgentID: agentID,
RuntimeID: claimRuntimeID,
PrepareLeaseSecs: prepareLeaseDuration.Seconds(),
RuntimeStaleSecs: RuntimeClaimFreshnessSeconds,
})
claimAgentMs = time.Since(t0).Milliseconds()
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
outcome = "no_tasks"
return nil
}
outcome = "error_claim"
return fmt.Errorf("claim task: %w", err)
}
// An idle task-owned direct-chat row may already be visible as the
// positional queue head. Normal completion reanchors a successor beside
// the assistant outcome before commit; this claim-time query is the
// compatibility fallback for an older or otherwise out-of-order row.
// It only moves that row inside the claim transaction, so full-list and
// cursor readers see the corrected boundary once it is dispatched. Retry
// children keep the root input owner and stale-dispatch reclaim uses a
// separate query, so neither path moves an already-visible turn again.
if task.ChatSessionID.Valid && task.ChatInputTaskID.Valid && task.ChatInputTaskID == task.ID {
t0 = time.Now()
if err := qtx.ReanchorClaimedDirectChatInput(ctx, db.ReanchorClaimedDirectChatInputParams{
DispatchedAt: task.DispatchedAt,
TaskID: task.ID,
}); err != nil {
reanchorMs = time.Since(t0).Milliseconds()
outcome = "error_reanchor_chat_input"
return fmt.Errorf("reanchor claimed direct chat input: %w", err)
}
reanchorMs = time.Since(t0).Milliseconds()
}
claimedTask := task
claimed = &claimedTask
return nil
})
if err != nil {
if outcome == "unknown" {
outcome = "error_transaction"
}
return nil, err
}
if claimed == nil {
return nil, nil
}
s.trackTaskForReclaim(*claimed, reclaimCheckAfter)
slog.Info("task claimed", "task_id", util.UUIDToString(claimed.ID), "agent_id", util.UUIDToString(agentID))
s.captureTaskDispatched(ctx, *claimed)
// Refresh agent status from active tasks. This avoids a stale unconditional
// working write racing after a just-cancelled claim.
t0 := time.Now()
s.ReconcileAgentStatus(ctx, agentID)
updateStatusMs = time.Since(t0).Milliseconds()
// Broadcast task:dispatch. ResolveTaskWorkspaceID inside this path can
// re-query issue/chat_session/autopilot_run, so it can also be a real
// contributor to claim latency.
t0 = time.Now()
s.broadcastTaskDispatch(ctx, *claimed)
dispatchMs = time.Since(t0).Milliseconds()
outcome = "claimed"
return claimed, nil
}
// ClaimTaskForRuntime claims the next runnable task for a runtime while
// still respecting each agent's max_concurrent_tasks limit.
//
// Empty-claim fast path: when EmptyClaim is configured and a recent
// check verified the runtime had no queued tasks, returns immediately
// without touching Postgres. The cache is invalidated synchronously on
// every enqueue (notifyTaskAvailable), so a queued task becomes
// claimable on the next call rather than waiting for the TTL.
func (s *TaskService) ClaimTaskForRuntime(ctx context.Context, runtimeID pgtype.UUID) (*db.AgentTaskQueue, error) {
start := time.Now()
var (
outcome = "no_task"
listMs, loopMs int64
listCount, tried int
claimedFlag bool
)
defer func() {
totalMs := time.Since(start).Milliseconds()
if totalMs < 300 {
return
}
slog.Info("claim_for_runtime slow",
"runtime_id", util.UUIDToString(runtimeID),
"outcome", outcome,
"total_ms", totalMs,
"list_pending_ms", listMs,
"list_pending_count", listCount,
"agents_tried", tried,
"claim_loop_ms", loopMs,
"claimed", claimedFlag,
)
}()
runtimeKey := util.UUIDToString(runtimeID)
if err := s.PromoteDueDeferredTasksForRuntime(ctx, runtimeID); err != nil {
outcome = "error_promote_deferred"
s.recordClaimDecision(pgtype.UUID{}, runtimeID, outcome, claimScope{claimPath: claimPathRuntime}, claimDecisionDetail{})
return nil, err
}
// Keep stale-response recovery before EmptyClaim because the queued-only
// cache cannot represent dispatched work. ReclaimCheck independently skips
// the UPDATE until a task hint or bounded DB backstop is due.
checkStarted := time.Now()
if due := s.ReclaimCheck.DueRuntimeIDs(ctx, []string{runtimeKey}, checkStarted); len(due) > 0 {
reclaimCheckAfter := time.Now().Add(claimResponseRecoveryWindow + ReclaimCheckHintSafetyMargin)
stale, err := s.Queries.ReclaimStaleDispatchedTaskForRuntime(ctx, db.ReclaimStaleDispatchedTaskForRuntimeParams{
RuntimeID: runtimeID,
ClaimRecoverySecs: claimResponseRecoveryWindow.Seconds(),
PrepareLeaseSecs: prepareLeaseDuration.Seconds(),
RuntimeStaleSecs: RuntimeClaimFreshnessSeconds,
})
if err == nil {
s.ReclaimCheck.MarkChecked(
ctx,
[]string{runtimeKey},
checkStarted,
time.Now().Add(ReclaimCheckRetryInterval),
)
s.trackTaskForReclaim(stale, reclaimCheckAfter)
outcome = "reclaimed_dispatched"
s.recordClaimDecision(stale.AgentID, runtimeID, outcome, claimScope{claimPath: claimPathRuntime}, claimDecisionDetail{})
claimedFlag = true
slog.Info("stale dispatched task reclaimed",
"task_id", util.UUIDToString(stale.ID),
"runtime_id", runtimeKey,
"agent_id", util.UUIDToString(stale.AgentID),
)
return &stale, nil
}
if !errors.Is(err, pgx.ErrNoRows) {
outcome = "error_reclaim_dispatched"
s.recordClaimDecision(pgtype.UUID{}, runtimeID, outcome, claimScope{claimPath: claimPathRuntime}, claimDecisionDetail{})
return nil, fmt.Errorf("reclaim stale dispatched task: %w", err)
}
s.ReclaimCheck.MarkChecked(
ctx,
[]string{runtimeKey},
checkStarted,
time.Now().Add(ReclaimCheckRetryInterval),
)
}
if s.EmptyClaim.IsEmpty(ctx, runtimeKey) {
outcome = "empty_cache_hit"
s.recordClaimDecision(pgtype.UUID{}, runtimeID, outcome, claimScope{claimPath: claimPathRuntime}, claimDecisionDetail{})
return nil, nil
}
// Sample the invalidation version BEFORE the SELECT. If a
// concurrent enqueue Bumps between this read and the post-SELECT
// MarkEmpty, the next IsEmpty will see the empty key tagged with
// a stale version and reject it — closing the race that would
// otherwise stall the just-queued task until the empty key's TTL
// expired.
preSelectVersion := s.EmptyClaim.CurrentVersion(ctx, runtimeKey)
t0 := time.Now()
tasks, err := s.Queries.ListQueuedClaimCandidatesByRuntime(ctx, runtimeID)
listMs = time.Since(t0).Milliseconds()
listCount = len(tasks)
if err != nil {
outcome = "error_list"
s.recordClaimDecision(pgtype.UUID{}, runtimeID, outcome, claimScope{claimPath: claimPathRuntime}, claimDecisionDetail{})
return nil, fmt.Errorf("list queued claim candidates: %w", err)
}
if len(tasks) == 0 {
s.EmptyClaim.MarkEmpty(ctx, runtimeKey, preSelectVersion)
outcome = "empty_db"
s.recordClaimDecision(pgtype.UUID{}, runtimeID, outcome, claimScope{claimPath: claimPathRuntime}, claimDecisionDetail{})
return nil, nil
}
loopStart := time.Now()
triedAgents := map[string]struct{}{}
// Candidate rows per agent, counted before the loop so each claimTask call
// can report whether the runtime saw work for that agent (the #8551
// attribution split between candidates_blocked and no_candidates).
candidatesByAgent := make(map[string]int, len(tasks))
for i := range tasks {
candidatesByAgent[util.UUIDToString(tasks[i].AgentID)]++
}
var claimed *db.AgentTaskQueue
for _, candidate := range tasks {
agentKey := util.UUIDToString(candidate.AgentID)
if _, seen := triedAgents[agentKey]; seen {
continue
}
triedAgents[agentKey] = struct{}{}
tried++
task, err := s.claimTask(ctx, candidate.AgentID, runtimeID, claimScope{
claimPath: claimPathRuntime,
candidates: candidatesByAgent[agentKey],
})
if err != nil {
loopMs = time.Since(loopStart).Milliseconds()
outcome = "error_claim"
return nil, err
}
if task != nil && task.RuntimeID == runtimeID {
claimed = task
break
}
}
loopMs = time.Since(loopStart).Milliseconds()
if claimed != nil {
claimedFlag = true
outcome = "claimed"
} else if len(tasks) > 0 {
// Work was visible for at least one agent but every claim attempt lost
// (capacity cap, SQL serialization fence, or runtime binding). This is
// the key signal for attributing queue waits to a fence rather than to
// an empty queue (#8551).
slog.Debug("claim poll exhausted candidates without a claim",
"runtime_id", runtimeKey,
"candidates", len(tasks),
)
}
return claimed, nil
}
// ErrClaimDeliveryAuthz signals that the final delivery gate rejected the
// claimed task: the current agent/runtime authorization no longer holds at the
// delivery boundary. The task is settled by the caller through the existing
// FailTask path; no claim payload is dispatched.
type ClaimDeliveryAuthzError struct {
Reason string
Detail string
}
func (e *ClaimDeliveryAuthzError) Error() string {
return "claim delivery authorization failed: " + e.Reason + ": " + e.Detail
}
// FinalizeTaskClaim atomically persists the task-scoped agent token, an
// optional short-lived daemon token used by the Remote MCP broker, the
// comparable issue state this payload was built from, and, for a comment-backed
// task, the exact comment ids embedded in the response. The handler must call
// this only after the full payload has been built and before writing any
// response bytes. A failure rolls every write back so the claim can be safely
// returned to the queue.
//
// issueSnapshot is empty for tasks with no issue (chat, quick-create,
// autopilot) and for an issue claim whose snapshot could not be encoded; the
// write is then skipped and the NEXT run on that issue reports "not compared"
// rather than a wrong "unchanged" (MUL-7344). Unlike the comment receipt it is
// NOT gated on the task being comment-backed: an assignment run that recorded
// no snapshot leaves the following comment-triggered run with no baseline.
//
// The optional authorize closure runs INSIDE the same transaction, after the
// gate has re-read the current runtime row under a FOR UPDATE row lock. That
// makes the authorization decision and the task-token/daemon-token writes one
// atomic unit: a concurrent runtime re-registration that would change owner_id
// blocks until the gate commits, so the owner the gate authorized against is
// the owner the tokens were minted for — no stale-snapshot delivery window.
// The closure receives the in-transaction token params so it can normalize
// identity fields from the locked rows before the token is inserted. It
// returns a *ClaimDeliveryAuthzError to reject delivery (every other error
// rolls the claim back like any other finalize failure).
func (s *TaskService) FinalizeTaskClaim(
ctx context.Context,
task db.AgentTaskQueue,
token db.CreateTaskTokenParams,
deliveredCommentIDs []pgtype.UUID,
recordCommentReceipt bool,
authorize func(qtx *db.Queries, token *db.CreateTaskTokenParams) error,
issueSnapshot []byte,
daemonTokens ...db.CreateDaemonTokenParams,
) ([]pgtype.UUID, error) {
if len(daemonTokens) > 1 {
return nil, fmt.Errorf("finalize task claim: expected at most one daemon token, got %d", len(daemonTokens))
}
receipt := task.DeliveredCommentIds
err := s.runInTx(ctx, func(qtx *db.Queries) error {
if authorize != nil {
if err := authorize(qtx, &token); err != nil {
return fmt.Errorf("authorize claim delivery: %w", err)
}
}
if _, err := qtx.CreateTaskToken(ctx, token); err != nil {
return fmt.Errorf("create task token: %w", err)
}
if len(daemonTokens) == 1 {
// Opportunistic bounded cleanup keeps short-lived per-task daemon
// credentials from accumulating without adding another sweeper.
if err := qtx.DeleteExpiredDaemonTokens(ctx); err != nil {
return fmt.Errorf("delete expired daemon tokens: %w", err)
}
// Bind the credential to the same owner the task token was just
// minted for (the locked runtime owner, set by authorize), so it
// can never reach a runtime another member later registers under
// this daemon_id (MUL-8154).
daemonToken := daemonTokens[0]
daemonToken.OwnerID = token.UserID
if !daemonToken.OwnerID.Valid {
return fmt.Errorf("create remote MCP daemon token: runtime owner required")
}
if _, err := qtx.CreateDaemonToken(ctx, daemonToken); err != nil {
return fmt.Errorf("create remote MCP daemon token: %w", err)
}
}
if len(issueSnapshot) > 0 {
// Same CAS columns as the receipt below, so a stale handler cannot
// write a snapshot over a newer reclaim's, or after the run has
// started. A no-op update (0 rows) is not an error: it means this
// claim generation is no longer current, and the newer one records
// its own snapshot.
if err := qtx.SetTaskIssueSnapshot(ctx, db.SetTaskIssueSnapshotParams{
IssueSnapshot: issueSnapshot,
TaskID: task.ID,
RuntimeID: task.RuntimeID,
DispatchedAt: task.DispatchedAt,
}); err != nil {
return fmt.Errorf("set task issue snapshot: %w", err)
}
}
if !recordCommentReceipt {
return nil
}
persisted, err := qtx.SetTaskDeliveredCommentIDs(ctx, db.SetTaskDeliveredCommentIDsParams{
DeliveredCommentIds: deliveredCommentIDs,
TaskID: task.ID,
RuntimeID: task.RuntimeID,
DispatchedAt: task.DispatchedAt,
ExpectedTriggerCommentID: task.TriggerCommentID,
})
if err != nil {
return fmt.Errorf("set delivered comment ids: %w", err)
}
receipt = persisted
return nil
})
if err != nil {
return nil, err
}
return receipt, nil
}
// RequeueTaskAfterClaimFailure immediately releases an exact dispatched claim
// whose payload finalization failed before the HTTP response was written. The
// SQL CAS includes dispatched_at so a late handler cannot roll back a newer
// reclaim. This is not a fresh enqueue: do not duplicate queued analytics.
func (s *TaskService) RequeueTaskAfterClaimFailure(ctx context.Context, task db.AgentTaskQueue) (*db.AgentTaskQueue, error) {
requeued, err := s.Queries.RequeueAgentTaskAfterClaimFailure(ctx, db.RequeueAgentTaskAfterClaimFailureParams{
TaskID: task.ID,
RuntimeID: task.RuntimeID,
DispatchedAt: task.DispatchedAt,
})
if err != nil {
return nil, fmt.Errorf("requeue task after claim failure: %w", err)
}
s.forgetTaskReclaim(requeued)
s.ReconcileAgentStatus(ctx, requeued.AgentID)
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, requeued)
s.notifyTaskAvailable(requeued)
slog.Info("task requeued after claim finalization failure",
"task_id", util.UUIDToString(requeued.ID),
"runtime_id", util.UUIDToString(requeued.RuntimeID),
)
return &requeued, nil
}
// ClaimTasksForRuntimes is the machine-level (MUL-4257) batch counterpart of
// ClaimTaskForRuntime: it claims up to maxTasks tasks across every runtime in
// runtimeIDs in a single call, so a daemon can poll for all of its runtimes
// with one HTTP request and a constant number of DB queries instead of one
// request (and one promote/reclaim/list cycle) per runtime.
//
// It preserves the exact per-runtime semantics, just set-ified:
// 1. promote due deferred tasks across the set (one UPDATE);
// 2. reclaim up to maxTasks stale-dispatched tasks across the set (one UPDATE)
// — done before the empty-cache check because a lost claim response moves
// the task out of `queued`, which the empty-queued cache cannot represent;
// 3. short-circuit runtimes whose empty-claim verdict is cached, sampling the
// invalidation version for the rest BEFORE the candidate SELECT;
// 4. list queued candidates across the non-empty set (one SELECT);
// 5. mark still-empty runtimes so their next idle poll skips Postgres;
// 6. claim per distinct agent via the runtime-scoped helper, preserving
// per-(issue, agent) serialization, the agent concurrency cap, and every
// dispatch side effect until maxTasks is reached.
//
// The returned slice contains both reclaimed and freshly-claimed tasks, each
// already carrying its runtime_id so the daemon routes it to the matching
// runtime locally.
func (s *TaskService) ClaimTasksForRuntimes(ctx context.Context, runtimeIDs []pgtype.UUID, maxTasks int) ([]db.AgentTaskQueue, error) {
if len(runtimeIDs) == 0 || maxTasks <= 0 {
return nil, nil
}
// De-dup runtime IDs defensively so MarkEmpty/version bookkeeping stays
// unambiguous even if a daemon ever sends a duplicate.
seen := make(map[string]struct{}, len(runtimeIDs))
uniqueIDs := make([]pgtype.UUID, 0, len(runtimeIDs))
runtimeInSet := make(map[string]struct{}, len(runtimeIDs))
for _, rid := range runtimeIDs {
key := util.UUIDToString(rid)
runtimeInSet[key] = struct{}{}
if _, dup := seen[key]; dup {
continue
}
seen[key] = struct{}{}
uniqueIDs = append(uniqueIDs, rid)
}
claimed := make([]db.AgentTaskQueue, 0, maxTasks)
// 1. Promote due deferred tasks across the whole set (promote-first, like
// the singular path). Replay the per-row side effects the singular service
// method PromoteDueDeferredTasksForRuntime performs — crucially
// EmptyClaim.Bump (via NotifyTaskEnqueued → notifyTaskAvailable) so a
// just-promoted deferred task invalidates its runtime's cached empty
// verdict BEFORE the empty-cache filter in step 3; otherwise a stale
// MarkEmpty from a prior idle poll would short-circuit the runtime and the
// promoted task would sit unclaimed until the empty key's TTL. Also emits
// the deferred→queued UI event and the enqueue analytics sample.
s.cancelSupersededDeferredRetries(ctx, uniqueIDs)
promoted, err := s.Queries.PromoteDueDeferredTasksForRuntimes(ctx, db.PromoteDueDeferredTasksForRuntimesParams{
RuntimeIds: uniqueIDs,
RuntimeStaleSecs: RuntimeClaimFreshnessSeconds,
})
if isDuplicatePendingTaskErr(err) {
// Same tolerance as the single-runtime path, and it matters more here:
// one contended row would otherwise fail the claim for EVERY runtime in
// the batch. Promote nothing this tick and let the claim continue.
slog.Info("promote deferred tasks (batch): slot taken by a concurrent enqueue, skipping this tick")
promoted = nil
} else if err != nil {
s.recordClaimDecision(pgtype.UUID{}, pgtype.UUID{}, "error_promote_deferred", claimScope{claimPath: claimPathBatch}, claimDecisionDetail{})
return nil, fmt.Errorf("promote deferred tasks: %w", err)
}
for _, task := range promoted {
slog.Info("deferred task promoted (batch)",
"task_id", util.UUIDToString(task.ID),
"runtime_id", util.UUIDToString(task.RuntimeID),
"agent_id", util.UUIDToString(task.AgentID),
)
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, task)
s.NotifyTaskEnqueued(ctx, task)
}
// 2. Reclaim lost-response dispatched tasks when any runtime's task schedule
// or bounded DB backstop is due. The query always receives the complete
// machine-level set so per-runtime backstops stay aligned and fixed UPDATE
// setup/locking cost is paid at a predictable cadence. A nil/missing/failed
// cache preserves the historical query path.
runtimeKeys := make([]string, 0, len(uniqueIDs))
for _, rid := range uniqueIDs {
runtimeKeys = append(runtimeKeys, util.UUIDToString(rid))
}
checkStarted := time.Now()
dueKeys := s.ReclaimCheck.DueRuntimeIDs(ctx, runtimeKeys, checkStarted)
var reclaimed []db.AgentTaskQueue
var reclaimCheckAfter time.Time
if len(dueKeys) > 0 {
reclaimCheckAfter = time.Now().Add(claimResponseRecoveryWindow + ReclaimCheckHintSafetyMargin)
reclaimed, err = s.Queries.ReclaimStaleDispatchedTasksForRuntimes(ctx, db.ReclaimStaleDispatchedTasksForRuntimesParams{
RuntimeIds: uniqueIDs,
ClaimRecoverySecs: claimResponseRecoveryWindow.Seconds(),
PrepareLeaseSecs: prepareLeaseDuration.Seconds(),
RuntimeStaleSecs: RuntimeClaimFreshnessSeconds,
MaxTasks: int32(maxTasks),
})
if err != nil {
s.recordClaimDecision(pgtype.UUID{}, pgtype.UUID{}, "error_reclaim_dispatched", claimScope{claimPath: claimPathBatch}, claimDecisionDetail{})
return nil, fmt.Errorf("reclaim stale dispatched tasks: %w", err)
}
// The UPDATE's fixed setup/locking cost dominates array width. Query and
// advance the complete machine-level set together so per-runtime backstops
// cannot drift into one UPDATE on almost every daemon poll.
s.ReclaimCheck.MarkChecked(
ctx,
runtimeKeys,
checkStarted,
time.Now().Add(ReclaimCheckRetryInterval),
)
}
for i := range reclaimed {
s.trackTaskForReclaim(reclaimed[i], reclaimCheckAfter)
claimed = append(claimed, reclaimed[i])
s.recordClaimDecision(reclaimed[i].AgentID, reclaimed[i].RuntimeID, "reclaimed_dispatched", claimScope{claimPath: claimPathBatch}, claimDecisionDetail{})
slog.Info("stale dispatched task reclaimed (batch)",
"task_id", util.UUIDToString(reclaimed[i].ID),
"runtime_id", util.UUIDToString(reclaimed[i].RuntimeID),
"agent_id", util.UUIDToString(reclaimed[i].AgentID),
)
}
if len(claimed) >= maxTasks {
return claimed[:maxTasks], nil
}
// 3. Empty-cache short-circuit + version sampling for the remaining runtimes.
nonEmpty := make([]pgtype.UUID, 0, len(uniqueIDs))
versions := make(map[string]int64, len(uniqueIDs))
for _, rid := range uniqueIDs {
key := util.UUIDToString(rid)
if s.EmptyClaim.IsEmpty(ctx, key) {
continue
}
versions[key] = s.EmptyClaim.CurrentVersion(ctx, key)
nonEmpty = append(nonEmpty, rid)
}
if len(nonEmpty) == 0 {
s.recordClaimDecision(pgtype.UUID{}, pgtype.UUID{}, "empty_cache_hit", claimScope{claimPath: claimPathBatch}, claimDecisionDetail{})
slog.Debug("batch claim: all runtimes empty-cached", "runtimes", len(uniqueIDs))
return claimed, nil
}
// 4. One candidate SELECT across the non-empty set.
candidates, err := s.Queries.ListQueuedClaimCandidatesByRuntimes(ctx, nonEmpty)
if err != nil {
s.recordClaimDecision(pgtype.UUID{}, pgtype.UUID{}, "error_list", claimScope{claimPath: claimPathBatch}, claimDecisionDetail{})
// Steps 2/6 commit reclaimed/claimed tasks in their own transactions,
// so `claimed` may already hold tasks dispatched server-side. Dropping
// them with a 500 makes the daemon HTTP-fall-back and claim a SECOND
// batch into the same free slots (the first batch then waits for stale
// reclaim) — the same double-claim this PR set out to remove
// (MUL-4257). Prefer partial success: hand back what committed so the
// handler finalizes and returns it; the errored candidates stay queued
// for the next poll.
if len(claimed) > 0 {
slog.Error("batch claim: candidate query failed after partial success; returning claimed tasks to avoid loss",
"error", err, "claimed", len(claimed))
return claimed, nil
}
return nil, fmt.Errorf("list queued claim candidates: %w", err)
}
// 5. Mark runtimes with zero candidates empty so their next idle poll skips
// Postgres. Runtimes that had at least one candidate are intentionally not
// marked (positive results always re-check the DB, matching the singular
// path).
withCandidates := make(map[string]struct{}, len(candidates))
for i := range candidates {
withCandidates[util.UUIDToString(candidates[i].RuntimeID)] = struct{}{}
}
for _, rid := range nonEmpty {
key := util.UUIDToString(rid)
if _, ok := withCandidates[key]; !ok {
s.EmptyClaim.MarkEmpty(ctx, key, versions[key])
}
}
if len(candidates) == 0 {
// Poll-level decision: the candidate query saw nothing claimable for
// any runtime in the batch (it already filters wakeup validity and
// agent/runtime binding, so this is about visible candidates, not
// about the queue being empty).
s.recordClaimDecision(pgtype.UUID{}, pgtype.UUID{}, "empty_db", claimScope{claimPath: claimPathBatch}, claimDecisionDetail{})
return claimed, nil
}
// 6. Claim per distinct agent through the runtime-scoped helper, preserving
// per-(issue, agent) serialization, capacity caps, and dispatch side effects.
triedAgents := make(map[string]struct{}, len(candidates))
// Candidate rows per agent for the claim-decision scope; see the singular
// path above for why it is counted before the loop.
candidatesByAgent := make(map[string]int, len(candidates))
for i := range candidates {
candidatesByAgent[util.UUIDToString(candidates[i].AgentID)]++
}
for i := range candidates {
if len(claimed) >= maxTasks {
slog.Debug("batch claim: reached max tasks",
"claimed", len(claimed),
"candidates_remaining", len(candidates)-i,
)
break
}
agentKey := util.UUIDToString(candidates[i].AgentID)
if _, tried := triedAgents[agentKey]; tried {
continue
}
triedAgents[agentKey] = struct{}{}
task, err := s.claimTask(ctx, candidates[i].AgentID, candidates[i].RuntimeID, claimScope{
claimPath: claimPathBatch,
candidates: candidatesByAgent[agentKey],
})
if err != nil {
// Each scoped claim commits in its own transaction, so earlier
// iterations (and step-2 reclaims) are already dispatched
// server-side. Returning nil here would drop them and force the
// daemon to double-claim via HTTP fallback (MUL-4257). Return the
// partial batch instead; the failed agent's task stays queued.
if len(claimed) > 0 {
slog.Error("batch claim: claim task failed after partial success; returning claimed tasks to avoid loss",
"error", err, "claimed", len(claimed))
return claimed, nil
}
return nil, fmt.Errorf("claim task: %w", err)
}
if task == nil {
continue
}
// The SQL claim is scoped to the candidate runtime. Retain this guard as
// a defensive contract check so a future query change cannot route work
// to a runtime this daemon does not host.
if _, ok := runtimeInSet[util.UUIDToString(task.RuntimeID)]; !ok {
slog.Debug("batch claim: skipping claimed task outside requested runtime set",
"task_id", util.UUIDToString(task.ID),
"runtime_id", util.UUIDToString(task.RuntimeID),
)
continue
}
claimed = append(claimed, *task)
}
return claimed, nil
}
// cancelSupersededDeferredRetries drops deferred auto-retry rows that an active
// task already supersedes, so a single rerun click still produces exactly one
// more run. Runs immediately before promotion — promotion is the moment a stale
// retry would otherwise become claimable. Best-effort: a failure here must not
// stop the claim loop, it just leaves the row for the next tick.
func (s *TaskService) cancelSupersededDeferredRetries(ctx context.Context, runtimeIDs []pgtype.UUID) {
if len(runtimeIDs) == 0 {
return
}
cancelled, err := s.Queries.CancelSupersededDeferredRetriesForRuntimes(ctx, runtimeIDs)
if err != nil {
slog.Warn("cancel superseded deferred retries failed", "error", err)
return
}
for _, task := range cancelled {
slog.Info("deferred auto-retry cancelled: superseded by an active task",
"task_id", util.UUIDToString(task.ID),
"issue_id", util.UUIDToString(task.IssueID),
"agent_id", util.UUIDToString(task.AgentID),
)
s.captureTaskCancelled(ctx, task)
s.ReconcileAgentStatus(ctx, task.AgentID)
s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, task)
}
}
func (s *TaskService) PromoteDueDeferredTasksForRuntime(ctx context.Context, runtimeID pgtype.UUID) error {
s.cancelSupersededDeferredRetries(ctx, []pgtype.UUID{runtimeID})
tasks, err := s.Queries.PromoteDueDeferredTasksForRuntime(ctx, db.PromoteDueDeferredTasksForRuntimeParams{
RuntimeID: runtimeID,
RuntimeStaleSecs: RuntimeClaimFreshnessSeconds,
})
if isDuplicatePendingTaskErr(err) {
// The NOT EXISTS fence inside the query cannot see an enqueue that has
// not committed yet, so a rerun landing in the same instant still
// collides here. One row losing its slot must not fail the whole claim:
// the row stays deferred and the next tick — by which time the rerun is
// visible — skips it cleanly. Costs one poll interval, never a stall.
slog.Info("promote due deferred tasks: slot taken by a concurrent enqueue, skipping this tick",
"runtime_id", util.UUIDToString(runtimeID))
return nil
}
if err != nil {
return fmt.Errorf("promote due deferred tasks: %w", err)
}
for _, task := range tasks {
slog.Info("deferred task promoted",
"task_id", util.UUIDToString(task.ID),
"runtime_id", util.UUIDToString(runtimeID),
"agent_id", util.UUIDToString(task.AgentID),
)
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, task)
s.NotifyTaskEnqueued(ctx, task)
}
return nil
}
// maybeLogClaimSlow emits one structured log per ClaimTask call when its total
// latency exceeds 300ms, so the prod tail can be diagnosed without flooding
// logs at normal poll rates. Called via defer so it captures the full path
// including post-claim updateAgentStatus / broadcastTaskDispatch (both of
// which can hit the DB) and any error exit.
func (s *TaskService) maybeLogClaimSlow(agentID pgtype.UUID, outcome string, start time.Time, getAgentMs, countRunningMs, claimAgentMs, reanchorMs, updateStatusMs, dispatchMs int64) {
totalMs := time.Since(start).Milliseconds()
if totalMs < 300 {
return
}
slog.Info("claim_task slow",
"agent_id", util.UUIDToString(agentID),
"outcome", outcome,
"total_ms", totalMs,
"get_agent_ms", getAgentMs,
"count_running_ms", countRunningMs,
"claim_agent_ms", claimAgentMs,
"reanchor_chat_input_ms", reanchorMs,
"update_status_ms", updateStatusMs,
"dispatch_ms", dispatchMs,
)
}
// TaskStartCapabilities records the capabilities offered for this exact run.
// Persist these in the same transaction as running, before cancellation can win.
type TaskStartCapabilities struct {
Supplement bool
ProcessCleanup bool
}
// StartTask transitions a dispatched task to running.
// Local agents manage issue status via the CLI; Qoder runs use guarded server transitions.
func (s *TaskService) StartTask(ctx context.Context, taskID pgtype.UUID, capabilities ...TaskStartCapabilities) (*db.AgentTaskQueue, error) {
var offered TaskStartCapabilities
if len(capabilities) > 0 {
offered = capabilities[0]
}
var task db.AgentTaskQueue
var change *qoderIssueChange
err := s.runInTx(ctx, func(q *db.Queries) error {
var err error
task, err = q.StartAgentTaskWithSupplement(ctx, db.StartAgentTaskWithSupplementParams{TaskID: taskID, EnableTaskSupplement: offered.Supplement, EnableProcessCleanup: offered.ProcessCleanup})
if err != nil {
return err
}
change, err = syncQoderIssue(ctx, q, task, false)
return err
})
if err != nil {
return nil, fmt.Errorf("start task: %w", err)
}
s.publishQoderIssueChange(ctx, change)
s.taskStarted(ctx, task)
return &task, nil
}
// StartTaskForClaim serializes the ownership check and transition with reclaim,
// cancellation and other start requests. A replay linearizes at the locked read;
// a cancellation that commits later can still cancel the acknowledged task.
func (s *TaskService) StartTaskForClaim(ctx context.Context, claim db.LockAgentTaskStartClaimParams, capabilities ...TaskStartCapabilities) (*db.AgentTaskQueue, error) {
if !claim.ID.Valid || !claim.RuntimeID.Valid || !claim.DispatchedAt.Valid {
return nil, fmt.Errorf("start task: incomplete claim")
}
tx, err := s.TxStarter.Begin(ctx)
if err != nil {
return nil, fmt.Errorf("begin task start: %w", err)
}
defer tx.Rollback(ctx)
qtx := s.Queries.WithTx(tx)
task, err := qtx.LockAgentTaskStartClaim(ctx, claim)
if err != nil {
return nil, fmt.Errorf("lock task start claim: %w", err)
}
var change *qoderIssueChange
replay := task.Status == "running"
if !replay {
var offered TaskStartCapabilities
if len(capabilities) > 0 {
offered = capabilities[0]
}
task, err = qtx.StartAgentTaskWithSupplement(ctx, db.StartAgentTaskWithSupplementParams{
TaskID: task.ID,
EnableTaskSupplement: offered.Supplement,
EnableProcessCleanup: offered.ProcessCleanup,
})
if err != nil {
return nil, fmt.Errorf("start claimed task: %w", err)
}
}
if !replay {
change, err = syncQoderIssue(ctx, qtx, task, false)
if err != nil {
return nil, err
}
}
if err := tx.Commit(ctx); err != nil {
return nil, fmt.Errorf("commit task start: %w", err)
}
s.publishQoderIssueChange(ctx, change)
if !replay {
s.taskStarted(ctx, task)
}
return &task, nil
}
func (s *TaskService) taskStarted(ctx context.Context, task db.AgentTaskQueue) {
s.forgetTaskReclaim(task)
slog.Info("task started", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(task.IssueID))
s.captureTaskStarted(ctx, task)
// A local-directory waiter was reconciled out of the persisted working
// status while parked. Restore working as soon as it enters running; the
// normal dispatched -> running path is already working, so this is
// intentionally idempotent there.
s.ReconcileAgentStatus(ctx, task.AgentID)
// Tell every connected workspace WS client that this task transitioned
// (dispatched | waiting_local_directory) → running. Without this, the
// workspace-wide `agentTaskSnapshot` query only refreshes on the 30s
// staleTime, so any UI that distinguishes "queued" from "running" (e.g.
// the issue-card agent activity indicator) lags by up to half a minute
// on the transition users care about most.
s.broadcastTaskEvent(ctx, protocol.EventTaskRunning, task)
}
// ExtendTaskPrepareLease keeps a claimed-but-not-started task protected while
// the daemon resolves cached inputs and prepares the execution environment.
func (s *TaskService) ExtendTaskPrepareLease(ctx context.Context, taskID, runtimeID pgtype.UUID) (*db.AgentTaskQueue, error) {
task, err := s.Queries.ExtendAgentTaskPrepareLease(ctx, db.ExtendAgentTaskPrepareLeaseParams{
ID: taskID,
RuntimeID: runtimeID,
LeaseSecs: prepareLeaseDuration.Seconds(),
})
if err != nil {
return nil, fmt.Errorf("extend task prepare lease: %w", err)
}
if task.Status == "dispatched" {
// Use the successful response time as a conservative application-clock
// approximation of the DB lease start. Scheduling slightly late by query
// latency is safer than an early failed check suppressing the hint until
// the next backstop.
s.extendTaskReclaimHint(task, time.Now().Add(prepareLeaseDuration))
} else {
s.forgetTaskReclaim(task)
}
return &task, nil
}
// MarkTaskWaitingLocalDirectory parks a dispatched task in the
// waiting_local_directory state while the daemon waits for another in-flight
// task to release the project_resource path lock. reason carries a short
// human-readable hint (typically the contested path) that the UI surfaces
// next to the status. Returns the updated row so the daemon can confirm the
// transition and so the broadcast carries the up-to-date snapshot.
func (s *TaskService) MarkTaskWaitingLocalDirectory(ctx context.Context, taskID pgtype.UUID, reason string) (*db.AgentTaskQueue, error) {
reason = sanitizeWaitReason(reason)
task, err := s.Queries.MarkAgentTaskWaitingLocalDirectory(ctx, db.MarkAgentTaskWaitingLocalDirectoryParams{
ID: taskID,
WaitReason: pgtype.Text{String: reason, Valid: reason != ""},
PrepareLeaseSecs: prepareLeaseDuration.Seconds(),
})
if err != nil {
return nil, fmt.Errorf("mark task waiting_local_directory: %w", err)
}
s.forgetTaskReclaim(task)
slog.Info("task waiting_local_directory",
"task_id", util.UUIDToString(task.ID),
"issue_id", util.UUIDToString(task.IssueID),
"reason", reason,
)
// waiting_local_directory is owned/queued work, not executing work. The
// claim path marked the agent working while the row was dispatched, so
// reconcile immediately when it parks instead of leaving that persisted
// status stale until a terminal transition.
s.ReconcileAgentStatus(ctx, task.AgentID)
// Carry the reason on the event, not just in the row. Without it the client
// learns WHY a task parked only on the follow-up refetch, so the pill spends
// that round-trip saying nothing useful — which is the gap this hint exists
// to close. Already sanitized above, so the socket cannot carry a path the
// REST payload would have withheld.
extra := map[string]any{}
if reason != "" {
extra["wait_reason"] = reason
}
s.broadcastTaskEvent(ctx, protocol.EventTaskWaitingLocalDirectory, task, extra)
return &task, nil
}
// legacyLocalDirectoryWaitPrefix is how daemons before the display-name change
// opened a wait reason: the literal word, a space, then the absolute path.
const legacyLocalDirectoryWaitPrefix = "local_directory "
// sanitizeWaitReason drops a wait reason that carries an absolute filesystem
// path, then trims what remains.
//
// Version skew makes this load-bearing rather than defensive. Current daemons
// send a display name (localDirectoryAssignment.DisplayName) precisely because
// this text reaches every client on the session and lands in screenshots. An
// OLDER daemon paired with this server still sends `local_directory /Users/
// <name>/repo (held by task abc12345)`, and since clients now render the field
// instead of ignoring it, passing that through would put the user's home path
// — and account name — on screen. The leak would be introduced BY surfacing
// the field, so the guard belongs with it.
//
// Dropping the whole string rather than editing it: clients already handle an
// absent reason (older servers never sent one) by showing the plain waiting
// label, which is exactly the pre-change behaviour and is never wrong.
//
// Remove once no supported daemon emits the legacy format.
func sanitizeWaitReason(reason string) string {
reason = strings.TrimSpace(reason)
rest, ok := strings.CutPrefix(reason, legacyLocalDirectoryWaitPrefix)
if ok && startsWithAbsolutePath(rest) {
return ""
}
return reason
}
// startsWithAbsolutePath reports whether s opens with an absolute path on any
// platform the daemon runs on: POSIX (/…), Windows drive (C:\… or C:/…), or a
// UNC share (\\host\share). Checked against the daemon's own output, so the
// question is only "did an old daemon put a path here", not general parsing.
//
// It deliberately does NOT match the holder clause a current daemon appends —
// `local_directory (held by task abc12345)` is what this server produces when
// a directory is genuinely NAMED "local_directory", and that name is the user's
// own label, not a path.
func startsWithAbsolutePath(s string) bool {
if s == "" {
return false
}
if s[0] == '/' || strings.HasPrefix(s, `\\`) {
return true
}
// Drive-letter form: a single ASCII letter, a colon, then a separator.
if len(s) >= 3 && s[1] == ':' && (s[2] == '\\' || s[2] == '/') {
c := s[0]
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}
return false
}
// CompleteTask marks a task as completed.
// Issue status is NOT changed here — the agent manages it via the CLI.
//
// For chat tasks, CompleteAgentTask and the chat_session resume-pointer
// update run in a single transaction. This closes a race where the next
// queued chat message could be claimed in the window between the task
// flipping to 'completed' and chat_session.session_id being refreshed,
// causing the new task to resume against a stale (or NULL) session.
// durableWorkDir is terminal delivery metadata, not a resume pointer: it is
// populated only after the daemon confirms a disposable worktree is gone.
func (s *TaskService) CompleteTask(ctx context.Context, taskID pgtype.UUID, result []byte, sessionID, workDir, branchName string, sessionRolloutMissing bool, retiredSessionID, durableWorkDir string) (*db.AgentTaskQueue, error) {
task, _, err := s.CompleteTaskWithTransition(ctx, taskID, result, sessionID, workDir, branchName, sessionRolloutMissing, retiredSessionID, durableWorkDir)
return task, err
}
// CompleteTaskWithTransition reports whether this call won the running ->
// completed compare-and-swap. Callers with transaction-external side effects
// must only run them when transitioned is true; a replay against an already
// terminal task is still an idempotent success but must not emit them again.
func (s *TaskService) CompleteTaskWithTransition(ctx context.Context, taskID pgtype.UUID, result []byte, sessionID, workDir, branchName string, sessionRolloutMissing bool, retiredSessionID, durableWorkDir string) (*db.AgentTaskQueue, bool, error) {
var task db.AgentTaskQueue
var change *qoderIssueChange
// chatAssistantMsg is the single assistant outcome row written for a chat
// task inside the completion transaction below. It is broadcast (chat:done)
// only after the transaction commits.
var chatAssistantMsg *db.ChatMessage
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
if err := lockChatSessionForTaskWrite(ctx, qtx, taskID); err != nil {
return err
}
t, err := qtx.CompleteAgentTask(ctx, db.CompleteAgentTaskParams{
ID: taskID,
Result: result,
SessionID: pgtype.Text{String: sessionID, Valid: sessionID != ""},
WorkDir: pgtype.Text{String: workDir, Valid: workDir != ""},
DurableWorkDir: pgtype.Text{String: durableWorkDir, Valid: durableWorkDir != ""},
BranchName: pgtype.Text{String: branchName, Valid: branchName != ""},
SessionRolloutMissing: sessionRolloutMissing,
RetiredSessionID: pgtype.Text{String: retiredSessionID, Valid: retiredSessionID != ""},
})
if err != nil {
return err
}
task = t
change, err = syncQoderIssue(ctx, qtx, t, true)
if err != nil {
return err
}
// Atomic with the status flip: a crash between the two would leave a
// finished obligation looking pending forever.
if err := SettleTerminalTaskState(ctx, qtx, t); err != nil {
return err
}
if t.ChatSessionID.Valid {
// Pin the chat_session's runtime_id alongside the session_id so the
// next claim can apply the runtime-guard. Both fields move together:
// when there's no session_id to record, leave runtime_id untouched
// (NULL → COALESCE keeps the existing value).
var sessionRuntimeID pgtype.UUID
if sessionID != "" {
sessionRuntimeID = t.RuntimeID
}
// COALESCE in SQL guarantees empty inputs don't wipe the
// existing resume pointer; we still surface DB errors.
if err := qtx.UpdateChatSessionSession(ctx, db.UpdateChatSessionSessionParams{
ID: t.ChatSessionID,
SessionID: pgtype.Text{String: sessionID, Valid: sessionID != ""},
WorkDir: pgtype.Text{String: workDir, Valid: workDir != ""},
RuntimeID: sessionRuntimeID,
}); err != nil {
return fmt.Errorf("update chat session resume pointer: %w", err)
}
// A turn that recovered by abandoning its session still has to
// retire it here. The COALESCE above cannot: a fresh retry that
// succeeds without emitting a new id passes sessionID == "", which
// preserves the pointer — and the pointer is the poisoned session
// (GH #6066). Runs after the update so a real new id wins first and
// the match guard then finds nothing to clear.
if retiredSessionID != "" {
if err := qtx.ClearChatSessionSessionIfMatches(ctx, db.ClearChatSessionSessionIfMatchesParams{
ID: t.ChatSessionID,
SessionID: pgtype.Text{String: retiredSessionID, Valid: true},
RuntimeID: t.RuntimeID,
}); err != nil {
return fmt.Errorf("clear retired chat session resume pointer: %w", err)
}
}
// Write the assistant outcome in the SAME transaction as the status
// flip and resume-pointer update (MUL-4351). For a task-owned direct
// task this is exactly one row (message or no_response); for a
// legacy/channel task an empty output writes no row (see
// writeChatCompletionOutcome). Failing here rolls the whole completion
// back so the daemon retries the terminal callback, and the status CAS
// above guarantees a replay can't write a second row.
msg, err := s.writeChatCompletionOutcome(ctx, qtx, t, result)
if err != nil {
return fmt.Errorf("write chat assistant outcome: %w", err)
}
chatAssistantMsg = msg
}
return nil
}); err != nil {
// When parallel agents race, a task may already be completed,
// cancelled, or failed by the time this call runs. The UPDATE
// … WHERE status = 'running' returns no rows in that case.
// Treat it as an idempotent success — same pattern as CancelTask.
if existing, lookupErr := s.Queries.GetAgentTask(ctx, taskID); lookupErr == nil {
if errors.Is(err, pgx.ErrNoRows) {
slog.Info("complete task: already finalized",
"task_id", util.UUIDToString(taskID),
"current_status", existing.Status,
"agent_id", util.UUIDToString(existing.AgentID),
)
return &existing, false, nil
}
slog.Warn("complete task failed",
"task_id", util.UUIDToString(taskID),
"current_status", existing.Status,
"issue_id", util.UUIDToString(existing.IssueID),
"chat_session_id", util.UUIDToString(existing.ChatSessionID),
"agent_id", util.UUIDToString(existing.AgentID),
"error", err,
)
} else {
slog.Warn("complete task failed: task not found",
"task_id", util.UUIDToString(taskID),
"lookup_error", lookupErr,
)
}
return nil, false, fmt.Errorf("complete task: %w", err)
}
s.publishQoderIssueChange(ctx, change)
slog.Info("task completed", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(task.IssueID))
s.captureTaskCompleted(ctx, task)
// Invariant: every completed issue task must have at least one agent
// comment on the issue, so the user always sees something when a run
// ends. If the agent posted a comment during execution (result, progress
// ping, or CLI reply), HasAgentCommentedSince returns true and we skip.
// Otherwise, synthesize one from the final output. For comment-triggered
// tasks, TriggerCommentID threads the fallback under the original comment;
// for assignment-triggered tasks it is NULL and the fallback is top-level.
// Chat tasks have no IssueID and are handled separately below.
if task.IssueID.Valid {
suppressNoActionComment, err := HasSquadLeaderNoActionEvaluationForTask(ctx, s.Queries, task)
if err != nil {
slog.Warn("checking squad leader no_action evaluation failed",
"task_id", util.UUIDToString(task.ID),
"issue_id", util.UUIDToString(task.IssueID),
"agent_id", util.UUIDToString(task.AgentID),
"error", err,
)
}
agentCommented, _ := s.Queries.HasAgentCommentedSince(ctx, db.HasAgentCommentedSinceParams{
IssueID: task.IssueID,
AuthorID: task.AgentID,
Since: task.StartedAt,
})
// A scheduled wakeup check that found nothing new ends with a check-in
// instead of a comment (see IssueWakeupService.CheckIn).
if !suppressNoActionComment && !agentCommented && !HasWakeupCheckin(task) {
var payload protocol.TaskCompletedPayload
if err := json.Unmarshal(result, &payload); err == nil {
if payload.Output != "" {
// Match the CLI's --content / --description behavior: agents that
// emit literal `\n` 4-char sequences (Python/JSON-style) get them
// decoded into real newlines before the comment hits the DB. See
// util.UnescapeBackslashEscapes for the exact contract.
body := util.UnescapeBackslashEscapes(payload.Output)
if task.TriggerCommentID.Valid && isTrivialDoneOutput(body) {
slog.Warn("suppressing trivial comment-trigger fallback output",
"task_id", util.UUIDToString(task.ID),
"issue_id", util.UUIDToString(task.IssueID),
"agent_id", util.UUIDToString(task.AgentID),
)
} else {
// Redact first, then bound: a runaway raw-stream Output (GH #5455)
// must never reach the issue thread, even as a clipped excerpt.
content := truncateFallbackCommentBody(redact.Text(body), maxSynthesizedFallbackCommentRunes)
s.createAgentComment(ctx, task.IssueID, task.AgentID, content, "comment", task.TriggerCommentID, task.ID)
}
}
}
}
}
// Quick-create tasks: locate the issue the agent just created and push
// an inbox confirmation to the requester. The agent has no issue / chat
// link, so the regular completion paths above don't apply. We find the
// new issue by querying for the most recent issue this agent created in
// the requester's workspace since the task started — more robust than
// parsing the agent's stdout for an identifier.
if qc, ok := s.parseQuickCreateContext(task); ok {
s.notifyQuickCreateCompleted(ctx, task, qc, result)
}
// For chat tasks, broadcast chat:done AFTER commit. The single assistant
// outcome row (message or no_response) and the resume pointer were already
// persisted inside the transaction above. Unread is derived from the read
// cursor (chat_session.last_read_at) vs the assistant messages after it — a
// no_response row has role='assistant' and a fresh created_at, so it counts
// as unread just like a text reply, no per-reply stamping needed.
if task.ChatSessionID.Valid {
// The assistant outcome row (message / no_response) and any attachment
// binding were written inside the completion transaction above by
// writeChatCompletionOutcome. Broadcast chat:done AFTER commit.
//
// The quick-actions decision is made here, before the broadcast, so the
// pending flag and the pass that resolves it can never disagree: a
// client only shows the skeleton placeholder when generation is
// actually about to run.
suggest := s.chatQuickActionsEligible(ctx, task, chatAssistantMsg)
s.broadcastChatDone(ctx, task, chatAssistantMsg, suggest)
if suggest {
// Detached: the reply is already delivered and the user's next turn
// must never wait on suggestions.
s.GenerateChatQuickActionsAsync(task, ChatQuickActionsAutomatic)
}
}
// Reconcile agent status
s.ReconcileAgentStatus(ctx, task.AgentID)
// Broadcast
s.broadcastTaskEvent(ctx, protocol.EventTaskCompleted, task)
return &task, true, nil
}
// chatNoResponseFallback is the non-empty English body stored on a no_response
// assistant row. New clients render a localized "no text reply this turn"
// message keyed on message_kind='no_response'; older clients that ignore
// message_kind still show this text instead of an empty bubble (MUL-4351).
const chatNoResponseFallback = "The agent finished this turn without a text reply."
// writeChatCompletionOutcome writes the assistant chat_message outcome for a
// completed chat task inside the caller's completion transaction, returning the
// row (nil when none is written).
//
// Direct (web/mobile) tasks get the explicit single-outcome contract: a
// non-empty final output becomes an ordinary assistant message, and an
// empty/whitespace output becomes a visible no_response outcome carrying a
// non-empty English fallback body. It never auto-retries: an empty output is a
// legitimate terminal result (a tool-only turn) and re-running it would repeat
// side effects already performed.
//
// Channel (Slack/Lark) and legacy tasks keep the prior behavior: a non-empty
// output writes an ordinary assistant message, but an EMPTY output writes NO
// row, so chat:done carries empty content and the channel outbound silently
// drops it (MUL-4351 review): the no_response fallback body must never be
// pushed to an external channel. Channel tasks now own a sealed input batch
// (chat_input_task_id set) just like direct tasks, so the discriminator is the
// immutable channel_ingested stamp on the owned batch — keyed by the batch
// owner id so an auto-retry clone (which inherits chat_input_task_id) reaches
// the same verdict as its parent — while a NULL owner marks a legacy task.
func (s *TaskService) writeChatCompletionOutcome(ctx context.Context, qtx *db.Queries, task db.AgentTaskQueue, result []byte) (*db.ChatMessage, error) {
// result is the daemon request re-marshalled by the handler, so it is always
// valid JSON; an empty Output is the only case this branch cares about.
var payload protocol.TaskCompletedPayload
_ = json.Unmarshal(result, &payload)
// Same unescape as the issue-comment path: literal `\n` from agent stdout
// becomes a real newline so the chat panel renders paragraph breaks.
body := util.UnescapeBackslashEscapes(payload.Output)
// Strip any in-band quick-actions footer from EVERY chat completion — the
// reserved syntax must never reach a stored transcript. This includes the
// agent-initiated intro turn (chat_input_task_id NULL), which previously
// fell outside the strip gate and leaked the raw footer into its content;
// channel outputs never carry the syntax, so the split is a no-op there.
//
// The parsed footer is DISCARDED: suggestions come from the server-side pass
// now (SupplementChatQuickActions). Honouring a footer here would pin a
// pre-upgrade session to the retired, lower-quality in-band suggestions and
// suppress the pass that would have replaced them.
body, _ = splitChatQuickActions(body)
isEmpty := strings.TrimSpace(body) == ""
// MUL-4899 completion-boundary observation. Measures whether the delivery
// contract in the runtime brief is actually landing on the chat surface.
// Strictly non-blocking: the reply is written either way.
s.observeChatOutputLocalPath(task, body)
// Attachments the agent uploaded during this task (tagged with task_id, not
// yet bound to any owner) are part of this reply. They make an empty-text
// turn a real image/file response — NOT a no_response — and need a row to
// hang on. Count + bind run on qtx so message creation and binding are one
// atomic outcome.
wsUUID, _ := util.ParseUUID(s.ResolveTaskWorkspaceID(ctx, task))
var pendingAttachments int64
if wsUUID.Valid {
n, err := qtx.CountUnboundChatAttachmentsForTask(ctx, db.CountUnboundChatAttachmentsForTaskParams{
WorkspaceID: wsUUID,
TaskID: task.ID,
})
if err != nil {
return nil, fmt.Errorf("count chat attachments: %w", err)
}
pendingAttachments = n
}
// Channel/legacy empty completion with nothing to show: emit no assistant
// row, only an empty chat:done for typing/lifecycle. Keeps the Slack/Lark
// silent-drop path — the outbound patcher forwards any non-empty content
// verbatim, so the fallback body must never be written for a channel task.
// Attachments still force a row — the agent produced a deliverable the
// user must see.
if isEmpty && pendingAttachments == 0 {
if !task.ChatInputTaskID.Valid {
return nil, nil // legacy task
}
channelIngested, err := qtx.TaskHasChannelIngestedMessages(ctx, task.ChatInputTaskID)
if err != nil {
return nil, fmt.Errorf("check chat completion channel provenance: %w", err)
}
if channelIngested {
return nil, nil // channel task
}
}
params := db.CreateChatMessageParams{
ID: dbid.NewV7(),
ChatSessionID: task.ChatSessionID,
Role: "assistant",
TaskID: task.ID,
ElapsedMs: computeChatElapsedMs(task),
}
switch {
case !isEmpty:
params.Content = redact.Text(body)
// message_kind left NULL → COALESCE defaults to 'message'.
//
// The one exception is now a deploy-window case. The server writes the
// opening directly (MUL-5827), so no new task ever produces one — but a
// kickoff task enqueued by the previous server can still be claimed by
// this one, and its reply IS that member's opening. Leaving it a plain
// 'message' would permanently cost that session its starter cards.
//
// Gated on "the batch is a kickoff and nothing else", which is precisely
// the old shape: the new kickoff rides in alongside a real member
// message, and stamping that turn would render the cards a second time
// under a reply that is not an opening.
//
// Keyed on chatInputOwnerID, not task.ID: an auto-retry clone gets a
// fresh id while inheriting the root's chat_input_task_id (MUL-4351),
// and the kickoff user row stays bound to the root.
openingOnly, err := qtx.TaskInputIsOnboardingKickoffOnly(ctx, chatInputOwnerID(task))
if err != nil {
return nil, fmt.Errorf("check onboarding kickoff input: %w", err)
}
if openingOnly.Bool {
params.MessageKind = pgtype.Text{String: protocol.ChatMessageKindOnboardingOpening, Valid: true}
}
case pendingAttachments > 0:
// Image/file-only reply: a real 'message' outcome with empty text — the
// attachment cards ARE the response, so it must not read as no_response.
params.Content = ""
default:
// Task-owned direct task, empty output, no attachments: explicit,
// visible no_response outcome.
params.Content = chatNoResponseFallback
params.MessageKind = pgtype.Text{String: protocol.ChatMessageKindNoResponse, Valid: true}
}
row, err := createAssistantChatMessage(ctx, qtx, params)
if err != nil {
return nil, err
}
// Bind the task's still-unclaimed attachments to the reply we just wrote.
if pendingAttachments > 0 && wsUUID.Valid {
bound, err := qtx.BindChatAttachmentsToMessage(ctx, db.BindChatAttachmentsToMessageParams{
ChatMessageID: row.ID,
WorkspaceID: wsUUID,
TaskID: task.ID,
})
if err != nil {
return nil, fmt.Errorf("bind chat attachments: %w", err)
}
if len(bound) > 0 {
slog.Info("bound chat attachments to assistant reply",
"task_id", util.UUIDToString(task.ID),
"message_id", util.UUIDToString(row.ID),
"count", len(bound),
)
}
}
return &row, nil
}
// observeChatOutputLocalPath records a metric when a chat reply references a
// runtime-local path (MUL-4899). Observation only — it never mutates the reply,
// never fails the completion, and makes no claim to have fixed anything.
//
// Two hard constraints shape it:
//
// 1. Lexical only. The path lives on the daemon's machine, so the server cannot
// os.Stat it the way the CLI lint can. That leaves two signals it can be
// confident about without guessing: a `file://` URL, and the task's own
// recorded work_dir as a prefix. Anything subtler would be a guess, and a
// guess is not worth a false signal on a dashboard.
// 2. No path, no body text, and no fragment of either may reach the metric or
// the log — only the classification and the task id.
func (s *TaskService) observeChatOutputLocalPath(task db.AgentTaskQueue, body string) {
if s.Metrics == nil || strings.TrimSpace(body) == "" {
return
}
kind := ""
switch {
case strings.Contains(strings.ToLower(body), "file://"):
kind = "file_url"
case task.WorkDir.Valid && task.WorkDir.String != "" && strings.Contains(body, task.WorkDir.String):
kind = "workdir_path"
default:
return
}
s.Metrics.RecordChatOutputLocalPath(kind)
slog.Warn("chat reply references a runtime-local path",
"task_id", util.UUIDToString(task.ID),
"kind", kind,
)
}
// FailTask marks a task as failed.
// Issue status is NOT changed here — the agent manages it via the CLI.
//
// sessionID/workDir are optional: when the agent established a real session
// before failing (e.g. crashed mid-conversation, was cancelled, or hit a
// tool error), the daemon should pass them so we can preserve the resume
// pointer on both the task row and the chat_session — otherwise the next
// chat turn would silently start a brand-new session and lose memory.
// durableWorkDir is persisted on the task only; it never replaces the actual
// workDir used for session resumption.
//
// failureReason is a coarse classifier consumed by the auto-retry path.
// Pass "" when unknown — the server runs the raw error text through
// taskfailure.Classify so the persisted failure_reason still lands in
// the canonical refined taxonomy rather than the legacy "agent_error"
// coarse bucket. Daemon callers that already produced a refined reason
// (via classifyPoisonedError, the timeout / runtime classifier, etc.)
// will have their value preserved untouched.
func (s *TaskService) FailTask(ctx context.Context, taskID pgtype.UUID, errMsg, sessionID, workDir, branchName, failureReason string, sessionRolloutMissing bool, retiredSessionID, durableWorkDir string) (*db.AgentTaskQueue, error) {
task, _, err := s.FailTaskWithTransition(ctx, taskID, errMsg, sessionID, workDir, branchName, failureReason, sessionRolloutMissing, retiredSessionID, durableWorkDir)
return task, err
}
// FailTaskWithTransition is the failure counterpart to
// CompleteTaskWithTransition. The bool is false for an idempotent replay that
// observed an already-terminal row.
func (s *TaskService) FailTaskWithTransition(ctx context.Context, taskID pgtype.UUID, errMsg, sessionID, workDir, branchName, failureReason string, sessionRolloutMissing bool, retiredSessionID, durableWorkDir string) (*db.AgentTaskQueue, bool, error) {
// Strip bytes PostgreSQL cannot store before anything else reads errMsg, so
// the classifier, the transaction and every downstream consumer see the one
// text we will actually persist (GH #7098). Kept at the service boundary
// rather than only in the /fail handler because failClaimedTaskBeforeLaunch
// reaches FailTask without going through request decoding.
errMsg = util.SanitizeTextForPostgres(errMsg)
// MUL-2946: synthesise a refined reason from the error text whenever the
// caller didn't supply one. This is the last write-path guard against
// "agent_error" coarse rows ending up in agent_task_queue.failure_reason
// — every other path either provides a classified reason directly
// (sweepers writing 'queued_expired' / 'runtime_offline' / 'timeout'
// / 'runtime_recovery' via SQL) or runs the daemon's classifyPoisonedError
// + taskfailure.Classify chain.
if failureReason == "" {
failureReason = taskfailure.Classify(errMsg).String()
}
// MUL-5370: daemons upgrade on their own cadence, so a fix that depends on
// a new daemon-side label only reaches hosts that happened to update. An
// older daemon reports a *non-empty* catchall for a failed skill-bundle
// download, which the branch above deliberately leaves alone — without this
// the retry and the actionable copy would both skip every un-upgraded host.
// Runs after the empty-reason branch so a legacy reason synthesised there
// is normalised too, and before the retry pre-compute below so the upgraded
// reason is what decides retry eligibility.
failureReason = taskfailure.NormalizeDaemonReason(failureReason, errMsg).String()
// Pre-compute the auto-retry so the retry child can be created inside the
// SAME transaction as the fail (MUL-4351). Doing it atomically closes the
// window between the fail committing and the retry appearing during which a
// newer chat task could claim the now-idle session and jump ahead of the
// retry. The overlay build can do network I/O (Composio), so we resolve it
// here — before the transaction — and only for retryable failures, so the
// common agent_error path skips this work entirely.
var (
wantRetry bool
retryOverlay runtimeMCPOverlayData
retryFireAt pgtype.Timestamptz
retryMaxAttempts pgtype.Int4
)
if retryableReasons[failureReason] {
if parent, perr := s.Queries.GetAgentTask(ctx, taskID); perr != nil {
slog.Warn("fail task auto-retry: load parent failed",
"task_id", util.UUIDToString(taskID), "error", perr)
} else if retryEligible(failureReason, parent) {
wantRetry = true
// Persist the reason-aware effective budget into the child so the
// retry chain self-describes (e.g. provider_network → max_attempts=3),
// rather than leaking a contradictory attempt=N/max_attempts=2 row.
retryMaxAttempts = pgtype.Int4{Int32: retryAttemptCeiling(failureReason, parent.MaxAttempts), Valid: true}
// Defer this attempt when the reason's schedule calls for a backoff
// (provider_network's final attempt waits ~5s); a zero delay leaves
// fire_at NULL so the child is created immediately-claimable.
if delay := retryDelayForAttempt(failureReason, parent.Attempt); delay > 0 {
retryFireAt = pgtype.Timestamptz{Time: time.Now().Add(delay), Valid: true}
}
if agent, aerr := s.Queries.GetAgent(ctx, parent.AgentID); aerr != nil {
// Best-effort: a missing overlay is not retry-fatal — the child
// simply runs without the Composio overlay.
slog.Warn("fail task auto-retry: load agent for overlay failed",
"task_id", util.UUIDToString(taskID),
"agent_id", util.UUIDToString(parent.AgentID), "error", aerr)
} else {
retryOverlay = s.buildRuntimeMCPOverlay(ctx, parent.OriginatorUserID, agent)
}
}
}
var task db.AgentTaskQueue
var retried *db.AgentTaskQueue
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
if err := lockChatSessionForTaskWrite(ctx, qtx, taskID); err != nil {
return err
}
t, err := qtx.FailAgentTask(ctx, db.FailAgentTaskParams{
ID: taskID,
Error: pgtype.Text{String: errMsg, Valid: true},
FailureReason: pgtype.Text{String: failureReason, Valid: failureReason != ""},
SessionID: pgtype.Text{String: sessionID, Valid: sessionID != ""},
WorkDir: pgtype.Text{String: workDir, Valid: workDir != ""},
DurableWorkDir: pgtype.Text{String: durableWorkDir, Valid: durableWorkDir != ""},
// A failed run can still have produced a branch: worktree mode
// commits whatever the agent left before tearing the worktree down,
// precisely so partial work survives. Dropping the name here would
// leave that commit with no pointer to it.
BranchName: pgtype.Text{String: branchName, Valid: branchName != ""},
SessionRolloutMissing: sessionRolloutMissing,
RetiredSessionID: pgtype.Text{String: retiredSessionID, Valid: retiredSessionID != ""},
})
if errors.Is(err, pgx.ErrNoRows) && failureReason == taskfailure.ReasonProcessCleanupUnconfirmed.String() {
t, err = qtx.FailCancelledTaskProcessCleanup(ctx, db.FailCancelledTaskProcessCleanupParams{ID: taskID, ErrorMessage: errMsg})
}
if err != nil {
return err
}
task = t
// Atomic with the status flip, same as the completion path. A failed
// coordinator that already received the recovery comment has consumed
// the obligation: the pre-existing delivered_comment_ids coverage check
// never looked at the covering task's status either.
if err := SettleTerminalTaskState(ctx, qtx, t); err != nil {
return err
}
// Keep resume-unsafe sessions on the task row for observability, but
// do not promote them to the chat-level resume pointer.
//
// Declining to overwrite is not enough on its own: the claim handler
// reads chat_session.session_id BEFORE consulting
// GetLastChatTaskSession, so a pointer already holding the dead
// session bypasses every filter that query applies and the next turn
// resumes it (GH #6066). Clear it in this same transaction, matched on
// the exact session and runtime so a concurrent turn's newer pointer
// is left alone. ResumeUnsafeFailure (not the reason alone) so an
// un-upgraded daemon's agent_error.unknown row is caught by the error
// text too.
if t.ChatSessionID.Valid && ResumeUnsafeFailure(failureReason, errMsg) {
deadSession := sessionID
if deadSession == "" {
deadSession = t.SessionID.String
}
if deadSession != "" {
if err := qtx.ClearChatSessionSessionIfMatches(ctx, db.ClearChatSessionSessionIfMatchesParams{
ID: t.ChatSessionID,
SessionID: pgtype.Text{String: deadSession, Valid: true},
RuntimeID: t.RuntimeID,
}); err != nil {
return fmt.Errorf("clear poisoned chat session resume pointer: %w", err)
}
}
}
// A session the run explicitly retired must go too, whatever the
// terminal status: the fresh-session retry that replaced it may well
// have succeeded, and the pointer would otherwise still name the
// transcript it retried away from.
if t.ChatSessionID.Valid && retiredSessionID != "" {
if err := qtx.ClearChatSessionSessionIfMatches(ctx, db.ClearChatSessionSessionIfMatchesParams{
ID: t.ChatSessionID,
SessionID: pgtype.Text{String: retiredSessionID, Valid: true},
RuntimeID: t.RuntimeID,
}); err != nil {
return fmt.Errorf("clear retired chat session resume pointer: %w", err)
}
}
// ResumeUnsafeFailure, not resumeUnsafeFailureReason: the reason-only
// check passes an un-upgraded daemon's agent_error.unknown row, which
// would re-pin the very session the clear above just removed.
if t.ChatSessionID.Valid && !ResumeUnsafeFailure(failureReason, errMsg) {
// Pin the chat_session's runtime_id alongside the session_id so the
// next claim can apply the runtime-guard. Both fields move together:
// when there's no session_id to record, leave runtime_id untouched
// (NULL → COALESCE keeps the existing value).
var sessionRuntimeID pgtype.UUID
if sessionID != "" {
sessionRuntimeID = t.RuntimeID
}
if err := qtx.UpdateChatSessionSession(ctx, db.UpdateChatSessionSessionParams{
ID: t.ChatSessionID,
SessionID: pgtype.Text{String: sessionID, Valid: sessionID != ""},
WorkDir: pgtype.Text{String: workDir, Valid: workDir != ""},
RuntimeID: sessionRuntimeID,
}); err != nil {
return fmt.Errorf("update chat session resume pointer: %w", err)
}
}
// Create the retry child atomically with the fail. CreateRetryTask reads
// the just-failed parent row (same tx), so it inherits chat_input_task_id
// and the bumped chat-retry priority; broadcast/notify happen after commit.
// This check is an optimisation, NOT the safety mechanism. It is a plain
// count at READ COMMITTED and takes no lock, so a rerun can always commit
// between it and the insert below; it only saves the work when a
// successor is already visible and gives the skip a readable log line.
// Correctness under that race belongs to CreateRetryTask's
// ON CONFLICT DO NOTHING, which yields the slot instead of raising and so
// can never abort this transaction — which also carries the parent's
// failed status.
createRetry := wantRetry
if createRetry {
successor, herr := hasRunnableSuccessor(ctx, qtx, t)
if herr != nil {
return fmt.Errorf("check runnable successor: %w", herr)
}
if successor {
slog.Info("fail task auto-retry skipped: a successor is already pending",
"task_id", util.UUIDToString(taskID),
"issue_id", util.UUIDToString(t.IssueID),
"agent_id", util.UUIDToString(t.AgentID),
)
createRetry = false
}
}
// The queue door, on the one insert path that cannot fail loudly. A
// refusal here must never abort the transaction — it also carries the
// parent's failed status, and losing that leaves the task stuck in
// 'running' — so both a Triage issue and an unreadable status skip the
// retry instead of returning, exactly as the ErrNoRows case below does.
if createRetry {
if gerr := guardIssueNotInTriage(ctx, qtx, t.IssueID, OriginDerived); gerr != nil {
slog.Info("fail task auto-retry skipped: issue does not run",
"task_id", util.UUIDToString(taskID),
"issue_id", util.UUIDToString(t.IssueID),
"error", gerr,
)
createRetry = false
}
}
if createRetry {
child, cerr := qtx.CreateRetryTask(ctx, db.CreateRetryTaskParams{
NewTaskID: dbid.NewV7(),
ID: taskID,
FireAt: retryFireAt,
MaxAttempts: retryMaxAttempts,
RuntimeMcpOverlay: retryOverlay.Overlay,
RuntimeConnectedApps: retryOverlay.ConnectedApps,
})
switch {
case cerr == nil:
transferErr := transferPendingSourceContextToRetry(ctx, qtx, t, child)
if errors.Is(transferErr, pgx.ErrNoRows) {
deleted, deleteErr := qtx.DeleteUnstartedQuickCreateRetryTask(ctx, child.ID)
if deleteErr != nil || deleted != 1 {
return fmt.Errorf("discard source context retry without attach authority: changed=%d: %w", deleted, deleteErr)
}
slog.Info("fail task auto-retry skipped: source context already transferred or attached",
"task_id", util.UUIDToString(taskID),
"source_context_retry_id", util.UUIDToString(child.ID),
)
} else if transferErr != nil {
return fmt.Errorf("transfer pending source context to retry: %w", transferErr)
} else {
if err := qtx.CopyChannelTaskDelivery(ctx, db.CopyChannelTaskDeliveryParams{
ChildTaskID: child.ID, ParentTaskID: taskID,
}); err != nil {
return fmt.Errorf("copy retry channel delivery: %w", err)
}
retried = &child
}
case errors.Is(cerr, pgx.ErrNoRows):
// The statement wrote nothing: either the owning workspace was
// torn down mid-flight, or a rerun took the pending slot after
// the unlocked check above (ON CONFLICT DO NOTHING). Neither is
// a reason to abort — this transaction still owns the parent's
// failed status, and losing that would leave the task stuck in
// 'running'. Record the failure and move on without a retry.
slog.Info("fail task auto-retry not created: no row written",
"task_id", util.UUIDToString(taskID),
"issue_id", util.UUIDToString(t.IssueID),
"agent_id", util.UUIDToString(t.AgentID),
)
default:
return fmt.Errorf("create retry task: %w", cerr)
}
}
// A terminal non-retried chat failure is a visible assistant outcome.
// Persist it while the session lock and failure transaction are still
// held, then reanchor the next direct head. Otherwise the successor could
// be claimed between the status flip and this row, placing its user input
// before the failure it follows.
if t.ChatSessionID.Valid && retried == nil {
// This turn is dead, so anything it owned has to move on. An adopted
// onboarding kickoff would otherwise stay bound to a task that will
// never run again: the next turn would reach Mika with no onboarding
// context and no record that she had already greeted the member, and
// she would introduce herself a second time (MUL-5827). The query
// hands it to the session's next queued turn — including one the
// member queued while THIS turn was still running, which adoption at
// send time could not have caught.
//
// Gated on retried == nil on purpose. A retry child inherits the
// root's chat_input_task_id, and the kickoff stays bound to that
// root, so the retry still reads it — releasing here would strip
// the context off a turn that is about to run.
if err := qtx.ReleaseOnboardingKickoffFromTask(ctx, chatInputOwnerID(t)); err != nil {
return fmt.Errorf("release onboarding kickoff: %w", err)
}
if _, err := createAssistantChatMessage(ctx, qtx, db.CreateChatMessageParams{
ID: dbid.NewV7(),
ChatSessionID: t.ChatSessionID,
Role: "assistant",
Content: redact.Text(errMsg),
TaskID: t.ID,
FailureReason: pgtype.Text{String: failureReason, Valid: failureReason != ""},
ElapsedMs: computeChatElapsedMs(t),
}); err != nil {
return fmt.Errorf("write chat failure outcome: %w", err)
}
}
return nil
}); err != nil {
if existing, lookupErr := s.Queries.GetAgentTask(ctx, taskID); lookupErr == nil {
if errors.Is(err, pgx.ErrNoRows) {
slog.Info("fail task: already finalized",
"task_id", util.UUIDToString(taskID),
"current_status", existing.Status,
"agent_id", util.UUIDToString(existing.AgentID),
)
return &existing, false, nil
}
slog.Warn("fail task failed",
"task_id", util.UUIDToString(taskID),
"current_status", existing.Status,
"issue_id", util.UUIDToString(existing.IssueID),
"chat_session_id", util.UUIDToString(existing.ChatSessionID),
"agent_id", util.UUIDToString(existing.AgentID),
"error", err,
)
} else {
slog.Warn("fail task failed: task not found",
"task_id", util.UUIDToString(taskID),
"lookup_error", lookupErr,
)
}
return nil, false, fmt.Errorf("fail task: %w", err)
}
slog.Warn("task failed", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(task.IssueID), "error", errMsg, "failure_reason", failureReason)
s.captureTaskFailed(ctx, task)
// The auto-retry child (if any) was created inside the transaction above so
// no newer chat task could jump ahead of it. Surface it now: broadcast
// queued first, then notify the daemon — see EnqueueTaskForIssue for the
// ordering rationale. A deferred child (backoff armed via fire_at) is NOT
// queued yet: PromoteDueDeferredTasksForRuntime emits its queued event and
// daemon wakeup when fire_at arrives, so announcing it here would be wrong.
if retried != nil {
slog.Info("task auto-retry enqueued",
"parent_task_id", util.UUIDToString(task.ID),
"child_task_id", util.UUIDToString(retried.ID),
"reason", failureReason,
"attempt", retried.Attempt,
"max_attempts", retried.MaxAttempts,
"status", retried.Status,
)
if retried.Status == "queued" {
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, *retried)
s.NotifyTaskEnqueued(ctx, *retried)
}
}
// A delegated task that has reached a terminal failure must hand control
// back to the task that delegated it. This runs only after the existing
// retry decision: retryable failures keep their current policy and remain
// silent while a child attempt is pending. The recovery signal is a
// platform-authored comment on the source task's issue, routed explicitly to
// that task's agent; recoverDelegatedTaskFailure coalesces it with an
// existing coordinator run and deduplicates by the failed task id.
if retried == nil && failureReason != taskfailure.ReasonProcessCleanupUnconfirmed.String() && failureReason != taskfailure.ReasonProcessCleanupBlocked.String() {
_, recoveryErr := s.recoverDelegatedTaskFailure(ctx, task)
if recoveryErr != nil {
slog.Warn("delegated task failure recovery failed",
"task_id", util.UUIDToString(task.ID),
"delegated_from_task_id", util.UUIDToString(task.DelegatedFromTaskID),
"error", recoveryErr,
)
}
}
// Skip the per-failure system comment when we'll immediately retry —
// the new task will surface its own status to the user, and we don't
// want to spam the issue with "task timed out" messages on every
// daemon hiccup. Delegated failures keep this existing failed-issue comment
// in addition to the coordinator recovery signal, preserving visibility on
// both sides of a cross-issue handoff.
if errMsg != "" && task.IssueID.Valid && retried == nil {
s.createAgentComment(ctx, task.IssueID, task.AgentID, redact.Text(errMsg), "system", task.TriggerCommentID, task.ID)
}
// Quick-create tasks: push a failure inbox notification to the
// requester so they can either retry or fall back to the advanced form
// without losing their original prompt. Skipped when an auto-retry is
// pending — the new attempt will write its own outcome.
if retried == nil {
if qc, ok := s.parseQuickCreateContext(task); ok {
attached, attachedErr := s.sourceContextAttachedByTask(ctx, task, qc)
switch {
case attachedErr != nil:
slog.Error("quick-create failure: source context outcome lookup failed",
"task_id", util.UUIDToString(task.ID), "error", attachedErr)
s.notifyQuickCreateUnconfirmed(ctx, task, qc)
case attached:
// The CLI create committed before the runtime reported its own
// failure. The attached context is authoritative proof that the
// target exists, so reconcile the normal success inbox rather than
// inviting a duplicate retry from a misleading failure row.
s.notifyQuickCreateCompleted(ctx, task, qc, nil)
default:
s.notifyQuickCreateFailed(ctx, task, qc, errMsg)
}
}
}
// Reconcile agent status
s.ReconcileAgentStatus(ctx, task.AgentID)
// Broadcast. Channel subscribers need the same redacted failure text that
// was persisted in the chat transcript. A retry-pending attempt stays silent
// because its child reports the eventual terminal outcome.
s.broadcastTaskFailedEvent(ctx, task, errMsg, failureReason, retried != nil)
return &task, true, nil
}
// retryableReasons enumerates failure reasons that the auto-retry path is
// allowed to act on. Agent-side errors (compile failures, model rejections,
// etc.) are intentionally excluded — those are real problems that the user
// should see, not infrastructure flakiness.
//
// The one agent_error.* exception is provider_network: a mid-stream provider
// disconnect (e.g. Claude Code's "API Error: Connection closed mid-response")
// is transient infrastructure flakiness, not an agent decision. Unattended
// issue runs otherwise terminate on it, while interactive chat only survives
// because the CLI's own in-process retry happens to recover first — so we make
// the platform retry it directly (MUL-4910). It is resume-safe (not in
// resumeUnsafeFailureReason), so the retry child inherits the session and
// continues the truncated conversation rather than restarting from scratch.
// skill_bundle_unavailable is retryable for the same reason: the agent process
// never started, so there is nothing to be idempotent about, and every bundle
// that did download is already cached on disk — a retry resumes from there
// instead of re-fetching the whole set (MUL-5370).
var retryableReasons = map[string]bool{
string(taskfailure.ReasonRuntimeOffline): true,
string(taskfailure.ReasonRuntimeRecovery): true,
string(taskfailure.ReasonTimeout): true,
"codex_semantic_inactivity": true,
string(taskfailure.ReasonAgentProviderNetwork): true,
string(taskfailure.ReasonSkillBundleUnavailable): true,
}
// runtime_offline retries start deferred, not queued: their positive fire_at
// routes them through health-gated promotion after a fresh runtime heartbeat
// returns. The queue sweeper also exempts this retry lineage, covering the race
// where a runtime disconnects again after promotion but before claim. This
// delay is only a state marker, not the reconnect policy; the heartbeat gate is
// authoritative.
//
// Transient provider stream cuts (provider_network) get a bespoke three-tier
// schedule (MUL-4910): first run + immediate retry + one retry deferred ~5s.
// A blip that survives the immediate retry gets a short cooldown before the
// final attempt instead of firing back-to-back. Every other retryable reason
// keeps the task's generic max_attempts ceiling and retries immediately.
const (
runtimeOfflineRetryDeferral = time.Second
providerNetworkMaxAttempts = 3
providerNetworkFinalRetryWait = 5 * time.Second
)
// retryAttemptCeiling reports how many attempts the auto-retry path allows for
// a failure reason. It only ever WIDENS the task's generic max_attempts, and
// only for reasons with a bespoke schedule; everything else keeps the column's
// value (default 2 = first run + one retry).
//
// max_attempts <= 1 explicitly disables auto-retry (055_task_lease_and_retry.up
// .sql: "1 disables retry"), so it is never overridden — a disabled task must
// not be revived by a raised ceiling. Callers persist this value into the retry
// child (CreateRetryTask's max_attempts) so the row stays self-consistent:
// provider_network's chain records attempt=3, max_attempts=3, not a
// contradictory attempt=3, max_attempts=2 (MUL-4910).
func retryAttemptCeiling(reason string, taskMaxAttempts int32) int32 {
if taskMaxAttempts <= 1 {
return taskMaxAttempts
}
if reason == string(taskfailure.ReasonAgentProviderNetwork) && taskMaxAttempts < providerNetworkMaxAttempts {
return providerNetworkMaxAttempts
}
return taskMaxAttempts
}
// retryDelayForAttempt reports how long to defer the NEXT attempt after a
// failure at failedAttempt. runtime_offline always gets a positive fire_at so
// it waits for the health-gated promotion path. provider_network's final
// attempt is deferred ~5s; every other retry remains immediate (zero delay →
// the child is created 'queued', claimable at once). Callers pass the returned
// delay to CreateRetryTask via fire_at.
func retryDelayForAttempt(reason string, failedAttempt int32) time.Duration {
if reason == string(taskfailure.ReasonRuntimeOffline) {
return runtimeOfflineRetryDeferral
}
if reason == string(taskfailure.ReasonAgentProviderNetwork) &&
failedAttempt >= providerNetworkMaxAttempts-1 {
return providerNetworkFinalRetryWait
}
return 0
}
func resumeUnsafeFailureReason(reason string) bool {
switch reason {
// Failures that poison the agent CONVERSATION (not the workdir): resuming
// the same session would immediately replay the stuck/oversized state.
// Keep in sync with the GetLastTaskSession / GetLastChatTaskSession resume
// blacklists. (CreateRetryTask's fresh-session CASE WHEN only needs the
// subset of these that is also auto-retryable, currently
// codex_semantic_inactivity.)
// codex_resume_oversized is the strongest member of this set: a codex
// rollout only ever grows, so a thread whose resume response already
// overflowed the reader will overflow on every future attempt too.
case "iteration_limit", "agent_fallback_message", "api_invalid_request", "codex_semantic_inactivity", "agent_error.context_overflow", "codex_resume_oversized", "opencode_clock_skew":
return true
default:
return false
}
}
// ResumeUnsafeFailure reports whether a failed task's agent session must NOT be
// resumed on a retry. It combines the failure_reason poison set
// (resumeUnsafeFailureReason) with the SAME defense-in-depth on raw error text
// that the GetLastTaskSession / GetLastChatTaskSession resume queries apply: an
// Anthropic 400 invalid_request_error means the conversation history itself is
// unprocessable even when failure_reason was mis- or un-classified (legacy
// 'agent_error' rows written before MUL-1921, or deploy-window rows). Callers
// that only have a failure_reason (e.g. at fail time) may pass an empty
// errorText.
//
// This is the shared source of truth for the manual-retry claim path, which
// reads the exact source task instead of GetLastTaskSession and would otherwise
// bypass the error-text guard.
func ResumeUnsafeFailure(failureReason, errorText string) bool {
if resumeUnsafeFailureReason(failureReason) {
return true
}
lower := strings.ToLower(errorText)
if strings.Contains(lower, "400") && strings.Contains(lower, "invalid_request_error") {
return true
}
// Provider credential-resolution failures are deterministic on resume: the
// missing api_key / auth_token / auth header is baked into the session's
// provider state, so a rerun must start fresh instead of replaying the same
// auth error on the recorded (agent, issue) session. taskfailure.Classify
// deliberately leaves this error as agent_error.unknown, so this
// reason-independent text guard is the load-bearing protection for both new
// and already persisted rows. Keep it in sync with the GetLastTaskSession /
// GetLastChatTaskSession resume queries.
//
// The phrase itself lives in taskfailure.AuthMethodUnresolved, shared with
// the daemon's in-turn fresh-session retry gate so the two layers cannot
// disagree about which errors mean "this session can never be resumed".
if taskfailure.AuthMethodUnresolved(errorText) {
return true
}
// Same defense-in-depth for the provider-agnostic empty-message shape:
// a daemon too old to carry classifyPoisonedError's new branch reports
// agent_error.unknown, and without this the manual-retry path would
// happily resume the transcript the provider just refused (GH #6066).
return taskfailure.UnresumableHistory(errorText)
}
// retryEligible reports whether a failed task qualifies for an automatic retry
// attempt: an infrastructure-shaped failure_reason, remaining attempt budget,
// not an autopilot run, not a Triage run, and linked to an issue or chat
// session. Shared by FailTask's in-transaction retry and the orphan sweeper's
// MaybeRetryFailedTask so both agree on which failures re-run.
//
// A Triage run is excluded for the reason autopilot runs are (MUL-7189 §5.6):
// it has its own recovery. Re-triaging reads the entry as it stands now, which
// a human may have edited in the meantime, so replaying the failed attempt is
// never what the workspace wants.
func retryEligible(failureReason string, t db.AgentTaskQueue) bool {
return retryableReasons[failureReason] &&
t.Attempt < retryAttemptCeiling(failureReason, t.MaxAttempts) &&
!t.AutopilotRunID.Valid &&
!IsTriageTask(t) &&
(t.IssueID.Valid || t.ChatSessionID.Valid || isSourceContextQuickCreateTask(t))
}
func isSourceContextQuickCreateTask(task db.AgentTaskQueue) bool {
if len(task.Context) == 0 || task.IssueID.Valid || task.ChatSessionID.Valid || task.AutopilotRunID.Valid {
return false
}
var quickCreate QuickCreateContext
return json.Unmarshal(task.Context, &quickCreate) == nil &&
quickCreate.Type == QuickCreateContextType && quickCreate.SourceContextID != ""
}
// hasRunnableSuccessor reports whether another not-yet-started task already
// occupies the single queued/dispatched slot that
// idx_one_pending_task_per_issue_agent_v2 allows per (issue, agent).
//
// Both auto-retry paths consult this before inserting a retry child. A manual
// rerun can now be enqueued BEHIND a still-running task instead of cancelling
// it, so by the time that task fails its slot may already be taken, and the
// retry exists to give the issue a runnable successor that already exists.
//
// This is advisory only: the query takes no lock, so a rerun committing after it
// returns is always possible. CreateRetryTask's ON CONFLICT DO NOTHING is what
// makes losing that race harmless. Skipping early just avoids pointless work and
// records why no retry was made.
//
// Chat / quick-create tasks carry no issue_id, so the index cannot apply to them
// and their retries can never collide.
func hasRunnableSuccessor(ctx context.Context, q *db.Queries, task db.AgentTaskQueue) (bool, error) {
if !task.IssueID.Valid {
return false, nil
}
return q.HasPendingTaskForIssueAndAgentInThread(ctx, db.HasPendingTaskForIssueAndAgentInThreadParams{
ThreadCommentID: task.TriggerCommentID,
IssueID: task.IssueID,
AgentID: task.AgentID,
})
}
// MaybeRetryFailedTask spawns a fresh queued attempt for a recently-failed
// task when the failure was infrastructure-shaped (daemon crash, runtime
// went offline, dispatch/run timeout) and the task hasn't exhausted its
// max_attempts budget. The child task inherits agent/runtime/issue/chat
// links and, for resume-safe failures, the parent's session_id/work_dir so
// the agent can resume the conversation when the backend supports it. Returns
// the new task, or nil when no retry was created.
//
// Autopilot tasks are NOT auto-retried here; the autopilot scheduler owns
// its own re-run cadence and we don't want to double-fire it.
func (s *TaskService) MaybeRetryFailedTask(ctx context.Context, parent db.AgentTaskQueue) (*db.AgentTaskQueue, error) {
if parent.Status != "failed" {
return nil, nil
}
reason := ""
if parent.FailureReason.Valid {
reason = parent.FailureReason.String
}
if !retryableReasons[reason] {
return nil, nil
}
// Use the reason-aware ceiling, not the raw max_attempts column, so an
// orphaned provider_network task recovered on its 2nd attempt is still
// allowed its deferred 3rd attempt (retryAttemptCeiling raises the ceiling
// to 3). Kept in sync with retryEligible below, which applies the same
// ceiling to the primary FailTask path.
if parent.Attempt >= retryAttemptCeiling(reason, parent.MaxAttempts) {
slog.Info("task auto-retry skipped: budget exhausted",
"task_id", util.UUIDToString(parent.ID),
"attempt", parent.Attempt,
"max_attempts", parent.MaxAttempts,
"ceiling", retryAttemptCeiling(reason, parent.MaxAttempts),
)
return nil, nil
}
// Autopilot has its own retry semantics (don't double-trigger) and a task
// with no issue/chat link has nowhere to report its retry — retryEligible
// covers both, keeping this sweeper path in sync with FailTask's in-tx retry.
if !retryEligible(reason, parent) {
return nil, nil
}
var runtimeMCPOverlay runtimeMCPOverlayData
agent, agentErr := s.Queries.GetAgent(ctx, parent.AgentID)
if agentErr != nil {
// Best-effort: failing to resolve the agent for the overlay is not
// retry-fatal. Log and continue — the daemon will reject the claim
// later if the agent is genuinely gone.
slog.Warn("task auto-retry: load agent for overlay failed",
"parent_task_id", util.UUIDToString(parent.ID),
"agent_id", util.UUIDToString(parent.AgentID),
"error", agentErr,
)
} else {
runtimeMCPOverlay = s.buildRuntimeMCPOverlay(ctx, parent.OriginatorUserID, agent)
}
// Mirror FailTask's in-tx backoff + effective-budget persistence: defer the
// final provider_network attempt ~5s via fire_at (zero delay leaves fire_at
// NULL for an immediate child), and write the reason-aware ceiling into the
// child's max_attempts so the retry chain stays self-consistent.
var retryFireAt pgtype.Timestamptz
if delay := retryDelayForAttempt(reason, parent.Attempt); delay > 0 {
retryFireAt = pgtype.Timestamptz{Time: time.Now().Add(delay), Valid: true}
}
// Same advisory slot check as FailTask's path, for the same reason: skip the
// work when a successor is already visible. Losing the race is handled by
// CreateRetryTask yielding the slot, which this caller reads as "no retry".
if successor, herr := hasRunnableSuccessor(ctx, s.Queries, parent); herr != nil {
slog.Warn("task auto-retry: successor check failed; attempting retry anyway",
"parent_task_id", util.UUIDToString(parent.ID), "error", herr)
} else if successor {
slog.Info("task auto-retry skipped: a successor is already pending",
"parent_task_id", util.UUIDToString(parent.ID),
"issue_id", util.UUIDToString(parent.IssueID),
"agent_id", util.UUIDToString(parent.AgentID),
)
return nil, nil
}
if s.TxStarter == nil {
return nil, errors.New("task auto-retry: transaction starter is required")
}
tx, err := s.TxStarter.Begin(ctx)
if err != nil {
return nil, fmt.Errorf("task auto-retry: begin transaction: %w", err)
}
defer tx.Rollback(ctx)
qtx := s.Queries.WithTx(tx)
if err := guardIssueNotInTriage(ctx, qtx, parent.IssueID, OriginDerived); err != nil {
if errors.Is(err, ErrIssueInTriage) {
slog.Info("task auto-retry skipped: issue is in triage",
"parent_task_id", util.UUIDToString(parent.ID),
"issue_id", util.UUIDToString(parent.IssueID))
return nil, nil
}
return nil, err
}
child, err := qtx.CreateRetryTask(ctx, db.CreateRetryTaskParams{
NewTaskID: dbid.NewV7(),
ID: parent.ID,
FireAt: retryFireAt,
MaxAttempts: pgtype.Int4{Int32: retryAttemptCeiling(reason, parent.MaxAttempts), Valid: true},
RuntimeMcpOverlay: runtimeMCPOverlay.Overlay,
RuntimeConnectedApps: runtimeMCPOverlay.ConnectedApps,
})
if errors.Is(err, pgx.ErrNoRows) {
// Workspace torn down, or the pending slot was taken between the check
// above and this insert. Same contract as FailTask's path: no retry, no
// error.
slog.Info("task auto-retry not created: no row written",
"parent_task_id", util.UUIDToString(parent.ID),
"reason", reason,
)
return nil, nil
}
if err != nil {
slog.Warn("task auto-retry failed",
"parent_task_id", util.UUIDToString(parent.ID),
"reason", reason,
"error", err,
)
return nil, err
}
if err := transferPendingSourceContextToRetry(ctx, qtx, parent, child); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
slog.Info("task auto-retry skipped: source context already transferred or attached",
"parent_task_id", util.UUIDToString(parent.ID),
"source_context_retry_id", util.UUIDToString(child.ID),
)
return nil, nil
}
return nil, err
}
if err := qtx.CopyChannelTaskDelivery(ctx, db.CopyChannelTaskDeliveryParams{
ChildTaskID: child.ID, ParentTaskID: parent.ID,
}); err != nil {
return nil, fmt.Errorf("copy auto-retry channel delivery: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return nil, fmt.Errorf("task auto-retry: commit: %w", err)
}
slog.Info("task auto-retry enqueued",
"parent_task_id", util.UUIDToString(parent.ID),
"child_task_id", util.UUIDToString(child.ID),
"reason", reason,
"attempt", child.Attempt,
"max_attempts", child.MaxAttempts,
"status", child.Status,
)
// A queued child transitions ∅ → queued (same as EnqueueTaskFor*): broadcast
// queued first, then notify the daemon — see EnqueueTaskForIssue for ordering
// rationale. A deferred child (backoff armed) stays inert until
// PromoteDueDeferredTasksForRuntime fires its queued event + wakeup.
if child.Status == "queued" {
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, child)
s.NotifyTaskEnqueued(ctx, child)
}
return &child, nil
}
func transferPendingSourceContextToRetry(ctx context.Context, q *db.Queries, parent, child db.AgentTaskQueue) error {
if len(parent.Context) == 0 {
return nil
}
var quickCreate QuickCreateContext
if json.Unmarshal(parent.Context, &quickCreate) != nil || quickCreate.Type != QuickCreateContextType || quickCreate.SourceContextID == "" {
return nil
}
workspaceID, err := util.ParseUUID(quickCreate.WorkspaceID)
if err != nil {
return err
}
contextID, err := util.ParseUUID(quickCreate.SourceContextID)
if err != nil {
return err
}
_, err = q.TransferPendingIssueSourceContextTask(ctx, db.TransferPendingIssueSourceContextTaskParams{
NewTaskID: child.ID, WorkspaceID: workspaceID, ID: contextID, OldTaskID: parent.ID,
})
return err
}
// RerunIssue creates a fresh queued task for an agent on the issue. Used by
// the manual rerun endpoint.
//
// Target agent resolution:
// - sourceTaskID Valid: rerun the agent that ran that task (and reuse its
// leader/worker role). This is what the execution log retry button uses
// so a per-row retry survives a subsequent assignee change and correctly
// re-fires the squad worker or mention agent whose row was clicked. The
// source task's trigger_comment_id is also inherited (when the caller
// didn't pass one) so a per-row rerun of a comment- or mention-triggered
// task stays comment-triggered — the daemon's buildCommentPrompt path
// keys on TriggerCommentID, and losing it would degrade the rerun into
// a generic issue run that no longer carries the original comment.
// - sourceTaskID empty: fall back to the issue's current assignee (agent
// or squad leader). This preserves the CLI / API contract for callers
// that have an issue ID but no specific task to target.
//
// Unless fresh is requested, a retry reuses the source workdir when it exists on
// disk (MUL-4869): a transient failure — network, provider 5xx/rate-limit,
// runtime_offline, timeout, or an auth/quota/config error the user has since
// fixed — should not throw away the work already done. Only the agent SESSION
// is conditionally resumed, and that decision is made later by the daemon claim
// handler from the SOURCE task (via rerun_of_task_id), NOT baked into this row.
// enqueueRerunTask pins force_fresh_session=true so an old claim handler during
// a rolling deploy degrades to a clean start rather than resuming a different
// execution; the new claim handler ignores the flag for reruns and resumes the
// session only when the source failure did not poison the conversation (see
// service.ResumeUnsafeFailure) and the source ran on the same runtime. When the
// dir is objectively unreusable (GC'd, absent on the claiming runtime, or never
// recorded) the daemon falls back to a fresh workdir. Auto-retry of an orphaned
// mid-flight failure (HandleFailedTasks → MaybeRetryFailedTask →
// CreateRetryTask) takes its own path, so MUL-1128's mid-flight resume contract
// is preserved.
//
// ErrRerunInvokeNotAllowed signals that RerunIssue refused to rerun because the
// current operator may not invoke the resolved target agent. The handler maps it
// to a structured 403 (no task was cancelled or created).
var ErrRerunInvokeNotAllowed = errors.New("rerun: operator not allowed to invoke target agent")
// IsFreshRerun reports whether a manual rerun explicitly discards continuity.
// The source lineage stays intact for attribution and execution history.
func IsFreshRerun(task db.AgentTaskQueue) bool {
var context struct {
Fresh bool `json:"fresh_rerun"`
}
return json.Unmarshal(task.Context, &context) == nil && context.Fresh
}
// Only tasks belonging to the target agent on this issue are cancelled.
// Tasks owned by other agents on the same issue (e.g. a parallel
// @-mention agent) are left alone — rerun must not collateral-cancel
// them.
//
// canInvoke re-validates that the current operator may invoke the RESOLVED
// target agent, keyed on the historical agent for a task_id rerun and on the
// current assignee/leader otherwise (MUL-4525). It runs AFTER the target is
// resolved but BEFORE any prior task is cancelled or a new one is created, so a
// caller who can see the issue but cannot invoke its private agent cannot use
// rerun as a back door — and a blocked rerun mutates nothing. Pass nil only
// from trusted internal callers (tests, backfill) that have already gated.
func (s *TaskService) RerunIssue(ctx context.Context, issueID pgtype.UUID, sourceTaskID pgtype.UUID, triggerCommentID pgtype.UUID, actorUserID pgtype.UUID, fresh bool, canInvoke func(agent db.Agent) bool) (*db.AgentTaskQueue, error) {
issue, err := s.Queries.GetIssue(ctx, issueID)
if err != nil {
return nil, fmt.Errorf("load issue: %w", err)
}
// In Triage a rerun follows its source, and the decision is made here —
// before anything is cancelled. The queue door would refuse a derived rerun
// anyway, but this path cancels the prior run on its way there, so a late
// refusal would leave the issue with one fewer run and no new one.
//
// Naming no source means "run the assignee again", which is the derived
// executor Triage does not have. Naming a discussion run is repeating a
// conversation the member started, so it is allowed; naming the triage run
// itself is refused further down, with its own reason.
rerunOrigin := OriginDerived
if sourceTaskID.Valid {
rerunOrigin = OriginNamed
} else if issue.TriageState.Valid {
return nil, ErrIssueInTriage
}
// Determine the target agent for the rerun.
var (
agentID pgtype.UUID
isLeader bool
squadID pgtype.UUID
coalescedCommentIDs []pgtype.UUID
)
if sourceTaskID.Valid {
sourceTask, err := s.Queries.GetAgentTask(ctx, sourceTaskID)
if err != nil {
return nil, fmt.Errorf("load source task: %w", err)
}
if !sourceTask.IssueID.Valid || util.UUIDToString(sourceTask.IssueID) != util.UUIDToString(issueID) {
return nil, fmt.Errorf("source task does not belong to this issue")
}
// A triage run is not an execution run to repeat (MUL-7189 §5.6). This
// is the only path that names a source task directly, and it would
// otherwise outlive Triage: after accept the issue is runnable again, so
// nothing above stops a rerun pointed at the triage task — which resumes
// its session through rerun_of_task_id (the claim handler reads the
// named source, not GetLastTaskSession) and targets the triager rather
// than the issue's assignee. Redoing triage is re-triage, a different
// action on a different status.
if IsTriageTask(sourceTask) {
return nil, ErrRerunSourceIsTriage
}
agentID = sourceTask.AgentID
isLeader = sourceTask.IsLeaderTask
// Carry the source task's squad provenance so a rerun of a leader
// task still injects the squad briefing at claim time (see migration
// 127 / daemon claim handler).
squadID = sourceTask.SquadID
// Inherit trigger provenance so a per-row rerun of a comment- or
// mention-triggered task stays a comment-triggered task. Without
// this the daemon's buildCommentPrompt path is skipped (it keys on
// TriggerCommentID) and the rerun degrades into a generic issue
// run that has lost the original comment context. Only override
// when the caller didn't pass one explicitly.
if !triggerCommentID.Valid {
coalescedCommentIDs = append([]pgtype.UUID{}, sourceTask.CoalescedCommentIds...)
sourceTriggerLive := sourceTask.TriggerCommentID.Valid
if sourceTriggerLive {
// A trigger deleted while it had replies keeps its row as a
// tombstone instead of clearing trigger_comment_id; repair the
// plan exactly as for a removed trigger.
trigger, err := s.Queries.GetComment(ctx, sourceTask.TriggerCommentID)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return nil, fmt.Errorf("load source trigger comment: %w", err)
}
sourceTriggerLive = err == nil && !trigger.DeletedAt.Valid
}
if sourceTriggerLive {
triggerCommentID = sourceTask.TriggerCommentID
} else if len(coalescedCommentIDs) > 0 {
triggerCommentID, coalescedCommentIDs, err = s.promoteNewestSurvivingComment(ctx, coalescedCommentIDs)
if err != nil {
return nil, fmt.Errorf("repair source comment plan: %w", err)
}
}
}
} else {
switch {
case issue.AssigneeType.String == "agent" && issue.AssigneeID.Valid:
agentID = issue.AssigneeID
case issue.AssigneeType.String == "squad" && issue.AssigneeID.Valid:
squad, err := s.Queries.GetSquad(ctx, issue.AssigneeID)
if err != nil {
return nil, fmt.Errorf("issue is assigned to a squad but squad not found")
}
agentID = squad.LeaderID
isLeader = true
squadID = issue.AssigneeID
default:
return nil, fmt.Errorf("issue is not assigned to an agent or squad")
}
}
// Re-validate invoke permission on the RESOLVED target before mutating
// anything (MUL-4525). For a task_id rerun this gates the historical agent,
// so a since-reassigned issue can't be used to re-fire a private agent the
// operator may only view. A block fails closed: no prior task is cancelled,
// no new task is created.
if canInvoke != nil {
targetAgent, err := s.Queries.GetAgent(ctx, agentID)
if err != nil {
return nil, fmt.Errorf("load target agent: %w", err)
}
if !canInvoke(targetAgent) {
return nil, ErrRerunInvokeNotAllowed
}
}
// Clear only the tasks that have not begun executing. Those are the rows the
// fresh enqueue would collide with under
// idx_one_pending_task_per_issue_agent_v2, and replacing them costs no work
// while keeping the new run attributed to the rerunning member (MUL-4302 §5)
// rather than inheriting whoever created the pending row.
//
// A running / waiting_local_directory task is deliberately left alone: an
// agent is executing in it. Neither status appears in that unique index, so
// the enqueue below inserts a queued row BEHIND the active one, and
// ClaimAgentTask's per-(issue, agent) serialization holds it there until the
// active run reaches a terminal state. Rerun used to cancel these too, so
// asking an agent for another pass silently killed the pass it was still
// working on; interrupting an in-flight run is what CancelTask /
// `multica issue cancel-task` is for.
clearPendingSlot := func() int {
var cancelled []db.AgentTaskQueue
// Atomic with the cancel: a pending coordinator task can already hold a
// recovery receipt, and nothing downstream could repair a settlement
// that failed after this committed.
cerr := s.runInTx(ctx, func(qtx *db.Queries) error {
var err error
cancelled, err = qtx.CancelPendingTasksByIssueAndAgentInThread(ctx, db.CancelPendingTasksByIssueAndAgentInThreadParams{
ThreadCommentID: triggerCommentID,
IssueID: issueID,
AgentID: agentID,
})
if err != nil {
return err
}
return SettleTerminalTaskState(ctx, qtx, cancelled...)
})
if cerr != nil {
slog.Warn("rerun: cancel pending tasks failed",
"issue_id", util.UUIDToString(issueID),
"agent_id", util.UUIDToString(agentID),
"error", cerr,
)
}
for _, t := range cancelled {
s.captureTaskCancelled(ctx, t)
s.ReconcileAgentStatus(ctx, t.AgentID)
s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, t)
}
return len(cancelled)
}
cancelledCount := clearPendingSlot()
// A manual rerun is a NEW direct_human trigger attributed to the rerunning
// member, not the original run's human (MUL-4302 §5); actorUserID carries them.
// sourceTaskID is the rerun lineage: it rides the CreateAgentTask insert
// (rerun_of_task_id) so the queued event / daemon claim never sees a NULL
// lineage, and it stays distinct from system-retry's retry_of_task_id (§5).
task, err := s.enqueueRerunTask(ctx, issue, agentID, triggerCommentID, coalescedCommentIDs, isLeader, squadID, actorUserID, sourceTaskID, rerunOrigin, fresh)
if pendingSlotTakenErr(err) {
// The clear above and this enqueue are separate commits, so a system
// retry created by a concurrent FailTask can take the pending slot in
// between. CreateRetryTask yields the slot when it is already occupied,
// but it cannot yield to a row that does not exist yet, so a retry
// committing inside this window gets there first. Clear once more and
// retry: the deliberate human action is the one that should hold the
// slot. Bounded to a single extra attempt — a second collision would mean
// something is enqueueing in a loop, which is worth surfacing rather than
// spinning on.
slog.Info("issue rerun: pending slot taken concurrently, reclaiming",
"issue_id", util.UUIDToString(issueID),
"agent_id", util.UUIDToString(agentID),
)
cancelledCount += clearPendingSlot()
task, err = s.enqueueRerunTask(ctx, issue, agentID, triggerCommentID, coalescedCommentIDs, isLeader, squadID, actorUserID, sourceTaskID, rerunOrigin, fresh)
}
if err != nil {
return nil, err
}
slog.Info("issue rerun enqueued",
"task_id", util.UUIDToString(task.ID),
"issue_id", util.UUIDToString(issueID),
"agent_id", util.UUIDToString(agentID),
"source_task_id", util.UUIDToString(sourceTaskID),
"is_leader", isLeader,
"cancelled_pending", cancelledCount,
)
return &task, nil
}
// promoteNewestSurvivingComment repairs a manual rerun whose original trigger
// was deleted (the FK clears trigger_comment_id, or the trigger is a tombstone,
// while the UUID-array plan survives). Tombstones never count as survivors.
// Promoting before enqueue lets the normal enqueue path recompute originator
// and user-scoped connected-app capabilities from the real comment, rather
// than carrying the deleted trigger's stale security context.
func (s *TaskService) promoteNewestSurvivingComment(ctx context.Context, ids []pgtype.UUID) (pgtype.UUID, []pgtype.UUID, error) {
type survivingComment struct {
id pgtype.UUID
createdAt time.Time
}
survivors := make([]survivingComment, 0, len(ids))
seen := make(map[string]struct{}, len(ids))
for _, id := range ids {
if !id.Valid {
continue
}
key := util.UUIDToString(id)
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
comment, err := s.Queries.GetComment(ctx, id)
if errors.Is(err, pgx.ErrNoRows) {
continue
}
if err != nil {
return pgtype.UUID{}, nil, err
}
if comment.DeletedAt.Valid {
continue
}
survivors = append(survivors, survivingComment{id: comment.ID, createdAt: comment.CreatedAt.Time})
}
if len(survivors) == 0 {
return pgtype.UUID{}, nil, nil
}
newest := 0
for i := 1; i < len(survivors); i++ {
if survivors[i].createdAt.After(survivors[newest].createdAt) ||
(survivors[i].createdAt.Equal(survivors[newest].createdAt) &&
util.UUIDToString(survivors[i].id) > util.UUIDToString(survivors[newest].id)) {
newest = i
}
}
remaining := make([]pgtype.UUID, 0, len(survivors)-1)
for i, comment := range survivors {
if i != newest {
remaining = append(remaining, comment.id)
}
}
return survivors[newest].id, remaining, nil
}
// enqueueRerunTask enqueues a fresh task for the given agent on the issue.
// When the target agent is the issue's single-agent assignee we use the
// assignee-driven path (enqueueIssueTask) so the issue-assignee bookkeeping
// stays in sync; otherwise (squad member, prior assignee that has since been
// reassigned, mention agent) we use the mention path.
//
// force_fresh_session is pinned to true on every rerun row on purpose. It is
// the rollback-safe legacy signal: an OLD claim handler (mid rolling deploy)
// gates the whole resume lookup on !force_fresh_session, so it starts clean
// instead of resuming via the (agent, issue) most-recent query — which could
// pick a different execution than the one the user clicked. The NEW claim
// handler ignores this flag for reruns and instead reads the exact source task
// (rerun_of_task_id) to reuse its workdir and, when the failure did not poison
// the conversation, resume its session (MUL-4869). An explicit fresh rerun stamps
// context.fresh_rerun in the same insert so claim cannot race ahead of that choice.
func (s *TaskService) enqueueRerunTask(ctx context.Context, issue db.Issue, agentID pgtype.UUID, triggerCommentID pgtype.UUID, coalescedCommentIDs []pgtype.UUID, isLeader bool, squadID pgtype.UUID, actorUserID pgtype.UUID, rerunOfTaskID pgtype.UUID, origin RunOrigin, fresh bool) (db.AgentTaskQueue, error) {
if issue.AssigneeType.String == "agent" && issue.AssigneeID.Valid &&
util.UUIDToString(issue.AssigneeID) == util.UUIDToString(agentID) {
return s.enqueueIssueTaskWithCommentPlan(ctx, issue, triggerCommentID, coalescedCommentIDs, true, "", actorUserID, rerunOfTaskID, pgtype.Timestamptz{}, origin, fresh)
}
return s.enqueueMentionTaskWithCommentPlan(ctx, issue, agentID, triggerCommentID, coalescedCommentIDs, isLeader, squadID, true, "", actorUserID, rerunOfTaskID, origin, fresh)
}
// The bulk terminal writes below are the sweeper, archive and daemon-recovery
// paths that finalize many tasks in one statement. They exist on TaskService rather than
// being called as bare queries so the statement and all terminal-state
// settlement share a transaction.
//
// That is not a stylistic preference. HandleFailedTasks and
// CaptureCancelledTasks run AFTER their caller committed, so a settlement
// issued there could not be rolled back — and could not be repaired either,
// because ListPendingDelegatedFailureRecoveries excludes a comment whose
// covering task is terminal and already holds the receipt. Such a row would
// neither replay nor settle; it would sit in the partial index forever,
// restoring the unbounded history scan this change set removes.
// FailTasksForOfflineRuntimes fails in-flight tasks whose runtime stayed
// offline past the reconnect grace.
func (s *TaskService) FailTasksForOfflineRuntimes(ctx context.Context, arg db.FailTasksForOfflineRuntimesParams) ([]db.AgentTaskQueue, error) {
return s.terminateTasksInTx(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) {
return qtx.FailTasksForOfflineRuntimes(ctx, arg)
})
}
// FailExpiredRuntimeReconnectRetries fails deferred retries that reached their
// terminal reconnect deadline.
func (s *TaskService) FailExpiredRuntimeReconnectRetries(ctx context.Context, arg db.FailExpiredRuntimeReconnectRetriesParams) ([]db.AgentTaskQueue, error) {
return s.terminateTasksInTx(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) {
return qtx.FailExpiredRuntimeReconnectRetries(ctx, arg)
})
}
// FailStaleTasks fails claimed work whose runtime stopped reporting.
func (s *TaskService) FailStaleTasks(ctx context.Context, arg db.FailStaleTasksParams) ([]db.AgentTaskQueue, error) {
return s.terminateTasksInTx(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) {
return qtx.FailStaleTasks(ctx, arg)
})
}
// ExpireStaleQueuedTasks fails queued work whose runtime never came back.
func (s *TaskService) ExpireStaleQueuedTasks(ctx context.Context, arg db.ExpireStaleQueuedTasksParams) ([]db.AgentTaskQueue, error) {
return s.terminateTasksInTx(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) {
return qtx.ExpireStaleQueuedTasks(ctx, arg)
})
}
// RecoverOrphanedTasksForRuntime fails work a restarted daemon reports it lost.
func (s *TaskService) RecoverOrphanedTasksForRuntime(ctx context.Context, runtimeID pgtype.UUID) ([]db.AgentTaskQueue, error) {
return s.terminateTasksInTx(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) {
return qtx.RecoverOrphanedTasksForRuntime(ctx, runtimeID)
})
}
// CancelTasksForArchivedAgent cancels every active task belonging to an agent
// being archived and settles their recovery receipts in the same transaction.
//
// After commit, cancellation side effects are captured before chat tasks emit
// task:cancelled so existing consumers can clear processing indicators, release
// streams, and refresh chat state. The caller still publishes agent:archived;
// non-chat tasks keep their existing behavior.
func (s *TaskService) CancelTasksForArchivedAgent(ctx context.Context, agentID pgtype.UUID) ([]db.AgentTaskQueue, error) {
cancelled, err := s.terminateTasksInTx(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) {
return qtx.CancelAgentTasksByAgent(ctx, agentID)
})
if err != nil {
return nil, err
}
s.CaptureCancelledTasks(ctx, cancelled)
for _, task := range cancelled {
if task.ChatSessionID.Valid {
s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, task)
}
}
return cancelled, nil
}
func (s *TaskService) terminateTasksInTx(ctx context.Context, fail func(*db.Queries) ([]db.AgentTaskQueue, error)) ([]db.AgentTaskQueue, error) {
var failed []db.AgentTaskQueue
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
var err error
failed, err = fail(qtx)
if err != nil {
return err
}
return SettleTerminalTaskState(ctx, qtx, failed...)
}); err != nil {
return nil, err
}
return failed, nil
}
// HandleFailedTasks runs the post-failure side effects for a batch of
// freshly-failed tasks: optional auto-retry, task:failed event broadcast,
// agent status reconciliation, and (when an issue has no remaining active
// task and isn't being retried) resetting the issue back to todo so the
// daemon can pick it up again.
//
// The bulk failure paths — sweepers and recover-orphans — funnel through
// here so the same UI-consistency guarantees apply on each of them.
// FailTaskWithTransition runs its own failure side effects.
func (s *TaskService) HandleFailedTasks(ctx context.Context, tasks []db.AgentTaskQueue) int {
if len(tasks) == 0 {
return 0
}
affectedAgents := make(map[string]pgtype.UUID)
processedIssues := make(map[string]bool)
retriedIssues := make(map[string]bool)
retried := 0
for _, t := range tasks {
// Auto-retry first so the issue stays in_progress rather than
// flapping todo → in_progress within a tick.
retryPending := false
if child, _ := s.MaybeRetryFailedTask(ctx, t); child != nil {
retryPending = true
retried++
if t.IssueID.Valid {
retriedIssues[util.UUIDToString(t.IssueID)] = true
}
}
if !retryPending {
if _, err := s.recoverDelegatedTaskFailure(ctx, t); err != nil {
slog.Warn("handle failed tasks: delegated failure recovery failed",
"task_id", util.UUIDToString(t.ID),
"delegated_from_task_id", util.UUIDToString(t.DelegatedFromTaskID),
"error", err,
)
}
// Same terminal-failure comment FailTaskWithTransition posts, so
// swept tasks are not silent on their issue.
if t.IssueID.Valid && t.Error.String != "" {
s.createAgentComment(ctx, t.IssueID, t.AgentID, redact.Text(t.Error.String), "system", t.TriggerCommentID, t.ID)
}
}
failureReason := "agent_error"
if t.FailureReason.Valid && t.FailureReason.String != "" {
failureReason = t.FailureReason.String
}
s.captureTaskFailed(ctx, t)
workspaceID := ""
if t.IssueID.Valid {
if issue, err := s.Queries.GetIssue(ctx, t.IssueID); err == nil {
workspaceID = util.UUIDToString(issue.WorkspaceID)
// Reset stuck in_progress issues only when no other active
// task exists for the issue and no retry was just enqueued.
issueKey := util.UUIDToString(t.IssueID)
// Only "an agent is actively working" resets, and since
// MUL-7240 that is the fixed in_progress key alone. in_review
// and blocked are excluded because a human or an external
// dependency owns the issue then; a CUSTOM started status is
// excluded because custom statuses inherit lifecycle only, not
// the active-status recovery rule. Effective() no longer
// projects a nonterminal custom key onto a built-in, so this is
// a key comparison on purpose. (MUL-6243, MUL-7240)
effectiveStatus := issuestatus.Effective(ctx, s.Queries, issue.WorkspaceID, issue.Status)
if effectiveStatus == "in_progress" && !processedIssues[issueKey] && !retriedIssues[issueKey] {
processedIssues[issueKey] = true
hasActive, checkErr := s.Queries.HasActiveTaskForIssue(ctx, t.IssueID)
if checkErr != nil {
slog.Warn("handle failed tasks: active check failed",
"issue_id", issueKey,
"error", checkErr,
)
} else if !hasActive {
updatedIssue, updateErr := s.Queries.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{
SourceTaskID: t.ID,
ID: t.IssueID,
Status: "todo",
WorkspaceID: issue.WorkspaceID,
})
if updateErr != nil {
slog.Warn("handle failed tasks: reset stuck issue failed",
"issue_id", issueKey,
"error", updateErr,
)
} else {
// This direct reset bypasses the HTTP UpdateIssue
// handler that normally emits issue:updated, so emit
// it here too. Without it the board / status-filter
// caches keep showing the issue as in_progress until
// the next write touches it (#4648 / MUL-3782).
s.broadcastIssueUpdated(ctx, updatedIssue, issue.Status)
}
}
}
}
}
if workspaceID == "" {
workspaceID = s.ResolveTaskWorkspaceID(ctx, t)
}
s.publishTaskFailedEvent(workspaceID, t, t.Error.String, failureReason, retryPending)
affectedAgents[util.UUIDToString(t.AgentID)] = t.AgentID
}
for _, agentID := range affectedAgents {
s.ReconcileAgentStatus(ctx, agentID)
}
s.notifyTasksFinished(tasks)
return retried
}
const (
delegatedFailureErrorSummaryRunes = 800
delegatedFailureRecoveryMaxTaskAttempts = 3
delegatedFailureRecoveryCommentType = "progress_update"
)
// SettleTerminalTaskState applies every application-owned side effect of a
// task entering a terminal state. It must run with the same qtx as the status
// update so task completion and its dependent receipts commit atomically.
//
// Keeping this as the single terminal-settlement entry point is important:
// task supplements used to rely on an agent_task_queue trigger that performed
// a cross-table update invisibly. Explicit settlement preserves the existing
// task-row -> dependent-row lock order without making task status writes depend
// on database trigger behavior.
func SettleTerminalTaskState(ctx context.Context, q *db.Queries, tasks ...db.AgentTaskQueue) error {
if len(tasks) == 0 {
return nil
}
taskIDs := make([]pgtype.UUID, 0, len(tasks))
for _, task := range tasks {
taskIDs = append(taskIDs, task.ID)
}
if _, err := q.SettleTerminalTaskSupplements(ctx, taskIDs); err != nil {
return fmt.Errorf("settle terminal task supplements: %w", err)
}
return SettleDeliveredDelegatedFailureRecoveries(ctx, q, tasks...)
}
// SettleDeliveredDelegatedFailureRecoveries retires every delegated-failure
// recovery comment the given now-terminal tasks actually received, so those
// comments drop out of idx_comment_delegated_failure_unsettled instead of
// being re-proven settled by ListPendingDelegatedFailureRecoveries on every
// sweeper tick. Without it the outbox scan grows with total history even when
// it returns nothing.
//
// INVARIANT: every path that moves tasks to a terminal status must reach
// SettleTerminalTaskState with the same qtx as the terminal write — per-task
// writes and bulk cancellations alike, so every marker commits atomically with
// the status change or not at all. A row stranded by a
// committed-but-unsettled terminal write
// cannot be repaired later: ListPendingDelegatedFailureRecoveries excludes a
// comment whose covering task is already terminal and holds its receipt, so
// nothing replays the settlement and nothing else marks it, and the index
// reacquires the unbounded growth it exists to remove.
//
// For the same reason the post-commit helpers — BroadcastCancelledTasks,
// CaptureCancelledTasks, HandleFailedTasks — must never call this: they run
// after their caller has committed, where a failure here can neither roll back
// nor be compensated. Do not reintroduce a best-effort settlement outside the
// transaction; the old one was deleted, not kept.
//
// Call this only once the task is terminal. A dispatched task's receipt is
// still replaceable (SetTaskDeliveredCommentIDs), so settling earlier would
// freeze a legitimate reclaim window into a permanently lost recovery.
// SettleDelegatedFailureRecoveriesForTask re-checks the terminal status in SQL,
// so a mistaken early call updates nothing rather than losing a recovery.
//
// A task with no delivery receipt — nearly every task — costs a slice length
// check and no query.
func SettleDeliveredDelegatedFailureRecoveries(ctx context.Context, q *db.Queries, tasks ...db.AgentTaskQueue) error {
for _, t := range tasks {
if len(t.DeliveredCommentIds) == 0 {
continue
}
if _, err := q.SettleDelegatedFailureRecoveriesForTask(ctx, t.ID); err != nil {
return fmt.Errorf("settle delegated failure recoveries for task %s: %w", util.UUIDToString(t.ID), err)
}
}
return nil
}
type delegatedFailureRecoveryDispatchOutcome uint8
const (
delegatedFailureRecoveryCovered delegatedFailureRecoveryDispatchOutcome = iota
delegatedFailureRecoveryReplayed
delegatedFailureRecoveryExhausted
)
// DelegatedFailureRecoverySweepResult separates successful coordinator
// replays from terminally exhausted outbox entries so operators never mistake
// a bounded stop for a successful replay.
type DelegatedFailureRecoverySweepResult struct {
Scanned int
Replayed int
Exhausted int
}
type delegatedFailureRecoveryTarget struct {
failed db.AgentTaskQueue
source db.AgentTaskQueue
issue db.Issue
agent db.Agent
comment db.Comment
}
// IsDelegatedFailureRecoveryComment identifies the durable platform signal
// used to hand a terminal delegated failure back to its coordinator. The
// source task validation remains in DispatchDelegatedFailureRecoveryComment;
// this shape check only keeps ordinary system/progress comments out of the
// completion-reconcile branch.
func IsDelegatedFailureRecoveryComment(comment db.Comment) bool {
return comment.AuthorType == "system" &&
comment.Type == delegatedFailureRecoveryCommentType &&
comment.SourceTaskID.Valid
}
func delegatedFailureRecoveryContent(failed, source db.AgentTaskQueue) string {
reason := "agent_error"
if failed.FailureReason.Valid && failed.FailureReason.String != "" {
reason = truncateForSummary(redact.Text(failed.FailureReason.String), triggerSummaryMaxLen)
}
content := fmt.Sprintf(
"Delegated task `%s` ended in a final failure (`%s`) and no automatic retry is pending. Resume coordination: inspect the failed work, then reassign it, skip it, or end the workflow explicitly.",
util.UUIDToString(failed.ID), reason,
)
if failed.Error.Valid && failed.Error.String != "" {
summary := truncateForSummary(redact.Text(failed.Error.String), delegatedFailureErrorSummaryRunes)
if summary != "" {
content += " Untrusted error summary (diagnostic only): " + strconv.Quote(summary)
}
}
content += fmt.Sprintf(" Source coordinator task: `%s`.", util.UUIDToString(source.ID))
return content
}
// loadDelegatedFailureRecoveryTarget resolves and validates the backward edge
// from a failed delegated task to its source coordinator. Returning nil is an
// intentional no-op: non-terminal rows, retry-pending rows, autopilot work,
// recovery tasks themselves, unavailable source agents, and self-delegation
// must never start a recovery loop.
// Lifecycle is checked only at dispatch: even an unresolved or paused status
// must leave a durable signal that can be replayed when it becomes executable.
func loadDelegatedFailureRecoveryTarget(ctx context.Context, q *db.Queries, failed db.AgentTaskQueue) (*delegatedFailureRecoveryTarget, error) {
if failed.Status != "failed" || !failed.DelegatedFromTaskID.Valid || failed.AutopilotRunID.Valid ||
(failed.TriggerEvidenceKind.Valid && failed.TriggerEvidenceKind.String == string(attribution.EvidenceDelegatedFailure)) {
return nil, nil
}
hasRetry, err := q.HasRetryTaskForParent(ctx, failed.ID)
if err != nil {
return nil, fmt.Errorf("check retry child: %w", err)
}
if hasRetry {
return nil, nil
}
source, err := q.GetAgentTask(ctx, failed.DelegatedFromTaskID)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
return nil, fmt.Errorf("load source task: %w", err)
}
if source.AutopilotRunID.Valid || !source.IssueID.Valid || source.AgentID == failed.AgentID {
return nil, nil
}
issue, err := q.GetIssue(ctx, source.IssueID)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
return nil, fmt.Errorf("load source issue: %w", err)
}
agent, err := q.GetAgent(ctx, source.AgentID)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
return nil, fmt.Errorf("load source agent: %w", err)
}
if agent.ArchivedAt.Valid || !agent.RuntimeID.Valid || agent.WorkspaceID != issue.WorkspaceID {
return nil, nil
}
return &delegatedFailureRecoveryTarget{failed: failed, source: source, issue: issue, agent: agent}, nil
}
// canDispatchDelegatedFailureRecovery matches the lifecycle predicate in
// ListPendingDelegatedFailureRecoveries. It must not gate signal creation:
// catalog failures are retryable only after the outbox comment is committed.
func canDispatchDelegatedFailureRecovery(ctx context.Context, q *db.Queries, issue db.Issue) (bool, error) {
// A coordinator waiting in Triage is the entry's proposed owner, not its
// owner, so a worker failure must not wake it (MUL-7189 §2.3).
if issue.TriageState.Valid {
return false, nil
}
category, err := issuestatus.CategoryWithError(ctx, q, issue.WorkspaceID, issue.Status)
if err != nil {
return false, err
}
return issue.Status != issuestatus.Backlog &&
(category == issuestatus.CategoryUnstarted || category == issuestatus.CategoryStarted), nil
}
// ensureDelegatedFailureRecoveryComment creates one durable recovery signal
// per failed task. The failed row lock serializes FailTask and sweeper callers;
// the comment itself is platform-authored so it does not create subscriber or
// mention-notification side effects.
func (s *TaskService) ensureDelegatedFailureRecoveryComment(ctx context.Context, failedID pgtype.UUID) (*delegatedFailureRecoveryTarget, bool, error) {
var target *delegatedFailureRecoveryTarget
created := false
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
failed, err := qtx.GetAgentTaskForDelegatedFailureUpdate(ctx, failedID)
if err != nil {
return fmt.Errorf("lock failed task: %w", err)
}
target, err = loadDelegatedFailureRecoveryTarget(ctx, qtx, failed)
if err != nil || target == nil {
return err
}
comment, err := qtx.GetDelegatedFailureRecoveryComment(ctx, db.GetDelegatedFailureRecoveryCommentParams{
IssueID: target.issue.ID,
WorkspaceID: target.issue.WorkspaceID,
SourceTaskID: failed.ID,
})
if err == nil {
target.comment = comment
return nil
}
if !errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("find recovery comment: %w", err)
}
createdComment, err := qtx.CreateComment(ctx, db.CreateCommentParams{
ID: dbid.NewV7(),
IssueID: target.issue.ID,
WorkspaceID: target.issue.WorkspaceID,
AuthorType: "system",
AuthorID: pgtype.UUID{Valid: true},
Content: delegatedFailureRecoveryContent(target.failed, target.source),
Type: delegatedFailureRecoveryCommentType,
ParentID: target.source.TriggerCommentID,
SourceTaskID: failed.ID,
})
if err != nil {
return fmt.Errorf("create recovery comment: %w", err)
}
target.comment = createdComment.Comment()
created = true
return nil
}); err != nil {
return nil, false, err
}
if target == nil {
return nil, false, nil
}
if created && s.Bus != nil {
s.Bus.Publish(events.Event{
Type: protocol.EventCommentCreated,
WorkspaceID: util.UUIDToString(target.issue.WorkspaceID),
ActorType: "system",
ActorID: "",
Payload: map[string]any{
"comment": commentEventFields(target.comment),
"issue_title": target.issue.Title,
"issue_status": target.issue.Status,
},
})
}
return target, created, nil
}
func delegatedFailureRecoveryExhaustionContent(target *delegatedFailureRecoveryTarget) string {
return fmt.Sprintf(
"Automatic recovery for delegated task `%s` stopped after %d coordinator tasks ended before receiving the recovery signal. No more recovery tasks will be created automatically; resume or dismiss the work manually. Source coordinator task: `%s`.",
util.UUIDToString(target.failed.ID),
delegatedFailureRecoveryMaxTaskAttempts,
util.UUIDToString(target.source.ID),
)
}
func delegatedFailureRecoveryAttribution(target *delegatedFailureRecoveryTarget) (pgtype.UUID, pgtype.UUID) {
originator := target.failed.OriginatorUserID
accountable := target.failed.AccountableUserID
if originator.Valid {
accountable = originator
}
if !originator.Valid && !accountable.Valid {
originator = target.source.OriginatorUserID
accountable = target.source.AccountableUserID
if originator.Valid {
accountable = originator
}
}
return originator, accountable
}
// exhaustDelegatedFailureRecovery atomically settles the recovery outbox after
// its bounded automatic attempts, creates one visible system explanation, and
// notifies the responsible human. The bool reports whether this caller created
// that terminal outcome. Updating the newest attempt first serializes
// concurrent sweepers; the second caller then observes the explanation written
// by the first and does not report another exhaustion.
func (s *TaskService) exhaustDelegatedFailureRecovery(ctx context.Context, target *delegatedFailureRecoveryTarget) (bool, error) {
var exhaustedComment db.Comment
var exhaustedInbox db.InboxItem
created := false
inboxCreated := false
if err := s.runInTx(ctx, func(qtx *db.Queries) error {
if _, err := qtx.AcknowledgeExhaustedDelegatedFailureRecovery(ctx, db.AcknowledgeExhaustedDelegatedFailureRecoveryParams{
CommentID: target.comment.ID,
FailedTaskID: target.failed.ID,
MaxAttempts: delegatedFailureRecoveryMaxTaskAttempts,
}); err != nil {
return fmt.Errorf("acknowledge exhausted delegated failure recovery: %w", err)
}
// The receipt above lands on the newest attempt row, which may still be
// running, so the task-scoped settle cannot see it. Exhaustion is
// terminal on its own terms — the attempt budget is spent and the
// visible explanation below tells the user why nothing else will run —
// so retire the comment directly, in this same transaction.
if _, err := qtx.SettleDelegatedFailureRecoveryComment(ctx, target.comment.ID); err != nil {
return fmt.Errorf("settle exhausted delegated failure recovery: %w", err)
}
comment, err := qtx.GetDelegatedFailureRecoveryExhaustionComment(ctx, db.GetDelegatedFailureRecoveryExhaustionCommentParams{
IssueID: target.issue.ID,
WorkspaceID: target.issue.WorkspaceID,
SourceTaskID: target.failed.ID,
})
if err == nil {
exhaustedComment = comment
return nil
}
if !errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("find delegated failure exhaustion comment: %w", err)
}
createdComment, err := qtx.CreateComment(ctx, db.CreateCommentParams{
ID: dbid.NewV7(),
IssueID: target.issue.ID,
WorkspaceID: target.issue.WorkspaceID,
AuthorType: "system",
AuthorID: pgtype.UUID{Valid: true},
Content: delegatedFailureRecoveryExhaustionContent(target),
Type: "system",
ParentID: target.source.TriggerCommentID,
SourceTaskID: target.failed.ID,
})
if err != nil {
return fmt.Errorf("create delegated failure exhaustion comment: %w", err)
}
exhaustedComment = createdComment.Comment()
created = true
// Exhaustion deliberately does not @mention the coordinator agent: doing
// so would enqueue a fourth recovery run and defeat the attempt bound.
// Instead, create one durable action-required inbox item for the human
// who originated (or is accountable for) the delegated work.
_, recipient := delegatedFailureRecoveryAttribution(target)
if recipient.Valid {
_, err = qtx.GetMemberByUserAndWorkspace(ctx, db.GetMemberByUserAndWorkspaceParams{
UserID: recipient,
WorkspaceID: target.issue.WorkspaceID,
})
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return fmt.Errorf("validate delegated failure exhaustion recipient: %w", err)
}
details, err := json.Marshal(map[string]any{
"failed_task_id": util.UUIDToString(target.failed.ID),
"source_task_id": util.UUIDToString(target.source.ID),
"coordinator_agent_id": util.UUIDToString(target.agent.ID),
"max_attempts": delegatedFailureRecoveryMaxTaskAttempts,
})
if err != nil {
return fmt.Errorf("encode delegated failure exhaustion details: %w", err)
}
exhaustedInbox, inboxCreated, err = inbox.CreateIfEligible(ctx, qtx, db.CreateInboxItemParams{
ID: dbid.NewV7(),
WorkspaceID: target.issue.WorkspaceID,
RecipientType: "member",
RecipientID: recipient,
Type: "task_failed",
Severity: "action_required",
IssueID: target.issue.ID,
Title: target.issue.Title,
Body: pgtype.Text{String: exhaustedComment.Content, Valid: true},
ActorType: pgtype.Text{String: "system", Valid: true},
ActorID: pgtype.UUID{},
Details: details,
})
if err != nil {
return fmt.Errorf("create delegated failure exhaustion inbox item: %w", err)
}
}
return nil
}); err != nil {
return false, err
}
if created && s.Bus != nil {
s.Bus.Publish(events.Event{
Type: protocol.EventCommentCreated,
WorkspaceID: util.UUIDToString(target.issue.WorkspaceID),
ActorType: "system",
ActorID: "",
Payload: map[string]any{
"comment": commentEventFields(exhaustedComment),
"issue_title": target.issue.Title,
"issue_status": target.issue.Status,
},
})
}
if inboxCreated && s.Bus != nil {
s.Bus.Publish(events.Event{
Type: protocol.EventInboxNew,
WorkspaceID: util.UUIDToString(target.issue.WorkspaceID),
ActorType: "system",
ActorID: "",
Payload: map[string]any{"item": map[string]any{
"id": util.UUIDToString(exhaustedInbox.ID),
"workspace_id": util.UUIDToString(exhaustedInbox.WorkspaceID),
"recipient_type": exhaustedInbox.RecipientType,
"recipient_id": util.UUIDToString(exhaustedInbox.RecipientID),
"type": exhaustedInbox.Type,
"severity": exhaustedInbox.Severity,
"issue_id": util.UUIDToPtr(exhaustedInbox.IssueID),
"title": exhaustedInbox.Title,
"body": util.TextToPtr(exhaustedInbox.Body),
"read": exhaustedInbox.Read,
"archived": exhaustedInbox.Archived,
"created_at": util.TimestampToString(exhaustedInbox.CreatedAt),
"actor_type": util.TextToPtr(exhaustedInbox.ActorType),
"actor_id": util.UUIDToPtr(exhaustedInbox.ActorID),
"details": json.RawMessage(exhaustedInbox.Details),
"issue_status": target.issue.Status,
}},
})
}
return created, nil
}
// dispatchDelegatedFailureRecovery routes a recovery comment to the source
// coordinator without relying on generic mention parsing. A pre-claim task
// absorbs it; otherwise a dedicated queued successor is created. The only
// state that blocks both writes is a dispatched task (its claim is already
// built and the pending-task uniqueness slot is still held), so that narrow
// race records the comment as planned-but-undelivered and lets completion
// reconciliation schedule the follow-up. The three-pass loop closes state
// changes around those writes.
func (s *TaskService) dispatchDelegatedFailureRecovery(ctx context.Context, target *delegatedFailureRecoveryTarget, excludeTaskID pgtype.UUID) (delegatedFailureRecoveryDispatchOutcome, error) {
// Signal creation has committed before reaching this shared dispatch path.
// Refresh the issue because its status may have changed since creation or
// sweep selection; a paused/unreadable lifecycle never settles the signal.
//
// This is also the queue door for Triage on this path (MUL-7189 §2.3), and
// it needs no separate check: Triage is its own category, so it is neither
// unstarted nor started and canDispatchDelegatedFailureRecovery already
// refuses it. "Covered" is the honest outcome — no further dispatch is owed,
// and the run that would answer the comment arrives with accept.
issue, err := s.Queries.GetIssue(ctx, target.issue.ID)
if errors.Is(err, pgx.ErrNoRows) {
return delegatedFailureRecoveryCovered, nil
}
if err != nil {
return delegatedFailureRecoveryCovered, fmt.Errorf("reload recovery source issue: %w", err)
}
allowed, err := canDispatchDelegatedFailureRecovery(ctx, s.Queries, issue)
if err != nil || !allowed {
return delegatedFailureRecoveryCovered, err
}
target.issue = issue
const maxAttempts = 3
for attempt := 0; attempt < maxAttempts; attempt++ {
covered, err := s.Queries.HasTaskCoveringDelegatedFailureComment(ctx, db.HasTaskCoveringDelegatedFailureCommentParams{
IssueID: target.issue.ID,
AgentID: target.agent.ID,
CommentID: target.comment.ID,
ExcludeTaskID: excludeTaskID,
})
if err != nil {
return delegatedFailureRecoveryCovered, fmt.Errorf("check recovery coverage: %w", err)
}
if covered {
return delegatedFailureRecoveryCovered, nil
}
recoveryTasks, err := s.Queries.CountDelegatedFailureRecoveryTasks(ctx, target.failed.ID)
if err != nil {
return delegatedFailureRecoveryCovered, fmt.Errorf("count delegated failure recovery tasks: %w", err)
}
if recoveryTasks >= delegatedFailureRecoveryMaxTaskAttempts {
exhausted, err := s.exhaustDelegatedFailureRecovery(ctx, target)
if err != nil {
return delegatedFailureRecoveryCovered, err
}
if exhausted {
return delegatedFailureRecoveryExhausted, nil
}
return delegatedFailureRecoveryCovered, nil
}
if merged, err := s.Queries.MergeDelegatedFailureCommentIntoPendingTask(ctx, db.MergeDelegatedFailureCommentIntoPendingTaskParams{
CommentID: target.comment.ID,
TriggerSummary: s.buildCommentTriggerSummary(ctx, target.issue.WorkspaceID, target.comment.ID),
IssueID: target.issue.ID,
AgentID: target.agent.ID,
}); err == nil {
slog.Info("delegated failure recovery merged into pending coordinator task",
"failed_task_id", util.UUIDToString(target.failed.ID),
"coordinator_task_id", util.UUIDToString(merged.ID),
)
s.BroadcastTaskCoalesced(ctx, merged)
return delegatedFailureRecoveryReplayed, nil
} else if !errors.Is(err, pgx.ErrNoRows) {
return delegatedFailureRecoveryCovered, fmt.Errorf("merge recovery into pending task: %w", err)
}
originator, accountable := delegatedFailureRecoveryAttribution(target)
source := attribution.SourceDelegation
if !originator.Valid && !accountable.Valid {
source = attribution.SourceUnattributed
}
ruleVersionID := target.failed.RuleVersionID
if !ruleVersionID.Valid {
ruleVersionID = target.source.RuleVersionID
}
overlay := s.buildRuntimeMCPOverlay(ctx, originator, target.agent)
task, err := s.Queries.CreateAgentTask(ctx, db.CreateAgentTaskParams{
ID: dbid.NewV7(),
AgentID: target.agent.ID,
RuntimeID: target.agent.RuntimeID,
IssueID: target.issue.ID,
Priority: priorityToInt(target.issue.Priority),
TriggerCommentID: target.comment.ID,
TriggerSummary: s.buildCommentTriggerSummary(ctx, target.issue.WorkspaceID, target.comment.ID),
IsLeaderTask: pgtype.Bool{Bool: target.source.IsLeaderTask, Valid: target.source.IsLeaderTask},
SquadID: target.source.SquadID,
OriginatorUserID: originator,
AccountableUserID: accountable,
RuntimeMcpOverlay: overlay.Overlay,
RuntimeConnectedApps: overlay.ConnectedApps,
OriginatorSource: pgtype.Text{String: string(source), Valid: true},
DelegatedFromTaskID: target.failed.ID,
RuleVersionID: ruleVersionID,
TriggerEvidenceKind: pgtype.Text{String: string(attribution.EvidenceDelegatedFailure), Valid: true},
TriggerEvidenceRefID: target.failed.ID,
HeadSha: headShaText(s.ResolveIssueReviewSHA(ctx, target.issue.ID)),
})
if err == nil {
slog.Info("delegated failure recovery task enqueued",
"failed_task_id", util.UUIDToString(target.failed.ID),
"source_task_id", util.UUIDToString(target.source.ID),
"recovery_task_id", util.UUIDToString(task.ID),
"coordinator_agent_id", util.UUIDToString(target.agent.ID),
)
s.broadcastTaskEvent(ctx, protocol.EventTaskQueued, task)
s.NotifyTaskEnqueued(ctx, task)
return delegatedFailureRecoveryReplayed, nil
}
if !isDuplicatePendingTaskErr(err) {
return delegatedFailureRecoveryCovered, fmt.Errorf("create recovery task: %w", err)
}
// A dispatched task still owns the unique queued/dispatched slot, but
// its claim payload is immutable. Register the comment as undelivered so
// its completion reconciliation creates the successor. Running tasks do
// not own that slot, so they took the durable queued-successor path above.
if active, err := s.Queries.RegisterPlannedCommentForActiveTask(ctx, db.RegisterPlannedCommentForActiveTaskParams{
CommentID: target.comment.ID,
IssueID: target.issue.ID,
AgentID: target.agent.ID,
}); err == nil {
slog.Info("delegated failure recovery registered behind dispatched coordinator task",
"failed_task_id", util.UUIDToString(target.failed.ID),
"coordinator_task_id", util.UUIDToString(active.ID),
)
return delegatedFailureRecoveryReplayed, nil
} else if !errors.Is(err, pgx.ErrNoRows) {
return delegatedFailureRecoveryCovered, fmt.Errorf("register recovery on dispatched task: %w", err)
}
}
return delegatedFailureRecoveryCovered, fmt.Errorf("delegate failure recovery could not acquire coordinator task slot")
}
// recoverDelegatedTaskFailure is the shared post-terminal hook for FailTask and
// HandleFailedTasks. handled reports whether the failure was an eligible
// delegated terminal; the production terminal paths deliberately retain their
// legacy raw-error notice alongside the richer coordinator recovery signal.
func (s *TaskService) recoverDelegatedTaskFailure(ctx context.Context, failed db.AgentTaskQueue) (handled bool, err error) {
target, _, err := s.ensureDelegatedFailureRecoveryComment(ctx, failed.ID)
if err != nil || target == nil {
return false, err
}
_, err = s.dispatchDelegatedFailureRecovery(ctx, target, pgtype.UUID{})
return true, err
}
// RecoverPendingDelegatedFailures replays the durable recovery outbox. The
// platform recovery comment is the obligation; it is complete only while a task
// that carries it can still execute, or after a task records it in
// delivered_comment_ids. This lets a later sweeper repair a process crash or
// transient database error between comment creation and coordinator dispatch
// without producing duplicate runnable tasks.
func (s *TaskService) RecoverPendingDelegatedFailures(ctx context.Context, maxPerTick int32) (DelegatedFailureRecoverySweepResult, error) {
result := DelegatedFailureRecoverySweepResult{}
if maxPerTick <= 0 {
return result, nil
}
pending, err := s.Queries.ListPendingDelegatedFailureRecoveries(ctx, maxPerTick)
if err != nil {
return result, fmt.Errorf("list pending delegated failure recoveries: %w", err)
}
result.Scanned = len(pending)
errs := make([]error, 0)
for _, comment := range pending {
outcome, recoveryErr := s.dispatchDelegatedFailureRecoveryComment(ctx, comment, pgtype.UUID{})
if recoveryErr != nil {
errs = append(errs, fmt.Errorf("dispatch recovery comment %s: %w", util.UUIDToString(comment.ID), recoveryErr))
continue
}
switch outcome {
case delegatedFailureRecoveryReplayed:
result.Replayed++
case delegatedFailureRecoveryExhausted:
result.Exhausted++
}
}
return result, errors.Join(errs...)
}
// DispatchDelegatedFailureRecoveryComment is used by completion reconciliation
// when a recovery signal arrived after a coordinator task was claimed. The
// completed task is excluded from the coverage check because the comment was
// planned but not delivered to it; routing then merges/enqueues exactly one
// follow-up.
func (s *TaskService) DispatchDelegatedFailureRecoveryComment(ctx context.Context, comment db.Comment, completedTaskID pgtype.UUID) error {
_, err := s.dispatchDelegatedFailureRecoveryComment(ctx, comment, completedTaskID)
return err
}
func (s *TaskService) dispatchDelegatedFailureRecoveryComment(ctx context.Context, comment db.Comment, completedTaskID pgtype.UUID) (delegatedFailureRecoveryDispatchOutcome, error) {
if !IsDelegatedFailureRecoveryComment(comment) {
return delegatedFailureRecoveryCovered, nil
}
failed, err := s.Queries.GetAgentTask(ctx, comment.SourceTaskID)
if err != nil {
return delegatedFailureRecoveryCovered, fmt.Errorf("load failed recovery source: %w", err)
}
target, err := loadDelegatedFailureRecoveryTarget(ctx, s.Queries, failed)
if err != nil || target == nil {
return delegatedFailureRecoveryCovered, err
}
if target.issue.ID != comment.IssueID || target.issue.WorkspaceID != comment.WorkspaceID {
return delegatedFailureRecoveryCovered, fmt.Errorf("delegated failure recovery comment scope mismatch")
}
target.comment = comment
return s.dispatchDelegatedFailureRecovery(ctx, target, completedTaskID)
}
// runInTx executes fn inside a single DB transaction. If TxStarter is nil
// (e.g. some tests construct TaskService directly), fn runs against the
// regular Queries handle without transactional guarantees.
func (s *TaskService) runInTx(ctx context.Context, fn func(*db.Queries) error) error {
if s.TxStarter == nil {
return fn(s.Queries)
}
tx, err := s.TxStarter.Begin(ctx)
if err != nil {
return fmt.Errorf("begin tx: %w", err)
}
defer tx.Rollback(ctx)
if err := fn(s.Queries.WithTx(tx)); err != nil {
return err
}
return tx.Commit(ctx)
}
// ReportProgress broadcasts a progress update via the event bus.
func (s *TaskService) ReportProgress(ctx context.Context, taskID string, workspaceID string, summary string, step, total int) {
s.Bus.Publish(events.Event{
Type: protocol.EventTaskProgress,
WorkspaceID: workspaceID,
ActorType: "system",
ActorID: "",
TaskID: taskID,
Payload: protocol.TaskProgressPayload{
TaskID: taskID,
Summary: summary,
Step: step,
Total: total,
},
})
}
// ReconcileAgentStatus refreshes agent status from the current working task
// set. The query returns no row when the status is already correct, which
// avoids rewriting updated_at and broadcasting a zero-information event.
func (s *TaskService) ReconcileAgentStatus(ctx context.Context, agentID pgtype.UUID) {
agent, err := s.Queries.RefreshAgentStatusFromTasks(ctx, agentID)
if err != nil {
return
}
slog.Debug("agent status reconciled", "agent_id", util.UUIDToString(agentID), "status", agent.Status)
s.publishAgentStatus(agent)
}
func (s *TaskService) updateAgentStatus(ctx context.Context, agentID pgtype.UUID, status string) {
agent, err := s.Queries.UpdateAgentStatus(ctx, db.UpdateAgentStatusParams{
ID: agentID,
Status: status,
})
if err != nil {
return
}
s.publishAgentStatus(agent)
}
func (s *TaskService) publishAgentStatus(agent db.Agent) {
s.Bus.Publish(events.Event{
Type: protocol.EventAgentStatus,
WorkspaceID: util.UUIDToString(agent.WorkspaceID),
ActorType: "system",
ActorID: "",
Payload: map[string]any{"agent": agentToMap(agent)},
})
}
// LoadAgentSkills loads an agent's skills with their files for task execution.
//
// A read failure is REPORTED, never swallowed into a shorter skill set. Both
// reads are all-or-nothing for the agent's entire skill set — the file load
// covers every skill in one query — so a swallowed error does not degrade the
// payload, it silently replaces it: every skill loses its supporting files, or
// the agent loses every skill. Nothing downstream can tell that apart from an
// agent that genuinely has none, because the bundle hash is computed over
// whatever did load, so the daemon's own validation passes and the agent
// starts on rules it is missing. Callers must settle the failure (preserve the
// claim for redelivery, or 5xx the resolve) instead of dispatching that.
func (s *TaskService) LoadAgentSkills(ctx context.Context, agentID pgtype.UUID) ([]AgentSkillData, error) {
skills, err := s.Queries.ListAgentSkills(ctx, agentID)
if err != nil {
return nil, fmt.Errorf("list agent skills: %w", err)
}
if len(skills) == 0 {
return nil, nil
}
return s.skillsWithFiles(ctx, skills)
}
// skillsWithFiles loads the files of every given skill in ONE round trip
// instead of one query per skill, and assembles the result in the order the
// skills were given. Shared by the claim-time full load and the resolve-time
// scoped load so the two cannot drift on skill order, per-skill file order, or
// the nil (not empty) file list of a skill that has none.
func (s *TaskService) skillsWithFiles(ctx context.Context, skills []db.Skill) ([]AgentSkillData, error) {
skillIDs := make([]pgtype.UUID, len(skills))
for i, sk := range skills {
skillIDs[i] = sk.ID
}
// Group by skill_id in a single linear pass — the query orders by
// skill_id, path.
files, err := s.Queries.ListSkillFilesBySkillIDs(ctx, skillIDs)
if err != nil {
return nil, fmt.Errorf("list skill files for %d skills: %w", len(skills), err)
}
filesBySkill := make(map[string][]AgentSkillFileData, len(skills))
for _, f := range files {
id := util.UUIDToString(f.SkillID)
filesBySkill[id] = append(filesBySkill[id], AgentSkillFileData{Path: f.Path, Content: f.Content})
}
result := make([]AgentSkillData, 0, len(skills))
for _, sk := range skills {
result = append(result, AgentSkillData{
ID: util.UUIDToString(sk.ID),
Name: sk.Name,
Description: sk.Description,
Content: sk.Content,
Files: filesBySkill[util.UUIDToString(sk.ID)],
})
}
return result, nil
}
// LoadAgentSkillBundles returns every skill visible to an agent, including
// built-ins, with stable bundle hashes and lightweight refs for slim claims.
// It fails closed on a workspace-skill read error for the reason in
// LoadAgentSkills: a bundle set built from a partial read is indistinguishable
// from a correct one.
func (s *TaskService) LoadAgentSkillBundles(ctx context.Context, agentID pgtype.UUID, agentSystemKey string, legacyRedirects bool) ([]AgentSkillData, []AgentSkillRefData, error) {
skills, err := s.LoadAgentSkills(ctx, agentID)
if err != nil {
return nil, nil, err
}
skills = append(skills, s.BuiltinSkills(agentSystemKey, legacyRedirects)...)
bundles, refs := BuildAgentSkillBundles(skills)
return bundles, refs, nil
}
// BuiltinSkillID is the ref id a builtin skill is addressed by. Builtins have
// no database row, so their identity is their name — this is the one place
// that turns a name into that identity, for both the bundle builder that hands
// the id out and the resolver that looks one back up.
func BuiltinSkillID(name string) string { return "builtin:" + name }
// AgentSkillBundleKey is the (source, id) identity a daemon skill ref resolves
// by. Source is part of the key because a builtin and a workspace skill are
// different bundles even when they share a name.
func AgentSkillBundleKey(source, id string) string { return source + "\x00" + id }
// AgentSkillBundleRef is one skill a daemon is asking to resolve.
type AgentSkillBundleRef struct {
ID string
Source string
}
// LoadRequestedAgentSkillBundles returns bundles for EXACTLY the refs given,
// keyed by AgentSkillBundleKey. A ref the agent cannot see is simply absent
// from the map — the junction predicate in ListAgentSkillsByIDs is the
// authorization, so "no row" and "not allowed" are the same answer and the
// caller reports both as not-found.
//
// This exists because the daemon resolves one skill per request (GH #4505, so
// each download gets its own size-scaled deadline and caches independently).
// Serving those out of the agent's full bundle set made the server redo the
// whole agent on every request: N requests, each reading and hashing all N
// skills to return one. Loading only what was asked for makes that linear,
// which is why the resolve path must not reuse LoadAgentSkillBundles.
func (s *TaskService) LoadRequestedAgentSkillBundles(ctx context.Context, agentID pgtype.UUID, refs []AgentSkillBundleRef) (map[string]AgentSkillData, error) {
requestedIDs := make([]pgtype.UUID, 0, len(refs))
seenWorkspace := make(map[string]struct{}, len(refs))
wantBuiltin := make(map[string]struct{}, len(refs))
for _, ref := range refs {
switch ref.Source {
case skillbundle.SourceBuiltin:
wantBuiltin[ref.ID] = struct{}{}
case skillbundle.SourceWorkspace:
if _, ok := seenWorkspace[ref.ID]; ok {
continue
}
id, err := util.ParseUUID(ref.ID)
if err != nil {
// An unparseable id matches no row, which is the same outcome
// as an id the agent does not have. Skipping it keeps one
// malformed ref from failing the refs alongside it.
continue
}
seenWorkspace[ref.ID] = struct{}{}
requestedIDs = append(requestedIDs, id)
}
// Any other source has no server-side producer, so it resolves to
// nothing and the caller reports not-found.
}
var requested []AgentSkillData
if len(requestedIDs) > 0 {
skills, err := s.Queries.ListAgentSkillsByIDs(ctx, db.ListAgentSkillsByIDsParams{
AgentID: agentID,
SkillIds: requestedIDs,
})
if err != nil {
return nil, fmt.Errorf("list agent skills by ids: %w", err)
}
if len(skills) > 0 {
// Same fail-closed rule as LoadAgentSkills: a failed file read
// would produce a bundle that hashes and validates like a complete
// one, so it must never be served.
loaded, err := s.skillsWithFiles(ctx, skills)
if err != nil {
return nil, err
}
requested = append(requested, loaded...)
}
}
if len(wantBuiltin) > 0 {
// Every built-in, not the agent-scoped subset — see AllBuiltinSkills.
for _, builtin := range s.AllBuiltinSkills() {
if _, ok := wantBuiltin[BuiltinSkillID(builtin.Name)]; ok {
requested = append(requested, builtin)
}
}
}
bundles, _ := BuildAgentSkillBundles(requested)
resolved := make(map[string]AgentSkillData, len(bundles))
for _, bundle := range bundles {
resolved[AgentSkillBundleKey(bundle.Source, bundle.ID)] = bundle
}
return resolved, nil
}
func BuildAgentSkillBundles(skills []AgentSkillData) ([]AgentSkillData, []AgentSkillRefData) {
bundles := make([]AgentSkillData, 0, len(skills))
refs := make([]AgentSkillRefData, 0, len(skills))
for _, skill := range skills {
source := skill.Source
id := skill.ID
if source == "" {
if id == "" {
source = skillbundle.SourceBuiltin
} else {
source = skillbundle.SourceWorkspace
}
}
if id == "" && source == skillbundle.SourceBuiltin {
id = BuiltinSkillID(skill.Name)
}
skill.Source = source
skill.ID = id
files := make([]skillbundle.File, 0, len(skill.Files))
for _, file := range skill.Files {
files = append(files, skillbundle.File{Path: file.Path, Content: file.Content})
}
manifest := skillbundle.BuildManifest(skillbundle.Skill{
ID: skill.ID,
Source: skill.Source,
Name: skill.Name,
Description: skill.Description,
Content: skill.Content,
Files: files,
})
skill.Hash = manifest.Hash
skill.SizeBytes = manifest.SizeBytes
fileRefsByPath := make(map[string]skillbundle.FileRef, len(manifest.Files))
for _, file := range manifest.Files {
fileRefsByPath[file.Path] = file
}
for i := range skill.Files {
if ref, ok := fileRefsByPath[skill.Files[i].Path]; ok {
skill.Files[i].SHA256 = ref.SHA256
skill.Files[i].SizeBytes = ref.SizeBytes
}
}
bundles = append(bundles, skill)
refFiles := make([]AgentSkillFileRefData, 0, len(manifest.Files))
for _, file := range manifest.Files {
refFiles = append(refFiles, AgentSkillFileRefData{
Path: file.Path,
SHA256: file.SHA256,
SizeBytes: file.SizeBytes,
})
}
refs = append(refs, AgentSkillRefData{
ID: skill.ID,
Source: skill.Source,
Name: skill.Name,
Description: skill.Description,
Hash: manifest.Hash,
SizeBytes: manifest.SizeBytes,
FileCount: manifest.FileCount,
Files: refFiles,
})
}
return bundles, refs
}
// AgentSkillData represents a skill for task execution responses.
type AgentSkillData struct {
ID string `json:"id"`
Source string `json:"source,omitempty"`
Name string `json:"name"`
Description string `json:"description,omitempty"`
Hash string `json:"hash,omitempty"`
SizeBytes int64 `json:"size_bytes,omitempty"`
Content string `json:"content"`
Files []AgentSkillFileData `json:"files,omitempty"`
}
// AgentSkillFileData represents a supporting file within a skill.
type AgentSkillFileData struct {
Path string `json:"path"`
Content string `json:"content"`
SHA256 string `json:"sha256,omitempty"`
SizeBytes int64 `json:"size_bytes,omitempty"`
}
type AgentSkillRefData struct {
ID string `json:"id"`
Source string `json:"source"`
Name string `json:"name"`
Description string `json:"description,omitempty"`
Hash string `json:"hash"`
SizeBytes int64 `json:"size_bytes"`
FileCount int `json:"file_count"`
Files []AgentSkillFileRefData `json:"files,omitempty"`
}
type AgentSkillFileRefData struct {
Path string `json:"path"`
SHA256 string `json:"sha256"`
SizeBytes int64 `json:"size_bytes"`
}
// computeChatElapsedMs returns the wall-clock duration from task creation
// (user hit send) to terminal state (completed/failed). Stored on the
// assistant chat_message so the UI can render "Replied in 38s" /
// "Failed after 12s". Uses created_at — not started_at — because users
// experience total wait time, including queue + dispatch, not just the
// daemon's actual run time.
func computeChatElapsedMs(task db.AgentTaskQueue) pgtype.Int8 {
if !task.CompletedAt.Valid || !task.CreatedAt.Valid {
return pgtype.Int8{}
}
ms := task.CompletedAt.Time.Sub(task.CreatedAt.Time).Milliseconds()
if ms < 0 {
ms = 0
}
return pgtype.Int8{Int64: ms, Valid: true}
}
func priorityToInt(p string) int32 {
switch p {
case "urgent":
return 4
case "high":
return 3
case "medium":
return 2
case "low":
return 1
default:
return 0
}
}
// NotifyTaskEnqueued is the cross-package shim for callers outside
// TaskService (e.g. AutopilotService.dispatchRunOnly) that insert a
// row into agent_task_queue directly. Invalidates the empty-claim
// cache and kicks the daemon WS so the new task is claimed without
// waiting for the next poll.
func (s *TaskService) NotifyTaskEnqueued(ctx context.Context, task db.AgentTaskQueue) {
s.captureTaskQueued(ctx, task)
s.notifyTaskAvailable(task)
}
// NotifyTaskFinished invalidates a runtime's empty-claim verdict and emits a
// best-effort daemon wakeup after a task reaches a terminal state. The task ID
// is deliberately omitted from the wakeup payload: the completed task itself
// is not available; the hint only means that a queued successor may have
// become claimable because an agent-capacity or serialization barrier cleared.
func (s *TaskService) NotifyTaskFinished(task db.AgentTaskQueue) {
s.forgetTaskReclaim(task)
s.notifyRuntimeMayHaveWork(task.RuntimeID, "")
}
// notifyTasksFinished is the batch form used by bulk terminal transitions.
// Coalesce by runtime so cancelling many tasks on one machine produces one
// cache bump and one websocket hint rather than a burst of identical work.
func (s *TaskService) notifyTasksFinished(tasks []db.AgentTaskQueue) {
seen := make(map[string]struct{}, len(tasks))
for _, task := range tasks {
if !task.RuntimeID.Valid {
continue
}
s.forgetTaskReclaim(task)
runtimeKey := util.UUIDToString(task.RuntimeID)
if _, ok := seen[runtimeKey]; ok {
continue
}
seen[runtimeKey] = struct{}{}
s.notifyRuntimeMayHaveWork(task.RuntimeID, "")
}
}
// notifyTaskAvailable runs after a task has been inserted: bumps the
// runtime's invalidation version so any in-flight claim that is about
// to write an "empty" verdict will have it rejected on read, then
// kicks the daemon WS so the daemon claims without waiting for its
// next poll. Order matters — Bump must happen before the wakeup,
// otherwise the wakeup-driven claim could read the still-current
// empty verdict and return null.
func (s *TaskService) notifyTaskAvailable(task db.AgentTaskQueue) {
s.notifyRuntimeMayHaveWork(task.RuntimeID, util.UUIDToString(task.ID))
}
// notifyRuntimeMayHaveWork is the shared bump-before-wakeup primitive for both
// fresh enqueues and terminal transitions that can unblock queued work.
func (s *TaskService) notifyRuntimeMayHaveWork(runtimeID pgtype.UUID, taskID string) {
if !runtimeID.Valid {
return
}
runtimeKey := util.UUIDToString(runtimeID)
// Use a background context: the cache bump / wakeup must outlive
// the request that created the task, otherwise an early client
// disconnect could leave the empty verdict in place and stall the
// just-queued task until the TTL expires. The cache itself bounds
// every Redis call with a short timeout so a wedged Redis cannot
// block enqueue.
s.EmptyClaim.Bump(context.Background(), runtimeKey)
if s.Wakeup == nil {
return
}
s.Wakeup.NotifyTaskAvailable(runtimeKey, taskID)
}
func (s *TaskService) broadcastTaskDispatch(ctx context.Context, task db.AgentTaskQueue) {
var payload map[string]any
if task.Context != nil {
json.Unmarshal(task.Context, &payload)
}
if payload == nil {
payload = map[string]any{}
}
payload["task_id"] = util.UUIDToString(task.ID)
payload["runtime_id"] = util.UUIDToString(task.RuntimeID)
payload["issue_id"] = util.UUIDToString(task.IssueID)
payload["agent_id"] = util.UUIDToString(task.AgentID)
// chat_session_id is the routing key the chat window uses to writethrough
// `chatKeys.pendingTask` to status="running" the moment the daemon claims
// the task. Without it the pill stays stuck at "Queued" until completion.
if task.ChatSessionID.Valid {
payload["chat_session_id"] = util.UUIDToString(task.ChatSessionID)
}
workspaceID := s.ResolveTaskWorkspaceID(ctx, task)
if workspaceID == "" {
return
}
s.Bus.Publish(events.Event{
Type: protocol.EventTaskDispatch,
WorkspaceID: workspaceID,
ActorType: "system",
ActorID: "",
Payload: payload,
})
}
// taskEvent builds the shared task-lifecycle event contract. Scope hints are
// duplicated on the envelope intentionally: current listeners remain
// compatible with the payload map, while the realtime layer can route without
// decoding it once per-resource fanout is enabled.
func taskEvent(eventType, workspaceID string, task db.AgentTaskQueue, extra ...map[string]any) events.Event {
payload := map[string]any{
"task_id": util.UUIDToString(task.ID),
"agent_id": util.UUIDToString(task.AgentID),
"issue_id": util.UUIDToString(task.IssueID),
"status": task.Status,
}
e := events.Event{
Type: eventType,
WorkspaceID: workspaceID,
ActorType: "system",
ActorID: "",
TaskID: util.UUIDToString(task.ID),
Payload: payload,
}
if task.ChatSessionID.Valid {
chatSessionID := util.UUIDToString(task.ChatSessionID)
payload["chat_session_id"] = chatSessionID
e.ChatSessionID = chatSessionID
}
for _, fields := range extra {
for key, value := range fields {
payload[key] = value
}
}
return e
}
func (s *TaskService) publishTaskEvent(eventType, workspaceID string, task db.AgentTaskQueue, extra ...map[string]any) {
if workspaceID == "" {
return
}
s.Bus.Publish(taskEvent(eventType, workspaceID, task, extra...))
}
func (s *TaskService) broadcastTaskEvent(ctx context.Context, eventType string, task db.AgentTaskQueue, extra ...map[string]any) {
workspaceID := s.ResolveTaskWorkspaceID(ctx, task)
s.publishTaskEvent(eventType, workspaceID, task, extra...)
}
// taskFailedFields adds the terminal failure context required by channel
// outbounds without changing the long-standing map payload used by existing
// task event consumers. Error text is redacted and omitted while an automatic
// retry is pending, so consumers can distinguish an intermediate failed
// attempt from a user-visible terminal failure.
func taskFailedFields(errMsg, failureReason string, retryPending bool) map[string]any {
fields := map[string]any{
"failure_reason": failureReason,
"retry_pending": retryPending,
}
if errMsg != "" && !retryPending {
fields["error"] = redact.Text(errMsg)
}
return fields
}
func (s *TaskService) publishTaskFailedEvent(workspaceID string, task db.AgentTaskQueue, errMsg, failureReason string, retryPending bool) {
s.publishTaskEvent(protocol.EventTaskFailed, workspaceID, task, taskFailedFields(errMsg, failureReason, retryPending))
}
func (s *TaskService) broadcastTaskFailedEvent(ctx context.Context, task db.AgentTaskQueue, errMsg, failureReason string, retryPending bool) {
workspaceID := s.ResolveTaskWorkspaceID(ctx, task)
s.publishTaskFailedEvent(workspaceID, task, errMsg, failureReason, retryPending)
}
// ResolveTaskWorkspaceID determines the workspace ID for a task, best-effort.
// Returns "" when the workspace could not be determined, whether because the
// link target is genuinely gone or because a lookup failed.
//
// Use this only where "" is an acceptable answer — event broadcasts skip
// themselves rather than fabricating a workspace. Anything that turns the
// result into an HTTP status MUST use ResolveTaskWorkspaceIDChecked instead:
// collapsing both cases to "" is what let a transient DB error be reported to
// the daemon as `404 task not found`, which it acts on by killing a healthy
// run (MUL-7259 / GH #8272).
func (s *TaskService) ResolveTaskWorkspaceID(ctx context.Context, task db.AgentTaskQueue) string {
workspaceID, _ := s.ResolveTaskWorkspaceIDChecked(ctx, task)
return workspaceID
}
// ResolveTaskWorkspaceIDChecked resolves a task's workspace and keeps the two
// failure modes apart:
//
// - ("", nil) — every link this task carries was looked up successfully and
// the target is genuinely absent. The task is unreachable; a 404 is honest.
// - ("", err) — a lookup could not be completed (DB timeout, pool exhaustion,
// …). We do not know whether the task is reachable, and the caller must NOT
// report absence. This is a 5xx.
//
// For issue tasks the workspace comes from the issue, for chat tasks from the
// chat session, for autopilot tasks from the autopilot via its run, and for
// quick-create tasks from the context JSONB (they carry no link at all).
//
// A failed lookup does not stop the walk. If a later link resolves, its
// workspace is returned and the earlier error is dropped, because that answer
// is still trustworthy; the error is surfaced only when nothing resolved. That
// keeps a task carrying several links working during a partial outage while
// still refusing to call an unknown state "not found".
func (s *TaskService) ResolveTaskWorkspaceIDChecked(ctx context.Context, task db.AgentTaskQueue) (string, error) {
scope, err := s.ResolveTaskScopeChecked(ctx, task)
return scope.WorkspaceID, err
}
// TaskScope is where a task's realtime events belong.
type TaskScope struct {
WorkspaceID string
// ChatSessionOwnerID is the chat session creator when the workspace was
// resolved through the task's chat session, and "" otherwise.
ChatSessionOwnerID string
}
// ResolveTaskScope is ResolveTaskWorkspaceID plus the chat session owner, read
// from the same chat_session row so chat broadcasts cost no extra query.
func (s *TaskService) ResolveTaskScope(ctx context.Context, task db.AgentTaskQueue) TaskScope {
scope, _ := s.ResolveTaskScopeChecked(ctx, task)
return scope
}
// ResolveTaskScopeChecked is ResolveTaskWorkspaceIDChecked plus the chat
// session owner, with the same failure semantics.
func (s *TaskService) ResolveTaskScopeChecked(ctx context.Context, task db.AgentTaskQueue) (TaskScope, error) {
// isNotFound is the "genuinely absent" signal; every other error means the
// lookup itself did not complete. pgx.ErrNoRows is the only error the
// queries below use to say "this row does not exist".
var lookupErr error
note := func(err error) {
if err != nil && !errors.Is(err, pgx.ErrNoRows) && lookupErr == nil {
lookupErr = err
}
}
if task.IssueID.Valid {
issue, err := s.Queries.GetIssue(ctx, task.IssueID)
if err == nil {
return TaskScope{WorkspaceID: util.UUIDToString(issue.WorkspaceID)}, nil
}
note(err)
}
if task.ChatSessionID.Valid {
cs, err := s.Queries.GetChatSession(ctx, task.ChatSessionID)
if err == nil {
return TaskScope{
WorkspaceID: util.UUIDToString(cs.WorkspaceID),
ChatSessionOwnerID: util.UUIDToString(cs.CreatorID),
}, nil
}
note(err)
}
if task.AutopilotRunID.Valid {
run, err := s.Queries.GetAutopilotRun(ctx, task.AutopilotRunID)
if err == nil {
ap, apErr := s.Queries.GetAutopilot(ctx, run.AutopilotID)
if apErr == nil {
return TaskScope{WorkspaceID: util.UUIDToString(ap.WorkspaceID)}, nil
}
note(apErr)
}
note(err)
}
// Quick-create tasks have no issue / chat / autopilot link — workspace
// lives in the context JSONB. Returning "" here is what blocked
// requireDaemonTaskAccess (404 on /start, /progress, /complete, /fail
// for the daemon) and silently dropped task:dispatch / task:completed
// broadcasts, which is why quick-create tasks appeared stuck queued.
if qc, ok := s.parseQuickCreateContext(task); ok {
return TaskScope{WorkspaceID: qc.WorkspaceID}, nil
}
if lookupErr != nil {
return TaskScope{}, fmt.Errorf("resolve task workspace: %w", lookupErr)
}
return TaskScope{}, nil
}
func (s *TaskService) broadcastChatDone(ctx context.Context, task db.AgentTaskQueue, msg *db.ChatMessage, quickActionsPending bool) {
scope := s.ResolveTaskScope(ctx, task)
workspaceID := scope.WorkspaceID
if workspaceID == "" {
return
}
payload := protocol.ChatDonePayload{
ChatSessionID: util.UUIDToString(task.ChatSessionID),
TaskID: util.UUIDToString(task.ID),
QuickActionsPending: quickActionsPending,
}
if msg != nil {
payload.MessageID = util.UUIDToString(msg.ID)
payload.Content = msg.Content
payload.MessageKind = msg.MessageKind
if len(msg.QuickActions) > 0 {
_ = json.Unmarshal(msg.QuickActions, &payload.QuickActions)
}
if msg.CreatedAt.Valid {
payload.CreatedAt = msg.CreatedAt.Time.UTC().Format(time.RFC3339Nano)
}
if msg.ElapsedMs.Valid {
payload.ElapsedMs = msg.ElapsedMs.Int64
}
}
s.Bus.Publish(events.Event{
Type: protocol.EventChatDone,
WorkspaceID: workspaceID,
ActorType: "system",
ActorID: "",
ChatSessionID: util.UUIDToString(task.ChatSessionID),
ChatSessionOwnerID: scope.ChatSessionOwnerID,
Payload: payload,
})
}
// broadcastIssueUpdated publishes the issue:updated event the frontend's
// realtime reconcile (onIssueUpdated) relies on to move an issue between status
// columns / status filters and reconcile their bucket counts. prevStatus is the
// issue's status before the write so the client can gate that reconcile on
// status_changed.
//
// The `issue` payload is a map (IssueToMap), which the workspace WS fanout
// (listeners.go SubscribeAll) marshals and broadcasts as-is — that is what
// drives the UI reconcile. Note this does NOT cover the full HTTP UpdateIssue
// side effects: the activity-log and inbox listeners type-assert `issue` to a
// handler.IssueResponse and skip a map, so a background status reset does not
// emit status-change activity / notifications. That is intentional for the
// realtime-staleness fix (#4648 / MUL-3782); folding those side effects in
// would mean unifying the payload type and is left as a follow-up.
func (s *TaskService) broadcastIssueUpdated(ctx context.Context, issue db.Issue, prevStatus string) {
prefix := s.getIssuePrefix(issue.WorkspaceID)
s.Bus.Publish(events.Event{
Type: protocol.EventIssueUpdated,
WorkspaceID: util.UUIDToString(issue.WorkspaceID),
ActorType: "system",
ActorID: "",
Payload: map[string]any{
"issue": IssueToMapResolved(ctx, s.Queries, issue, prefix),
"status_changed": prevStatus != issue.Status,
"prev_status": prevStatus,
},
})
}
func (s *TaskService) getIssuePrefix(workspaceID pgtype.UUID) string {
ws, err := s.Queries.GetWorkspace(context.Background(), workspaceID)
if err != nil {
return ""
}
return ws.IssuePrefix
}
// commentEventFields renders the `comment` object carried by comment:created
// broadcasts published outside the HTTP handler (agent replies, delegated
// failure recovery, recovery exhaustion).
//
// created_at goes through util.TimestampToString — the SAME helper the REST
// CommentResponse uses — so the timestamp a client receives over WS is
// byte-identical to the one it gets when it refetches that comment. These
// payloads used to format the time with a literal "Z" suffix instead, which
// is not Go's Z07:00 offset token: pgx decodes timestamptz into the process
// location (pgtype/timestamptz.go, no ScanLocation configured), so on a
// deployment whose process TZ is not UTC the broadcast stamped local
// wall-clock digits with a UTC label. Clients then placed the entry a whole
// offset away from its real instant and moved it again once a refetch
// replaced the value — visible as timeline entries jumping position.
func commentEventFields(c db.Comment) map[string]any {
return map[string]any{
"id": util.UUIDToString(c.ID),
"issue_id": util.UUIDToString(c.IssueID),
"author_type": c.AuthorType,
"author_id": util.UUIDToString(c.AuthorID),
"content": c.Content,
"type": c.Type,
"parent_id": util.UUIDToPtr(c.ParentID),
"source_task_id": util.UUIDToPtr(c.SourceTaskID),
"created_at": util.TimestampToString(c.CreatedAt),
}
}
func (s *TaskService) createAgentComment(ctx context.Context, issueID, agentID pgtype.UUID, content, commentType string, parentID, sourceTaskID pgtype.UUID) {
if content == "" {
return
}
// Look up issue to get workspace ID for mention expansion and broadcasting.
issue, err := s.Queries.GetIssue(ctx, issueID)
if err != nil {
return
}
// Resolve the thread root for thread-level side effects without overwriting
// parentID. The stored parent_id must remain the exact comment being replied
// to; recursive thread reads recover the root when needed.
var rootComment *db.Comment
if parentID.Valid {
if root, err := s.Queries.GetThreadRoot(ctx, db.GetThreadRootParams{
CommentID: parentID,
WorkspaceID: issue.WorkspaceID,
}); err == nil {
rootComment = &root
}
}
created, err := s.Queries.CreateComment(ctx, db.CreateCommentParams{
ID: dbid.NewV7(),
IssueID: issueID,
WorkspaceID: issue.WorkspaceID,
AuthorType: "agent",
AuthorID: agentID,
Content: content,
Type: commentType,
ParentID: parentID,
SourceTaskID: sourceTaskID,
})
if err != nil {
return
}
comment := created.Comment()
commentFields := commentEventFields(comment)
commentFields["revision"] = comment.Revision
s.Bus.Publish(events.Event{
Type: protocol.EventCommentCreated,
WorkspaceID: util.UUIDToString(issue.WorkspaceID),
ActorType: "agent",
ActorID: util.UUIDToString(agentID),
Payload: map[string]any{
"comment": commentFields,
"issue_title": issue.Title,
"issue_status": issue.Status,
"issue_revision": created.IssueRevision,
},
})
s.AutoUnresolveThreadOnReply(ctx, rootComment, util.UUIDToString(issue.WorkspaceID), "agent", util.UUIDToString(agentID), sourceTaskID)
}
// AutoUnresolveThreadOnReply clears resolved_at on the thread root when a
// reply lands in a resolved thread, and broadcasts comment:unresolved. Shared
// between the user-facing Handler.CreateComment path and the agent-facing
// TaskService.createAgentComment path so the resolved-then-replied state can
// never desync (one of the bugs Emacs flagged on PR #2300). Errors are logged
// — the reply itself already committed, the desync is recoverable on next read.
func (s *TaskService) AutoUnresolveThreadOnReply(ctx context.Context, parent *db.Comment, workspaceID, actorType, actorID string, sourceTaskID pgtype.UUID) {
if parent == nil || !parent.ResolvedAt.Valid {
return
}
// This follow-up write is a consequence of the reply's run, not a new
// system action. Preserve that lineage for event subscriptions as well.
tx, err := s.TxStarter.Begin(ctx)
if err != nil {
slog.Warn("auto-unresolve transaction failed", "error", err)
return
}
defer tx.Rollback(ctx)
_, err = tx.Exec(ctx, `SELECT set_config('multica.actor_type',$1,true),set_config('multica.actor_id',$2,true),set_config('multica.source_task_id',$3,true)`, actorType, actorID, util.UUIDToString(sourceTaskID))
if err != nil {
slog.Warn("auto-unresolve source attribution failed", "error", err)
return
}
updated, err := s.Queries.WithTx(tx).UnresolveComment(ctx, parent.ID)
if err != nil {
slog.Warn("auto-unresolve on reply failed", "error", err, "comment_id", util.UUIDToString(parent.ID))
return
}
if err := tx.Commit(ctx); err != nil {
slog.Warn("auto-unresolve commit failed", "error", err)
return
}
s.Bus.Publish(events.Event{
Type: protocol.EventCommentUnresolved,
WorkspaceID: workspaceID,
ActorType: actorType,
ActorID: actorID,
Payload: map[string]any{
"comment": map[string]any{
"id": util.UUIDToString(updated.ID),
"issue_id": util.UUIDToString(updated.IssueID),
"author_type": updated.AuthorType,
"author_id": util.UUIDToString(updated.AuthorID),
"content": updated.Content,
"type": updated.Type,
"parent_id": util.UUIDToPtr(updated.ParentID),
"created_at": util.TimestampToString(updated.CreatedAt),
"updated_at": util.TimestampToString(updated.UpdatedAt),
"resolved_at": util.TimestampToPtr(updated.ResolvedAt),
"resolved_by_type": util.TextToPtr(updated.ResolvedByType),
"resolved_by_id": util.UUIDToPtr(updated.ResolvedByID),
"revision": updated.Revision,
},
},
})
}
// IssueToMap renders an issue row as the map shape the issue:created /
// issue:updated broadcast payloads carry under their "issue" key. It is the
// single source of truth for that shape wherever the event is published from
// outside the HTTP handler — autopilot and the channel engine's /issue command
// on issue:created, the background stuck-issue status reset on issue:updated.
// The workspace WS fanout marshals it as-is for the UI, and cmd/server's
// extractIssueFields reads id / creator_id / workspace_id off it to decide who
// to auto-subscribe.
//
// The map must stay key-compatible with handler.IssueResponse, the other
// rendering of the same event. Clients type both as a complete Issue and
// insert it straight into the list cache without runtime validation, so a
// field missing here is a field that reads back undefined until the next
// refetch — see TestIssueToMap_KeysMatchIssueResponse, which fails if the two
// renderings drift apart.
// builtInStatusCategory returns a status's public lifecycle category when it
// can be known without a catalog read.
func builtInStatusCategory(status string) string {
if category, ok := issuestatus.CategoryForBehavior(status); ok {
return issuestatus.WireCategory(status, category)
}
return ""
}
// IssueMapQuerier is what IssueToMapResolved reads: the status catalog and,
// for a duplicate, its original.
type IssueMapQuerier interface {
issuestatus.Querier
GetIssueRefInWorkspace(ctx context.Context, arg db.GetIssueRefInWorkspaceParams) (db.GetIssueRefInWorkspaceRow, error)
}
// IssueToMapResolved is IssueToMap with an AUTHORITATIVE status_category and
// status_name, both resolved through the catalog so a custom status is not
// emitted with blanks. Background events go through here; clients treat this
// payload as a complete issue and bucket it by category. (MUL-6243)
//
// Both fields come from ONE catalog read. Resolving them separately would
// double the query on every event carrying a custom status, and the HTTP
// rendering already shares a single read through its Resolver. (MUL-6749)
//
// duplicate_of is resolved too: a client patches its cache with this
// snapshot, so a null here would erase a mark the issue still carries
// (MUL-7349). Only a cancelled issue with a pointer costs a read.
func IssueToMapResolved(ctx context.Context, q IssueMapQuerier, issue db.Issue, issuePrefix string) map[string]any {
m := IssueToMap(issue, issuePrefix)
category, name := issuestatus.CategoryAndName(ctx, q, issue.WorkspaceID, issue.Status)
m["status_category"] = issuestatus.WireCategory(issue.Status, category)
m["status_name"] = name
if ref := resolveDuplicateOf(ctx, q, issue, issuePrefix); ref != nil {
m["duplicate_of"] = ref
}
return m
}
// resolveDuplicateOf renders the original a duplicate points at, in the shape
// of handler.IssueRefResponse, or nil when the issue carries no live mark: it
// is not cancelled, has no pointer, or its original is gone.
func resolveDuplicateOf(ctx context.Context, q IssueMapQuerier, issue db.Issue, issuePrefix string) map[string]any {
if issue.Status != issuestatus.Cancelled || !issue.DuplicateOfIssueID.Valid {
return nil
}
row, err := q.GetIssueRefInWorkspace(ctx, db.GetIssueRefInWorkspaceParams{
ID: issue.DuplicateOfIssueID,
WorkspaceID: issue.WorkspaceID,
})
if err != nil {
if !errors.Is(err, pgx.ErrNoRows) {
slog.Warn("resolve duplicate original failed",
"issue_id", util.UUIDToString(issue.ID), "error", err)
}
return nil
}
return map[string]any{
"id": util.UUIDToString(row.ID),
"identifier": IssueIdentifier(issuePrefix, row.Number),
"title": row.Title,
"status": row.Status,
}
}
func IssueToMap(issue db.Issue, issuePrefix string) map[string]any {
return map[string]any{
"id": util.UUIDToString(issue.ID),
"workspace_id": util.UUIDToString(issue.WorkspaceID),
"number": issue.Number,
"identifier": IssueIdentifier(issuePrefix, issue.Number),
"title": issue.Title,
"description": util.TextToPtr(issue.Description),
"status": issue.Status,
// Mirrors handler.IssueResponse.StatusCategory. Built-ins map to a
// public lifecycle category without a catalog lookup; custom statuses
// are filled by IssueToMapResolved. (MUL-6243)
"status_category": builtInStatusCategory(issue.Status),
// Mirrors handler.IssueResponse.StatusName. A built-in carries no name
// — clients localize those from the key — and a CUSTOM one is filled in
// by IssueToMapResolved, which has the catalog. Emitted unconditionally
// so this rendering cannot lose a key the HTTP one carries. (MUL-6749)
"status_name": "",
"priority": issue.Priority,
"assignee_type": util.TextToPtr(issue.AssigneeType),
"assignee_id": util.UUIDToPtr(issue.AssigneeID),
"creator_type": issue.CreatorType,
"creator_id": util.UUIDToString(issue.CreatorID),
"parent_issue_id": util.UUIDToPtr(issue.ParentIssueID),
// Mirrors handler.IssueResponse.DuplicateOf. Null is only true for a
// row with no live mark; IssueToMapResolved resolves it for the rest.
"duplicate_of": nil,
"project_id": util.UUIDToPtr(issue.ProjectID),
"position": issue.Position,
"stage": util.Int4ToPtr(issue.Stage),
"start_date": util.DateToPtr(issue.StartDate),
"due_date": util.DateToPtr(issue.DueDate),
"created_at": util.TimestampToString(issue.CreatedAt),
"updated_at": util.TimestampToString(issue.UpdatedAt),
"last_activity_at": util.TimestampToNanoPtr(issue.LastActivityAt),
"revision": issue.Revision,
"metadata": util.JSONObjectOrEmpty(issue.Metadata),
"properties": util.JSONObjectOrEmpty(issue.Properties),
}
}
// IssueIdentifier renders the human-facing issue key ("MUL-42"). Callers that
// resolve the workspace prefix defensively may pass "": a failed workspace
// lookup should not surface as a stray "-42", so the number stands alone as
// "#42". The HTTP layer never passes "" — handler.getIssuePrefix derives a
// prefix from the workspace name before rendering anything.
func IssueIdentifier(issuePrefix string, number int32) string {
if issuePrefix == "" {
return "#" + strconv.Itoa(int(number))
}
return issuePrefix + "-" + strconv.Itoa(int(number))
}
// parseQuickCreateContext returns the quick-create payload if the task's
// context JSONB contains type == "quick_create"; otherwise the bool is
// false so callers can short-circuit. Tasks linked to an issue / chat /
// autopilot are never quick-create even if they happen to carry a
// context blob, so those are filtered up front.
func (s *TaskService) parseQuickCreateContext(task db.AgentTaskQueue) (QuickCreateContext, bool) {
if task.IssueID.Valid || task.ChatSessionID.Valid || task.AutopilotRunID.Valid {
return QuickCreateContext{}, false
}
if len(task.Context) == 0 {
return QuickCreateContext{}, false
}
var qc QuickCreateContext
if err := json.Unmarshal(task.Context, &qc); err != nil {
return QuickCreateContext{}, false
}
if qc.Type != QuickCreateContextType {
return QuickCreateContext{}, false
}
return qc, true
}
func (s *TaskService) sourceContextAttachedByTask(ctx context.Context, task db.AgentTaskQueue, qc QuickCreateContext) (bool, error) {
if qc.SourceContextID == "" {
return false, nil
}
workspaceID, workspaceErr := util.ParseUUID(qc.WorkspaceID)
contextID, contextErr := util.ParseUUID(qc.SourceContextID)
if workspaceErr != nil || contextErr != nil {
return false, nil
}
row, err := s.Queries.GetIssueSourceContextByID(ctx, db.GetIssueSourceContextByIDParams{
WorkspaceID: workspaceID,
ID: contextID,
})
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, err
}
return row.State == "attached" && row.IssueID.Valid && row.OriginTaskID == task.ID, nil
}
// maxQuickCreateFailureDetailRunes bounds the failure reason lifted from a
// quick-create task's final output. A genuine CLI error (a duplicate message,
// a validation error) is short; an output far larger than this is a runaway
// raw-stream dump whose head is process narration rather than the real reason
// (same failure mode as GH #5455), so it is dropped to a generic notice rather
// than surfaced as the error.
const maxQuickCreateFailureDetailRunes = 2000
const quickCreateOversizedFailureDetail = "Quick create failed, but the agent's output was too large to show the reason safely. Check the task's execution log for details."
// quickCreateFailureDetail extracts a user-facing failure reason from a
// quick-create task's final output. The quick-create prompt instructs the agent
// to exit with the CLI error as its only output when `multica issue create`
// fails, so this normally carries the real reason (e.g. an active-duplicate
// message naming the existing issue). Returns "" when there is no usable output
// so the caller falls back to a generic message; redaction is applied by
// notifyQuickCreateFailed.
func quickCreateFailureDetail(result []byte) string {
var payload protocol.TaskCompletedPayload
if err := json.Unmarshal(result, &payload); err != nil {
return ""
}
// Same unescape as the comment-fallback path: literal `\n` sequences from
// agent stdout become real newlines before the reason reaches the user.
body := strings.TrimSpace(util.UnescapeBackslashEscapes(payload.Output))
if body == "" {
return ""
}
if utf8.RuneCountInString(body) > maxQuickCreateFailureDetailRunes {
return quickCreateOversizedFailureDetail
}
return body
}
// notifyQuickCreateCompleted writes a success inbox notification to the
// requester pointing at the issue the agent just created. The issue is
// stamped with origin_type=quick_create + origin_id=<task_id> by the
// daemon-injected MULTICA_QUICK_CREATE_TASK_ID env var, so this lookup is
// deterministic — robust against the same agent creating other issues in
// parallel (e.g. assignment task running while max_concurrent_tasks > 1
// permits another quick-create alongside it).
func (s *TaskService) notifyQuickCreateCompleted(ctx context.Context, task db.AgentTaskQueue, qc QuickCreateContext, result []byte) {
requesterID, err := util.ParseUUID(qc.RequesterID)
if err != nil {
slog.Warn("quick-create completion: invalid requester id", "task_id", util.UUIDToString(task.ID), "error", err)
return
}
workspaceID, err := util.ParseUUID(qc.WorkspaceID)
if err != nil {
slog.Warn("quick-create completion: invalid workspace id", "task_id", util.UUIDToString(task.ID), "error", err)
return
}
issue, err := s.Queries.GetIssueByOrigin(ctx, db.GetIssueByOriginParams{
WorkspaceID: workspaceID,
OriginType: pgtype.Text{String: "quick_create", Valid: true},
OriginID: task.ID,
})
if err != nil {
if !errors.Is(err, pgx.ErrNoRows) {
// The lookup itself failed (DB fault, timeout, …), not a confirmed
// "no issue": the agent may well have created the issue, so a
// failure inbox would misreport it. But the task is already
// completed and nothing retries this reconciliation, so returning
// silently would end the run with NO inbox result at all. Write a
// neutral, terminal notification instead — the user always gets a
// result, and it never asserts a failure we did not observe.
slog.Error("quick-create completion: issue lookup failed, writing unconfirmed inbox",
"task_id", util.UUIDToString(task.ID),
"agent_id", util.UUIDToString(task.AgentID),
"workspace_id", qc.WorkspaceID,
"error", err,
)
s.notifyQuickCreateUnconfirmed(ctx, task, qc)
return
}
// No issue created — the agent ran to completion but the CLI create
// call must have failed (most often the active-duplicate guard). The
// quick-create prompt tells the agent to exit with the CLI error as its
// only output, so prefer that as the failure reason instead of a
// generic string; fall back to notifyQuickCreateFailed's own default
// when the output is empty. This is what turns the opaque "agent
// finished without creating an issue" into the concrete reason (#5885).
detail := quickCreateFailureDetail(result)
slog.Warn("quick-create completion: no issue found, writing failure inbox",
"task_id", util.UUIDToString(task.ID),
"agent_id", util.UUIDToString(task.AgentID),
"workspace_id", qc.WorkspaceID,
"has_detail", detail != "",
)
s.notifyQuickCreateFailed(ctx, task, qc, detail)
return
}
// Link the new issue back to this task so subsequent reads (Activity tab,
// Recent work, etc.) can navigate to the result instead of leaving it on
// the "Creating issue" active wording. The task's source kind remains
// quick_create, derived from its typed context. Best-effort: a write failure
// here doesn't block the inbox notification, which is the more important
// signal to the user.
if err := s.Queries.LinkTaskToIssue(ctx, db.LinkTaskToIssueParams{
ID: task.ID,
IssueID: issue.ID,
}); err != nil {
slog.Warn("quick-create completion: link task→issue failed",
"task_id", util.UUIDToString(task.ID),
"issue_id", util.UUIDToString(issue.ID),
"error", err,
)
}
// Subscribing the requester used to happen here, at completion. It now
// happens at issue-creation time in the shared delegated-subscriber rule
// (cmd/server/subscriber_listeners.go → subscribeDelegatedHuman), which
// resolves the human from origin_type='quick_create' + the origin task's
// originator_user_id — the same origin waterfall attribution uses.
//
// This was one of three separate hand-rolled fixes for "an agent created
// this issue and no human is subscribed" (quick-create here, the autopilot
// subscriber template, and — missing entirely until MUL-5483 — ordinary
// agent-created sub-issues). Keeping a second write here would leave the
// same decision encoded in two places that can drift.
prefix := s.getIssuePrefix(workspaceID)
identifier := fmt.Sprintf("%s-%d", prefix, issue.Number)
details, _ := json.Marshal(map[string]any{
"task_id": util.UUIDToString(task.ID),
"agent_id": util.UUIDToString(task.AgentID),
"issue_id": util.UUIDToString(issue.ID),
"identifier": identifier,
"original_prompt": qc.Prompt,
})
item, inserted, err := inbox.CreateIfEligible(ctx, s.Queries, db.CreateInboxItemParams{
ID: dbid.NewV7(),
WorkspaceID: workspaceID,
RecipientType: "member",
RecipientID: requesterID,
Type: "quick_create_done",
Severity: "info",
IssueID: issue.ID,
Title: issue.Title,
Body: pgtype.Text{},
ActorType: pgtype.Text{String: "agent", Valid: true},
ActorID: task.AgentID,
Details: details,
})
if err != nil {
slog.Error("quick-create completion: inbox write failed", "task_id", util.UUIDToString(task.ID), "error", err)
return
}
if !inserted {
return
}
s.publishQuickCreateInbox(item, qc.WorkspaceID, util.UUIDToString(task.AgentID), issue.Status)
}
// Inbox types for the two non-success quick-create outcomes. They are distinct
// because clients render them differently: the failed type carries an explicit
// "Failed:" framing, which must not be applied to an outcome we could not
// verify. Older clients that predate the unconfirmed type fall through their
// existing default branch and render the row's title/body unchanged, which is
// already the neutral wording.
const (
inboxTypeQuickCreateFailed = "quick_create_failed"
inboxTypeQuickCreateUnconfirmed = "quick_create_unconfirmed"
)
// notifyQuickCreateFailed writes a failure inbox notification carrying the
// original prompt + agent ID so the frontend can render an "Edit as
// advanced form" entry that pre-fills the legacy create-issue modal
// without asking the user to retype. Use this only when the run is KNOWN not
// to have produced an issue; when that is merely unverified, use
// notifyQuickCreateUnconfirmed so the user is not told a definite failure.
func (s *TaskService) notifyQuickCreateFailed(ctx context.Context, task db.AgentTaskQueue, qc QuickCreateContext, errMsg string) {
if errMsg == "" {
errMsg = "Quick create did not finish successfully"
}
s.writeQuickCreateOutcomeInbox(ctx, task, qc, inboxTypeQuickCreateFailed, "Quick create failed", errMsg)
}
// quickCreateUnconfirmedDetail is the user-facing message for a quick-create
// run whose outcome could not be verified (the completion lookup itself
// failed). It must not claim failure: the agent may well have created the
// issue. It points at the one safe next step instead — check before retrying,
// so a retry cannot silently produce the duplicate the guard exists to prevent.
const quickCreateUnconfirmedDetail = "Couldn't confirm whether the issue was created. Check your recent issues before retrying — creating it again may produce a duplicate."
// notifyQuickCreateUnconfirmed writes a NEUTRAL terminal notification for a
// quick-create run whose outcome is unverified. The task is already completed
// and nothing re-runs this reconciliation, so returning without writing here
// would strand the requester with no inbox result at all — the failure mode
// this exists to prevent.
//
// It uses its own inbox type rather than reusing quick_create_failed: every
// client renders the failed type with a "Failed:" prefix, which would assert a
// failure we never observed no matter how neutral the title and body are.
// Severity stays action_required because the user does need to look.
func (s *TaskService) notifyQuickCreateUnconfirmed(ctx context.Context, task db.AgentTaskQueue, qc QuickCreateContext) {
s.writeQuickCreateOutcomeInbox(ctx, task, qc, inboxTypeQuickCreateUnconfirmed, "Quick create needs a check", quickCreateUnconfirmedDetail)
}
// quickCreateNotifyTimeout bounds the detached terminal-notification write in
// writeQuickCreateOutcomeInbox. Long enough for a healthy DB round-trip, short
// enough that a wedged pool cannot pin the completion goroutine.
const quickCreateNotifyTimeout = 5 * time.Second
// writeQuickCreateOutcomeInbox writes the shared inbox row used by both
// non-success outcomes (known failure and unverified outcome). Callers own the
// user-facing wording and the row type; the row shape — original prompt, agent
// id, redacted message — is identical so the frontend's recovery affordance
// keeps working for both.
func (s *TaskService) writeQuickCreateOutcomeInbox(ctx context.Context, task db.AgentTaskQueue, qc QuickCreateContext, inboxType, title, errMsg string) {
requesterID, err := util.ParseUUID(qc.RequesterID)
if err != nil {
return
}
workspaceID, err := util.ParseUUID(qc.WorkspaceID)
if err != nil {
return
}
// The task is already committed as completed and nothing retries this
// notification, so it must not die with the caller's context. This matters
// most on the unconfirmed path: the completion lookup may have failed
// precisely BECAUSE ctx was cancelled or timed out, and reusing that ctx
// would fail the write for the same reason — leaving the user with no
// result at all, the exact silent drop this path exists to prevent. Detach
// from cancellation, but keep a bound so a wedged DB cannot pin us.
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), quickCreateNotifyTimeout)
defer cancel()
details, _ := json.Marshal(map[string]any{
"task_id": util.UUIDToString(task.ID),
"agent_id": util.UUIDToString(task.AgentID),
"original_prompt": qc.Prompt,
"error": redact.Text(errMsg),
"source_context_id": qc.SourceContextID,
})
item, inserted, err := inbox.CreateIfEligible(ctx, s.Queries, db.CreateInboxItemParams{
ID: dbid.NewV7(),
WorkspaceID: workspaceID,
RecipientType: "member",
RecipientID: requesterID,
Type: inboxType,
Severity: "action_required",
IssueID: pgtype.UUID{},
Title: title,
Body: pgtype.Text{String: redact.Text(errMsg), Valid: true},
ActorType: pgtype.Text{String: "agent", Valid: true},
ActorID: task.AgentID,
Details: details,
})
if err != nil {
slog.Error("quick-create failure: inbox write failed", "task_id", util.UUIDToString(task.ID), "error", err)
return
}
if !inserted {
return
}
s.publishQuickCreateInbox(item, qc.WorkspaceID, util.UUIDToString(task.AgentID), "")
}
// publishQuickCreateInbox emits the WS event so the requester's inbox list
// updates immediately. Mirrors the payload shape used by the other inbox
// listeners (notification_listeners.go).
func (s *TaskService) publishQuickCreateInbox(item db.InboxItem, workspaceID, agentID, issueStatus string) {
resp := map[string]any{
"id": util.UUIDToString(item.ID),
"workspace_id": util.UUIDToString(item.WorkspaceID),
"recipient_type": item.RecipientType,
"recipient_id": util.UUIDToString(item.RecipientID),
"type": item.Type,
"severity": item.Severity,
"issue_id": util.UUIDToPtr(item.IssueID),
"title": item.Title,
"body": util.TextToPtr(item.Body),
"read": item.Read,
"archived": item.Archived,
"created_at": util.TimestampToString(item.CreatedAt),
"actor_type": util.TextToPtr(item.ActorType),
"actor_id": util.UUIDToPtr(item.ActorID),
"details": json.RawMessage(item.Details),
"issue_status": issueStatus,
}
s.Bus.Publish(events.Event{
Type: protocol.EventInboxNew,
WorkspaceID: workspaceID,
ActorType: "agent",
ActorID: agentID,
Payload: map[string]any{"item": resp},
})
}
// agentToMap builds a simple map for broadcasting agent status updates.
func agentToMap(a db.Agent) map[string]any {
var rc any
if a.RuntimeConfig != nil {
json.Unmarshal(a.RuntimeConfig, &rc)
}
return map[string]any{
"id": util.UUIDToString(a.ID),
"workspace_id": util.UUIDToString(a.WorkspaceID),
"runtime_id": util.UUIDToString(a.RuntimeID),
"name": a.Name,
"description": a.Description,
"avatar_url": util.TextToPtr(a.AvatarUrl),
"runtime_mode": a.RuntimeMode,
"runtime_config": rc,
"visibility": a.Visibility,
"status": a.Status,
"max_concurrent_tasks": a.MaxConcurrentTasks,
"owner_id": util.UUIDToPtr(a.OwnerID),
"skills": []any{},
"created_at": util.TimestampToString(a.CreatedAt),
"updated_at": util.TimestampToString(a.UpdatedAt),
"archived_at": util.TimestampToPtr(a.ArchivedAt),
"archived_by": util.UUIDToPtr(a.ArchivedBy),
}
}