mirror of
https://gh-proxy.org/https://github.com/klinsmaya/multica.git
synced 2026-11-08 01:16:45 +08:00
- issueworkflow.Brief is the only renderer: the claim carries it into every run's per-turn prompt; the editor preview and timeline show the same text. - Workflow handoffs stop writing the brief into handoff_note, so a person's note no longer drops the brief and the handoff framing fits again. - Agents that are not the issue's assignee get an observer brief without the step's commands. - Clearer rule override and narrower wakeup guidance; user text is stripped of control characters. - multica-platform skill documents project workflows. Co-authored-by: Cursor <[email protected]> Co-authored-by: multica-agent <[email protected]>
6158 lines
260 KiB
Go
6158 lines
260 KiB
Go
package handler
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log/slog"
|
||
"net/http"
|
||
"slices"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/go-chi/chi/v5"
|
||
"github.com/google/uuid"
|
||
"github.com/jackc/pgx/v5"
|
||
"github.com/jackc/pgx/v5/pgtype"
|
||
"github.com/multica-ai/multica/server/internal/analytics"
|
||
"github.com/multica-ai/multica/server/internal/auth"
|
||
"github.com/multica-ai/multica/server/internal/daemonws"
|
||
"github.com/multica-ai/multica/server/internal/integrations/slack"
|
||
"github.com/multica-ai/multica/server/internal/issuestatus"
|
||
"github.com/multica-ai/multica/server/internal/logger"
|
||
obsmetrics "github.com/multica-ai/multica/server/internal/metrics"
|
||
"github.com/multica-ai/multica/server/internal/middleware"
|
||
"github.com/multica-ai/multica/server/internal/runtimeapps"
|
||
"github.com/multica-ai/multica/server/internal/service"
|
||
"github.com/multica-ai/multica/server/internal/util"
|
||
"github.com/multica-ai/multica/server/pkg/agent"
|
||
db "github.com/multica-ai/multica/server/pkg/db/generated"
|
||
"github.com/multica-ai/multica/server/pkg/dbid"
|
||
"github.com/multica-ai/multica/server/pkg/protocol"
|
||
"github.com/multica-ai/multica/server/pkg/redact"
|
||
"github.com/multica-ai/multica/server/pkg/skillbundle"
|
||
"github.com/multica-ai/multica/server/pkg/taskfailure"
|
||
)
|
||
|
||
// claimPollHintMinDelay bounds a future mismatch between the hint query and
|
||
// deferred-task promotion to at most one follow-up claim per second. Under the
|
||
// shared eligibility fences an overdue task should normally be promoted by the
|
||
// current request, so this is a defense-in-depth floor rather than the steady
|
||
// state poll interval.
|
||
const claimPollHintMinDelay = time.Second
|
||
|
||
// Daemon workspace ownership helpers
|
||
|
||
// requireDaemonWorkspaceAccess verifies the caller has access to the given workspace.
|
||
// For daemon tokens (mdt_), compares the token's workspace ID directly.
|
||
// For PAT/JWT fallback, verifies user membership in the workspace.
|
||
func (h *Handler) requireDaemonWorkspaceAccess(w http.ResponseWriter, r *http.Request, workspaceID string) bool {
|
||
if workspaceID == "" {
|
||
writeError(w, http.StatusNotFound, "not found")
|
||
return false
|
||
}
|
||
|
||
// Daemon token: workspace must match.
|
||
if daemonWsID := middleware.DaemonWorkspaceIDFromContext(r.Context()); daemonWsID != "" {
|
||
if daemonWsID != workspaceID {
|
||
writeError(w, http.StatusNotFound, "not found")
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
// PAT/JWT fallback: check membership cache before hitting DB.
|
||
userID := requestUserID(r)
|
||
if userID != "" {
|
||
if h.MembershipCache.Get(r.Context(), userID, workspaceID) {
|
||
return true
|
||
}
|
||
}
|
||
|
||
_, ok := h.requireWorkspaceMember(w, r, workspaceID, "not found")
|
||
if ok && userID != "" {
|
||
h.MembershipCache.Set(r.Context(), userID, workspaceID)
|
||
}
|
||
return ok
|
||
}
|
||
|
||
// requireDaemonRuntimeAccess looks up a runtime and verifies the caller owns its workspace.
|
||
//
|
||
// Only pgx.ErrNoRows is treated as a real "runtime gone" 404 — the daemon uses
|
||
// that response to drop the stale runtime from its in-memory map and re-register,
|
||
// so collapsing transient DB errors into the same 404 would force the daemon to
|
||
// self-cleanup on a hiccup. Other DB errors become 500.
|
||
func (h *Handler) requireDaemonRuntimeAccess(w http.ResponseWriter, r *http.Request, runtimeID string) (db.AgentRuntime, bool) {
|
||
runtimeUUID, ok := parseUUIDOrBadRequest(w, runtimeID, "runtime_id")
|
||
if !ok {
|
||
return db.AgentRuntime{}, false
|
||
}
|
||
rt, err := h.getAgentRuntime(r.Context(), obsmetrics.RuntimeLookupSourceDaemonAPI, runtimeUUID)
|
||
if err != nil {
|
||
if isNotFound(err) {
|
||
writeError(w, http.StatusNotFound, "runtime not found")
|
||
return db.AgentRuntime{}, false
|
||
}
|
||
slog.Warn("get agent runtime failed", "runtime_id", runtimeID, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to load runtime")
|
||
return db.AgentRuntime{}, false
|
||
}
|
||
if !h.requireDaemonWorkspaceAccess(w, r, uuidToString(rt.WorkspaceID)) {
|
||
return db.AgentRuntime{}, false
|
||
}
|
||
return rt, true
|
||
}
|
||
|
||
// requireDaemonTaskAccess looks up a task and verifies the caller owns its workspace.
|
||
func (h *Handler) requireDaemonTaskAccess(w http.ResponseWriter, r *http.Request, taskID string) (db.AgentTaskQueue, bool) {
|
||
task, _, ok := h.requireDaemonTaskAccessWithWorkspace(w, r, taskID)
|
||
return task, ok
|
||
}
|
||
|
||
// requireDaemonTaskAccessWithWorkspace is the workspace-aware variant of
|
||
// requireDaemonTaskAccess. It returns the resolved workspace ID alongside
|
||
// the task row so callers that need to forward workspace_id into
|
||
// taskToResponse (powering RelativeWorkDir) don't have to repeat the
|
||
// ResolveTaskWorkspaceID lookup. The two helpers share their entire
|
||
// implementation; the simpler one is preserved for ergonomic call sites
|
||
// that genuinely don't need workspace_id.
|
||
func (h *Handler) requireDaemonTaskAccessWithWorkspace(w http.ResponseWriter, r *http.Request, taskID string) (db.AgentTaskQueue, string, bool) {
|
||
taskUUID, ok := parseUUIDOrBadRequest(w, taskID, "task_id")
|
||
if !ok {
|
||
return db.AgentTaskQueue{}, "", false
|
||
}
|
||
task, err := h.Queries.GetAgentTask(r.Context(), taskUUID)
|
||
if err != nil {
|
||
// Only treat pgx.ErrNoRows as a real "task gone" signal — daemon
|
||
// uses this 404 to interrupt the running agent, so a transient DB
|
||
// error must not be reported as a deletion.
|
||
if isNotFound(err) {
|
||
writeError(w, http.StatusNotFound, "task not found")
|
||
return db.AgentTaskQueue{}, "", false
|
||
}
|
||
slog.Warn("get agent task failed", "task_id", taskID, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to load task")
|
||
return db.AgentTaskQueue{}, "", false
|
||
}
|
||
|
||
// Same rule as the GetAgentTask branch above, one link further out: the
|
||
// daemon kills a running agent on this 404, so only a lookup that actually
|
||
// completed and found nothing may produce it. A DB timeout resolving the
|
||
// issue / chat session / autopilot leaves us unable to tell whether the
|
||
// task is reachable, and "I don't know" must not be reported as "deleted"
|
||
// (MUL-7259 / GH #8272 — the branch #2127 hardened above, missed here).
|
||
wsID, err := h.TaskService.ResolveTaskWorkspaceIDChecked(r.Context(), task)
|
||
if err != nil {
|
||
slog.Warn("resolve task workspace failed", "task_id", taskID, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to load task")
|
||
return db.AgentTaskQueue{}, "", false
|
||
}
|
||
if wsID == "" {
|
||
writeError(w, http.StatusNotFound, "task not found")
|
||
return db.AgentTaskQueue{}, "", false
|
||
}
|
||
|
||
if !h.requireDaemonWorkspaceAccess(w, r, wsID) {
|
||
return db.AgentTaskQueue{}, "", false
|
||
}
|
||
return task, wsID, true
|
||
}
|
||
|
||
// verifyDaemonWorkspaceAccess checks workspace access without writing an HTTP error.
|
||
// Used in loops where individual items may be skipped silently.
|
||
func (h *Handler) verifyDaemonWorkspaceAccess(r *http.Request, workspaceID string) bool {
|
||
if workspaceID == "" {
|
||
return false
|
||
}
|
||
if daemonWsID := middleware.DaemonWorkspaceIDFromContext(r.Context()); daemonWsID != "" {
|
||
return daemonWsID == workspaceID
|
||
}
|
||
userID := requestUserID(r)
|
||
if userID == "" {
|
||
return false
|
||
}
|
||
if h.MembershipCache.Get(r.Context(), userID, workspaceID) {
|
||
return true
|
||
}
|
||
_, err := h.getWorkspaceMember(r.Context(), userID, workspaceID)
|
||
if err != nil {
|
||
return false
|
||
}
|
||
h.MembershipCache.Set(r.Context(), userID, workspaceID)
|
||
return true
|
||
}
|
||
|
||
// Daemon Registration & Heartbeat
|
||
|
||
type DaemonRegisterRequest struct {
|
||
WorkspaceID string `json:"workspace_id"`
|
||
DaemonID string `json:"daemon_id"`
|
||
// LegacyDaemonIDs lists prior hostname-derived daemon_ids this machine
|
||
// may have registered under before switching to a persistent UUID. The
|
||
// handler merges any matching runtime rows into the new row so agents
|
||
// and tasks keep working without manual intervention.
|
||
LegacyDaemonIDs []string `json:"legacy_daemon_ids"`
|
||
DeviceName string `json:"device_name"`
|
||
CLIVersion string `json:"cli_version"` // multica CLI version
|
||
LaunchedBy string `json:"launched_by"` // "desktop" when spawned by the Electron app
|
||
Runtimes []struct {
|
||
Name string `json:"name"`
|
||
Type string `json:"type"`
|
||
Version string `json:"version"` // agent CLI version (claude/codex)
|
||
Status string `json:"status"`
|
||
// ProfileID, when non-empty, marks this as an instance of a custom
|
||
// runtime_profile (MUL-3284). Empty = built-in runtime (legacy path).
|
||
// Type carries the protocol family for both built-in and custom rows
|
||
// so task routing (agent.New) is unchanged.
|
||
ProfileID string `json:"profile_id"`
|
||
} `json:"runtimes"`
|
||
FailedProfiles []struct {
|
||
ProfileID string `json:"profile_id"`
|
||
CommandName string `json:"command_name"`
|
||
Reason string `json:"reason"`
|
||
} `json:"failed_profiles"`
|
||
}
|
||
|
||
type daemonWorkspaceReposResponse struct {
|
||
WorkspaceID string `json:"workspace_id"`
|
||
Repos []RepoData `json:"repos"`
|
||
ReposVersion string `json:"repos_version"`
|
||
Settings json.RawMessage `json:"settings,omitempty"`
|
||
}
|
||
|
||
func normalizeWorkspaceRepos(repos []RepoData) []RepoData {
|
||
if len(repos) == 0 {
|
||
return []RepoData{}
|
||
}
|
||
|
||
normalized := make([]RepoData, 0, len(repos))
|
||
seen := make(map[string]struct{}, len(repos))
|
||
for _, repo := range repos {
|
||
url := strings.TrimSpace(repo.URL)
|
||
if url == "" {
|
||
continue
|
||
}
|
||
if _, exists := seen[url]; exists {
|
||
continue
|
||
}
|
||
seen[url] = struct{}{}
|
||
normalized = append(normalized, RepoData{URL: url, Description: repo.Description})
|
||
}
|
||
return normalized
|
||
}
|
||
|
||
func workspaceReposVersion(repos []RepoData) string {
|
||
urls := make([]string, 0, len(repos))
|
||
for _, repo := range repos {
|
||
if repo.URL == "" {
|
||
continue
|
||
}
|
||
urls = append(urls, repo.URL)
|
||
}
|
||
sort.Strings(urls)
|
||
sum := sha256.Sum256([]byte(strings.Join(urls, "\n")))
|
||
return hex.EncodeToString(sum[:])
|
||
}
|
||
|
||
func parseWorkspaceRepos(raw []byte) []RepoData {
|
||
if len(raw) == 0 {
|
||
return []RepoData{}
|
||
}
|
||
|
||
var repos []RepoData
|
||
if err := json.Unmarshal(raw, &repos); err != nil {
|
||
return []RepoData{}
|
||
}
|
||
return normalizeWorkspaceRepos(repos)
|
||
}
|
||
|
||
func workspaceReposResponse(workspaceID string, raw []byte, settingsRaw []byte) daemonWorkspaceReposResponse {
|
||
repos := parseWorkspaceRepos(raw)
|
||
resp := daemonWorkspaceReposResponse{
|
||
WorkspaceID: workspaceID,
|
||
Repos: repos,
|
||
ReposVersion: workspaceReposVersion(repos),
|
||
}
|
||
if len(settingsRaw) > 0 {
|
||
resp.Settings = json.RawMessage(settingsRaw)
|
||
}
|
||
return resp
|
||
}
|
||
|
||
// normalizeProvider canonicalizes a provider string for storage: trimmed and
|
||
// lowercased so client-side pricing lookups tolerate case drift. Returns "" for
|
||
// a blank input.
|
||
func normalizeProvider(s string) string {
|
||
return strings.ToLower(strings.TrimSpace(s))
|
||
}
|
||
|
||
// inheritMachineCustomName gives a freshly-inserted runtime the machine's
|
||
// shared custom name (MUL-4217) when the machine is already named, so adding a
|
||
// provider — or recording a failed custom-runtime profile — on a named machine
|
||
// doesn't leave a custom_name = NULL row that makes the machine title revert to
|
||
// its hostname. Both the normal runtime path and the failed-profile path write
|
||
// daemon_id-scoped rows that show up in the machine grouping, so both call this.
|
||
//
|
||
// Only fresh inserts with no name of their own and a daemon_id participate;
|
||
// existing rows keep whatever they already have. A lookup/update error is
|
||
// non-fatal — registration must still succeed — so the input row is returned
|
||
// unchanged on any failure.
|
||
func (h *Handler) inheritMachineCustomName(ctx context.Context, rt db.AgentRuntime, inserted bool) db.AgentRuntime {
|
||
if !inserted || rt.CustomName.Valid || !rt.DaemonID.Valid {
|
||
return rt
|
||
}
|
||
names, err := h.Queries.ListDaemonCustomNames(ctx, db.ListDaemonCustomNamesParams{
|
||
WorkspaceID: rt.WorkspaceID,
|
||
DaemonID: rt.DaemonID,
|
||
ExcludeID: rt.ID,
|
||
})
|
||
if err != nil {
|
||
return rt
|
||
}
|
||
shared, ok := sharedDaemonCustomName(names)
|
||
if !ok {
|
||
return rt
|
||
}
|
||
updated, err := h.Queries.UpdateAgentRuntimeCustomName(ctx, db.UpdateAgentRuntimeCustomNameParams{
|
||
CustomName: pgtype.Text{String: shared, Valid: true},
|
||
ID: rt.ID,
|
||
})
|
||
if err != nil {
|
||
return rt
|
||
}
|
||
return updated
|
||
}
|
||
|
||
var errRuntimeProfileDisabled = errors.New("runtime profile is disabled")
|
||
|
||
// upsertRuntimeWithProfile serializes custom-runtime registration with profile
|
||
// deletion. The profile row remains KEY SHARE locked until the runtime upsert
|
||
// commits; DeleteRuntimeProfile takes a conflicting UPDATE lock before it
|
||
// enumerates runtime rows. This closes the stale-read window where deletion
|
||
// could miss an instance inserted by a concurrently registering daemon.
|
||
func (h *Handler) upsertRuntimeWithProfile(
|
||
ctx context.Context,
|
||
workspaceID, profileID pgtype.UUID,
|
||
build func(db.RuntimeProfile) db.UpsertAgentRuntimeWithProfileParams,
|
||
) (db.UpsertAgentRuntimeWithProfileRow, db.RuntimeProfile, error) {
|
||
var row db.UpsertAgentRuntimeWithProfileRow
|
||
var profile db.RuntimeProfile
|
||
|
||
tx, err := h.TxStarter.Begin(ctx)
|
||
if err != nil {
|
||
return row, profile, fmt.Errorf("begin profile runtime registration: %w", err)
|
||
}
|
||
defer tx.Rollback(ctx)
|
||
qtx := h.Queries.WithTx(tx)
|
||
|
||
profile, err = qtx.LockRuntimeProfileForRegistration(ctx, db.LockRuntimeProfileForRegistrationParams{
|
||
ID: profileID,
|
||
WorkspaceID: workspaceID,
|
||
})
|
||
if err != nil {
|
||
return row, profile, fmt.Errorf("lock runtime profile: %w", err)
|
||
}
|
||
if !profile.Enabled {
|
||
return row, profile, errRuntimeProfileDisabled
|
||
}
|
||
|
||
row, err = qtx.UpsertAgentRuntimeWithProfile(ctx, build(profile))
|
||
if err != nil {
|
||
return row, profile, fmt.Errorf("upsert profile runtime: %w", err)
|
||
}
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return row, profile, fmt.Errorf("commit profile runtime registration: %w", err)
|
||
}
|
||
return row, profile, nil
|
||
}
|
||
|
||
// sharedDaemonCustomName returns the machine-level name shared by all of a
|
||
// daemon's runtimes — the same rule the frontend's sharedCustomName applies:
|
||
// every runtime must carry the identical non-empty custom_name. Returns
|
||
// ("", false) when the set is empty, any runtime is unnamed, or the names
|
||
// disagree (i.e. there is no single machine name to inherit).
|
||
func sharedDaemonCustomName(names []pgtype.Text) (string, bool) {
|
||
if len(names) == 0 {
|
||
return "", false
|
||
}
|
||
var first string
|
||
for i, n := range names {
|
||
if !n.Valid {
|
||
return "", false
|
||
}
|
||
v := strings.TrimSpace(n.String)
|
||
if v == "" {
|
||
return "", false
|
||
}
|
||
if i == 0 {
|
||
first = v
|
||
} else if v != first {
|
||
return "", false
|
||
}
|
||
}
|
||
return first, true
|
||
}
|
||
|
||
func (h *Handler) DaemonRegister(w http.ResponseWriter, r *http.Request) {
|
||
var req DaemonRegisterRequest
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
|
||
req.WorkspaceID = strings.TrimSpace(req.WorkspaceID)
|
||
req.DaemonID = strings.TrimSpace(req.DaemonID)
|
||
req.DeviceName = strings.TrimSpace(req.DeviceName)
|
||
|
||
if req.DaemonID == "" {
|
||
writeError(w, http.StatusBadRequest, "daemon_id is required")
|
||
return
|
||
}
|
||
if req.WorkspaceID == "" {
|
||
writeError(w, http.StatusBadRequest, "workspace_id is required")
|
||
return
|
||
}
|
||
if len(req.Runtimes) == 0 && len(req.FailedProfiles) == 0 {
|
||
writeError(w, http.StatusBadRequest, "at least one runtime or failed profile is required")
|
||
return
|
||
}
|
||
wsUUID, ok := parseUUIDOrBadRequest(w, req.WorkspaceID, "workspace_id")
|
||
if !ok {
|
||
return
|
||
}
|
||
req.WorkspaceID = uuidToString(wsUUID)
|
||
|
||
// Verify workspace access and resolve owner.
|
||
// Daemon tokens (mdt_) prove workspace access directly; OwnerID will be zero
|
||
// (the SQL COALESCE preserves any existing owner on upsert).
|
||
// PAT/JWT tokens require a membership check and set OwnerID from the member.
|
||
var ownerID pgtype.UUID
|
||
if daemonWsID := middleware.DaemonWorkspaceIDFromContext(r.Context()); daemonWsID != "" {
|
||
if daemonWsID != req.WorkspaceID {
|
||
writeError(w, http.StatusNotFound, "workspace not found")
|
||
return
|
||
}
|
||
// ownerID stays zero — COALESCE keeps the existing owner on upsert.
|
||
} else {
|
||
member, ok := h.requireWorkspaceMember(w, r, req.WorkspaceID, "workspace not found")
|
||
if !ok {
|
||
return
|
||
}
|
||
ownerID = member.UserID
|
||
}
|
||
|
||
ws, err := h.Queries.GetWorkspace(r.Context(), wsUUID)
|
||
if err != nil {
|
||
writeError(w, http.StatusNotFound, "workspace not found")
|
||
return
|
||
}
|
||
|
||
resp := make([]AgentRuntimeResponse, 0, len(req.Runtimes))
|
||
for _, runtime := range req.Runtimes {
|
||
provider := normalizeProvider(runtime.Type)
|
||
if provider == "" {
|
||
provider = "unknown"
|
||
}
|
||
name := strings.TrimSpace(runtime.Name)
|
||
if name == "" {
|
||
name = provider
|
||
if req.DeviceName != "" {
|
||
name = fmt.Sprintf("%s (%s)", provider, req.DeviceName)
|
||
}
|
||
}
|
||
deviceInfo := strings.TrimSpace(req.DeviceName)
|
||
if runtime.Version != "" && deviceInfo != "" {
|
||
deviceInfo = fmt.Sprintf("%s · %s", deviceInfo, runtime.Version)
|
||
} else if runtime.Version != "" {
|
||
deviceInfo = runtime.Version
|
||
}
|
||
status := "online"
|
||
if runtime.Status == "offline" {
|
||
status = "offline"
|
||
}
|
||
// Persist what this daemon advertised, so surfaces that cannot see the
|
||
// live request — the resource-save gate and the UI — decide from the
|
||
// same signal the claim path uses, instead of re-deriving it from a
|
||
// version string (MUL-5707).
|
||
metadata, _ := json.Marshal(map[string]any{
|
||
"version": runtime.Version,
|
||
"cli_version": req.CLIVersion,
|
||
"launched_by": req.LaunchedBy,
|
||
"capabilities": requestClientCapabilities(r),
|
||
})
|
||
|
||
var registered db.AgentRuntime
|
||
var inserted bool
|
||
isCustom := strings.TrimSpace(runtime.ProfileID) != ""
|
||
|
||
if isCustom {
|
||
profileUUID, pok := parseUUIDOrBadRequest(w, strings.TrimSpace(runtime.ProfileID), "profile_id")
|
||
if !pok {
|
||
return
|
||
}
|
||
// The profile must exist in this workspace and be enabled. Trust
|
||
// the profile's stored runtime identity over the daemon-sent type so
|
||
// the provider used for task routing cannot drift from the profile.
|
||
prow, profile, err := h.upsertRuntimeWithProfile(
|
||
r.Context(),
|
||
wsUUID,
|
||
profileUUID,
|
||
func(profile db.RuntimeProfile) db.UpsertAgentRuntimeWithProfileParams {
|
||
return db.UpsertAgentRuntimeWithProfileParams{
|
||
WorkspaceID: wsUUID,
|
||
DaemonID: strToText(req.DaemonID),
|
||
Name: name,
|
||
RuntimeMode: "local",
|
||
Provider: agent.ProfileRuntimeType(profile.RuntimeType, profile.ProtocolFamily),
|
||
Status: status,
|
||
DeviceInfo: deviceInfo,
|
||
Metadata: metadata,
|
||
OwnerID: ownerID,
|
||
ProfileID: profileUUID,
|
||
}
|
||
},
|
||
)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
writeError(w, http.StatusBadRequest, "unknown runtime profile: "+runtime.ProfileID)
|
||
return
|
||
}
|
||
if errors.Is(err, errRuntimeProfileDisabled) {
|
||
writeError(w, http.StatusConflict, "runtime profile is disabled: "+runtime.ProfileID)
|
||
return
|
||
}
|
||
if err != nil {
|
||
obsmetrics.RecordEvent(h.Analytics, h.Metrics, analytics.RuntimeFailed(
|
||
uuidToString(ownerID),
|
||
req.WorkspaceID,
|
||
req.DaemonID,
|
||
provider,
|
||
"registration_failed",
|
||
"db_error",
|
||
true,
|
||
))
|
||
slog.Warn("register runtime failed", append(logger.RequestAttrs(r), "error", err)...)
|
||
writeError(w, http.StatusInternalServerError, "failed to register runtime")
|
||
return
|
||
}
|
||
provider = agent.ProfileRuntimeType(profile.RuntimeType, profile.ProtocolFamily)
|
||
inserted = prow.Inserted
|
||
registered = db.AgentRuntime{
|
||
ID: prow.ID,
|
||
WorkspaceID: prow.WorkspaceID,
|
||
DaemonID: prow.DaemonID,
|
||
Name: prow.Name,
|
||
CustomName: prow.CustomName,
|
||
RuntimeMode: prow.RuntimeMode,
|
||
Provider: prow.Provider,
|
||
Status: prow.Status,
|
||
DeviceInfo: prow.DeviceInfo,
|
||
Metadata: prow.Metadata,
|
||
LastSeenAt: prow.LastSeenAt,
|
||
CreatedAt: prow.CreatedAt,
|
||
UpdatedAt: prow.UpdatedAt,
|
||
OwnerID: prow.OwnerID,
|
||
LegacyDaemonID: prow.LegacyDaemonID,
|
||
Visibility: prow.Visibility,
|
||
ProfileID: prow.ProfileID,
|
||
}
|
||
} else {
|
||
row, err := h.Queries.UpsertAgentRuntime(r.Context(), db.UpsertAgentRuntimeParams{
|
||
WorkspaceID: wsUUID,
|
||
DaemonID: strToText(req.DaemonID),
|
||
Name: name,
|
||
RuntimeMode: "local",
|
||
Provider: provider,
|
||
Status: status,
|
||
DeviceInfo: deviceInfo,
|
||
Metadata: metadata,
|
||
OwnerID: ownerID,
|
||
})
|
||
if err != nil {
|
||
obsmetrics.RecordEvent(h.Analytics, h.Metrics, analytics.RuntimeFailed(
|
||
uuidToString(ownerID),
|
||
req.WorkspaceID,
|
||
req.DaemonID,
|
||
provider,
|
||
"registration_failed",
|
||
"db_error",
|
||
true,
|
||
))
|
||
slog.Warn("register runtime failed", append(logger.RequestAttrs(r), "error", err)...)
|
||
writeError(w, http.StatusInternalServerError, "failed to register runtime")
|
||
return
|
||
}
|
||
inserted = row.Inserted
|
||
registered = db.AgentRuntime{
|
||
ID: row.ID,
|
||
WorkspaceID: row.WorkspaceID,
|
||
DaemonID: row.DaemonID,
|
||
Name: row.Name,
|
||
CustomName: row.CustomName,
|
||
RuntimeMode: row.RuntimeMode,
|
||
Provider: row.Provider,
|
||
Status: row.Status,
|
||
DeviceInfo: row.DeviceInfo,
|
||
Metadata: row.Metadata,
|
||
LastSeenAt: row.LastSeenAt,
|
||
CreatedAt: row.CreatedAt,
|
||
UpdatedAt: row.UpdatedAt,
|
||
OwnerID: row.OwnerID,
|
||
LegacyDaemonID: row.LegacyDaemonID,
|
||
Visibility: row.Visibility,
|
||
ProfileID: row.ProfileID,
|
||
}
|
||
}
|
||
|
||
// A brand-new runtime on an already-named machine inherits the machine's
|
||
// shared custom name so the machine title stays stable as providers come
|
||
// and go (MUL-4217). Shared with the failed-profile path below.
|
||
registered = h.inheritMachineCustomName(r.Context(), registered, inserted)
|
||
|
||
// Inserted is false for normal daemon reconnects/upserts, so
|
||
// runtime_ready is a first-ready-per-runtime-row signal.
|
||
if inserted {
|
||
obsmetrics.RecordEvent(h.Analytics, h.Metrics, analytics.RuntimeRegistered(
|
||
uuidToString(ownerID),
|
||
req.WorkspaceID,
|
||
uuidToString(registered.ID),
|
||
req.DaemonID,
|
||
provider,
|
||
runtime.Version,
|
||
req.CLIVersion,
|
||
))
|
||
if registered.Status == "online" {
|
||
obsmetrics.RecordEvent(h.Analytics, h.Metrics, analytics.RuntimeReady(
|
||
uuidToString(ownerID),
|
||
req.WorkspaceID,
|
||
uuidToString(registered.ID),
|
||
req.DaemonID,
|
||
provider,
|
||
0,
|
||
))
|
||
}
|
||
}
|
||
|
||
// Seamless migration from the previous hostname-derived identity. The
|
||
// daemon sends every legacy daemon_id it may have registered under
|
||
// (e.g. "host.local", "host", "host-staging"); for each match we
|
||
// reassign agents + tasks onto the new UUID-keyed row, then delete
|
||
// the stale row so there's only ever one runtime per machine.
|
||
//
|
||
// Only built-in runtimes participate: legacy rows predate custom
|
||
// profiles, so a profile-keyed instance never has a hostname-derived
|
||
// ancestor to merge, and mergeLegacyRuntimes scopes by provider alone
|
||
// (no profile_id), which could otherwise fold a built-in row into a
|
||
// custom one of the same provider.
|
||
if !isCustom {
|
||
h.mergeLegacyRuntimes(r, registered, provider, req.LegacyDaemonIDs)
|
||
}
|
||
|
||
resp = append(resp, runtimeToResponse(registered))
|
||
}
|
||
for _, failed := range req.FailedProfiles {
|
||
profileID := strings.TrimSpace(failed.ProfileID)
|
||
if profileID == "" {
|
||
continue
|
||
}
|
||
profileUUID, pok := parseUUIDOrBadRequest(w, profileID, "profile_id")
|
||
if !pok {
|
||
return
|
||
}
|
||
reason := strings.TrimSpace(failed.Reason)
|
||
if reason == "" {
|
||
reason = "custom runtime command could not be resolved"
|
||
}
|
||
commandName := strings.TrimSpace(failed.CommandName)
|
||
prow, _, err := h.upsertRuntimeWithProfile(
|
||
r.Context(),
|
||
wsUUID,
|
||
profileUUID,
|
||
func(profile db.RuntimeProfile) db.UpsertAgentRuntimeWithProfileParams {
|
||
name := profile.DisplayName
|
||
if req.DeviceName != "" {
|
||
name = fmt.Sprintf("%s (%s)", name, req.DeviceName)
|
||
}
|
||
resolvedCommandName := commandName
|
||
if resolvedCommandName == "" {
|
||
resolvedCommandName = profile.CommandName
|
||
}
|
||
// Capabilities belong on this row too. It is written AFTER the
|
||
// successful runtimes in the same request, so its last_seen_at is
|
||
// the newest for this daemon until the first heartbeat — and the
|
||
// worktree gates read the newest row. Omitting them made a failed
|
||
// profile look, for that window, like a daemon that advertises
|
||
// nothing (and now like a capability-blind server).
|
||
metadata, _ := json.Marshal(map[string]any{
|
||
"version": "",
|
||
"cli_version": req.CLIVersion,
|
||
"launched_by": req.LaunchedBy,
|
||
"capabilities": requestClientCapabilities(r),
|
||
"runtime_profile_registration_error": true,
|
||
"runtime_profile_failure_reason": reason,
|
||
"command_name": resolvedCommandName,
|
||
})
|
||
return db.UpsertAgentRuntimeWithProfileParams{
|
||
WorkspaceID: wsUUID,
|
||
DaemonID: strToText(req.DaemonID),
|
||
Name: name,
|
||
RuntimeMode: "local",
|
||
Provider: agent.ProfileRuntimeType(profile.RuntimeType, profile.ProtocolFamily),
|
||
Status: "offline",
|
||
DeviceInfo: strings.TrimSpace(req.DeviceName),
|
||
Metadata: metadata,
|
||
OwnerID: ownerID,
|
||
ProfileID: profileUUID,
|
||
}
|
||
},
|
||
)
|
||
if err != nil {
|
||
slog.Warn("failed to record runtime profile registration failure",
|
||
"workspace_id", req.WorkspaceID, "daemon_id", req.DaemonID,
|
||
"profile_id", profileID, "error", err)
|
||
continue
|
||
}
|
||
// Keep the failed-profile row consistent with the machine's name so it
|
||
// doesn't drag the machine title back to the hostname (MUL-4217).
|
||
h.inheritMachineCustomName(r.Context(), db.AgentRuntime{
|
||
ID: prow.ID,
|
||
WorkspaceID: prow.WorkspaceID,
|
||
DaemonID: prow.DaemonID,
|
||
CustomName: prow.CustomName,
|
||
}, prow.Inserted)
|
||
}
|
||
|
||
slog.Info("daemon registered", "workspace_id", req.WorkspaceID, "daemon_id", req.DaemonID, "runtimes_count", len(resp))
|
||
|
||
h.publish(protocol.EventDaemonRegister, req.WorkspaceID, "system", "", map[string]any{
|
||
"runtimes": resp,
|
||
})
|
||
|
||
repoResp := workspaceReposResponse(req.WorkspaceID, ws.Repos, ws.Settings)
|
||
|
||
writeJSON(w, http.StatusOK, map[string]any{
|
||
"runtimes": resp,
|
||
"repos": repoResp.Repos,
|
||
"repos_version": repoResp.ReposVersion,
|
||
"settings": repoResp.Settings,
|
||
})
|
||
}
|
||
|
||
// mergeLegacyRuntimes folds every runtime row keyed on a prior hostname-derived
|
||
// daemon_id into the newly registered UUID-keyed row. For each legacy id the
|
||
// lookup is case-insensitive and returns *all* matching rows — case-only drift
|
||
// may have already minted duplicates historically (e.g. `Foo.local` AND
|
||
// `foo.local` coexisting), and we need to consolidate every one of them, not
|
||
// just the first. Per match we reassign agents and tasks, record the legacy
|
||
// id on the new row for audit, then delete the stale row.
|
||
//
|
||
// Scoping by (workspace_id, provider) is sufficient since provider is single-
|
||
// runtime-per-daemon; `unique (workspace_id, daemon_id, provider)` prevents
|
||
// any two *exact* matches but the `LOWER(...)` comparison crosses that bound
|
||
// precisely when case-duplicate rows exist — which is the bug we're fixing.
|
||
// We also dedupe across legacy ids so overlapping candidates (e.g. `foo` and
|
||
// `foo.local` both resolving to the same stored row) don't double-process.
|
||
func (h *Handler) mergeLegacyRuntimes(r *http.Request, registered db.AgentRuntime, provider string, legacyIDs []string) {
|
||
newID := uuidToString(registered.ID)
|
||
merged := make(map[string]struct{})
|
||
|
||
for _, legacyID := range legacyIDs {
|
||
legacyID = strings.TrimSpace(legacyID)
|
||
if legacyID == "" {
|
||
continue
|
||
}
|
||
|
||
matches, err := h.Queries.FindLegacyRuntimesByDaemonID(r.Context(), db.FindLegacyRuntimesByDaemonIDParams{
|
||
WorkspaceID: registered.WorkspaceID,
|
||
Provider: provider,
|
||
DaemonID: legacyID,
|
||
})
|
||
if err != nil {
|
||
slog.Warn("legacy runtime merge: lookup failed", "legacy_daemon_id", legacyID, "error", err)
|
||
continue
|
||
}
|
||
for _, old := range matches {
|
||
oldID := uuidToString(old.ID)
|
||
if oldID == newID {
|
||
continue
|
||
}
|
||
if _, seen := merged[oldID]; seen {
|
||
continue
|
||
}
|
||
merged[oldID] = struct{}{}
|
||
|
||
if err := h.mergeLegacyRuntime(r.Context(), registered.ID, old.ID, legacyID, provider); err != nil {
|
||
slog.Warn("legacy runtime merge failed",
|
||
"legacy_daemon_id", legacyID, "old_runtime_id", oldID, "new_runtime_id", newID, "error", err)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// errRuntimeMergeFenced reports that the task-write fence refused the
|
||
// reassignment because the target runtime's workspace is being deleted or is
|
||
// already gone. The merge is abandoned, not retried in place: the next daemon
|
||
// registration re-runs it, and by then either the workspace is gone entirely or
|
||
// the teardown rolled back.
|
||
var errRuntimeMergeFenced = errors.New("runtime merge refused by the task-write fence")
|
||
|
||
// mergeLegacyRuntime folds one legacy runtime row into the freshly registered one,
|
||
// as a single transaction.
|
||
//
|
||
// Order is load-bearing. The fence comes first — workspace row, then both runtime
|
||
// rows FOR UPDATE — and tasks move next, because that statement carries the same
|
||
// fence predicate every task write uses and is the only step that can tell us the
|
||
// target workspace is being torn down. Its verdict is separate from its row
|
||
// count for exactly this reason: "0 tasks reassigned" is also what a legacy runtime
|
||
// with no history looks like, and treating a fenced refusal as success would take
|
||
// us straight to DeleteAgentRuntime below — where agent_task_queue's
|
||
// ON DELETE CASCADE would delete the history this merge is supposed to carry
|
||
// forward (MUL-5999 review).
|
||
//
|
||
// One transaction, because the fence's workspace lock lives only as long as the
|
||
// statement that took it. Run as four autocommit statements, the merge had two
|
||
// ways to leave the tenant inconsistent and the registration reporting success:
|
||
//
|
||
// - a transient failure on the agent reassignment left tasks pointing at the new
|
||
// runtime while their agents stayed on the old one, self-healing only on some
|
||
// later registration;
|
||
// - worse, in the gap after the task commit a workspace teardown could lock and
|
||
// delete the target runtime, whose UnbindTasksFromRuntime step clears
|
||
// runtime_id on the tasks that had just moved. The next merge looks for tasks
|
||
// by the OLD runtime id, so that history was permanently detached.
|
||
//
|
||
// Holding the fence lock from the first statement to COMMIT closes both: the
|
||
// teardown cannot start deleting the target until this merge finishes, and any
|
||
// failure rolls the whole merge back to its starting state.
|
||
func (h *Handler) mergeLegacyRuntime(ctx context.Context, newRuntimeID, oldRuntimeID pgtype.UUID, legacyID, provider string) error {
|
||
tx, err := h.TxStarter.Begin(ctx)
|
||
if err != nil {
|
||
return fmt.Errorf("begin merge tx: %w", err)
|
||
}
|
||
defer tx.Rollback(ctx)
|
||
qtx := h.Queries.WithTx(tx)
|
||
|
||
// Fence, in the same order every task write uses: workspace row first, then the
|
||
// owner rows. Without the FOR UPDATE on the runtimes, a concurrent enqueue
|
||
// against the OLD runtime slipped through after the task scan — writers hold
|
||
// only FOR KEY SHARE on the workspace, and so did this merge, and two KEY SHARE
|
||
// locks do not conflict — and DeleteAgentRuntime below then removed that
|
||
// brand-new task through ON DELETE CASCADE.
|
||
runtimeIDs := []pgtype.UUID{oldRuntimeID, newRuntimeID}
|
||
if err := qtx.LockWorkspaceForRuntimeMerge(ctx, runtimeIDs); err != nil {
|
||
return fmt.Errorf("lock workspace: %w", err)
|
||
}
|
||
locked, err := qtx.LockRuntimesForMerge(ctx, runtimeIDs)
|
||
if err != nil {
|
||
return fmt.Errorf("lock runtimes: %w", err)
|
||
}
|
||
if len(locked) != len(runtimeIDs) {
|
||
// One of them went away while we were queueing for the lock; the fence
|
||
// inside the reassignment would refuse anyway.
|
||
return errRuntimeMergeFenced
|
||
}
|
||
|
||
reassignment, err := qtx.ReassignTasksToRuntime(ctx, db.ReassignTasksToRuntimeParams{
|
||
NewRuntimeID: newRuntimeID,
|
||
OldRuntimeID: oldRuntimeID,
|
||
})
|
||
if err != nil {
|
||
return fmt.Errorf("reassign tasks: %w", err)
|
||
}
|
||
if !reassignment.FenceOk {
|
||
return errRuntimeMergeFenced
|
||
}
|
||
|
||
agents, err := qtx.ReassignAgentsToRuntime(ctx, db.ReassignAgentsToRuntimeParams{
|
||
NewRuntimeID: newRuntimeID,
|
||
OldRuntimeID: oldRuntimeID,
|
||
})
|
||
if err != nil {
|
||
return fmt.Errorf("reassign agents: %w", err)
|
||
}
|
||
|
||
// Inside the transaction this can no longer be best-effort: a failed statement
|
||
// poisons the transaction, so it either lands with the rest of the merge or the
|
||
// merge is rolled back.
|
||
if err := qtx.RecordRuntimeLegacyDaemonID(ctx, db.RecordRuntimeLegacyDaemonIDParams{
|
||
ID: newRuntimeID,
|
||
LegacyDaemonID: strToText(legacyID),
|
||
}); err != nil {
|
||
return fmt.Errorf("record legacy daemon_id: %w", err)
|
||
}
|
||
if err := qtx.DeleteAgentRuntime(ctx, oldRuntimeID); err != nil {
|
||
return fmt.Errorf("delete old runtime: %w", err)
|
||
}
|
||
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return fmt.Errorf("commit merge: %w", err)
|
||
}
|
||
h.NotifyRuntimeGone(uuidToString(oldRuntimeID))
|
||
|
||
slog.Info("legacy runtime merged",
|
||
"legacy_daemon_id", legacyID,
|
||
"old_runtime_id", uuidToString(oldRuntimeID),
|
||
"new_runtime_id", uuidToString(newRuntimeID),
|
||
"provider", provider,
|
||
"agents_reassigned", agents,
|
||
"tasks_reassigned", reassignment.ReassignedTasks,
|
||
)
|
||
return nil
|
||
}
|
||
|
||
func (h *Handler) GetDaemonWorkspaceRepos(w http.ResponseWriter, r *http.Request) {
|
||
workspaceID := strings.TrimSpace(chi.URLParam(r, "workspaceId"))
|
||
if !h.requireDaemonWorkspaceAccess(w, r, workspaceID) {
|
||
return
|
||
}
|
||
|
||
ws, err := h.Queries.GetWorkspace(r.Context(), parseUUID(workspaceID))
|
||
if err != nil {
|
||
writeError(w, http.StatusNotFound, "workspace not found")
|
||
return
|
||
}
|
||
|
||
writeJSON(w, http.StatusOK, workspaceReposResponse(workspaceID, ws.Repos, ws.Settings))
|
||
}
|
||
|
||
// setRuntimeOffline flips a runtime offline, recording the daemon's reason when
|
||
// it sent one. An absent or unusable reason falls back to the plain offline
|
||
// write: a malformed payload must never cost the caller the state change it
|
||
// actually asked for.
|
||
func (h *Handler) setRuntimeOffline(ctx context.Context, runtimeID pgtype.UUID, reason json.RawMessage) error {
|
||
if len(reason) > 0 && json.Valid(reason) {
|
||
return h.Queries.SetAgentRuntimeOfflineWithReason(ctx, db.SetAgentRuntimeOfflineWithReasonParams{
|
||
ID: runtimeID,
|
||
OfflineReason: reason,
|
||
})
|
||
}
|
||
return h.Queries.SetAgentRuntimeOffline(ctx, runtimeID)
|
||
}
|
||
|
||
// DaemonDeregister marks runtimes as offline when the daemon shuts down.
|
||
func (h *Handler) DaemonDeregister(w http.ResponseWriter, r *http.Request) {
|
||
var req struct {
|
||
RuntimeIDs []string `json:"runtime_ids"`
|
||
// OfflineReasons is optional and keyed by runtime id. Present only for
|
||
// causes the user must repair — a daemon shutting down sends none, so an
|
||
// older daemon simply keeps today's behaviour (MUL-6164).
|
||
OfflineReasons map[string]json.RawMessage `json:"offline_reasons"`
|
||
}
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
|
||
if len(req.RuntimeIDs) == 0 {
|
||
writeError(w, http.StatusBadRequest, "runtime_ids is required")
|
||
return
|
||
}
|
||
runtimeUUIDs, ok := parseUUIDSliceOrBadRequest(w, req.RuntimeIDs, "runtime_ids")
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
// Track affected workspaces for WS notifications.
|
||
affectedWorkspaces := make(map[string]bool)
|
||
|
||
// Batch the runtime lookups instead of one GetAgentRuntime per id (N+1),
|
||
// while keeping the MUL-6884 per-source attribution: getAgentRuntimes
|
||
// records one multica_agent_runtime_lookup_total result per requested id.
|
||
// A read error is NOT "the rows don't exist": fail closed with 500 (like
|
||
// ListRuntimesForClaim) so a transient blip can't report a successful
|
||
// deregister while every runtime silently stays online until the liveness
|
||
// sweep reaps it. GetMany keys rows by canonical UUID string so a
|
||
// differently-cased request id still matches; a genuinely missing id is
|
||
// simply absent from the map and falls through to the "runtime not found"
|
||
// skip below, preserving the per-runtime verify / setOffline semantics.
|
||
runtimesByID, err := h.getAgentRuntimes(r.Context(), obsmetrics.RuntimeLookupSourceDaemonAPI, runtimeUUIDs)
|
||
if err != nil {
|
||
slog.Error("deregister: batch runtime lookup failed", "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to load runtimes")
|
||
return
|
||
}
|
||
|
||
for i, rid := range req.RuntimeIDs {
|
||
// Look up the runtime and verify ownership. Key by the parsed UUID's
|
||
// canonical form (runtimeUUIDs[i]) so a differently-cased request id
|
||
// still matches the row; rid is kept for OfflineReasons and logging.
|
||
rt, ok := runtimesByID[uuidToString(runtimeUUIDs[i])]
|
||
if !ok {
|
||
slog.Warn("deregister: runtime not found", "runtime_id", rid)
|
||
continue
|
||
}
|
||
|
||
wsID := uuidToString(rt.WorkspaceID)
|
||
if !h.verifyDaemonWorkspaceAccess(r, wsID) {
|
||
slog.Warn("deregister: workspace mismatch", "runtime_id", rid)
|
||
continue
|
||
}
|
||
|
||
if err := h.setRuntimeOffline(r.Context(), rt.ID, req.OfflineReasons[rid]); err != nil {
|
||
slog.Warn("deregister: failed to set offline", "runtime_id", rid, "error", err)
|
||
continue
|
||
}
|
||
obsmetrics.RecordEvent(h.Analytics, h.Metrics, analytics.RuntimeOffline(
|
||
uuidToString(rt.OwnerID),
|
||
wsID,
|
||
uuidToString(rt.ID),
|
||
rt.DaemonID.String,
|
||
rt.Provider,
|
||
))
|
||
|
||
affectedWorkspaces[wsID] = true
|
||
}
|
||
|
||
// Notify frontend clients so they re-fetch runtime list.
|
||
for wsID := range affectedWorkspaces {
|
||
h.publish(protocol.EventDaemonRegister, wsID, "system", "", map[string]any{
|
||
"action": "deregister",
|
||
})
|
||
}
|
||
|
||
slog.Info("daemon deregistered", "runtime_ids", req.RuntimeIDs)
|
||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||
}
|
||
|
||
type DaemonHeartbeatRequest struct {
|
||
RuntimeID string `json:"runtime_id"`
|
||
SupportsBatchImport bool `json:"supports_batch_import,omitempty"`
|
||
}
|
||
|
||
// heartbeatHasPendingTimeout bounds the cheap HasPending probe on the
|
||
// heartbeat hot path. Probes are read-only (ZCARD in Redis) so a timeout is
|
||
// ack-safe: the worst case is "we didn't find out if anything was queued this
|
||
// tick" and the next heartbeat (default 15s later) will try again.
|
||
//
|
||
// PopPending is deliberately NOT bounded this way — its Redis implementation
|
||
// runs a Lua claim script whose ZREM + SET-running side effects cannot be
|
||
// cleanly un-run from the client side if the context expires mid-script. We
|
||
// therefore only invoke PopPending after HasPending confirms there is work
|
||
// to claim, so we never start a claim we might have to abort.
|
||
// Kept as a variable so the slow-probe test can preserve this relationship
|
||
// without paying the production timeout; production never reassigns it.
|
||
var heartbeatHasPendingTimeout = 1 * time.Second
|
||
|
||
// maxLocalSkillImportBatch is how many pending import requests the heartbeat
|
||
// handler pops per cycle. Higher values let the daemon process more imports
|
||
// in parallel but increase per-heartbeat latency.
|
||
//
|
||
// Timeout invariant: IMPORT_CONCURRENCY (views/.../runtime-local-skill-import-panel.tsx)
|
||
// × heartbeat period (~15s) must stay within runtimeLocalSkillPendingTimeout
|
||
// (runtime_local_skills.go), and IMPORT_POLL_TIMEOUT_MS (core/runtimes/local-skills.ts)
|
||
// must exceed pendingTimeout + runningTimeout.
|
||
const maxLocalSkillImportBatch = 10
|
||
|
||
// runtimeLivenessTTL is how long a Redis liveness record stays valid before
|
||
// expiring. The daemon refreshes it every heartbeat (~15s), so this just
|
||
// needs to be a few heartbeats long — the value (90s) tolerates ~6 missed
|
||
// beats before Redis declares the runtime dead.
|
||
//
|
||
// It is intentionally shorter than the sweeper's stale threshold (150s in
|
||
// cmd/server/runtime_sweeper.go). That ordering is safe and desirable:
|
||
// Redis can declare a runtime dead before the DB stale window opens, and
|
||
// the sweeper will simply ignore it until the DB column also crosses the
|
||
// threshold. The unsafe direction would be the opposite (Redis claiming
|
||
// "alive" past the DB stale window, masking a truly dead runtime when the
|
||
// sweeper consults Redis as the source of truth) — that cannot happen here.
|
||
const runtimeLivenessTTL = 90 * time.Second
|
||
|
||
// runtimeHeartbeatDBFlushInterval is the maximum staleness we tolerate on
|
||
// agent_runtime.last_seen_at while Redis is the active liveness source. When
|
||
// last_seen_at gets older than this, the heartbeat path schedules a DB write
|
||
// so (a) the UI's "last seen" display stays bounded and (b) the sweeper's
|
||
// DB-only fallback path (used when an IsAliveBatch call to Redis errors) does
|
||
// not false-positive on alive-but-Redis-only runtimes.
|
||
//
|
||
// Load-bearing invariant: this must be strictly less than the sweeper's
|
||
// stale threshold (150s in cmd/server/runtime_sweeper.go) MINUS one daemon
|
||
// heartbeat cycle (~15s) MINUS the BatchedHeartbeatScheduler tick interval
|
||
// (~30s). Worst-case DB age for an alive runtime is therefore bounded by
|
||
// flush + heartbeat + batchTick = 60 + 15 + 30 = 105s, leaving a 45s buffer
|
||
// below the 150s stale window. If you tune any of these constants, recompute
|
||
// the chain and keep at least a one-tick buffer.
|
||
//
|
||
// We intentionally keep the per-runtime flush throttle at 60s (rather than
|
||
// pushing it higher) so a crashed runtime is detected within ~150s instead
|
||
// of ~10 minutes. The bulk of the DB-pressure win comes from batched
|
||
// coalescing in HeartbeatScheduler — at 70 online runtimes that collapses
|
||
// ~17 single-row UPDATE/s into ~0.03 bulk UPDATE/s (one per batch tick),
|
||
// independent of how the per-runtime throttle is tuned.
|
||
const runtimeHeartbeatDBFlushInterval = 60 * time.Second
|
||
|
||
func (h *Handler) DaemonHeartbeat(w http.ResponseWriter, r *http.Request) {
|
||
start := time.Now()
|
||
authPath := middleware.DaemonAuthPathFromContext(r.Context())
|
||
var (
|
||
outcome = "unauth"
|
||
runtimeID string
|
||
decodeMs, runtimeLookupMs, workspaceCheckMs int64
|
||
authMs, updateMs, probeModelMs, popModelMs, probeSkillsMs, popSkillsMs, probeImportMs, popImportMs int64
|
||
probeModelTimedOut, probeSkillsTimedOut, probeImportTimedOut bool
|
||
)
|
||
defer func() {
|
||
logHeartbeatEndpointSlow(runtimeID, outcome, authPath, start, decodeMs, runtimeLookupMs, workspaceCheckMs, authMs, updateMs, probeModelMs, popModelMs, probeSkillsMs, popSkillsMs, probeImportMs, popImportMs, probeModelTimedOut, probeSkillsTimedOut, probeImportTimedOut)
|
||
}()
|
||
|
||
decodeStart := time.Now()
|
||
var req DaemonHeartbeatRequest
|
||
decodeErr := json.NewDecoder(r.Body).Decode(&req)
|
||
decodeMs = time.Since(decodeStart).Milliseconds()
|
||
if decodeErr != nil {
|
||
outcome = "bad_body"
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
|
||
if req.RuntimeID == "" {
|
||
outcome = "missing_runtime_id"
|
||
writeError(w, http.StatusBadRequest, "runtime_id is required")
|
||
return
|
||
}
|
||
runtimeID = req.RuntimeID
|
||
|
||
// Inlined and instrumented version of requireDaemonRuntimeAccess so we
|
||
// can attribute the runtime-lookup and workspace-check sub-stages
|
||
// independently in slow-logs. Together with the auth_path label set by
|
||
// DaemonAuth middleware, this lets us tell whether prod heartbeat tail
|
||
// latency is in pgx pool acquisition (runtime_lookup_ms), in the PAT
|
||
// fallback workspace-membership query (workspace_check_ms), or upstream.
|
||
runtimeUUID, ok := parseUUIDOrBadRequest(w, req.RuntimeID, "runtime_id")
|
||
if !ok {
|
||
outcome = "bad_runtime_id"
|
||
return
|
||
}
|
||
lookupStart := time.Now()
|
||
rt, lookupErr := h.getAgentRuntime(r.Context(), obsmetrics.RuntimeLookupSourceHeartbeatHTTP, runtimeUUID)
|
||
runtimeLookupMs = time.Since(lookupStart).Milliseconds()
|
||
if lookupErr != nil {
|
||
// Only pgx.ErrNoRows means the runtime row is gone. Daemon reads this
|
||
// 404 as a signal to drop the stale runtime locally; treating a
|
||
// transient DB error the same way would force daemons to self-cleanup
|
||
// on a hiccup.
|
||
if isNotFound(lookupErr) {
|
||
outcome = "runtime_not_found"
|
||
writeError(w, http.StatusNotFound, "runtime not found")
|
||
return
|
||
}
|
||
outcome = "runtime_lookup_error"
|
||
slog.Warn("get agent runtime failed", "runtime_id", req.RuntimeID, "error", lookupErr)
|
||
writeError(w, http.StatusInternalServerError, "failed to load runtime")
|
||
return
|
||
}
|
||
wsCheckStart := time.Now()
|
||
wsOK := h.requireDaemonWorkspaceAccess(w, r, uuidToString(rt.WorkspaceID))
|
||
workspaceCheckMs = time.Since(wsCheckStart).Milliseconds()
|
||
if !wsOK {
|
||
outcome = "workspace_denied"
|
||
return
|
||
}
|
||
authMs = time.Since(start).Milliseconds()
|
||
|
||
updateStart := time.Now()
|
||
if err := h.recordHeartbeat(r.Context(), rt); err != nil {
|
||
updateMs = time.Since(updateStart).Milliseconds()
|
||
outcome = "error_update"
|
||
writeError(w, http.StatusInternalServerError, "heartbeat failed")
|
||
return
|
||
}
|
||
updateMs = time.Since(updateStart).Milliseconds()
|
||
|
||
ack, m, err := h.processHeartbeat(r.Context(), runtimeID, req.SupportsBatchImport)
|
||
probeModelMs = m.ProbeModelMs
|
||
popModelMs = m.PopModelMs
|
||
probeSkillsMs = m.ProbeSkillsMs
|
||
popSkillsMs = m.PopSkillsMs
|
||
probeImportMs = m.ProbeImportMs
|
||
popImportMs = m.PopImportMs
|
||
probeModelTimedOut = m.ProbeModelTimedOut
|
||
probeSkillsTimedOut = m.ProbeSkillsTimedOut
|
||
probeImportTimedOut = m.ProbeImportTimedOut
|
||
if err != nil {
|
||
outcome = "error_update"
|
||
writeError(w, http.StatusInternalServerError, "heartbeat failed")
|
||
return
|
||
}
|
||
|
||
outcome = "ok"
|
||
// Preserve the existing HTTP response shape: the runtime_id field is new
|
||
// in the WS path and would be redundant noise on the HTTP path where the
|
||
// caller already knows which runtime it asked about.
|
||
resp := map[string]any{"status": ack.Status}
|
||
if ack.PendingUpdate != nil {
|
||
resp["pending_update"] = ack.PendingUpdate
|
||
}
|
||
if ack.PendingModelList != nil {
|
||
resp["pending_model_list"] = ack.PendingModelList
|
||
}
|
||
if ack.PendingLocalSkills != nil {
|
||
resp["pending_local_skills"] = ack.PendingLocalSkills
|
||
}
|
||
if ack.PendingLocalSkillImport != nil {
|
||
resp["pending_local_skill_import"] = ack.PendingLocalSkillImport
|
||
}
|
||
if len(ack.PendingLocalSkillImports) > 0 {
|
||
resp["pending_local_skill_imports"] = ack.PendingLocalSkillImports
|
||
}
|
||
writeJSON(w, http.StatusOK, resp)
|
||
}
|
||
|
||
// HandleDaemonWSHeartbeat is the daemonws.HeartbeatHandler entry point. The
|
||
// WebSocket upgrade already batch-authenticated the fixed runtime set and
|
||
// captured each runtime's liveness state in a connection lease, so the hot
|
||
// path never reads agent_runtime. HTTP heartbeat remains the stateless lookup
|
||
// fallback for a daemon that stops receiving WebSocket acknowledgements.
|
||
func (h *Handler) HandleDaemonWSHeartbeat(ctx context.Context, identity daemonws.ClientIdentity, runtimeID string, supportsBatchImport bool) (*protocol.DaemonHeartbeatAckPayload, error) {
|
||
lease := identity.RuntimeLeases[runtimeID]
|
||
if lease == nil {
|
||
return nil, fmt.Errorf("runtime not in connection lease")
|
||
}
|
||
// Defensive consistency assertion only: the workspace scope and lease came
|
||
// from the same connection-time query. This does not re-authorize against DB.
|
||
if !identity.AllowsWorkspace(lease.Snapshot().WorkspaceID) {
|
||
return nil, fmt.Errorf("runtime not in connection workspace")
|
||
}
|
||
if err := h.recordHeartbeatLease(ctx, runtimeID, lease); err != nil {
|
||
if isNotFound(err) {
|
||
if h.DaemonRuntimeGone != nil {
|
||
h.NotifyRuntimeGone(runtimeID)
|
||
return nil, nil
|
||
}
|
||
return runtimeGoneHeartbeatAck(runtimeID), nil
|
||
}
|
||
return nil, err
|
||
}
|
||
ack, _, err := h.processHeartbeat(ctx, runtimeID, supportsBatchImport)
|
||
return ack, err
|
||
}
|
||
|
||
func runtimeGoneHeartbeatAck(runtimeID string) *protocol.DaemonHeartbeatAckPayload {
|
||
return &protocol.DaemonHeartbeatAckPayload{
|
||
RuntimeID: runtimeID,
|
||
Status: protocol.HeartbeatStatusRuntimeGone,
|
||
RuntimeGone: true,
|
||
}
|
||
}
|
||
|
||
// recordHeartbeat marks the runtime as alive. When LivenessStore is available
|
||
// (Redis configured and reachable) it writes a TTL'd liveness key and skips
|
||
// the DB row write on most beats — the DB is only updated on the
|
||
// offline→online transition or once per runtimeHeartbeatDBFlushInterval to
|
||
// keep last_seen_at fresh enough for the UI and the DB-fallback sweeper.
|
||
//
|
||
// When LivenessStore is unavailable (no Redis configured) or any Touch call
|
||
// errors, recordHeartbeat falls back to writing the DB on every beat — that
|
||
// is the original behavior and keeps the sweeper's DB-only path correct.
|
||
//
|
||
// The actual DB write is delegated to h.HeartbeatScheduler so production can
|
||
// coalesce many runtimes' bumps into one bulk UPDATE per tick. See
|
||
// heartbeat_scheduler.go for the two implementations.
|
||
func (h *Handler) recordHeartbeat(ctx context.Context, rt db.AgentRuntime) error {
|
||
return h.recordHeartbeatState(ctx, rt.ID, uuidToString(rt.ID), heartbeatLivenessState{
|
||
Status: rt.Status,
|
||
LastSeenAt: rt.LastSeenAt.Time,
|
||
LastSeenAtValid: rt.LastSeenAt.Valid,
|
||
WorkspaceID: rt.WorkspaceID,
|
||
}, nil)
|
||
}
|
||
|
||
func (h *Handler) recordHeartbeatLease(ctx context.Context, runtimeID string, lease *daemonws.RuntimeLease) error {
|
||
runtimeUUID, err := util.ParseUUID(runtimeID)
|
||
if err != nil {
|
||
return fmt.Errorf("invalid runtime_id: %w", err)
|
||
}
|
||
state := lease.Snapshot()
|
||
// Lenient parse: the workspace ID only feeds the recovery refresh payload.
|
||
// An invalid value suppresses the event instead of failing the heartbeat.
|
||
wsUUID, _ := util.ParseUUID(state.WorkspaceID)
|
||
return h.recordHeartbeatState(ctx, runtimeUUID, runtimeID, heartbeatLivenessState{
|
||
Status: state.Status,
|
||
LastSeenAt: state.LastSeenAt,
|
||
LastSeenAtValid: state.LastSeenAtValid,
|
||
WorkspaceID: wsUUID,
|
||
}, lease.MarkDBWriteScheduled)
|
||
}
|
||
|
||
type heartbeatLivenessState struct {
|
||
Status string
|
||
LastSeenAt time.Time
|
||
LastSeenAtValid bool
|
||
// WorkspaceID feeds the recovery lifecycle refresh; it is already known
|
||
// from the lease snapshot or the HTTP row and never re-read from the DB.
|
||
WorkspaceID pgtype.UUID
|
||
}
|
||
|
||
func (h *Handler) recordHeartbeatState(
|
||
ctx context.Context,
|
||
runtimeUUID pgtype.UUID,
|
||
runtimeID string,
|
||
state heartbeatLivenessState,
|
||
markDBWriteScheduled func(time.Time),
|
||
) error {
|
||
now := time.Now()
|
||
|
||
// Decide whether the DB row needs a write *before* touching Redis, so a
|
||
// Touch failure can simply force needDBWrite=true without re-evaluating
|
||
// the structural reasons.
|
||
needDBWrite := !h.LivenessStore.Available() ||
|
||
state.Status != "online" ||
|
||
!state.LastSeenAtValid ||
|
||
now.Sub(state.LastSeenAt) >= runtimeHeartbeatDBFlushInterval
|
||
|
||
if h.LivenessStore.Available() {
|
||
if err := h.LivenessStore.Touch(ctx, runtimeID, runtimeLivenessTTL); err != nil {
|
||
// Redis hiccup: degrade transparently to the DB-only path for
|
||
// this beat. The sweeper falls back to its DB threshold the
|
||
// same way when IsAliveBatch fails, so end-to-end correctness
|
||
// is preserved.
|
||
slog.Warn("liveness touch failed; falling back to DB heartbeat",
|
||
"runtime_id", runtimeID, "error", err)
|
||
needDBWrite = true
|
||
}
|
||
}
|
||
|
||
if !needDBWrite {
|
||
return nil
|
||
}
|
||
|
||
// Status transitions remain synchronous so the acknowledgement cannot race
|
||
// dependent work that expects an online row. The steady-state online bump
|
||
// is ID-only and may be coalesced by the production scheduler.
|
||
if state.Status != "online" || !state.LastSeenAtValid {
|
||
// The conditional update reports whether this beat performed the
|
||
// offline → online flip. Only an actual flip publishes the lifecycle
|
||
// refresh; a beat that lost the race (or a never-seen row already
|
||
// online) stays silent and keeps the unconditional update below so
|
||
// last_seen_at is bumped and pgx.ErrNoRows is preserved for deletions.
|
||
flipped, err := h.Queries.MarkAgentRuntimeOnlineIfOffline(ctx, runtimeUUID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if flipped > 0 {
|
||
// Reuse daemon:register so web and mobile invalidate both runtime and
|
||
// agent projections. Ordinary online heartbeats stay silent.
|
||
if state.WorkspaceID.Valid {
|
||
h.PublishRuntimeRefresh(uuidToString(state.WorkspaceID), "system", "", "heartbeat_recovery")
|
||
}
|
||
} else if _, err := h.Queries.MarkAgentRuntimeOnline(ctx, runtimeUUID); err != nil {
|
||
return err
|
||
}
|
||
if markDBWriteScheduled != nil {
|
||
markDBWriteScheduled(now)
|
||
}
|
||
return nil
|
||
}
|
||
if err := h.HeartbeatScheduler.Schedule(ctx, runtimeUUID, state.WorkspaceID); err != nil {
|
||
return err
|
||
}
|
||
if markDBWriteScheduled != nil {
|
||
markDBWriteScheduled(now)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// heartbeatMetrics carries per-stage timings out of processHeartbeat so the
|
||
// HTTP slow-log can stay structured. The WS path discards them.
|
||
type heartbeatMetrics struct {
|
||
ProbeModelMs, PopModelMs, ProbeSkillsMs, PopSkillsMs, ProbeImportMs, PopImportMs int64
|
||
ProbeModelTimedOut, ProbeSkillsTimedOut, ProbeImportTimedOut bool
|
||
}
|
||
|
||
// processHeartbeat pulls pending actions for both HTTP and WebSocket
|
||
// heartbeats using only the runtime ID. Each transport records liveness first:
|
||
// HTTP uses its stateless runtime row, while WebSocket uses the connection
|
||
// lease. Auth and request decoding also remain transport-specific.
|
||
func (h *Handler) processHeartbeat(ctx context.Context, runtimeID string, supportsBatchImport bool) (*protocol.DaemonHeartbeatAckPayload, heartbeatMetrics, error) {
|
||
var m heartbeatMetrics
|
||
|
||
slog.Debug("daemon heartbeat", "runtime_id", runtimeID)
|
||
|
||
ack := &protocol.DaemonHeartbeatAckPayload{
|
||
RuntimeID: runtimeID,
|
||
Status: "ok",
|
||
ServerCapabilities: []string{protocol.DaemonCapabilityRPCV1},
|
||
}
|
||
|
||
probeUpdateCtx, cancelProbeUpdate := context.WithTimeout(ctx, heartbeatHasPendingTimeout)
|
||
hasUpdate, probeUpdateErr := h.UpdateStore.HasPending(probeUpdateCtx, runtimeID)
|
||
cancelProbeUpdate()
|
||
switch {
|
||
case probeUpdateErr == nil && hasUpdate:
|
||
pending, popUpdateErr := h.UpdateStore.PopPending(ctx, runtimeID)
|
||
if popUpdateErr != nil {
|
||
slog.Warn("update PopPending failed", "error", popUpdateErr, "runtime_id", runtimeID)
|
||
} else if pending != nil {
|
||
ack.PendingUpdate = &protocol.DaemonHeartbeatPendingUpdate{
|
||
ID: pending.ID,
|
||
TargetVersion: pending.TargetVersion,
|
||
}
|
||
}
|
||
case probeUpdateErr != nil:
|
||
if errors.Is(probeUpdateErr, context.DeadlineExceeded) || errors.Is(probeUpdateErr, context.Canceled) {
|
||
slog.Warn("update HasPending timed out", "runtime_id", runtimeID)
|
||
} else {
|
||
slog.Warn("update HasPending failed", "error", probeUpdateErr, "runtime_id", runtimeID)
|
||
}
|
||
}
|
||
|
||
// Probe then claim the model list queue. Same pattern as the local-skill
|
||
// queues below — a slow shared store cannot stall the heartbeat on
|
||
// empty-queue ticks, but the claim itself runs unbounded because its
|
||
// Lua side effects cannot be safely aborted mid-script.
|
||
probeModelStart := time.Now()
|
||
probeModelCtx, cancelProbeModel := context.WithTimeout(ctx, heartbeatHasPendingTimeout)
|
||
hasModel, probeModelErr := h.ModelListStore.HasPending(probeModelCtx, runtimeID)
|
||
cancelProbeModel()
|
||
m.ProbeModelMs = time.Since(probeModelStart).Milliseconds()
|
||
switch {
|
||
case probeModelErr == nil && hasModel:
|
||
popStart := time.Now()
|
||
pendingModel, popErr := h.ModelListStore.PopPending(ctx, runtimeID)
|
||
m.PopModelMs = time.Since(popStart).Milliseconds()
|
||
if popErr != nil {
|
||
slog.Warn("model list PopPending failed", "error", popErr, "runtime_id", runtimeID)
|
||
} else if pendingModel != nil {
|
||
ack.PendingModelList = &protocol.DaemonHeartbeatPendingModelList{ID: pendingModel.ID}
|
||
}
|
||
case probeModelErr != nil:
|
||
if errors.Is(probeModelErr, context.DeadlineExceeded) || errors.Is(probeModelErr, context.Canceled) {
|
||
m.ProbeModelTimedOut = true
|
||
slog.Warn("model list HasPending timed out", "runtime_id", runtimeID, "elapsed_ms", m.ProbeModelMs)
|
||
} else {
|
||
slog.Warn("model list HasPending failed", "error", probeModelErr, "runtime_id", runtimeID)
|
||
}
|
||
}
|
||
|
||
// Probe then claim the local-skill list queue. The probe is bounded so a
|
||
// slow shared store cannot stall the heartbeat on empty-queue ticks; the
|
||
// claim runs unbounded (it inherits only ctx) because its Lua side
|
||
// effects cannot be safely aborted mid-script.
|
||
probeSkillsStart := time.Now()
|
||
probeSkillsCtx, cancelProbeSkills := context.WithTimeout(ctx, heartbeatHasPendingTimeout)
|
||
hasSkills, probeErr := h.LocalSkillListStore.HasPending(probeSkillsCtx, runtimeID)
|
||
cancelProbeSkills()
|
||
m.ProbeSkillsMs = time.Since(probeSkillsStart).Milliseconds()
|
||
switch {
|
||
case probeErr == nil && hasSkills:
|
||
popStart := time.Now()
|
||
pendingSkills, popErr := h.LocalSkillListStore.PopPending(ctx, runtimeID)
|
||
m.PopSkillsMs = time.Since(popStart).Milliseconds()
|
||
if popErr != nil {
|
||
slog.Warn("local skill list PopPending failed", "error", popErr, "runtime_id", runtimeID)
|
||
} else if pendingSkills != nil {
|
||
ack.PendingLocalSkills = &protocol.DaemonHeartbeatPendingLocalSkills{ID: pendingSkills.ID}
|
||
}
|
||
case probeErr != nil:
|
||
if errors.Is(probeErr, context.DeadlineExceeded) || errors.Is(probeErr, context.Canceled) {
|
||
m.ProbeSkillsTimedOut = true
|
||
slog.Warn("local skill list HasPending timed out", "runtime_id", runtimeID, "elapsed_ms", m.ProbeSkillsMs)
|
||
} else {
|
||
slog.Warn("local skill list HasPending failed", "error", probeErr, "runtime_id", runtimeID)
|
||
}
|
||
}
|
||
|
||
probeImportStart := time.Now()
|
||
probeImportCtx, cancelProbeImport := context.WithTimeout(ctx, heartbeatHasPendingTimeout)
|
||
hasImport, probeErr := h.LocalSkillImportStore.HasPending(probeImportCtx, runtimeID)
|
||
cancelProbeImport()
|
||
m.ProbeImportMs = time.Since(probeImportStart).Milliseconds()
|
||
switch {
|
||
case probeErr == nil && hasImport:
|
||
popStart := time.Now()
|
||
if supportsBatchImport {
|
||
pendingImports, popErr := h.LocalSkillImportStore.PopPendingBatch(ctx, runtimeID, maxLocalSkillImportBatch)
|
||
m.PopImportMs = time.Since(popStart).Milliseconds()
|
||
if popErr != nil {
|
||
slog.Warn("local skill import PopPendingBatch failed", "error", popErr, "runtime_id", runtimeID, "claimed", len(pendingImports))
|
||
}
|
||
// Always dispatch whatever was claimed — even on partial
|
||
// failure the claimed requests have already transitioned to
|
||
// running in the store. Dropping them here would leave them
|
||
// stranded until the running timeout.
|
||
if len(pendingImports) > 0 {
|
||
// Backwards compat: singular field carries the first item so
|
||
// old daemons that don't know the plural field still get one.
|
||
ack.PendingLocalSkillImport = &protocol.DaemonHeartbeatPendingLocalSkillImport{
|
||
ID: pendingImports[0].ID,
|
||
SkillKey: pendingImports[0].SkillKey,
|
||
}
|
||
batch := make([]protocol.DaemonHeartbeatPendingLocalSkillImport, 0, len(pendingImports))
|
||
for _, p := range pendingImports {
|
||
batch = append(batch, protocol.DaemonHeartbeatPendingLocalSkillImport{
|
||
ID: p.ID,
|
||
SkillKey: p.SkillKey,
|
||
})
|
||
}
|
||
ack.PendingLocalSkillImports = batch
|
||
}
|
||
} else {
|
||
pendingImport, popErr := h.LocalSkillImportStore.PopPending(ctx, runtimeID)
|
||
m.PopImportMs = time.Since(popStart).Milliseconds()
|
||
if popErr != nil {
|
||
slog.Warn("local skill import PopPending failed", "error", popErr, "runtime_id", runtimeID)
|
||
} else if pendingImport != nil {
|
||
ack.PendingLocalSkillImport = &protocol.DaemonHeartbeatPendingLocalSkillImport{
|
||
ID: pendingImport.ID,
|
||
SkillKey: pendingImport.SkillKey,
|
||
}
|
||
}
|
||
}
|
||
case probeErr != nil:
|
||
if errors.Is(probeErr, context.DeadlineExceeded) || errors.Is(probeErr, context.Canceled) {
|
||
m.ProbeImportTimedOut = true
|
||
slog.Warn("local skill import HasPending timed out", "runtime_id", runtimeID, "elapsed_ms", m.ProbeImportMs)
|
||
} else {
|
||
slog.Warn("local skill import HasPending failed", "error", probeErr, "runtime_id", runtimeID)
|
||
}
|
||
}
|
||
|
||
return ack, m, nil
|
||
}
|
||
|
||
// logHeartbeatEndpointSlow emits one structured log when /api/daemon/heartbeat
|
||
// exceeds 500ms, splitting auth / update / probe / pop phases for both queues
|
||
// so the prod tail can be attributed without flooding logs at normal rates.
|
||
// auth_ms is further decomposed into decode_ms, runtime_lookup_ms, and
|
||
// workspace_check_ms; auth_path labels which token kind authenticated the
|
||
// request ("daemon_token", "pat", or "jwt"). Mirrors logClaimEndpointSlow.
|
||
func logHeartbeatEndpointSlow(runtimeID, outcome, authPath string, start time.Time, decodeMs, runtimeLookupMs, workspaceCheckMs, authMs, updateMs, probeModelMs, popModelMs, probeSkillsMs, popSkillsMs, probeImportMs, popImportMs int64, probeModelTimedOut, probeSkillsTimedOut, probeImportTimedOut bool) {
|
||
totalMs := time.Since(start).Milliseconds()
|
||
if totalMs < 500 && !probeModelTimedOut && !probeSkillsTimedOut && !probeImportTimedOut {
|
||
return
|
||
}
|
||
slog.Info("heartbeat_endpoint slow",
|
||
"runtime_id", runtimeID,
|
||
"outcome", outcome,
|
||
"auth_path", authPath,
|
||
"total_ms", totalMs,
|
||
"auth_ms", authMs,
|
||
"decode_ms", decodeMs,
|
||
"runtime_lookup_ms", runtimeLookupMs,
|
||
"workspace_check_ms", workspaceCheckMs,
|
||
"update_ms", updateMs,
|
||
"probe_model_ms", probeModelMs,
|
||
"pop_model_ms", popModelMs,
|
||
"probe_skills_ms", probeSkillsMs,
|
||
"pop_skills_ms", popSkillsMs,
|
||
"probe_import_ms", probeImportMs,
|
||
"pop_import_ms", popImportMs,
|
||
"probe_model_timed_out", probeModelTimedOut,
|
||
"probe_skills_timed_out", probeSkillsTimedOut,
|
||
"probe_import_timed_out", probeImportTimedOut,
|
||
)
|
||
}
|
||
|
||
// logClaimEndpointSlow emits one structured log when the /tasks/claim endpoint
|
||
// exceeds 500ms, splitting auth / claim / response-build phases so the prod
|
||
// tail can be diagnosed without flooding logs at normal poll rates.
|
||
func logClaimEndpointSlow(runtimeID, outcome string, start time.Time, authMs, claimMs, buildMs int64, payloadBytes, agentSkillCount, builtinSkillCount, skillPayloadBytes int) {
|
||
totalMs := time.Since(start).Milliseconds()
|
||
if totalMs < 500 {
|
||
return
|
||
}
|
||
slog.Info("claim_endpoint slow",
|
||
"runtime_id", runtimeID,
|
||
"outcome", outcome,
|
||
"total_ms", totalMs,
|
||
"auth_ms", authMs,
|
||
"claim_ms", claimMs,
|
||
"build_ms", buildMs,
|
||
"payload_bytes", payloadBytes,
|
||
"agent_skill_count", agentSkillCount,
|
||
"builtin_skill_count", builtinSkillCount,
|
||
"skill_payload_bytes", skillPayloadBytes,
|
||
)
|
||
}
|
||
|
||
// requestHasClientCapability reports whether the caller advertised a capability
|
||
// in X-Client-Capabilities. Daemons and app clients share the header.
|
||
// requestClientCapabilities returns the advertised capability list, trimmed and
|
||
// with empties dropped. Nil when the header is absent — which is exactly what an
|
||
// older daemon looks like.
|
||
func requestClientCapabilities(r *http.Request) []string {
|
||
raw := strings.Split(r.Header.Get("X-Client-Capabilities"), ",")
|
||
out := make([]string, 0, len(raw))
|
||
for _, part := range raw {
|
||
if trimmed := strings.TrimSpace(part); trimmed != "" {
|
||
out = append(out, trimmed)
|
||
}
|
||
}
|
||
if len(out) == 0 {
|
||
return nil
|
||
}
|
||
return out
|
||
}
|
||
|
||
// runtimeHasCapability reports whether a stored runtime row advertised the
|
||
// capability at registration. Absent metadata means an older daemon that never
|
||
// sent the header, so this fails closed.
|
||
func runtimeHasCapability(metadata []byte, capability string) bool {
|
||
if len(metadata) == 0 {
|
||
return false
|
||
}
|
||
var m struct {
|
||
Capabilities []string `json:"capabilities"`
|
||
}
|
||
if err := json.Unmarshal(metadata, &m); err != nil {
|
||
return false
|
||
}
|
||
for _, c := range m.Capabilities {
|
||
if c == capability {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func requestHasClientCapability(r *http.Request, capability string) bool {
|
||
for _, part := range strings.Split(r.Header.Get("X-Client-Capabilities"), ",") {
|
||
if strings.TrimSpace(part) == capability {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func parseRuntimeConnectedAppsForClaim(raw []byte, taskID pgtype.UUID) []runtimeapps.ConnectedApp {
|
||
raw = bytes.TrimSpace(raw)
|
||
if len(raw) == 0 || bytes.Equal(raw, []byte("null")) {
|
||
return nil
|
||
}
|
||
var apps []runtimeapps.ConnectedApp
|
||
if err := json.Unmarshal(raw, &apps); err != nil {
|
||
slog.Warn("daemon claim: unmarshal runtime_connected_apps failed",
|
||
"task_id", uuidToString(taskID),
|
||
"error", err,
|
||
)
|
||
return nil
|
||
}
|
||
return apps
|
||
}
|
||
|
||
// repairStaleCommentPlanIfNeeded handles the edit/delete race where a claimed
|
||
// task's trigger was deleted (trigger_comment_id cleared, or pointing at a
|
||
// tombstone) but coalesced_comment_ids survive: such a task must never be
|
||
// dispatched as a generic assignment — its user-scoped MCP overlay still
|
||
// belongs to the deleted author, and the prompt would read issue history
|
||
// exposing that stale user's capabilities. When it applies, the task is
|
||
// cancelled and its surviving comments are replayed through normal routing
|
||
// (which recomputes originator + connected-app context).
|
||
//
|
||
// Returns handled=true when the task must NOT be dispatched — either a clean
|
||
// repair (failure==nil) or a hard failure (failure!=nil, carrying the
|
||
// status/message/outcome the per-runtime endpoint renders). handled=false means
|
||
// proceed with a normal claim. Shared by the per-runtime and batch claim
|
||
// handlers so the batch path can't silently drop surviving comments (MUL-4257).
|
||
func (h *Handler) repairStaleCommentPlanIfNeeded(ctx context.Context, task *db.AgentTaskQueue, runtimeWorkspaceID string) (handled bool, failure *claimBuildFailure) {
|
||
if len(task.CoalescedCommentIds) == 0 {
|
||
return false, nil
|
||
}
|
||
if task.TriggerCommentID.Valid {
|
||
// A trigger deleted while it had replies stays as a tombstone instead of
|
||
// clearing trigger_comment_id (#8296); repair it like a removed one.
|
||
trigger, err := h.Queries.GetComment(ctx, task.TriggerCommentID)
|
||
if err != nil || !trigger.DeletedAt.Valid {
|
||
return false, nil
|
||
}
|
||
}
|
||
if !task.IssueID.Valid {
|
||
return true, &claimBuildFailure{outcome: "error_stale_comment_plan", status: http.StatusInternalServerError, message: "comment task has no issue"}
|
||
}
|
||
issue, loadErr := h.Queries.GetIssue(ctx, task.IssueID)
|
||
if loadErr != nil {
|
||
return true, &claimBuildFailure{outcome: "error_stale_comment_plan", status: http.StatusInternalServerError, message: "failed to repair stale comment task"}
|
||
}
|
||
if uuidToString(issue.WorkspaceID) != runtimeWorkspaceID {
|
||
if _, cancelErr := h.TaskService.CancelTask(ctx, task.ID); cancelErr != nil {
|
||
slog.Error("task claim: cancel stale cross-workspace task failed",
|
||
"task_id", uuidToString(task.ID), "error", cancelErr)
|
||
}
|
||
return true, &claimBuildFailure{outcome: "error_workspace", status: http.StatusInternalServerError, message: "task workspace isolation check failed"}
|
||
}
|
||
cancelled, cancelErr := h.TaskService.CancelTask(ctx, task.ID)
|
||
if cancelErr != nil {
|
||
return true, &claimBuildFailure{outcome: "error_stale_comment_plan", status: http.StatusInternalServerError, message: "failed to repair stale comment task"}
|
||
}
|
||
h.retriggerCancelledTaskSurvivors(ctx, issue, []db.AgentTaskQueue{*cancelled}, pgtype.UUID{})
|
||
return true, nil
|
||
}
|
||
|
||
// claimBatchMaxTasksCap bounds how many tasks a single machine-level batch
|
||
// claim may return, so one request can neither build an unbounded payload nor
|
||
// hold the DB for an unbounded number of per-agent claim transactions. The
|
||
// daemon never asks for more than its free execution-slot count anyway.
|
||
const claimBatchMaxTasksCap = 32
|
||
|
||
// ClaimTasksByRuntime is the machine-level (MUL-4257) batch claim endpoint. A
|
||
// daemon posts every runtime_id it hosts plus its free execution-slot count and
|
||
// receives up to max_tasks already-claimed tasks in ONE round trip — each
|
||
// carrying its runtime_id so the daemon routes it to the matching runtime
|
||
// locally. This collapses the per-runtime idle-poll fan-out (one HTTP request
|
||
// plus one promote/reclaim/list cycle per runtime) into a single request backed
|
||
// by = ANY merged queries.
|
||
//
|
||
// Each returned task goes through the same FinalizeTaskClaim as the per-runtime
|
||
// endpoint, so the task-scoped token AND the comment-delivery receipt
|
||
// (delivered_comment_ids) are persisted atomically; a finalization failure
|
||
// requeues that exact claim and omits it from the batch. Unknown/unauthorized
|
||
// runtime_ids are skipped silently (a daemon may send a just-deleted runtime;
|
||
// it self-heals via the heartbeat path).
|
||
func (h *Handler) ClaimTasksByRuntime(w http.ResponseWriter, r *http.Request) {
|
||
start := time.Now()
|
||
buildDeadline, responseDeadline := batchClaimDeadlines(r.Context(), start)
|
||
buildCtx, cancelBuild := context.WithDeadline(r.Context(), buildDeadline)
|
||
defer cancelBuild()
|
||
r = r.WithContext(buildCtx)
|
||
var req struct {
|
||
DaemonID string `json:"daemon_id"`
|
||
RuntimeIDs []string `json:"runtime_ids"`
|
||
MaxTasks int `json:"max_tasks"`
|
||
}
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
|
||
// Machine-level ownership (MUL-4257 review): the batch claim is scoped to a
|
||
// single daemon. daemon_id is required so the server can reject any
|
||
// runtime_id that belongs to a different machine (guards against a stale /
|
||
// crossed runtime set claiming another daemon's tasks — which would land
|
||
// local_directory / machine-local work on the wrong host). For an mdt_
|
||
// token the body daemon_id must equal the token's daemon_id, so a
|
||
// workspace-scoped token can't spoof a peer.
|
||
if req.DaemonID == "" {
|
||
writeError(w, http.StatusBadRequest, "daemon_id is required")
|
||
return
|
||
}
|
||
if ctxDaemonID := middleware.DaemonIDFromContext(r.Context()); ctxDaemonID != "" && ctxDaemonID != req.DaemonID {
|
||
writeError(w, http.StatusForbidden, "daemon_id does not match token")
|
||
return
|
||
}
|
||
|
||
// max_tasks semantics (MUL-4257 review): a negative count is malformed; zero
|
||
// is a valid "no free slots" poll that must claim nothing — never coerce to
|
||
// 1, which would dispatch a task the daemon cannot run and strand it until
|
||
// stale reclaim. Positive counts are capped so one request can't build an
|
||
// unbounded payload.
|
||
if req.MaxTasks < 0 {
|
||
writeError(w, http.StatusBadRequest, "max_tasks must not be negative")
|
||
return
|
||
}
|
||
if req.MaxTasks == 0 {
|
||
writeMeasuredJSON(w, http.StatusOK, map[string]any{"tasks": []AgentTaskResponse{}})
|
||
return
|
||
}
|
||
maxTasks := req.MaxTasks
|
||
if maxTasks > claimBatchMaxTasksCap {
|
||
maxTasks = claimBatchMaxTasksCap
|
||
}
|
||
|
||
// Parse + de-dup requested ids with the NON-panicking parser (MUL-4257
|
||
// review): the handler-local parseUUID panics on malformed input, which
|
||
// would turn a single bad id into a 500. Invalid ids are skipped, matching
|
||
// this endpoint's "unknown id skipped" semantics. Key by the canonical uuid
|
||
// string so the post-claim lookup by task.RuntimeID always matches.
|
||
idByKey := make(map[string]pgtype.UUID, len(req.RuntimeIDs))
|
||
for _, rid := range req.RuntimeIDs {
|
||
ruid, err := util.ParseUUID(rid)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
idByKey[util.UUIDToString(ruid)] = ruid
|
||
}
|
||
if len(idByKey) == 0 {
|
||
writeMeasuredJSON(w, http.StatusOK, map[string]any{"tasks": []AgentTaskResponse{}})
|
||
return
|
||
}
|
||
ids := make([]pgtype.UUID, 0, len(idByKey))
|
||
for _, id := range idByKey {
|
||
ids = append(ids, id)
|
||
}
|
||
|
||
// Resolve all requested runtimes in one query (instead of a point lookup
|
||
// per runtime), then authorize each; skip (don't fail) unknown/unauthorized
|
||
// ids so a single stale runtime can't sink the whole batch.
|
||
//
|
||
// This read goes through RuntimeLookup like every other agent_runtime read
|
||
// by id (MUL-6884), so the claim path is attributed on
|
||
// multica_agent_runtime_lookup_total instead of being invisible on it. That
|
||
// matters more here than on any other caller: both /tasks/claim and /claim
|
||
// route to this handler and the WebSocket claim RPC replays through it, so
|
||
// an unattributed read here would make the busiest reader in the system
|
||
// look idle next to once-per-shutdown deregisters.
|
||
foundByID, err := h.getAgentRuntimes(r.Context(), obsmetrics.RuntimeLookupSourceDaemonAPI, ids)
|
||
if err != nil {
|
||
writeError(w, http.StatusInternalServerError, "failed to load runtimes")
|
||
return
|
||
}
|
||
// Iterate ids rather than ranging the returned map: authorized[] is passed
|
||
// to ClaimTasksForRuntimes, where maxTasks can cut the set off partway, so
|
||
// map iteration order would decide which runtimes get the remaining slots.
|
||
// (ids is itself built from a map above, so this pins the order to one
|
||
// source rather than making it fully deterministic — worth tightening, but
|
||
// not in this change.)
|
||
//
|
||
// runtimeByID must end up holding ONLY authorized rows: the post-claim loop
|
||
// below treats a miss in it as a stray cross-daemon claim and drops the
|
||
// task. GetMany returns every row it found, so the authorized subset is
|
||
// collected separately rather than reusing its map.
|
||
runtimeByID := make(map[string]db.AgentRuntime, len(foundByID))
|
||
authorized := make([]pgtype.UUID, 0, len(foundByID))
|
||
for _, id := range ids {
|
||
rt, ok := foundByID[uuidToString(id)]
|
||
if !ok {
|
||
continue
|
||
}
|
||
if !h.verifyDaemonWorkspaceAccess(r, uuidToString(rt.WorkspaceID)) {
|
||
continue
|
||
}
|
||
// Group-ownership check (mirrors the WS path, daemon_ws.go): a runtime
|
||
// bound to a different daemon must not be claimed by this one. Runtimes
|
||
// with a NULL daemon_id (e.g. cloud runtimes) are not machine-pinned, so
|
||
// they stay claimable — same tolerance as the WS handler.
|
||
if rt.DaemonID.Valid && rt.DaemonID.String != req.DaemonID {
|
||
continue
|
||
}
|
||
runtimeByID[uuidToString(rt.ID)] = rt
|
||
authorized = append(authorized, rt.ID)
|
||
}
|
||
if len(authorized) == 0 {
|
||
writeMeasuredJSON(w, http.StatusOK, map[string]any{"tasks": []AgentTaskResponse{}})
|
||
return
|
||
}
|
||
|
||
claimed, err := h.TaskService.ClaimTasksForRuntimes(r.Context(), authorized, maxTasks)
|
||
if err != nil {
|
||
slog.Warn("claim tasks failed", append(logger.RequestAttrs(r), "error", err)...)
|
||
writeError(w, http.StatusInternalServerError, "failed to claim tasks")
|
||
return
|
||
}
|
||
|
||
out := make([]AgentTaskResponse, 0, len(claimed))
|
||
for i := range claimed {
|
||
if r.Context().Err() != nil {
|
||
break
|
||
}
|
||
task := claimed[i]
|
||
rt, ok := runtimeByID[uuidToString(task.RuntimeID)]
|
||
if !ok {
|
||
// Service guards claims to the authorized set; a miss here would be
|
||
// a stray cross-daemon claim. Leave it for the owning daemon's
|
||
// reclaim path rather than shipping it to the wrong machine.
|
||
continue
|
||
}
|
||
rtWorkspaceID := uuidToString(rt.WorkspaceID)
|
||
// Stale comment-plan repair must run for the batch path too: otherwise a
|
||
// task whose trigger was deleted (only coalesced survive) would be
|
||
// finalized+dispatched with no comment input, silently dropping the
|
||
// surviving user comment. On repair (or hard failure) the task is
|
||
// cancelled / left for reclaim and omitted from the batch.
|
||
if handled, _ := h.repairStaleCommentPlanIfNeeded(r.Context(), &task, rtWorkspaceID); handled {
|
||
continue
|
||
}
|
||
resp, deliveredCommentIDs, issueSnapshot, _, _, failure := h.buildClaimedTaskResponse(r, &task, rt, uuidToString(task.RuntimeID), rtWorkspaceID)
|
||
if failure != nil {
|
||
// Builder rejected this task (workspace isolation / chat-input);
|
||
// it has already settled the task where the failure requires it.
|
||
// Undelivered claims are recovered below with a separate budget.
|
||
continue
|
||
}
|
||
if !rt.OwnerID.Valid {
|
||
slog.Error("batch claim: runtime owner missing; cancelling task to avoid unscoped agent credentials",
|
||
"task_id", uuidToString(task.ID), "runtime_id", uuidToString(task.RuntimeID))
|
||
if _, cerr := h.TaskService.CancelTask(r.Context(), task.ID); cerr != nil {
|
||
slog.Error("batch claim: cancel after missing runtime owner failed",
|
||
"task_id", uuidToString(task.ID), "error", cerr)
|
||
}
|
||
continue
|
||
}
|
||
tokenStr, terr := auth.GenerateAgentTaskToken()
|
||
if terr != nil {
|
||
slog.Error("batch claim: generate task token failed; requeueing claim",
|
||
"task_id", uuidToString(task.ID), "error", terr)
|
||
continue
|
||
}
|
||
remoteMCPToken, daemonTokens, derr := remoteMCPDaemonTokenForClaim(resp, rt)
|
||
if derr != nil {
|
||
slog.Error("batch claim: generate Remote MCP daemon token failed; requeueing claim",
|
||
"task_id", uuidToString(task.ID), "error", derr)
|
||
continue
|
||
}
|
||
// Route through the SAME finalization as the per-runtime endpoint so the
|
||
// token and the comment-delivery receipt (delivered_comment_ids for
|
||
// comment/coalesced-comment tasks) are persisted atomically; on failure
|
||
// the exact claim is requeued and omitted from this batch. The shared
|
||
// final delivery gate re-verifies current runtime ownership inside the
|
||
// same transaction; a rejected task is settled via FailTask and skipped
|
||
// while valid tasks in the same batch continue.
|
||
commentBackedTask := task.TriggerCommentID.Valid || len(task.CoalescedCommentIds) > 0
|
||
receipt, deliveryFailure, ferr := h.finalizeClaimDelivery(r.Context(), &task, rt, uuidToString(task.RuntimeID), rtWorkspaceID, &resp, db.CreateTaskTokenParams{
|
||
ID: dbid.NewV7(),
|
||
TokenHash: auth.HashToken(tokenStr),
|
||
TaskID: task.ID,
|
||
AgentID: task.AgentID,
|
||
WorkspaceID: parseUUID(resp.WorkspaceID),
|
||
UserID: rt.OwnerID,
|
||
ExpiresAt: pgtype.Timestamptz{Time: time.Now().Add(24 * time.Hour), Valid: true},
|
||
}, deliveredCommentIDs, commentBackedTask, issueSnapshot, daemonTokens...)
|
||
if ferr != nil {
|
||
slog.Error("batch claim: finalize task claim failed; requeueing claim",
|
||
"task_id", uuidToString(task.ID), "error", ferr)
|
||
continue
|
||
}
|
||
if deliveryFailure != nil {
|
||
if !deliveryFailure.settled {
|
||
slog.Error("batch claim: delivery rejection settlement failed",
|
||
"task_id", uuidToString(task.ID), "outcome", deliveryFailure.outcome,
|
||
"error", deliveryFailure.message)
|
||
}
|
||
continue
|
||
}
|
||
resp.AuthToken = tokenStr
|
||
resp.RemoteMCPDaemonToken = remoteMCPToken
|
||
resp.DeliveredCommentIDs = uuidStringsOrEmpty(receipt)
|
||
out = append(out, resp)
|
||
}
|
||
|
||
if len(out) > 0 {
|
||
slog.Info("tasks claimed by runtime batch",
|
||
"runtimes", len(authorized), "requested_max", maxTasks, "claimed", len(out),
|
||
"total_ms", time.Since(start).Milliseconds())
|
||
}
|
||
response := map[string]any{"tasks": out}
|
||
// Only opted-in daemons understand this additive response metadata. Query
|
||
// after the claim so a future fire_at can shorten the long healthy-WS safety
|
||
// poll; a task that crossed fire_at during this request yields a bounded
|
||
// follow-up hint and is promoted on the next claim. If the lookup fails,
|
||
// omit the support bit so the daemon conservatively retains its ordinary
|
||
// PollInterval. Omitted claims also retain that short poll in case recovery
|
||
// failed or the task_available wakeup raced with this response.
|
||
if len(out) == len(claimed) && len(out) < maxTasks && requestHasClientCapability(r, protocol.DaemonCapabilityClaimPollHintsV1) {
|
||
nextDeferred, nextErr := h.Queries.NextDeferredTaskFireAtForRuntimes(r.Context(), db.NextDeferredTaskFireAtForRuntimesParams{
|
||
RuntimeIds: authorized,
|
||
RuntimeStaleSecs: service.RuntimeClaimFreshnessSeconds,
|
||
})
|
||
if nextErr != nil {
|
||
slog.Warn("batch claim: next deferred task lookup failed; retaining short client poll",
|
||
"error", nextErr)
|
||
} else {
|
||
response["claim_poll_hint_supported"] = true
|
||
if nextDeferred.Valid {
|
||
response["next_deferred_task_after_ms"] = claimPollHintDelay(time.Now(), nextDeferred.Time).Milliseconds()
|
||
}
|
||
}
|
||
}
|
||
if len(out) != len(claimed) {
|
||
// HTTP/1.1 cannot process another request until this handler returns.
|
||
// Keep the next claim off this connection while recovery is running.
|
||
w.Header().Set("Connection", "close")
|
||
}
|
||
writeMeasuredJSON(w, http.StatusOK, response)
|
||
if len(out) != len(claimed) {
|
||
// Deliver successful payloads before batch recovery side effects. Cache
|
||
// invalidation, event listeners and cross-node wakeups can use their own
|
||
// timeouts, so the recovery context alone cannot bound the response wait.
|
||
// HTTP already advertises Content-Length; flushing sends the whole body.
|
||
// WS keeps cleanup in the existing RPC worker after queuing its response.
|
||
cleanup := func() {
|
||
h.requeueUndeliveredBatchClaims(r.Context(), claimed, out, responseDeadline)
|
||
}
|
||
if !daemonws.AfterRPCResponse(r.Context(), cleanup) {
|
||
_ = http.NewResponseController(w).Flush()
|
||
cleanup()
|
||
}
|
||
}
|
||
}
|
||
|
||
const (
|
||
batchClaimRecoveryTimeout = time.Second
|
||
batchClaimResponseMargin = 250 * time.Millisecond
|
||
)
|
||
|
||
// Reserve recovery and response time INSIDE the installed HTTP client's budget.
|
||
// The WS response grace is for transport, not additional database work. Shorter
|
||
// caller deadlines get proportionally smaller reserves; longer ones cannot
|
||
// extend the HTTP-compatible batch budget.
|
||
func batchClaimDeadlines(ctx context.Context, start time.Time) (build, response time.Time) {
|
||
deadline := start.Add(protocol.DaemonBatchClaimRequestTimeout)
|
||
if caller, ok := ctx.Deadline(); ok && caller.Before(deadline) {
|
||
deadline = caller
|
||
}
|
||
budget := max(deadline.Sub(start), 0)
|
||
response = deadline.Add(-min(batchClaimResponseMargin, budget/10))
|
||
build = response.Add(-min(batchClaimRecoveryTimeout, budget/4))
|
||
return build, response
|
||
}
|
||
|
||
// Share one deadline for recovery DB work across the batch. Post-commit cache,
|
||
// event and wakeup side effects may outlive it, so callers deliver the response
|
||
// first while retaining cleanup in the current HTTP/RPC request worker.
|
||
func (h *Handler) requeueUndeliveredBatchClaims(ctx context.Context, claimed []db.AgentTaskQueue, delivered []AgentTaskResponse, responseDeadline time.Time) {
|
||
if len(claimed) == len(delivered) {
|
||
return
|
||
}
|
||
deliveredIDs := make(map[string]bool, len(delivered))
|
||
for _, task := range delivered {
|
||
deliveredIDs[task.ID] = true
|
||
}
|
||
deadline := time.Now().Add(batchClaimRecoveryTimeout)
|
||
if responseDeadline.Before(deadline) {
|
||
deadline = responseDeadline
|
||
}
|
||
ctx, cancel := context.WithDeadline(context.WithoutCancel(ctx), deadline)
|
||
defer cancel()
|
||
for _, task := range claimed {
|
||
if deliveredIDs[uuidToString(task.ID)] {
|
||
continue
|
||
}
|
||
if ctx.Err() != nil {
|
||
slog.Warn("batch claim: recovery budget exhausted; stale reclaim will recover remaining claims", "error", ctx.Err())
|
||
break
|
||
}
|
||
// The existing CAS matches runtime + dispatched_at and requires an
|
||
// unstarted dispatched row. Settled, already requeued, started, and
|
||
// newer claims must remain untouched (ErrNoRows).
|
||
if _, err := h.TaskService.RequeueTaskAfterClaimFailure(ctx, task); err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||
slog.Error("batch claim: undelivered claim recovery failed; stale reclaim will recover it",
|
||
"task_id", uuidToString(task.ID), "error", err)
|
||
}
|
||
}
|
||
}
|
||
|
||
func claimPollHintDelay(now, fireAt time.Time) time.Duration {
|
||
delay := fireAt.Sub(now)
|
||
if delay < claimPollHintMinDelay {
|
||
return claimPollHintMinDelay
|
||
}
|
||
return delay
|
||
}
|
||
|
||
// finalizeClaimDelivery is the shared final delivery gate for the singular and
|
||
// batch claim paths. It re-verifies the current agent/runtime authorization
|
||
// INSIDE the FinalizeTaskClaim transaction, under a FOR UPDATE row lock on the
|
||
// runtime row, so the decision and the task-token mint commit atomically:
|
||
//
|
||
// - runtime.owner_id is re-read authoritatively (no stale claim-time
|
||
// snapshot), and a concurrent re-registration that would change owner_id
|
||
// blocks until this gate commits — closing the TOCTOU window between the
|
||
// claim-time runtime read and daemon delivery (PUCK-89 blocker 1);
|
||
// - the private-runtime owner fence re-runs against the locked row;
|
||
// - agent.runtime_id still equals the claimed task's runtime (rebind fence);
|
||
// - agent ownership still satisfies the private-runtime binding.
|
||
// - response requesting-user identity is refreshed from that same owner;
|
||
//
|
||
// On mismatch the task is settled through the existing
|
||
// failClaimedTaskBeforeLaunch -> TaskService.FailTask path and never returned
|
||
// to the daemon. The delivery failure reports whether settlement succeeded so
|
||
// callers can keep their poll semantics (singular: 200 {"task":null} only
|
||
// after settlement; batch: skip the task, keep returning valid ones).
|
||
func (h *Handler) finalizeClaimDelivery(
|
||
ctx context.Context,
|
||
task *db.AgentTaskQueue,
|
||
runtime db.AgentRuntime,
|
||
runtimeID, runtimeWorkspaceID string,
|
||
response *AgentTaskResponse,
|
||
token db.CreateTaskTokenParams,
|
||
deliveredCommentIDs []pgtype.UUID,
|
||
recordCommentReceipt bool,
|
||
issueSnapshot []byte,
|
||
daemonTokens ...db.CreateDaemonTokenParams,
|
||
) (receipt []pgtype.UUID, deliveryFailure *claimBuildFailure, err error) {
|
||
var agentOwnerID pgtype.UUID
|
||
authorize := func(qtx *db.Queries, tokenParams *db.CreateTaskTokenParams) error {
|
||
// FOR UPDATE on the runtime row: any concurrent
|
||
// UpsertAgentRuntime/visibility flip that would change owner_id or
|
||
// visibility blocks until this transaction commits, so the values read
|
||
// here are the ones delivery is authorized against.
|
||
locked, lerr := qtx.LockAgentRuntime(ctx, runtime.ID)
|
||
if lerr != nil {
|
||
return fmt.Errorf("lock runtime for delivery authorization: %w", lerr)
|
||
}
|
||
if response != nil {
|
||
// The response was built from the claim-time runtime snapshot. Clear
|
||
// that identity before resolving the locked owner so a failed lookup
|
||
// can never leak stale user context to the daemon.
|
||
response.RequestingUserName = ""
|
||
response.RequestingUserProfileDescription = ""
|
||
}
|
||
agent, aerr := qtx.GetAgentForUpdate(ctx, task.AgentID)
|
||
if aerr != nil {
|
||
return fmt.Errorf("re-read agent for delivery authorization: %w", aerr)
|
||
}
|
||
if agent.RuntimeID != task.RuntimeID {
|
||
return &service.ClaimDeliveryAuthzError{
|
||
Reason: "error_agent_runtime_changed",
|
||
Detail: "agent runtime changed before delivery",
|
||
}
|
||
}
|
||
if locked.Visibility == "private" && locked.OwnerID.Valid &&
|
||
(!agent.OwnerID.Valid || agent.OwnerID != locked.OwnerID) {
|
||
agentOwnerID = agent.OwnerID
|
||
return &service.ClaimDeliveryAuthzError{
|
||
Reason: "error_runtime_access_denied",
|
||
Detail: "private runtime does not permit task agent at delivery gate",
|
||
}
|
||
}
|
||
if !locked.OwnerID.Valid {
|
||
return &service.ClaimDeliveryAuthzError{
|
||
Reason: "error_runtime_owner_missing",
|
||
Detail: "runtime owner missing before task delivery",
|
||
}
|
||
}
|
||
if owner, oerr := qtx.GetUser(ctx, locked.OwnerID); oerr != nil {
|
||
slog.Debug("failed to load locked runtime owner for brief injection",
|
||
"runtime_id", runtimeID,
|
||
"owner_id", uuidToString(locked.OwnerID),
|
||
"error", oerr,
|
||
)
|
||
} else if response != nil {
|
||
response.RequestingUserName = owner.Name
|
||
response.RequestingUserProfileDescription = owner.ProfileDescription
|
||
}
|
||
// The runtime row is locked for the duration of FinalizeTaskClaim. Use
|
||
// its current owner as the task-token identity rather than the stale
|
||
// claim-time snapshot captured by the caller.
|
||
tokenParams.UserID = locked.OwnerID
|
||
return nil
|
||
}
|
||
|
||
receipt, err = h.TaskService.FinalizeTaskClaim(ctx, *task, token, deliveredCommentIDs, recordCommentReceipt, authorize, issueSnapshot, daemonTokens...)
|
||
if err == nil {
|
||
return receipt, nil, nil
|
||
}
|
||
var authzErr *service.ClaimDeliveryAuthzError
|
||
if !errors.As(err, &authzErr) {
|
||
return nil, nil, err
|
||
}
|
||
// Authorization rejected at the delivery boundary: settle through the
|
||
// existing failure path so the task never reaches the daemon.
|
||
switch authzErr.Reason {
|
||
case "error_agent_runtime_changed":
|
||
failure := h.failClaimedTaskBeforeLaunch(
|
||
ctx, task,
|
||
"The agent moved to another runtime before this task could start. Retry the task to run it on the agent's current runtime.",
|
||
taskfailure.ReasonInvalidTaskIdentity,
|
||
"error_agent_runtime_changed", http.StatusConflict, "agent runtime changed before task delivery",
|
||
)
|
||
return nil, failure, nil
|
||
default:
|
||
userMessage := "This private runtime cannot run the assigned agent because the agent and runtime have different owners."
|
||
if authzErr.Reason == "error_runtime_owner_missing" {
|
||
userMessage = "This runtime cannot run the assigned agent because the runtime has no owner."
|
||
} else if !agentOwnerID.Valid {
|
||
userMessage = "This private runtime cannot run the assigned agent because the agent has no owner."
|
||
}
|
||
failure := h.failClaimedTaskBeforeLaunch(
|
||
ctx, task,
|
||
userMessage,
|
||
taskfailure.ReasonRuntimeAccessDenied,
|
||
"error_runtime_access_denied", http.StatusForbidden, "private runtime does not permit task agent",
|
||
)
|
||
return nil, failure, nil
|
||
}
|
||
}
|
||
|
||
// claimBuildFailure captures a pre-response failure from
|
||
// buildClaimedTaskResponse (workspace isolation, chat-input load/empty, ...) so
|
||
// the per-runtime handler can render the exact status/message/outcome and the
|
||
// batch handler can skip the task. Any task cancellation is already performed
|
||
// inside the builder before it returns one.
|
||
type claimBuildFailure struct {
|
||
outcome string
|
||
status int
|
||
message string
|
||
settled bool
|
||
}
|
||
|
||
// rejectClaimSourceLoad settles a claim whose SOURCE row — the issue, chat
|
||
// session, autopilot_run or autopilot the task hangs off — could not be read.
|
||
//
|
||
// Every one of these branches used to swallow the error and fall through on
|
||
// `err == nil`. That is a fail-open: taskToResponse has already seeded
|
||
// resp.WorkspaceID with the RUNTIME's workspace, so a swallowed read ran on to
|
||
// the builder's backstop isolation check, which compared that seeded value
|
||
// against itself and passed. The agent was then dispatched with no issue, no
|
||
// chat input, or no autopilot instructions at all — and, for chat, past the
|
||
// empty-input guard that lives inside the success branch.
|
||
//
|
||
// The two failures settle differently:
|
||
//
|
||
// - The read FAILED (transient: DB blip, timeout, reset). Preserve the
|
||
// just-dispatched task for redelivery. The batch endpoint requeues
|
||
// omitted claims within its response budget; the singular endpoint
|
||
// retains stale-dispatch recovery, as for chat-input reads (MUL-4351).
|
||
// - The row is GONE (ErrNoRows). Cancel: the task can never run, and
|
||
// preserving it would spin in the reclaim loop forever. Legacy FKs
|
||
// currently keep every one of these states from arising —
|
||
// agent_task_queue.issue_id ON DELETE CASCADE,
|
||
// agent_task_queue.chat_session_id ON DELETE SET NULL, and
|
||
// autopilot_run.autopilot_id ON DELETE CASCADE feeding
|
||
// agent_task_queue.autopilot_run_id ON DELETE SET NULL — so this branch is
|
||
// defensive. It is what protects the claim path when those FKs are dropped
|
||
// for the repo's no-FK rule.
|
||
func (h *Handler) rejectClaimSourceLoad(ctx context.Context, task *db.AgentTaskQueue, err error, source, sourceID string) *claimBuildFailure {
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
slog.Error("task claim: source row is missing; cancelling task",
|
||
"task_id", uuidToString(task.ID), "source", source, "source_id", sourceID)
|
||
if _, cerr := h.TaskService.CancelTask(ctx, task.ID); cerr != nil {
|
||
slog.Error("task claim: cancel after missing source row failed",
|
||
"task_id", uuidToString(task.ID), "error", cerr)
|
||
}
|
||
return &claimBuildFailure{
|
||
outcome: "error_source_missing",
|
||
status: http.StatusInternalServerError,
|
||
message: fmt.Sprintf("task is missing its %s", source),
|
||
}
|
||
}
|
||
slog.Error("task claim: source load failed; preserving task for redelivery",
|
||
"task_id", uuidToString(task.ID), "source", source, "source_id", sourceID, "error", err)
|
||
return &claimBuildFailure{
|
||
outcome: "error_source_load",
|
||
status: http.StatusInternalServerError,
|
||
message: fmt.Sprintf("failed to load task %s", source),
|
||
}
|
||
}
|
||
|
||
// rejectClaimSkillLoad preserves a claim whose agent skills could not be read.
|
||
// Like other transient source failures, the batch endpoint requeues it within
|
||
// its response budget; the singular endpoint retains stale-dispatch recovery.
|
||
//
|
||
// Dispatching with whatever loaded is not the safer half-measure it looks
|
||
// like. LoadAgentSkills reads the whole skill set in two queries, so a failure
|
||
// means every skill loses its supporting files or the agent loses every skill,
|
||
// and the ref hashes sent to the daemon are computed over that same truncated
|
||
// content — so the daemon validates the bundle, caches it, and the agent runs
|
||
// with rules silently missing. There is no ErrNoRows branch to mirror: an
|
||
// agent with no skills is a legitimate empty result, not a missing row.
|
||
func (h *Handler) rejectClaimSkillLoad(task *db.AgentTaskQueue, err error) *claimBuildFailure {
|
||
slog.Error("task claim: agent skill load failed; preserving task for redelivery",
|
||
"task_id", uuidToString(task.ID), "agent_id", uuidToString(task.AgentID), "error", err)
|
||
return &claimBuildFailure{
|
||
outcome: "error_skill_load",
|
||
status: http.StatusInternalServerError,
|
||
message: "failed to load agent skills",
|
||
}
|
||
}
|
||
|
||
// rejectClaimOnWorkspaceMismatch enforces the claim's tenant boundary against
|
||
// the workspace that OWNS the task's context (issue / chat session / autopilot
|
||
// / quick-create), which is the only authority for MULTICA_WORKSPACE_ID in the
|
||
// agent env. An empty value would make the CLI silently fall back to the
|
||
// user-global config and talk to whatever workspace the user happened to last
|
||
// configure; a value that doesn't match the runtime's workspace means upstream
|
||
// routed a foreign-workspace task here. Both cases hard-fail AND cancel the
|
||
// just-dispatched task so the queue / agent status don't sit stuck until the
|
||
// stale-task sweeper fires minutes later.
|
||
//
|
||
// Every branch calls this as soon as it knows its authoritative workspace and
|
||
// BEFORE it hydrates project context, so "prove ownership, then load" is a
|
||
// property of the structure rather than a check each branch remembers to
|
||
// repeat. Returns nil when the claim may proceed.
|
||
func (h *Handler) rejectClaimOnWorkspaceMismatch(ctx context.Context, task *db.AgentTaskQueue, resolvedWorkspaceID, runtimeID, runtimeWorkspaceID string, hasQuickCreate bool) *claimBuildFailure {
|
||
if resolvedWorkspaceID != "" && resolvedWorkspaceID == runtimeWorkspaceID {
|
||
return nil
|
||
}
|
||
slog.Error("task claim: workspace isolation check failed, cancelling task",
|
||
"task_id", uuidToString(task.ID),
|
||
"runtime_id", runtimeID,
|
||
"runtime_workspace", runtimeWorkspaceID,
|
||
"resolved_workspace", resolvedWorkspaceID,
|
||
"has_issue", task.IssueID.Valid,
|
||
"has_chat", task.ChatSessionID.Valid,
|
||
"has_autopilot_run", task.AutopilotRunID.Valid,
|
||
"has_quick_create", hasQuickCreate,
|
||
)
|
||
if _, cerr := h.TaskService.CancelTask(ctx, task.ID); cerr != nil {
|
||
slog.Error("task claim: cancel after workspace check failed",
|
||
"task_id", uuidToString(task.ID), "error", cerr)
|
||
}
|
||
return &claimBuildFailure{
|
||
outcome: "error_workspace",
|
||
status: http.StatusInternalServerError,
|
||
message: "task workspace isolation check failed",
|
||
}
|
||
}
|
||
|
||
// remoteMCPDaemonTokenForClaim prepares the short-lived credential the daemon
|
||
// uses to resolve write-only Remote MCP secrets for this task. The raw token is
|
||
// returned only in the claim response; its hash is committed atomically with
|
||
// the task-scoped agent token by FinalizeTaskClaim.
|
||
func remoteMCPDaemonTokenForClaim(resp AgentTaskResponse, runtime db.AgentRuntime) (string, []db.CreateDaemonTokenParams, error) {
|
||
if len(resp.RemoteMCPConnections) == 0 {
|
||
return "", nil, nil
|
||
}
|
||
if !runtime.DaemonID.Valid || strings.TrimSpace(runtime.DaemonID.String) == "" {
|
||
return "", nil, errors.New("runtime daemon_id is required for Remote MCP")
|
||
}
|
||
raw, err := auth.GenerateDaemonToken()
|
||
if err != nil {
|
||
return "", nil, fmt.Errorf("generate Remote MCP daemon token: %w", err)
|
||
}
|
||
return raw, []db.CreateDaemonTokenParams{{
|
||
TokenHash: auth.HashToken(raw),
|
||
WorkspaceID: runtime.WorkspaceID,
|
||
DaemonID: runtime.DaemonID.String,
|
||
ExpiresAt: pgtype.Timestamptz{Time: time.Now().Add(24 * time.Hour), Valid: true},
|
||
}}, nil
|
||
}
|
||
|
||
// failClaimedTaskBeforeLaunch settles a durable claim-time rejection before
|
||
// the daemon ever receives the task. Leaving these failures in dispatched
|
||
// makes the stale-claim reaper deliver the same impossible task forever and
|
||
// leaves chat showing "starting" with no actionable outcome. If settlement
|
||
// itself fails, release the exact claim so a later attempt can retry the gate.
|
||
func (h *Handler) failClaimedTaskBeforeLaunch(
|
||
ctx context.Context,
|
||
task *db.AgentTaskQueue,
|
||
userMessage string,
|
||
failureReason taskfailure.Reason,
|
||
outcome string,
|
||
status int,
|
||
claimMessage string,
|
||
) *claimBuildFailure {
|
||
if _, err := h.TaskService.FailTask(
|
||
ctx,
|
||
task.ID,
|
||
userMessage,
|
||
"",
|
||
"",
|
||
"",
|
||
failureReason.String(),
|
||
false,
|
||
"",
|
||
"",
|
||
); err != nil {
|
||
slog.Error("task claim: fail rejected task failed; requeueing claim",
|
||
"task_id", uuidToString(task.ID),
|
||
"outcome", outcome,
|
||
"error", err,
|
||
)
|
||
if _, requeueErr := h.TaskService.RequeueTaskAfterClaimFailure(ctx, *task); requeueErr != nil {
|
||
slog.Error("task claim: requeue after rejected-task settlement failure failed; stale reclaim will recover it",
|
||
"task_id", uuidToString(task.ID),
|
||
"outcome", outcome,
|
||
"error", requeueErr,
|
||
)
|
||
}
|
||
return &claimBuildFailure{
|
||
outcome: outcome + "_settle",
|
||
status: http.StatusInternalServerError,
|
||
message: "failed to settle a task rejected before launch",
|
||
}
|
||
}
|
||
return &claimBuildFailure{outcome: outcome, status: status, message: claimMessage, settled: true}
|
||
}
|
||
|
||
func chatSessionResumeFallbackNeeded(priorSessionID, priorWorkDir string) bool {
|
||
return priorSessionID == "" || priorWorkDir == ""
|
||
}
|
||
|
||
func rerunSourceMatchesTaskScope(task, source db.AgentTaskQueue) bool {
|
||
if task.AgentID != source.AgentID {
|
||
return false
|
||
}
|
||
// A triage run is never a source to continue from (MUL-7189 §5.6). The
|
||
// exclusion GetLastTaskSession applies cannot reach this branch, which
|
||
// resolves the session from the task the user named rather than from the
|
||
// (agent, issue) lookup. RerunIssue already refuses a triage source, so this
|
||
// is the second lock on the same door — and the one that still holds for a
|
||
// row written by an older server mid rolling-deploy.
|
||
if service.IsTriageTask(source) {
|
||
return false
|
||
}
|
||
if task.IssueID.Valid {
|
||
return source.IssueID.Valid && task.IssueID == source.IssueID
|
||
}
|
||
if task.ChatSessionID.Valid {
|
||
return source.ChatSessionID.Valid && task.ChatSessionID == source.ChatSessionID
|
||
}
|
||
return false
|
||
}
|
||
|
||
// applyFreshSessionRetryWorkdir resolves the prior pointers for an automatic
|
||
// retry that must start a fresh session. CreateRetryTask forces one only when
|
||
// the parent's failure poisoned the conversation (currently
|
||
// codex_semantic_inactivity), and still copies the parent's work_dir onto the
|
||
// child: a poisoned conversation says nothing about the files it left behind,
|
||
// the same contract the manual-retry branch applies (MUL-4869, MUL-7034).
|
||
//
|
||
// The workdir is offered only to a daemon whose `multica repo checkout` keeps
|
||
// an existing checkout's work (DaemonCapabilityCheckoutKeepsWorkV1). The fresh
|
||
// session has no memory of that work and will fetch its repositories again;
|
||
// an older daemon's checkout resets the checkout and deletes the work being
|
||
// kept, so it gets a fresh directory as before. The daemon validates the
|
||
// directory before reusing it and falls back to a fresh Prepare when it is
|
||
// gone. Either way the failed attempt's working memory does not come back, so
|
||
// the continuity gap is disclosed.
|
||
func applyFreshSessionRetryWorkdir(task db.AgentTaskQueue, resp *AgentTaskResponse, daemonKeepsCheckoutWork bool) {
|
||
if daemonKeepsCheckoutWork && task.WorkDir.Valid {
|
||
resp.PriorWorkDir = task.WorkDir.String
|
||
}
|
||
resp.PriorSessionResumeUnavailable = true
|
||
}
|
||
|
||
func claimResponseAgentIdentityMatches(resp AgentTaskResponse) bool {
|
||
return resp.AgentID != "" && resp.Agent != nil && resp.Agent.ID == resp.AgentID
|
||
}
|
||
|
||
// buildClaimedTaskResponse assembles the full daemon claim payload for a
|
||
// single already-claimed task and computes the exact comment ids embedded in
|
||
// it (deliveredCommentIDs). Shared by the per-runtime handler
|
||
// (ClaimTaskByRuntime) and the machine-level batch handler
|
||
// (ClaimTasksByRuntime, MUL-4257) so both build byte-identical payloads and
|
||
// feed the same delivery receipt into FinalizeTaskClaim. A non-nil failure
|
||
// means the task must not be dispatched; the builder has already cancelled it
|
||
// where the failure semantics require it.
|
||
func (h *Handler) buildClaimedTaskResponse(r *http.Request, task *db.AgentTaskQueue, runtime db.AgentRuntime, runtimeID, runtimeWorkspaceID string) (resp AgentTaskResponse, deliveredCommentIDs []pgtype.UUID, issueSnapshot []byte, agentSkillCount, builtinSkillCount int, failure *claimBuildFailure) {
|
||
// Build response with fresh agent data (name + skills + custom_env + custom_args).
|
||
resp = taskToResponse(*task, runtimeWorkspaceID)
|
||
if err := (&service.IssueWakeupService{Tasks: h.TaskService}).CheckClaim(r.Context(), *task); err != nil {
|
||
if !errors.Is(err, service.ErrWakeupForbidden) {
|
||
return resp, nil, nil, 0, 0, h.rejectClaimSourceLoad(r.Context(), task, err, "wakeup", resp.WakeupID)
|
||
}
|
||
return resp, nil, nil, 0, 0, h.failClaimedTaskBeforeLaunch(r.Context(), task, "Wakeup is disabled or its authorization is no longer available.", taskfailure.ReasonInvalidTaskIdentity, "wakeup_unavailable", http.StatusConflict, "wakeup unavailable")
|
||
}
|
||
// Preserve PostgreSQL's microseconds: second-resolution display timestamps
|
||
// cannot distinguish stale claims reclaimed within the same second.
|
||
if task.DispatchedAt.Valid {
|
||
generation := task.DispatchedAt.Time.UTC().Format(time.RFC3339Nano)
|
||
resp.DispatchedAt = &generation
|
||
resp.StartClaimSupported = true
|
||
}
|
||
var issueNumber int32
|
||
// Claim-only capability: this server resolves the squad-leader role on the
|
||
// wire (is_leader_task / squad_id), so the daemon must not re-derive it
|
||
// from the briefing text. Set unconditionally — on every claim, leader or
|
||
// not — because its absence is what tells an upgraded daemon it is talking
|
||
// to a server too old to have answered the question (MUL-5811).
|
||
resp.LeaderRoleResolved = true
|
||
// Agent-trigger plugin hooks, as tools. A failure here degrades to no
|
||
// tools rather than failing the claim: a plugin that cannot be listed must
|
||
// not stop an agent from working on the issue.
|
||
if h.PluginService != nil && h.pluginsV1Enabled(r.Context()) {
|
||
if tools, toolErr := h.PluginService.AgentHookTools(r.Context(), parseUUID(runtimeWorkspaceID)); toolErr != nil {
|
||
slog.Warn("plugins: could not list agent hook tools", "workspace_id", runtimeWorkspaceID, "error", toolErr)
|
||
} else {
|
||
resp.PluginHookTools = tools
|
||
}
|
||
// mcp-transport hooks ride the EXISTING broker: the connection shape is
|
||
// what validatePinnedRemoteMCPTools already reads, so an approved tool
|
||
// that went missing or whose schema drifted refuses at startup without
|
||
// any new enforcement code.
|
||
if connections, connErr := h.PluginService.AgentMCPConnections(r.Context(), parseUUID(runtimeWorkspaceID)); connErr != nil {
|
||
slog.Warn("plugins: could not list agent MCP connections", "workspace_id", runtimeWorkspaceID, "error", connErr)
|
||
} else if len(connections) > 0 {
|
||
resp.RemoteMCPConnections = append(resp.RemoteMCPConnections, connections...)
|
||
}
|
||
}
|
||
supportsCoalescedComments := requestHasClientCapability(r, protocol.DaemonCapabilityCoalescedCommentsV1)
|
||
// Empty-but-non-nil so pgx persists '{}' rather than NULL for tasks without
|
||
// comment input. Comment tasks replace this with the ids actually embedded
|
||
// in the capability-aware response built below.
|
||
deliveredCommentIDs = []pgtype.UUID{}
|
||
composioMCPEnabled := h.composioMCPAppsEnabled(r.Context())
|
||
if composioMCPEnabled {
|
||
resp.ConnectedApps = parseRuntimeConnectedAppsForClaim(task.RuntimeConnectedApps, task.ID)
|
||
}
|
||
agent, err := h.Queries.GetAgent(r.Context(), task.AgentID)
|
||
if err != nil {
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
// The current schema cascades agent deletion to task rows, so a
|
||
// durable orphan task cannot normally reach this branch. It remains a
|
||
// fail-closed guard for a deletion racing response assembly or for
|
||
// externally corrupted data; settlement may report that the task row
|
||
// already disappeared, but no claim payload is dispatched either way.
|
||
slog.Error("daemon claim: task agent no longer exists; refusing dispatch",
|
||
"task_id", uuidToString(task.ID), "agent_id", uuidToString(task.AgentID))
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, h.failClaimedTaskBeforeLaunch(
|
||
r.Context(), task,
|
||
"Task identity is invalid: the assigned agent no longer exists.",
|
||
taskfailure.ReasonInvalidTaskIdentity,
|
||
"error_invalid_task_identity", http.StatusConflict, "task agent no longer exists",
|
||
)
|
||
}
|
||
slog.Error("daemon claim: load task agent failed; requeueing claim",
|
||
"task_id", uuidToString(task.ID), "agent_id", uuidToString(task.AgentID), "error", err)
|
||
if _, requeueErr := h.TaskService.RequeueTaskAfterClaimFailure(r.Context(), *task); requeueErr != nil {
|
||
slog.Error("daemon claim: requeue after agent load failure failed; stale reclaim will recover it",
|
||
"task_id", uuidToString(task.ID), "error", requeueErr)
|
||
}
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, &claimBuildFailure{
|
||
outcome: "error_load_task_agent",
|
||
status: http.StatusInternalServerError,
|
||
message: "failed to load task agent",
|
||
}
|
||
}
|
||
// The SQL claim narrows candidates and repeats the access predicate before
|
||
// changing task state, but Agent mutations do not stay locked through HTTP
|
||
// response assembly. Recheck the freshly loaded Agent here so a rebind or
|
||
// owner change that committed after the claim cannot reach the daemon.
|
||
if agent.RuntimeID != task.RuntimeID {
|
||
slog.Warn("daemon claim: agent runtime changed before delivery; refusing dispatch",
|
||
"task_id", uuidToString(task.ID),
|
||
"agent_id", uuidToString(task.AgentID),
|
||
"task_runtime_id", uuidToString(task.RuntimeID),
|
||
"agent_runtime_id", uuidToString(agent.RuntimeID),
|
||
)
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, h.failClaimedTaskBeforeLaunch(
|
||
r.Context(), task,
|
||
"The agent moved to another runtime before this task could start. Retry the task to run it on the agent's current runtime.",
|
||
taskfailure.ReasonInvalidTaskIdentity,
|
||
"error_agent_runtime_changed", http.StatusConflict, "agent runtime changed before task delivery",
|
||
)
|
||
}
|
||
if runtime.Visibility == "private" && runtime.OwnerID.Valid &&
|
||
(!agent.OwnerID.Valid || agent.OwnerID != runtime.OwnerID) {
|
||
userMessage := "This private runtime cannot run the assigned agent because the agent and runtime have different owners."
|
||
if !agent.OwnerID.Valid {
|
||
userMessage = "This private runtime cannot run the assigned agent because the agent has no owner."
|
||
}
|
||
slog.Warn("daemon claim: private runtime no longer permits task agent; refusing dispatch",
|
||
"task_id", uuidToString(task.ID),
|
||
"agent_id", uuidToString(task.AgentID),
|
||
"runtime_id", runtimeID,
|
||
"agent_owner_valid", agent.OwnerID.Valid,
|
||
)
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, h.failClaimedTaskBeforeLaunch(
|
||
r.Context(), task,
|
||
userMessage,
|
||
taskfailure.ReasonRuntimeAccessDenied,
|
||
"error_runtime_access_denied", http.StatusForbidden, "private runtime does not permit task agent",
|
||
)
|
||
}
|
||
useSkillRefs := requestHasClientCapability(r, protocol.DaemonCapabilitySkillBundlesV1)
|
||
// A daemon older than the multica-platform merge assembles a brief that
|
||
// still names the built-ins this server stopped shipping. It cannot be
|
||
// fixed from here — the brief lives in the daemon binary — so the missing
|
||
// capability buys that daemon a redirect stub under the old name instead of
|
||
// a dangling pointer. Capability, not version: the version string is only
|
||
// ever shown to humans.
|
||
legacySkillRedirects := !requestHasClientCapability(r, protocol.DaemonCapabilityPlatformSkillV1)
|
||
var customEnv map[string]string
|
||
if agent.CustomEnv != nil {
|
||
if err := json.Unmarshal(agent.CustomEnv, &customEnv); err != nil {
|
||
slog.Warn("failed to unmarshal agent custom_env", "agent_id", uuidToString(agent.ID), "error", err)
|
||
}
|
||
}
|
||
var customArgs []string
|
||
if agent.CustomArgs != nil {
|
||
if err := json.Unmarshal(agent.CustomArgs, &customArgs); err != nil {
|
||
slog.Warn("failed to unmarshal agent custom_args", "agent_id", uuidToString(agent.ID), "error", err)
|
||
}
|
||
}
|
||
var mcpConfig json.RawMessage
|
||
if agent.McpConfig != nil {
|
||
mcpConfig = json.RawMessage(agent.McpConfig)
|
||
}
|
||
// Fold in the workspace MCP servers this agent has been explicitly
|
||
// given (GH #6062). Only bound AND enabled servers are read, so a
|
||
// workspace library entry nobody added reaches nothing. Read on every
|
||
// claim, exactly like the agent column, so an admin's edit or a toggle
|
||
// lands on the agent's next task with nothing to restart. Errors —
|
||
// including a failed read — leave the agent config untouched: a broken
|
||
// shared entry must never take away servers the agent runs with today.
|
||
if bound, err := h.Queries.ListEnabledAgentMcpServers(r.Context(), agent.ID); err != nil {
|
||
slog.Warn("daemon claim: load agent mcp servers failed; using agent mcp_config",
|
||
"task_id", uuidToString(task.ID), "agent_id", uuidToString(agent.ID), "error", err)
|
||
} else if len(bound) > 0 {
|
||
bindings := make([]WorkspaceMcpBinding, 0, len(bound))
|
||
for _, server := range bound {
|
||
bindings = append(bindings, WorkspaceMcpBinding{Name: server.Name, Config: json.RawMessage(server.Config)})
|
||
}
|
||
if resolved, err := ResolveAgentMcpConfig(bindings, mcpConfig); err != nil {
|
||
slog.Warn("daemon claim: resolve agent mcp servers failed; falling back to agent mcp_config",
|
||
"task_id", uuidToString(task.ID), "agent_id", uuidToString(agent.ID), "error", err)
|
||
} else {
|
||
mcpConfig = resolved
|
||
}
|
||
}
|
||
// Layer the per-task overlay (set at enqueue from the initiator
|
||
// user's active integrations — currently Composio) on top of the
|
||
// agent's saved mcp_config. Overlay wins on server-name collisions
|
||
// because it carries the live user-scoped session URL. Errors are
|
||
// logged but never fail the claim: a broken overlay must not prevent
|
||
// the agent from running with its base config.
|
||
if composioMCPEnabled && len(task.RuntimeMcpOverlay) > 0 {
|
||
if merged, err := mergeMCPOverlay(mcpConfig, json.RawMessage(task.RuntimeMcpOverlay)); err != nil {
|
||
slog.Warn("daemon claim: merge runtime_mcp_overlay failed; falling back to agent mcp_config", "task_id", uuidToString(task.ID), "error", err)
|
||
} else {
|
||
mcpConfig = merged
|
||
}
|
||
}
|
||
// runtime_config is stored as JSONB and may legitimately be the
|
||
// empty object `{}` for agents that haven't opted into any
|
||
// provider-specific tuning. Forward only non-empty payloads so the
|
||
// daemon's per-provider decoders treat absent-or-empty identically.
|
||
var runtimeConfig json.RawMessage
|
||
if rc := bytes.TrimSpace(agent.RuntimeConfig); len(rc) > 0 && !bytes.Equal(rc, []byte("{}")) && !bytes.Equal(rc, []byte("null")) {
|
||
runtimeConfig = json.RawMessage(agent.RuntimeConfig)
|
||
}
|
||
resp.Agent = &TaskAgentData{
|
||
ID: uuidToString(agent.ID),
|
||
Name: agent.Name,
|
||
Instructions: agent.Instructions,
|
||
CustomEnv: customEnv,
|
||
CustomArgs: customArgs,
|
||
McpConfig: mcpConfig,
|
||
Model: agent.Model.String,
|
||
ThinkingLevel: agent.ThinkingLevel.String,
|
||
ServiceTier: agent.ServiceTier.String,
|
||
RuntimeConfig: runtimeConfig,
|
||
DisabledRuntimeSkills: disabledRuntimeSkillsFor(agent.DisabledRuntimeSkills, runtimeID, runtime.Provider),
|
||
}
|
||
// System agents carry a product-owned instruction layer that ships with
|
||
// this binary instead of being copied into their row at creation. That
|
||
// is what makes it hot-updatable: editing the embedded file and
|
||
// deploying reaches every existing workspace on its next task, with no
|
||
// migration and no client upgrade. agent.Instructions holds only the
|
||
// workspace's own notes, so a release can never overwrite them.
|
||
//
|
||
// Composing here covers every task kind, because this is the single
|
||
// place a claimed task's agent payload is assembled.
|
||
if agent.SystemKey.String == service.MikaSystemKey {
|
||
resp.Agent.Instructions = service.ComposeMikaInstructions(agent.Name, agent.Instructions)
|
||
}
|
||
if useSkillRefs {
|
||
_, skillRefs, err := h.TaskService.LoadAgentSkillBundles(r.Context(), task.AgentID, agent.SystemKey.String, legacySkillRedirects)
|
||
if err != nil {
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, h.rejectClaimSkillLoad(task, err)
|
||
}
|
||
agentSkillCount = len(skillRefs)
|
||
resp.Agent.SkillRefs = skillRefs
|
||
} else {
|
||
skills, err := h.TaskService.LoadAgentSkills(r.Context(), task.AgentID)
|
||
if err != nil {
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, h.rejectClaimSkillLoad(task, err)
|
||
}
|
||
agentSkillCount = len(skills)
|
||
builtinSkills := h.TaskService.BuiltinSkills(agent.SystemKey.String, legacySkillRedirects)
|
||
builtinSkillCount = len(builtinSkills)
|
||
skills = append(skills, builtinSkills...)
|
||
resp.Agent.Skills = skills
|
||
}
|
||
if !claimResponseAgentIdentityMatches(resp) {
|
||
responseAgentID := ""
|
||
if resp.Agent != nil {
|
||
responseAgentID = resp.Agent.ID
|
||
}
|
||
slog.Error("daemon claim: response agent identity mismatch; refusing dispatch",
|
||
"task_id", uuidToString(task.ID), "task_agent_id", resp.AgentID, "response_agent_id", responseAgentID)
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, h.failClaimedTaskBeforeLaunch(
|
||
r.Context(), task,
|
||
"Task identity is invalid: the task and response agent disagree.",
|
||
taskfailure.ReasonInvalidTaskIdentity,
|
||
"error_invalid_task_identity", http.StatusConflict, "task response agent identity mismatch",
|
||
)
|
||
}
|
||
|
||
// Resolve the runtime owner's profile description so the daemon can
|
||
// inject "## Requesting User" into the brief. Empty fields short-circuit
|
||
// the heading entirely on the daemon side; cloud / system runtimes with
|
||
// no owner stay anonymous. Failure here must not block claim — the agent
|
||
// can still run without the user-context section.
|
||
if runtime.OwnerID.Valid {
|
||
if owner, err := h.Queries.GetUser(r.Context(), runtime.OwnerID); err == nil {
|
||
resp.RequestingUserName = owner.Name
|
||
resp.RequestingUserProfileDescription = owner.ProfileDescription
|
||
} else {
|
||
slog.Debug("failed to load runtime owner for brief injection",
|
||
"runtime_id", runtimeID,
|
||
"owner_id", uuidToString(runtime.OwnerID),
|
||
"error", err,
|
||
)
|
||
}
|
||
}
|
||
|
||
// Include workspace ID and repos so the daemon can set up worktrees.
|
||
// Project context, repo precedence and the tenant/failure rules around both
|
||
// live in resolveClaimProjectContext, which every claim path shares.
|
||
if task.IssueID.Valid {
|
||
issue, issueErr := h.Queries.GetIssue(r.Context(), task.IssueID)
|
||
if issueErr != nil {
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount,
|
||
h.rejectClaimSourceLoad(r.Context(), task, issueErr, "issue", uuidToString(task.IssueID))
|
||
}
|
||
resp.WorkspaceID = uuidToString(issue.WorkspaceID)
|
||
if failure := h.rejectClaimOnWorkspaceMismatch(r.Context(), task, resp.WorkspaceID, runtimeID, runtimeWorkspaceID, false); failure != nil {
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, failure
|
||
}
|
||
resp.ThreadName = issue.Title
|
||
issueNumber = issue.Number
|
||
|
||
// Issue-state delta (MUL-7344). Every field below already sits on the
|
||
// `issue` row this claim loaded, so this costs one extra read — the
|
||
// PREVIOUS run's snapshot — and never a second GetIssue.
|
||
//
|
||
// The current status and assignee ship whether or not the comparison
|
||
// succeeded: a run that has to read the issue anyway loses nothing by
|
||
// also being told them, and a run that does not still needs them for
|
||
// workflow step 3 ("is it already in progress", "is it mine").
|
||
resp.IssueStatus = issue.Status
|
||
if issue.AssigneeType.Valid {
|
||
resp.IssueAssigneeType = issue.AssigneeType.String
|
||
}
|
||
if issue.AssigneeID.Valid {
|
||
resp.IssueAssigneeID = uuidToString(issue.AssigneeID)
|
||
}
|
||
// This claim's own snapshot, recorded by FinalizeTaskClaim so a LATER
|
||
// run that resumes this one's session can compare against what THIS run
|
||
// was actually handed. Building it here, from the issue row already
|
||
// loaded, is what makes the snapshot mean "the issue as of this claim".
|
||
currentSnapshot := buildIssueStateSnapshot(issue)
|
||
if encoded, err := json.Marshal(currentSnapshot); err != nil {
|
||
// A snapshot that cannot be encoded is simply not recorded; the
|
||
// next run reports "not compared" and reads the issue, which is
|
||
// the behaviour that predates the column.
|
||
slog.Warn("claim: could not encode issue snapshot",
|
||
"task_id", uuidToString(task.ID), "issue_id", uuidToString(task.IssueID), "error", err)
|
||
} else {
|
||
issueSnapshot = encoded
|
||
}
|
||
// Both deltas are computed AFTER the resume source is resolved, further
|
||
// down, because both must be measured from the run whose session this
|
||
// claim actually hands back — see resumeAnchor there.
|
||
var commentDeltaScope *commentCountScope
|
||
var resumeAnchor *resumedRunAnchor
|
||
|
||
// Squad-leader briefing injection: keyed off the task being a
|
||
// leader-task (is_leader_task) carrying a squad_id — NOT off the
|
||
// issue being assigned to a squad. The task flag is stamped at
|
||
// enqueue time and is true for every ISSUE-BOUND path that routes
|
||
// work to a squad leader: direct assign-to-squad, comment
|
||
// @squad-mention (even when the issue itself is assigned to a
|
||
// plain agent — the MUL-3724 case), sub-issue done callback,
|
||
// autopilot squad-assignee, and retry-clone inheritance. The old
|
||
// issue.AssigneeType=="squad" gate missed the comment-mention
|
||
// path, so the leader booted with zero squad context and
|
||
// degraded into doing the work itself instead of orchestrating.
|
||
//
|
||
// NOTE: quick-create tasks do NOT reach this block — they have a
|
||
// NULL issue_id (so the enclosing `task.IssueID.Valid` is false)
|
||
// and do NOT carry is_leader_task / squad_id columns. They route
|
||
// their squad through the task CONTEXT JSON (QuickCreateContext.
|
||
// SquadID) and get their briefing from the separate quick-create
|
||
// branch further below (search `qc.SquadID`). Do not "unify" the
|
||
// two by deleting that branch: it also sets resp.SquadID /
|
||
// resp.SquadName so the new issue defaults to the squad assignee,
|
||
// and there is no issue row to hang this column-based path on.
|
||
//
|
||
// We resolve the squad directly from task.SquadID rather than
|
||
// reverse-looking-up "which squad is this agent the leader of",
|
||
// which is ambiguous when one agent leads multiple squads. The
|
||
// uuidToString(squad.LeaderID) == resp.Agent.ID re-check is kept
|
||
// as a defensive gate: if the squad's leader was swapped after the
|
||
// task was enqueued, we never feed a stale briefing to a
|
||
// non-leader. It also doubles as the dangling-squad_id guard: a
|
||
// squad hard-deleted after enqueue makes GetSquadInWorkspace
|
||
// return no row (err != nil) — we skip injection silently, which
|
||
// is exactly the same observable result as "condition not
|
||
// matched". Claim still succeeds; no stale briefing is emitted.
|
||
// (No FK on squad_id — see migration 127.) We append (not replace)
|
||
// so per-agent instructions stay authoritative; the squad briefing
|
||
// stacks on top as task-specific squad context.
|
||
if task.IsLeaderTask {
|
||
injected := false
|
||
if resp.Agent != nil && task.SquadID.Valid {
|
||
if squad, err := h.Queries.GetSquadInWorkspace(r.Context(), db.GetSquadInWorkspaceParams{
|
||
ID: task.SquadID,
|
||
WorkspaceID: issue.WorkspaceID,
|
||
}); err == nil && uuidToString(squad.LeaderID) == resp.Agent.ID {
|
||
// Parent-status authority is deliberately NARROWER than
|
||
// briefing injection. Injection is keyed off is_leader_task
|
||
// (see above) and therefore also fires on the MUL-3724 path,
|
||
// where the issue belongs to a plain agent and this squad was
|
||
// only @mentioned for help. Granting status ownership there
|
||
// would let a guest squad push someone else's in-flight issue
|
||
// to in_review, so we gate it on the issue actually being
|
||
// assigned to this squad.
|
||
ownsIssueStatus := issue.AssigneeType.Valid &&
|
||
issue.AssigneeType.String == "squad" &&
|
||
uuidToString(issue.AssigneeID) == uuidToString(squad.ID)
|
||
briefing := buildSquadLeaderBriefing(r.Context(), h.Queries, squad, ownsIssueStatus)
|
||
if strings.TrimSpace(resp.Agent.Instructions) == "" {
|
||
resp.Agent.Instructions = briefing
|
||
} else {
|
||
resp.Agent.Instructions = resp.Agent.Instructions + "\n\n" + briefing
|
||
}
|
||
injected = true
|
||
slog.Debug("injected squad leader briefing",
|
||
"squad_id", uuidToString(squad.ID),
|
||
"squad_name", squad.Name,
|
||
"leader_agent_id", resp.Agent.ID,
|
||
"owns_issue_status", ownsIssueStatus,
|
||
)
|
||
}
|
||
}
|
||
// Every skip above (NULL squad_id, squad hard-deleted, leader
|
||
// swapped after enqueue) leaves a task the daemon must NOT run
|
||
// as a leader: it has no roster to delegate to and no protocol
|
||
// to follow. The daemon derives its leader role from this flag
|
||
// (MUL-5811), so clearing it here is what keeps
|
||
// "is_leader_task on the wire ⇔ briefing injected" true, and the
|
||
// run degrades to an ordinary agent turn exactly as it did when
|
||
// the daemon inferred the role from the briefing text itself.
|
||
if !injected {
|
||
resp.IsLeaderTask = false
|
||
slog.Warn("squad leader briefing not injected; claim delivered as a non-leader task",
|
||
"task_id", uuidToString(task.ID),
|
||
"squad_id", uuidToString(task.SquadID),
|
||
"agent_id", uuidToString(task.AgentID),
|
||
)
|
||
}
|
||
}
|
||
|
||
projectCtx, projectErr := h.resolveClaimProjectContext(r.Context(), issue.ProjectID, issue.WorkspaceID)
|
||
if projectErr != nil {
|
||
slog.Error("issue claim: load project context failed; preserving task for redelivery",
|
||
"task_id", uuidToString(task.ID),
|
||
"issue_id", uuidToString(issue.ID),
|
||
"error", projectErr)
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, &claimBuildFailure{
|
||
outcome: "error_project_context",
|
||
status: http.StatusInternalServerError,
|
||
message: "failed to load project context",
|
||
}
|
||
}
|
||
projectCtx.applyTo(&resp)
|
||
// Project workflow (MUL-7420): best-effort, like the status catalog
|
||
// below — a failure degrades to the workspace-wide status rules.
|
||
resp.ProjectWorkflow = h.claimProjectWorkflow(r.Context(), issue, task)
|
||
|
||
// Load every planned input as one chronological, de-duplicated set.
|
||
// The trigger is included here so the delivery receipt can only contain
|
||
// comments whose body we successfully embedded. Missing/deleted rows are
|
||
// intentionally absent and remain eligible for reconciliation. A stable
|
||
// payload budget always keeps the primary trigger, then admits an oldest-
|
||
// first prefix of additional comments; overflow is reconciled later.
|
||
// Workspace-scoped load (MUL-4252) so a foreign comment UUID resolves to
|
||
// "missing" instead of leaking another tenant's text into the prompt.
|
||
plannedCommentIDs := append([]pgtype.UUID{}, task.CoalescedCommentIds...)
|
||
if task.TriggerCommentID.Valid {
|
||
plannedCommentIDs = append(plannedCommentIDs, task.TriggerCommentID)
|
||
}
|
||
loadedComments := h.buildCoalescedCommentData(r.Context(), runtime.WorkspaceID, plannedCommentIDs)
|
||
triggerCommentID := uuidToString(task.TriggerCommentID)
|
||
var deliveredComments []CoalescedCommentData
|
||
triggerLoaded := false
|
||
for _, comment := range loadedComments {
|
||
if comment.ID == triggerCommentID {
|
||
triggerLoaded = true
|
||
break
|
||
}
|
||
}
|
||
if task.TriggerCommentID.Valid && triggerLoaded {
|
||
deliveredComments = selectCommentDelivery(
|
||
loadedComments,
|
||
triggerCommentID,
|
||
!supportsCoalescedComments,
|
||
maxClaimCommentPayloadBytes,
|
||
)
|
||
}
|
||
// If the persisted trigger body cannot be loaded, fail closed on comment
|
||
// coverage for this claim. The trigger snapshot CAS below also rejects a
|
||
// concurrent edit/delete that changes the FK after this read.
|
||
deliveredCommentIDs = commentDataIDs(deliveredComments)
|
||
// taskToResponse exposes the enqueue plan to UI task-list callers. A
|
||
// daemon claim must instead advertise only the structured ids actually
|
||
// present in this payload, especially when the delivery budget truncates.
|
||
resp.CoalescedCommentIDs = nil
|
||
for _, comment := range deliveredComments {
|
||
if comment.ID == triggerCommentID {
|
||
// Populate the actual payload from the same successful read that
|
||
// earned the receipt. The richer GetComment lookup below resolves
|
||
// initiator ids and count hints, but a transient second-read failure
|
||
// must never acknowledge a body that was not embedded.
|
||
resp.TriggerCommentContent = comment.Content
|
||
resp.TriggerThreadID = comment.ThreadID
|
||
resp.TriggerAuthorType = comment.AuthorType
|
||
resp.TriggerAuthorName = comment.AuthorName
|
||
continue
|
||
}
|
||
resp.CoalescedCommentIDs = append(resp.CoalescedCommentIDs, comment.ID)
|
||
resp.CoalescedComments = append(resp.CoalescedComments, comment)
|
||
}
|
||
|
||
// Fetch the triggering comment content so the daemon can embed it
|
||
// directly in the agent prompt (prevents the agent from ignoring comments
|
||
// when stale output files exist in a reused workdir). Also surface the
|
||
// comment author's kind and display name so the agent knows whether it
|
||
// was triggered by a human or by another agent — a signal used by the
|
||
// harness instructions to avoid mention loops between agents.
|
||
effectiveTriggerUUID := task.TriggerCommentID
|
||
if effectiveTriggerUUID.Valid {
|
||
// Scope by the runtime's workspace so a task row carrying a foreign
|
||
// comment UUID can never pull another workspace's comment text into
|
||
// this agent's prompt. The task's issue workspace is asserted equal
|
||
// to runtime.WorkspaceID below, so this is the right tenant (MUL-4252).
|
||
// A deleted trigger's tombstone is treated like a removed row.
|
||
if comment, err := h.Queries.GetCommentInWorkspace(r.Context(), db.GetCommentInWorkspaceParams{
|
||
ID: effectiveTriggerUUID,
|
||
WorkspaceID: runtime.WorkspaceID,
|
||
}); err == nil && !comment.DeletedAt.Valid {
|
||
resp.TriggerCommentContent = comment.Content
|
||
resp.TriggerThreadID = uuidToString(comment.ID)
|
||
if comment.ParentID.Valid {
|
||
if root, err := h.Queries.GetCommentThreadRootID(r.Context(), comment.ID); err == nil {
|
||
resp.TriggerThreadID = uuidToString(root)
|
||
}
|
||
}
|
||
resp.TriggerAuthorType = comment.AuthorType
|
||
// Preserve the direct trigger actor independently from the human
|
||
// whose authority this run uses. They can differ on delegated runs
|
||
// and manual reruns of comment-triggered tasks.
|
||
switch comment.AuthorType {
|
||
case "agent":
|
||
if comment.AuthorID.Valid {
|
||
if a, err := h.Queries.GetAgent(r.Context(), comment.AuthorID); err == nil {
|
||
resp.TriggerAuthorName = a.Name
|
||
}
|
||
}
|
||
case "member":
|
||
// For member-authored comments, AuthorID is a user UUID
|
||
// (see handler.resolveActor) — look up the user's display name.
|
||
if comment.AuthorID.Valid {
|
||
if u, err := h.Queries.GetUser(r.Context(), comment.AuthorID); err == nil {
|
||
resp.TriggerAuthorName = u.Name
|
||
}
|
||
}
|
||
}
|
||
// Count comments that arrived issue-wide since this agent's last
|
||
// run, so the daemon can tell it the full catch-up volume up front
|
||
// (the prompt then steers it to read the triggering thread first).
|
||
// Anchor = the prior task's started_at (never completed_at: a long
|
||
// run would miss comments posted while it ran). Cold start (no prior
|
||
// task) → no anchor → no hint. Excludes the agent's own comments and
|
||
// the triggering comment itself because that body is already
|
||
// injected into the prompt. Best-effort: any DB error or zero count
|
||
// leaves the hint suppressed.
|
||
//
|
||
// NewCommentsDeltaKnown is set on the success path REGARDLESS of
|
||
// the count, and is the only thing that distinguishes "the server
|
||
// looked and there is nothing" from "the server could not look".
|
||
// The count fields stay suppressed at zero — the daemon has no
|
||
// hint to render from a zero — but the daemon must still be able
|
||
// to tell a computed zero from a failed read, because only the
|
||
// computed one may waive the workflow's comment scan (MUL-6984).
|
||
//
|
||
// The anchor is not known yet: it is the started_at of the run
|
||
// whose session this claim resumes, which is resolved further
|
||
// down (MUL-7344). Record what the count needs and run it there.
|
||
commentDeltaScope = &commentCountScope{
|
||
AnchorID: effectiveTriggerUUID,
|
||
IssueID: comment.IssueID,
|
||
WorkspaceID: comment.WorkspaceID,
|
||
}
|
||
}
|
||
}
|
||
|
||
if !supportsCoalescedComments {
|
||
// Legacy daemons ignore the structured coalesced fields. Fold every
|
||
// successfully loaded comment into the one trigger field they already
|
||
// understand, then hide the structured fields to avoid duplicate prompt
|
||
// sections in intermediate daemons that understand them but do not yet
|
||
// advertise the capability.
|
||
if len(resp.CoalescedComments) > 0 || (resp.TriggerCommentContent == "" && len(deliveredComments) > 0) {
|
||
resp.TriggerCommentContent = formatLegacyCommentBundle(deliveredComments)
|
||
}
|
||
resp.CoalescedCommentIDs = nil
|
||
resp.CoalescedComments = nil
|
||
} else if resp.TriggerCommentContent == "" && len(deliveredComments) > 0 {
|
||
// A deleted newest trigger must not suppress the structured earlier
|
||
// comments: buildCommentPrompt renders them inside its trigger-content
|
||
// branch. The missing id itself is not acknowledged in the receipt.
|
||
resp.TriggerCommentContent = "The newest triggering comment is no longer available. Address every earlier comment included below."
|
||
}
|
||
|
||
// Resolve the prior agent session / workdir to resume.
|
||
if task.RerunOfTaskID.Valid {
|
||
// Manual retry: resume precisely from the source task the user
|
||
// clicked, NOT the most-recent (agent, issue) row — a parallel task
|
||
// on the same issue must never hijack the resume (MUL-4869). The
|
||
// workdir is ALWAYS reused when it still exists; the session is
|
||
// resumed only when the source failure did not poison the
|
||
// conversation AND the source ran on this runtime.
|
||
//
|
||
// Resume-safety is computed HERE from the source task, not read off
|
||
// task.ForceFreshSession: RerunIssue pins that flag to true so an OLD
|
||
// claim handler mid rolling-deploy degrades to a clean start instead
|
||
// of resuming a different execution via the (agent, issue) lookup.
|
||
// service.ResumeUnsafeFailure mirrors GetLastTaskSession, including
|
||
// its 400/invalid_request_error text defense for legacy /
|
||
// mis-classified rows that the exact-source path would otherwise miss.
|
||
//
|
||
// When the source workdir is gone (GC'd), absent on this runtime, or
|
||
// was never recorded (failed too early), execenv.Reuse falls back to a
|
||
// fresh Prepare and gateResumeToReusedWorkdir drops the now-unusable
|
||
// session — reuse is best-effort, never a silent swap onto a stale
|
||
// directory. PriorWorkDir is offered regardless of runtime (a shared
|
||
// mount may still resolve it); only the per-cwd session is
|
||
// runtime-gated.
|
||
if src, err := h.Queries.GetAgentTask(r.Context(), task.RerunOfTaskID); err == nil && rerunSourceMatchesTaskScope(*task, src) {
|
||
if src.WorkDir.Valid {
|
||
resp.PriorWorkDir = src.WorkDir.String
|
||
}
|
||
if !service.ResumeUnsafeFailure(src.FailureReason.String, src.Error.String) &&
|
||
src.SessionID.Valid && src.RuntimeID == task.RuntimeID {
|
||
resp.PriorSessionID = src.SessionID.String
|
||
// The deltas date from the run we actually resume, which on
|
||
// this path is the operator-chosen source — routinely NOT
|
||
// the newest run on the issue. nil unless that run proved it
|
||
// delivered its prompt to the provider (MUL-7344).
|
||
resumeAnchor = newResumedRunAnchor(src.Status, src.StartedAt, src.IssueSnapshot)
|
||
}
|
||
// MUL-5305: if the source task withheld its Codex session because
|
||
// the rollout was missing, this rerun has nothing resumable from it
|
||
// — disclose the gap rather than silently starting fresh.
|
||
if src.SessionRolloutMissing {
|
||
resp.PriorSessionResumeUnavailable = true
|
||
}
|
||
} else if err == nil {
|
||
slog.Warn("daemon claim: rerun source belongs to another agent or scope; starting fresh",
|
||
"task_id", uuidToString(task.ID),
|
||
"task_agent_id", uuidToString(task.AgentID),
|
||
"task_issue_id", uuidToString(task.IssueID),
|
||
"source_task_id", uuidToString(src.ID),
|
||
"source_agent_id", uuidToString(src.AgentID),
|
||
"source_issue_id", uuidToString(src.IssueID),
|
||
)
|
||
resp.PriorSessionResumeUnavailable = true
|
||
}
|
||
} else if !task.ForceFreshSession {
|
||
// Non-rerun follow-up on the same issue: resume the most recent
|
||
// (agent, issue) session so the agent keeps the issue's conversation
|
||
// context across turns. The "Focus on THIS comment" guard in
|
||
// prompt.go defends against inheriting the prior turn's "Done."
|
||
// marker, and GetLastTaskSession already excludes poisoned sessions.
|
||
if prior, err := h.Queries.GetLastTaskSession(r.Context(), db.GetLastTaskSessionParams{
|
||
AgentID: task.AgentID,
|
||
IssueID: task.IssueID,
|
||
}); err == nil && prior.SessionID.Valid {
|
||
if prior.RuntimeID == task.RuntimeID {
|
||
resp.PriorSessionID = prior.SessionID.String
|
||
// Same rule as the rerun path: date the deltas from the run
|
||
// this session belongs to. GetLastTaskSession skips poisoned
|
||
// and retired sessions, so `prior` can be an older run than
|
||
// the newest one on the issue; it also accepts failed and
|
||
// cancelled rows, which keep the session resumable but prove
|
||
// nothing about delivery (MUL-7344).
|
||
resumeAnchor = newResumedRunAnchor(prior.Status, prior.StartedAt, prior.IssueSnapshot)
|
||
}
|
||
if prior.WorkDir.Valid {
|
||
resp.PriorWorkDir = prior.WorkDir.String
|
||
}
|
||
}
|
||
// MUL-5305: if the most recent terminal task withheld its Codex
|
||
// session because the rollout was missing, GetLastTaskSession fell
|
||
// back to an older session (or none). Disclose the continuity gap so
|
||
// the next run tells the user the most recent turn's context could not
|
||
// be carried over — even when that older session resumes cleanly,
|
||
// which the resume-presence gate would otherwise pass silently.
|
||
if missing, err := h.Queries.GetLatestTaskRolloutMissing(r.Context(), db.GetLatestTaskRolloutMissingParams{
|
||
AgentID: task.AgentID,
|
||
IssueID: task.IssueID,
|
||
}); err == nil && missing {
|
||
resp.PriorSessionResumeUnavailable = true
|
||
}
|
||
} else if task.RetryOfTaskID.Valid {
|
||
// Automatic retry that must start a fresh session: continue in the
|
||
// parent's workdir, never its session. A force_fresh task with no
|
||
// retry lineage still resumes nothing.
|
||
applyFreshSessionRetryWorkdir(*task, &resp, requestHasClientCapability(r, protocol.DaemonCapabilityCheckoutKeepsWorkV1))
|
||
}
|
||
|
||
// Both deltas, now that the resume source is known (MUL-7344).
|
||
//
|
||
// resumeAnchor is non-nil only where a session was actually ADOPTED
|
||
// above, so the run being dated is the run whose context the agent will
|
||
// continue from. Every other path — no prior session, a session on
|
||
// another runtime, a forced-fresh retry, a rerun whose source failure
|
||
// poisoned the conversation — leaves it nil, reports neither delta, and
|
||
// the daemon falls back to the reads it has always performed. That is
|
||
// the conservative direction: the cost of not reporting is one extra
|
||
// read, while the cost of reporting against the wrong run is an agent
|
||
// continuing on stale requirements with nothing to signal it.
|
||
//
|
||
// PriorSessionResumeUnavailable is deliberately NOT consulted here. It
|
||
// flags a Codex rollout that went missing, which is only one of the ways
|
||
// the resumed session can be older than the newest run; the daemon
|
||
// applies it as its own separate gate.
|
||
//
|
||
// newResumedRunAnchor already rejected every source that cannot date a
|
||
// delta — anything that did not finish a turn, and so never proved the
|
||
// prompt reached the session. A non-nil anchor here is a completed run.
|
||
if resumeAnchor != nil {
|
||
if prev, ok := decodeIssueStateSnapshot(resumeAnchor.IssueSnapshot); ok {
|
||
resp.IssueStateDeltaKnown = true
|
||
resp.IssueChangedFields = currentSnapshot.changedFieldsSince(prev)
|
||
}
|
||
// Comments that arrived issue-wide since the resumed run started
|
||
// (never completed_at: a long run would miss comments posted while
|
||
// it ran). Excludes the agent's own comments and the triggering
|
||
// comment itself, whose body is already injected into the prompt.
|
||
//
|
||
// NewCommentsDeltaKnown is set on the success path REGARDLESS of the
|
||
// count: it is the only thing separating "the server looked and
|
||
// there is nothing" from "the server could not look", and only the
|
||
// former may waive the workflow's comment scan (MUL-6984). The count
|
||
// fields stay suppressed at zero — there is no hint to render from a
|
||
// zero, and the anchor would only invite a read returning nothing.
|
||
if commentDeltaScope != nil {
|
||
if cnt, err := h.Queries.CountNewCommentsSince(r.Context(), db.CountNewCommentsSinceParams{
|
||
AnchorID: commentDeltaScope.AnchorID,
|
||
IssueID: commentDeltaScope.IssueID,
|
||
WorkspaceID: commentDeltaScope.WorkspaceID,
|
||
Since: resumeAnchor.StartedAt,
|
||
AuthorID: task.AgentID,
|
||
}); err == nil {
|
||
resp.NewCommentsDeltaKnown = true
|
||
if cnt > 0 {
|
||
resp.NewCommentCount = int(cnt)
|
||
resp.NewCommentsSince = resumeAnchor.StartedAt.Time.UTC().Format(time.RFC3339)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Chat task: populate workspace/session info from the chat_session table.
|
||
if task.ChatSessionID.Valid {
|
||
cs, chatErr := h.Queries.GetChatSession(r.Context(), task.ChatSessionID)
|
||
if chatErr != nil {
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount,
|
||
h.rejectClaimSourceLoad(r.Context(), task, chatErr, "chat session", uuidToString(task.ChatSessionID))
|
||
}
|
||
resp.WorkspaceID = uuidToString(cs.WorkspaceID)
|
||
if failure := h.rejectClaimOnWorkspaceMismatch(r.Context(), task, resp.WorkspaceID, runtimeID, runtimeWorkspaceID, false); failure != nil {
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, failure
|
||
}
|
||
resp.ChatSessionID = uuidToString(cs.ID)
|
||
resp.ThreadName = cs.Title
|
||
// Legacy compatibility: agent creation no longer creates intro chats,
|
||
// but historical is_agent_intro sessions can still be resumed. Such a
|
||
// session carries no user message on its opening turn, so flag it for
|
||
// the historical self-introduction prompt (MUL-4230). The column stays true for the
|
||
// session's whole life, so gate the intro prompt on the session still
|
||
// having zero human messages — otherwise every follow-up turn after the
|
||
// creator replies would re-run the "introduce yourself" prompt and the
|
||
// agent keeps repeating the same introduction (MUL-4259).
|
||
if cs.IsAgentIntro {
|
||
if hasUser, herr := h.Queries.ChatSessionHasUserMessage(r.Context(), cs.ID); herr != nil {
|
||
slog.Warn("chat intro gate: has-user-message check failed",
|
||
"chat_session_id", uuidToString(cs.ID), "error", herr)
|
||
} else {
|
||
resp.ChatIntro = !hasUser
|
||
}
|
||
}
|
||
// A task-level delivery snapshot, not the Chat's historical binding,
|
||
// decides whether this run is operating for an external audience.
|
||
// Web/Desktop/Mobile turns in an old channel-originated Chat have no
|
||
// snapshot and remain private to Multica after /new rotates the route.
|
||
delivery, deliveryErr := h.Queries.GetChannelTaskDelivery(r.Context(), task.ID)
|
||
if deliveryErr == nil {
|
||
resp.ChatChannelType = delivery.ChannelType
|
||
resp.ChatType = delivery.ChatType
|
||
resp.ChatChannelDeliversFiles = h.channelDeliversFiles(delivery.ChannelType)
|
||
if delivery.ChannelType == string(slack.TypeSlack) {
|
||
resp.ChatInThread = delivery.ChannelThreadID.Valid &&
|
||
delivery.ChannelThreadID.String != "" &&
|
||
delivery.ChannelThreadID.String != delivery.ChannelMessageID.String
|
||
}
|
||
} else if !errors.Is(deliveryErr, pgx.ErrNoRows) {
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount,
|
||
h.rejectClaimSourceLoad(r.Context(), task, deliveryErr, "channel task delivery", uuidToString(task.ID))
|
||
}
|
||
// A web chat can opt into the same durable project context as an
|
||
// issue-bound task.
|
||
projectCtx, projectErr := h.resolveClaimProjectContext(r.Context(), cs.ProjectID, cs.WorkspaceID)
|
||
if projectErr != nil {
|
||
slog.Error("chat claim: load project context failed; preserving task for redelivery",
|
||
"task_id", uuidToString(task.ID),
|
||
"chat_session_id", uuidToString(cs.ID),
|
||
"error", projectErr)
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, &claimBuildFailure{
|
||
outcome: "error_project_context",
|
||
status: http.StatusInternalServerError,
|
||
message: "failed to load project context",
|
||
}
|
||
}
|
||
projectCtx.applyTo(&resp)
|
||
if !task.ForceFreshSession && !task.ChannelContextRevision.Valid {
|
||
// Resume chat sessions only when the stored pointer was produced
|
||
// by the same runtime as the claiming task. When the chat_session
|
||
// pointer is missing (legacy NULL runtime_id), stale (last task
|
||
// failed before reporting completion), or runtime-mismatched, fall
|
||
// back to the most recent task row that recorded a session_id —
|
||
// otherwise a single failed turn would silently drop the entire
|
||
// conversation memory on the next message. The fallback also
|
||
// requires runtime to match.
|
||
if cs.SessionID.Valid && cs.RuntimeID.Valid && cs.RuntimeID == task.RuntimeID {
|
||
resp.PriorSessionID = cs.SessionID.String
|
||
}
|
||
if cs.WorkDir.Valid {
|
||
resp.PriorWorkDir = cs.WorkDir.String
|
||
}
|
||
}
|
||
// Resolve the user-message input batch for this run. A task-owned
|
||
// task (chat_input_task_id set) reads exactly the user messages
|
||
// tagged with its own input owner, so a message that arrived after
|
||
// this turn was sealed can never be absorbed here. Direct-chat
|
||
// tasks have owned their single message since MUL-4351; channel
|
||
// (Slack/Lark) tasks now seal their trailing batch at enqueue too.
|
||
// Only legacy tasks created before that deploy carry a NULL owner
|
||
// and keep the trailing-message selector — the run of user messages
|
||
// after the last assistant row, which also covers a debounced burst
|
||
// (MUL-2968: "看上海天气" then "还有青岛" must both be delivered) —
|
||
// so a rolling deploy never replays their history. Attachments are
|
||
// collected per included message so the agent can
|
||
// `multica attachment download <id>` (the inline markdown URL is
|
||
// signed + 30-min expiring on the CDN).
|
||
var unanswered []db.ChatMessage
|
||
var inputLoadErr error
|
||
if task.ChatInputTaskID.Valid {
|
||
unanswered, inputLoadErr = h.Queries.ListChatInputMessages(r.Context(), task.ChatInputTaskID)
|
||
} else if msgs, err := h.Queries.ListChatMessagesForLegacyTask(r.Context(), cs.ID); err == nil {
|
||
unanswered = trailingUserMessages(msgs)
|
||
} else {
|
||
inputLoadErr = err
|
||
}
|
||
// A read failure must NOT masquerade as "zero input". Preserve the
|
||
// just-dispatched task for batch recovery (or singular stale reclaim)
|
||
// and reject its payload, rather than cancelling a valid direct task
|
||
// on a transient DB error (MUL-4351 review).
|
||
if inputLoadErr != nil {
|
||
slog.Error("chat claim: load chat input messages failed; preserving task for redelivery",
|
||
"task_id", uuidToString(task.ID),
|
||
"chat_session_id", uuidToString(cs.ID),
|
||
"error", inputLoadErr)
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, &claimBuildFailure{
|
||
outcome: "error_chat_input_load",
|
||
status: http.StatusInternalServerError,
|
||
message: "failed to load chat input",
|
||
}
|
||
}
|
||
|
||
// Input ownership is the claim's fail-closed boundary. Resume-history
|
||
// reads belong after it: a task that cannot load its input is preserved
|
||
// for redelivery and must not spend two more queries before returning.
|
||
if !task.ForceFreshSession {
|
||
contextRevision := pgtype.Int8{}
|
||
if task.ChannelContextRevision.Valid {
|
||
contextRevision = task.ChannelContextRevision
|
||
}
|
||
// GetLastChatTaskSession currently has exactly two consumers: the
|
||
// prior session and prior workdir fields. Keep this guard coupled to
|
||
// both so adding a third consumer cannot silently skip its fallback.
|
||
// Context-scoped channel tasks always resolve both fields from task
|
||
// rows in their own generation; the Chat-wide pointer may belong to a
|
||
// different generation when independent debounce timers race.
|
||
if task.ChannelContextRevision.Valid || chatSessionResumeFallbackNeeded(resp.PriorSessionID, resp.PriorWorkDir) {
|
||
h.Metrics.RecordChatClaimSessionFallbackNeeded()
|
||
started := time.Now()
|
||
prior, err := h.Queries.GetLastChatTaskSession(r.Context(), db.GetLastChatTaskSessionParams{
|
||
ChatSessionID: cs.ID,
|
||
ChannelContextRevision: contextRevision,
|
||
})
|
||
h.Metrics.ObserveChatClaimLastSessionQuery(time.Since(started).Seconds())
|
||
switch {
|
||
case err == nil && prior.SessionID.Valid:
|
||
h.Metrics.RecordChatClaimSessionFallbackHit()
|
||
if resp.PriorSessionID == "" && prior.RuntimeID == task.RuntimeID {
|
||
resp.PriorSessionID = prior.SessionID.String
|
||
}
|
||
if prior.WorkDir.Valid && resp.PriorWorkDir == "" {
|
||
resp.PriorWorkDir = prior.WorkDir.String
|
||
}
|
||
case errors.Is(err, pgx.ErrNoRows):
|
||
h.Metrics.RecordChatClaimSessionFallbackMiss()
|
||
case err == nil:
|
||
// Defensive only: the SQL excludes NULL session ids, but
|
||
// preserve miss semantics if that contract ever changes.
|
||
h.Metrics.RecordChatClaimSessionFallbackMiss()
|
||
default:
|
||
h.Metrics.RecordChatClaimSessionFallbackError()
|
||
}
|
||
}
|
||
|
||
// MUL-5305: continuity-gap disclosure is independent of whether
|
||
// either pointer field needed fallback, so this query stays
|
||
// unconditional for non-force-fresh chat claims.
|
||
started := time.Now()
|
||
missing, err := h.Queries.GetLatestChatTaskRolloutMissing(r.Context(), db.GetLatestChatTaskRolloutMissingParams{
|
||
ChatSessionID: cs.ID,
|
||
ChannelContextRevision: contextRevision,
|
||
})
|
||
h.Metrics.ObserveChatClaimRolloutMissingQuery(time.Since(started).Seconds())
|
||
if err == nil && missing {
|
||
resp.PriorSessionResumeUnavailable = true
|
||
}
|
||
} else if task.RetryOfTaskID.Valid {
|
||
// Same as the issue branch. The retry lineage is what separates this
|
||
// from a user-requested fresh start (the Lark fresh-session command),
|
||
// which still inherits nothing.
|
||
applyFreshSessionRetryWorkdir(*task, &resp, requestHasClientCapability(r, protocol.DaemonCapabilityCheckoutKeepsWorkV1))
|
||
}
|
||
|
||
parts := make([]string, 0, len(unanswered))
|
||
// Batch attachment lookups instead of one query per unanswered message
|
||
// (N+1). The batch orders globally by created_at, so grouping by
|
||
// chat_message_id keeps each message's attachments in created_at order,
|
||
// and iterating unanswered below in its original order reproduces the
|
||
// exact append order of the previous per-message loop.
|
||
messageIDs := make([]pgtype.UUID, 0, len(unanswered))
|
||
for _, m := range unanswered {
|
||
messageIDs = append(messageIDs, m.ID)
|
||
}
|
||
attsByMessage := make(map[string][]db.Attachment, len(unanswered))
|
||
if len(messageIDs) > 0 {
|
||
atts, attErr := h.Queries.ListAttachmentsByChatMessageIDs(r.Context(), db.ListAttachmentsByChatMessageIDsParams{
|
||
Column1: messageIDs,
|
||
WorkspaceID: parseUUID(resp.WorkspaceID),
|
||
})
|
||
// A read failure must NOT masquerade as "no attachments". Attachment
|
||
// IDs are the agent's only handle for downloading user files, and
|
||
// batching widened the blast radius from one message's attachments to
|
||
// the whole turn's. Fail closed exactly like the chat-input load above
|
||
// (MUL-6788 review): preserve the just-dispatched task so the
|
||
// stale-dispatched reclaim redelivers it, rather than starting the run
|
||
// with files silently missing.
|
||
if attErr != nil {
|
||
slog.Error("chat claim: load chat attachments failed; preserving task for redelivery",
|
||
"task_id", uuidToString(task.ID),
|
||
"chat_session_id", uuidToString(cs.ID),
|
||
"error", attErr)
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, &claimBuildFailure{
|
||
outcome: "error_chat_attachment_load",
|
||
status: http.StatusInternalServerError,
|
||
message: "failed to load chat attachments",
|
||
}
|
||
}
|
||
for _, a := range atts {
|
||
mid := uuidToString(a.ChatMessageID)
|
||
attsByMessage[mid] = append(attsByMessage[mid], a)
|
||
}
|
||
}
|
||
for _, m := range unanswered {
|
||
if strings.TrimSpace(m.Content) != "" {
|
||
parts = append(parts, m.Content)
|
||
}
|
||
for _, a := range attsByMessage[uuidToString(m.ID)] {
|
||
resp.ChatMessageAttachments = append(resp.ChatMessageAttachments, ChatAttachmentMeta{
|
||
ID: uuidToString(a.ID),
|
||
Filename: a.Filename,
|
||
ContentType: a.ContentType,
|
||
})
|
||
}
|
||
}
|
||
resp.ChatMessage = strings.Join(parts, "\n\n")
|
||
|
||
// Fail closed: a task-owned direct task that resolves to no user text
|
||
// (and is not the agent's proactive intro) must never dispatch an
|
||
// empty prompt. The send path creates the owning user message in the
|
||
// same transaction as the task, so this only fires on genuinely
|
||
// corrupt state — cancel the just-dispatched task and reject the claim
|
||
// rather than run the agent with nothing to answer (MUL-4351).
|
||
if task.ChatInputTaskID.Valid && !resp.ChatIntro && strings.TrimSpace(resp.ChatMessage) == "" {
|
||
slog.Error("chat claim: task-owned direct task has no user input; cancelling",
|
||
"task_id", uuidToString(task.ID),
|
||
"chat_session_id", uuidToString(cs.ID),
|
||
"chat_input_task_id", uuidToString(task.ChatInputTaskID),
|
||
)
|
||
if _, cerr := h.TaskService.CancelTask(r.Context(), task.ID); cerr != nil {
|
||
slog.Error("chat claim: cancel after empty input failed",
|
||
"task_id", uuidToString(task.ID), "error", cerr)
|
||
}
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, &claimBuildFailure{
|
||
outcome: "error_empty_chat_input",
|
||
status: http.StatusInternalServerError,
|
||
message: "chat task has no user input",
|
||
}
|
||
}
|
||
|
||
if strings.TrimSpace(resp.ThreadName) == "" && resp.ChatMessage != "" {
|
||
resp.ThreadName = resp.ChatMessage
|
||
}
|
||
}
|
||
|
||
// Autopilot run_only task: resolve workspace from autopilot_run →
|
||
// autopilot, and include the autopilot instructions because there is no
|
||
// issue for the agent to fetch.
|
||
if task.AutopilotRunID.Valid {
|
||
run, runErr := h.Queries.GetAutopilotRun(r.Context(), task.AutopilotRunID)
|
||
if runErr != nil {
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount,
|
||
h.rejectClaimSourceLoad(r.Context(), task, runErr, "autopilot run", uuidToString(task.AutopilotRunID))
|
||
}
|
||
ap, apErr := h.Queries.GetAutopilot(r.Context(), run.AutopilotID)
|
||
if apErr != nil {
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount,
|
||
h.rejectClaimSourceLoad(r.Context(), task, apErr, "autopilot", uuidToString(run.AutopilotID))
|
||
}
|
||
|
||
// The autopilot owns this task's workspace. taskToResponse seeded
|
||
// resp.WorkspaceID with the RUNTIME's workspace, so the old
|
||
// `if resp.WorkspaceID == ""` guard never fired and left the isolation
|
||
// check comparing the runtime workspace against itself. Assign
|
||
// authoritatively, then prove ownership before exposing any
|
||
// autopilot-owned context.
|
||
resp.WorkspaceID = uuidToString(ap.WorkspaceID)
|
||
if failure := h.rejectClaimOnWorkspaceMismatch(r.Context(), task, resp.WorkspaceID, runtimeID, runtimeWorkspaceID, false); failure != nil {
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, failure
|
||
}
|
||
|
||
resp.AutopilotID = uuidToString(run.AutopilotID)
|
||
resp.AutopilotSource = run.Source
|
||
if run.TriggerPayload != nil {
|
||
resp.AutopilotTriggerPayload = json.RawMessage(run.TriggerPayload)
|
||
}
|
||
resp.AutopilotTitle = ap.Title
|
||
resp.ThreadName = ap.Title
|
||
if ap.Description.Valid {
|
||
resp.AutopilotDescription = ap.Description.String
|
||
}
|
||
|
||
// A run_only autopilot has no issue from which to inherit project
|
||
// context, so its configured project is hydrated here. That gives the
|
||
// daemon the same resource contract as issue-bound and quick-create
|
||
// tasks: project repositories scope the checkout, while local_directory
|
||
// lets the daemon select the bound path and write its managed manifest.
|
||
projectCtx, projectErr := h.resolveClaimProjectContext(r.Context(), ap.ProjectID, ap.WorkspaceID)
|
||
if projectErr != nil {
|
||
slog.Error("autopilot claim: load project context failed; preserving task for redelivery",
|
||
"task_id", uuidToString(task.ID),
|
||
"autopilot_id", uuidToString(run.AutopilotID),
|
||
"error", projectErr)
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, &claimBuildFailure{
|
||
outcome: "error_project_context",
|
||
status: http.StatusInternalServerError,
|
||
message: "failed to load project context",
|
||
}
|
||
}
|
||
projectCtx.applyTo(&resp)
|
||
}
|
||
|
||
// Quick-create task: no issue / chat / autopilot link — workspace and
|
||
// prompt come from the task's context JSONB. Resolve workspace from
|
||
// there so the isolation check below has something to compare.
|
||
hasQuickCreate := false
|
||
if task.Context != nil && !task.IssueID.Valid && !task.ChatSessionID.Valid && !task.AutopilotRunID.Valid {
|
||
var qc service.QuickCreateContext
|
||
if json.Unmarshal(task.Context, &qc) == nil && qc.Type == service.QuickCreateContextType {
|
||
hasQuickCreate = true
|
||
resp.QuickCreatePrompt = qc.Prompt
|
||
resp.QuickCreatePriority = qc.Priority
|
||
resp.QuickCreateDueDate = qc.DueDate
|
||
resp.QuickCreateAttachmentIDs = append([]string(nil), qc.AttachmentIDs...)
|
||
resp.ThreadName = qc.Prompt
|
||
resp.WorkspaceID = qc.WorkspaceID
|
||
if failure := h.rejectClaimOnWorkspaceMismatch(r.Context(), task, resp.WorkspaceID, runtimeID, runtimeWorkspaceID, true); failure != nil {
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, failure
|
||
}
|
||
if qc.SourceContextID != "" {
|
||
workspaceID, workspaceErr := util.ParseUUID(qc.WorkspaceID)
|
||
contextID, contextIDErr := util.ParseUUID(qc.SourceContextID)
|
||
if workspaceErr != nil || contextIDErr != nil {
|
||
return resp, nil, nil, 0, 0, h.failClaimedTaskBeforeLaunch(
|
||
r.Context(), task,
|
||
"Captured source context is invalid. Start again from the branch point.",
|
||
taskfailure.ReasonAgentUnknown,
|
||
"error_source_context_invalid", http.StatusConflict, "captured source context is invalid",
|
||
)
|
||
}
|
||
captured, contextErr := h.Queries.GetIssueSourceContextByID(r.Context(), db.GetIssueSourceContextByIDParams{WorkspaceID: workspaceID, ID: contextID})
|
||
if contextErr != nil && !errors.Is(contextErr, pgx.ErrNoRows) {
|
||
if _, requeueErr := h.TaskService.RequeueTaskAfterClaimFailure(r.Context(), *task); requeueErr != nil {
|
||
slog.Error("quick-create claim: requeue after source context lookup failed", "task_id", uuidToString(task.ID), "error", requeueErr)
|
||
}
|
||
return resp, nil, nil, 0, 0, &claimBuildFailure{
|
||
outcome: "error_source_context_load", status: http.StatusInternalServerError, message: "failed to load captured source context",
|
||
}
|
||
}
|
||
if contextErr != nil || captured.State != "pending" || captured.OriginTaskID != task.ID {
|
||
return resp, nil, nil, 0, 0, h.failClaimedTaskBeforeLaunch(
|
||
r.Context(), task,
|
||
"Captured source context is no longer available. Start again from the branch point.",
|
||
taskfailure.ReasonAgentUnknown,
|
||
"error_source_context_unavailable", http.StatusConflict, "captured source context is unavailable",
|
||
)
|
||
}
|
||
resp.QuickCreateSourceContext = append(json.RawMessage(nil), captured.Snapshot...)
|
||
}
|
||
// When the user picked a project in the modal, surface its title
|
||
// and resources to the daemon so the agent has the same context
|
||
// it would for an issue-bound task. A prompt-supplied project id
|
||
// that does not parse is treated as no project at all.
|
||
var quickCreateProjectID pgtype.UUID
|
||
if qc.ProjectID != "" {
|
||
if parsed, err := util.ParseUUID(qc.ProjectID); err == nil {
|
||
quickCreateProjectID = parsed
|
||
}
|
||
}
|
||
projectCtx, projectErr := h.resolveClaimProjectContext(r.Context(), quickCreateProjectID, parseUUID(qc.WorkspaceID))
|
||
if projectErr != nil {
|
||
slog.Error("quick-create claim: load project context failed; preserving task for redelivery",
|
||
"task_id", uuidToString(task.ID),
|
||
"project_id", qc.ProjectID,
|
||
"error", projectErr)
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, &claimBuildFailure{
|
||
outcome: "error_project_context",
|
||
status: http.StatusInternalServerError,
|
||
message: "failed to load project context",
|
||
}
|
||
}
|
||
projectCtx.applyTo(&resp)
|
||
|
||
// Parent-issue resolution for quick-create tasks opened from
|
||
// "Add sub issue". The handler already verified workspace
|
||
// membership at submit time; here we re-fetch to pull the
|
||
// human-readable identifier (e.g. MUL-123) the agent will
|
||
// reference in the prompt. If the parent was deleted between
|
||
// submit and claim we surface the UUID anyway — the agent
|
||
// still passes `--parent <uuid>` and the server-side create
|
||
// will fail loud, which is a better outcome than silently
|
||
// dropping the sub-issue intent.
|
||
if qc.ParentIssueID != "" {
|
||
parentExists := true
|
||
if parentUUID, err := util.ParseUUID(qc.ParentIssueID); err == nil {
|
||
if wsUUID, wsErr := util.ParseUUID(qc.WorkspaceID); wsErr == nil {
|
||
parent, perr := h.Queries.GetIssueInWorkspace(r.Context(), db.GetIssueInWorkspaceParams{
|
||
ID: parentUUID,
|
||
WorkspaceID: wsUUID,
|
||
})
|
||
if perr == nil && parent.ID.Valid {
|
||
if ws, werr := h.Queries.GetWorkspace(r.Context(), wsUUID); werr == nil {
|
||
resp.ParentIssueIdentifier = ws.IssuePrefix + "-" + strconv.Itoa(int(parent.Number))
|
||
}
|
||
} else if qc.SourceContextID != "" && errors.Is(perr, pgx.ErrNoRows) {
|
||
// A contextual quick-create already owns an immutable
|
||
// snapshot. If its source was deleted before this first
|
||
// issue exists, create the eventual target as top-level —
|
||
// the same state application-layer detach would have
|
||
// produced had deletion happened just after creation.
|
||
// Ordinary Add sub-issue keeps its historical fail-loud
|
||
// behavior when its parent disappears.
|
||
parentExists = false
|
||
}
|
||
}
|
||
}
|
||
if parentExists {
|
||
resp.ParentIssueID = qc.ParentIssueID
|
||
}
|
||
}
|
||
|
||
// Squad-leader briefing injection for quick-create tasks. When
|
||
// the user picked a squad in the modal, the task runs on the
|
||
// squad's leader agent (resolved by the handler). Surface the
|
||
// same Operating Protocol + Roster + user Instructions that
|
||
// issue-bound squad tasks see, so the leader can decide to
|
||
// delegate before opening the issue.
|
||
if resp.Agent != nil && qc.SquadID != "" {
|
||
wsUUID, wsErr := util.ParseUUID(qc.WorkspaceID)
|
||
squadUUID, sqErr := util.ParseUUID(qc.SquadID)
|
||
if wsErr == nil && sqErr == nil {
|
||
if squad, err := h.Queries.GetSquadInWorkspace(r.Context(), db.GetSquadInWorkspaceParams{
|
||
ID: squadUUID,
|
||
WorkspaceID: wsUUID,
|
||
}); err == nil && uuidToString(squad.LeaderID) == resp.Agent.ID {
|
||
// Quick-create has no issue yet — there is no parent
|
||
// status to own on this turn. Once the leader opens the
|
||
// issue with the squad as assignee, the issue-bound
|
||
// claim path above grants ownership.
|
||
briefing := buildSquadLeaderBriefing(r.Context(), h.Queries, squad, false)
|
||
if strings.TrimSpace(resp.Agent.Instructions) == "" {
|
||
resp.Agent.Instructions = briefing
|
||
} else {
|
||
resp.Agent.Instructions = resp.Agent.Instructions + "\n\n" + briefing
|
||
}
|
||
// Surface the squad identity to the daemon so the
|
||
// quick-create prompt defaults the new issue's
|
||
// assignee to the squad, not the leader agent.
|
||
resp.SquadID = uuidToString(squad.ID)
|
||
resp.SquadName = squad.Name
|
||
slog.Debug("injected squad leader briefing for quick-create",
|
||
"squad_id", uuidToString(squad.ID),
|
||
"squad_name", squad.Name,
|
||
"leader_agent_id", resp.Agent.ID,
|
||
)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Catch-all workspace isolation check. Every context branch above already
|
||
// ran this the moment it resolved its owning workspace, which is what makes
|
||
// "prove ownership, then hydrate" structural rather than per-branch
|
||
// discipline. This final call is the backstop for a task that reached here
|
||
// without any branch claiming it.
|
||
if failure := h.rejectClaimOnWorkspaceMismatch(r.Context(), task, resp.WorkspaceID, runtimeID, runtimeWorkspaceID, hasQuickCreate); failure != nil {
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, failure
|
||
}
|
||
|
||
// Workspace-level Context (workspace.context DB column) — the per-workspace
|
||
// system prompt that workspace owners set in Settings → General. Inject it
|
||
// into the brief regardless of task kind (issue / chat / autopilot /
|
||
// quick-create) so every agent running in the workspace sees the same
|
||
// shared context. Empty string when the owner hasn't set one; the daemon
|
||
// skips rendering the heading in that case.
|
||
if ws, err := h.Queries.GetWorkspace(r.Context(), parseUUID(resp.WorkspaceID)); err == nil {
|
||
resp.WorkspaceSlug = ws.Slug
|
||
if issueNumber > 0 {
|
||
resp.IssueIdentifier = service.IssueIdentifier(ws.IssuePrefix, issueNumber)
|
||
}
|
||
if ws.Context.Valid {
|
||
resp.WorkspaceContext = ws.Context.String
|
||
}
|
||
} else {
|
||
slog.Warn("task claim: failed to load workspace for context injection",
|
||
"task_id", uuidToString(task.ID),
|
||
"workspace_id", resp.WorkspaceID,
|
||
"error", err,
|
||
)
|
||
}
|
||
|
||
// Workspace status catalog (MUL-6460): active CUSTOM statuses only, so the
|
||
// daemon can render them into the brief's status-command line. Read on every
|
||
// claim, like the agent row, so an admin's edit lands on the next task.
|
||
// Failure degrades to the built-in-only brief rather than failing the claim.
|
||
if entries, err := h.Queries.ListIssueStatusEntries(r.Context(), db.ListIssueStatusEntriesParams{
|
||
WorkspaceID: parseUUID(resp.WorkspaceID),
|
||
IncludeArchived: false,
|
||
}); err != nil {
|
||
slog.Warn("task claim: failed to load issue status catalog for brief injection",
|
||
"task_id", uuidToString(task.ID),
|
||
"workspace_id", resp.WorkspaceID,
|
||
"error", err,
|
||
)
|
||
} else {
|
||
for _, entry := range entries {
|
||
if entry.IsSystem {
|
||
continue
|
||
}
|
||
if len(resp.IssueStatuses) == taskIssueStatusCap {
|
||
resp.IssueStatusesOmitted++
|
||
continue
|
||
}
|
||
// Older daemons only render the seven legacy category values.
|
||
resp.IssueStatuses = append(resp.IssueStatuses, TaskIssueStatusData{
|
||
Key: entry.Key,
|
||
Name: entry.Name,
|
||
Category: issuestatus.WireCategory(entry.Key, entry.Category),
|
||
Description: entry.Description,
|
||
})
|
||
}
|
||
}
|
||
|
||
// Last gate before dispatch: refuse to hand a worktree-mode local_directory
|
||
// task to a daemon that cannot implement the mode.
|
||
//
|
||
// The save-time gate in project_resource.go cannot cover this. It checks the
|
||
// version at the moment the resource is written; a machine downgraded
|
||
// afterwards still claims tasks, and an old daemon json-skips execution_mode
|
||
// entirely — it would run the task IN PLACE, editing the working copy the
|
||
// user asked to isolate. That is the exact outcome worktree mode exists to
|
||
// prevent, so it fails closed here, against the version of the runtime that
|
||
// is actually claiming.
|
||
if reason := worktreeClaimBlockReason(
|
||
resp.ProjectResources,
|
||
runtime,
|
||
requestHasClientCapability(r, protocol.DaemonCapabilityLocalWorktreeV1),
|
||
); reason != "" {
|
||
slog.Error("task claim: runtime too old for worktree mode; cancelling rather than running in place",
|
||
"task_id", uuidToString(task.ID),
|
||
"runtime_id", runtimeID,
|
||
"daemon_id", runtime.DaemonID.String,
|
||
"reason", reason,
|
||
)
|
||
// Cancel rather than leave it dispatched: the resource is pinned to this
|
||
// daemon, so redelivery would hand it straight back to the same too-old
|
||
// runtime forever.
|
||
//
|
||
// Cancel WITH the reason persisted on the row. The 4xx below only
|
||
// reaches the daemon's log, and the batch-claim path drops the task from
|
||
// its response entirely — so without this the user is left with a task
|
||
// that says "cancelled" and nothing else, for a condition only they can
|
||
// fix (update the app on that machine).
|
||
//
|
||
// Through the TaskService, not a raw query: cancellation has side
|
||
// effects beyond the row — audit capture, chat settle, agent status
|
||
// reconcile, the task:cancelled broadcast that clears live cards, and
|
||
// NotifyTaskFinished waking capacity/serial waiters. A direct query
|
||
// leaves all of those stale.
|
||
if _, cerr := h.TaskService.CancelTaskWithReason(r.Context(), task.ID, reason, "local_directory_error"); cerr != nil {
|
||
// The cancel did not commit, so the row is still claimed. The
|
||
// daemon is about to be refused, so left alone the task would
|
||
// strand in dispatched until stale reclaim, with no visible
|
||
// reason. Requeue it instead — the next claim re-runs this gate
|
||
// and retries the cancel — and report a 5xx so the daemon reads
|
||
// this as a transient server problem, not a claim refusal.
|
||
slog.Error("task claim: cancel after worktree version gate failed; requeueing so the gate can run again",
|
||
"task_id", uuidToString(task.ID), "error", cerr)
|
||
if _, rerr := h.TaskService.RequeueTaskAfterClaimFailure(r.Context(), *task); rerr != nil {
|
||
slog.Error("task claim: requeue after worktree-gate cancel failure failed; stale reclaim will recover it",
|
||
"task_id", uuidToString(task.ID), "error", rerr)
|
||
}
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, &claimBuildFailure{
|
||
outcome: "error_worktree_gate_cancel",
|
||
status: http.StatusInternalServerError,
|
||
message: "failed to cancel a worktree task blocked by daemon version; task requeued",
|
||
}
|
||
}
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, &claimBuildFailure{
|
||
outcome: "error_worktree_daemon_version",
|
||
status: http.StatusUnprocessableEntity,
|
||
message: reason,
|
||
}
|
||
}
|
||
|
||
// Wakeup rules that waited for this run hand it their inputs now, after
|
||
// every gate passed, and only for a daemon that renders them; otherwise
|
||
// they keep their inputs and start their own run.
|
||
if requestHasClientCapability(r, protocol.DaemonCapabilityJoinedWakeupsV1) {
|
||
joined, err := (&service.IssueWakeupService{Tasks: h.TaskService}).JoinWaitingWakeups(r.Context(), *task)
|
||
if err != nil {
|
||
slog.Warn("daemon claim: waiting wakeups keep their inputs", "task_id", uuidToString(task.ID), "error", err)
|
||
} else {
|
||
task.Context = joined
|
||
resp.WakeupJoined = service.JoinedWakeupNotes(joined)
|
||
}
|
||
}
|
||
|
||
// Hydrate attribution only after every source/workspace/version gate has
|
||
// passed so a rejected claim cannot receive another user's profile data.
|
||
// The existing flat initiator fields carry the run's authorization human to
|
||
// installed daemons as well as current ones. Direct trigger authors remain
|
||
// available separately through trigger_author_*.
|
||
h.hydrateTaskAttributions(r.Context(), []*TaskAttribution{resp.Attribution})
|
||
if resp.Attribution != nil && resp.Attribution.Originator != nil {
|
||
originator := resp.Attribution.Originator
|
||
resp.InitiatorType = "member"
|
||
resp.InitiatorID = originator.ID
|
||
resp.InitiatorName = originator.Name
|
||
resp.InitiatorEmail = originator.Email
|
||
}
|
||
|
||
return resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, nil
|
||
}
|
||
|
||
// worktreeClaimBlockReason returns a user-facing reason when this runtime must
|
||
// not run the task, or "" when it may proceed.
|
||
//
|
||
// The decision is made on the CAPABILITY the daemon advertised on this very
|
||
// request, not on its version string. A daemon without the implementation does
|
||
// not merely lose concurrency — it json-skips execution_mode and runs the task
|
||
// in place, editing the working copy the user asked to isolate. Version strings
|
||
// could not answer that reliably: the git-describe dev-build exemption that
|
||
// keeps `make daemon` unblocked let a v0.4.23-era daemon straight through, and
|
||
// two tasks silently ran in the user's own directory (MUL-5707).
|
||
//
|
||
// Only resources bound to the claiming runtime's own daemon are considered: a
|
||
// project may carry one local_directory per machine, and another machine's
|
||
// worktree resource says nothing about this one's ability to run the task.
|
||
func worktreeClaimBlockReason(resources []ProjectResourceData, runtime db.AgentRuntime, hasWorktreeCapability bool) string {
|
||
if !runtime.DaemonID.Valid || runtime.DaemonID.String == "" {
|
||
return ""
|
||
}
|
||
if hasWorktreeCapability {
|
||
return ""
|
||
}
|
||
for _, res := range resources {
|
||
if res.ResourceType != "local_directory" {
|
||
continue
|
||
}
|
||
var ref localDirectoryRef
|
||
if err := json.Unmarshal(res.ResourceRef, &ref); err != nil {
|
||
continue
|
||
}
|
||
if ref.ExecutionMode != localDirectoryModeWorktree || ref.DaemonID != runtime.DaemonID.String {
|
||
continue
|
||
}
|
||
return fmt.Sprintf(
|
||
"This machine's Multica runtime does not support parallel (worktree) mode, which %q is set to use. "+
|
||
"Update the Multica app on that machine to the latest version, then re-run this task. "+
|
||
"Refusing to run rather than falling back to editing the directory directly, which is what this mode exists to prevent.",
|
||
ref.LocalPath)
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// ClaimTaskByRuntime atomically claims the next queued task for a runtime.
|
||
// The response includes the agent's name and skills, fetched fresh from the DB.
|
||
func (h *Handler) ClaimTaskByRuntime(w http.ResponseWriter, r *http.Request) {
|
||
runtimeID := chi.URLParam(r, "runtimeId")
|
||
start := time.Now()
|
||
|
||
var (
|
||
outcome = "unauth"
|
||
authMs, claimMs, buildMs int64
|
||
payloadBytes int
|
||
agentSkillCount int
|
||
builtinSkillCount int
|
||
skillPayloadBytes int
|
||
buildStart time.Time
|
||
)
|
||
defer func() {
|
||
// Emit at function exit so error / unauth paths also carry timing.
|
||
// build_ms is computed from buildStart only when we entered the
|
||
// response-build phase (otherwise stays 0).
|
||
if !buildStart.IsZero() {
|
||
buildMs = time.Since(buildStart).Milliseconds()
|
||
}
|
||
logClaimEndpointSlow(runtimeID, outcome, start, authMs, claimMs, buildMs, payloadBytes, agentSkillCount, builtinSkillCount, skillPayloadBytes)
|
||
}()
|
||
|
||
// Verify the caller owns this runtime's workspace. The runtime's
|
||
// workspace_id is the authoritative value a claimed task must match
|
||
// below — a task whose resolved workspace doesn't equal this runtime's
|
||
// workspace is rejected even if it was enqueued against this
|
||
// runtime_id (defense-in-depth against upstream routing bugs).
|
||
runtime, ok := h.requireDaemonRuntimeAccess(w, r, runtimeID)
|
||
if !ok {
|
||
return
|
||
}
|
||
runtimeWorkspaceID := uuidToString(runtime.WorkspaceID)
|
||
authMs = time.Since(start).Milliseconds()
|
||
|
||
claimStart := time.Now()
|
||
task, err := h.TaskService.ClaimTaskForRuntime(r.Context(), parseUUID(runtimeID))
|
||
claimMs = time.Since(claimStart).Milliseconds()
|
||
if err != nil {
|
||
outcome = "error_claim"
|
||
slog.Warn("claim task failed", append(logger.RequestAttrs(r), "error", err)...)
|
||
writeError(w, http.StatusInternalServerError, "failed to claim task")
|
||
return
|
||
}
|
||
|
||
if task == nil {
|
||
slog.Debug("no task to claim", "runtime_id", runtimeID)
|
||
payloadBytes, _ = writeMeasuredJSON(w, http.StatusOK, map[string]any{"task": nil})
|
||
outcome = "no_task"
|
||
return
|
||
}
|
||
if len(task.CoalescedCommentIds) > 0 {
|
||
handled, failure := h.repairStaleCommentPlanIfNeeded(r.Context(), task, runtimeWorkspaceID)
|
||
if handled {
|
||
if failure != nil {
|
||
outcome = failure.outcome
|
||
writeError(w, failure.status, failure.message)
|
||
return
|
||
}
|
||
outcome = "repaired_stale_comment_plan"
|
||
payloadBytes, _ = writeMeasuredJSON(w, http.StatusOK, map[string]any{"task": nil})
|
||
return
|
||
}
|
||
}
|
||
|
||
outcome = "claimed"
|
||
buildStart = time.Now()
|
||
|
||
resp, deliveredCommentIDs, issueSnapshot, agentSkillCount, builtinSkillCount, failure := h.buildClaimedTaskResponse(r, task, runtime, runtimeID, runtimeWorkspaceID)
|
||
if failure != nil {
|
||
outcome = failure.outcome
|
||
if failure.settled {
|
||
payloadBytes, _ = writeMeasuredJSON(w, http.StatusOK, map[string]any{"task": nil})
|
||
return
|
||
}
|
||
writeError(w, failure.status, failure.message)
|
||
return
|
||
}
|
||
commentBackedTask := task.TriggerCommentID.Valid || len(task.CoalescedCommentIds) > 0
|
||
requeueFailedClaim := func(reason string) {
|
||
if _, err := h.TaskService.RequeueTaskAfterClaimFailure(r.Context(), *task); err != nil {
|
||
slog.Error("task claim: failed to requeue after finalization error",
|
||
"task_id", uuidToString(task.ID),
|
||
"reason", reason,
|
||
"error", err,
|
||
)
|
||
}
|
||
}
|
||
// Mint a task-scoped `mat_` token bound to (agent, task, workspace,
|
||
// owner). The daemon will inject this as MULTICA_TOKEN into the agent
|
||
// process instead of its own credential, so any API call the agent
|
||
// makes — even one that strips X-Agent-ID / X-Task-ID headers — is
|
||
// recognized server-side as actor=agent, closing the lateral-movement
|
||
// path on human-only endpoints (e.g. `/api/agents/{id}/env`). Runtime
|
||
// owner is required because task tokens are still bound to an owning user;
|
||
// without one, fail the claim explicitly instead of letting the daemon
|
||
// fall back to a member/owner credential. MUL-3292.
|
||
// Token expires after the queue/runtime upper bound (24h) so it survives
|
||
// long-running tasks but cannot outlive a forgotten one.
|
||
if !runtime.OwnerID.Valid {
|
||
outcome = "error_token"
|
||
slog.Error("task claim: runtime owner missing; cancelling task to avoid unscoped agent credentials",
|
||
"task_id", uuidToString(task.ID),
|
||
"runtime_id", runtimeID,
|
||
"workspace_id", runtimeWorkspaceID,
|
||
)
|
||
if _, cerr := h.TaskService.CancelTask(r.Context(), task.ID); cerr != nil {
|
||
slog.Error("task claim: cancel after missing runtime owner failed",
|
||
"task_id", uuidToString(task.ID), "error", cerr)
|
||
}
|
||
writeError(w, http.StatusInternalServerError, "runtime owner required to mint task token")
|
||
return
|
||
}
|
||
tokenStr, terr := auth.GenerateAgentTaskToken()
|
||
if terr != nil {
|
||
outcome = "error_token"
|
||
slog.Error("task claim: failed to generate agent task token",
|
||
"task_id", uuidToString(task.ID), "error", terr)
|
||
requeueFailedClaim("token_generation")
|
||
writeError(w, http.StatusInternalServerError, "failed to mint task token")
|
||
return
|
||
}
|
||
remoteMCPToken, daemonTokens, derr := remoteMCPDaemonTokenForClaim(resp, runtime)
|
||
if derr != nil {
|
||
outcome = "error_remote_mcp_token"
|
||
slog.Error("task claim: failed to generate Remote MCP daemon token",
|
||
"task_id", uuidToString(task.ID), "error", derr)
|
||
requeueFailedClaim("remote_mcp_token_generation")
|
||
writeError(w, http.StatusInternalServerError, "failed to mint Remote MCP daemon token")
|
||
return
|
||
}
|
||
receipt, deliveryFailure, ferr := h.finalizeClaimDelivery(r.Context(), task, runtime, runtimeID, runtimeWorkspaceID, &resp, db.CreateTaskTokenParams{
|
||
ID: dbid.NewV7(),
|
||
TokenHash: auth.HashToken(tokenStr),
|
||
TaskID: task.ID,
|
||
AgentID: task.AgentID,
|
||
WorkspaceID: parseUUID(resp.WorkspaceID),
|
||
UserID: runtime.OwnerID,
|
||
ExpiresAt: pgtype.Timestamptz{Time: time.Now().Add(24 * time.Hour), Valid: true},
|
||
}, deliveredCommentIDs, commentBackedTask, issueSnapshot, daemonTokens...)
|
||
if ferr != nil {
|
||
outcome = "error_claim_finalize"
|
||
slog.Error("task claim: failed to finalize token and comment delivery receipt",
|
||
"task_id", uuidToString(task.ID), "error", ferr)
|
||
// FinalizeTaskClaim is transactional, so its newly generated token is
|
||
// rolled back with the receipt. Never delete by task here: a stale
|
||
// reclaim can race the original daemon starting, and broad revocation
|
||
// would invalidate that already-authorized execution.
|
||
requeueFailedClaim("token_and_delivery_receipt")
|
||
writeError(w, http.StatusInternalServerError, "failed to finalize task claim")
|
||
return
|
||
}
|
||
if deliveryFailure != nil {
|
||
if !deliveryFailure.settled {
|
||
outcome = deliveryFailure.outcome
|
||
writeError(w, deliveryFailure.status, deliveryFailure.message)
|
||
return
|
||
}
|
||
// The final delivery gate rejected the task (agent rebound, or the
|
||
// runtime's current ownership no longer permits this agent). The task
|
||
// is already terminal via the FailTask path — return the same empty
|
||
// successful poll the queued-mismatch settle path returns.
|
||
outcome = deliveryFailure.outcome
|
||
payloadBytes, _ = writeMeasuredJSON(w, http.StatusOK, map[string]any{"task": nil})
|
||
return
|
||
}
|
||
resp.AuthToken = tokenStr
|
||
resp.RemoteMCPDaemonToken = remoteMCPToken
|
||
task.DeliveredCommentIds = receipt
|
||
resp.DeliveredCommentIDs = uuidStringsOrEmpty(receipt)
|
||
|
||
slog.Info("task claimed by runtime", "task_id", uuidToString(task.ID), "runtime_id", runtimeID, "agent_id", uuidToString(task.AgentID), "prior_session", resp.PriorSessionID)
|
||
if resp.Agent != nil && len(resp.Agent.Skills) > 0 {
|
||
if skillPayload, err := json.Marshal(resp.Agent.Skills); err == nil {
|
||
skillPayloadBytes = len(skillPayload)
|
||
}
|
||
} else if resp.Agent != nil && len(resp.Agent.SkillRefs) > 0 {
|
||
if skillPayload, err := json.Marshal(resp.Agent.SkillRefs); err == nil {
|
||
skillPayloadBytes = len(skillPayload)
|
||
}
|
||
}
|
||
payloadBytes, _ = writeMeasuredJSON(w, http.StatusOK, map[string]any{"task": resp})
|
||
}
|
||
|
||
type resolveSkillBundlesRequest struct {
|
||
Skills []resolveSkillBundleRef `json:"skills"`
|
||
}
|
||
|
||
type resolveSkillBundleRef struct {
|
||
ID string `json:"id"`
|
||
Source string `json:"source"`
|
||
Hash string `json:"hash"`
|
||
}
|
||
|
||
// ResolveTaskSkillBundles returns full skill content for refs from a slim
|
||
// claim. The daemon calls this after claim and before execenv.Prepare so
|
||
// runtimes still see complete local skill files at startup.
|
||
//
|
||
// If a requested hash no longer matches the agent's current skill bundle, the
|
||
// endpoint returns the current bundle and hash. Stage 1 does not snapshot skill
|
||
// content at claim time; the daemon validates the returned bundle before
|
||
// writing it to cache and materializing it.
|
||
func (h *Handler) ResolveTaskSkillBundles(w http.ResponseWriter, r *http.Request) {
|
||
runtimeID := chi.URLParam(r, "runtimeId")
|
||
taskID := chi.URLParam(r, "taskId")
|
||
|
||
runtime, ok := h.requireDaemonRuntimeAccess(w, r, runtimeID)
|
||
if !ok {
|
||
return
|
||
}
|
||
task, taskWorkspaceID, ok := h.requireDaemonTaskAccessWithWorkspace(w, r, taskID)
|
||
if !ok {
|
||
return
|
||
}
|
||
if taskWorkspaceID != uuidToString(runtime.WorkspaceID) || uuidToString(task.RuntimeID) != runtimeID {
|
||
writeError(w, http.StatusNotFound, "task not found")
|
||
return
|
||
}
|
||
if task.Status != "dispatched" && task.Status != "waiting_local_directory" {
|
||
writeError(w, http.StatusConflict, "task is not preparing")
|
||
return
|
||
}
|
||
|
||
var req resolveSkillBundlesRequest
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
if len(req.Skills) == 0 {
|
||
writeJSON(w, http.StatusOK, map[string]any{"bundles": []service.AgentSkillData{}})
|
||
return
|
||
}
|
||
|
||
// Validate before reading: a malformed ref is rejected the same way it
|
||
// always was, now without paying for a load first.
|
||
wanted := make([]service.AgentSkillBundleRef, 0, len(req.Skills))
|
||
for _, ref := range req.Skills {
|
||
if ref.ID == "" || ref.Source == "" || ref.Hash == "" {
|
||
writeError(w, http.StatusBadRequest, "invalid skill ref")
|
||
return
|
||
}
|
||
wanted = append(wanted, service.AgentSkillBundleRef{ID: ref.ID, Source: ref.Source})
|
||
}
|
||
|
||
// Load ONLY what was asked for. The daemon resolves one skill per request,
|
||
// so serving these out of the agent's full bundle set meant reading and
|
||
// hashing every skill the agent has, once per request, to return one of
|
||
// them — quadratic in skill count across a cold dispatch.
|
||
allowed, err := h.TaskService.LoadRequestedAgentSkillBundles(r.Context(), task.AgentID, wanted)
|
||
if err != nil {
|
||
// 5xx, not a partial answer: the daemon's resolve retry can recover a
|
||
// transient read, and a bundle assembled from a failed read would pass
|
||
// its client-side validation and be cached as if it were complete.
|
||
slog.Error("resolve skill bundles: load agent skills failed",
|
||
"task_id", uuidToString(task.ID), "agent_id", uuidToString(task.AgentID), "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to load skill bundles")
|
||
return
|
||
}
|
||
|
||
resolved := make([]service.AgentSkillData, 0, len(req.Skills))
|
||
for _, ref := range req.Skills {
|
||
bundle, ok := allowed[service.AgentSkillBundleKey(ref.Source, ref.ID)]
|
||
if !ok {
|
||
writeError(w, http.StatusNotFound, "skill bundle not found")
|
||
return
|
||
}
|
||
if ref.Source == skillbundle.SourcePlugin && bundle.Hash != ref.Hash {
|
||
writeError(w, http.StatusConflict, "pinned plugin skill bundle hash mismatch")
|
||
return
|
||
}
|
||
resolved = append(resolved, bundle)
|
||
}
|
||
|
||
writeJSON(w, http.StatusOK, map[string]any{"bundles": resolved})
|
||
}
|
||
|
||
// trailingUserMessages returns the run of user messages after the last
|
||
// assistant message in a chronologically-ordered chat history — the set the
|
||
// agent has NOT yet replied to. The agent resumes its prior session and only
|
||
// learns of new input through the claim response's chat_message, so a single
|
||
// run that covers a debounced burst (MUL-2968) must deliver every one of
|
||
// these, not just the latest. Every completed or failed run writes an
|
||
// assistant row, so the anchor advances one turn at a time; the result is the
|
||
// whole slice on the first turn and exactly the new message(s) thereafter.
|
||
// Legacy channel tasks also stop at the first unexpired media marker so an old
|
||
// run cannot consume a placeholder that has not been bound yet. New channel
|
||
// tasks use task-owned input batches and do not depend on this fallback.
|
||
func trailingUserMessages(msgs []db.ChatMessage) []db.ChatMessage {
|
||
start := 0
|
||
for i := len(msgs) - 1; i >= 0; i-- {
|
||
if msgs[i].Role != "user" {
|
||
start = i + 1
|
||
break
|
||
}
|
||
}
|
||
msgs = msgs[start:]
|
||
now := time.Now()
|
||
for i := range msgs {
|
||
pending := msgs[i].ChannelMediaPendingUntil
|
||
if pending.Valid && pending.Time.After(now) {
|
||
return msgs[:i]
|
||
}
|
||
}
|
||
return msgs
|
||
}
|
||
|
||
// ListPendingTasksByRuntime returns queued/dispatched tasks for a runtime.
|
||
func (h *Handler) ListPendingTasksByRuntime(w http.ResponseWriter, r *http.Request) {
|
||
runtimeID := chi.URLParam(r, "runtimeId")
|
||
|
||
// Verify the caller owns this runtime's workspace.
|
||
runtime, ok := h.requireDaemonRuntimeAccess(w, r, runtimeID)
|
||
if !ok {
|
||
return
|
||
}
|
||
workspaceID := uuidToString(runtime.WorkspaceID)
|
||
|
||
tasks, err := h.Queries.ListPendingTasksByRuntime(r.Context(), parseUUID(runtimeID))
|
||
if err != nil {
|
||
writeError(w, http.StatusInternalServerError, "failed to list pending tasks")
|
||
return
|
||
}
|
||
|
||
resp := make([]AgentTaskResponse, len(tasks))
|
||
for i, t := range tasks {
|
||
resp[i] = taskToResponse(t, workspaceID)
|
||
}
|
||
writeJSON(w, http.StatusOK, resp)
|
||
}
|
||
|
||
// Task Lifecycle (called by daemon)
|
||
|
||
// ExtendTaskPrepareLease keeps a dispatched task protected while the daemon is
|
||
// resolving startup inputs and preparing the execution environment.
|
||
func (h *Handler) ExtendTaskPrepareLease(w http.ResponseWriter, r *http.Request) {
|
||
runtimeID := chi.URLParam(r, "runtimeId")
|
||
taskID := chi.URLParam(r, "taskId")
|
||
|
||
runtime, ok := h.requireDaemonRuntimeAccess(w, r, runtimeID)
|
||
if !ok {
|
||
return
|
||
}
|
||
task, taskWorkspaceID, ok := h.requireDaemonTaskAccessWithWorkspace(w, r, taskID)
|
||
if !ok {
|
||
return
|
||
}
|
||
if taskWorkspaceID != uuidToString(runtime.WorkspaceID) || uuidToString(task.RuntimeID) != runtimeID {
|
||
writeError(w, http.StatusNotFound, "task not found")
|
||
return
|
||
}
|
||
|
||
updated, err := h.TaskService.ExtendTaskPrepareLease(r.Context(), parseUUID(taskID), parseUUID(runtimeID))
|
||
if err != nil {
|
||
slog.Warn("extend task prepare lease failed", "task_id", taskID, "runtime_id", runtimeID, "error", err)
|
||
writeError(w, http.StatusBadRequest, err.Error())
|
||
return
|
||
}
|
||
|
||
writeJSON(w, http.StatusOK, taskToResponse(*updated, taskWorkspaceID))
|
||
}
|
||
|
||
// StartTask marks a dispatched task as running.
|
||
func (h *Handler) StartTask(w http.ResponseWriter, r *http.Request) {
|
||
taskID := chi.URLParam(r, "taskId")
|
||
|
||
// Verify the caller owns this task's workspace.
|
||
_, workspaceID, ok := h.requireDaemonTaskAccessWithWorkspace(w, r, taskID)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
var req struct {
|
||
Capabilities []string `json:"capabilities"`
|
||
RuntimeID string `json:"runtime_id"`
|
||
DispatchedAt string `json:"dispatched_at"`
|
||
}
|
||
if r.ContentLength != 0 {
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
}
|
||
enableTaskSupplement := slices.Contains(req.Capabilities, protocol.DaemonCapabilityTaskSupplementV1)
|
||
var task *db.AgentTaskQueue
|
||
var err error
|
||
legacy := req.RuntimeID == "" && req.DispatchedAt == ""
|
||
if legacy {
|
||
// Older daemons send {}. Keep their single-winner behavior; in
|
||
// particular, they cannot acknowledge an already-running task.
|
||
task, err = h.TaskService.StartTask(r.Context(), parseUUID(taskID), enableTaskSupplement)
|
||
} else {
|
||
runtimeID, ok := parseUUIDOrBadRequest(w, req.RuntimeID, "runtime_id")
|
||
if !ok {
|
||
return
|
||
}
|
||
generation, parseErr := time.Parse(time.RFC3339Nano, req.DispatchedAt)
|
||
if parseErr != nil || generation.Nanosecond()%1000 != 0 {
|
||
writeError(w, http.StatusBadRequest, "dispatched_at must be a PostgreSQL claim timestamp")
|
||
return
|
||
}
|
||
task, err = h.TaskService.StartTaskForClaim(r.Context(), db.LockAgentTaskStartClaimParams{
|
||
ID: parseUUID(taskID), RuntimeID: runtimeID,
|
||
DispatchedAt: pgtype.Timestamptz{Time: generation, Valid: true},
|
||
}, enableTaskSupplement)
|
||
}
|
||
if err != nil {
|
||
slog.Warn("start task failed", "task_id", taskID, "error", err)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
status := http.StatusConflict
|
||
if legacy {
|
||
status = http.StatusBadRequest
|
||
}
|
||
writeError(w, status, "task claim is stale or task is no longer startable")
|
||
} else {
|
||
writeError(w, http.StatusInternalServerError, "failed to start task")
|
||
}
|
||
return
|
||
}
|
||
|
||
slog.Info("task started", "task_id", taskID, "agent_id", uuidToString(task.AgentID))
|
||
resp := taskToResponse(*task, workspaceID)
|
||
// Echo the capability the server actually committed for this exact run.
|
||
// A daemon must use this response rather than its own offer: an old server
|
||
// ignores the offer and omits the field, which keeps daemon-first rollouts
|
||
// fail closed without requiring synchronized deployment.
|
||
if task.IssueID.Valid {
|
||
if capability, capabilityErr := h.Queries.GetTaskSupplementCapability(r.Context(), task.ID); capabilityErr == nil {
|
||
resp.SupplementCapability = capability.Capability
|
||
} else if !errors.Is(capabilityErr, pgx.ErrNoRows) {
|
||
slog.Warn("start task: failed to load negotiated supplement capability", "task_id", taskID, "error", capabilityErr)
|
||
}
|
||
}
|
||
writeJSON(w, http.StatusOK, resp)
|
||
}
|
||
|
||
// TaskWaitLocalDirectoryRequest is the body the daemon POSTs when it parks
|
||
// a freshly-dispatched task on a busy local_directory path.
|
||
type TaskWaitLocalDirectoryRequest struct {
|
||
// Reason is a short hint surfaced by the UI alongside the status —
|
||
// typically "<path>" or "<path> (holder: <task short id>)". Small
|
||
// enough to fit on the issue card. Empty is accepted; the column is
|
||
// nullable on the server.
|
||
Reason string `json:"reason"`
|
||
}
|
||
|
||
// MarkTaskWaitingLocalDirectory transitions a dispatched task to
|
||
// waiting_local_directory. Called by the daemon when, after claiming a task
|
||
// whose project carries a local_directory resource, it discovers another
|
||
// in-flight task already holds the path's mutex.
|
||
func (h *Handler) MarkTaskWaitingLocalDirectory(w http.ResponseWriter, r *http.Request) {
|
||
taskID := chi.URLParam(r, "taskId")
|
||
|
||
_, workspaceID, ok := h.requireDaemonTaskAccessWithWorkspace(w, r, taskID)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
var req TaskWaitLocalDirectoryRequest
|
||
if r.ContentLength != 0 {
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
}
|
||
|
||
task, err := h.TaskService.MarkTaskWaitingLocalDirectory(r.Context(), parseUUID(taskID), req.Reason)
|
||
if err != nil {
|
||
slog.Warn("mark task waiting_local_directory failed", "task_id", taskID, "error", err)
|
||
writeError(w, http.StatusBadRequest, err.Error())
|
||
return
|
||
}
|
||
|
||
writeJSON(w, http.StatusOK, taskToResponse(*task, workspaceID))
|
||
}
|
||
|
||
// ReportTaskProgress broadcasts a progress update.
|
||
type TaskProgressRequest struct {
|
||
Summary string `json:"summary"`
|
||
Step int `json:"step"`
|
||
Total int `json:"total"`
|
||
}
|
||
|
||
func (h *Handler) ReportTaskProgress(w http.ResponseWriter, r *http.Request) {
|
||
taskID := chi.URLParam(r, "taskId")
|
||
|
||
var req TaskProgressRequest
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
|
||
// Verify ownership and resolve workspace ID.
|
||
task, ok := h.requireDaemonTaskAccess(w, r, taskID)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
workspaceID := ""
|
||
if task.IssueID.Valid {
|
||
if issue, err := h.Queries.GetIssue(r.Context(), task.IssueID); err == nil {
|
||
workspaceID = uuidToString(issue.WorkspaceID)
|
||
}
|
||
}
|
||
|
||
h.TaskService.ReportProgress(r.Context(), taskID, workspaceID, req.Summary, req.Step, req.Total)
|
||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||
}
|
||
|
||
// CompleteTask marks a running task as completed.
|
||
type TaskCompleteRequest struct {
|
||
PRURL string `json:"pr_url"`
|
||
Output string `json:"output"`
|
||
SessionID string `json:"session_id"` // Claude session ID for future resumption
|
||
WorkDir string `json:"work_dir"` // working directory used during execution
|
||
// DurableWorkDir is the configured project directory that replaces a
|
||
// disposable task worktree after the daemon confirms the worktree is gone.
|
||
DurableWorkDir string `json:"durable_work_dir,omitempty"`
|
||
// BranchName is the branch this run delivered its work on. Worktree-mode
|
||
// local_directory tasks never touch the user's working copy, so this is the
|
||
// only pointer to where the changes went. Empty for every other task kind.
|
||
BranchName string `json:"branch_name,omitempty"`
|
||
// SessionRolloutMissing: the daemon withheld this task's Codex session
|
||
// because its rollout was missing (MUL-5305). Clear the resume pointer and
|
||
// flag the continuity gap for the next claim.
|
||
SessionRolloutMissing bool `json:"session_rollout_missing,omitempty"`
|
||
// RetiredSessionID: a session this run proved unresumable and abandoned
|
||
// (GH #6066). Distinct from an empty SessionID, which only means "nothing
|
||
// to report" — this says "never hand this id to a later run". Older
|
||
// daemons omit it, which is exactly the pre-fix behaviour.
|
||
RetiredSessionID string `json:"retired_session_id,omitempty"`
|
||
}
|
||
|
||
// sanitizeTaskCompleteRequest / sanitizeTaskFailRequest scrub every
|
||
// caller-supplied string on a terminal task callback. Both request types are
|
||
// flat bags of strings, so this is exhaustive by construction — but that also
|
||
// means a NEW string field must be added here, or it reopens GH #7098 through a
|
||
// fresh door. The task-row columns these feed (error, work_dir,
|
||
// durable_work_dir, branch_name, session_id) are all TEXT, and result is
|
||
// JSONB; neither tolerates a NUL.
|
||
func sanitizeTaskCompleteRequest(req *TaskCompleteRequest) {
|
||
req.PRURL = util.SanitizeTextForPostgres(req.PRURL)
|
||
req.Output = util.SanitizeTextForPostgres(req.Output)
|
||
req.SessionID = util.SanitizeTextForPostgres(req.SessionID)
|
||
req.WorkDir = util.SanitizeTextForPostgres(req.WorkDir)
|
||
req.DurableWorkDir = util.SanitizeTextForPostgres(req.DurableWorkDir)
|
||
req.BranchName = util.SanitizeTextForPostgres(req.BranchName)
|
||
req.RetiredSessionID = util.SanitizeTextForPostgres(req.RetiredSessionID)
|
||
}
|
||
|
||
func sanitizeTaskFailRequest(req *TaskFailRequest) {
|
||
req.Error = util.SanitizeTextForPostgres(req.Error)
|
||
req.SessionID = util.SanitizeTextForPostgres(req.SessionID)
|
||
req.WorkDir = util.SanitizeTextForPostgres(req.WorkDir)
|
||
req.DurableWorkDir = util.SanitizeTextForPostgres(req.DurableWorkDir)
|
||
req.FailureReason = util.SanitizeTextForPostgres(req.FailureReason)
|
||
req.BranchName = util.SanitizeTextForPostgres(req.BranchName)
|
||
req.RetiredSessionID = util.SanitizeTextForPostgres(req.RetiredSessionID)
|
||
}
|
||
|
||
func (h *Handler) CompleteTask(w http.ResponseWriter, r *http.Request) {
|
||
taskID := chi.URLParam(r, "taskId")
|
||
|
||
// Verify the caller owns this task's workspace.
|
||
_, workspaceID, ok := h.requireDaemonTaskAccessWithWorkspace(w, r, taskID)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
var req TaskCompleteRequest
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
// Strip bytes PostgreSQL cannot store BEFORE anything reads this payload
|
||
// (GH #7098). The whole request is marshalled into agent_task_queue.result,
|
||
// a JSONB column, and encoding/json renders an embedded NUL as \u0000 —
|
||
// which JSONB rejects (SQLSTATE 22P05), rolling the completion transaction
|
||
// back and leaving the task stuck in 'running' forever. Sanitizing here
|
||
// rather than just before the Marshal is deliberate: the context-exhaustion
|
||
// re-route below feeds req.Output into the failure classifier, and that
|
||
// classifier must see exactly the text we are going to persist.
|
||
sanitizeTaskCompleteRequest(&req)
|
||
|
||
// GH #6402: a daemon whose backend does not (yet) read the provider's
|
||
// structured terminal reason reports a context-exhausted run as a clean
|
||
// success, with the CLI's "your context window is full" notice as the
|
||
// answer. Re-route it to the failure path here so the fix does not have to
|
||
// wait for every installed daemon to update: an un-upgraded host would
|
||
// otherwise keep publishing that notice as the agent's reply AND keep the
|
||
// dead session pinned as the resume pointer, which is a permanently stuck
|
||
// (agent, issue) pair rather than a mislabelled row. Same argument, and the
|
||
// same shared classifier, as taskfailure.NormalizeDaemonReason on the fail
|
||
// boundary (MUL-5370). A current daemon classifies this before it ever calls
|
||
// /complete, so this branch is dead weight for it — by design.
|
||
if taskfailure.ContextExhaustedCompletion(req.Output) {
|
||
slog.Warn("complete task: output is a provider context-exhaustion notice, recording as failed",
|
||
"task_id", taskID,
|
||
"failure_reason", taskfailure.ReasonAgentContextOverflow,
|
||
)
|
||
h.failTask(w, r, taskID, workspaceID, TaskFailRequest{
|
||
Error: req.Output,
|
||
FailureReason: string(taskfailure.ReasonAgentContextOverflow),
|
||
SessionID: req.SessionID,
|
||
WorkDir: req.WorkDir,
|
||
DurableWorkDir: req.DurableWorkDir,
|
||
// Carry the branch across the reroute. The run still delivered one:
|
||
// it ran out of context, it did not fail to produce anything, and
|
||
// dropping the name here would hide the work it did commit.
|
||
BranchName: req.BranchName,
|
||
SessionRolloutMissing: req.SessionRolloutMissing,
|
||
RetiredSessionID: req.RetiredSessionID,
|
||
})
|
||
return
|
||
}
|
||
|
||
result, _ := json.Marshal(req)
|
||
// MUL-5305: SessionRolloutMissing is applied inside CompleteTask's terminal
|
||
// transaction (force session_id NULL + flag the row), so an auto-retry the
|
||
// same commit creates and wakes can never observe the withheld pointer or a
|
||
// missing continuity-gap flag.
|
||
task, transitioned, err := h.TaskService.CompleteTaskWithTransition(r.Context(), parseUUID(taskID), result, req.SessionID, req.WorkDir, req.BranchName, req.SessionRolloutMissing, req.RetiredSessionID, req.DurableWorkDir)
|
||
if err != nil {
|
||
// A CompleteTask error is an infrastructure failure (transaction /
|
||
// assistant-outcome write), not a bad request: an already-finalized
|
||
// callback is treated as idempotent success and returns no error. Return
|
||
// 5xx so the daemon retries the terminal callback and the completion —
|
||
// including the single chat outcome row — lands exactly once (MUL-4351).
|
||
slog.Warn("complete task failed", "task_id", taskID, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to complete task")
|
||
return
|
||
}
|
||
if !transitioned {
|
||
writeJSON(w, http.StatusOK, taskToResponse(*task, workspaceID))
|
||
return
|
||
}
|
||
|
||
h.emitIssueExecutedOnFirstCompletion(r, task)
|
||
|
||
// MUL-4195: guarantee at-least-once processing. If a member posted a
|
||
// deliberate comment while this run was executing (or one was merged into
|
||
// it after its context was built), schedule a single follow-up so the
|
||
// input is not silently dropped. Agent replays are restricted to explicit
|
||
// mentions and recorded worker inputs; see reconcileCommentsOnCompletion.
|
||
h.reconcileCommentsOnCompletion(r.Context(), task)
|
||
// The terminal transaction and completion reconciliation are committed.
|
||
// Wake the owning runtime now so queued work that was blocked by this
|
||
// task's agent capacity or serialization key is re-claimed immediately.
|
||
h.TaskService.NotifyTaskFinished(*task)
|
||
|
||
// Best-effort revoke of any agent task token minted at claim time.
|
||
// The token would naturally expire at the 24h watermark and is also
|
||
// cascaded on agent_task deletion, but eagerly deleting it on
|
||
// completion shrinks the window where a compromised agent process
|
||
// can keep making API calls after its task finishes. Failure here is
|
||
// non-fatal; the expiry / cascade are the durable guards.
|
||
if err := h.Queries.DeleteTaskTokensByTask(r.Context(), task.ID); err != nil {
|
||
slog.Warn("complete task: failed to revoke task tokens", "task_id", uuidToString(task.ID), "error", err)
|
||
}
|
||
|
||
slog.Info("task completed", "task_id", taskID, "agent_id", uuidToString(task.AgentID))
|
||
writeJSON(w, http.StatusOK, taskToResponse(*task, workspaceID))
|
||
}
|
||
|
||
// emitIssueExecutedOnFirstCompletion atomically flips issue.first_executed_at
|
||
// and fires the issue_executed analytics event iff this is the first task on
|
||
// the issue to reach terminal done. Retries / re-assignments / comment-
|
||
// triggered follow-ups hit the WHERE first_executed_at IS NULL clause and
|
||
// no-op, so the funnel counts unique issues, not tasks.
|
||
func (h *Handler) emitIssueExecutedOnFirstCompletion(r *http.Request, task *db.AgentTaskQueue) {
|
||
if task == nil {
|
||
return
|
||
}
|
||
marked, err := h.Queries.MarkIssueFirstExecuted(r.Context(), task.IssueID)
|
||
if err != nil {
|
||
if !isNotFound(err) {
|
||
slog.Warn("analytics: mark issue first-executed failed", "issue_id", uuidToString(task.IssueID), "error", err)
|
||
}
|
||
return
|
||
}
|
||
var durationMS int64
|
||
if task.StartedAt.Valid && task.CompletedAt.Valid {
|
||
durationMS = task.CompletedAt.Time.Sub(task.StartedAt.Time).Milliseconds()
|
||
}
|
||
taskContext := h.TaskService.AnalyticsContextForTask(r.Context(), *task)
|
||
// distinct_id prefers the human creator so agent-driven events flow into
|
||
// the issue-author's person profile (same place signup and
|
||
// workspace_created land). Agent-created issues keep the agent id with a
|
||
// prefix so PostHog doesn't merge them into a user by accident.
|
||
distinct := uuidToString(marked.CreatorID)
|
||
if marked.CreatorType == "agent" {
|
||
distinct = "agent:" + distinct
|
||
}
|
||
obsmetrics.RecordEvent(h.Analytics, h.Metrics, analytics.IssueExecuted(
|
||
distinct,
|
||
uuidToString(marked.WorkspaceID),
|
||
uuidToString(marked.ID),
|
||
uuidToString(task.ID),
|
||
uuidToString(task.AgentID),
|
||
taskContext.Source,
|
||
taskContext.RuntimeMode,
|
||
taskContext.Provider,
|
||
durationMS,
|
||
))
|
||
}
|
||
|
||
// reconcileCommentsOnCompletion replays qualifying comments absent from this
|
||
// run's delivered_comment_ids receipt. Planned/coalesced IDs do not prove the
|
||
// provider received them, particularly with older daemons or deleted comments.
|
||
//
|
||
// Anchor at task creation and exclude the delivered set. A dispatch timestamp
|
||
// would miss a comment whose pre-claim merge lost a race to claiming the task.
|
||
// Replay chronologically through normal enqueue so comments coalesce into one
|
||
// follow-up. That follow-up's later creation time and own receipt bound repetition.
|
||
//
|
||
// Preserve member routing and accepted explicit agent routes, current invocation
|
||
// permissions, and self-trigger guards. Do not invent implicit agent conversation
|
||
// routes or wake an unrelated agent from another agent's completion.
|
||
func (h *Handler) reconcileCommentsOnCompletion(ctx context.Context, task *db.AgentTaskQueue) {
|
||
if task == nil || !task.IssueID.Valid || !task.AgentID.Valid || !task.CreatedAt.Valid {
|
||
return
|
||
}
|
||
plannedCommentIDs := append([]pgtype.UUID{}, task.CoalescedCommentIds...)
|
||
if task.TriggerCommentID.Valid {
|
||
plannedCommentIDs = append(plannedCommentIDs, task.TriggerCommentID)
|
||
}
|
||
comments, err := h.Queries.ListReconcilableCommentsForIssueSince(ctx, db.ListReconcilableCommentsForIssueSinceParams{
|
||
CommentThreadID: task.CommentThreadID,
|
||
IssueID: task.IssueID,
|
||
Since: task.CreatedAt,
|
||
PlannedCommentIds: plannedCommentIDs,
|
||
AgentID: task.AgentID,
|
||
})
|
||
if err != nil {
|
||
slog.Warn("reconcile comments on completion: list comments failed",
|
||
"issue_id", uuidToString(task.IssueID), "task_id", uuidToString(task.ID), "error", err)
|
||
return
|
||
}
|
||
if len(comments) == 0 {
|
||
return
|
||
}
|
||
// The delivered set is the claim-time receipt, not the enqueue-time plan.
|
||
// Legacy tasks backfill only the primary trigger, deliberately replaying
|
||
// coalesced inputs that their daemon may have ignored.
|
||
delivered := make(map[string]struct{}, len(task.DeliveredCommentIds))
|
||
for _, id := range task.DeliveredCommentIds {
|
||
if id.Valid {
|
||
delivered[uuidToString(id)] = struct{}{}
|
||
}
|
||
}
|
||
issue, err := h.Queries.GetIssue(ctx, task.IssueID)
|
||
if err != nil {
|
||
slog.Warn("reconcile comments on completion: load issue failed",
|
||
"issue_id", uuidToString(task.IssueID), "error", err)
|
||
return
|
||
}
|
||
agentID := uuidToString(task.AgentID)
|
||
scheduled := 0
|
||
for i := range comments {
|
||
c := comments[i]
|
||
if _, ok := delivered[uuidToString(c.ID)]; ok {
|
||
// Already delivered to this run (trigger or pre-claim coalesced).
|
||
continue
|
||
}
|
||
if isNoteComment(c.Content) {
|
||
continue
|
||
}
|
||
// A delegated failure recovery signal is platform-authored and targets
|
||
// the exact source coordinator recorded by the failed task. It bypasses
|
||
// generic comment routing (which deliberately treats system authors as
|
||
// unattributed) and excludes this just-completed task from the coverage
|
||
// check because the signal was planned after claim, not delivered to it.
|
||
if service.IsDelegatedFailureRecoveryComment(c) {
|
||
if err := h.TaskService.DispatchDelegatedFailureRecoveryComment(ctx, c, task.ID); err != nil {
|
||
slog.Warn("reconcile comments on completion: delegated failure recovery replay failed",
|
||
"issue_id", uuidToString(task.IssueID),
|
||
"task_id", uuidToString(task.ID),
|
||
"comment_id", uuidToString(c.ID),
|
||
"error", err,
|
||
)
|
||
} else {
|
||
scheduled++
|
||
}
|
||
continue
|
||
}
|
||
var parentComment *db.Comment
|
||
if c.ParentID.Valid {
|
||
// Scope to the issue's workspace; a comment's parent is always in the
|
||
// same workspace, so this only fails closed against a stray foreign
|
||
// UUID rather than changing behavior (MUL-4252).
|
||
if parent, err := h.Queries.GetCommentInWorkspace(ctx, db.GetCommentInWorkspaceParams{
|
||
ID: c.ParentID,
|
||
WorkspaceID: issue.WorkspaceID,
|
||
}); err == nil {
|
||
parentComment = &parent
|
||
}
|
||
}
|
||
// Compute what this comment would trigger, then keep ONLY the agent
|
||
// that just completed — never the full fan-out (that would re-wake
|
||
// unrelated `@other-agent` targets).
|
||
//
|
||
// The comment is routed under its OWN author_type. A member is its own
|
||
// originator. For an agent author, the originator is the human at the
|
||
// top of that agent's trigger chain (resolved from the comment's source
|
||
// task); canInvokeAgent judges an agent→agent (A2A) mention by that
|
||
// originator, not the immediate agent principal (MUL-3963).
|
||
actorType := c.AuthorType
|
||
actorID := uuidToString(c.AuthorID)
|
||
originatorUserID := actorID
|
||
if actorType != "member" {
|
||
originatorUserID = uuidToString(h.TaskService.ResolveOriginatorFromTriggerComment(ctx, issue.WorkspaceID, c.ID))
|
||
}
|
||
triggers, _ := h.computeCommentAgentTriggers(ctx, issue, c.Content, parentComment, actorType, actorID, commentTriggerComputeOptions{
|
||
ExcludeTriggerCommentID: c.ID,
|
||
AuthoringTaskID: c.SourceTaskID,
|
||
OriginatorUserID: originatorUserID,
|
||
})
|
||
// Agent replies discovered only by timestamp must not start a new
|
||
// conversation. Replay explicit mentions, or a worker reply that the
|
||
// creation path already accepted and recorded in this run's input plan.
|
||
// Recomputed routing still checks current permissions and the self guard.
|
||
if actorType != "member" {
|
||
triggers = keepReplayableAgentTriggers(triggers, slices.Contains(plannedCommentIDs, c.ID))
|
||
}
|
||
scoped := make([]commentAgentTrigger, 0, 1)
|
||
for _, trigger := range triggers {
|
||
if uuidToString(trigger.Agent.ID) == agentID {
|
||
scoped = append(scoped, trigger)
|
||
}
|
||
}
|
||
if len(scoped) == 0 {
|
||
continue
|
||
}
|
||
// The first qualifying comment enqueues the follow-up task; later ones
|
||
// find it AlreadyPending and merge in, so all undelivered comments end
|
||
// up covered by a single bounded run.
|
||
//
|
||
// A BLOCKED replay must not be discarded (#5914, Elon round 8): the slot
|
||
// can be held by a task that will never see this comment (it predates that
|
||
// task and is not in its planned ids), so dropping the failure here is
|
||
// exactly how a promised follow-up is lost. Hand the obligation to that
|
||
// blocker instead, keeping it alive until some run provably covers it.
|
||
if res := h.enqueueCommentAgentTriggers(ctx, issue, c.ID, scoped)[agentID]; res.status == DispatchBlocked {
|
||
headSha := h.TaskService.ResolveIssueReviewSHAParam(ctx, task.IssueID)
|
||
if h.propagateUncoveredCommentObligation(ctx, issue, scoped[0], c.ID, headSha) {
|
||
slog.Info("reconcile comments on completion: replay blocked, obligation handed to the active task",
|
||
"issue_id", uuidToString(task.IssueID), "agent_id", agentID, "comment_id", uuidToString(c.ID))
|
||
} else {
|
||
// The slot is held by a DIFFERENT-head queued task: it can neither
|
||
// cover this comment nor accept it (merging would let an old-head
|
||
// run consume a new-head request — TEN-356). Today's schema has no
|
||
// safe place to park the obligation, so surface it loudly rather
|
||
// than let it disappear quietly.
|
||
slog.Error("reconcile comments on completion: replay blocked and obligation could not be handed off; comment needs a durable obligation record",
|
||
"issue_id", uuidToString(task.IssueID), "agent_id", agentID, "comment_id", uuidToString(c.ID),
|
||
"reason", res.reason)
|
||
}
|
||
}
|
||
scheduled++
|
||
}
|
||
if scheduled > 0 {
|
||
slog.Info("reconcile comments on completion: scheduled follow-up",
|
||
"issue_id", uuidToString(task.IssueID),
|
||
"completed_task_id", uuidToString(task.ID),
|
||
"agent_id", agentID,
|
||
"undelivered_comments", scheduled)
|
||
}
|
||
}
|
||
|
||
// keepReplayableAgentTriggers preserves explicit mentions (MUL-4304) and
|
||
// accepted worker replies recorded in the completing run's plan. A timestamp
|
||
// alone never authorizes replay of an implicit agent route. Member comments
|
||
// retain their full routing and do not pass through this filter.
|
||
func keepReplayableAgentTriggers(triggers []commentAgentTrigger, planned bool) []commentAgentTrigger {
|
||
if len(triggers) == 0 {
|
||
return triggers
|
||
}
|
||
filtered := make([]commentAgentTrigger, 0, len(triggers))
|
||
for _, trigger := range triggers {
|
||
switch trigger.Source {
|
||
case commentTriggerSourceMentionAgent, commentTriggerSourceMentionSquadLeader:
|
||
filtered = append(filtered, trigger)
|
||
case commentTriggerSourceIssueAssignee, commentTriggerSourceThreadParent:
|
||
if planned && trigger.NonLeaderAgentReply {
|
||
filtered = append(filtered, trigger)
|
||
}
|
||
}
|
||
}
|
||
return filtered
|
||
}
|
||
|
||
// buildCoalescedCommentData loads the full detail of each comment that was
|
||
// folded into a not-yet-started run (MUL-4195) so the claim response can embed
|
||
// them and the prompt can address each without assuming they share the
|
||
// triggering thread (review should-fix #3). Thread id follows the same rule as
|
||
// the triggering comment (parent id when the comment is a reply, else the
|
||
// comment's own id). Missing comments (deleted / wrong workspace) are skipped
|
||
// rather than failing the claim. The set is bounded by how many comments a user
|
||
// fires before a run starts, so the per-comment lookups stay cheap.
|
||
func (h *Handler) buildCoalescedCommentData(ctx context.Context, workspaceID pgtype.UUID, ids []pgtype.UUID) []CoalescedCommentData {
|
||
if len(ids) == 0 {
|
||
return nil
|
||
}
|
||
out := make([]CoalescedCommentData, 0, len(ids))
|
||
seen := make(map[string]struct{}, len(ids))
|
||
for _, id := range ids {
|
||
if !id.Valid {
|
||
continue
|
||
}
|
||
idString := uuidToString(id)
|
||
if _, ok := seen[idString]; ok {
|
||
continue
|
||
}
|
||
seen[idString] = struct{}{}
|
||
// Workspace-scoped so a foreign comment UUID resolves to "missing"
|
||
// (skipped) instead of leaking another tenant's text into the prompt
|
||
// (MUL-4252). Matches this function's documented skip-on-missing rule.
|
||
comment, err := h.Queries.GetCommentInWorkspace(ctx, db.GetCommentInWorkspaceParams{
|
||
ID: id,
|
||
WorkspaceID: workspaceID,
|
||
})
|
||
// A tombstone (deleted while it still had replies) is as missing as a
|
||
// removed row: there is no body left to deliver.
|
||
if err != nil || comment.DeletedAt.Valid {
|
||
continue
|
||
}
|
||
data := CoalescedCommentData{
|
||
ID: uuidToString(comment.ID),
|
||
ThreadID: uuidToString(comment.ID),
|
||
AuthorType: comment.AuthorType,
|
||
Content: comment.Content,
|
||
CreatedAt: timestampToString(comment.CreatedAt),
|
||
}
|
||
if comment.ParentID.Valid {
|
||
root, err := h.Queries.GetCommentThreadRootID(ctx, comment.ID)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
data.ThreadID = uuidToString(root)
|
||
}
|
||
if comment.AuthorID.Valid {
|
||
switch comment.AuthorType {
|
||
case "agent":
|
||
if a, err := h.Queries.GetAgent(ctx, comment.AuthorID); err == nil {
|
||
data.AuthorName = a.Name
|
||
}
|
||
case "member":
|
||
if u, err := h.Queries.GetUser(ctx, comment.AuthorID); err == nil {
|
||
data.AuthorName = u.Name
|
||
}
|
||
}
|
||
}
|
||
out = append(out, data)
|
||
}
|
||
if len(out) == 0 {
|
||
return nil
|
||
}
|
||
sort.Slice(out, func(i, j int) bool {
|
||
left, leftErr := time.Parse(time.RFC3339Nano, out[i].CreatedAt)
|
||
right, rightErr := time.Parse(time.RFC3339Nano, out[j].CreatedAt)
|
||
if leftErr == nil && rightErr == nil && !left.Equal(right) {
|
||
return left.Before(right)
|
||
}
|
||
if out[i].CreatedAt != out[j].CreatedAt {
|
||
return out[i].CreatedAt < out[j].CreatedAt
|
||
}
|
||
return out[i].ID < out[j].ID
|
||
})
|
||
return out
|
||
}
|
||
|
||
func commentDataIDs(comments []CoalescedCommentData) []pgtype.UUID {
|
||
if len(comments) == 0 {
|
||
return []pgtype.UUID{}
|
||
}
|
||
ids := make([]pgtype.UUID, 0, len(comments))
|
||
for _, comment := range comments {
|
||
ids = append(ids, parseUUID(comment.ID))
|
||
}
|
||
return ids
|
||
}
|
||
|
||
const maxClaimCommentPayloadBytes = 512 << 10 // 512 KiB of comment input per claim
|
||
|
||
// selectCommentDelivery applies a deterministic budget to comment input. The
|
||
// primary trigger is mandatory when it still exists, even when that single
|
||
// comment exceeds the budget; one comment must never become unclaimable. Extra
|
||
// comments are admitted as an oldest-first prefix, so overflow remains a stable
|
||
// suffix for completion reconciliation and cannot starve across retries.
|
||
func selectCommentDelivery(comments []CoalescedCommentData, triggerID string, legacy bool, limit int) []CoalescedCommentData {
|
||
if len(comments) == 0 {
|
||
return nil
|
||
}
|
||
mandatoryID := triggerID
|
||
mandatoryFound := false
|
||
for _, comment := range comments {
|
||
if comment.ID == mandatoryID {
|
||
mandatoryFound = true
|
||
break
|
||
}
|
||
}
|
||
if !mandatoryFound {
|
||
// The planned trigger may have been deleted. Keep the newest available
|
||
// comment so the claim still makes progress and reconcile can pick up the
|
||
// remainder.
|
||
mandatoryID = comments[len(comments)-1].ID
|
||
}
|
||
|
||
selected := map[string]struct{}{mandatoryID: {}}
|
||
used := commentDeliveryBaseSize(legacy) + commentDeliveryEntrySize(commentByID(comments, mandatoryID), legacy)
|
||
for _, comment := range comments {
|
||
if comment.ID == mandatoryID {
|
||
continue
|
||
}
|
||
cost := commentDeliveryEntrySize(comment, legacy)
|
||
if limit > 0 && used+cost > limit {
|
||
break
|
||
}
|
||
selected[comment.ID] = struct{}{}
|
||
used += cost
|
||
}
|
||
|
||
out := make([]CoalescedCommentData, 0, len(selected))
|
||
for _, comment := range comments {
|
||
if _, ok := selected[comment.ID]; ok {
|
||
out = append(out, comment)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func commentByID(comments []CoalescedCommentData, id string) CoalescedCommentData {
|
||
for _, comment := range comments {
|
||
if comment.ID == id {
|
||
return comment
|
||
}
|
||
}
|
||
return CoalescedCommentData{}
|
||
}
|
||
|
||
func commentDeliveryBaseSize(legacy bool) int {
|
||
if legacy {
|
||
return escapedJSONStringContentSize(legacyCommentBundleHeader)
|
||
}
|
||
return 2 // JSON array brackets
|
||
}
|
||
|
||
func commentDeliveryEntrySize(comment CoalescedCommentData, legacy bool) int {
|
||
if legacy {
|
||
return escapedJSONStringContentSize(formatLegacyCommentEntry(comment))
|
||
}
|
||
encoded, err := json.Marshal(comment)
|
||
if err != nil {
|
||
return maxClaimCommentPayloadBytes + 1
|
||
}
|
||
return len(encoded) + 1 // comma between JSON array entries
|
||
}
|
||
|
||
func escapedJSONStringContentSize(value string) int {
|
||
encoded, err := json.Marshal(value)
|
||
if err != nil || len(encoded) < 2 {
|
||
return maxClaimCommentPayloadBytes + 1
|
||
}
|
||
// json.Marshal adds the surrounding string quotes. The remaining bytes
|
||
// match the escaping cost paid when the legacy bundle is nested in the
|
||
// claim response.
|
||
return len(encoded) - 2
|
||
}
|
||
|
||
const legacyCommentBundleHeader = "This run covers multiple distinct issue comments. Address every comment below in chronological order; do not treat this bundle as one rewritten comment.\n"
|
||
|
||
// formatLegacyCommentBundle carries every planned comment through the one
|
||
// field understood by daemons that predate coalesced-comments-v1. Delimiters,
|
||
// ids and thread ids keep distinct instructions attributable and fetchable.
|
||
func formatLegacyCommentBundle(comments []CoalescedCommentData) string {
|
||
if len(comments) == 0 {
|
||
return ""
|
||
}
|
||
var b strings.Builder
|
||
b.WriteString(legacyCommentBundleHeader)
|
||
for _, comment := range comments {
|
||
b.WriteString(formatLegacyCommentEntry(comment))
|
||
}
|
||
return strings.TrimSpace(b.String())
|
||
}
|
||
|
||
func formatLegacyCommentEntry(comment CoalescedCommentData) string {
|
||
var b strings.Builder
|
||
fmt.Fprintf(&b, "\n--- comment %s", comment.ID)
|
||
if comment.ThreadID != "" {
|
||
fmt.Fprintf(&b, " [thread %s]", comment.ThreadID)
|
||
}
|
||
if comment.AuthorType != "" || comment.AuthorName != "" {
|
||
fmt.Fprintf(&b, " [author %s", comment.AuthorType)
|
||
if comment.AuthorName != "" {
|
||
fmt.Fprintf(&b, ": %s", comment.AuthorName)
|
||
}
|
||
b.WriteString("]")
|
||
}
|
||
if comment.CreatedAt != "" {
|
||
fmt.Fprintf(&b, " [created %s]", comment.CreatedAt)
|
||
}
|
||
b.WriteString(" ---\n")
|
||
b.WriteString(comment.Content)
|
||
fmt.Fprintf(&b, "\n--- end comment %s ---\n", comment.ID)
|
||
return b.String()
|
||
}
|
||
|
||
// ReportTaskUsage stores per-task token usage. Called independently of
|
||
// complete/fail so usage is captured even when tasks fail or are blocked.
|
||
type TaskUsagePayload struct {
|
||
Provider string `json:"provider"`
|
||
Model string `json:"model"`
|
||
InputTokens int64 `json:"input_tokens"`
|
||
OutputTokens int64 `json:"output_tokens"`
|
||
CacheReadTokens int64 `json:"cache_read_tokens"`
|
||
CacheWriteTokens int64 `json:"cache_write_tokens"`
|
||
// CostUSDTicks is the provider's own price for this usage in 1e-10 USD.
|
||
// Absent or 0 from every daemon that doesn't have one (older builds, and
|
||
// every provider except Grok today) — stored as NULL so the reader knows
|
||
// to fall back to rate-table estimation rather than reading a real $0.
|
||
CostUSDTicks int64 `json:"cost_usd_ticks"`
|
||
}
|
||
|
||
// authoritativeCostTicks converts a reported cost into the nullable column.
|
||
// Only a positive figure is authoritative: 0 is what a daemon that knows
|
||
// nothing about cost sends, and storing it would claim a genuine $0 spend and
|
||
// suppress the estimate. Negative is nonsense from a malformed report.
|
||
func authoritativeCostTicks(ticks int64) pgtype.Int8 {
|
||
if ticks <= 0 {
|
||
return pgtype.Int8{}
|
||
}
|
||
return pgtype.Int8{Int64: ticks, Valid: true}
|
||
}
|
||
|
||
func (h *Handler) ReportTaskUsage(w http.ResponseWriter, r *http.Request) {
|
||
taskID := chi.URLParam(r, "taskId")
|
||
|
||
// Verify the caller owns this task's workspace.
|
||
task, ok := h.requireDaemonTaskAccess(w, r, taskID)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
var req struct {
|
||
Usage []TaskUsagePayload `json:"usage"`
|
||
}
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
|
||
// Provider is lowercased on write so client-side pricing lookups tolerate
|
||
// case drift. An empty provider (an older daemon that omits the field) is
|
||
// stamped from the task's runtime, so generic model ids like `auto` still
|
||
// resolve to a provider instead of landing as '' and pricing $0.
|
||
var runtimeProvider string
|
||
runtimeProviderLoaded := false
|
||
for _, u := range req.Usage {
|
||
provider := normalizeProvider(u.Provider)
|
||
if provider == "" {
|
||
if !runtimeProviderLoaded {
|
||
if rt, err := h.getAgentRuntime(r.Context(), obsmetrics.RuntimeLookupSourceTask, task.RuntimeID); err == nil {
|
||
runtimeProvider = normalizeProvider(rt.Provider)
|
||
} else {
|
||
slog.Warn("load runtime provider for usage backfill failed",
|
||
"task_id", taskID, "runtime_id", uuidToString(task.RuntimeID), "error", err)
|
||
}
|
||
runtimeProviderLoaded = true
|
||
}
|
||
provider = runtimeProvider
|
||
}
|
||
if err := h.Queries.UpsertTaskUsage(r.Context(), db.UpsertTaskUsageParams{
|
||
TaskID: parseUUID(taskID),
|
||
Provider: provider,
|
||
Model: u.Model,
|
||
InputTokens: u.InputTokens,
|
||
OutputTokens: u.OutputTokens,
|
||
CacheReadTokens: u.CacheReadTokens,
|
||
CacheWriteTokens: u.CacheWriteTokens,
|
||
CostUsdTicks: authoritativeCostTicks(u.CostUSDTicks),
|
||
}); err != nil {
|
||
slog.Warn("upsert task usage failed", "task_id", taskID, "model", u.Model, "error", err)
|
||
continue
|
||
}
|
||
h.TaskService.CaptureTaskUsage(r.Context(), task, provider, u.Model, u.InputTokens, u.OutputTokens, u.CacheReadTokens, u.CacheWriteTokens, u.CostUSDTicks)
|
||
|
||
// Surface prompt-cache effectiveness per run so cache hit rates are
|
||
// observable in logs, not just queryable from runtime_usage. The ratio
|
||
// is cached input over total input-side tokens; a persistently low
|
||
// value flags a prompt prefix that is not being reused across runs
|
||
// (e.g. volatile values poisoning the cacheable prefix). MUL-3887.
|
||
if totalInput := u.InputTokens + u.CacheReadTokens + u.CacheWriteTokens; totalInput > 0 {
|
||
slog.Info("task prompt-cache usage",
|
||
"task_id", taskID,
|
||
"provider", provider,
|
||
"model", u.Model,
|
||
"input_tokens", u.InputTokens,
|
||
"output_tokens", u.OutputTokens,
|
||
"cache_read_tokens", u.CacheReadTokens,
|
||
"cache_write_tokens", u.CacheWriteTokens,
|
||
"cache_read_ratio", float64(u.CacheReadTokens)/float64(totalInput),
|
||
)
|
||
}
|
||
}
|
||
|
||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||
}
|
||
|
||
// GetTaskStatus returns the current status of a task.
|
||
// Used by the daemon to detect terminal/interruption signals (cancelled,
|
||
// failed, completed) while a task is executing mid-flight.
|
||
func (h *Handler) GetTaskStatus(w http.ResponseWriter, r *http.Request) {
|
||
taskID := chi.URLParam(r, "taskId")
|
||
taskUUID, ok := parseUUIDOrBadRequest(w, taskID, "task_id")
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
// This endpoint is polled by every in-flight daemon task. Load only the
|
||
// status and the owning agent's workspace instead of the full task plus its
|
||
// optional issue / chat / autopilot source chain.
|
||
task, err := h.Queries.GetAgentTaskStatus(r.Context(), taskUUID)
|
||
if err != nil {
|
||
// A missing row is the daemon's hard interruption signal. Keep transient
|
||
// database failures distinct so a brief outage cannot kill healthy work.
|
||
if isNotFound(err) {
|
||
writeError(w, http.StatusNotFound, "task not found")
|
||
return
|
||
}
|
||
slog.Warn("get agent task status failed", "task_id", taskID, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to load task")
|
||
return
|
||
}
|
||
|
||
if !h.requireDaemonWorkspaceAccess(w, r, uuidToString(task.WorkspaceID)) {
|
||
return
|
||
}
|
||
|
||
writeJSON(w, http.StatusOK, map[string]string{"status": task.Status})
|
||
}
|
||
|
||
// FailTask marks a running task as failed.
|
||
type TaskFailRequest struct {
|
||
Error string `json:"error"`
|
||
SessionID string `json:"session_id,omitempty"`
|
||
WorkDir string `json:"work_dir,omitempty"`
|
||
DurableWorkDir string `json:"durable_work_dir,omitempty"`
|
||
FailureReason string `json:"failure_reason,omitempty"`
|
||
// BranchName: a failed run can still have produced a branch — worktree mode
|
||
// commits whatever the agent left before tearing the worktree down. Report
|
||
// it so a partially-successful run is still findable.
|
||
BranchName string `json:"branch_name,omitempty"`
|
||
// SessionRolloutMissing: the daemon withheld this task's Codex session
|
||
// because its rollout was missing (MUL-5305). Clear the resume pointer and
|
||
// flag the continuity gap for the next claim.
|
||
SessionRolloutMissing bool `json:"session_rollout_missing,omitempty"`
|
||
// RetiredSessionID: a session this run proved unresumable and abandoned
|
||
// (GH #6066). Distinct from an empty SessionID, which only means "nothing
|
||
// to report" — this says "never hand this id to a later run". Older
|
||
// daemons omit it, which is exactly the pre-fix behaviour.
|
||
RetiredSessionID string `json:"retired_session_id,omitempty"`
|
||
}
|
||
|
||
func (h *Handler) FailTask(w http.ResponseWriter, r *http.Request) {
|
||
taskID := chi.URLParam(r, "taskId")
|
||
|
||
// Verify the caller owns this task's workspace.
|
||
_, workspaceID, ok := h.requireDaemonTaskAccessWithWorkspace(w, r, taskID)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
var req TaskFailRequest
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
// TaskService.FailTask normalizes req.Error itself, but every other field
|
||
// here lands in a TEXT column too and a NUL in any one of them fails the
|
||
// same transaction (GH #7098).
|
||
sanitizeTaskFailRequest(&req)
|
||
|
||
h.failTask(w, r, taskID, workspaceID, req)
|
||
}
|
||
|
||
// failTask records a terminal failure and writes the response. Shared by the
|
||
// /fail endpoint and by CompleteTask's context-exhaustion normalization so a
|
||
// run re-classified at the /complete boundary lands through exactly the same
|
||
// transaction, token revocation and runtime wake-up as one the daemon reported
|
||
// as failed itself.
|
||
func (h *Handler) failTask(w http.ResponseWriter, r *http.Request, taskID, workspaceID string, req TaskFailRequest) {
|
||
// MUL-5305: SessionRolloutMissing is applied inside FailTask's terminal
|
||
// transaction — forcing session_id NULL (overriding the COALESCE that would
|
||
// keep a stale mid-flight pin) and flagging the row in the same commit that
|
||
// creates and wakes the auto-retry, so the retry can never claim the withheld
|
||
// pointer or miss the continuity gap.
|
||
task, transitioned, err := h.TaskService.FailTaskWithTransition(r.Context(), parseUUID(taskID), req.Error, req.SessionID, req.WorkDir, req.BranchName, req.FailureReason, req.SessionRolloutMissing, req.RetiredSessionID, req.DurableWorkDir)
|
||
if err != nil {
|
||
// A FailTask error is an infrastructure failure (the terminal
|
||
// transaction that also clears the withheld session, writes the
|
||
// continuity-gap flag, and creates the auto-retry rolled back), not a bad
|
||
// request. Return 5xx so the daemon's terminal callback — which treats a
|
||
// 400 as permanent and bails without retrying (postJSONWithRetry /
|
||
// isTransientError) — retries and the fail, gap flag, and retry land
|
||
// exactly once (MUL-5305). An invalid request body still returns 400 above.
|
||
slog.Warn("fail task failed", "task_id", taskID, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to fail task")
|
||
return
|
||
}
|
||
if !transitioned {
|
||
writeJSON(w, http.StatusOK, taskToResponse(*task, workspaceID))
|
||
return
|
||
}
|
||
h.TaskService.NotifyTaskFinished(*task)
|
||
|
||
// Best-effort revoke of the mat_ task token minted at claim. Same
|
||
// rationale as CompleteTask — eager deletion shrinks the post-
|
||
// terminal window. The 24h expiry / cascade are the durable guards.
|
||
if err := h.Queries.DeleteTaskTokensByTask(r.Context(), task.ID); err != nil {
|
||
slog.Warn("fail task: failed to revoke task tokens", "task_id", uuidToString(task.ID), "error", err)
|
||
}
|
||
|
||
slog.Info("task failed", "task_id", taskID, "agent_id", uuidToString(task.AgentID), "task_error", req.Error, "failure_reason", req.FailureReason)
|
||
writeJSON(w, http.StatusOK, taskToResponse(*task, workspaceID))
|
||
}
|
||
|
||
// Task Messages (live agent output)
|
||
|
||
type TaskMessageRequest struct {
|
||
// CallID is an opaque tool-call identity scoped to one backend execution.
|
||
CallID string `json:"call_id,omitempty"`
|
||
Seq int `json:"seq"`
|
||
Type string `json:"type"`
|
||
Tool string `json:"tool,omitempty"`
|
||
Content string `json:"content,omitempty"`
|
||
Input map[string]any `json:"input,omitempty"`
|
||
Output string `json:"output,omitempty"`
|
||
// CreatedAt is optional so installed daemons can roll forward or back
|
||
// independently of the server. Missing values keep the database-time fallback.
|
||
CreatedAt *time.Time `json:"created_at,omitempty"`
|
||
// OutputTruncated is absent from every older daemon's payload, so it stays
|
||
// a pointer all the way to the column: nil must persist as NULL (unknown),
|
||
// never as false. Recording an unmeasured output as complete is the one
|
||
// claim this data is not entitled to make.
|
||
OutputTruncated *bool `json:"output_truncated,omitempty"`
|
||
}
|
||
|
||
type TaskMessageBatchRequest struct {
|
||
Messages []TaskMessageRequest `json:"messages"`
|
||
}
|
||
|
||
// ReportTaskMessages receives a batch of agent execution messages from the daemon.
|
||
func (h *Handler) ReportTaskMessages(w http.ResponseWriter, r *http.Request) {
|
||
taskID := chi.URLParam(r, "taskId")
|
||
|
||
var req TaskMessageBatchRequest
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
if len(req.Messages) == 0 {
|
||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||
return
|
||
}
|
||
|
||
// Verify the caller owns this task's workspace. The access check already
|
||
// resolves the workspace id (it needs it to authorize the daemon), so take
|
||
// it from there instead of re-running GetIssue / GetChatSession: this
|
||
// endpoint fires every 500ms for every running task, and that second
|
||
// lookup was a whole extra query per batch on the hottest write path in
|
||
// the system (MUL-6523).
|
||
task, wsID, ok := h.requireDaemonTaskAccessWithWorkspace(w, r, taskID)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
// Broadcast reach is deliberately unchanged: only issue- and chat-backed
|
||
// tasks streamed live messages before, and widening that to autopilot /
|
||
// quick-create tasks is a product decision, not part of this optimization.
|
||
workspaceID := ""
|
||
if task.IssueID.Valid || task.ChatSessionID.Valid {
|
||
workspaceID = wsID
|
||
}
|
||
|
||
// Column-wise parameters for the single batch insert below. Building them
|
||
// as parallel arrays rather than as one JSON document is deliberate: these
|
||
// strings were just decoded out of the request, and re-encoding content /
|
||
// output into JSON would make the server escape every byte a second time
|
||
// and Postgres parse the envelope back out — measurably worse on exactly
|
||
// the large messages that are already the most expensive (MUL-6523). Empty
|
||
// string means SQL NULL, applied by the query's NULLIF.
|
||
n := len(req.Messages)
|
||
params := db.CreateTaskMessagesParams{
|
||
TaskID: parseUUID(taskID),
|
||
Ids: make([]pgtype.UUID, 0, n),
|
||
Seqs: make([]int32, 0, n),
|
||
Types: make([]string, 0, n),
|
||
Tools: make([]string, 0, n),
|
||
CallIds: make([]string, 0, n),
|
||
Contents: make([]string, 0, n),
|
||
Inputs: make([]string, 0, n),
|
||
Outputs: make([]string, 0, n),
|
||
// Optional for mixed-version rollout. Older daemons omit the event
|
||
// timestamp, and one missing or implausible value makes the whole batch
|
||
// use database time so paired events never mix clocks.
|
||
CreatedAts: make([]string, 0, n),
|
||
// Tri-state through a text[]: "" is NULL, matching how the query maps
|
||
// every other nullable column in this batch.
|
||
OutputTruncations: make([]string, 0, n),
|
||
}
|
||
createdAts := taskMessageCreatedAts(req.Messages, time.Now().UTC())
|
||
for i, msg := range req.Messages {
|
||
id, err := uuid.NewV7()
|
||
if err != nil {
|
||
slog.Error("failed to generate task message id", "task_id", taskID, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to persist task message")
|
||
return
|
||
}
|
||
|
||
// Redact sensitive information before persisting or broadcasting.
|
||
msg.Content = redact.Text(msg.Content)
|
||
msg.Output = redact.Text(msg.Output)
|
||
msg.Input = redact.InputMap(msg.Input)
|
||
|
||
// redact.Text only masks secret-shaped substrings; it has no opinion on
|
||
// bytes PostgreSQL refuses to store. Tool output is the likeliest
|
||
// carrier of a stray NUL in the whole system — an agent that cats a
|
||
// binary, or a Windows tool emitting UTF-16 — and this endpoint has no
|
||
// retry on the daemon side, so an unsanitized batch is silently lost
|
||
// (GH #7098). Input is a JSONB column, so it needs the deep walk: the
|
||
// offending byte can sit at any depth of a tool's arguments. Batching
|
||
// raises the stakes: one bad byte now fails the whole statement rather
|
||
// than a single row, which is inherent to one-statement writes.
|
||
msg.Type = util.SanitizeTextForPostgres(msg.Type)
|
||
msg.Tool = util.SanitizeTextForPostgres(msg.Tool)
|
||
msg.CallID = util.SanitizeTextForPostgres(msg.CallID)
|
||
msg.Content = util.SanitizeTextForPostgres(msg.Content)
|
||
msg.Output = util.SanitizeTextForPostgres(msg.Output)
|
||
if msg.Input != nil {
|
||
if cleaned, ok := util.SanitizeJSONForPostgres(msg.Input).(map[string]any); ok {
|
||
msg.Input = cleaned
|
||
}
|
||
}
|
||
|
||
inputJSON := ""
|
||
if msg.Input != nil {
|
||
// Fail loud rather than dropping the field: a tool call whose
|
||
// arguments silently vanish is worse than a 500 the daemon logs,
|
||
// because the transcript then shows a tool_use with no input and
|
||
// nothing anywhere records that it was lost.
|
||
encoded, err := json.Marshal(msg.Input)
|
||
if err != nil {
|
||
slog.Error("failed to encode task message input", "task_id", taskID, "seq", msg.Seq, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to persist task message")
|
||
return
|
||
}
|
||
inputJSON = string(encoded)
|
||
}
|
||
|
||
params.Ids = append(params.Ids, pgtype.UUID{Bytes: [16]byte(id), Valid: true})
|
||
params.Seqs = append(params.Seqs, int32(msg.Seq))
|
||
params.Types = append(params.Types, msg.Type)
|
||
params.Tools = append(params.Tools, msg.Tool)
|
||
params.CallIds = append(params.CallIds, msg.CallID)
|
||
params.Contents = append(params.Contents, msg.Content)
|
||
params.Inputs = append(params.Inputs, inputJSON)
|
||
params.Outputs = append(params.Outputs, msg.Output)
|
||
params.CreatedAts = append(params.CreatedAts, createdAts[i])
|
||
params.OutputTruncations = append(params.OutputTruncations, boolArrayElement(msg.OutputTruncated))
|
||
}
|
||
|
||
created, err := h.Queries.CreateTaskMessages(r.Context(), params)
|
||
if err != nil {
|
||
slog.Error("failed to create task messages", "task_id", taskID, "count", n, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to persist task message")
|
||
return
|
||
}
|
||
|
||
if workspaceID != "" {
|
||
// CreateTaskMessages orders its result by seq, which is the daemon's
|
||
// own within-batch ordering — so this publishes in the same order the
|
||
// per-message loop did. A bare INSERT ... RETURNING has no row-order
|
||
// guarantee, and subscribers render these events as they arrive, so the
|
||
// ordering lives in the query rather than in the clients.
|
||
for _, m := range created {
|
||
// The ordered CTE makes sqlc name the row type after the query
|
||
// rather than reusing the table model; the columns are the table's,
|
||
// in order, so the conversion is checked by the compiler and breaks
|
||
// loudly if the query ever stops returning the whole row.
|
||
h.publishTask(protocol.EventTaskMessage, workspaceID, "system", "", taskID,
|
||
taskMessageToPayload(db.TaskMessage(m), taskID, uuidToString(task.IssueID)))
|
||
}
|
||
}
|
||
|
||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||
}
|
||
|
||
// AckTaskCancelled receives the daemon's acknowledgement that it observed a
|
||
// task cancellation and finished flushing the transcript. Settles the chat
|
||
// finalization that CancelTaskWithResult deferred for a started-but-empty
|
||
// transcript (#5219); idempotent when nothing was deferred.
|
||
// TaskCancelAckRequest is the body of the daemon's cancel acknowledgement.
|
||
type TaskCancelAckRequest struct {
|
||
// BranchName: a cancelled worktree task has already committed whatever the
|
||
// agent produced — the worktree is finalized before the daemon learns of
|
||
// the cancellation. The rest of the result is discarded on this path, so
|
||
// this is the only channel that can report where the work went.
|
||
BranchName string `json:"branch_name,omitempty"`
|
||
// DurableWorkDir is present only when Finalize confirmed the disposable
|
||
// worktree was removed and the configured project directory is authoritative.
|
||
DurableWorkDir string `json:"durable_work_dir,omitempty"`
|
||
// ErrorMessage / FailureReason: set when the cancelled run's worktree
|
||
// Finalize ABORTED — there is no branch, and the error text naming the
|
||
// preserved worktree is the only pointer left to the agent's work.
|
||
ErrorMessage string `json:"error_message,omitempty"`
|
||
FailureReason string `json:"failure_reason,omitempty"`
|
||
}
|
||
|
||
func (h *Handler) AckTaskCancelled(w http.ResponseWriter, r *http.Request) {
|
||
taskID := chi.URLParam(r, "taskId")
|
||
task, ok := h.requireDaemonTaskAccess(w, r, taskID)
|
||
if !ok {
|
||
return
|
||
}
|
||
// Body is optional: older daemons send `{}`, and a decode failure must not
|
||
// break the cancellation contract this endpoint exists for.
|
||
var req TaskCancelAckRequest
|
||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||
// Same persistence hazard as /fail and /complete: these strings go straight
|
||
// into TEXT columns, and this endpoint fails LOUD on write errors, so an
|
||
// unsanitized NUL here turns a cancelled task's only pointer to its work
|
||
// into an endless 500 retry loop (GH #7098).
|
||
req.ErrorMessage = util.SanitizeTextForPostgres(req.ErrorMessage)
|
||
req.FailureReason = util.SanitizeTextForPostgres(req.FailureReason)
|
||
req.BranchName = util.SanitizeTextForPostgres(req.BranchName)
|
||
req.DurableWorkDir = util.SanitizeTextForPostgres(req.DurableWorkDir)
|
||
|
||
// Terminal deliveries first, failing LOUD on persistence errors: these
|
||
// fields are the only pointer to a cancelled task's work, and the daemon
|
||
// retries this ack on transient failures — a warn-and-200 would turn one
|
||
// DB blip into a permanently undiscoverable branch. Both writes never
|
||
// overwrite already-recorded values, so replays are idempotent — and both
|
||
// carry a status='cancelled' CAS, because the daemon acks on EVERY
|
||
// terminal status it observes: a late ack from a stale run must not stamp
|
||
// its branch or error onto a completed/failed row whose own callback is
|
||
// the authoritative channel. A CAS-refused write is a deliberate no-op
|
||
// (Exec reports no error), so the ack still returns 200 — there is
|
||
// nothing for the daemon to retry — and the rebroadcast below is guarded
|
||
// by the same status check inside RebroadcastCancelledTask.
|
||
delivered := false
|
||
if durableWorkDir := strings.TrimSpace(req.DurableWorkDir); durableWorkDir != "" {
|
||
if err := h.Queries.SetAgentTaskDurableWorkDir(r.Context(), db.SetAgentTaskDurableWorkDirParams{
|
||
ID: task.ID,
|
||
DurableWorkDir: pgtype.Text{String: durableWorkDir, Valid: true},
|
||
}); err != nil {
|
||
slog.Error("cancel ack: record durable work directory failed",
|
||
"task_id", taskID, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to record durable work directory")
|
||
return
|
||
}
|
||
delivered = true
|
||
}
|
||
if branch := strings.TrimSpace(req.BranchName); branch != "" {
|
||
if err := h.Queries.SetAgentTaskBranchName(r.Context(), db.SetAgentTaskBranchNameParams{
|
||
ID: task.ID,
|
||
BranchName: pgtype.Text{String: branch, Valid: true},
|
||
}); err != nil {
|
||
slog.Error("cancel ack: record branch name failed",
|
||
"task_id", taskID, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to record branch name")
|
||
return
|
||
}
|
||
delivered = true
|
||
}
|
||
if msg := strings.TrimSpace(req.ErrorMessage); msg != "" {
|
||
reason := strings.TrimSpace(req.FailureReason)
|
||
if err := h.Queries.SetAgentTaskErrorIfEmpty(r.Context(), db.SetAgentTaskErrorIfEmptyParams{
|
||
ID: task.ID,
|
||
Error: pgtype.Text{String: msg, Valid: true},
|
||
FailureReason: pgtype.Text{String: reason, Valid: reason != ""},
|
||
}); err != nil {
|
||
slog.Error("cancel ack: record preserved-work error failed",
|
||
"task_id", taskID, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to record task error")
|
||
return
|
||
}
|
||
delivered = true
|
||
}
|
||
if delivered {
|
||
// The task:cancelled broadcast fired at cancel time, before this ack —
|
||
// clients may already hold a refetched row without the branch/error
|
||
// and will not refetch again on their own.
|
||
h.TaskService.RebroadcastCancelledTask(r.Context(), task.ID)
|
||
}
|
||
h.TaskService.FinalizeDeferredCancelledChat(r.Context(), task.ID)
|
||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||
}
|
||
|
||
func taskMessageToPayload(m db.TaskMessage, taskID, issueID string) protocol.TaskMessagePayload {
|
||
var input map[string]any
|
||
if m.Input != nil {
|
||
json.Unmarshal(m.Input, &input)
|
||
}
|
||
createdAt := ""
|
||
if m.CreatedAt.Valid {
|
||
createdAt = m.CreatedAt.Time.UTC().Format(time.RFC3339Nano)
|
||
}
|
||
return protocol.TaskMessagePayload{
|
||
TaskID: taskID,
|
||
IssueID: issueID,
|
||
Seq: int(m.Seq),
|
||
Type: m.Type,
|
||
Tool: m.Tool.String,
|
||
CallID: m.CallID.String,
|
||
Content: m.Content.String,
|
||
Input: input,
|
||
Output: m.Output.String,
|
||
OutputTruncated: util.BoolToPtr(m.OutputTruncated),
|
||
CreatedAt: createdAt,
|
||
}
|
||
}
|
||
|
||
// boolArrayElement encodes a tri-state bool for the text[] parameters the batch
|
||
// insert uses. Empty string is the query's NULL, so an unmeasured output stays
|
||
// unknown rather than collapsing into false.
|
||
func boolArrayElement(v *bool) string {
|
||
if v == nil {
|
||
return ""
|
||
}
|
||
if *v {
|
||
return "true"
|
||
}
|
||
return "false"
|
||
}
|
||
|
||
const maxTaskMessageClockSkew = 2 * time.Minute
|
||
|
||
// taskMessageCreatedAt encodes the daemon-observed event time for the batch
|
||
// query's nullable text[] transport. Empty means an older daemon omitted the
|
||
// field or its wall clock is implausibly far from the server, so PostgreSQL
|
||
// falls back to its normal insertion time.
|
||
func taskMessageCreatedAt(v *time.Time, serverNow time.Time) string {
|
||
if v == nil || v.IsZero() {
|
||
return ""
|
||
}
|
||
if skew := v.Sub(serverNow); skew < -maxTaskMessageClockSkew || skew > maxTaskMessageClockSkew {
|
||
return ""
|
||
}
|
||
return v.UTC().Format(time.RFC3339Nano)
|
||
}
|
||
|
||
// taskMessageCreatedAts keeps every timestamp in a report batch on the same
|
||
// clock. If any event lacks a plausible daemon timestamp, returning all empty
|
||
// values makes PostgreSQL assign one database timestamp to the whole batch.
|
||
func taskMessageCreatedAts(messages []TaskMessageRequest, serverNow time.Time) []string {
|
||
createdAts := make([]string, len(messages))
|
||
for i := range messages {
|
||
createdAts[i] = taskMessageCreatedAt(messages[i].CreatedAt, serverNow)
|
||
if createdAts[i] == "" {
|
||
clear(createdAts)
|
||
return createdAts
|
||
}
|
||
}
|
||
return createdAts
|
||
}
|
||
|
||
// ListTaskMessages returns the persisted messages for a task (for catch-up after reconnect).
|
||
func (h *Handler) ListTaskMessages(w http.ResponseWriter, r *http.Request) {
|
||
taskID := chi.URLParam(r, "taskId")
|
||
|
||
// Verify the caller owns this task's workspace.
|
||
task, ok := h.requireDaemonTaskAccess(w, r, taskID)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
var (
|
||
messages []db.TaskMessage
|
||
err error
|
||
)
|
||
if sinceStr := r.URL.Query().Get("since"); sinceStr != "" {
|
||
sinceSeq, parseErr := strconv.Atoi(sinceStr)
|
||
if parseErr != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid since parameter")
|
||
return
|
||
}
|
||
messages, err = h.Queries.ListTaskMessagesSince(r.Context(), db.ListTaskMessagesSinceParams{
|
||
TaskID: parseUUID(taskID),
|
||
Seq: int32(sinceSeq),
|
||
})
|
||
} else {
|
||
messages, err = h.Queries.ListTaskMessages(r.Context(), parseUUID(taskID))
|
||
}
|
||
if err != nil {
|
||
writeError(w, http.StatusInternalServerError, "failed to list task messages")
|
||
return
|
||
}
|
||
|
||
issueID := uuidToString(task.IssueID)
|
||
|
||
resp := make([]protocol.TaskMessagePayload, len(messages))
|
||
for i, m := range messages {
|
||
resp[i] = taskMessageToPayload(m, taskID, issueID)
|
||
}
|
||
|
||
writeJSON(w, http.StatusOK, resp)
|
||
}
|
||
|
||
// GetActiveTaskForIssue returns all currently active tasks for an issue.
|
||
// Returns { tasks: [...] } array (may be empty).
|
||
func (h *Handler) GetActiveTaskForIssue(w http.ResponseWriter, r *http.Request) {
|
||
issueID := chi.URLParam(r, "id")
|
||
issue, ok := h.loadIssueForUser(w, r, issueID)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
tasks, err := h.Queries.ListActiveTasksByIssue(r.Context(), issue.ID)
|
||
if err != nil {
|
||
tasks = nil
|
||
}
|
||
|
||
workspaceID := uuidToString(issue.WorkspaceID)
|
||
resp := make([]AgentTaskResponse, len(tasks))
|
||
for i, t := range tasks {
|
||
resp[i] = taskToResponse(t, workspaceID)
|
||
}
|
||
// Same issue-facing attribution surface as ListTasksByIssue — hydrate names.
|
||
h.hydrateTaskAttributions(r.Context(), attributionsOf(resp))
|
||
|
||
writeJSON(w, http.StatusOK, map[string]any{"tasks": resp})
|
||
}
|
||
|
||
// CancelTask cancels a running or queued task by ID.
|
||
// Verifies both that the URL-parameter issue belongs to the caller's workspace
|
||
// and that the task belongs to that same issue — a task UUID from a different
|
||
// issue (in any workspace) must not be cancellable through this route.
|
||
func (h *Handler) CancelTask(w http.ResponseWriter, r *http.Request) {
|
||
issueID := chi.URLParam(r, "id")
|
||
issue, ok := h.loadIssueForUser(w, r, issueID)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
taskID := chi.URLParam(r, "taskId")
|
||
existing, err := h.Queries.GetAgentTask(r.Context(), parseUUID(taskID))
|
||
if err != nil || uuidToString(existing.IssueID) != uuidToString(issue.ID) {
|
||
writeError(w, http.StatusNotFound, "task not found")
|
||
return
|
||
}
|
||
|
||
workspaceID := uuidToString(issue.WorkspaceID)
|
||
actorType, actorID := h.resolveActor(r, requestUserID(r), workspaceID)
|
||
task, err := h.TaskService.CancelTaskByUser(
|
||
r.Context(),
|
||
existing.ID,
|
||
h.taskCancellationActor(r.Context(), actorType, actorID),
|
||
)
|
||
if err != nil {
|
||
slog.Warn("cancel task failed", "task_id", taskID, "error", err)
|
||
writeError(w, http.StatusBadRequest, err.Error())
|
||
return
|
||
}
|
||
|
||
slog.Info("task cancelled by user", "task_id", taskID, "issue_id", uuidToString(task.IssueID))
|
||
resp := taskToResponse(*task, workspaceID)
|
||
// Keep this issue-scoped surface consistent with the list endpoints so a
|
||
// cancelled row keeps its resolved "on behalf of" name in the UI.
|
||
h.hydrateTaskAttributions(r.Context(), []*TaskAttribution{resp.Attribution})
|
||
writeJSON(w, http.StatusOK, resp)
|
||
}
|
||
|
||
// familyActiveRunCap bounds a scope=family read.
|
||
//
|
||
// It is a response-size budget set to the shape of the normal case — a handful
|
||
// of runs in flight at once (maintainer call, MUL-6846) — NOT a bound derived
|
||
// from how much work can really be in flight. A family can legitimately exceed
|
||
// it by a lot: a single agent may be configured up to
|
||
// agentconfig.MaxMaxConcurrentTasks (50) on its own, this feature exists
|
||
// precisely for the case where SEVERAL agents work a family at once, and the
|
||
// returned set includes queued / dispatched / waiting_local_directory rows,
|
||
// which no execution-slot limit bounds at all — a parent that fans out 200
|
||
// children can have 200 of them enqueued.
|
||
//
|
||
// So truncation here is an ordinary outcome, not a pathological one, and that
|
||
// is exactly why it must be reported. A cap that truncated silently would be
|
||
// worse than no cap: "I saw no run on that sibling" and "the answer was cut
|
||
// off" would look identical, and an agent would read the second as the first.
|
||
// The handler asks for one row more than it will return and sets
|
||
// HeaderActiveRunsTruncated when that extra row comes back. The query orders
|
||
// running-first, so what the budget drops is the least decision-relevant.
|
||
const familyActiveRunCap = 20
|
||
|
||
// HeaderActiveRunsTruncated tells a caller its coordination read hit
|
||
// familyActiveRunCap and is therefore an incomplete picture of who is working
|
||
// in this family. Same shape and convention as HeaderTimelineTruncated: the
|
||
// signal rides a header because the response body is a bare array that existing
|
||
// callers parse positionally.
|
||
const HeaderActiveRunsTruncated = "X-Active-Runs-Truncated"
|
||
|
||
// ActiveRunSummary is one in-flight run as the coordination read reports it:
|
||
// which issue, which agent, what state, since when, and the task id to follow
|
||
// up with `multica issue run-messages`.
|
||
//
|
||
// Deliberately NOT AgentTaskResponse. That type is the execution log's row —
|
||
// result, work_dir, attribution, coalesced comment ids — and it costs roughly
|
||
// 5x the bytes of this one. A caller asking "is anyone working next to me?"
|
||
// reads none of those fields, and it is an agent spending its own context on
|
||
// the answer, so the payload is cut to what the question needs.
|
||
type ActiveRunSummary struct {
|
||
TaskID string `json:"task_id"`
|
||
IssueID string `json:"issue_id"`
|
||
IssueIdentifier string `json:"issue_identifier"`
|
||
IssueTitle string `json:"issue_title"`
|
||
AgentID string `json:"agent_id"`
|
||
Status string `json:"status"`
|
||
CreatedAt string `json:"created_at"`
|
||
StartedAt *string `json:"started_at,omitempty"`
|
||
}
|
||
|
||
// ListTasksByIssue returns tasks for an issue — the execution history behind
|
||
// the issue-detail sidebar, and the coordination reads behind
|
||
// `multica issue runs --active` / `--siblings`.
|
||
//
|
||
// Two optional query params narrow or widen it; with neither, the response is
|
||
// byte-identical to what it has always been (full history, newest first), which
|
||
// is what the UI and the CLI's short-task-ID resolver both depend on:
|
||
//
|
||
// - active=true — restrict to in-flight statuses, the same set the
|
||
// issue-detail "agent live" banner calls active.
|
||
// - scope=family — widen from this issue to its sub-issue family: the
|
||
// issue's parent (or the issue itself, when it has no parent) plus every
|
||
// child of that parent. Implies active=true, because a full execution
|
||
// history across a whole family is unbounded and answers no question anyone
|
||
// asked.
|
||
func (h *Handler) ListTasksByIssue(w http.ResponseWriter, r *http.Request) {
|
||
issueID := chi.URLParam(r, "id")
|
||
issue, ok := h.loadIssueForUser(w, r, issueID)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
scope := r.URL.Query().Get("scope")
|
||
if scope != "" && scope != "issue" && scope != "family" {
|
||
writeError(w, http.StatusBadRequest, "scope must be 'issue' or 'family'")
|
||
return
|
||
}
|
||
// Parse rather than compare. `active=tru` answering with the full history
|
||
// under a 200 is the same silent-downgrade failure the unknown-scope check
|
||
// above rejects: the caller asked who is here NOW and would be handed every
|
||
// run that ever finished, which reads as "nobody is here" only after it has
|
||
// paid for the whole log. Same shape as the comment-list boolean params.
|
||
activeOnly := false
|
||
if activeStr := r.URL.Query().Get("active"); activeStr != "" {
|
||
switch activeStr {
|
||
case "true":
|
||
activeOnly = true
|
||
case "false":
|
||
default:
|
||
writeError(w, http.StatusBadRequest, "invalid active parameter; expected boolean")
|
||
return
|
||
}
|
||
}
|
||
|
||
workspaceID := uuidToString(issue.WorkspaceID)
|
||
|
||
if scope == "family" {
|
||
// Root the family at the parent when there is one, so a child sees its
|
||
// siblings; at the issue itself otherwise, so a parent sees its children
|
||
// and a standalone issue degenerates to its own active runs. One rule,
|
||
// both directions.
|
||
root := issue.ID
|
||
if issue.ParentIssueID.Valid {
|
||
root = issue.ParentIssueID
|
||
}
|
||
// One row beyond the cap, so a full page can be told apart from a
|
||
// truncated one without a second count query.
|
||
rows, err := h.Queries.ListActiveTasksByIssueFamily(r.Context(), db.ListActiveTasksByIssueFamilyParams{
|
||
WorkspaceID: issue.WorkspaceID,
|
||
RootIssueID: root,
|
||
RowLimit: familyActiveRunCap + 1,
|
||
})
|
||
if err != nil {
|
||
writeError(w, http.StatusInternalServerError, "failed to list tasks")
|
||
return
|
||
}
|
||
if len(rows) > familyActiveRunCap {
|
||
rows = rows[:familyActiveRunCap]
|
||
w.Header().Set(HeaderActiveRunsTruncated, "true")
|
||
}
|
||
summaries := make([]ActiveRunSummary, len(rows))
|
||
for i, row := range rows {
|
||
summaries[i] = ActiveRunSummary{
|
||
TaskID: uuidToString(row.TaskID),
|
||
IssueID: uuidToString(row.IssueID),
|
||
// Rows span several issues here, so each one has to carry the
|
||
// issue it belongs to — a caller cannot label it from the task.
|
||
IssueIdentifier: service.IssueIdentifier(row.IssuePrefix, row.IssueNumber),
|
||
IssueTitle: row.IssueTitle,
|
||
AgentID: uuidToString(row.AgentID),
|
||
Status: row.Status,
|
||
CreatedAt: timestampToString(row.CreatedAt),
|
||
StartedAt: timestampToPtr(row.StartedAt),
|
||
}
|
||
}
|
||
// No attribution hydration either: it was the single largest field on
|
||
// the old payload and needed its own query, and "on behalf of whom" is
|
||
// an execution-log question, not a coordination one.
|
||
writeJSON(w, http.StatusOK, summaries)
|
||
return
|
||
}
|
||
|
||
var tasks []db.AgentTaskQueue
|
||
var err error
|
||
if activeOnly {
|
||
tasks, err = h.Queries.ListActiveTasksByIssue(r.Context(), issue.ID)
|
||
} else {
|
||
tasks, err = h.Queries.ListTasksByIssue(r.Context(), issue.ID)
|
||
}
|
||
if err != nil {
|
||
writeError(w, http.StatusInternalServerError, "failed to list tasks")
|
||
return
|
||
}
|
||
|
||
tasks = visibleTaskHistory(tasks)
|
||
resp := make([]AgentTaskResponse, len(tasks))
|
||
for i, t := range tasks {
|
||
resp[i] = taskToResponse(t, workspaceID)
|
||
}
|
||
h.hydrateTaskSupplementMetadata(r.Context(), r, issue.WorkspaceID, tasks, resp)
|
||
// Execution-log rows render the "on behalf of <member>" badge, so this
|
||
// issue-facing surface must resolve initiator/originator names (departed-safe,
|
||
// one batch) — otherwise the badge falls back to "someone" on issue detail.
|
||
h.hydrateTaskAttributions(r.Context(), attributionsOf(resp))
|
||
// Usage belongs to the execution log, not to a coordination read.
|
||
// ListIssueTaskUsage returns a row per (task, provider, model) for EVERY
|
||
// task the issue ever ran, so hydrating it on the active path would keep
|
||
// paying the full-history cost this filter exists to remove — and pay it
|
||
// for a column that is near-empty on runs that have not finished.
|
||
if !activeOnly {
|
||
h.hydrateTaskUsage(r.Context(), issue.ID, resp)
|
||
}
|
||
|
||
writeJSON(w, http.StatusOK, resp)
|
||
}
|
||
|
||
// hydrateTaskUsage attaches each run's own token usage to the execution-log
|
||
// rows. One query for the whole issue, then a map join — not one query per
|
||
// task, which would be an N+1 over a list the UI always renders in full.
|
||
//
|
||
// Usage is display metadata: a failure here must not take the execution log
|
||
// down with it, so the error is swallowed and every row keeps its nil Usage,
|
||
// which the UI already renders as "no usage recorded".
|
||
func (h *Handler) hydrateTaskUsage(ctx context.Context, issueID pgtype.UUID, resp []AgentTaskResponse) {
|
||
if len(resp) == 0 {
|
||
return
|
||
}
|
||
|
||
rows, err := h.Queries.ListIssueTaskUsage(ctx, issueID)
|
||
if err != nil || len(rows) == 0 {
|
||
return
|
||
}
|
||
|
||
byTask := make(map[string][]TaskUsageData, len(resp))
|
||
for _, row := range rows {
|
||
appendTaskUsage(byTask, row.TaskID, row.Provider, row.Model,
|
||
row.InputTokens, row.OutputTokens, row.CacheReadTokens,
|
||
row.CacheWriteTokens, row.CostUsdTicks)
|
||
}
|
||
attachTaskUsage(resp, byTask)
|
||
}
|
||
|
||
// hydrateAgentTaskUsage attaches the same per-run accounting shape used by the
|
||
// issue execution log to an agent's user-facing task history. One agent-scoped
|
||
// query covers exactly the returned task IDs, avoiding unrelated history and an
|
||
// N+1 query per task.
|
||
func (h *Handler) hydrateAgentTaskUsage(ctx context.Context, agentID pgtype.UUID, taskIDs []pgtype.UUID, resp []AgentTaskResponse) error {
|
||
if len(resp) == 0 {
|
||
return nil
|
||
}
|
||
|
||
rows, err := h.Queries.ListAgentTaskUsage(ctx, db.ListAgentTaskUsageParams{
|
||
AgentID: agentID,
|
||
TaskIds: taskIDs,
|
||
})
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if len(rows) == 0 {
|
||
return nil
|
||
}
|
||
|
||
byTask := make(map[string][]TaskUsageData, len(resp))
|
||
for _, row := range rows {
|
||
appendTaskUsage(byTask, row.TaskID, row.Provider, row.Model,
|
||
row.InputTokens, row.OutputTokens, row.CacheReadTokens,
|
||
row.CacheWriteTokens, row.CostUsdTicks)
|
||
}
|
||
attachTaskUsage(resp, byTask)
|
||
return nil
|
||
}
|
||
|
||
func appendTaskUsage(
|
||
byTask map[string][]TaskUsageData,
|
||
taskID pgtype.UUID,
|
||
provider string,
|
||
model string,
|
||
inputTokens int64,
|
||
outputTokens int64,
|
||
cacheReadTokens int64,
|
||
cacheWriteTokens int64,
|
||
costUsdTicks pgtype.Int8,
|
||
) {
|
||
var cost *int64
|
||
if costUsdTicks.Valid {
|
||
value := costUsdTicks.Int64
|
||
cost = &value
|
||
}
|
||
id := uuidToString(taskID)
|
||
byTask[id] = append(byTask[id], TaskUsageData{
|
||
Provider: provider,
|
||
Model: model,
|
||
InputTokens: inputTokens,
|
||
OutputTokens: outputTokens,
|
||
CacheReadTokens: cacheReadTokens,
|
||
CacheWriteTokens: cacheWriteTokens,
|
||
CostUsdTicks: cost,
|
||
})
|
||
}
|
||
|
||
func attachTaskUsage(resp []AgentTaskResponse, byTask map[string][]TaskUsageData) {
|
||
for i := range resp {
|
||
if usage, ok := byTask[resp[i].ID]; ok {
|
||
resp[i].Usage = usage
|
||
}
|
||
}
|
||
}
|
||
|
||
// ListTaskMessagesByUser returns task messages for a task.
|
||
// Used by the frontend under regular user auth (not daemon auth).
|
||
// Verifies the task belongs to the caller's workspace.
|
||
func (h *Handler) ListTaskMessagesByUser(w http.ResponseWriter, r *http.Request) {
|
||
taskID := chi.URLParam(r, "taskId")
|
||
taskUUID, ok := parseUUIDOrBadRequest(w, taskID, "task_id")
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
task, err := h.Queries.GetAgentTask(r.Context(), taskUUID)
|
||
if err != nil {
|
||
if !isNotFound(err) {
|
||
slog.Warn("get agent task failed", "task_id", taskID, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to load task")
|
||
return
|
||
}
|
||
writeError(w, http.StatusNotFound, "task not found")
|
||
return
|
||
}
|
||
|
||
// Verify the task belongs to the caller's workspace. A failed lookup is a
|
||
// 5xx here too: this endpoint does not drive the daemon's interrupt, but
|
||
// telling a reader "this task does not exist" because the DB blinked is
|
||
// the same lie, and it trains clients to give up on a retryable error.
|
||
wsID, err := h.TaskService.ResolveTaskWorkspaceIDChecked(r.Context(), task)
|
||
if err != nil {
|
||
slog.Warn("resolve task workspace failed", "task_id", taskID, "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to load task")
|
||
return
|
||
}
|
||
// Mismatch stays a 404 rather than a 403: a task in another workspace must
|
||
// be indistinguishable from one that does not exist.
|
||
if wsID == "" || wsID != middleware.WorkspaceIDFromContext(r.Context()) {
|
||
writeError(w, http.StatusNotFound, "task not found")
|
||
return
|
||
}
|
||
|
||
var (
|
||
messages []db.TaskMessage
|
||
queryErr error
|
||
)
|
||
if sinceStr := r.URL.Query().Get("since"); sinceStr != "" {
|
||
sinceSeq, parseErr := strconv.Atoi(sinceStr)
|
||
if parseErr != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid since parameter")
|
||
return
|
||
}
|
||
messages, queryErr = h.Queries.ListTaskMessagesSince(r.Context(), db.ListTaskMessagesSinceParams{
|
||
TaskID: taskUUID,
|
||
Seq: int32(sinceSeq),
|
||
})
|
||
} else {
|
||
messages, queryErr = h.Queries.ListTaskMessages(r.Context(), taskUUID)
|
||
}
|
||
if queryErr != nil {
|
||
writeError(w, http.StatusInternalServerError, "failed to list task messages")
|
||
return
|
||
}
|
||
|
||
issueID := uuidToString(task.IssueID)
|
||
|
||
resp := make([]protocol.TaskMessagePayload, len(messages))
|
||
for i, m := range messages {
|
||
resp[i] = taskMessageToPayload(m, taskID, issueID)
|
||
}
|
||
|
||
writeJSON(w, http.StatusOK, resp)
|
||
}
|
||
|
||
// GetIssueUsage returns aggregated token usage for all tasks belonging to an issue.
|
||
func (h *Handler) GetIssueUsage(w http.ResponseWriter, r *http.Request) {
|
||
issueID := chi.URLParam(r, "id")
|
||
issue, ok := h.loadIssueForUser(w, r, issueID)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
row, err := h.Queries.GetIssueUsageSummary(r.Context(), issue.ID)
|
||
if err != nil {
|
||
writeError(w, http.StatusInternalServerError, "failed to get issue usage")
|
||
return
|
||
}
|
||
|
||
writeJSON(w, http.StatusOK, map[string]any{
|
||
"total_input_tokens": row.TotalInputTokens,
|
||
"total_output_tokens": row.TotalOutputTokens,
|
||
"total_cache_read_tokens": row.TotalCacheReadTokens,
|
||
"total_cache_write_tokens": row.TotalCacheWriteTokens,
|
||
// Cost split — see the note on DashboardUsageDailyResponse.
|
||
"cost_usd_ticks": row.TotalCostUsdTicks,
|
||
"uncosted_input_tokens": row.UncostedInputTokens,
|
||
"uncosted_output_tokens": row.UncostedOutputTokens,
|
||
"uncosted_cache_read_tokens": row.UncostedCacheReadTokens,
|
||
"uncosted_cache_write_tokens": row.UncostedCacheWriteTokens,
|
||
// task_count is the legacy count of tasks with usage. Keep it stable for
|
||
// installed clients while the explicit coverage fields distinguish runs
|
||
// from metered runs.
|
||
"task_count": row.TaskCount,
|
||
"terminal_task_count": row.TerminalTaskCount,
|
||
"metered_task_count": row.MeteredTaskCount,
|
||
"unreported_task_count": row.UnreportedTaskCount,
|
||
})
|
||
}
|
||
|
||
const (
|
||
maxIssueGCBatchSize = 500
|
||
maxIssueGCBatchBodyBytes = 64 << 10
|
||
)
|
||
|
||
type batchIssueGCCheckRequest struct {
|
||
IssueIDs []string `json:"issue_ids"`
|
||
}
|
||
|
||
type batchIssueGCCheckItem struct {
|
||
ID string `json:"id"`
|
||
Found bool `json:"found"`
|
||
Status string `json:"status,omitempty"`
|
||
Category string `json:"category,omitempty"`
|
||
UpdatedAt *time.Time `json:"updated_at,omitempty"`
|
||
}
|
||
|
||
// issueGCWire encodes an issue's lifecycle for both GC check endpoints: the
|
||
// four-value `category` a current daemon decides on, and the legacy
|
||
// seven-value `status` enum installed daemons still match literally.
|
||
//
|
||
// `status` predates custom statuses — the daemon tests it against the 7
|
||
// built-in keys and fails closed on anything else (isKnownIssueStatus in
|
||
// daemon/gc.go) — so the stored key cannot be handed back raw. A nonterminal
|
||
// custom key silently disables the GCCompletedTaskTTL full-cleanup path, and a
|
||
// four-value category is not in that vocabulary either. GC consumes exactly one
|
||
// lifecycle fact ("is this issue terminal?"), which is why projecting a custom
|
||
// status onto a built-in key here grants it no built-in behavior anywhere else.
|
||
//
|
||
// A status with no lifecycle category — an unresolvable key after a failed
|
||
// catalog read, or a status created since the resolver's snapshot — is returned
|
||
// raw with no category, so every daemon fails closed and reclaims artifacts
|
||
// only. (MUL-7364)
|
||
//
|
||
// Triage does not reach this: it is not a status, and a Triage entry carries an
|
||
// ordinary non-terminal one, which already reclaims artifacts only.
|
||
func issueGCWire(status, category string) (wireStatus, wireCategory string) {
|
||
if !issuestatus.IsCategory(category) {
|
||
return status, ""
|
||
}
|
||
return issuestatus.WireCategory(status, category), category
|
||
}
|
||
|
||
// BatchIssueGCCheck returns one explicit result for every requested issue ID.
|
||
// The query is workspace-scoped at the SQL layer; missing rows and IDs owned by
|
||
// another workspace both become found=false so the endpoint is not an
|
||
// enumeration oracle. Requests are capped because installed daemons run this
|
||
// endpoint periodically and must not be able to produce unbounded DB work.
|
||
func (h *Handler) BatchIssueGCCheck(w http.ResponseWriter, r *http.Request) {
|
||
workspaceID := chi.URLParam(r, "workspaceId")
|
||
workspaceUUID, ok := parseUUIDOrBadRequest(w, workspaceID, "workspace_id")
|
||
if !ok {
|
||
return
|
||
}
|
||
if !h.requireDaemonWorkspaceAccess(w, r, workspaceID) {
|
||
return
|
||
}
|
||
|
||
r.Body = http.MaxBytesReader(w, r.Body, maxIssueGCBatchBodyBytes)
|
||
var req batchIssueGCCheckRequest
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||
return
|
||
}
|
||
if len(req.IssueIDs) > maxIssueGCBatchSize {
|
||
writeError(w, http.StatusBadRequest, "too many issue_ids")
|
||
return
|
||
}
|
||
|
||
parsedIDs := make([]pgtype.UUID, 0, len(req.IssueIDs))
|
||
canonicalIDs := make([]string, 0, len(req.IssueIDs))
|
||
for _, issueID := range req.IssueIDs {
|
||
parsedID, err := util.ParseUUID(issueID)
|
||
if err != nil {
|
||
writeError(w, http.StatusBadRequest, "invalid issue_id")
|
||
return
|
||
}
|
||
parsedIDs = append(parsedIDs, parsedID)
|
||
canonicalIDs = append(canonicalIDs, uuidToString(parsedID))
|
||
}
|
||
|
||
rows := make(map[string]db.ListIssueGCStatusesRow, len(parsedIDs))
|
||
if len(parsedIDs) > 0 {
|
||
result, err := h.Queries.ListIssueGCStatuses(r.Context(), db.ListIssueGCStatusesParams{
|
||
WorkspaceID: workspaceUUID,
|
||
IssueIds: parsedIDs,
|
||
})
|
||
if err != nil {
|
||
slog.Warn("list issue GC statuses failed", "workspace_id", workspaceID, "count", len(parsedIDs), "error", err)
|
||
writeError(w, http.StatusInternalServerError, "failed to check issues")
|
||
return
|
||
}
|
||
for _, row := range result {
|
||
rows[uuidToString(row.ID)] = row
|
||
}
|
||
}
|
||
|
||
// ONE resolver for the whole batch, not a point lookup per row. The
|
||
// package-level helpers issue a GetIssueStatusEntryByKey for every custom
|
||
// key they see, so resolving inside this loop cost up to maxIssueGCBatchSize
|
||
// catalog queries per request — on an endpoint whose entire purpose is to
|
||
// replace per-issue requests, and which every installed daemon runs on a
|
||
// timer. The resolver reads the catalog lazily and at most once, so an
|
||
// all-built-in batch still costs zero. (MUL-6243)
|
||
resolver := issuestatus.NewResolver(workspaceUUID)
|
||
items := make([]batchIssueGCCheckItem, 0, len(req.IssueIDs))
|
||
for i, issueID := range req.IssueIDs {
|
||
row, found := rows[canonicalIDs[i]]
|
||
item := batchIssueGCCheckItem{ID: issueID, Found: found}
|
||
if found {
|
||
// The daemon consumes this purely as a machine signal ("is this
|
||
// issue terminal, so its workdir can be reclaimed?") and has no
|
||
// database of its own, so the lifecycle is resolved here.
|
||
// Normalizing server-side also means daemons that predate custom
|
||
// statuses keep making correct GC decisions. (MUL-6243)
|
||
item.Status, item.Category = issueGCWire(row.Status,
|
||
resolver.Category(r.Context(), h.issueStatusCatalog(), row.Status))
|
||
updatedAt := row.UpdatedAt.Time
|
||
item.UpdatedAt = &updatedAt
|
||
}
|
||
items = append(items, item)
|
||
}
|
||
writeJSON(w, http.StatusOK, map[string]any{"issues": items})
|
||
}
|
||
|
||
// GetIssueGCCheck returns minimal issue info needed by older daemon GC loops.
|
||
// Gated on workspace access so a daemon token scoped to workspace A cannot
|
||
// read issue metadata from workspace B via UUID enumeration.
|
||
func (h *Handler) GetIssueGCCheck(w http.ResponseWriter, r *http.Request) {
|
||
issueID := chi.URLParam(r, "issueId")
|
||
issueUUID, ok := parseUUIDOrBadRequest(w, issueID, "issue_id")
|
||
if !ok {
|
||
return
|
||
}
|
||
issue, err := h.Queries.GetIssueGCStatus(r.Context(), issueUUID)
|
||
if err != nil {
|
||
writeError(w, http.StatusNotFound, "issue not found")
|
||
return
|
||
}
|
||
if !h.requireDaemonWorkspaceAccess(w, r, uuidToString(issue.WorkspaceID)) {
|
||
return
|
||
}
|
||
// Same reasoning as BatchIssueGCCheck: normalize server-side so the
|
||
// daemon's terminal-status test stays correct. (MUL-6243)
|
||
status, category := issueGCWire(issue.Status,
|
||
issuestatus.Category(r.Context(), h.issueStatusCatalog(), issue.WorkspaceID, issue.Status))
|
||
writeJSON(w, http.StatusOK, map[string]any{
|
||
"status": status,
|
||
"category": category,
|
||
"updated_at": issue.UpdatedAt.Time,
|
||
})
|
||
}
|
||
|
||
// GetChatSessionGCCheck returns the status and updated_at of a chat session
|
||
// for the daemon GC loop. A 404 here means the session was hard-deleted
|
||
// (DeleteChatSession in chat.go runs a real DELETE), which the daemon treats
|
||
// as an immediate-clean signal — the user's explicit delete is the strongest
|
||
// reclaim authorization we can get.
|
||
//
|
||
// Same anti-enumeration shape as GetIssueGCCheck: workspace mismatch returns
|
||
// the same 404 so a scoped daemon token can't probe other workspaces.
|
||
func (h *Handler) GetChatSessionGCCheck(w http.ResponseWriter, r *http.Request) {
|
||
sessionID := chi.URLParam(r, "sessionId")
|
||
sessionUUID, ok := parseUUIDOrBadRequest(w, sessionID, "session_id")
|
||
if !ok {
|
||
return
|
||
}
|
||
session, err := h.Queries.GetChatSession(r.Context(), sessionUUID)
|
||
if err != nil {
|
||
writeError(w, http.StatusNotFound, "chat session not found")
|
||
return
|
||
}
|
||
if !h.requireDaemonWorkspaceAccess(w, r, uuidToString(session.WorkspaceID)) {
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusOK, map[string]any{
|
||
"status": session.Status,
|
||
"updated_at": session.UpdatedAt.Time,
|
||
})
|
||
}
|
||
|
||
// GetAutopilotRunGCCheck returns the status and completed_at of an autopilot
|
||
// run for the daemon GC loop. The daemon decides purely on terminal status:
|
||
// an autopilot run's workdir is never reused, so a terminal run is reclaimed on
|
||
// sight while non-terminal status is a skip signal — completed_at is returned
|
||
// for the API contract and diagnostics, not as a TTL anchor.
|
||
//
|
||
// Workspace ownership is resolved via the parent autopilot row.
|
||
func (h *Handler) GetAutopilotRunGCCheck(w http.ResponseWriter, r *http.Request) {
|
||
runID := chi.URLParam(r, "runId")
|
||
runUUID, ok := parseUUIDOrBadRequest(w, runID, "run_id")
|
||
if !ok {
|
||
return
|
||
}
|
||
run, err := h.Queries.GetAutopilotRun(r.Context(), runUUID)
|
||
if err != nil {
|
||
writeError(w, http.StatusNotFound, "autopilot run not found")
|
||
return
|
||
}
|
||
autopilot, err := h.Queries.GetAutopilot(r.Context(), run.AutopilotID)
|
||
if err != nil {
|
||
// Parent autopilot is gone — treat as not found rather than 500
|
||
// so the daemon can fall through to its orphan-by-mtime path.
|
||
writeError(w, http.StatusNotFound, "autopilot run not found")
|
||
return
|
||
}
|
||
if !h.requireDaemonWorkspaceAccess(w, r, uuidToString(autopilot.WorkspaceID)) {
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusOK, map[string]any{
|
||
"status": run.Status,
|
||
"completed_at": run.CompletedAt.Time,
|
||
})
|
||
}
|
||
|
||
// GetTaskGCCheck returns the agent_task_queue status for quick-create cleanup.
|
||
// Quick-create tasks have no parent record (no issue_id at WriteGCMeta time,
|
||
// no chat session, no autopilot run) so the daemon keys GC directly on the
|
||
// task row itself.
|
||
func (h *Handler) GetTaskGCCheck(w http.ResponseWriter, r *http.Request) {
|
||
taskID := chi.URLParam(r, "taskId")
|
||
task, ok := h.requireDaemonTaskAccess(w, r, taskID)
|
||
if !ok {
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusOK, map[string]any{
|
||
"status": task.Status,
|
||
"completed_at": task.CompletedAt.Time,
|
||
})
|
||
}
|