package handler import ( "context" "encoding/json" "errors" "fmt" "log/slog" "net/http" "strconv" "strings" "time" "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" obsmetrics "github.com/multica-ai/multica/server/internal/metrics" "github.com/multica-ai/multica/server/internal/service" "github.com/multica-ai/multica/server/internal/util" "github.com/multica-ai/multica/server/pkg/agent" db "github.com/multica-ai/multica/server/pkg/db/generated" "github.com/multica-ai/multica/server/pkg/protocol" ) type AgentRuntimeResponse struct { ID string `json:"id"` WorkspaceID string `json:"workspace_id"` DaemonID *string `json:"daemon_id"` Name string `json:"name"` // CustomName is the user-set display override (MUL-4217); null when the // runtime still uses its daemon-proposed Name. Clients show // CustomName ?? Name and seed the rename field from this raw value. CustomName *string `json:"custom_name"` RuntimeMode string `json:"runtime_mode"` Provider string `json:"provider"` LaunchHeader string `json:"launch_header"` Status string `json:"status"` DeviceInfo string `json:"device_info"` Metadata any `json:"metadata"` OwnerID *string `json:"owner_id"` // Visibility is "private" (default — only the owner can bind agents) or // "public" (any workspace member can). See migration 083 and // canUseRuntimeForAgent. Visibility string `json:"visibility"` // ProfileID is set when this runtime is an instance of a custom // runtime_profile (MUL-3284); null for built-in runtimes. ProfileID *string `json:"profile_id"` LastSeenAt *string `json:"last_seen_at"` CreatedAt string `json:"created_at"` UpdatedAt string `json:"updated_at"` } func runtimeToResponse(rt db.AgentRuntime) AgentRuntimeResponse { var metadata any if rt.Metadata != nil { json.Unmarshal(rt.Metadata, &metadata) } if metadata == nil { metadata = map[string]any{} } return AgentRuntimeResponse{ ID: uuidToString(rt.ID), WorkspaceID: uuidToString(rt.WorkspaceID), DaemonID: textToPtr(rt.DaemonID), Name: rt.Name, CustomName: textToPtr(rt.CustomName), RuntimeMode: rt.RuntimeMode, Provider: rt.Provider, LaunchHeader: agent.LaunchHeader(rt.Provider), Status: rt.Status, DeviceInfo: rt.DeviceInfo, Metadata: metadata, OwnerID: uuidToPtr(rt.OwnerID), Visibility: rt.Visibility, ProfileID: uuidToPtr(rt.ProfileID), LastSeenAt: timestampToPtr(rt.LastSeenAt), CreatedAt: timestampToString(rt.CreatedAt), UpdatedAt: timestampToString(rt.UpdatedAt), } } // Runtime Usage type RuntimeUsageResponse struct { RuntimeID string `json:"runtime_id"` Date string `json:"date"` Provider string `json:"provider"` Model string `json:"model"` InputTokens int64 `json:"input_tokens"` OutputTokens int64 `json:"output_tokens"` CacheReadTokens int64 `json:"cache_read_tokens"` CacheWriteTokens int64 `json:"cache_write_tokens"` // Cost split: `CostUSDTicks` is what the provider itself charged for the // rows behind this aggregate (1e-10 USD), and the `Uncosted*` token // counts are the tokens from rows the provider did NOT price. The client // reports authoritative + estimate(uncosted), so a window mixing both // kinds of row stays whole. See migration 213. CostUSDTicks int64 `json:"cost_usd_ticks"` UncostedInputTokens int64 `json:"uncosted_input_tokens"` UncostedOutputTokens int64 `json:"uncosted_output_tokens"` UncostedCacheReadTokens int64 `json:"uncosted_cache_read_tokens"` UncostedCacheWriteTokens int64 `json:"uncosted_cache_write_tokens"` } // GetRuntimeUsage returns daily token usage for a runtime, aggregated from // per-task usage records captured by the daemon. This is scoped to // Daemon-executed tasks only (i.e. excludes users' local CLI usage of the // same tool). func (h *Handler) GetRuntimeUsage(w http.ResponseWriter, r *http.Request) { runtimeID := chi.URLParam(r, "runtimeId") rt, _, ok := h.requireRuntimeReadAccess(w, r, obsmetrics.RuntimeLookupSourceRuntimeAPI, runtimeID) if !ok { return } // All runtime reports render in the viewer's tz. viewTZ := h.resolveViewingTZ(r) since := parseSinceParamInTZ(r, 90, viewTZ) resp, err := h.listRuntimeUsage(r.Context(), rt.ID, viewTZ, since) if err != nil { writeError(w, http.StatusInternalServerError, "failed to list usage") return } writeJSON(w, http.StatusOK, resp) } // listRuntimeUsage reads the daily-bucketed trend from task_usage_hourly, // applying the viewer's tz to project bucket_hour into local days. func (h *Handler) listRuntimeUsage(ctx context.Context, runtimeID pgtype.UUID, tz string, since pgtype.Timestamptz) ([]RuntimeUsageResponse, error) { resolvedRuntimeID := uuidToString(runtimeID) rows, err := h.Queries.ListRuntimeUsage(ctx, db.ListRuntimeUsageParams{ RuntimeID: runtimeID, Since: since, Tz: tz, }) if err != nil { return nil, err } resp := make([]RuntimeUsageResponse, len(rows)) for i, row := range rows { resp[i] = RuntimeUsageResponse{ RuntimeID: resolvedRuntimeID, Date: row.Date.Time.Format("2006-01-02"), Provider: row.Provider, Model: row.Model, InputTokens: row.InputTokens, OutputTokens: row.OutputTokens, CacheReadTokens: row.CacheReadTokens, CacheWriteTokens: row.CacheWriteTokens, CostUSDTicks: row.CostUsdTicks, UncostedInputTokens: row.UncostedInputTokens, UncostedOutputTokens: row.UncostedOutputTokens, UncostedCacheReadTokens: row.UncostedCacheReadTokens, UncostedCacheWriteTokens: row.UncostedCacheWriteTokens, } } return resp, nil } // GetRuntimeTaskActivity returns hourly task activity distribution for a runtime. func (h *Handler) GetRuntimeTaskActivity(w http.ResponseWriter, r *http.Request) { runtimeID := chi.URLParam(r, "runtimeId") rt, _, ok := h.requireRuntimeReadAccess(w, r, obsmetrics.RuntimeLookupSourceRuntimeAPI, runtimeID) if !ok { return } viewTZ := h.resolveViewingTZ(r) rows, err := h.Queries.GetRuntimeTaskHourlyActivity(r.Context(), db.GetRuntimeTaskHourlyActivityParams{ RuntimeID: rt.ID, Tz: viewTZ, }) if err != nil { writeError(w, http.StatusInternalServerError, "failed to get task activity") return } type HourlyActivity struct { Hour int `json:"hour"` Count int `json:"count"` } resp := make([]HourlyActivity, len(rows)) for i, row := range rows { resp[i] = HourlyActivity{Hour: int(row.Hour), Count: int(row.Count)} } writeJSON(w, http.StatusOK, resp) } // RuntimeUsageByAgentResponse is one (agent, provider, model) row of "Cost by // agent". provider + model stay on the wire because cost is computed // client-side from a model pricing table (intentionally not stored server-side // so pricing changes don't require a back-fill); provider disambiguates bare // model ids that collide across providers. The client groups by agent_id and sums. type RuntimeUsageByAgentResponse struct { AgentID string `json:"agent_id"` Provider string `json:"provider"` Model string `json:"model"` InputTokens int64 `json:"input_tokens"` OutputTokens int64 `json:"output_tokens"` CacheReadTokens int64 `json:"cache_read_tokens"` CacheWriteTokens int64 `json:"cache_write_tokens"` // Cost split: `CostUSDTicks` is what the provider itself charged for the // rows behind this aggregate (1e-10 USD), and the `Uncosted*` token // counts are the tokens from rows the provider did NOT price. The client // reports authoritative + estimate(uncosted), so a window mixing both // kinds of row stays whole. See migration 213. CostUSDTicks int64 `json:"cost_usd_ticks"` UncostedInputTokens int64 `json:"uncosted_input_tokens"` UncostedOutputTokens int64 `json:"uncosted_output_tokens"` UncostedCacheReadTokens int64 `json:"uncosted_cache_read_tokens"` UncostedCacheWriteTokens int64 `json:"uncosted_cache_write_tokens"` TaskCount int32 `json:"task_count"` } // GetRuntimeUsageByAgent returns per-agent token aggregates for a runtime // since the cutoff window. Drives the runtime-detail "Cost by agent" tab. func (h *Handler) GetRuntimeUsageByAgent(w http.ResponseWriter, r *http.Request) { runtimeID := chi.URLParam(r, "runtimeId") rt, _, ok := h.requireRuntimeReadAccess(w, r, obsmetrics.RuntimeLookupSourceRuntimeAPI, runtimeID) if !ok { return } // No date bucketing — tz only sets the cutoff boundary so "last 30 // days" means 30 of the viewer's days. viewTZ := h.resolveViewingTZ(r) since := parseSinceParamInTZ(r, 30, viewTZ) rows, err := h.Queries.ListRuntimeUsageByAgent(r.Context(), db.ListRuntimeUsageByAgentParams{ RuntimeID: rt.ID, Since: since, }) if err != nil { writeError(w, http.StatusInternalServerError, "failed to list usage by agent") return } resp := make([]RuntimeUsageByAgentResponse, len(rows)) for i, row := range rows { resp[i] = RuntimeUsageByAgentResponse{ AgentID: uuidToString(row.AgentID), Provider: row.Provider, Model: row.Model, InputTokens: row.InputTokens, OutputTokens: row.OutputTokens, CacheReadTokens: row.CacheReadTokens, CacheWriteTokens: row.CacheWriteTokens, CostUSDTicks: row.CostUsdTicks, UncostedInputTokens: row.UncostedInputTokens, UncostedOutputTokens: row.UncostedOutputTokens, UncostedCacheReadTokens: row.UncostedCacheReadTokens, UncostedCacheWriteTokens: row.UncostedCacheWriteTokens, TaskCount: row.TaskCount, } } writeJSON(w, http.StatusOK, resp) } // RuntimeUsageByHourResponse is one (hour, model) row. Hours with zero // activity are omitted by the SQL — clients fill the gap to render a // continuous 0..23 axis. Model is preserved for client-side cost math. type RuntimeUsageByHourResponse struct { Hour int `json:"hour"` Model string `json:"model"` InputTokens int64 `json:"input_tokens"` OutputTokens int64 `json:"output_tokens"` CacheReadTokens int64 `json:"cache_read_tokens"` CacheWriteTokens int64 `json:"cache_write_tokens"` // Cost split: `CostUSDTicks` is what the provider itself charged for the // rows behind this aggregate (1e-10 USD), and the `Uncosted*` token // counts are the tokens from rows the provider did NOT price. The client // reports authoritative + estimate(uncosted), so a window mixing both // kinds of row stays whole. See migration 213. CostUSDTicks int64 `json:"cost_usd_ticks"` UncostedInputTokens int64 `json:"uncosted_input_tokens"` UncostedOutputTokens int64 `json:"uncosted_output_tokens"` UncostedCacheReadTokens int64 `json:"uncosted_cache_read_tokens"` UncostedCacheWriteTokens int64 `json:"uncosted_cache_write_tokens"` TaskCount int32 `json:"task_count"` } // GetRuntimeUsageByHour returns hourly (0..23) token aggregates for a // runtime since the cutoff window. Drives the "By hour" tab. // // The hour-of-day axis is bucketed in the viewer's tz like every other // report — the same timezone resolved by resolveViewingTZ from the request's // `?tz=` param or the authenticated user's stored user.timezone. func (h *Handler) GetRuntimeUsageByHour(w http.ResponseWriter, r *http.Request) { runtimeID := chi.URLParam(r, "runtimeId") rt, _, ok := h.requireRuntimeReadAccess(w, r, obsmetrics.RuntimeLookupSourceRuntimeAPI, runtimeID) if !ok { return } viewTZ := h.resolveViewingTZ(r) since := parseSinceParamInTZ(r, 30, viewTZ) rows, err := h.Queries.GetRuntimeUsageByHour(r.Context(), db.GetRuntimeUsageByHourParams{ RuntimeID: rt.ID, Since: since, Tz: viewTZ, }) if err != nil { writeError(w, http.StatusInternalServerError, "failed to get usage by hour") return } resp := make([]RuntimeUsageByHourResponse, len(rows)) for i, row := range rows { resp[i] = RuntimeUsageByHourResponse{ Hour: int(row.Hour), Model: row.Model, InputTokens: row.InputTokens, OutputTokens: row.OutputTokens, CacheReadTokens: row.CacheReadTokens, CacheWriteTokens: row.CacheWriteTokens, CostUSDTicks: row.CostUsdTicks, UncostedInputTokens: row.UncostedInputTokens, UncostedOutputTokens: row.UncostedOutputTokens, UncostedCacheReadTokens: row.UncostedCacheReadTokens, UncostedCacheWriteTokens: row.UncostedCacheWriteTokens, TaskCount: row.TaskCount, } } writeJSON(w, http.StatusOK, resp) } // sinceFromDays is the pure, now-injectable core of parseSinceParamInTZ. // Given the current instant, a day count and an IANA location, it returns // the instant of local midnight `days` days before `now`'s local calendar // day. `now` is a parameter so the DST boundary maths can be tested at // pinned dates (see TestSinceFromDays). // // The cutoff yields N+1 calendar buckets (today-days … today inclusive). // The extra day versus a naive "-(days-1)" is deliberate headroom, not an // off-by-one: // - Runtime detail's sliceWindow filters `date >= today-days` (closed) and // its prior-window delta reaches back to today-2*days, so the today-days // bucket MUST exist or the oldest bar / KPI delta silently loses data. // - The workspace dashboard re-filters client-side with -(days-1); the one // extra day the backend returns is trimmed there — harmless. // // Do not "tighten" this to -(days-1): it would break the runtime detail page. func sinceFromDays(now time.Time, days int, loc *time.Location) time.Time { local := now.In(loc) startOfToday := time.Date(local.Year(), local.Month(), local.Day(), 0, 0, 0, 0, loc) return startOfToday.AddDate(0, 0, -days) } // parseSinceParamInTZ parses the "days" query parameter into a cutoff // timestamptz. Anchors the cutoff to start-of-day-(N) in the supplied IANA zone so that // `days=N` returns full N+1 calendar buckets in that zone (today's partial // bucket + N prior full days). If tzName is empty or unparseable, falls back // to UTC — never returns an error so handlers stay simple. func parseSinceParamInTZ(r *http.Request, defaultDays int, tzName string) pgtype.Timestamptz { return parseDaysCutoff(r, defaultDays, tzName, 0) } // parseExactSinceParamInTZ is parseSinceParamInTZ without the extra day of // headroom: `days=N` yields exactly N calendar buckets (today's partial // bucket + N-1 prior full days), which is the window the workspace dashboard // actually displays. // // The N+1 cutoff exists so date-bucketed series can reach one bucket further // back than they render (runtime detail's prior-window delta needs it), and // the dashboard trims the surplus client-side with `-(days-1)`. A response // with NO date dimension cannot be trimmed that way, so an aggregate served // off the N+1 cutoff silently covers one more day than the chart beside it. // Endpoints whose rows carry no date use this variant instead. func parseExactSinceParamInTZ(r *http.Request, defaultDays int, tzName string) pgtype.Timestamptz { return parseDaysCutoff(r, defaultDays, tzName, 1) } // parseDaysCutoff is the shared body of the two cutoff parsers. `trimDays` // pulls the cutoff forward, so 0 keeps the N+1 headroom and 1 closes the // window to exactly N calendar days. // dayWindowNow is the clock every `days=N` cutoff reads. // // A variable so a test can pin it. Without one, a fixture and the cutoff that // has to contain it read the wall clock at two different moments — the fixture // when it is built, the cutoff when the request is handled, with inserts and a // rollup in between. A suite that crosses midnight in that gap builds its run // in one day and then asks for the next day's window, and the row it just // wrote is excluded. Production always reads the real clock. var dayWindowNow = time.Now func parseDaysCutoff( r *http.Request, defaultDays int, tzName string, trimDays int, ) pgtype.Timestamptz { days := defaultDays if d := r.URL.Query().Get("days"); d != "" { if parsed, err := strconv.Atoi(d); err == nil && parsed > 0 && parsed <= 365 { days = parsed } } loc, err := time.LoadLocation(tzName) if err != nil || loc == nil { loc = time.UTC } // Guard the floor: days is already >= 1 here, and trimming a 1-day // window by one would put the cutoff at start-of-today+0 — still correct // ("today only"), which is exactly what days=1 means. return pgtype.Timestamptz{ Time: sinceFromDays(dayWindowNow(), days-trimDays, loc), Valid: true, } } // resolveViewingTZ resolves the IANA tz to render the response in: // `?tz=` query param, else the authenticated user's stored // user.timezone, else "UTC". Invalid values fall through rather than // erroring — tz is a display concern. // // The browser app always sends `?tz=` (resolved client-side by // useViewingTimezone), so the `GetUser` lookup below is a COLD fallback // hit only by API clients / older builds that omit the param — it is not // a hot path. Do not replicate this DB-read pattern into a handler that // runs without a `?tz=`-supplying client in front of it. func (h *Handler) resolveViewingTZ(r *http.Request) string { if tz := strings.TrimSpace(r.URL.Query().Get("tz")); tz != "" { if loc, err := time.LoadLocation(tz); err == nil && loc != nil { return tz } } if userID := requestUserID(r); userID != "" { uid, err := util.ParseUUID(userID) if err != nil { slog.Warn("resolveViewingTZ: malformed X-User-ID, falling back to UTC", "path", r.URL.Path, "user_id", userID) } if err == nil { slog.Debug("resolveViewingTZ cold path: ?tz= missing, reading user.timezone", "path", r.URL.Path, "user_id", userID) if user, err := h.Queries.GetUser(r.Context(), uid); err == nil && user.Timezone.Valid { stored := strings.TrimSpace(user.Timezone.String) if stored != "" { if loc, err := time.LoadLocation(stored); err == nil && loc != nil { return stored } } } } } return "UTC" } // UpdateAgentRuntimeRequest is the JSON body accepted by PATCH /api/runtimes/:id. // Only fields users may legitimately edit are listed; other runtime metadata // (provider, daemon_id, status…) flows in from the daemon and is read-only here. type UpdateAgentRuntimeRequest struct { // Visibility flips a runtime between "private" (default — only the owner // can bind agents) and "public" (any workspace member can). Runtime owner // only, gated by canSetRuntimeVisibility — narrower than the rest of this // request, which admins may also send. Visibility *string `json:"visibility,omitempty"` // CustomName sets or clears a user-facing display override (MUL-4217). // An empty / whitespace-only string clears it (revert to the // daemon-proposed name). Owner / workspace admin only. CustomName *string `json:"custom_name,omitempty"` // ApplyToMachine, when true alongside CustomName, applies the name to // every runtime sharing this runtime's daemon_id (a machine hosts one // runtime per provider) instead of just this one. Ignored when the // runtime has no daemon_id. ApplyToMachine bool `json:"apply_to_machine,omitempty"` } // maxRuntimeCustomNameLen caps a runtime's custom name. Default names are // short (e.g. "Claude (host.local)"); 100 chars is generous headroom while // keeping the picker rows and machine headers from overflowing. const maxRuntimeCustomNameLen = 100 // UpdateAgentRuntime handles PATCH /api/runtimes/:id. Currently visibility // is editable; the request shape is open-ended so future fields (display // name, description) can be added without a route change. // Workspace-membership-checked; write access is gated by canEditRuntime, and // `visibility` carries the narrower owner-only gate on top (MUL-6126). func (h *Handler) UpdateAgentRuntime(w http.ResponseWriter, r *http.Request) { runtimeID := chi.URLParam(r, "runtimeId") runtimeUUID, ok := parseUUIDOrBadRequest(w, runtimeID, "runtime_id") if !ok { return } rt, err := h.getAgentRuntime(r.Context(), obsmetrics.RuntimeLookupSourceRuntimeAPI, runtimeUUID) if err != nil { writeError(w, http.StatusNotFound, "runtime not found") return } member, ok := h.requireWorkspaceMember(w, r, uuidToString(rt.WorkspaceID), "runtime not found") if !ok { return } if !canEditRuntime(member, rt) { writeError(w, http.StatusForbidden, "you can only edit your own runtimes") return } var req UpdateAgentRuntimeRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid JSON body") return } // Validate every field before any mutation so a bad value in one field // can't leave a partially-applied PATCH. var ( newVisibility string needVisibility bool ) if req.Visibility != nil { v := *req.Visibility if v != "private" && v != "public" { writeError(w, http.StatusBadRequest, "visibility must be 'private' or 'public'") return } // Only a real change is gated. An admin renaming a runtime through a // PATCH-as-PUT client that echoes the unchanged visibility back is not // trying to share anyone's machine, so that no-op is dropped rather // than turned into a 403 — same tolerance UpdateAgent applies to // resubmitted agent permissions. if v != rt.Visibility { if !canSetRuntimeVisibility(member, rt) { writeError(w, http.StatusForbidden, "only the runtime owner can change its visibility") return } newVisibility = v needVisibility = true } } if req.CustomName != nil { if len([]rune(strings.TrimSpace(*req.CustomName))) > maxRuntimeCustomNameLen { writeError(w, http.StatusBadRequest, "custom name is too long") return } } changed := false if needVisibility { updated, err := h.Queries.UpdateAgentRuntimeVisibility(r.Context(), db.UpdateAgentRuntimeVisibilityParams{ ID: runtimeUUID, Visibility: newVisibility, }) if err != nil { slog.Error("UpdateAgentRuntimeVisibility failed", "error", err, "runtime_id", runtimeID) writeError(w, http.StatusInternalServerError, "failed to update runtime") return } rt = updated changed = true } if req.CustomName != nil { // An empty / whitespace-only name clears the override (NULL), so the // runtime falls back to its daemon-proposed Name. trimmed := strings.TrimSpace(*req.CustomName) customName := pgtype.Text{String: trimmed, Valid: trimmed != ""} if req.ApplyToMachine && rt.DaemonID.Valid { // Non-admins may only relabel their own runtimes on the machine; // owners/admins rename every runtime sharing the daemon_id. A NULL // owner filter means "all runtimes on this machine". var ownerFilter pgtype.UUID if !roleAllowed(member.Role, "owner", "admin") { ownerFilter = member.UserID } rows, err := h.Queries.UpdateAgentRuntimeCustomNameByDaemon(r.Context(), db.UpdateAgentRuntimeCustomNameByDaemonParams{ CustomName: customName, WorkspaceID: rt.WorkspaceID, DaemonID: rt.DaemonID, OwnerID: ownerFilter, }) if err != nil { slog.Error("UpdateAgentRuntimeCustomNameByDaemon failed", "error", err, "runtime_id", runtimeID) writeError(w, http.StatusInternalServerError, "failed to update runtime") return } // The actor always owns (or admins) the runtime addressed by :id, // so it is among the updated rows — surface it in the response. for _, row := range rows { if uuidToString(row.ID) == uuidToString(runtimeUUID) { rt = row break } } changed = true } else { updated, err := h.Queries.UpdateAgentRuntimeCustomName(r.Context(), db.UpdateAgentRuntimeCustomNameParams{ CustomName: customName, ID: runtimeUUID, }) if err != nil { slog.Error("UpdateAgentRuntimeCustomName failed", "error", err, "runtime_id", runtimeID) writeError(w, http.StatusInternalServerError, "failed to update runtime") return } rt = updated changed = true } } if changed { // Notify connected clients that runtime metadata changed so the // list/detail pages refresh — matches the pattern used by // DeleteAgentRuntime. h.publish(protocol.EventDaemonRegister, uuidToString(rt.WorkspaceID), "member", uuidToString(member.UserID), map[string]any{ "action": "update", }) } writeJSON(w, http.StatusOK, runtimeToResponse(rt)) } func canEditRuntime(member db.Member, rt db.AgentRuntime) bool { if roleAllowed(member.Role, "owner", "admin") { return true } return rt.OwnerID.Valid && uuidToString(rt.OwnerID) == uuidToString(member.UserID) } // getAgentRuntime reads one agent_runtime row by id and attributes the read to // source, which labels multica_agent_runtime_lookup_total (MUL-6884). Pick the // obsmetrics.RuntimeLookupSource* constant that names the product behaviour // driving the read, not the file the call happens to live in: a poll loop // counted as generic API traffic is exactly the confusion the metric exists to // remove. func (h *Handler) getAgentRuntime(ctx context.Context, source string, id pgtype.UUID) (db.AgentRuntime, error) { return h.runtimeLookup(source).Get(ctx, id) } // getAgentRuntimes is the batch sibling of getAgentRuntime: one query for many // ids, attributed to source the same way (MUL-6788). It returns the rows keyed // by canonical UUID string and surfaces the read error so batch callers fail // closed instead of treating a failed read as "no rows exist". func (h *Handler) getAgentRuntimes(ctx context.Context, source string, ids []pgtype.UUID) (map[string]db.AgentRuntime, error) { return h.runtimeLookup(source).GetMany(ctx, ids) } // runtimeLookup is the same reader, unexecuted, for handlers that hand it to a // shared readiness helper instead of reading the row themselves. func (h *Handler) runtimeLookup(source string) service.RuntimeLookup { return service.RuntimeLookup{Queries: h.Queries, Metrics: h.Metrics, Source: source} } // requireRuntimeReadAccess protects runtime data and machine-triggering // capabilities. Governance access is deliberately separate: workspace owners // and admins may list, rename, or delete another member's private runtime, // but a private machine is readable or usable only by its owner. Returning // 404 for that case prevents a known runtime ID from becoming an oracle. // // source names the product behaviour behind the read for // multica_agent_runtime_lookup_total (MUL-6884). It matters here more than // anywhere else: this one gate serves both a rarely-opened usage tab and // several 500ms browser poll loops, and counting them together would hide the // polling the metric exists to measure. func (h *Handler) requireRuntimeReadAccess(w http.ResponseWriter, r *http.Request, source, runtimeID string) (db.AgentRuntime, db.Member, bool) { runtimeUUID, ok := parseUUIDOrBadRequest(w, runtimeID, "runtime_id") if !ok { return db.AgentRuntime{}, db.Member{}, false } rt, err := h.getAgentRuntime(r.Context(), source, runtimeUUID) if err != nil { writeError(w, http.StatusNotFound, "runtime not found") return db.AgentRuntime{}, db.Member{}, false } member, ok := h.requireWorkspaceMember(w, r, uuidToString(rt.WorkspaceID), "runtime not found") if !ok || !canUseRuntimeForAgent(member, rt) { if ok { writeError(w, http.StatusNotFound, "runtime not found") } return db.AgentRuntime{}, db.Member{}, false } return rt, member, true } // runtimeLiveProfile returns the custom runtime profile that owns rt, if that // profile still exists in the same workspace. A profile-backed instance whose // profile is gone is an orphan and stays directly deletable (MUL-4158). // // The profile row itself — not just "one exists" — is what the caller needs: // the refusal it writes names the profile, so the user can tell which shared // definition they would be reaching for if they followed the old advice. func (h *Handler) runtimeLiveProfile(ctx context.Context, rt db.AgentRuntime) (db.RuntimeProfile, bool, error) { if !rt.ProfileID.Valid { return db.RuntimeProfile{}, false, nil } profile, err := h.Queries.GetRuntimeProfileForWorkspace(ctx, db.GetRuntimeProfileForWorkspaceParams{ ID: rt.ProfileID, WorkspaceID: rt.WorkspaceID, }) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return db.RuntimeProfile{}, false, nil } return db.RuntimeProfile{}, false, err } return profile, true, nil } // profileInstanceDeleteRefusal explains why this one runtime row cannot be // deleted on its own, and — the part that matters — what the user should // actually do instead. // // The previous wording said only "delete its runtime profile instead", which // is actively harmful advice for the case that produces this error most often // (GH #8456, #6671): a retired machine's leftover row inside a profile that // other, healthy machines still use. Following it means reaching for a // workspace-wide delete that takes those machines' runtimes with it, and that // a bound agent will refuse anyway. So the refusal now leads with the outcome // the user wants — an offline row is reclaimed automatically — and states the // blast radius of the profile delete rather than recommending it. // // blockers are the non-archived user agents bound to rt, matching the predicate // retention GC applies; known is false when that read failed. They decide two // things. Whether the promise of automatic cleanup is one this server can keep // at all — GC skips a runtime that still has a bound agent — and, when it is // not, which of those blockers the user can actually do anything about. Mika is // a user-kind agent that can be neither archived nor moved, so "reassign or // archive them" is not a universal instruction here either. func profileInstanceDeleteRefusal(rt db.AgentRuntime, profile db.RuntimeProfile, blockers profileInstanceBlockers) map[string]any { known := blockers.known ttlDays := service.OfflineRuntimeTTLDays() name := rt.Name if rt.CustomName.Valid && strings.TrimSpace(rt.CustomName.String) != "" { name = rt.CustomName.String } lead := fmt.Sprintf( "cannot delete %q on its own: it is registered from the custom runtime profile %q.", name, profile.DisplayName, ) scope := "Deleting the profile instead would remove this runtime on every machine that registered it, not just this one." parts := []string{lead} switch { case rt.Status == "online": parts = append(parts, fmt.Sprintf( "It is still online, so its daemon would register it again. Stop that daemon first; Multica then removes the runtime automatically after %d days offline, once no agent is bound to it and nothing is still running on it.", ttlDays, )) case !known: // Blocker set unavailable; promise only what holds regardless of it. parts = append(parts, fmt.Sprintf( "It is offline, and Multica removes offline runtimes automatically after %d days, once no agent is bound to them and nothing is still running on them.", ttlDays, )) case len(blockers.agents) > 0 || blockers.undrainedTasks > 0: // GC needs BOTH gone. Naming only the agents would send a user who // clears them straight back here a week later, still waiting on a task // nothing told them about — a deferred run left behind when its agent // was rebound elsewhere is the ordinary way this happens. var holds []string if n := len(blockers.agents); n > 0 { holds = append(holds, fmt.Sprintf("%d agent(s) are still bound to it", n)) } if n := blockers.undrainedTasks; n > 0 { holds = append(holds, fmt.Sprintf("%d unfinished task(s) belong to it or to agents bound to it", n)) } parts = append(parts, fmt.Sprintf( "It is offline, but %s, which holds it in place; Multica removes the runtime automatically after %d days offline once that is cleared.", strings.Join(holds, " and "), ttlDays, )) parts = append(parts, blockingAgentRemedies(blockingAgentClassesFromAgents(blockers.agents), blockingAgentScopeInstance)...) if blockers.undrainedTasks > 0 { parts = append(parts, "Let those tasks finish, or cancel them — one can be running on a different machine if its agent was moved there.") } default: parts = append(parts, fmt.Sprintf( "It is offline with no agents bound and nothing still running on it, so Multica removes it automatically after %d days offline — this row will be reclaimed without any action from you.", ttlDays, )) } parts = append(parts, scope) msg := strings.Join(parts, " ") resp := map[string]any{ "error": msg, "code": "runtime_profile_instance_delete_unsupported", // Structured companions to the sentence above so a client can render // its own localized copy instead of echoing the English (see // writeErrorCode's rationale). The sentence stays the fallback. "profile_id": uuidToString(profile.ID), "profile_name": profile.DisplayName, "runtime_status": rt.Status, "last_seen_at": timestampToPtr(rt.LastSeenAt), "auto_cleanup_after_days": ttlDays, } if known { resp["active_agent_count"] = len(blockers.agents) resp["undrained_task_count"] = blockers.undrainedTasks } return resp } // profileInstanceBlockers is everything retention GC checks before it will // reclaim an offline runtime, which is more than its candidate query asks for: // the candidate scan wants no non-archived user agent and no runtime-owned task // with completed_at NULL, and then gcRuntime re-checks the drain across every // user agent bound to the runtime, archived ones included. Reporting any subset // of that promises a cleanup the sweeper then skips. type profileInstanceBlockers struct { agents []db.Agent undrainedTasks int64 known bool } // profileInstanceRefusalBlockers reads what would stop retention GC from // reclaiming this runtime. A read failure is not worth failing the request // over: it only costs the refusal its most specific sentence, so report it as // unknown and let the caller fall back to the cautious wording. // // Already bounded — a single runtime's bound agents, unlike a profile's, are // capped by what one machine can host. func (h *Handler) profileInstanceRefusalBlockers(ctx context.Context, runtimeID pgtype.UUID) profileInstanceBlockers { agents, err := h.Queries.ListActiveAgentsByRuntime(ctx, runtimeID) if err != nil { slog.Warn("profile instance refusal: active agent lookup failed", "runtime_id", uuidToString(runtimeID), "error", err) return profileInstanceBlockers{} } // The same drain gate gcRuntime applies, and deliberately not just the // candidate query's runtime-owned predicate: gcRuntime widens it to every // user agent bound to this runtime, archived included, and skips the delete // when any of them still owns a non-terminal task. That task can sit on a // different machine — an agent moved away leaves its deferred run behind — // so a check scoped to this runtime's own rows reports a row as reclaimable // that the sweeper will pass over every hour. agentIDs, err := h.Queries.ListUserAgentIDsByRuntime(ctx, runtimeID) if err != nil { slog.Warn("profile instance refusal: bound agent id lookup failed", "runtime_id", uuidToString(runtimeID), "error", err) return profileInstanceBlockers{} } tasks, err := h.Queries.CountUndrainedTasksByRuntimeOrAgent(ctx, db.CountUndrainedTasksByRuntimeOrAgentParams{ RuntimeIds: []pgtype.UUID{runtimeID}, AgentIds: agentIDs, }) if err != nil { slog.Warn("profile instance refusal: undrained task lookup failed", "runtime_id", uuidToString(runtimeID), "error", err) return profileInstanceBlockers{} } return profileInstanceBlockers{agents: agents, undrainedTasks: tasks, known: true} } // canUseRuntimeForAgent reports whether a workspace member is allowed to // bind a new agent to — or move an existing agent onto — the given runtime. // A `public` runtime is usable by anyone in the workspace; a `private` one is // usable only by its owner, with NO workspace owner/admin override (MUL-6126): // a private runtime is someone's own machine, and running an agent on it // spends their credentials and their local files, which is not an // administrative decision. Sharing it is the owner's call, which is why // canSetRuntimeVisibility is owner-only too — otherwise an admin would keep // the same override with one extra step. // // An ownerless runtime is refused whatever its visibility: task claim needs an // owner to mint the agent's task token and cancels the task without one // (MUL-3292), so binding to one only produces agents that fail at run time. // // This is the same rule the clients enforce (`isRuntimeUsableForUser` in // packages/core/runtimes/access.ts), so UI, API and CLI agree. See migration // 083 for the visibility column. func canUseRuntimeForAgent(member db.Member, rt db.AgentRuntime) bool { if !rt.OwnerID.Valid { return false } if rt.Visibility == "public" { return true } return uuidToString(rt.OwnerID) == uuidToString(member.UserID) } // canSetRuntimeVisibility reports whether a member may flip this runtime // between `private` and `public`. Owner-only, and deliberately narrower than // canEditRuntime: visibility is the owner's consent to lend their machine to // the workspace, so an admin who could flip it would still hold the override // canUseRuntimeForAgent removed. Admins keep canEditRuntime for the // organisational actions — rename, delete — that do not hand anyone else's // credentials out. func canSetRuntimeVisibility(member db.Member, rt db.AgentRuntime) bool { return rt.OwnerID.Valid && uuidToString(rt.OwnerID) == uuidToString(member.UserID) } type runtimeLister interface { ListAgentRuntimes(ctx context.Context, workspaceID pgtype.UUID) ([]db.AgentRuntime, error) ListVisibleAgentRuntimes(ctx context.Context, arg db.ListVisibleAgentRuntimesParams) ([]db.AgentRuntime, error) } func runtimesVisibleTo(ctx context.Context, q runtimeLister, member db.Member) ([]db.AgentRuntime, error) { if roleAllowed(member.Role, "owner", "admin") { return q.ListAgentRuntimes(ctx, member.WorkspaceID) } return q.ListVisibleAgentRuntimes(ctx, db.ListVisibleAgentRuntimesParams{ WorkspaceID: member.WorkspaceID, OwnerID: member.UserID, }) } func (h *Handler) ListAgentRuntimes(w http.ResponseWriter, r *http.Request) { workspaceID := h.resolveWorkspaceID(r) userID, ok := requireUserID(w, r) if !ok { return } member, ok := h.requireWorkspaceMember(w, r, workspaceID, "workspace not found") if !ok { return } var runtimes []db.AgentRuntime var err error if ownerFilter := r.URL.Query().Get("owner"); ownerFilter == "me" { runtimes, err = h.Queries.ListAgentRuntimesByOwner(r.Context(), db.ListAgentRuntimesByOwnerParams{ WorkspaceID: parseUUID(workspaceID), OwnerID: parseUUID(userID), }) } else { runtimes, err = runtimesVisibleTo(r.Context(), h.Queries, member) } if err != nil { writeError(w, http.StatusInternalServerError, "failed to list runtimes") return } resp := make([]AgentRuntimeResponse, len(runtimes)) for i, rt := range runtimes { resp[i] = runtimeToResponse(rt) } writeJSON(w, http.StatusOK, resp) } // DeleteAgentRuntime deletes a runtime after permission and dependency checks. // // The strict variant: refuses with 409 + structured `runtime_has_active_agents` // when any non-archived agent is still bound to the runtime, and returns the // blocking agent list in the response body so the front-end can pivot to the // confirm dialog without an extra round-trip. The confirmed variant lives at // POST /api/runtimes/:id/unbind-agents-and-delete (UnbindAgentsAndDeleteRuntime // below) and runs the multi-write teardown inside a single transaction. // PublishRuntimeTeardown fans out a committed teardown. The caller controls // actor metadata and whether to append a runtime-list refresh so automatic GC // can deduplicate that refresh once per workspace and batch. func (h *Handler) PublishRuntimeTeardown(ctx context.Context, res service.RuntimeTeardownResult, wsID, actorType, actorID, action string, publishRuntimeRefresh bool) { if h.TaskService != nil && len(res.CancelledTasks) > 0 { // The teardown deletes the runtime's system agents, and a system agent's // chat sessions go with it, so the workspace of a cancelled chat task is // no longer resolvable from the task row. It is this workspace. h.TaskService.BroadcastCancelledTasks(ctx, wsID, res.CancelledTasks) } for _, a := range res.UnboundAgents { // agent:status is the generic "this agent changed" broadcast the agent // update path already uses; subscribers refresh the row and see // runtime_bound=false. No agent:archived here — nothing was archived. h.publish(protocol.EventAgentStatus, wsID, actorType, actorID, map[string]any{ "agent": broadcastAgentResponse(h.agentToResponse(a)), }) } for _, a := range res.PausedAutopilots { h.publish(protocol.EventAutopilotUpdated, wsID, actorType, actorID, map[string]any{ "autopilot": autopilotToResponse(a, nil), }) } if publishRuntimeRefresh { h.PublishRuntimeRefresh(wsID, actorType, actorID, action) } } // PublishRuntimeRefresh asks connected clients to refetch runtime state. func (h *Handler) PublishRuntimeRefresh(wsID, actorType, actorID, action string) { h.publish(protocol.EventDaemonRegister, wsID, actorType, actorID, map[string]any{"action": action}) } func (h *Handler) publishRuntimeTeardown(ctx context.Context, res service.RuntimeTeardownResult, wsID, userID string) { h.PublishRuntimeTeardown(ctx, res, wsID, "member", userID, "delete", true) } func (h *Handler) DeleteAgentRuntime(w http.ResponseWriter, r *http.Request) { runtimeID := chi.URLParam(r, "runtimeId") runtimeUUID, ok := parseUUIDOrBadRequest(w, runtimeID, "runtime_id") if !ok { return } rt, err := h.getAgentRuntime(r.Context(), obsmetrics.RuntimeLookupSourceRuntimeAPI, runtimeUUID) if err != nil { writeError(w, http.StatusNotFound, "runtime not found") return } wsID := uuidToString(rt.WorkspaceID) member, ok := h.requireWorkspaceMember(w, r, wsID, "runtime not found") if !ok { return } // Permission: owner/admin can delete any runtime; members can only delete their own. if !canEditRuntime(member, rt) { writeError(w, http.StatusForbidden, "you can only delete your own runtimes") return } userID := uuidToString(member.UserID) profile, hasLiveProfile, err := h.runtimeLiveProfile(r.Context(), rt) if err != nil { writeError(w, http.StatusInternalServerError, "failed to check runtime profile") return } if hasLiveProfile { blockers := h.profileInstanceRefusalBlockers(r.Context(), rt.ID) writeJSON(w, http.StatusConflict, profileInstanceDeleteRefusal(rt, profile, blockers)) return } if rt.ProfileID.Valid { slog.Warn("deleting orphaned profile-backed runtime instance", "runtime_id", uuidToString(rt.ID), "profile_id", uuidToString(rt.ProfileID), "workspace_id", wsID, "deleted_by", userID) } // Check if any active (non-archived) agents are bound to this runtime. // Surface them on the 409 so the dialog can render the cascade plan // directly from this response — saves a second round-trip when the // user clicked Delete from a stale list page. activeAgents, err := h.Queries.ListActiveAgentsByRuntime(r.Context(), rt.ID) if err != nil { writeError(w, http.StatusInternalServerError, "failed to check runtime dependencies") return } // Refuse before any teardown-side effects while active agents are still // bound. The user confirms the plan through // POST /runtimes/:id/unbind-agents-and-delete, which reuses the same // teardown once the confirmed set is verified. if len(activeAgents) > 0 { writeJSON(w, http.StatusConflict, h.runtimeHasActiveAgentsResponse(activeAgents)) return } tx, err := h.TxStarter.Begin(r.Context()) if err != nil { writeError(w, http.StatusInternalServerError, "failed to delete runtime") return } defer tx.Rollback(r.Context()) qtx := h.Queries.WithTx(tx) // Revalidate under the runtime row lock. Agent/task inserts take a // KEY SHARE lock through their runtime FK, so no active agent can appear // after this check and then be silently unbound by the teardown. if _, err := qtx.LockAgentRuntime(r.Context(), rt.ID); err != nil { writeError(w, http.StatusInternalServerError, "failed to lock runtime") return } if _, err := qtx.ListUserAgentsByRuntimeForUpdate(r.Context(), rt.ID); err != nil { writeError(w, http.StatusInternalServerError, "failed to lock runtime dependencies") return } activeAgents, err = qtx.ListActiveAgentsByRuntimeForUpdate(r.Context(), rt.ID) if err != nil { writeError(w, http.StatusInternalServerError, "failed to check runtime dependencies") return } if len(activeAgents) > 0 { writeJSON(w, http.StatusConflict, h.runtimeHasActiveAgentsResponse(activeAgents)) return } // Same teardown the confirmed path runs: unbind the runtime's agents and // their task history, cancel what was still active, remove only the system // agents. There is no active agent here by definition, but archived ones and // their history can still be bound to this runtime. teardown, err := service.TeardownRuntime(r.Context(), qtx, rt.ID, service.RuntimeTeardownOptions{CancelNonTerminalTasks: true}) if err != nil { if errors.Is(err, service.ErrRuntimeNotDrained) { slog.Error("runtime delete aborted: tasks not drained", "runtime_id", uuidToString(rt.ID), "error", err) writeJSON(w, http.StatusConflict, map[string]any{ "error": "the runtime still has tasks in flight; retry in a moment.", "code": "runtime_delete_not_drained", }) return } if errors.Is(err, service.ErrRuntimeWorkspaceMismatch) { slog.Error("runtime delete aborted: agent workspace mismatch", "runtime_id", uuidToString(rt.ID), "error", err) writeJSON(w, http.StatusConflict, map[string]any{ "error": "the runtime has an invalid cross-workspace agent binding.", "code": "runtime_delete_workspace_mismatch", }) return } slog.Error("runtime delete teardown failed", "runtime_id", uuidToString(rt.ID), "error", err) writeError(w, http.StatusInternalServerError, "failed to delete runtime") return } if err := qtx.DeleteAgentRuntime(r.Context(), rt.ID); err != nil { writeError(w, http.StatusInternalServerError, "failed to delete runtime") return } if err := tx.Commit(r.Context()); err != nil { writeError(w, http.StatusInternalServerError, "failed to delete runtime") return } h.NotifyRuntimeGone(uuidToString(rt.ID)) slog.Info("runtime deleted", "runtime_id", uuidToString(rt.ID), "deleted_by", userID, "agents_unbound", len(teardown.UnboundAgents), "tasks_cancelled", len(teardown.CancelledTasks), "autopilots_paused", len(teardown.PausedAutopilots), ) h.publishRuntimeTeardown(r.Context(), teardown, wsID, userID) writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) } // runtimeHasActiveAgentsResponse builds the structured 409 body shared by // DeleteAgentRuntime (light-mode block) and UnbindAgentsAndDeleteRuntime // (cascade-plan-changed). The shape is: // // { // "error": "...", // "code": "runtime_has_active_agents" | "runtime_delete_plan_changed", // "active_agents": [AgentResponse, ...] // } // // Front-end branches on `code`. The caller picks which code to send; this // helper just normalises the agent serialisation and the error string. func (h *Handler) runtimeHasActiveAgentsResponse(agents []db.Agent) map[string]any { resp := make([]AgentResponse, len(agents)) for i, a := range agents { resp[i] = h.agentToResponse(a) } return map[string]any{ "error": "cannot delete runtime: it has active agents bound to it. Reassign them or confirm unbinding them first.", "code": "runtime_has_active_agents", "active_agents": resp, } } // unbindAgentsAndDeleteRuntimeRequest is the wire shape for the confirmed // delete endpoint. expected_active_agent_ids is the snapshot the user just // confirmed in the dialog — the server compares it to the live set inside the // transaction and refuses with runtime_delete_plan_changed if anything moved // between dialog open and confirm. That guarantees the user is approving the // exact agent set that will be unbound, even if a teammate adds or archives an // agent in the same window. // // The compared set is deliberately still "active agents on this runtime": it is // what installed clients send, and widening it would make every older client's // request mismatch and 409 forever, leaving the runtime undeletable. Extra // information for the dialog belongs in read-only fields, not in this set. type unbindAgentsAndDeleteRuntimeRequest struct { ExpectedActiveAgentIDs []string `json:"expected_active_agent_ids"` } // UnbindAgentsAndDeleteRuntime is the confirmed delete entry point: unbind every // user agent bound to the runtime, pause affected Autopilots, cancel active // tasks, detach task history, hard-delete only the system agents, and finally // delete the runtime row — all inside a single transaction so a partial failure // never leaves a runtime half-torn-down. // // Before MUL-5559 this archived those agents and then hard-deleted the rows, // destroying every conversation with them; the dialog said "archive", so what // the user agreed to was not what happened. Now nothing of the user's is // destroyed: the agents survive unbound and need a new runtime to run again. // // Transaction order follows the reference revoke flow in // revokeAndRemoveMember (workspace_revoke.go) so the two paths share the same // race-safety properties: the dispatcher can't claim a task whose runtime is // about to vanish, and post-commit publish events emit the same // task:cancelled → agent:status/autopilot:updated → daemon:register fan-out. // // The expected_active_agent_ids check is the load-bearing piece for the UX: // the front-end snapshots the agent list when the dialog opens and presents // the user a checkbox confirmation; if a teammate adds or archives an agent // while that dialog is open, this endpoint refuses with // runtime_delete_plan_changed and the latest list, so the user never confirms // a stale plan. // // Served at POST /api/runtimes/:id/unbind-agents-and-delete and, for installed // clients, the original /archive-agents-and-delete path. func (h *Handler) UnbindAgentsAndDeleteRuntime(w http.ResponseWriter, r *http.Request) { runtimeID := chi.URLParam(r, "runtimeId") runtimeUUID, ok := parseUUIDOrBadRequest(w, runtimeID, "runtime_id") if !ok { return } var req unbindAgentsAndDeleteRuntimeRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid request body") return } expected, ok := parseExpectedActiveAgentIDs(req.ExpectedActiveAgentIDs) if !ok { writeError(w, http.StatusBadRequest, "expected_active_agent_ids must be a list of valid UUIDs") return } rt, err := h.getAgentRuntime(r.Context(), obsmetrics.RuntimeLookupSourceRuntimeAPI, runtimeUUID) if err != nil { writeError(w, http.StatusNotFound, "runtime not found") return } wsID := uuidToString(rt.WorkspaceID) member, ok := h.requireWorkspaceMember(w, r, wsID, "runtime not found") if !ok { return } if !canEditRuntime(member, rt) { writeError(w, http.StatusForbidden, "you can only delete your own runtimes") return } userID := uuidToString(member.UserID) profile, hasLiveProfile, err := h.runtimeLiveProfile(r.Context(), rt) if err != nil { writeError(w, http.StatusInternalServerError, "failed to check runtime profile") return } if hasLiveProfile { blockers := h.profileInstanceRefusalBlockers(r.Context(), rt.ID) writeJSON(w, http.StatusConflict, profileInstanceDeleteRefusal(rt, profile, blockers)) return } if rt.ProfileID.Valid { slog.Warn("deleting orphaned profile-backed runtime instance via cascade", "runtime_id", uuidToString(rt.ID), "profile_id", uuidToString(rt.ProfileID), "workspace_id", wsID, "deleted_by", userID) } tx, err := h.TxStarter.Begin(r.Context()) if err != nil { writeError(w, http.StatusInternalServerError, "failed to start transaction") return } defer tx.Rollback(r.Context()) qtx := h.Queries.WithTx(tx) // Lock the runtime row first. PostgreSQL's FK validation on // agent.runtime_id requires FOR KEY SHARE on the parent runtime row, // which conflicts with FOR UPDATE — so any concurrent INSERT or // UPDATE that would point a new/moved agent at this runtime now // blocks until our tx finishes. This is the "兜底" lock that keeps // new actives from appearing between our snapshot and our unbind. if _, err := qtx.LockAgentRuntime(r.Context(), rt.ID); err != nil { writeError(w, http.StatusInternalServerError, "failed to lock runtime") return } if _, err := qtx.ListUserAgentsByRuntimeForUpdate(r.Context(), rt.ID); err != nil { writeError(w, http.StatusInternalServerError, "failed to lock runtime dependencies") return } // Re-list active agents inside the transaction, with FOR UPDATE on // each row so a concurrent archive/move of one of those existing // agents also blocks until we commit. Comparing against the expected // set here closes the dialog-open / user-confirm race: even if a // teammate creates or archives an agent on this runtime while the // dialog was open, the user is approving exactly the set the server // is about to unbind. currentActive, err := qtx.ListActiveAgentsByRuntimeForUpdate(r.Context(), rt.ID) if err != nil { writeError(w, http.StatusInternalServerError, "failed to enumerate active agents") return } if !activeAgentSetMatches(currentActive, expected) { // Refuse with the latest snapshot so the front-end can re-render // the dialog and force a fresh user confirmation. Reuses the // shared response helper but overrides the code to a planning // signal so the dialog can distinguish "you opened from a stale // page" from "the plan you confirmed just changed under you". body := h.runtimeHasActiveAgentsResponse(currentActive) body["code"] = "runtime_delete_plan_changed" body["error"] = "the active agent set changed; please review and confirm again." writeJSON(w, http.StatusConflict, body) return } // Single teardown, shared with the light DELETE path: unbind every user // agent (active and archived) plus their task history, cancel what was // running or queued, and hard-delete only the system agents. Nothing the // user configured is destroyed — the agents just need a new runtime. teardown, err := service.TeardownRuntime(r.Context(), qtx, rt.ID, service.RuntimeTeardownOptions{CancelNonTerminalTasks: true}) if err != nil { if errors.Is(err, service.ErrRuntimeNotDrained) { slog.Error("runtime delete aborted: tasks not drained", "runtime_id", uuidToString(rt.ID), "error", err) writeJSON(w, http.StatusConflict, map[string]any{ "error": "the runtime still has tasks in flight; retry in a moment.", "code": "runtime_delete_not_drained", }) return } slog.Error("runtime delete teardown failed", "runtime_id", uuidToString(rt.ID), "error", err) writeError(w, http.StatusInternalServerError, "failed to unbind agents") return } // Finally delete the runtime row itself. if err := qtx.DeleteAgentRuntime(r.Context(), rt.ID); err != nil { writeError(w, http.StatusInternalServerError, "failed to delete runtime") return } if err := tx.Commit(r.Context()); err != nil { writeError(w, http.StatusInternalServerError, "failed to commit transaction") return } h.NotifyRuntimeGone(uuidToString(rt.ID)) h.publishRuntimeTeardown(r.Context(), teardown, wsID, userID) slog.Info("runtime deleted, agents unbound", "runtime_id", uuidToString(rt.ID), "deleted_by", userID, "agents_unbound", len(teardown.UnboundAgents), "tasks_cancelled", len(teardown.CancelledTasks), "autopilots_paused", len(teardown.PausedAutopilots), ) writeJSON(w, http.StatusOK, map[string]any{ "status": "ok", "agents_unbound": len(teardown.UnboundAgents), "tasks_cancelled": len(teardown.CancelledTasks), "autopilots_paused": len(teardown.PausedAutopilots), // Deprecated mirror of agents_unbound: installed clients built against // the archive-and-delete contract read this key. The count is the same // set of agents; they are no longer archived. "agents_archived": len(teardown.UnboundAgents), }) } // parseExpectedActiveAgentIDs validates the cascade endpoint's // expected_active_agent_ids list. nil / empty is allowed (an empty set is a // valid plan: "I confirmed there are no active agents" — the cascade then // just deletes the runtime without unbinding an active agent). Returns ok=false on // any malformed UUID so the handler responds 400 instead of silently // matching a different set. func parseExpectedActiveAgentIDs(raw []string) (map[string]struct{}, bool) { out := make(map[string]struct{}, len(raw)) for _, s := range raw { u, err := util.ParseUUID(s) if err != nil || !u.Valid { return nil, false } out[uuidToString(u)] = struct{}{} } return out, true } // activeAgentSetMatches reports whether the live set of active agents on the // runtime matches the snapshot the front-end confirmed. Order-insensitive // because the front-end may render in any order; size + membership is what // matters for "did the plan change?". func activeAgentSetMatches(current []db.Agent, expected map[string]struct{}) bool { if len(current) != len(expected) { return false } for _, a := range current { if _, ok := expected[uuidToString(a.ID)]; !ok { return false } } return true }