Files
multica/server/pkg/agent/qoder.go
Bohan Jiang 1921d18e14 MUL-7120 fix(agent): stop ACP ghost sessions from wedging a conversation forever (#8155)
Fixes #8116.

A chat conversation could be killed permanently by one early failure: the
daemon pinned a backend session id from a run that died during setup, and
every later message then failed resuming an id the CLI had never persisted.
Fixing the model config did not help — the run died at session/resume, before
any model handling — so the only escape was abandoning the conversation and
losing its context.

Two independent defects had to line up, and both are fixed:

- Eight backends (qoder, kiro, grok, hermes, kimi, traecli, reasonix, dim)
  published a fresh session's id when setup failed before any prompt. The
  existing clear was gated on opts.ResumeSessionID != "", so it only ever
  covered resumes. hermes, grok and dim additionally pinned the id to the task
  row before set_model ran, where FailAgentTask's COALESCE made it
  unretractable; that pin now goes out once setup has succeeded.

- Six backends (qoder, hermes, kimi via session/resume; kiro, traecli, grok via
  session/load) returned a bare failed Result from the resume RPC, so
  shouldRetryWithFreshSession read the absent flag as "checked, not a
  rejection" and never retried. They now share classifyACPResumeFailure, which
  splits cancellation and timeout out first — a user who cancels mid-resume
  must not be reported as a rejected session.

That second defect is why the blast radius is wider than the report: those six
never consulted the classifier on the resume path at all, so any permanent
resume failure wedged the conversation, not just a ghost pointer.

isACPResumeRejected widens the vocabulary at the resume boundary only, where
the RPC's single job is turning an id into a live session. It stays positive
evidence rather than treating every resume error as a rejection: session/resume
carries mcpServers, so an unreachable MCP server, a handshake crash or an
expired credential fail there too and none is cured by starting over. It
carries no error-code gate, deliberately — the reported qodercli frame is
-32602, but that rests on the reporter's capture and gating on it would make
the fix miss its own bug if a runtime picked a code outside the set.

No server-side change. The daemon's in-turn retry already retires the pointer
through retiredSessionID, so existing wedged conversations recover on their
first message after the daemon upgrade, with no backfill.

Behaviour change: a fresh setup-stage failure no longer reports a session id,
so three set_model tests and dim's config-failure resume test asserted the old
contract and were updated. dim's coverage of fail-closed config re-application
on resume is preserved, driven from a run that completed a prompt.

Reviewed by Steve Jobs over three rounds; the predicate was tightened twice as
a result — an over-wide adjacency window that matched auth errors, and a
stop-list for request-shaped complaints.
2026-09-08 15:03:03 +08:00

456 lines
14 KiB
Go

package agent
import (
"context"
"fmt"
"io"
"os/exec"
"strings"
"sync"
"sync/atomic"
"time"
)
// qoderBlockedArgs are flags hardcoded by the daemon that must not be
// overridden by user-configured custom_args. Qoder enters ACP mode via the
// global `--acp` flag; letting users strip or duplicate it would break the
// daemon↔Qoder transport contract. `--yolo` is daemon-owned so headless ACP
// always runs in bypass-permissions mode (see Qoder ACP docs). The legacy `acp`
// subcommand style is blocked so custom_args cannot switch the binary mode.
var qoderBlockedArgs = map[string]blockedArgMode{
"--acp": blockedStandalone,
"acp": blockedStandalone,
"--yolo": blockedStandalone,
}
// qoderBackend implements Backend for both Qoder CLI binaries by spawning
// `<binary> --yolo --acp` and communicating via the ACP (Agent Communication
// Protocol) JSON-RPC 2.0 transport over stdin/stdout.
//
// Qoder CLI uses global flags (`--yolo`, `--acp`), not an `acp` subcommand. We
// reuse hermesClient like Hermes/Kimi/Kiro and mirror their streaming gate so
// history replay flushed during session/setup does not corrupt the streamed
// output or leave the UI stuck on a stale assistant chunk.
type qoderBackend struct {
cfg Config
defaultExecutable string
}
func qoderDefaultBinary(providerType string) string {
if providerType == "qoderclicn" {
return "qoderclicn"
}
return "qodercli"
}
var qoderReaderDrainGrace = 2 * time.Second
type qoderMessageStream struct {
ch chan Message
mu sync.Mutex
closed bool
}
func newQoderMessageStream(size int) *qoderMessageStream {
return &qoderMessageStream{ch: make(chan Message, size)}
}
func (s *qoderMessageStream) send(msg Message) {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return
}
trySend(s.ch, msg)
}
func (s *qoderMessageStream) close() {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return
}
s.closed = true
close(s.ch)
}
func (b *qoderBackend) Execute(ctx context.Context, prompt string, opts ExecOptions) (*Session, error) {
execPath := b.cfg.ExecutablePath
if execPath == "" {
execPath = b.defaultExecutable
}
if _, err := exec.LookPath(execPath); err != nil {
return nil, fmt.Errorf("%s executable not found at %q: %w", b.defaultExecutable, execPath, err)
}
// Translate the agent's mcp_config (Claude-style object of objects) into
// the array shape ACP session/new and session/resume expect. Reuse the
// shared converter so remote MCP `headers` (e.g. Authorization) survive as
// [{name, value}] and output is deterministic. Fail closed on malformed
// JSON so the launch surfaces the real error instead of silently dropping
// every MCP server.
mcpServers, err := buildACPMcpServers(opts.McpConfig, b.cfg.Logger)
if err != nil {
return nil, fmt.Errorf("qoder: invalid mcp_config: %w", err)
}
timeout := opts.Timeout
runCtx, cancel := runContext(ctx, timeout)
qoderArgs := append(
[]string{"--yolo", "--acp"},
filterCustomArgs(opts.CustomArgs, qoderBlockedArgs, b.cfg.Logger)...,
)
cmd := b.cfg.commandAt(execPath).exec(runCtx, qoderArgs...)
hideAgentWindow(cmd)
b.cfg.logAgentCommand(cmd, newAgentCommandLogArgs(qoderArgs))
if opts.Cwd != "" {
cmd.Dir = opts.Cwd
}
cmd.Env = buildEnv(b.cfg.Env)
stdout, err := cmd.StdoutPipe()
if err != nil {
cancel()
return nil, fmt.Errorf("qoder stdout pipe: %w", err)
}
stdin, err := cmd.StdinPipe()
if err != nil {
cancel()
return nil, fmt.Errorf("qoder stdin pipe: %w", err)
}
// StderrPipe + an explicit copier give us a join point (`stderrDone`)
// that fires before the failure-promotion decision; see the matching
// comment in hermes.go for why the io.MultiWriter form races with
// stopReason=end_turn under load (a terminal provider error can land
// after we've already read finalOutput and reported "completed").
providerErr := newACPProviderErrorSniffer("qoder")
stderr, err := cmd.StderrPipe()
if err != nil {
cancel()
return nil, fmt.Errorf("qoder stderr pipe: %w", err)
}
if err := startOwnedProcessTree(cmd, b.cfg.Logger); err != nil {
cancel()
return nil, fmt.Errorf("start qoder: %w", err)
}
stderrSink := io.MultiWriter(newLogWriter(b.cfg.Logger, "[qoder:stderr] "), providerErr)
stderrDone := make(chan struct{})
go func() {
defer close(stderrDone)
_, _ = io.Copy(stderrSink, stderr)
}()
b.cfg.Logger.Info("qoder acp started", "pid", cmd.Process.Pid, "cwd", opts.Cwd)
msgStream := newQoderMessageStream(256)
resCh := make(chan Result, 1)
// Qoder emits interim narration and the final answer as the same ACP
// AgentMessageChunk type; the tracker keeps only the post-tool-call block
// for Result.Output while retaining the full text for error detection.
var deliverable acpDeliverableTracker
var streamingCurrentTurn atomic.Bool
promptDone := make(chan hermesPromptResult, 1)
activity := make(chan struct{}, 1)
c := &hermesClient{
cfg: b.cfg,
stdin: stdin,
pending: make(map[int]*pendingRPC),
pendingTools: make(map[string]*pendingToolCall),
acceptNotification: func(string) bool {
return streamingCurrentTurn.Load()
},
onActivity: func() {
select {
case activity <- struct{}{}:
default:
}
},
onMessage: func(msg Message) {
if !streamingCurrentTurn.Load() {
return
}
if msg.Type == MessageToolUse {
msg.Tool = kimiToolNameFromTitle(msg.Tool)
}
deliverable.observe(msg)
msgStream.send(msg)
},
onPromptDone: func(result hermesPromptResult) {
if !streamingCurrentTurn.Load() {
return
}
select {
case promptDone <- result:
default:
}
},
}
readerDone := make(chan struct{})
go func() {
defer close(readerDone)
scanner := newAgentStreamScanner(stdout)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
c.handleLine(line)
}
c.closeAllPending(fmt.Errorf("qoder process exited"))
}()
go func() {
defer cancel()
defer msgStream.close()
defer close(resCh)
defer func() {
stdin.Close()
_ = cmd.Wait()
releaseProcessGroup(cmd)
}()
startTime := time.Now()
finalStatus := "completed"
var finalError string
var sessionID string
// Set when the ACP runtime refuses the session we asked to
// resume. Only that is curable by starting a fresh session, so
// handshake/network failures below must leave it false.
var resumeRejected bool
effectiveModel := strings.TrimSpace(opts.Model)
initResult, err := c.request(runCtx, "initialize", map[string]any{
"protocolVersion": 1,
"clientInfo": map[string]any{
"name": "multica-agent-sdk",
"version": "0.2.0",
},
"clientCapabilities": map[string]any{},
})
if err != nil {
finalStatus = "failed"
finalError = fmt.Sprintf("qoder initialize failed: %v", err)
resCh <- Result{Status: finalStatus, Error: finalError, DurationMs: time.Since(startTime).Milliseconds()}
return
}
// Drop MCP entries whose remote transport the runtime didn't
// advertise. See the matching comment in hermes.go for why
// unconditionally sending http/sse to a stdio-only ACP runtime
// tanks the whole session/new.
mcpServers = filterACPMcpServersByCapability(mcpServers, extractACPMcpCapabilities(initResult), "qoder", b.cfg)
cwd := opts.Cwd
if cwd == "" {
cwd = "."
}
if opts.ResumeSessionID != "" {
result, err := c.request(runCtx, "session/resume", map[string]any{
"cwd": cwd,
"sessionId": opts.ResumeSessionID,
"mcpServers": mcpServers,
})
if err != nil {
// A runtime that refuses the recorded id has to say so here:
// without ResumeRejected the daemon reads the bare failure as
// "checked, not a rejection", keeps the pointer and replays the
// same dead session on every later turn (GH #8116).
finalStatus, finalError, resumeRejected = classifyACPResumeFailure(
runCtx, "qoder", "session/resume", err, timeout, b.cfg.Logger)
resCh <- Result{Status: finalStatus, Error: finalError, DurationMs: time.Since(startTime).Milliseconds(), ResumeRejected: resumeRejected}
return
}
var changed bool
sessionID, changed = resolveResumedSessionID(opts.ResumeSessionID, result)
if changed {
b.cfg.Logger.Warn("agent returned a different session id on resume — original was likely lost; continuing with the new id",
"backend", "qoder",
"requested", opts.ResumeSessionID,
"actual", sessionID,
)
}
if effectiveModel == "" {
effectiveModel = extractACPCurrentModelID(result)
}
} else {
result, err := c.request(runCtx, "session/new", map[string]any{
"cwd": cwd,
"mcpServers": mcpServers,
})
if err != nil {
finalStatus = "failed"
finalError = fmt.Sprintf("qoder session/new failed: %v", err)
resCh <- Result{Status: finalStatus, Error: finalError, DurationMs: time.Since(startTime).Milliseconds()}
return
}
sessionID = extractACPSessionID(result)
if sessionID == "" {
finalStatus = "failed"
finalError = "qoder session/new returned no session ID"
resCh <- Result{Status: finalStatus, Error: finalError, DurationMs: time.Since(startTime).Milliseconds()}
return
}
if effectiveModel == "" {
effectiveModel = extractACPCurrentModelID(result)
}
}
c.sessionID = sessionID
b.cfg.Logger.Info("qoder session created", "session_id", sessionID)
if opts.Model != "" {
if _, err := c.request(runCtx, "session/set_model", map[string]any{
"sessionId": sessionID,
"modelId": opts.Model,
}); err != nil {
b.cfg.Logger.Warn("qoder set_session_model failed", "error", err, "requested_model", opts.Model)
finalStatus = "failed"
finalError = fmt.Sprintf("qoder could not switch to model %q: %v", opts.Model, err)
if setupFailureWithholdsSessionID(opts) {
sessionID = ""
} else if isACPSessionNotFound(err) {
// On a resumed session with a model override, the dead
// session surfaces here instead of at session/prompt.
// Clear the id so the daemon's resume-failure fallback
// retries fresh and stores the replacement session.
b.cfg.Logger.Warn("resumed session not found at set_model time; clearing session id so the daemon retries fresh",
"backend", "qoder",
"session_id", sessionID,
)
sessionID = ""
resumeRejected = true
}
resCh <- Result{
Status: finalStatus,
Error: finalError,
DurationMs: time.Since(startTime).Milliseconds(),
SessionID: sessionID,
ResumeRejected: resumeRejected,
}
return
}
b.cfg.Logger.Info("qoder session model set", "model", opts.Model)
}
userText := prompt
if opts.SystemPrompt != "" {
userText = opts.SystemPrompt + "\n\n---\n\n" + prompt
}
// Flip just before session/prompt so history replay flushed during setup
// is dropped; every notification for this turn is processed afterward.
streamingCurrentTurn.Store(true)
_, err = c.request(runCtx, "session/prompt", map[string]any{
"sessionId": sessionID,
"prompt": []map[string]any{
{"type": "text", "text": userText},
},
})
if err != nil {
if runCtx.Err() == context.DeadlineExceeded {
finalStatus = "timeout"
finalError = fmt.Sprintf("qoder timed out after %s", timeout)
} else if runCtx.Err() == context.Canceled {
finalStatus = "aborted"
finalError = "execution cancelled"
} else {
finalStatus = "failed"
finalError = fmt.Sprintf("qoder session/prompt failed: %v", err)
if opts.ResumeSessionID != "" && isACPSessionNotFound(err) {
// The runtime may echo the requested id from
// session/resume and only reject it at prompt time.
// Empty SessionID lets the daemon retry with a fresh
// session instead of pinning future runs to the stale id.
b.cfg.Logger.Warn("resumed session not found at prompt time; clearing session id so the daemon retries fresh",
"backend", "qoder",
"session_id", sessionID,
)
sessionID = ""
resumeRejected = true
}
}
} else {
select {
case pr := <-promptDone:
if pr.stopReason == "cancelled" {
finalStatus = "aborted"
finalError = "qoder cancelled the prompt"
}
c.mergeUsage(pr.usage)
default:
}
waitForACPNotificationQuiescence(runCtx, activity, readerDone, acpNotificationQuietTime, qoderReaderDrainGrace)
}
duration := time.Since(startTime)
b.cfg.Logger.Info("qoder finished", "pid", cmd.Process.Pid, "status", finalStatus, "duration", duration.Round(time.Millisecond).String())
stdin.Close()
cancel()
// Qoder ACP may keep the process — and the stdout/stderr pipes — open
// after session/prompt returns (it can leave a child holding the
// inherited fds). The prompt response is already terminal, so bound the
// drain: wait for both the stdout reader and the stderr copier, but no
// longer than the grace window. CommandContext cancellation above is
// what actually tears the process down. Draining stderr here is what
// makes the provider-error promotion below see a terminal marker before
// we decide the final status.
drainCtx, drainCancel := context.WithTimeout(context.Background(), qoderReaderDrainGrace)
select {
case <-readerDone:
case <-drainCtx.Done():
}
select {
case <-stderrDone:
case <-drainCtx.Done():
}
drainCancel()
// The stdout reader may still run after the grace window. Flip the
// stream gate before this goroutine's defer closes msgStream; if a
// late reader already passed the gate, qoderMessageStream serializes
// send and close so the late send is dropped instead of panicking.
streamingCurrentTurn.Store(false)
finalOutput, providerErrorOutput := deliverable.result()
// Promote completed→failed when stderr or the agent text stream show a
// terminal upstream-LLM failure (HTTP 4xx / rate-limit / expired token).
// Mirrors hermes/kimi/kiro; without it a run that exhausts retries still
// reports "completed" because session/prompt ends with stopReason=end_turn
// even though qodercli wrote a terminal error to stderr.
finalStatus, finalError = promoteACPResultOnProviderError(finalStatus, finalError, providerErrorOutput, providerErr)
u := c.accumulatedUsage()
var usageMap map[string]TokenUsage
if acpUsagePresent(u) {
model := effectiveModel
if model == "" {
model = "unknown"
}
usageMap = map[string]TokenUsage{model: u}
}
resCh <- Result{
Status: finalStatus,
Output: finalOutput,
Error: finalError,
DurationMs: duration.Milliseconds(),
SessionID: sessionID,
ResumeRejected: resumeRejected,
Usage: usageMap,
}
}()
return &Session{Messages: msgStream.ch, Result: resCh}, nil
}