package taskfailure import "testing" // TestClassifyEmptyAndWhitespace pins the empty/whitespace contract. // Daemon callers should never hand us empty error text — but if they // do, returning the catchall is safer than panicking. func TestClassifyEmptyAndWhitespace(t *testing.T) { t.Parallel() cases := []string{"", " ", "\n\t \n"} for _, in := range cases { if got := Classify(in); got != ReasonAgentUnknown { t.Errorf("Classify(%q) = %q, want %q", in, got, ReasonAgentUnknown) } } } // TestClassifyRules walks every classifier rule with a real-world // sample taken from MUL-1949's db-boy production analysis (top error // prefixes from `agent_task_queue.error` over a 7-day window). When // MUL-1949's SQL grows a new rule, add a fixture here so the in-flight // classifier and the offline backfill stay in lock-step. // // One test case per rule is the minimum bar; rules with notable // boundary conditions (e.g. the 5xx regex) get a dedicated subtest // further down. func TestClassifyRules(t *testing.T) { t.Parallel() cases := []struct { name string in string want Reason }{ // 1. Context overflow. {"context length exceeded", "Error: context length exceeded for model gpt-4", ReasonAgentContextOverflow}, {"context_length_exceeded code", `{"error":{"code":"context_length_exceeded"}}`, ReasonAgentContextOverflow}, {"maximum context", "Maximum context window of 200000 tokens has been exceeded", ReasonAgentContextOverflow}, {"prompt is too long", "API Error: prompt is too long: 250000 tokens > 200000 maximum", ReasonAgentContextOverflow}, {"context size has been exceeded", "context size has been exceeded; consider /compact", ReasonAgentContextOverflow}, {"token limit", "Hit the token limit for this conversation", ReasonAgentContextOverflow}, // GH #6360, verbatim from Claude Code 2.1.x. The turn is not // rejected with a 400 — the response comes back with stop_reason // "model_context_window_exceeded" and the CLI prints this line. {"claude code context window limit", "API Error: The model has reached its context window limit.", ReasonAgentContextOverflow}, {"raw stop reason", `{"stop_reason":"model_context_window_exceeded"}`, ReasonAgentContextOverflow}, // 2. Missing config. {"missing env var", "Missing environment variable: `MIFY_API_KEY`.", ReasonAgentMissingConfig}, {"missing api_key", "Failed to authenticate: missing api_key in config", ReasonAgentMissingConfig}, {"api key required", "An api key is required to use this provider", ReasonAgentMissingConfig}, {"no llm provider configured", "no llm provider configured; set OPENAI_API_KEY", ReasonAgentMissingConfig}, {"no provider configured", "no provider configured for runtime", ReasonAgentMissingConfig}, // 3. Provider auth / access. {"401", "API Error: 401 Unauthorized", ReasonAgentProviderAuthOrAccess}, {"403", "API Error: 403 Forbidden", ReasonAgentProviderAuthOrAccess}, {"unauthorized text", "Request unauthorized for this organization", ReasonAgentProviderAuthOrAccess}, {"login required", "login required: please run /login", ReasonAgentProviderAuthOrAccess}, {"not logged in", "Not logged in · Please run /login", ReasonAgentProviderAuthOrAccess}, {"please login again", "Session expired, please login again", ReasonAgentProviderAuthOrAccess}, {"refresh token", "refresh token has expired", ReasonAgentProviderAuthOrAccess}, {"invalid api key", "Invalid API key provided", ReasonAgentProviderAuthOrAccess}, {"access token", "access token has been revoked", ReasonAgentProviderAuthOrAccess}, {"subscription access", "Your organization has disabled Claude subscription access for Claude Code", ReasonAgentProviderAuthOrAccess}, {"does not have access", "Your account does not have access to this model", ReasonAgentProviderAuthOrAccess}, {"may not have access", "you may not have access to claude-3-opus", ReasonAgentProviderAuthOrAccess}, // 4. Provider quota / billing. {"402", "API Error: 402 Payment Required", ReasonAgentProviderQuotaLimit}, {"insufficient_balance", `{"error":{"code":"insufficient_balance"}}`, ReasonAgentProviderQuotaLimit}, {"balance is too low", "balance is too low to make this request", ReasonAgentProviderQuotaLimit}, {"monthly usage limit", "You've hit your org's monthly usage limit", ReasonAgentProviderQuotaLimit}, {"usage limit", "Account exceeded the daily usage limit", ReasonAgentProviderQuotaLimit}, {"hit your limit ascii", "you've hit your limit; upgrade to continue", ReasonAgentProviderQuotaLimit}, {"hit your limit curly", "you\u2019ve hit your limit", ReasonAgentProviderQuotaLimit}, {"credits", "Your account has 0 credits remaining", ReasonAgentProviderQuotaLimit}, {"quota", "quota exceeded for project foo", ReasonAgentProviderQuotaLimit}, // 5. Capacity / rate limit. {"429", "API Error: 429 Too Many Requests", ReasonAgentProviderCapacityOrRateLimit}, {"529", "Server overloaded: HTTP 529", ReasonAgentProviderCapacityOrRateLimit}, {"rate limit", "rate limit exceeded for tier 3", ReasonAgentProviderCapacityOrRateLimit}, {"overloaded", "overloaded_error: please retry", ReasonAgentProviderCapacityOrRateLimit}, {"no capacity available", "no capacity available; try again later", ReasonAgentProviderCapacityOrRateLimit}, // 6. Provider 5xx / server error. {"server had an error", "the server had an error processing your request", ReasonAgentProviderServerError}, {"provider returned error", "provider returned error: malformed response", ReasonAgentProviderServerError}, {"internal error", "An internal error occurred while serving the request", ReasonAgentProviderServerError}, {"500 with delimiter", "API Error: 500 Internal Server Error", ReasonAgentProviderServerError}, {"503 anywhere", "got HTTP 503 from provider", ReasonAgentProviderServerError}, {"503 at start", "503 service degraded", ReasonAgentProviderServerError}, {"504 at end", "upstream returned 504", ReasonAgentProviderServerError}, {"service unavailable", "service unavailable, retry later", ReasonAgentProviderServerError}, {"bad gateway", "Bad Gateway: upstream rejected", ReasonAgentProviderServerError}, // 7. Provider network. {"stream disconnected", "stream disconnected before completion", ReasonAgentProviderNetwork}, {"connection closed mid-response", "API Error: Connection closed mid-response. The response above may be incomplete.", ReasonAgentProviderNetwork}, {"connection closed with exit status wins over process failure", "claude exited with error: exit status 1\nAPI Error: Connection closed mid-response.", ReasonAgentProviderNetwork}, {"error sending request", "error sending request for url (https://api.example.com/v1)", ReasonAgentProviderNetwork}, {"unable to connect", "unable to connect to provider", ReasonAgentProviderNetwork}, {"dial tcp", "dial tcp 1.2.3.4:443: connect: connection refused", ReasonAgentProviderNetwork}, {"connection refused alone", "connection refused", ReasonAgentProviderNetwork}, {"connectionrefused single", "ConnectionRefused", ReasonAgentProviderNetwork}, {"dns", "dns lookup failed", ReasonAgentProviderNetwork}, {"i/o timeout", "read tcp 1.2.3.4:443: i/o timeout", ReasonAgentProviderNetwork}, // MUL-5370: every Go-side context deadline used to land in // agent_error.unknown, which is not on the retry allowlist — a // transient stall became a terminal failure with a useless label. {"context deadline exceeded", "context deadline exceeded", ReasonAgentProviderNetwork}, {"wrapped context deadline", `Post "https://api.example.com/v1": context deadline exceeded`, ReasonAgentProviderNetwork}, {"http client timeout", `Get "https://api.example.com": net/http: request canceled (Client.Timeout exceeded while awaiting headers)`, ReasonAgentProviderNetwork}, // #6522: all three OpenCode terminal-signal guard failures are silent // provider stream cuts. The two "terminal signal" variants used to hit // rule 13 by accident (the word "signal") and the empty-step one fell // to agent_error.unknown; neither bucket is retryable. {"opencode step open at EOF", "opencode stream ended without a terminal signal (step still open at EOF)", ReasonAgentProviderNetwork}, {"opencode continuation never started", "opencode stream ended without a terminal signal (last step required a continuation that never started)", ReasonAgentProviderNetwork}, {"opencode empty final step", "opencode stream ended on an empty step (no text, no tool call, no reported usage) — the provider produced nothing", ReasonAgentProviderNetwork}, {"opencode empty step with process exit appended", "opencode stream ended on an empty step (no text, no tool call, no reported usage) — the provider produced nothing; opencode exited with error: exit status 1", ReasonAgentProviderNetwork}, // BHD-135: Pi's OpenAI-compatible SDK wording for a dropped LiteLLM // call. Bare strings, then the same strings glued to "exit status 1" // after pi-print-clean-exit forces a non-zero wrap-up. {"pi connection error", "Connection error.", ReasonAgentProviderNetwork}, {"pi connection error with exit status wins over process failure", "Connection error.; pi exited with error: exit status 1", ReasonAgentProviderNetwork}, {"pi request timed out", "Request timed out.", ReasonAgentProviderNetwork}, {"pi request timed out with exit status wins over process failure", "Request timed out.; pi exited with error: exit status 1", ReasonAgentProviderNetwork}, {"omp connection error with exit status wins over process failure", "Connection error.; omp exited with error: exit status 1", ReasonAgentProviderNetwork}, {"codearts step open at EOF", "codearts stream ended without a terminal signal (step still open at EOF)", ReasonAgentProviderNetwork}, // 8. Model not found / unavailable. {"model not found", "Error: model claude-3-opus-99 not found", ReasonAgentModelNotFoundOrUnavailable}, {"model not found phrase", "the model was not found in this account", ReasonAgentModelNotFoundOrUnavailable}, {"unknown model", "unknown model 'foo-1.0'", ReasonAgentModelNotFoundOrUnavailable}, {"selected model", "the selected model is no longer supported", ReasonAgentModelNotFoundOrUnavailable}, {"http 404", "HTTP 404: model endpoint not registered", ReasonAgentModelNotFoundOrUnavailable}, {"404 page not found", "404 page not found", ReasonAgentModelNotFoundOrUnavailable}, // 9. Empty / unparseable output. {"returned empty output", "openclaw returned empty output", ReasonAgentEmptyOrUnparseableOutput}, {"returned no parseable output", "kimi returned no parseable output", ReasonAgentEmptyOrUnparseableOutput}, // 10. Agent timeout. {"timed out after", "claude timed out after 2h0m0s", ReasonAgentTimeout}, // 11. Runtime missing executable. {"executable not found", "executable not found in $PATH", ReasonAgentRuntimeMissingExecutable}, {"exec format error", "start claude: fork/exec /usr/lib/node_modules/@anthropic-ai/claude-code/bin/claude.exe: exec format error", ReasonAgentRuntimeMissingExecutable}, // 12. Runtime version unsupported. {"below the minimum supported version", "claude CLI 0.1.0 is below the minimum supported version 0.5.0", ReasonAgentRuntimeVersionUnsupported}, {"requires a newer version", "this protocol requires a newer version of the runtime", ReasonAgentRuntimeVersionUnsupported}, // 13. Process failure. {"exit status", "agent exit status 137", ReasonAgentProcessFailure}, {"signal", "agent terminated by signal: killed", ReasonAgentProcessFailure}, {"panic", "panic: runtime error: invalid memory address", ReasonAgentProcessFailure}, {"sigsegv", "fatal error: SIGSEGV", ReasonAgentProcessFailure}, {"process exited", "process exited with status 1", ReasonAgentProcessFailure}, {"pipe has been ended", "the pipe has been ended", ReasonAgentProcessFailure}, {"file already closed", "write |1: file already closed", ReasonAgentProcessFailure}, {"initialize failed", "initialize failed: backend not ready", ReasonAgentProcessFailure}, // 14. Catchall. {"unrecognized", "the agent gave up for reasons unknown", ReasonAgentUnknown}, {"sentence with no marker", "Hello world.", ReasonAgentUnknown}, // Pi's two short provider messages must not become broad substring // matches: local tool and MCP failures are deterministic and retrying // them only repeats the same failure. {"local tool connection error is not provider network", "local tool connection error while opening its database", ReasonAgentUnknown}, {"mcp request timeout is not provider network", "MCP server request timed out while loading configuration", ReasonAgentUnknown}, {"local connection error with exit remains process failure", "MCP server connection error; agent exited with error: exit status 1", ReasonAgentProcessFailure}, // 15. Digit-boundary regression: 3-digit HTTP status codes must NOT // match when embedded in a longer number. Before the fix these // landed in provider auth/quota/capacity buckets, masking hard // process failures under a provider reason and polluting failure // observability. {"402 embedded not quota", "agent consumed 402913 tokens before crashing", ReasonAgentUnknown}, {"529 embedded not capacity", "request latency was 15290ms; then it panicked: signal killed", ReasonAgentProcessFailure}, {"403 embedded not auth", "processed 4030 items, then exit status 1", ReasonAgentProcessFailure}, {"401 embedded not auth", "job 24019 finished, process exited with status 2", ReasonAgentProcessFailure}, {"429 embedded not capacity", "seq 14290 unknown outcome", ReasonAgentUnknown}, // Genuine status codes with a boundary still classify correctly. {"402 boundary still quota", "API Error: 402 Payment Required", ReasonAgentProviderQuotaLimit}, {"403 boundary still auth", "HTTP 403 Forbidden", ReasonAgentProviderAuthOrAccess}, {"429 boundary still capacity", "got 429 from provider", ReasonAgentProviderCapacityOrRateLimit}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { if got := Classify(c.in); got != c.want { t.Fatalf("Classify(%q) = %q, want %q", c.in, got, c.want) } }) } } // TestClassifyOrderingPriorities pins the rule precedence between // overlapping rules. These cases caught regressions during MUL-2946 PR1 // review: the SQL CASE ordering matters and a naive Go switch could // silently route them differently. func TestClassifyOrderingPriorities(t *testing.T) { t.Parallel() cases := []struct { name string in string want Reason }{ // "token limit" mentions both "context-ish" tokens AND // "limit". The context_overflow rule must win because the // quota-limit rule's "limit" trigger would otherwise swallow // it. {"token limit beats quota", "you exceeded the token limit", ReasonAgentContextOverflow}, // 401 + missing api_key: the missing_config rule runs before // auth precisely so we don't classify a config error as an // auth rejection. {"missing api key beats 401", "missing api_key for openai (401 returned downstream)", ReasonAgentMissingConfig}, // Some Anthropic-compatible providers return 403 for a transient // concurrency rejection. The semantic witness must beat generic auth and // token/context matching even when the CLI prefixes both misleadingly. {"403 concurrent request limit beats auth", "Failed to authenticate. API Error: 403 You've reached your concurrent request limit. Please wait for your ongoing requests to finish and try again.", ReasonAgentProviderCapacityOrRateLimit}, {"access token concurrent request limit beats context", "Failed to refresh access token. API Error: 403 You've reached your concurrent request limit.", ReasonAgentProviderCapacityOrRateLimit}, // Kimi Code reports an exhausted subscription window as HTTP 403. The // usage-limit witness must beat the bare 403 auth rule, while an // unrelated 403 stays an auth failure. {"403 usage limit beats auth", "API Error: 403 You've reached your 5-hour usage limit", ReasonAgentProviderQuotaLimit}, {"403 usage limit with prefix beats auth", "Failed to authenticate. API Error: 403 {\"error\":{\"message\":\"You've reached your 5-hour usage limit\"}}", ReasonAgentProviderQuotaLimit}, {"access token 403 usage limit beats context", "Failed to refresh access token. API Error: 403 You've reached your 5-hour usage limit", ReasonAgentProviderQuotaLimit}, {"plain 403 stays auth", "API Error: 403 Forbidden", ReasonAgentProviderAuthOrAccess}, // Both "429" and "rate limit" present — should still land in // the capacity bucket, not the quota bucket. {"429 rate limit", "API Error: 429 rate limit reached", ReasonAgentProviderCapacityOrRateLimit}, // "exit status" co-occurring with a stronger upstream marker // — the upstream classification should win because the // process_failure rule is checked last. {"exit status with 401 upstream", "exit status 1: API Error: 401 Unauthorized", ReasonAgentProviderAuthOrAccess}, {"windows codex process start", "start codex: fork/exec C:\\invalid\\codex.exe: %1 is not a valid Win32 application.", ReasonAgentProcessFailure}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { if got := Classify(c.in); got != c.want { t.Errorf("Classify(%q) = %q, want %q", c.in, got, c.want) } }) } } func TestNormalizeDaemonReasonUpgradesConcurrentRequestLimit(t *testing.T) { t.Parallel() const raw = "Failed to refresh access token. API Error: 403 You've reached your concurrent request limit." for _, reason := range []string{ string(ReasonAgentContextOverflow), string(ReasonAgentProviderAuthOrAccess), string(ReasonAgentUnknown), "agent_error", } { if got := NormalizeDaemonReason(reason, raw); got != ReasonAgentProviderCapacityOrRateLimit { t.Errorf("NormalizeDaemonReason(%q, concurrent request rejection) = %q, want %q", reason, got, ReasonAgentProviderCapacityOrRateLimit) } } if got := NormalizeDaemonReason(string(ReasonAgentProviderAuthOrAccess), "API Error: 403 Forbidden"); got != ReasonAgentProviderAuthOrAccess { t.Errorf("plain 403 auth rejection changed to %q", got) } if got := NormalizeDaemonReason(string(ReasonAgentContextOverflow), "you exceeded the token limit"); got != ReasonAgentContextOverflow { t.Errorf("ordinary token overflow changed to %q", got) } } func TestNormalizeDaemonReasonUpgradesUsageLimit403(t *testing.T) { t.Parallel() for _, raw := range []string{ "API Error: 403 You've reached your 5-hour usage limit", "Failed to refresh access token. API Error: 403 You've reached your 5-hour usage limit", } { for _, reason := range []string{ string(ReasonAgentContextOverflow), string(ReasonAgentProviderAuthOrAccess), string(ReasonAgentUnknown), "agent_error", } { if got := NormalizeDaemonReason(reason, raw); got != ReasonAgentProviderQuotaLimit { t.Errorf("NormalizeDaemonReason(%q, %q) = %q, want %q", reason, raw, got, ReasonAgentProviderQuotaLimit) } } } if got := NormalizeDaemonReason(string(ReasonAgentProviderAuthOrAccess), "API Error: 403 Forbidden"); got != ReasonAgentProviderAuthOrAccess { t.Errorf("plain 403 auth rejection changed to %q", got) } } // TestClassify5xxRegex pins the boundary behavior of the 5xx HTTP // status detector. The SQL classifier uses an anchored regex // `(^|[^0-9])5[0-9][0-9]([^0-9]|$)`; this Go classifier mirrors it via // providerHTTP5xxRe. Without the anchors, "1500ms" and "1.5.0" would // be misclassified as a server error. func TestClassify5xxRegex(t *testing.T) { t.Parallel() hits := []string{ "503", " 504 ", "got 502 from upstream", "upstream returned 599\n", } for _, in := range hits { if got := Classify(in); got != ReasonAgentProviderServerError { t.Errorf("Classify(%q) = %q, want %q", in, got, ReasonAgentProviderServerError) } } misses := []string{ "1500ms latency observed", "version 1.5.0 unsupported", "5000 tokens generated", "agent slept for 1500 seconds", } for _, in := range misses { if got := Classify(in); got == ReasonAgentProviderServerError { t.Errorf("Classify(%q) = %q, want NOT provider_server_error", in, got) } } } // TestClassifyAlwaysReturnsAgentSide guarantees Classify never returns // a platform-side reason. Platform-side reasons originate from // sweepers / scheduler / poisoned classifier paths that don't pass // through Classify; the in-flight classifier's job is exclusively to // pick among the 14 agent_error.* sub-reasons (or fall back to // ReasonAgentUnknown). A future change that accidentally returned, // say, ReasonRuntimeOffline from Classify would break Prometheus // label semantics — pin it here. func TestClassifyAlwaysReturnsAgentSide(t *testing.T) { t.Parallel() samples := []string{ "", "random text", "401 Unauthorized", "context length exceeded", "503 internal server error", "timed out after 2h0m0s", "exit status 1", } for _, s := range samples { got := Classify(s) if !got.IsAgentError() { t.Errorf("Classify(%q) = %q, must be agent_error.* (in-flight classifier never returns platform-side reasons)", s, got) } } } // TestNormalizeDaemonReason is the mixed-version regression for MUL-5370. // // The daemon-side fix labels a failed skill-bundle download structurally, but // installed daemons upgrade on their own cadence. An un-upgraded daemon reports // a NON-EMPTY catchall, which FailTask's "classify only when empty" guard // deliberately preserves — so without this normalisation the fix would reach // only hosts that happened to update: no auto-retry (the catchall is not on the // retry allowlist) and generic chat copy, on exactly the hosts most likely to // be hitting the bug. func TestNormalizeDaemonReason(t *testing.T) { t.Parallel() const legacyErr = "resolve skill bundles: context deadline exceeded" cases := []struct { name string reason string raw string want Reason }{ { name: "old daemon catchall is upgraded", reason: string(ReasonAgentUnknown), raw: legacyErr, want: ReasonSkillBundleUnavailable, }, { // A daemon new enough to classify the deadline as network, but not // new enough to know the failure was a skill bundle. name: "old daemon network guess is upgraded", reason: string(ReasonAgentProviderNetwork), raw: legacyErr, want: ReasonSkillBundleUnavailable, }, { name: "pre-MUL-1949 coarse reason is upgraded", reason: "agent_error", raw: legacyErr, want: ReasonSkillBundleUnavailable, }, { name: "leading whitespace does not defeat the witness", reason: string(ReasonAgentUnknown), raw: " " + legacyErr, want: ReasonSkillBundleUnavailable, }, { // A current daemon already sends the right reason and a different // error string; nothing to do. name: "current daemon reason passes through", reason: string(ReasonSkillBundleUnavailable), raw: `skill bundle unavailable: skill "x" (id=1, 10 bytes) after 30s: context deadline exceeded`, want: ReasonSkillBundleUnavailable, }, { // The witness is a prefix, not a substring: an agent that merely // mentions the old wrapper in its output must not be relabelled. name: "prefix only, not substring", reason: string(ReasonAgentUnknown), raw: "the agent said it could not resolve skill bundles: and then gave up", want: ReasonAgentUnknown, }, { name: "unrelated reason with the witness is left alone", reason: string(ReasonAgentProviderAuthOrAccess), raw: legacyErr, want: ReasonAgentProviderAuthOrAccess, }, { name: "catchall without the witness is left alone", reason: string(ReasonAgentUnknown), raw: "claude exited with error: exit status 1", want: ReasonAgentUnknown, }, { name: "empty reason is left alone for the caller's classifier", reason: "", raw: legacyErr, want: Reason(""), }, // --- GH #6360: response-side context overflow. An un-upgraded daemon // classifies the wordings below as the catchall, which is on no resume // blacklist — so without this the over-full session stays pinned and // every later comment on the issue replays the same overflow. { name: "old daemon catchall on the claude code wording is upgraded", reason: string(ReasonAgentUnknown), raw: "API Error: The model has reached its context window limit.", want: ReasonAgentContextOverflow, }, { name: "old daemon catchall on the raw stop reason is upgraded", reason: string(ReasonAgentUnknown), raw: `{"stop_reason":"model_context_window_exceeded"}`, want: ReasonAgentContextOverflow, }, { name: "pre-MUL-1949 coarse reason on the overflow is upgraded", reason: "agent_error", raw: "API Error: The model has reached its context window limit.", want: ReasonAgentContextOverflow, }, { // The witness is matched case-insensitively, like Classify's. name: "witness casing does not defeat the upgrade", reason: string(ReasonAgentUnknown), raw: "API ERROR: THE MODEL HAS REACHED ITS CONTEXT WINDOW LIMIT.", want: ReasonAgentContextOverflow, }, { // A current daemon already classified it; nothing to do. name: "current daemon overflow reason passes through", reason: string(ReasonAgentContextOverflow), raw: "API Error: The model has reached its context window limit.", want: ReasonAgentContextOverflow, }, { // A refined reason means the old daemon matched an earlier rule on // this same text. That is a stronger statement about what ended the // run than the witness is, so it is left alone. name: "refined reason with the overflow witness is left alone", reason: string(ReasonAgentProcessFailure), raw: "claude exited with error: exit status 1: The model has reached its context window limit.", want: ReasonAgentProcessFailure, }, { name: "catchall without an overflow witness is left alone", reason: string(ReasonAgentUnknown), raw: "API Error: the model is overloaded", want: ReasonAgentUnknown, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { if got := NormalizeDaemonReason(tc.reason, tc.raw); got != tc.want { t.Errorf("NormalizeDaemonReason(%q, %q) = %q, want %q", tc.reason, tc.raw, got, tc.want) } }) } } // TestNormalizeDaemonReason_UpgradedReasonIsRetryable pins the property that // actually matters to the user: the normalised reason must be one the server // retries. If someone later drops skill_bundle_unavailable from // internal/service/task.go's retryableReasons, the label survives but the // self-healing this PR is for silently disappears. func TestNormalizeDaemonReason_UpgradedReasonIsPlatformSide(t *testing.T) { t.Parallel() got := NormalizeDaemonReason(string(ReasonAgentUnknown), "resolve skill bundles: context deadline exceeded") if got.IsAgentError() { t.Errorf("%q must be platform-side: the agent process never started", got) } } // TestProviderUnconfigured pins the predicate the daemon uses to decide whether // a failure is worth annotating with the HERMES_HOME it actually read (GH // #6872). The wrapped fixture is the shape the error really arrives in — the // runtime's message nested inside the ACP transport's JSON-RPC framing — so a // future refactor to equality matching fails here rather than in production. func TestProviderUnconfigured(t *testing.T) { t.Parallel() cases := []struct { name string in string want bool }{ { "wrapped acp error as the daemon receives it", `hermes session/new failed: session/new: Internal error (code=-32603, ` + `data={"details":"No LLM provider configured. Run ` + "`hermes model`" + ` to select a provider."})`, true, }, {"bare runtime message", "No LLM provider configured. Run `hermes model` to select a provider.", true}, {"lowercased by a forwarder", "error: no llm provider configured", true}, // A credential that exists but was rejected is a different failure with // a different fix; the annotation would misdirect the user. {"rejected credential", "401 unauthorized: invalid api key", false}, // The provider WAS resolved — it just could not be reached. {"provider unreachable", "connection refused: https://example.invalid/v1", false}, {"empty", "", false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { t.Parallel() if got := ProviderUnconfigured(tc.in); got != tc.want { t.Errorf("ProviderUnconfigured(%q) = %v, want %v", tc.in, got, tc.want) } }) } } // TestProviderUnconfiguredAgreesWithClassify keeps the shared phrase honest: // the predicate and rule 2 read the same const, so anything the predicate // recognises must still be filed as a config problem. If these two ever // disagree, the daemon would be annotating failures the platform files as // something else entirely. func TestProviderUnconfiguredAgreesWithClassify(t *testing.T) { t.Parallel() const errText = "No LLM provider configured. Run `hermes model` to select a provider." if !ProviderUnconfigured(errText) { t.Fatal("precondition: the predicate should match its own phrase") } if got := Classify(errText); got != ReasonAgentMissingConfig { t.Errorf("Classify(%q) = %q, want %q", errText, got, ReasonAgentMissingConfig) } } // TestNormalizeDaemonReasonUpgradesOpenclawCLITimeout covers the mixed-version // window for #7112. A daemon that predates the timeout sentinel classifies the // failure from its text and lands on agent_error.provider_network, which is // worse than merely imprecise: that reason is on the auto-retry allowlist, so // every attempt re-pays the same 8-11s stall and fails identically, and the // chat bubble tells the user to check a network that was never involved. // Recognising the wire shape fixes both the moment the server deploys. func TestNormalizeDaemonReasonUpgradesOpenclawCLITimeout(t *testing.T) { t.Parallel() const rawError = "prepare execution environment: execenv: prepare openclaw config: " + "locate openclaw active config: openclaw config file: context deadline exceeded " + "(process: signal: killed)" for _, legacy := range []string{ string(ReasonAgentProviderNetwork), string(ReasonAgentUnknown), "agent_error", } { if got := NormalizeDaemonReason(legacy, rawError); got != ReasonRuntimeCLITimeout { t.Errorf("NormalizeDaemonReason(%q, openclaw timeout) = %q, want %q", legacy, got, ReasonRuntimeCLITimeout) } } // A current daemon already reports the precise reason; normalization must // leave it alone. if got := NormalizeDaemonReason(string(ReasonRuntimeCLITimeout), rawError); got != ReasonRuntimeCLITimeout { t.Errorf("NormalizeDaemonReason(runtime_cli_timeout) = %q, want it preserved", got) } // Both witnesses are required. A real provider-side deadline, and an // openclaw prep failure that is not a timeout, must keep their own // classification — the global "deadline exceeded" rule still belongs to // genuine network stalls. unrelated := map[string]struct { reason string rawError string want Reason }{ "provider deadline stays provider_network": { reason: string(ReasonAgentProviderNetwork), rawError: "API Error: context deadline exceeded while streaming from provider", want: ReasonAgentProviderNetwork, }, "openclaw prep failure without a timeout is untouched": { reason: string(ReasonAgentUnknown), rawError: "execenv: prepare openclaw config: read openclaw agents.list: exit status 1", want: ReasonAgentUnknown, }, } for name, tc := range unrelated { if got := NormalizeDaemonReason(tc.reason, tc.rawError); got != tc.want { t.Errorf("%s: NormalizeDaemonReason = %q, want %q", name, got, tc.want) } } } // TestClassifyKeepsDeadlineExceededAsProviderNetwork pins the rule the fix // deliberately did NOT touch. Local runtime CLI timeouts are recognised // structurally, upstream of Classify; the text rule still has to serve real // provider stalls, which are transient and retryable. func TestClassifyKeepsDeadlineExceededAsProviderNetwork(t *testing.T) { t.Parallel() if got := Classify("post to provider: context deadline exceeded"); got != ReasonAgentProviderNetwork { t.Errorf("Classify(provider deadline) = %q, want %q", got, ReasonAgentProviderNetwork) } }