mirror of
https://gh-proxy.org/https://github.com/klinsmaya/multica.git
synced 2026-11-08 09:26:50 +08:00
小七
ebc7c4696b
The failure classifier's status-code regexes guarded against longer numbers only: `(^|[^0-9])(429|529)([^0-9]|$)` and friends. Workspace and task IDs are hex, so a path like ~/multica_workspaces_x/<slug>-0000c429f4b/<task>/worktree embeds "429" between letters, which passed the digit-only guard — and the capacity rule runs before the process-failure and unknown rules. Every failed task quoting such a path was labeled agent_error.provider_capacity_or_rate_limit, sending users to check their provider accounts for a 429 no provider ever returned. The same gap misfiled paths containing hex "401"/"402"/"403"/"5xx" into the auth, quota and server-error buckets. Widen all five boundary guards (capacity, auth, quota, 5xx, and the 403 usage-limit regex shared with NormalizeDaemonReason) from `[^0-9]` to `[^0-9a-z]`. The classifier lowercases its input before matching, so letters are the only remaining embedded context to exclude. Genuine status codes (space/punctuation-delimited, or bare) still match. The MUL-1949 offline backfill SQL these patterns mirror is not in this repo; regenerating it needs the same widened boundary to stay in lock-step (noted in the pattern comments). Fixes #9146 Generated-by: GLM-5.3-Flash (ZCode)
670 lines
34 KiB
Go
670 lines
34 KiB
Go
package taskfailure
|
|
|
|
import "testing"
|
|
|
|
// TestClassifyEmptyAndWhitespace pins the empty/whitespace contract.
|
|
// Daemon callers should never hand us empty error text — but if they
|
|
// do, returning the catchall is safer than panicking.
|
|
func TestClassifyEmptyAndWhitespace(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
cases := []string{"", " ", "\n\t \n"}
|
|
for _, in := range cases {
|
|
if got := Classify(in); got != ReasonAgentUnknown {
|
|
t.Errorf("Classify(%q) = %q, want %q", in, got, ReasonAgentUnknown)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestClassifyRules walks every classifier rule with a real-world
|
|
// sample taken from MUL-1949's db-boy production analysis (top error
|
|
// prefixes from `agent_task_queue.error` over a 7-day window). When
|
|
// MUL-1949's SQL grows a new rule, add a fixture here so the in-flight
|
|
// classifier and the offline backfill stay in lock-step.
|
|
//
|
|
// One test case per rule is the minimum bar; rules with notable
|
|
// boundary conditions (e.g. the 5xx regex) get a dedicated subtest
|
|
// further down.
|
|
func TestClassifyRules(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
cases := []struct {
|
|
name string
|
|
in string
|
|
want Reason
|
|
}{
|
|
// 1. Context overflow.
|
|
{"context length exceeded", "Error: context length exceeded for model gpt-4", ReasonAgentContextOverflow},
|
|
{"context_length_exceeded code", `{"error":{"code":"context_length_exceeded"}}`, ReasonAgentContextOverflow},
|
|
{"maximum context", "Maximum context window of 200000 tokens has been exceeded", ReasonAgentContextOverflow},
|
|
{"prompt is too long", "API Error: prompt is too long: 250000 tokens > 200000 maximum", ReasonAgentContextOverflow},
|
|
{"context size has been exceeded", "context size has been exceeded; consider /compact", ReasonAgentContextOverflow},
|
|
{"token limit", "Hit the token limit for this conversation", ReasonAgentContextOverflow},
|
|
// GH #6360, verbatim from Claude Code 2.1.x. The turn is not
|
|
// rejected with a 400 — the response comes back with stop_reason
|
|
// "model_context_window_exceeded" and the CLI prints this line.
|
|
{"claude code context window limit", "API Error: The model has reached its context window limit.", ReasonAgentContextOverflow},
|
|
{"raw stop reason", `{"stop_reason":"model_context_window_exceeded"}`, ReasonAgentContextOverflow},
|
|
|
|
// 2. Missing config.
|
|
{"missing env var", "Missing environment variable: `MIFY_API_KEY`.", ReasonAgentMissingConfig},
|
|
{"missing api_key", "Failed to authenticate: missing api_key in config", ReasonAgentMissingConfig},
|
|
{"api key required", "An api key is required to use this provider", ReasonAgentMissingConfig},
|
|
{"no llm provider configured", "no llm provider configured; set OPENAI_API_KEY", ReasonAgentMissingConfig},
|
|
{"no provider configured", "no provider configured for runtime", ReasonAgentMissingConfig},
|
|
|
|
// 3. Provider auth / access.
|
|
{"401", "API Error: 401 Unauthorized", ReasonAgentProviderAuthOrAccess},
|
|
{"403", "API Error: 403 Forbidden", ReasonAgentProviderAuthOrAccess},
|
|
{"unauthorized text", "Request unauthorized for this organization", ReasonAgentProviderAuthOrAccess},
|
|
{"login required", "login required: please run /login", ReasonAgentProviderAuthOrAccess},
|
|
{"not logged in", "Not logged in · Please run /login", ReasonAgentProviderAuthOrAccess},
|
|
{"please login again", "Session expired, please login again", ReasonAgentProviderAuthOrAccess},
|
|
{"refresh token", "refresh token has expired", ReasonAgentProviderAuthOrAccess},
|
|
{"invalid api key", "Invalid API key provided", ReasonAgentProviderAuthOrAccess},
|
|
{"access token", "access token has been revoked", ReasonAgentProviderAuthOrAccess},
|
|
{"subscription access", "Your organization has disabled Claude subscription access for Claude Code", ReasonAgentProviderAuthOrAccess},
|
|
{"does not have access", "Your account does not have access to this model", ReasonAgentProviderAuthOrAccess},
|
|
{"may not have access", "you may not have access to claude-3-opus", ReasonAgentProviderAuthOrAccess},
|
|
|
|
// 4. Provider quota / billing.
|
|
{"402", "API Error: 402 Payment Required", ReasonAgentProviderQuotaLimit},
|
|
{"insufficient_balance", `{"error":{"code":"insufficient_balance"}}`, ReasonAgentProviderQuotaLimit},
|
|
{"balance is too low", "balance is too low to make this request", ReasonAgentProviderQuotaLimit},
|
|
{"monthly usage limit", "You've hit your org's monthly usage limit", ReasonAgentProviderQuotaLimit},
|
|
{"usage limit", "Account exceeded the daily usage limit", ReasonAgentProviderQuotaLimit},
|
|
{"hit your limit ascii", "you've hit your limit; upgrade to continue", ReasonAgentProviderQuotaLimit},
|
|
{"hit your limit curly", "you\u2019ve hit your limit", ReasonAgentProviderQuotaLimit},
|
|
{"credits", "Your account has 0 credits remaining", ReasonAgentProviderQuotaLimit},
|
|
{"quota", "quota exceeded for project foo", ReasonAgentProviderQuotaLimit},
|
|
|
|
// 5. Capacity / rate limit.
|
|
{"429", "API Error: 429 Too Many Requests", ReasonAgentProviderCapacityOrRateLimit},
|
|
{"529", "Server overloaded: HTTP 529", ReasonAgentProviderCapacityOrRateLimit},
|
|
{"rate limit", "rate limit exceeded for tier 3", ReasonAgentProviderCapacityOrRateLimit},
|
|
{"overloaded", "overloaded_error: please retry", ReasonAgentProviderCapacityOrRateLimit},
|
|
{"no capacity available", "no capacity available; try again later", ReasonAgentProviderCapacityOrRateLimit},
|
|
|
|
// 6. Provider 5xx / server error.
|
|
{"server had an error", "the server had an error processing your request", ReasonAgentProviderServerError},
|
|
{"provider returned error", "provider returned error: malformed response", ReasonAgentProviderServerError},
|
|
{"internal error", "An internal error occurred while serving the request", ReasonAgentProviderServerError},
|
|
{"500 with delimiter", "API Error: 500 Internal Server Error", ReasonAgentProviderServerError},
|
|
{"503 anywhere", "got HTTP 503 from provider", ReasonAgentProviderServerError},
|
|
{"503 at start", "503 service degraded", ReasonAgentProviderServerError},
|
|
{"504 at end", "upstream returned 504", ReasonAgentProviderServerError},
|
|
{"service unavailable", "service unavailable, retry later", ReasonAgentProviderServerError},
|
|
{"bad gateway", "Bad Gateway: upstream rejected", ReasonAgentProviderServerError},
|
|
|
|
// 7. Provider network.
|
|
{"stream disconnected", "stream disconnected before completion", ReasonAgentProviderNetwork},
|
|
{"connection closed mid-response", "API Error: Connection closed mid-response. The response above may be incomplete.", ReasonAgentProviderNetwork},
|
|
{"connection closed with exit status wins over process failure", "claude exited with error: exit status 1\nAPI Error: Connection closed mid-response.", ReasonAgentProviderNetwork},
|
|
{"error sending request", "error sending request for url (https://api.example.com/v1)", ReasonAgentProviderNetwork},
|
|
{"unable to connect", "unable to connect to provider", ReasonAgentProviderNetwork},
|
|
{"dial tcp", "dial tcp 1.2.3.4:443: connect: connection refused", ReasonAgentProviderNetwork},
|
|
{"connection refused alone", "connection refused", ReasonAgentProviderNetwork},
|
|
{"connectionrefused single", "ConnectionRefused", ReasonAgentProviderNetwork},
|
|
{"dns", "dns lookup failed", ReasonAgentProviderNetwork},
|
|
{"i/o timeout", "read tcp 1.2.3.4:443: i/o timeout", ReasonAgentProviderNetwork},
|
|
// MUL-5370: every Go-side context deadline used to land in
|
|
// agent_error.unknown, which is not on the retry allowlist — a
|
|
// transient stall became a terminal failure with a useless label.
|
|
{"context deadline exceeded", "context deadline exceeded", ReasonAgentProviderNetwork},
|
|
{"wrapped context deadline", `Post "https://api.example.com/v1": context deadline exceeded`, ReasonAgentProviderNetwork},
|
|
{"http client timeout", `Get "https://api.example.com": net/http: request canceled (Client.Timeout exceeded while awaiting headers)`, ReasonAgentProviderNetwork},
|
|
// #6522: all three OpenCode terminal-signal guard failures are silent
|
|
// provider stream cuts. The two "terminal signal" variants used to hit
|
|
// rule 13 by accident (the word "signal") and the empty-step one fell
|
|
// to agent_error.unknown; neither bucket is retryable.
|
|
{"opencode step open at EOF", "opencode stream ended without a terminal signal (step still open at EOF)", ReasonAgentProviderNetwork},
|
|
{"opencode continuation never started", "opencode stream ended without a terminal signal (last step required a continuation that never started)", ReasonAgentProviderNetwork},
|
|
{"opencode empty final step", "opencode stream ended on an empty step (no text, no tool call, no reported usage) — the provider produced nothing", ReasonAgentProviderNetwork},
|
|
{"opencode empty step with process exit appended", "opencode stream ended on an empty step (no text, no tool call, no reported usage) — the provider produced nothing; opencode exited with error: exit status 1", ReasonAgentProviderNetwork},
|
|
// BHD-135: Pi's OpenAI-compatible SDK wording for a dropped LiteLLM
|
|
// call. Bare strings, then the same strings glued to "exit status 1"
|
|
// after pi-print-clean-exit forces a non-zero wrap-up.
|
|
{"pi connection error", "Connection error.", ReasonAgentProviderNetwork},
|
|
{"pi connection error with exit status wins over process failure", "Connection error.; pi exited with error: exit status 1", ReasonAgentProviderNetwork},
|
|
{"pi request timed out", "Request timed out.", ReasonAgentProviderNetwork},
|
|
{"pi request timed out with exit status wins over process failure", "Request timed out.; pi exited with error: exit status 1", ReasonAgentProviderNetwork},
|
|
{"omp connection error with exit status wins over process failure", "Connection error.; omp exited with error: exit status 1", ReasonAgentProviderNetwork},
|
|
{"codearts step open at EOF", "codearts stream ended without a terminal signal (step still open at EOF)", ReasonAgentProviderNetwork},
|
|
|
|
// 8. Model not found / unavailable.
|
|
{"model not found", "Error: model claude-3-opus-99 not found", ReasonAgentModelNotFoundOrUnavailable},
|
|
{"model not found phrase", "the model was not found in this account", ReasonAgentModelNotFoundOrUnavailable},
|
|
{"unknown model", "unknown model 'foo-1.0'", ReasonAgentModelNotFoundOrUnavailable},
|
|
{"selected model", "the selected model is no longer supported", ReasonAgentModelNotFoundOrUnavailable},
|
|
{"http 404", "HTTP 404: model endpoint not registered", ReasonAgentModelNotFoundOrUnavailable},
|
|
{"404 page not found", "404 page not found", ReasonAgentModelNotFoundOrUnavailable},
|
|
|
|
// 9. Empty / unparseable output.
|
|
{"returned empty output", "openclaw returned empty output", ReasonAgentEmptyOrUnparseableOutput},
|
|
{"returned no parseable output", "kimi returned no parseable output", ReasonAgentEmptyOrUnparseableOutput},
|
|
|
|
// 10. Agent timeout.
|
|
{"timed out after", "claude timed out after 2h0m0s", ReasonAgentTimeout},
|
|
|
|
// 11. Runtime missing executable.
|
|
{"executable not found", "executable not found in $PATH", ReasonAgentRuntimeMissingExecutable},
|
|
{"exec format error", "start claude: fork/exec /usr/lib/node_modules/@anthropic-ai/claude-code/bin/claude.exe: exec format error", ReasonAgentRuntimeMissingExecutable},
|
|
|
|
// 12. Runtime version unsupported.
|
|
{"below the minimum supported version", "claude CLI 0.1.0 is below the minimum supported version 0.5.0", ReasonAgentRuntimeVersionUnsupported},
|
|
{"requires a newer version", "this protocol requires a newer version of the runtime", ReasonAgentRuntimeVersionUnsupported},
|
|
|
|
// 13. Process failure.
|
|
{"exit status", "agent exit status 137", ReasonAgentProcessFailure},
|
|
{"signal", "agent terminated by signal: killed", ReasonAgentProcessFailure},
|
|
{"panic", "panic: runtime error: invalid memory address", ReasonAgentProcessFailure},
|
|
{"sigsegv", "fatal error: SIGSEGV", ReasonAgentProcessFailure},
|
|
{"process exited", "process exited with status 1", ReasonAgentProcessFailure},
|
|
{"pipe has been ended", "the pipe has been ended", ReasonAgentProcessFailure},
|
|
{"file already closed", "write |1: file already closed", ReasonAgentProcessFailure},
|
|
{"initialize failed", "initialize failed: backend not ready", ReasonAgentProcessFailure},
|
|
|
|
// 14. Catchall.
|
|
{"unrecognized", "the agent gave up for reasons unknown", ReasonAgentUnknown},
|
|
{"sentence with no marker", "Hello world.", ReasonAgentUnknown},
|
|
// Pi's two short provider messages must not become broad substring
|
|
// matches: local tool and MCP failures are deterministic and retrying
|
|
// them only repeats the same failure.
|
|
{"local tool connection error is not provider network", "local tool connection error while opening its database", ReasonAgentUnknown},
|
|
{"mcp request timeout is not provider network", "MCP server request timed out while loading configuration", ReasonAgentUnknown},
|
|
{"local connection error with exit remains process failure", "MCP server connection error; agent exited with error: exit status 1", ReasonAgentProcessFailure},
|
|
|
|
// 15. Digit-boundary regression: 3-digit HTTP status codes must NOT
|
|
// match when embedded in a longer number. Before the fix these
|
|
// landed in provider auth/quota/capacity buckets, masking hard
|
|
// process failures under a provider reason and polluting failure
|
|
// observability.
|
|
{"402 embedded not quota", "agent consumed 402913 tokens before crashing", ReasonAgentUnknown},
|
|
{"529 embedded not capacity", "request latency was 15290ms; then it panicked: signal killed", ReasonAgentProcessFailure},
|
|
{"403 embedded not auth", "processed 4030 items, then exit status 1", ReasonAgentProcessFailure},
|
|
{"401 embedded not auth", "job 24019 finished, process exited with status 2", ReasonAgentProcessFailure},
|
|
{"429 embedded not capacity", "seq 14290 unknown outcome", ReasonAgentUnknown},
|
|
// Genuine status codes with a boundary still classify correctly.
|
|
{"402 boundary still quota", "API Error: 402 Payment Required", ReasonAgentProviderQuotaLimit},
|
|
{"403 boundary still auth", "HTTP 403 Forbidden", ReasonAgentProviderAuthOrAccess},
|
|
{"429 boundary still capacity", "got 429 from provider", ReasonAgentProviderCapacityOrRateLimit},
|
|
|
|
// 16. Letter-boundary regression (#9146): hex IDs in workspace and
|
|
// task paths embed digit runs like "429" between letters
|
|
// ("...c429f4b..."), which the digit-only boundary guard let
|
|
// through — every failure quoting such a path landed in the
|
|
// provider capacity/auth/quota/server-error buckets and sent
|
|
// users to check their provider accounts for a 429 no provider
|
|
// ever returned. Letters must count as embedded context, same
|
|
// as digits.
|
|
{"429 in hex id not capacity", "local_directory worktree: refusing to record branch agent/a/b: " +
|
|
"the delivered commit a806f863 no longer contains e8574f2b, the commit this turn started from; " +
|
|
"the task worktree is preserved at /Users/u/multica_workspaces_x/ws-0000c429f4b/b-1/worktree", ReasonAgentUnknown},
|
|
{"403 in hex id not auth", "refusing to write /users/u/ws-c403f1b9/worktree", ReasonAgentUnknown},
|
|
{"401 in hex id not auth", "artifact stored at /tmp/run-00be401d7c/output.log", ReasonAgentUnknown},
|
|
{"402 in hex id not quota", "cache dir /var/cache/ws-0000d402aa9f removed", ReasonAgentUnknown},
|
|
{"5xx in hex id not server error", "log at /ws/branch-c503ba4e/worktree", ReasonAgentUnknown},
|
|
// A model name carrying "500" before a letter is an identifier,
|
|
// not a status code — same widening, same rationale.
|
|
{"model name 500 suffix not server error", "artifact model-pack qwen3-500b saved", ReasonAgentUnknown},
|
|
// Widening must not dull real detection: a genuine status code
|
|
// next to hex IDs still classifies.
|
|
{"429 still capacity alongside hex ids", "commit e8574f2b retry failed: HTTP 429 too many requests", ReasonAgentProviderCapacityOrRateLimit},
|
|
{"503 still server error alongside letters", "model qwen3-500b request died with HTTP 503", ReasonAgentProviderServerError},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
if got := Classify(c.in); got != c.want {
|
|
t.Fatalf("Classify(%q) = %q, want %q", c.in, got, c.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestClassifyOrderingPriorities pins the rule precedence between
|
|
// overlapping rules. These cases caught regressions during MUL-2946 PR1
|
|
// review: the SQL CASE ordering matters and a naive Go switch could
|
|
// silently route them differently.
|
|
func TestClassifyOrderingPriorities(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
cases := []struct {
|
|
name string
|
|
in string
|
|
want Reason
|
|
}{
|
|
// "token limit" mentions both "context-ish" tokens AND
|
|
// "limit". The context_overflow rule must win because the
|
|
// quota-limit rule's "limit" trigger would otherwise swallow
|
|
// it.
|
|
{"token limit beats quota", "you exceeded the token limit", ReasonAgentContextOverflow},
|
|
|
|
// 401 + missing api_key: the missing_config rule runs before
|
|
// auth precisely so we don't classify a config error as an
|
|
// auth rejection.
|
|
{"missing api key beats 401", "missing api_key for openai (401 returned downstream)", ReasonAgentMissingConfig},
|
|
|
|
// Some Anthropic-compatible providers return 403 for a transient
|
|
// concurrency rejection. The semantic witness must beat generic auth and
|
|
// token/context matching even when the CLI prefixes both misleadingly.
|
|
{"403 concurrent request limit beats auth", "Failed to authenticate. API Error: 403 You've reached your concurrent request limit. Please wait for your ongoing requests to finish and try again.", ReasonAgentProviderCapacityOrRateLimit},
|
|
{"access token concurrent request limit beats context", "Failed to refresh access token. API Error: 403 You've reached your concurrent request limit.", ReasonAgentProviderCapacityOrRateLimit},
|
|
|
|
// Kimi Code reports an exhausted subscription window as HTTP 403. The
|
|
// usage-limit witness must beat the bare 403 auth rule, while an
|
|
// unrelated 403 stays an auth failure.
|
|
{"403 usage limit beats auth", "API Error: 403 You've reached your 5-hour usage limit", ReasonAgentProviderQuotaLimit},
|
|
{"403 usage limit with prefix beats auth", "Failed to authenticate. API Error: 403 {\"error\":{\"message\":\"You've reached your 5-hour usage limit\"}}", ReasonAgentProviderQuotaLimit},
|
|
{"access token 403 usage limit beats context", "Failed to refresh access token. API Error: 403 You've reached your 5-hour usage limit", ReasonAgentProviderQuotaLimit},
|
|
{"plain 403 stays auth", "API Error: 403 Forbidden", ReasonAgentProviderAuthOrAccess},
|
|
|
|
// Both "429" and "rate limit" present — should still land in
|
|
// the capacity bucket, not the quota bucket.
|
|
{"429 rate limit", "API Error: 429 rate limit reached", ReasonAgentProviderCapacityOrRateLimit},
|
|
|
|
// "exit status" co-occurring with a stronger upstream marker
|
|
// — the upstream classification should win because the
|
|
// process_failure rule is checked last.
|
|
{"exit status with 401 upstream", "exit status 1: API Error: 401 Unauthorized", ReasonAgentProviderAuthOrAccess},
|
|
{"windows codex process start", "start codex: fork/exec C:\\invalid\\codex.exe: %1 is not a valid Win32 application.", ReasonAgentProcessFailure},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
if got := Classify(c.in); got != c.want {
|
|
t.Errorf("Classify(%q) = %q, want %q", c.in, got, c.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNormalizeDaemonReasonUpgradesConcurrentRequestLimit(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const raw = "Failed to refresh access token. API Error: 403 You've reached your concurrent request limit."
|
|
|
|
for _, reason := range []string{
|
|
string(ReasonAgentContextOverflow),
|
|
string(ReasonAgentProviderAuthOrAccess),
|
|
string(ReasonAgentUnknown),
|
|
"agent_error",
|
|
} {
|
|
if got := NormalizeDaemonReason(reason, raw); got != ReasonAgentProviderCapacityOrRateLimit {
|
|
t.Errorf("NormalizeDaemonReason(%q, concurrent request rejection) = %q, want %q", reason, got, ReasonAgentProviderCapacityOrRateLimit)
|
|
}
|
|
}
|
|
|
|
if got := NormalizeDaemonReason(string(ReasonAgentProviderAuthOrAccess), "API Error: 403 Forbidden"); got != ReasonAgentProviderAuthOrAccess {
|
|
t.Errorf("plain 403 auth rejection changed to %q", got)
|
|
}
|
|
if got := NormalizeDaemonReason(string(ReasonAgentContextOverflow), "you exceeded the token limit"); got != ReasonAgentContextOverflow {
|
|
t.Errorf("ordinary token overflow changed to %q", got)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeDaemonReasonUpgradesUsageLimit403(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
for _, raw := range []string{
|
|
"API Error: 403 You've reached your 5-hour usage limit",
|
|
"Failed to refresh access token. API Error: 403 You've reached your 5-hour usage limit",
|
|
} {
|
|
for _, reason := range []string{
|
|
string(ReasonAgentContextOverflow),
|
|
string(ReasonAgentProviderAuthOrAccess),
|
|
string(ReasonAgentUnknown),
|
|
"agent_error",
|
|
} {
|
|
if got := NormalizeDaemonReason(reason, raw); got != ReasonAgentProviderQuotaLimit {
|
|
t.Errorf("NormalizeDaemonReason(%q, %q) = %q, want %q", reason, raw, got, ReasonAgentProviderQuotaLimit)
|
|
}
|
|
}
|
|
}
|
|
|
|
if got := NormalizeDaemonReason(string(ReasonAgentProviderAuthOrAccess), "API Error: 403 Forbidden"); got != ReasonAgentProviderAuthOrAccess {
|
|
t.Errorf("plain 403 auth rejection changed to %q", got)
|
|
}
|
|
}
|
|
|
|
// TestClassify5xxRegex pins the boundary behavior of the 5xx HTTP
|
|
// status detector. The SQL classifier uses an anchored regex
|
|
// `(^|[^0-9])5[0-9][0-9]([^0-9]|$)`; this Go classifier mirrors it via
|
|
// providerHTTP5xxRe. Without the anchors, "1500ms" and "1.5.0" would
|
|
// be misclassified as a server error.
|
|
func TestClassify5xxRegex(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hits := []string{
|
|
"503",
|
|
" 504 ",
|
|
"got 502 from upstream",
|
|
"upstream returned 599\n",
|
|
}
|
|
for _, in := range hits {
|
|
if got := Classify(in); got != ReasonAgentProviderServerError {
|
|
t.Errorf("Classify(%q) = %q, want %q", in, got, ReasonAgentProviderServerError)
|
|
}
|
|
}
|
|
|
|
misses := []string{
|
|
"1500ms latency observed",
|
|
"version 1.5.0 unsupported",
|
|
"5000 tokens generated",
|
|
"agent slept for 1500 seconds",
|
|
}
|
|
for _, in := range misses {
|
|
if got := Classify(in); got == ReasonAgentProviderServerError {
|
|
t.Errorf("Classify(%q) = %q, want NOT provider_server_error", in, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestClassifyAlwaysReturnsAgentSide guarantees Classify never returns
|
|
// a platform-side reason. Platform-side reasons originate from
|
|
// sweepers / scheduler / poisoned classifier paths that don't pass
|
|
// through Classify; the in-flight classifier's job is exclusively to
|
|
// pick among the 14 agent_error.* sub-reasons (or fall back to
|
|
// ReasonAgentUnknown). A future change that accidentally returned,
|
|
// say, ReasonRuntimeOffline from Classify would break Prometheus
|
|
// label semantics — pin it here.
|
|
func TestClassifyAlwaysReturnsAgentSide(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
samples := []string{
|
|
"",
|
|
"random text",
|
|
"401 Unauthorized",
|
|
"context length exceeded",
|
|
"503 internal server error",
|
|
"timed out after 2h0m0s",
|
|
"exit status 1",
|
|
}
|
|
for _, s := range samples {
|
|
got := Classify(s)
|
|
if !got.IsAgentError() {
|
|
t.Errorf("Classify(%q) = %q, must be agent_error.* (in-flight classifier never returns platform-side reasons)", s, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestNormalizeDaemonReason is the mixed-version regression for MUL-5370.
|
|
//
|
|
// The daemon-side fix labels a failed skill-bundle download structurally, but
|
|
// installed daemons upgrade on their own cadence. An un-upgraded daemon reports
|
|
// a NON-EMPTY catchall, which FailTask's "classify only when empty" guard
|
|
// deliberately preserves — so without this normalisation the fix would reach
|
|
// only hosts that happened to update: no auto-retry (the catchall is not on the
|
|
// retry allowlist) and generic chat copy, on exactly the hosts most likely to
|
|
// be hitting the bug.
|
|
func TestNormalizeDaemonReason(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const legacyErr = "resolve skill bundles: context deadline exceeded"
|
|
|
|
cases := []struct {
|
|
name string
|
|
reason string
|
|
raw string
|
|
want Reason
|
|
}{
|
|
{
|
|
name: "old daemon catchall is upgraded",
|
|
reason: string(ReasonAgentUnknown),
|
|
raw: legacyErr,
|
|
want: ReasonSkillBundleUnavailable,
|
|
},
|
|
{
|
|
// A daemon new enough to classify the deadline as network, but not
|
|
// new enough to know the failure was a skill bundle.
|
|
name: "old daemon network guess is upgraded",
|
|
reason: string(ReasonAgentProviderNetwork),
|
|
raw: legacyErr,
|
|
want: ReasonSkillBundleUnavailable,
|
|
},
|
|
{
|
|
name: "pre-MUL-1949 coarse reason is upgraded",
|
|
reason: "agent_error",
|
|
raw: legacyErr,
|
|
want: ReasonSkillBundleUnavailable,
|
|
},
|
|
{
|
|
name: "leading whitespace does not defeat the witness",
|
|
reason: string(ReasonAgentUnknown),
|
|
raw: " " + legacyErr,
|
|
want: ReasonSkillBundleUnavailable,
|
|
},
|
|
{
|
|
// A current daemon already sends the right reason and a different
|
|
// error string; nothing to do.
|
|
name: "current daemon reason passes through",
|
|
reason: string(ReasonSkillBundleUnavailable),
|
|
raw: `skill bundle unavailable: skill "x" (id=1, 10 bytes) after 30s: context deadline exceeded`,
|
|
want: ReasonSkillBundleUnavailable,
|
|
},
|
|
{
|
|
// The witness is a prefix, not a substring: an agent that merely
|
|
// mentions the old wrapper in its output must not be relabelled.
|
|
name: "prefix only, not substring",
|
|
reason: string(ReasonAgentUnknown),
|
|
raw: "the agent said it could not resolve skill bundles: and then gave up",
|
|
want: ReasonAgentUnknown,
|
|
},
|
|
{
|
|
name: "unrelated reason with the witness is left alone",
|
|
reason: string(ReasonAgentProviderAuthOrAccess),
|
|
raw: legacyErr,
|
|
want: ReasonAgentProviderAuthOrAccess,
|
|
},
|
|
{
|
|
name: "catchall without the witness is left alone",
|
|
reason: string(ReasonAgentUnknown),
|
|
raw: "claude exited with error: exit status 1",
|
|
want: ReasonAgentUnknown,
|
|
},
|
|
{
|
|
name: "empty reason is left alone for the caller's classifier",
|
|
reason: "",
|
|
raw: legacyErr,
|
|
want: Reason(""),
|
|
},
|
|
|
|
// --- GH #6360: response-side context overflow. An un-upgraded daemon
|
|
// classifies the wordings below as the catchall, which is on no resume
|
|
// blacklist — so without this the over-full session stays pinned and
|
|
// every later comment on the issue replays the same overflow.
|
|
{
|
|
name: "old daemon catchall on the claude code wording is upgraded",
|
|
reason: string(ReasonAgentUnknown),
|
|
raw: "API Error: The model has reached its context window limit.",
|
|
want: ReasonAgentContextOverflow,
|
|
},
|
|
{
|
|
name: "old daemon catchall on the raw stop reason is upgraded",
|
|
reason: string(ReasonAgentUnknown),
|
|
raw: `{"stop_reason":"model_context_window_exceeded"}`,
|
|
want: ReasonAgentContextOverflow,
|
|
},
|
|
{
|
|
name: "pre-MUL-1949 coarse reason on the overflow is upgraded",
|
|
reason: "agent_error",
|
|
raw: "API Error: The model has reached its context window limit.",
|
|
want: ReasonAgentContextOverflow,
|
|
},
|
|
{
|
|
// The witness is matched case-insensitively, like Classify's.
|
|
name: "witness casing does not defeat the upgrade",
|
|
reason: string(ReasonAgentUnknown),
|
|
raw: "API ERROR: THE MODEL HAS REACHED ITS CONTEXT WINDOW LIMIT.",
|
|
want: ReasonAgentContextOverflow,
|
|
},
|
|
{
|
|
// A current daemon already classified it; nothing to do.
|
|
name: "current daemon overflow reason passes through",
|
|
reason: string(ReasonAgentContextOverflow),
|
|
raw: "API Error: The model has reached its context window limit.",
|
|
want: ReasonAgentContextOverflow,
|
|
},
|
|
{
|
|
// A refined reason means the old daemon matched an earlier rule on
|
|
// this same text. That is a stronger statement about what ended the
|
|
// run than the witness is, so it is left alone.
|
|
name: "refined reason with the overflow witness is left alone",
|
|
reason: string(ReasonAgentProcessFailure),
|
|
raw: "claude exited with error: exit status 1: The model has reached its context window limit.",
|
|
want: ReasonAgentProcessFailure,
|
|
},
|
|
{
|
|
name: "catchall without an overflow witness is left alone",
|
|
reason: string(ReasonAgentUnknown),
|
|
raw: "API Error: the model is overloaded",
|
|
want: ReasonAgentUnknown,
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if got := NormalizeDaemonReason(tc.reason, tc.raw); got != tc.want {
|
|
t.Errorf("NormalizeDaemonReason(%q, %q) = %q, want %q", tc.reason, tc.raw, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestNormalizeDaemonReason_UpgradedReasonIsRetryable pins the property that
|
|
// actually matters to the user: the normalised reason must be one the server
|
|
// retries. If someone later drops skill_bundle_unavailable from
|
|
// internal/service/task.go's retryableReasons, the label survives but the
|
|
// self-healing this PR is for silently disappears.
|
|
func TestNormalizeDaemonReason_UpgradedReasonIsPlatformSide(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := NormalizeDaemonReason(string(ReasonAgentUnknown), "resolve skill bundles: context deadline exceeded")
|
|
if got.IsAgentError() {
|
|
t.Errorf("%q must be platform-side: the agent process never started", got)
|
|
}
|
|
}
|
|
|
|
// TestProviderUnconfigured pins the predicate the daemon uses to decide whether
|
|
// a failure is worth annotating with the HERMES_HOME it actually read (GH
|
|
// #6872). The wrapped fixture is the shape the error really arrives in — the
|
|
// runtime's message nested inside the ACP transport's JSON-RPC framing — so a
|
|
// future refactor to equality matching fails here rather than in production.
|
|
func TestProviderUnconfigured(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
cases := []struct {
|
|
name string
|
|
in string
|
|
want bool
|
|
}{
|
|
{
|
|
"wrapped acp error as the daemon receives it",
|
|
`hermes session/new failed: session/new: Internal error (code=-32603, ` +
|
|
`data={"details":"No LLM provider configured. Run ` + "`hermes model`" + ` to select a provider."})`,
|
|
true,
|
|
},
|
|
{"bare runtime message", "No LLM provider configured. Run `hermes model` to select a provider.", true},
|
|
{"lowercased by a forwarder", "error: no llm provider configured", true},
|
|
// A credential that exists but was rejected is a different failure with
|
|
// a different fix; the annotation would misdirect the user.
|
|
{"rejected credential", "401 unauthorized: invalid api key", false},
|
|
// The provider WAS resolved — it just could not be reached.
|
|
{"provider unreachable", "connection refused: https://example.invalid/v1", false},
|
|
{"empty", "", false},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
if got := ProviderUnconfigured(tc.in); got != tc.want {
|
|
t.Errorf("ProviderUnconfigured(%q) = %v, want %v", tc.in, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestProviderUnconfiguredAgreesWithClassify keeps the shared phrase honest:
|
|
// the predicate and rule 2 read the same const, so anything the predicate
|
|
// recognises must still be filed as a config problem. If these two ever
|
|
// disagree, the daemon would be annotating failures the platform files as
|
|
// something else entirely.
|
|
func TestProviderUnconfiguredAgreesWithClassify(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const errText = "No LLM provider configured. Run `hermes model` to select a provider."
|
|
if !ProviderUnconfigured(errText) {
|
|
t.Fatal("precondition: the predicate should match its own phrase")
|
|
}
|
|
if got := Classify(errText); got != ReasonAgentMissingConfig {
|
|
t.Errorf("Classify(%q) = %q, want %q", errText, got, ReasonAgentMissingConfig)
|
|
}
|
|
}
|
|
|
|
// TestNormalizeDaemonReasonUpgradesOpenclawCLITimeout covers the mixed-version
|
|
// window for #7112. A daemon that predates the timeout sentinel classifies the
|
|
// failure from its text and lands on agent_error.provider_network, which is
|
|
// worse than merely imprecise: that reason is on the auto-retry allowlist, so
|
|
// every attempt re-pays the same 8-11s stall and fails identically, and the
|
|
// chat bubble tells the user to check a network that was never involved.
|
|
// Recognising the wire shape fixes both the moment the server deploys.
|
|
func TestNormalizeDaemonReasonUpgradesOpenclawCLITimeout(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const rawError = "prepare execution environment: execenv: prepare openclaw config: " +
|
|
"locate openclaw active config: openclaw config file: context deadline exceeded " +
|
|
"(process: signal: killed)"
|
|
|
|
for _, legacy := range []string{
|
|
string(ReasonAgentProviderNetwork),
|
|
string(ReasonAgentUnknown),
|
|
"agent_error",
|
|
} {
|
|
if got := NormalizeDaemonReason(legacy, rawError); got != ReasonRuntimeCLITimeout {
|
|
t.Errorf("NormalizeDaemonReason(%q, openclaw timeout) = %q, want %q", legacy, got, ReasonRuntimeCLITimeout)
|
|
}
|
|
}
|
|
|
|
// A current daemon already reports the precise reason; normalization must
|
|
// leave it alone.
|
|
if got := NormalizeDaemonReason(string(ReasonRuntimeCLITimeout), rawError); got != ReasonRuntimeCLITimeout {
|
|
t.Errorf("NormalizeDaemonReason(runtime_cli_timeout) = %q, want it preserved", got)
|
|
}
|
|
|
|
// Both witnesses are required. A real provider-side deadline, and an
|
|
// openclaw prep failure that is not a timeout, must keep their own
|
|
// classification — the global "deadline exceeded" rule still belongs to
|
|
// genuine network stalls.
|
|
unrelated := map[string]struct {
|
|
reason string
|
|
rawError string
|
|
want Reason
|
|
}{
|
|
"provider deadline stays provider_network": {
|
|
reason: string(ReasonAgentProviderNetwork),
|
|
rawError: "API Error: context deadline exceeded while streaming from provider",
|
|
want: ReasonAgentProviderNetwork,
|
|
},
|
|
"openclaw prep failure without a timeout is untouched": {
|
|
reason: string(ReasonAgentUnknown),
|
|
rawError: "execenv: prepare openclaw config: read openclaw agents.list: exit status 1",
|
|
want: ReasonAgentUnknown,
|
|
},
|
|
}
|
|
for name, tc := range unrelated {
|
|
if got := NormalizeDaemonReason(tc.reason, tc.rawError); got != tc.want {
|
|
t.Errorf("%s: NormalizeDaemonReason = %q, want %q", name, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestClassifyKeepsDeadlineExceededAsProviderNetwork pins the rule the fix
|
|
// deliberately did NOT touch. Local runtime CLI timeouts are recognised
|
|
// structurally, upstream of Classify; the text rule still has to serve real
|
|
// provider stalls, which are transient and retryable.
|
|
func TestClassifyKeepsDeadlineExceededAsProviderNetwork(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
if got := Classify("post to provider: context deadline exceeded"); got != ReasonAgentProviderNetwork {
|
|
t.Errorf("Classify(provider deadline) = %q, want %q", got, ReasonAgentProviderNetwork)
|
|
}
|
|
}
|