fix(auth): retry and backoff cloudflare challenge 403 errors

Introduce Cloudflare challenge detection for 403 errors in the Auth
Manager. Apply a progressive rate-limiting cooldown ladder using the
existing BackoffLevel field instead of a hard 30-minute credentials
suspension. This ensures challenged requests fall through to subsequent
credentials and recover exponentially.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
sususu98
2026-06-03 10:28:43 +08:00
co-authored by Claude Opus 4.8
parent 7682accd95
commit 45f58d4f91
2 changed files with 125 additions and 0 deletions
+71
View File
@@ -2351,6 +2351,30 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
state.NextRetryAfter = next
suspendReason = "model_not_supported"
shouldSuspendModel = true
} else if isCloudflareChallengeResultError(result.Error) {
var next time.Time
backoffLevel := state.Quota.BackoffLevel
if !disableCooling {
cooldown, nextLevel := nextQuotaCooldown(backoffLevel, disableCooling)
if cooldown < 10*time.Second {
cooldown = 10 * time.Second
}
if cooldown > 0 {
next = now.Add(cooldown)
}
backoffLevel = nextLevel
}
state.NextRetryAfter = next
state.StatusMessage = "cloudflare challenge"
if auth.LastError != nil {
auth.StatusMessage = "cloudflare challenge"
}
state.Quota = QuotaState{
Exceeded: true,
Reason: "cloudflare challenge",
NextRecoverAt: next,
BackoffLevel: backoffLevel,
}
} else {
switch statusCode {
case 401:
@@ -2750,6 +2774,27 @@ func isModelSupportResultError(err *Error) bool {
return isModelSupportErrorMessage(err.Message)
}
func isCloudflareChallengeErrorMessage(message string) bool {
lower := strings.ToLower(strings.TrimSpace(message))
return strings.Contains(lower, "challenge-platform") ||
strings.Contains(lower, "cf-mitigated") ||
strings.Contains(lower, "challenge") ||
(strings.Contains(lower, "cloudflare") && strings.Contains(lower, "<html"))
}
func isCloudflareChallengeError(err error) bool {
if err == nil {
return false
}
return isCloudflareChallengeErrorMessage(err.Error())
}
func isCloudflareChallengeResultError(err *Error) bool {
if err == nil {
return false
}
return isCloudflareChallengeErrorMessage(err.Message)
}
func isRequestScopedNotFoundMessage(message string) bool {
if message == "" {
return false
@@ -2777,6 +2822,9 @@ func isRequestInvalidError(err error) bool {
if err == nil {
return false
}
if isCloudflareChallengeError(err) {
return false
}
if isModelSupportError(err) {
return false
}
@@ -2818,6 +2866,29 @@ func applyAuthFailureState(auth *Auth, resultErr *Error, retryAfter *time.Durati
}
}
statusCode := statusCodeFromResult(resultErr)
if isCloudflareChallengeResultError(resultErr) {
auth.StatusMessage = "cloudflare challenge"
var next time.Time
backoffLevel := auth.Quota.BackoffLevel
if !disableCooling {
cooldown, nextLevel := nextQuotaCooldown(backoffLevel, disableCooling)
if cooldown < 10*time.Second {
cooldown = 10 * time.Second
}
if cooldown > 0 {
next = now.Add(cooldown)
}
backoffLevel = nextLevel
}
auth.Quota = QuotaState{
Exceeded: true,
Reason: "cloudflare challenge",
NextRecoverAt: next,
BackoffLevel: backoffLevel,
}
auth.NextRetryAfter = next
return
}
switch statusCode {
case 401:
auth.StatusMessage = "unauthorized"
@@ -570,6 +570,60 @@ func TestManager_MarkResult_RespectsAuthDisableCoolingOverride_On403(t *testing.
}
}
func TestManager_MarkResult_CloudflareChallenge_On403(t *testing.T) {
prev := quotaCooldownDisabled.Load()
quotaCooldownDisabled.Store(false)
t.Cleanup(func() { quotaCooldownDisabled.Store(prev) })
m := NewManager(nil, nil, nil)
auth := &Auth{
ID: "auth-cf-403",
Provider: "claude",
}
if _, errRegister := m.Register(context.Background(), auth); errRegister != nil {
t.Fatalf("register auth: %v", errRegister)
}
model := "test-model-cf-403"
reg := registry.GetGlobalRegistry()
reg.RegisterClient(auth.ID, "claude", []*registry.ModelInfo{{ID: model}})
t.Cleanup(func() { reg.UnregisterClient(auth.ID) })
m.MarkResult(context.Background(), Result{
AuthID: auth.ID,
Provider: "claude",
Model: model,
Success: false,
Error: &Error{HTTPStatus: http.StatusForbidden, Message: "cf-mitigated: challenge"},
})
updated, ok := m.GetByID(auth.ID)
if !ok || updated == nil {
t.Fatalf("expected auth to be present")
}
state := updated.ModelStates[model]
if state == nil {
t.Fatalf("expected model state to be present")
}
if state.NextRetryAfter.IsZero() {
t.Fatalf("expected NextRetryAfter to be non-zero for cloudflare challenge")
}
diff := time.Until(state.NextRetryAfter)
if diff < 5*time.Second || diff > 25*time.Second {
t.Fatalf("expected NextRetryAfter to be ~10 seconds, got %v", diff)
}
if state.StatusMessage != "cloudflare challenge" {
t.Fatalf("expected StatusMessage to be 'cloudflare challenge', got %s", state.StatusMessage)
}
// Because Cloudflare Challenge is treated as transient (no suspension),
// the model should NOT be suspended in the global registry, so count > 0.
if count := reg.GetModelCount(model); count <= 0 {
t.Fatalf("expected model count > 0 for cloudflare challenge transient cooldown, got %d", count)
}
}
func TestManager_Execute_DisableCooling_DoesNotBlackoutAfter403(t *testing.T) {
prev := quotaCooldownDisabled.Load()
quotaCooldownDisabled.Store(false)