From 531ed27c23ba6721c47e5b6109da1be574b3429e Mon Sep 17 00:00:00 2001 From: Ali Alwan <11404482+Ali-Al1@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:33:07 +0000 Subject: [PATCH 1/4] fix(server): Claude pool limits keep the earliest gateway reset When every account in a CLIProxyAPI pool is cooling down, the proxy reports the earliest account reset either as "(reset after 1h2m3s)" text or as a model_cooldown body with reset_seconds. The Claude adapter only read the text form, and only directly after a 429's "API Error:" prefix or in 9router's exact wrapped 503 shape, so pool limits failed as provider errors or as usage limits with no reset. Read the reset from either form on a 429, a wrapped gateway 503, or any model_cooldown body, and class a model_cooldown with a reset as a usage limit whatever status carries it. auth_not_found and other errors with no reset stay provider errors. Co-Authored-By: Claude Code --- .../Adapters/ClaudeAdapterV2.test.ts | 143 ++++++++++++++++++ .../Adapters/ClaudeAdapterV2.ts | 71 ++++++--- 2 files changed, 191 insertions(+), 23 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 4877eed64f146..705d2f4aba6a0 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -7631,6 +7631,149 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + const poolModel = "mimo-v2.6-flash-all-pool"; + const poolCooldownBody = (resetSeconds: number) => + `{"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down","model":"${poolModel}","reset_seconds":${resetSeconds},"reset_time":"1h2m3s"}}`; + it.effect.each([ + { + name: "a 429 whose cooling-down message names the earliest reset", + status: 429, + error: "rate_limit" as const, + result: `API Error: Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 2h54m21s)`, + seconds: 2 * 3_600 + 54 * 60 + 21, + }, + { + name: "a 429 reset outside the API Error prefix", + status: 429, + error: "rate_limit" as const, + result: `Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 4m 6s)`, + seconds: 246, + }, + { + name: "a 429 model_cooldown body with reset_seconds", + status: 429, + error: "rate_limit" as const, + result: `API Error: 429 ${poolCooldownBody(3723)}`, + seconds: 3723, + }, + { + name: "a 429 JSON-escaped model_cooldown body", + status: 429, + error: "rate_limit" as const, + result: `API Error: 429 ${JSON.stringify(poolCooldownBody(3723))}`, + seconds: 3723, + }, + { + name: "a 429 Claude rate_limit_error body with reset_seconds", + status: 429, + error: "rate_limit" as const, + result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down","reset_seconds":3723}}`, + seconds: 3723, + }, + { + name: "a model_cooldown body on a non-429 status", + status: 500, + error: "server_error" as const, + result: `API Error: 500 ${poolCooldownBody(3723)}`, + seconds: 3723, + }, + { + name: "a wrapped gateway 503 whose message names the reset", + status: 503, + error: "server_error" as const, + result: `API Error: 503 [cpa/${poolModel}] [429]: {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (reset after 1h2m3s)"}}`, + seconds: 3723, + }, + { + name: "a wrapped gateway 503 with reset_seconds", + status: 503, + error: "server_error" as const, + result: `API Error: 503 [cpa/${poolModel}] [429]: {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down","reset_seconds":246}}`, + seconds: 246, + }, + { + name: "a 429 model_cooldown body with no time left", + status: 429, + error: "rate_limit" as const, + result: `API Error: 429 ${poolCooldownBody(0)}`, + seconds: 0, + usageLimit: true, + }, + { + name: "a non-429 model_cooldown body with no time left", + status: 500, + error: "server_error" as const, + result: `API Error: 500 ${poolCooldownBody(0)}`, + seconds: 0, + }, + { + name: "a pool with no auth candidates", + status: 503, + error: "server_error" as const, + result: `API Error: 503 auth_not_found: no auth candidates (providers=xiaomi,opencode-go, model=${poolModel})`, + seconds: 0, + }, + { + name: "a JSON auth_not_found body", + status: 503, + error: "server_error" as const, + result: `API Error: 503 {"error":{"code":"auth_not_found","message":"no auth candidates (providers=xiaomi, model=${poolModel})"}}`, + seconds: 0, + }, + ])("reads a credential pool's earliest reset from $name", (input) => + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-pool-cooldown"), + text: "Continue.", + attachments: [], + }), + ); + yield* Queue.offerAll(harness.sdkMessages, [ + makeAssistantErrorFrame({ + uuid: "00000000-0000-4000-8000-000000000656", + error: input.error, + text: input.result, + }), + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000657", + result: input.result, + terminalReason: "api_error", + isError: true, + apiErrorStatus: input.status, + }), + ]); + const terminal = yield* Queue.take(harness.terminalReceipts); + assert.equal(terminal.status, "failed"); + if (terminal.status !== "failed") return; + const usageLimit = input.seconds > 0 || input.usageLimit === true; + assert.equal(terminal.failure.class, usageLimit ? "usage_limit" : "provider_error"); + assert.equal( + terminal.failure.resetAt ?? null, + input.seconds > 0 + ? DateTime.formatIso( + DateTime.makeUnsafe(DateTime.toEpochMillis(now) + input.seconds * 1_000), + ) + : null, + ); + if (input.seconds > 0) assert.equal(terminal.failure.retryable, true); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect("keeps the measured context when a usage limit ends the turn", () => Effect.gen(function* () { const harness = yield* makeWakeHarness; diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 2824552b32ea7..ad48b5e33f44c 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4091,19 +4091,40 @@ function claudeUsageLimitResetAt(resetsAt: unknown): string | null { : null; } -function gatewayCooldownResetAt( - cooldown: RegExpExecArray | null, - completedAt: DateTime.Utc, -): string | undefined { - if (cooldown === null) return undefined; - const seconds = - Number(cooldown[1] ?? 0) * 3_600 + Number(cooldown[2] ?? 0) * 60 + Number(cooldown[3] ?? 0); - const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000; +const GATEWAY_WRAPPED_RATE_LIMIT = + /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",/; +const GATEWAY_RESET_AFTER = /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/; +// CLIProxyAPI's model_cooldown body; the CLI may relay it raw or JSON-escaped. +const GATEWAY_RESET_SECONDS = /\\?"reset_seconds\\?"\s*:\s*(\d+)/; +const GATEWAY_MODEL_COOLDOWN = /\\?"code\\?"\s*:\s*\\?"model_cooldown\\?"/; + +function resetAtAfterSeconds(seconds: number, receivedAt: DateTime.Utc): string | undefined { + const resetMs = DateTime.toEpochMillis(receivedAt) + seconds * 1_000; return seconds > 0 && resetMs < 8.64e15 ? DateTime.formatIso(DateTime.makeUnsafe(resetMs)) : undefined; } +// A gateway names its earliest credential reset as "(reset after 1h2m3s)" text +// or, from CLIProxyAPI, as a model_cooldown body's reset_seconds. +function gatewayCooldownResetAt(text: string, receivedAt: DateTime.Utc): string | undefined { + const cooldown = GATEWAY_RESET_AFTER.exec(text); + const fromText = + cooldown === null + ? undefined + : resetAtAfterSeconds( + Number(cooldown[1] ?? 0) * 3_600 + + Number(cooldown[2] ?? 0) * 60 + + Number(cooldown[3] ?? 0), + receivedAt, + ); + if (fromText !== undefined) return fromText; + const resetSeconds = GATEWAY_RESET_SECONDS.exec(text); + return resetSeconds === null + ? undefined + : resetAtAfterSeconds(Number(resetSeconds[1]), receivedAt); +} + function providerFailureFromResult( message: SDKResultMessage, receivedAt: DateTime.Utc, @@ -4115,13 +4136,9 @@ function providerFailureFromResult( message.subtype === "success" && message.is_error && message.terminal_reason === "api_error" && - message.api_error_status === 503 - ? gatewayCooldownResetAt( - /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec( - message.result, - ), - receivedAt, - ) + message.api_error_status === 503 && + GATEWAY_WRAPPED_RATE_LIMIT.test(message.result) + ? gatewayCooldownResetAt(message.result, receivedAt) : undefined; if (gatewayResetAt !== undefined) { return makeProviderFailure({ @@ -4132,8 +4149,16 @@ function providerFailureFromResult( resetAt: gatewayResetAt, }); } + // CLIProxyAPI reports an exhausted credential pool as model_cooldown with + // the earliest credential's reset, whatever status carries it. + const poolCooldownResetAt = + message.subtype === "success" && message.is_error && GATEWAY_MODEL_COOLDOWN.test(message.result) + ? gatewayCooldownResetAt(message.result, receivedAt) + : undefined; const failureClass = - (message.subtype === "success" && message.api_error_status === 429) || usageLimited + (message.subtype === "success" && message.api_error_status === 429) || + usageLimited || + poolCooldownResetAt !== undefined ? "usage_limit" : "provider_error"; const listedError = resultUserFacingError(message); @@ -4161,12 +4186,9 @@ function providerFailureFromResult( const apiErrorStatus = message.api_error_status ?? null; // API-key sessions get no rate_limit_info, so a gateway's 429 carries its reset only as text. const resetAt = - apiErrorStatus === 429 && message.result.startsWith("API Error:") - ? gatewayCooldownResetAt( - /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(message.result), - receivedAt, - ) - : undefined; + apiErrorStatus === 429 && (message.is_error || message.result.startsWith("API Error:")) + ? gatewayCooldownResetAt(message.result, receivedAt) + : poolCooldownResetAt; return makeProviderFailure({ message: listedError ?? structuredError ?? failureHint ?? message.result, code: @@ -4174,7 +4196,10 @@ function providerFailureFromResult( ? (message.terminal_reason ?? "sdk_result_error") : `api_error_${apiErrorStatus}`, class: failureClass, - retryable: apiErrorStatus === 429 || apiErrorStatus === 529 ? true : null, + retryable: + apiErrorStatus === 429 || apiErrorStatus === 529 || poolCooldownResetAt !== undefined + ? true + : null, resetAt: resetAt ?? null, }); } From dc30d01a553a6c81747039945a43c41a3e447afd Mon Sep 17 00:00:00 2001 From: Ali Alwan <11404482+Ali-Al1@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:17:36 +0000 Subject: [PATCH 2/4] fix(server): a pool's last error never sets its reset CLIProxyAPI appends the last upstream error to an exhausted pool's 429, e.g. "(reset after 2h54m21s) (last error: GoUsageLimitError (reset after 9h0m0s))". The pool's own reset is the first match. When that first match is "(reset after 0s)", or the pool names no reset, the parser no longer falls back to a reset_seconds quoted inside "(last error: ...)": that is one credential's reset, not the pool's. Adds the proxy-shaped 429 in its raw Claude body and rendered forms, and both last-error fallback cases, to the pool reset table. Co-Authored-By: Claude Code --- .../Adapters/ClaudeAdapterV2.test.ts | 29 +++++++++++++++++++ .../Adapters/ClaudeAdapterV2.ts | 20 ++++++------- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 705d2f4aba6a0..5ec3859446667 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -7649,6 +7649,35 @@ describe("ClaudeAdapterV2 background wake turns", () => { result: `Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 4m 6s)`, seconds: 246, }, + { + name: "a 429 Claude body whose last error names a later reset", + status: 429, + error: "rate_limit" as const, + result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (reset after 2h54m21s) (last error: GoUsageLimitError (reset after 9h0m0s))"}}`, + seconds: 2 * 3_600 + 54 * 60 + 21, + }, + { + name: "a rendered 429 whose last error names a later reset", + status: 429, + error: "rate_limit" as const, + result: `API Error: Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 2h54m21s) (last error: GoUsageLimitError (reset after 9h0m0s))`, + seconds: 2 * 3_600 + 54 * 60 + 21, + }, + { + name: "a 429 with no time left and a reset_seconds in its last error", + status: 429, + error: "rate_limit" as const, + result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (reset after 0s) (last error: {\\"reset_seconds\\":32400})"}}`, + seconds: 0, + usageLimit: true, + }, + { + name: "a model_cooldown whose only reset_seconds is in its last error", + status: 500, + error: "server_error" as const, + result: `API Error: 500 {"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down (last error: {\\"reset_seconds\\":32400})"}}`, + seconds: 0, + }, { name: "a 429 model_cooldown body with reset_seconds", status: 429, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index ad48b5e33f44c..cf0baf769cf6a 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4109,17 +4109,15 @@ function resetAtAfterSeconds(seconds: number, receivedAt: DateTime.Utc): string // or, from CLIProxyAPI, as a model_cooldown body's reset_seconds. function gatewayCooldownResetAt(text: string, receivedAt: DateTime.Utc): string | undefined { const cooldown = GATEWAY_RESET_AFTER.exec(text); - const fromText = - cooldown === null - ? undefined - : resetAtAfterSeconds( - Number(cooldown[1] ?? 0) * 3_600 + - Number(cooldown[2] ?? 0) * 60 + - Number(cooldown[3] ?? 0), - receivedAt, - ); - if (fromText !== undefined) return fromText; - const resetSeconds = GATEWAY_RESET_SECONDS.exec(text); + if (cooldown !== null) { + return resetAtAfterSeconds( + Number(cooldown[1] ?? 0) * 3_600 + Number(cooldown[2] ?? 0) * 60 + Number(cooldown[3] ?? 0), + receivedAt, + ); + } + // A reset_seconds quoted in "(last error: …)" is one credential's, not the pool's. + const lastError = text.indexOf("(last error:"); + const resetSeconds = GATEWAY_RESET_SECONDS.exec(lastError < 0 ? text : text.slice(0, lastError)); return resetSeconds === null ? undefined : resetAtAfterSeconds(Number(resetSeconds[1]), receivedAt); From a2ed31bae30f0842458286df425394775d567f82 Mon Sep 17 00:00:00 2001 From: Ali-Al1 <11404482+Ali-Al1@users.noreply.github.com> Date: Mon, 9 Nov 2026 08:14:19 +0000 Subject: [PATCH 3/4] ci: retrigger pull_request merge against fixed base Empty commit: same tree as dc30d01 so GitHub rebuilds pull/3/merge against base/upstream-main-920ffcdd04-hosted-ci @ c45fdebb (gitfile classifier fix from #4). Co-Authored-By: Claude Code From e956f66d4621821dabbbbd9ad05d8b7cada2e0f1 Mon Sep 17 00:00:00 2001 From: Ali-Al1 Date: Mon, 9 Nov 2026 09:08:31 +0000 Subject: [PATCH 4/4] fix(server): ignore a pool's last error for both reset forms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A gateway's "(last error: …)" quotes one credential's error, not the pool's earliest reset. The "(reset after …)" scan read the whole result while the reset_seconds fallback was cut at the marker, so a hint inside the last error was taken as the pool's reset and a top-level reset_seconds after the message was dropped. Remove each last error up to the end of the JSON string holding it (or the end of rendered text) and run both scans on what remains. Co-Authored-By: Claude Code --- .../Adapters/ClaudeAdapterV2.test.ts | 51 +++++++++++++++++++ .../Adapters/ClaudeAdapterV2.ts | 23 +++++++-- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 5ec3859446667..0bcd40e0a8cda 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -7638,6 +7638,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a 429 whose cooling-down message names the earliest reset", status: 429, + code: "api_error_429", error: "rate_limit" as const, result: `API Error: Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 2h54m21s)`, seconds: 2 * 3_600 + 54 * 60 + 21, @@ -7645,6 +7646,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a 429 reset outside the API Error prefix", status: 429, + code: "api_error_429", error: "rate_limit" as const, result: `Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 4m 6s)`, seconds: 246, @@ -7652,6 +7654,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a 429 Claude body whose last error names a later reset", status: 429, + code: "api_error_429", error: "rate_limit" as const, result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (reset after 2h54m21s) (last error: GoUsageLimitError (reset after 9h0m0s))"}}`, seconds: 2 * 3_600 + 54 * 60 + 21, @@ -7659,6 +7662,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a rendered 429 whose last error names a later reset", status: 429, + code: "api_error_429", error: "rate_limit" as const, result: `API Error: Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 2h54m21s) (last error: GoUsageLimitError (reset after 9h0m0s))`, seconds: 2 * 3_600 + 54 * 60 + 21, @@ -7666,6 +7670,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a 429 with no time left and a reset_seconds in its last error", status: 429, + code: "api_error_429", error: "rate_limit" as const, result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (reset after 0s) (last error: {\\"reset_seconds\\":32400})"}}`, seconds: 0, @@ -7674,13 +7679,49 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a model_cooldown whose only reset_seconds is in its last error", status: 500, + code: "api_error_500", error: "server_error" as const, result: `API Error: 500 {"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down (last error: {\\"reset_seconds\\":32400})"}}`, seconds: 0, }, + { + name: "a 429 Claude body whose only reset is in its last error", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (last error: GoUsageLimitError (reset after 9h0m0s))"}}`, + seconds: 0, + usageLimit: true, + }, + { + name: "a rendered 429 whose only reset is in its last error", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: Request rejected (429) · All credentials for model ${poolModel} are cooling down (last error: GoUsageLimitError (reset after 9h0m0s))`, + seconds: 0, + usageLimit: true, + }, + { + name: "a model_cooldown body with no time left in its last error and reset_seconds after it", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: 429 {"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down (last error: GoUsageLimitError (reset after 0s))","model":"${poolModel}","reset_seconds":600}}`, + seconds: 600, + }, + { + name: "a model_cooldown body in CLIProxyAPI's field order with a last error", + status: 500, + code: "api_error_500", + error: "server_error" as const, + result: `API Error: 500 {"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down (last error: {\\"error\\":{\\"message\\":\\"quota exceeded\\"}})","model":"${poolModel}","reset_seconds":3723}}`, + seconds: 3723, + }, { name: "a 429 model_cooldown body with reset_seconds", status: 429, + code: "api_error_429", error: "rate_limit" as const, result: `API Error: 429 ${poolCooldownBody(3723)}`, seconds: 3723, @@ -7688,6 +7729,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a 429 JSON-escaped model_cooldown body", status: 429, + code: "api_error_429", error: "rate_limit" as const, result: `API Error: 429 ${JSON.stringify(poolCooldownBody(3723))}`, seconds: 3723, @@ -7695,6 +7737,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a 429 Claude rate_limit_error body with reset_seconds", status: 429, + code: "api_error_429", error: "rate_limit" as const, result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down","reset_seconds":3723}}`, seconds: 3723, @@ -7702,6 +7745,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a model_cooldown body on a non-429 status", status: 500, + code: "api_error_500", error: "server_error" as const, result: `API Error: 500 ${poolCooldownBody(3723)}`, seconds: 3723, @@ -7709,6 +7753,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a wrapped gateway 503 whose message names the reset", status: 503, + code: "api_error_429", error: "server_error" as const, result: `API Error: 503 [cpa/${poolModel}] [429]: {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (reset after 1h2m3s)"}}`, seconds: 3723, @@ -7716,6 +7761,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a wrapped gateway 503 with reset_seconds", status: 503, + code: "api_error_429", error: "server_error" as const, result: `API Error: 503 [cpa/${poolModel}] [429]: {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down","reset_seconds":246}}`, seconds: 246, @@ -7723,6 +7769,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a 429 model_cooldown body with no time left", status: 429, + code: "api_error_429", error: "rate_limit" as const, result: `API Error: 429 ${poolCooldownBody(0)}`, seconds: 0, @@ -7731,6 +7778,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a non-429 model_cooldown body with no time left", status: 500, + code: "api_error_500", error: "server_error" as const, result: `API Error: 500 ${poolCooldownBody(0)}`, seconds: 0, @@ -7738,6 +7786,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a pool with no auth candidates", status: 503, + code: "api_error_503", error: "server_error" as const, result: `API Error: 503 auth_not_found: no auth candidates (providers=xiaomi,opencode-go, model=${poolModel})`, seconds: 0, @@ -7745,6 +7794,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "a JSON auth_not_found body", status: 503, + code: "api_error_503", error: "server_error" as const, result: `API Error: 503 {"error":{"code":"auth_not_found","message":"no auth candidates (providers=xiaomi, model=${poolModel})"}}`, seconds: 0, @@ -7782,6 +7832,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { if (terminal.status !== "failed") return; const usageLimit = input.seconds > 0 || input.usageLimit === true; assert.equal(terminal.failure.class, usageLimit ? "usage_limit" : "provider_error"); + assert.equal(terminal.failure.code, input.code); assert.equal( terminal.failure.resetAt ?? null, input.seconds > 0 diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index cf0baf769cf6a..0ad6cfcb7824e 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4105,19 +4105,34 @@ function resetAtAfterSeconds(seconds: number, receivedAt: DateTime.Utc): string : undefined; } +// Drops each "(last error: …)", which quotes one credential's error rather than +// the pool's. It runs to the end of the JSON string holding it (the first quote +// with no backslash before it), so fields after that string survive, or to the +// end of rendered, non-JSON text. +function withoutLastError(text: string): string { + let rest = text; + for (let start = rest.indexOf("(last error:"); start >= 0; start = rest.indexOf("(last error:")) { + let end = start; + do { + end = rest.indexOf('"', end + 1); + } while (end > 0 && rest[end - 1] === "\\"); + rest = rest.slice(0, start) + (end < 0 ? "" : rest.slice(end)); + } + return rest; +} + // A gateway names its earliest credential reset as "(reset after 1h2m3s)" text // or, from CLIProxyAPI, as a model_cooldown body's reset_seconds. function gatewayCooldownResetAt(text: string, receivedAt: DateTime.Utc): string | undefined { - const cooldown = GATEWAY_RESET_AFTER.exec(text); + const poolText = withoutLastError(text); + const cooldown = GATEWAY_RESET_AFTER.exec(poolText); if (cooldown !== null) { return resetAtAfterSeconds( Number(cooldown[1] ?? 0) * 3_600 + Number(cooldown[2] ?? 0) * 60 + Number(cooldown[3] ?? 0), receivedAt, ); } - // A reset_seconds quoted in "(last error: …)" is one credential's, not the pool's. - const lastError = text.indexOf("(last error:"); - const resetSeconds = GATEWAY_RESET_SECONDS.exec(lastError < 0 ? text : text.slice(0, lastError)); + const resetSeconds = GATEWAY_RESET_SECONDS.exec(poolText); return resetSeconds === null ? undefined : resetAtAfterSeconds(Number(resetSeconds[1]), receivedAt);