diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 4877eed64f146..0bcd40e0a8cda 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -7631,6 +7631,229 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + const poolModel = "mimo-v2.6-flash-all-pool"; + const poolCooldownBody = (resetSeconds: number) => + `{"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down","model":"${poolModel}","reset_seconds":${resetSeconds},"reset_time":"1h2m3s"}}`; + it.effect.each([ + { + name: "a 429 whose cooling-down message names the earliest reset", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 2h54m21s)`, + seconds: 2 * 3_600 + 54 * 60 + 21, + }, + { + name: "a 429 reset outside the API Error prefix", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 4m 6s)`, + seconds: 246, + }, + { + name: "a 429 Claude body whose last error names a later reset", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (reset after 2h54m21s) (last error: GoUsageLimitError (reset after 9h0m0s))"}}`, + seconds: 2 * 3_600 + 54 * 60 + 21, + }, + { + name: "a rendered 429 whose last error names a later reset", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 2h54m21s) (last error: GoUsageLimitError (reset after 9h0m0s))`, + seconds: 2 * 3_600 + 54 * 60 + 21, + }, + { + name: "a 429 with no time left and a reset_seconds in its last error", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (reset after 0s) (last error: {\\"reset_seconds\\":32400})"}}`, + seconds: 0, + usageLimit: true, + }, + { + name: "a model_cooldown whose only reset_seconds is in its last error", + status: 500, + code: "api_error_500", + error: "server_error" as const, + result: `API Error: 500 {"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down (last error: {\\"reset_seconds\\":32400})"}}`, + seconds: 0, + }, + { + name: "a 429 Claude body whose only reset is in its last error", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (last error: GoUsageLimitError (reset after 9h0m0s))"}}`, + seconds: 0, + usageLimit: true, + }, + { + name: "a rendered 429 whose only reset is in its last error", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: Request rejected (429) · All credentials for model ${poolModel} are cooling down (last error: GoUsageLimitError (reset after 9h0m0s))`, + seconds: 0, + usageLimit: true, + }, + { + name: "a model_cooldown body with no time left in its last error and reset_seconds after it", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: 429 {"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down (last error: GoUsageLimitError (reset after 0s))","model":"${poolModel}","reset_seconds":600}}`, + seconds: 600, + }, + { + name: "a model_cooldown body in CLIProxyAPI's field order with a last error", + status: 500, + code: "api_error_500", + error: "server_error" as const, + result: `API Error: 500 {"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down (last error: {\\"error\\":{\\"message\\":\\"quota exceeded\\"}})","model":"${poolModel}","reset_seconds":3723}}`, + seconds: 3723, + }, + { + name: "a 429 model_cooldown body with reset_seconds", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: 429 ${poolCooldownBody(3723)}`, + seconds: 3723, + }, + { + name: "a 429 JSON-escaped model_cooldown body", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: 429 ${JSON.stringify(poolCooldownBody(3723))}`, + seconds: 3723, + }, + { + name: "a 429 Claude rate_limit_error body with reset_seconds", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down","reset_seconds":3723}}`, + seconds: 3723, + }, + { + name: "a model_cooldown body on a non-429 status", + status: 500, + code: "api_error_500", + error: "server_error" as const, + result: `API Error: 500 ${poolCooldownBody(3723)}`, + seconds: 3723, + }, + { + name: "a wrapped gateway 503 whose message names the reset", + status: 503, + code: "api_error_429", + error: "server_error" as const, + result: `API Error: 503 [cpa/${poolModel}] [429]: {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (reset after 1h2m3s)"}}`, + seconds: 3723, + }, + { + name: "a wrapped gateway 503 with reset_seconds", + status: 503, + code: "api_error_429", + error: "server_error" as const, + result: `API Error: 503 [cpa/${poolModel}] [429]: {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down","reset_seconds":246}}`, + seconds: 246, + }, + { + name: "a 429 model_cooldown body with no time left", + status: 429, + code: "api_error_429", + error: "rate_limit" as const, + result: `API Error: 429 ${poolCooldownBody(0)}`, + seconds: 0, + usageLimit: true, + }, + { + name: "a non-429 model_cooldown body with no time left", + status: 500, + code: "api_error_500", + error: "server_error" as const, + result: `API Error: 500 ${poolCooldownBody(0)}`, + seconds: 0, + }, + { + name: "a pool with no auth candidates", + status: 503, + code: "api_error_503", + error: "server_error" as const, + result: `API Error: 503 auth_not_found: no auth candidates (providers=xiaomi,opencode-go, model=${poolModel})`, + seconds: 0, + }, + { + name: "a JSON auth_not_found body", + status: 503, + code: "api_error_503", + error: "server_error" as const, + result: `API Error: 503 {"error":{"code":"auth_not_found","message":"no auth candidates (providers=xiaomi, model=${poolModel})"}}`, + seconds: 0, + }, + ])("reads a credential pool's earliest reset from $name", (input) => + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-pool-cooldown"), + text: "Continue.", + attachments: [], + }), + ); + yield* Queue.offerAll(harness.sdkMessages, [ + makeAssistantErrorFrame({ + uuid: "00000000-0000-4000-8000-000000000656", + error: input.error, + text: input.result, + }), + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000657", + result: input.result, + terminalReason: "api_error", + isError: true, + apiErrorStatus: input.status, + }), + ]); + const terminal = yield* Queue.take(harness.terminalReceipts); + assert.equal(terminal.status, "failed"); + if (terminal.status !== "failed") return; + const usageLimit = input.seconds > 0 || input.usageLimit === true; + assert.equal(terminal.failure.class, usageLimit ? "usage_limit" : "provider_error"); + assert.equal(terminal.failure.code, input.code); + assert.equal( + terminal.failure.resetAt ?? null, + input.seconds > 0 + ? DateTime.formatIso( + DateTime.makeUnsafe(DateTime.toEpochMillis(now) + input.seconds * 1_000), + ) + : null, + ); + if (input.seconds > 0) assert.equal(terminal.failure.retryable, true); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect("keeps the measured context when a usage limit ends the turn", () => Effect.gen(function* () { const harness = yield* makeWakeHarness; diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 2824552b32ea7..0ad6cfcb7824e 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4091,19 +4091,53 @@ function claudeUsageLimitResetAt(resetsAt: unknown): string | null { : null; } -function gatewayCooldownResetAt( - cooldown: RegExpExecArray | null, - completedAt: DateTime.Utc, -): string | undefined { - if (cooldown === null) return undefined; - const seconds = - Number(cooldown[1] ?? 0) * 3_600 + Number(cooldown[2] ?? 0) * 60 + Number(cooldown[3] ?? 0); - const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000; +const GATEWAY_WRAPPED_RATE_LIMIT = + /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",/; +const GATEWAY_RESET_AFTER = /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/; +// CLIProxyAPI's model_cooldown body; the CLI may relay it raw or JSON-escaped. +const GATEWAY_RESET_SECONDS = /\\?"reset_seconds\\?"\s*:\s*(\d+)/; +const GATEWAY_MODEL_COOLDOWN = /\\?"code\\?"\s*:\s*\\?"model_cooldown\\?"/; + +function resetAtAfterSeconds(seconds: number, receivedAt: DateTime.Utc): string | undefined { + const resetMs = DateTime.toEpochMillis(receivedAt) + seconds * 1_000; return seconds > 0 && resetMs < 8.64e15 ? DateTime.formatIso(DateTime.makeUnsafe(resetMs)) : undefined; } +// Drops each "(last error: …)", which quotes one credential's error rather than +// the pool's. It runs to the end of the JSON string holding it (the first quote +// with no backslash before it), so fields after that string survive, or to the +// end of rendered, non-JSON text. +function withoutLastError(text: string): string { + let rest = text; + for (let start = rest.indexOf("(last error:"); start >= 0; start = rest.indexOf("(last error:")) { + let end = start; + do { + end = rest.indexOf('"', end + 1); + } while (end > 0 && rest[end - 1] === "\\"); + rest = rest.slice(0, start) + (end < 0 ? "" : rest.slice(end)); + } + return rest; +} + +// A gateway names its earliest credential reset as "(reset after 1h2m3s)" text +// or, from CLIProxyAPI, as a model_cooldown body's reset_seconds. +function gatewayCooldownResetAt(text: string, receivedAt: DateTime.Utc): string | undefined { + const poolText = withoutLastError(text); + const cooldown = GATEWAY_RESET_AFTER.exec(poolText); + if (cooldown !== null) { + return resetAtAfterSeconds( + Number(cooldown[1] ?? 0) * 3_600 + Number(cooldown[2] ?? 0) * 60 + Number(cooldown[3] ?? 0), + receivedAt, + ); + } + const resetSeconds = GATEWAY_RESET_SECONDS.exec(poolText); + return resetSeconds === null + ? undefined + : resetAtAfterSeconds(Number(resetSeconds[1]), receivedAt); +} + function providerFailureFromResult( message: SDKResultMessage, receivedAt: DateTime.Utc, @@ -4115,13 +4149,9 @@ function providerFailureFromResult( message.subtype === "success" && message.is_error && message.terminal_reason === "api_error" && - message.api_error_status === 503 - ? gatewayCooldownResetAt( - /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec( - message.result, - ), - receivedAt, - ) + message.api_error_status === 503 && + GATEWAY_WRAPPED_RATE_LIMIT.test(message.result) + ? gatewayCooldownResetAt(message.result, receivedAt) : undefined; if (gatewayResetAt !== undefined) { return makeProviderFailure({ @@ -4132,8 +4162,16 @@ function providerFailureFromResult( resetAt: gatewayResetAt, }); } + // CLIProxyAPI reports an exhausted credential pool as model_cooldown with + // the earliest credential's reset, whatever status carries it. + const poolCooldownResetAt = + message.subtype === "success" && message.is_error && GATEWAY_MODEL_COOLDOWN.test(message.result) + ? gatewayCooldownResetAt(message.result, receivedAt) + : undefined; const failureClass = - (message.subtype === "success" && message.api_error_status === 429) || usageLimited + (message.subtype === "success" && message.api_error_status === 429) || + usageLimited || + poolCooldownResetAt !== undefined ? "usage_limit" : "provider_error"; const listedError = resultUserFacingError(message); @@ -4161,12 +4199,9 @@ function providerFailureFromResult( const apiErrorStatus = message.api_error_status ?? null; // API-key sessions get no rate_limit_info, so a gateway's 429 carries its reset only as text. const resetAt = - apiErrorStatus === 429 && message.result.startsWith("API Error:") - ? gatewayCooldownResetAt( - /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(message.result), - receivedAt, - ) - : undefined; + apiErrorStatus === 429 && (message.is_error || message.result.startsWith("API Error:")) + ? gatewayCooldownResetAt(message.result, receivedAt) + : poolCooldownResetAt; return makeProviderFailure({ message: listedError ?? structuredError ?? failureHint ?? message.result, code: @@ -4174,7 +4209,10 @@ function providerFailureFromResult( ? (message.terminal_reason ?? "sdk_result_error") : `api_error_${apiErrorStatus}`, class: failureClass, - retryable: apiErrorStatus === 429 || apiErrorStatus === 529 ? true : null, + retryable: + apiErrorStatus === 429 || apiErrorStatus === 529 || poolCooldownResetAt !== undefined + ? true + : null, resetAt: resetAt ?? null, }); }