fix(server): Claude pool limits keep the earliest gateway reset

apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts

@@ -7631,6 +7631,229 @@ describe("ClaudeAdapterV2 background wake turns", () => {
     ),
   );
 
+  const poolModel = "mimo-v2.6-flash-all-pool";
+  const poolCooldownBody = (resetSeconds: number) =>
+    `{"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down","model":"${poolModel}","reset_seconds":${resetSeconds},"reset_time":"1h2m3s"}}`;
+  it.effect.each([
+    {
+      name: "a 429 whose cooling-down message names the earliest reset",
+      status: 429,
+      code: "api_error_429",
+      error: "rate_limit" as const,
+      result: `API Error: Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 2h54m21s)`,
+      seconds: 2 * 3_600 + 54 * 60 + 21,
+    },
+    {
+      name: "a 429 reset outside the API Error prefix",
+      status: 429,
+      code: "api_error_429",
+      error: "rate_limit" as const,
+      result: `Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 4m 6s)`,
+      seconds: 246,
+    },
+    {
+      name: "a 429 Claude body whose last error names a later reset",
+      status: 429,
+      code: "api_error_429",
+      error: "rate_limit" as const,
+      result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (reset after 2h54m21s) (last error: GoUsageLimitError (reset after 9h0m0s))"}}`,
+      seconds: 2 * 3_600 + 54 * 60 + 21,
+    },
+    {
+      name: "a rendered 429 whose last error names a later reset",
+      status: 429,
+      code: "api_error_429",
+      error: "rate_limit" as const,
+      result: `API Error: Request rejected (429) · All credentials for model ${poolModel} are cooling down (reset after 2h54m21s) (last error: GoUsageLimitError (reset after 9h0m0s))`,
+      seconds: 2 * 3_600 + 54 * 60 + 21,
+    },
+    {
+      name: "a 429 with no time left and a reset_seconds in its last error",
+      status: 429,
+      code: "api_error_429",
+      error: "rate_limit" as const,
+      result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (reset after 0s) (last error: {\\"reset_seconds\\":32400})"}}`,
+      seconds: 0,
+      usageLimit: true,
+    },
+    {
+      name: "a model_cooldown whose only reset_seconds is in its last error",
+      status: 500,
+      code: "api_error_500",
+      error: "server_error" as const,
+      result: `API Error: 500 {"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down (last error: {\\"reset_seconds\\":32400})"}}`,
+      seconds: 0,
+    },
+    {
+      name: "a 429 Claude body whose only reset is in its last error",
+      status: 429,
+      code: "api_error_429",
+      error: "rate_limit" as const,
+      result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (last error: GoUsageLimitError (reset after 9h0m0s))"}}`,
+      seconds: 0,
+      usageLimit: true,
+    },
+    {
+      name: "a rendered 429 whose only reset is in its last error",
+      status: 429,
+      code: "api_error_429",
+      error: "rate_limit" as const,
+      result: `API Error: Request rejected (429) · All credentials for model ${poolModel} are cooling down (last error: GoUsageLimitError (reset after 9h0m0s))`,
+      seconds: 0,
+      usageLimit: true,
+    },
+    {
+      name: "a model_cooldown body with no time left in its last error and reset_seconds after it",
+      status: 429,
+      code: "api_error_429",
+      error: "rate_limit" as const,
+      result: `API Error: 429 {"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down (last error: GoUsageLimitError (reset after 0s))","model":"${poolModel}","reset_seconds":600}}`,
+      seconds: 600,
+    },
+    {
+      name: "a model_cooldown body in CLIProxyAPI's field order with a last error",
+      status: 500,
+      code: "api_error_500",
+      error: "server_error" as const,
+      result: `API Error: 500 {"error":{"code":"model_cooldown","message":"All credentials for model ${poolModel} are cooling down (last error: {\\"error\\":{\\"message\\":\\"quota exceeded\\"}})","model":"${poolModel}","reset_seconds":3723}}`,
+      seconds: 3723,
+    },
+    {
+      name: "a 429 model_cooldown body with reset_seconds",
+      status: 429,
+      code: "api_error_429",
+      error: "rate_limit" as const,
+      result: `API Error: 429 ${poolCooldownBody(3723)}`,
+      seconds: 3723,
+    },
+    {
+      name: "a 429 JSON-escaped model_cooldown body",
+      status: 429,
+      code: "api_error_429",
+      error: "rate_limit" as const,
+      result: `API Error: 429 ${JSON.stringify(poolCooldownBody(3723))}`,
+      seconds: 3723,
+    },
+    {
+      name: "a 429 Claude rate_limit_error body with reset_seconds",
+      status: 429,
+      code: "api_error_429",
+      error: "rate_limit" as const,
+      result: `API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down","reset_seconds":3723}}`,
+      seconds: 3723,
+    },
+    {
+      name: "a model_cooldown body on a non-429 status",
+      status: 500,
+      code: "api_error_500",
+      error: "server_error" as const,
+      result: `API Error: 500 ${poolCooldownBody(3723)}`,
+      seconds: 3723,
+    },
+    {
+      name: "a wrapped gateway 503 whose message names the reset",
+      status: 503,
+      code: "api_error_429",
+      error: "server_error" as const,
+      result: `API Error: 503 [cpa/${poolModel}] [429]: {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down (reset after 1h2m3s)"}}`,
+      seconds: 3723,
+    },
+    {
+      name: "a wrapped gateway 503 with reset_seconds",
+      status: 503,
+      code: "api_error_429",
+      error: "server_error" as const,
+      result: `API Error: 503 [cpa/${poolModel}] [429]: {"type":"error","error":{"type":"rate_limit_error","message":"All credentials for model ${poolModel} are cooling down","reset_seconds":246}}`,
+      seconds: 246,
+    },
+    {
+      name: "a 429 model_cooldown body with no time left",
+      status: 429,
+      code: "api_error_429",
+      error: "rate_limit" as const,
+      result: `API Error: 429 ${poolCooldownBody(0)}`,
+      seconds: 0,
+      usageLimit: true,
+    },
+    {
+      name: "a non-429 model_cooldown body with no time left",
+      status: 500,
+      code: "api_error_500",
+      error: "server_error" as const,
+      result: `API Error: 500 ${poolCooldownBody(0)}`,
+      seconds: 0,
+    },
+    {
+      name: "a pool with no auth candidates",
+      status: 503,
+      code: "api_error_503",
+      error: "server_error" as const,
+      result: `API Error: 503 auth_not_found: no auth candidates (providers=xiaomi,opencode-go, model=${poolModel})`,
+      seconds: 0,
+    },
+    {
+      name: "a JSON auth_not_found body",
+      status: 503,
+      code: "api_error_503",
+      error: "server_error" as const,
+      result: `API Error: 503 {"error":{"code":"auth_not_found","message":"no auth candidates (providers=xiaomi, model=${poolModel})"}}`,
+      seconds: 0,
+    },
+  ])("reads a credential pool's earliest reset from $name", (input) =>
+    Effect.gen(function* () {
+      const harness = yield* makeWakeHarness;
+      const now = yield* DateTime.now;
+      yield* harness.runtime.startTurn(
+        makeClaudeTestTurnInput({
+          threadId: harness.threadId,
+          providerThread: harness.providerThread,
+          now,
+          attemptId: RunAttemptId.make("attempt-pool-cooldown"),
+          text: "Continue.",
+          attachments: [],
+        }),
+      );
+      yield* Queue.offerAll(harness.sdkMessages, [
+        makeAssistantErrorFrame({
+          uuid: "00000000-0000-4000-8000-000000000656",
+          error: input.error,
+          text: input.result,
+        }),
+        makeResultFrame({
+          uuid: "00000000-0000-4000-8000-000000000657",
+          result: input.result,
+          terminalReason: "api_error",
+          isError: true,
+          apiErrorStatus: input.status,
+        }),
+      ]);
+      const terminal = yield* Queue.take(harness.terminalReceipts);
+      assert.equal(terminal.status, "failed");
+      if (terminal.status !== "failed") return;
+      const usageLimit = input.seconds > 0 || input.usageLimit === true;
+      assert.equal(terminal.failure.class, usageLimit ? "usage_limit" : "provider_error");
+      assert.equal(terminal.failure.code, input.code);
+      assert.equal(
+        terminal.failure.resetAt ?? null,
+        input.seconds > 0
+          ? DateTime.formatIso(
+              DateTime.makeUnsafe(DateTime.toEpochMillis(now) + input.seconds * 1_000),
+            )
+          : null,
+      );
+      if (input.seconds > 0) assert.equal(terminal.failure.retryable, true);
+    }).pipe(
+      Effect.provide(
+        Layer.mergeAll(
+          IdAllocator.layer,
+          McpProviderSessions.layer,
+          NodeServices.layer,
+          NodeCrypto.layer,
+        ),
+      ),
+    ),
+  );
+
   it.effect("keeps the measured context when a usage limit ends the turn", () =>
     Effect.gen(function* () {
       const harness = yield* makeWakeHarness;

apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts

@@ -4091,19 +4091,53 @@ function claudeUsageLimitResetAt(resetsAt: unknown): string | null {
     : null;
 }
 
-function gatewayCooldownResetAt(
-  cooldown: RegExpExecArray | null,
-  completedAt: DateTime.Utc,
-): string | undefined {
-  if (cooldown === null) return undefined;
-  const seconds =
-    Number(cooldown[1] ?? 0) * 3_600 + Number(cooldown[2] ?? 0) * 60 + Number(cooldown[3] ?? 0);
-  const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000;
+const GATEWAY_WRAPPED_RATE_LIMIT =
+  /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",/;
+const GATEWAY_RESET_AFTER = /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/;
+// CLIProxyAPI's model_cooldown body; the CLI may relay it raw or JSON-escaped.
+const GATEWAY_RESET_SECONDS = /\\?"reset_seconds\\?"\s*:\s*(\d+)/;
+const GATEWAY_MODEL_COOLDOWN = /\\?"code\\?"\s*:\s*\\?"model_cooldown\\?"/;
+
+function resetAtAfterSeconds(seconds: number, receivedAt: DateTime.Utc): string | undefined {
+  const resetMs = DateTime.toEpochMillis(receivedAt) + seconds * 1_000;
   return seconds > 0 && resetMs < 8.64e15
     ? DateTime.formatIso(DateTime.makeUnsafe(resetMs))
     : undefined;
 }
 
+// Drops each "(last error: …)", which quotes one credential's error rather than
+// the pool's. It runs to the end of the JSON string holding it (the first quote
+// with no backslash before it), so fields after that string survive, or to the
+// end of rendered, non-JSON text.
+function withoutLastError(text: string): string {
+  let rest = text;
+  for (let start = rest.indexOf("(last error:"); start >= 0; start = rest.indexOf("(last error:")) {
+    let end = start;
+    do {
+      end = rest.indexOf('"', end + 1);
+    } while (end > 0 && rest[end - 1] === "\\");
+    rest = rest.slice(0, start) + (end < 0 ? "" : rest.slice(end));
+  }
+  return rest;
+}
+
+// A gateway names its earliest credential reset as "(reset after 1h2m3s)" text
+// or, from CLIProxyAPI, as a model_cooldown body's reset_seconds.
+function gatewayCooldownResetAt(text: string, receivedAt: DateTime.Utc): string | undefined {
+  const poolText = withoutLastError(text);
+  const cooldown = GATEWAY_RESET_AFTER.exec(poolText);
+  if (cooldown !== null) {
+    return resetAtAfterSeconds(
+      Number(cooldown[1] ?? 0) * 3_600 + Number(cooldown[2] ?? 0) * 60 + Number(cooldown[3] ?? 0),
+      receivedAt,
+    );
+  }
+  const resetSeconds = GATEWAY_RESET_SECONDS.exec(poolText);
+  return resetSeconds === null
+    ? undefined
+    : resetAtAfterSeconds(Number(resetSeconds[1]), receivedAt);
+}
+
 function providerFailureFromResult(
   message: SDKResultMessage,
   receivedAt: DateTime.Utc,
@@ -4115,13 +4149,9 @@ function providerFailureFromResult(
     message.subtype === "success" &&
     message.is_error &&
     message.terminal_reason === "api_error" &&
-    message.api_error_status === 503
-      ? gatewayCooldownResetAt(
-          /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(
-            message.result,
-          ),
-          receivedAt,
-        )
+    message.api_error_status === 503 &&
+    GATEWAY_WRAPPED_RATE_LIMIT.test(message.result)
+      ? gatewayCooldownResetAt(message.result, receivedAt)
       : undefined;
   if (gatewayResetAt !== undefined) {
     return makeProviderFailure({
@@ -4132,8 +4162,16 @@ function providerFailureFromResult(
       resetAt: gatewayResetAt,
     });
   }
+  // CLIProxyAPI reports an exhausted credential pool as model_cooldown with
+  // the earliest credential's reset, whatever status carries it.
+  const poolCooldownResetAt =
+    message.subtype === "success" && message.is_error && GATEWAY_MODEL_COOLDOWN.test(message.result)
+      ? gatewayCooldownResetAt(message.result, receivedAt)
+      : undefined;
   const failureClass =
-    (message.subtype === "success" && message.api_error_status === 429) || usageLimited
+    (message.subtype === "success" && message.api_error_status === 429) ||
+    usageLimited ||
+    poolCooldownResetAt !== undefined
       ? "usage_limit"
       : "provider_error";
   const listedError = resultUserFacingError(message);
@@ -4161,20 +4199,20 @@ function providerFailureFromResult(
   const apiErrorStatus = message.api_error_status ?? null;
   // API-key sessions get no rate_limit_info, so a gateway's 429 carries its reset only as text.
   const resetAt =
-    apiErrorStatus === 429 && message.result.startsWith("API Error:")
-      ? gatewayCooldownResetAt(
-          /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(message.result),
-          receivedAt,
-        )
-      : undefined;
+    apiErrorStatus === 429 && (message.is_error || message.result.startsWith("API Error:"))
+      ? gatewayCooldownResetAt(message.result, receivedAt)
+      : poolCooldownResetAt;
   return makeProviderFailure({
     message: listedError ?? structuredError ?? failureHint ?? message.result,
     code:
       apiErrorStatus === null
         ? (message.terminal_reason ?? "sdk_result_error")
         : `api_error_${apiErrorStatus}`,
     class: failureClass,
-    retryable: apiErrorStatus === 429 || apiErrorStatus === 529 ? true : null,
+    retryable:
+      apiErrorStatus === 429 || apiErrorStatus === 529 || poolCooldownResetAt !== undefined
+        ? true
+        : null,
     resetAt: resetAt ?? null,
   });
 }