From 455c30aa7d87ade1374d408ef44cb8a8aeaf34bb Mon Sep 17 00:00:00 2001 From: Vadym O Date: Fri, 16 Oct 2026 11:37:00 +0800 Subject: [PATCH 1/2] fix(codex): record a 402 account move as a quota recovery A Codex 402 move was recorded as oauth-account-429, so request metrics counted it as rate_limit and the dashboard labelled it an OAuth 429. Add oauth-account-402 mapped to quota-exhausted and wire it into the cooldown set, the dashboard label map and all locale catalogs. Co-Authored-By: Claude Code --- gui/src/i18n/de.ts | 1 + gui/src/i18n/en.ts | 1 + gui/src/i18n/fr.ts | 1 + gui/src/i18n/ja.ts | 1 + gui/src/i18n/ko.ts | 1 + gui/src/i18n/pt.ts | 1 + gui/src/i18n/ru.ts | 1 + gui/src/i18n/tr.ts | 1 + gui/src/i18n/vi.ts | 1 + gui/src/i18n/zh-TW.ts | 1 + gui/src/i18n/zh.ts | 1 + gui/src/pages/Logs.tsx | 1 + src/lib/request-failure-model.ts | 2 ++ src/routing/analytics.ts | 1 + src/server/responses/core-codex-account.ts | 4 +-- src/usage/telemetry-contract.ts | 1 + tests/lib/failure-stage-model.test.ts | 4 ++- .../responses/responses-account-label.test.ts | 34 ++++++++++++++++++- 18 files changed, 54 insertions(+), 4 deletions(-) diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 3b784e086f7..22c5edac19d 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -1148,6 +1148,7 @@ export const de: Record = { "logs.detail.attempt.recovery.key401": "API-Schlüssel erneut authentifiziert", "logs.detail.attempt.recovery.oauthAccount403": "Kontovalidierung erforderlich (403)", "logs.detail.attempt.recovery.oauthAccount429": "Konto rate-limitiert (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Kontingent des Kontos erschöpft (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Veralteter verschlüsselter Zustand verworfen", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Reasoning-Aufwand reduziert", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Schnellmodus abgelehnt, erneut mit Standardtempo versucht", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index a10c82ade62..350a21304ae 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -1206,6 +1206,7 @@ export const en = { "logs.detail.attempt.recovery.key401": "API key re-authentication", "logs.detail.attempt.recovery.oauthAccount403": "Account validation required (403)", "logs.detail.attempt.recovery.oauthAccount429": "Account rate-limited (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Account quota exhausted (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Stale encrypted state dropped", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Reasoning effort downgraded", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Fast mode refused, retried at standard speed", diff --git a/gui/src/i18n/fr.ts b/gui/src/i18n/fr.ts index d61681fcfc9..a17e39b0bd0 100644 --- a/gui/src/i18n/fr.ts +++ b/gui/src/i18n/fr.ts @@ -1182,6 +1182,7 @@ export const fr: Record = { "logs.detail.attempt.recovery.key401": "Ré-authentification de la clé API", "logs.detail.attempt.recovery.oauthAccount403": "Validation du compte requise (403)", "logs.detail.attempt.recovery.oauthAccount429": "Compte limité (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Quota du compte épuisé (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "État chiffré obsolète abandonné", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Effort de raisonnement réduit", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Mode rapide refusé, nouvel essai à vitesse standard", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 1ae152fdee0..08799fa8f3f 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -1089,6 +1089,7 @@ export const ja: Record = { "logs.detail.attempt.recovery.key401": "API キーの再認証", "logs.detail.attempt.recovery.oauthAccount403": "アカウントの検証が必要 (403)", "logs.detail.attempt.recovery.oauthAccount429": "アカウントのレート制限 (429)", + "logs.detail.attempt.recovery.oauthAccount402": "アカウントのクォータ枯渇 (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "古い暗号化状態を破棄", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "推論強度を下げて再試行", "logs.detail.attempt.recovery.anthropicFastDowngrade": "高速モードが拒否され、標準速度で再試行", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index a7faa3c9e47..0781408ca97 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -1187,6 +1187,7 @@ export const ko: Record = { "logs.detail.attempt.recovery.key401": "API 키 재인증", "logs.detail.attempt.recovery.oauthAccount403": "계정 확인 필요 (403)", "logs.detail.attempt.recovery.oauthAccount429": "계정 속도 제한 (429)", + "logs.detail.attempt.recovery.oauthAccount402": "계정 할당량 소진 (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "만료된 암호화 상태 제거", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "추론 강도 하향", "logs.detail.attempt.recovery.anthropicFastDowngrade": "빠른 모드 거부, 표준 속도로 재시도", diff --git a/gui/src/i18n/pt.ts b/gui/src/i18n/pt.ts index dac0f93fdc8..5d0f1dd1761 100644 --- a/gui/src/i18n/pt.ts +++ b/gui/src/i18n/pt.ts @@ -1201,6 +1201,7 @@ export const pt: Record = { "logs.detail.attempt.recovery.key401": "Reautenticação da chave de API", "logs.detail.attempt.recovery.oauthAccount403": "Validação da conta necessária (403)", "logs.detail.attempt.recovery.oauthAccount429": "Conta com limite de taxa (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Cota da conta esgotada (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Estado criptografado obsoleto descartado", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Esforço de raciocínio reduzido", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Modo rápido recusado, repetido em velocidade padrão", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 8603d20b578..ad8863b0cce 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -1165,6 +1165,7 @@ export const ru: Record = { "logs.detail.attempt.recovery.key401": "Повторная аутентификация API-ключа", "logs.detail.attempt.recovery.oauthAccount403": "Требуется проверка аккаунта (403)", "logs.detail.attempt.recovery.oauthAccount429": "Ограничение частоты для аккаунта (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Квота аккаунта исчерпана (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Устаревшее зашифрованное состояние отброшено", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Уровень рассуждения понижен", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Быстрый режим отклонён, повтор со стандартной скоростью", diff --git a/gui/src/i18n/tr.ts b/gui/src/i18n/tr.ts index 008e8be47e4..147e954d1c9 100644 --- a/gui/src/i18n/tr.ts +++ b/gui/src/i18n/tr.ts @@ -1192,6 +1192,7 @@ export const tr: Record = { "logs.detail.attempt.recovery.key401": "API anahtarı yeniden doğrulandı", "logs.detail.attempt.recovery.oauthAccount403": "Hesap doğrulaması gerekli (403)", "logs.detail.attempt.recovery.oauthAccount429": "Hesap hız sınırına takıldı (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Hesap kotası tükendi (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Eski şifreli durum bırakıldı", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Akıl yürütme düzeyi düşürüldü", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Hızlı mod reddedildi, standart hızda yeniden denendi", diff --git a/gui/src/i18n/vi.ts b/gui/src/i18n/vi.ts index 0f432ac4923..18fa8b93963 100644 --- a/gui/src/i18n/vi.ts +++ b/gui/src/i18n/vi.ts @@ -1186,6 +1186,7 @@ export const vi: Record = { "logs.detail.attempt.recovery.key401": "Xác thực lại khóa API", "logs.detail.attempt.recovery.oauthAccount403": "Cần xác minh tài khoản (403)", "logs.detail.attempt.recovery.oauthAccount429": "Tài khoản bị giới hạn tần suất (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Tài khoản đã hết hạn mức (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Đã bỏ trạng thái mã hóa cũ", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Đã giảm mức suy luận", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Chế độ nhanh bị từ chối, thử lại ở tốc độ chuẩn", diff --git a/gui/src/i18n/zh-TW.ts b/gui/src/i18n/zh-TW.ts index 1bc86ff154c..783ea79a4fc 100644 --- a/gui/src/i18n/zh-TW.ts +++ b/gui/src/i18n/zh-TW.ts @@ -3002,6 +3002,7 @@ export const zhTW: Record = { "logs.detail.attempt.recovery.key401": "API 金鑰重新驗證", "logs.detail.attempt.recovery.oauthAccount403": "需要驗證帳號 (403)", "logs.detail.attempt.recovery.oauthAccount429": "帳號速率受限 (429)", + "logs.detail.attempt.recovery.oauthAccount402": "帳號配額已用盡 (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "已捨棄過期加密狀態", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "已降低推理強度", "logs.detail.attempt.recovery.anthropicFastDowngrade": "快速模式遭拒，改以標準速度重試", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index a0240c48491..dacb18d70ea 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -1160,6 +1160,7 @@ export const zh: Record = { "logs.detail.attempt.recovery.key401": "API 密钥重新认证", "logs.detail.attempt.recovery.oauthAccount403": "需要验证账号 (403)", "logs.detail.attempt.recovery.oauthAccount429": "账号速率受限 (429)", + "logs.detail.attempt.recovery.oauthAccount402": "账号配额已用尽 (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "已丢弃过期加密状态", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "已降低推理强度", "logs.detail.attempt.recovery.anthropicFastDowngrade": "快速模式被拒，按标准速度重试", diff --git a/gui/src/pages/Logs.tsx b/gui/src/pages/Logs.tsx index 922634b69ec..a1f5858f394 100644 --- a/gui/src/pages/Logs.tsx +++ b/gui/src/pages/Logs.tsx @@ -355,6 +355,7 @@ const RECOVERY_KIND_KEYS = { "rate-limit-429": "logs.detail.attempt.recovery.rateLimit429", "anthropic-oauth-429": "logs.detail.attempt.recovery.anthropicOauth429", "oauth-account-429": "logs.detail.attempt.recovery.oauthAccount429", + "oauth-account-402": "logs.detail.attempt.recovery.oauthAccount402", "image-413": "logs.detail.attempt.recovery.image413", "empty-completion": "logs.detail.attempt.recovery.emptyCompletion", "console-go-upload-retry": "logs.detail.attempt.recovery.consoleGoUpload", diff --git a/src/lib/request-failure-model.ts b/src/lib/request-failure-model.ts index 7ac0cb9ba61..dc924ac2874 100644 --- a/src/lib/request-failure-model.ts +++ b/src/lib/request-failure-model.ts @@ -223,6 +223,8 @@ const RECOVERY_KIND_CAUSE = { "rate-limit-429": "rate-limit", "anthropic-oauth-429": "rate-limit", "oauth-account-429": "rate-limit", + // A 402 is the account's quota, not a transient limit; only another account can serve it. + "oauth-account-402": "quota-exhausted", "image-413": "payload-too-large", // The gateway rejects a body it accepts seconds later and the replay is byte-identical, so // nothing about the payload was wrong; the origin declined to take it at that moment. diff --git a/src/routing/analytics.ts b/src/routing/analytics.ts index 5c4411c3b8c..3eb2e540a14 100644 --- a/src/routing/analytics.ts +++ b/src/routing/analytics.ts @@ -122,6 +122,7 @@ const COOLDOWN_RECOVERY_KINDS = new Set([ "oauth-account-403", "anthropic-oauth-429", "oauth-account-429", + "oauth-account-402", ]); function percentile(sorted: number[], p: number): number | undefined { diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index de3152da035..7f73f04c4ce 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -975,8 +975,8 @@ export async function retryCodexPoolOnAlternateAccount( const maxRetrySends = retrySameConfirmedAccount ? 7 : 1; let retrySendCount = 0; // The gated-model 400 has no roster kind; it stays unattributed as before. - const moveRecovery: AttemptRecoveryKind | undefined = outcomeStatus === 429 || outcomeStatus === 402 - ? "oauth-account-429" + const moveRecovery: AttemptRecoveryKind | undefined = outcomeStatus === 429 ? "oauth-account-429" + : outcomeStatus === 402 ? "oauth-account-402" : isTransientUpstreamStatus(outcomeStatus) ? "transient-5xx" : undefined; let upstreamResponse: Response; try { diff --git a/src/usage/telemetry-contract.ts b/src/usage/telemetry-contract.ts index 157d372f7c7..69247569cdf 100644 --- a/src/usage/telemetry-contract.ts +++ b/src/usage/telemetry-contract.ts @@ -32,6 +32,7 @@ export const ATTEMPT_RECOVERY_KIND_ROSTER = Object.freeze([ "rate-limit-429", "anthropic-oauth-429", "oauth-account-429", + "oauth-account-402", "image-413", "console-go-upload-retry", "opaque-blob-rejection", diff --git a/tests/lib/failure-stage-model.test.ts b/tests/lib/failure-stage-model.test.ts index 893f85b6178..078942991fd 100644 --- a/tests/lib/failure-stage-model.test.ts +++ b/tests/lib/failure-stage-model.test.ts @@ -199,8 +199,10 @@ describe("recovery kinds speak the shared dictionary", () => { * the durable log were forbidden and unfunded. */ test("no recorded recovery classifies as a resend that cannot help", () => { + // Except the Codex 402 account move: re-asking the exhausted account is futile, and the move + // to another account is a route decision this table explicitly does not make. const futile = ATTEMPT_RECOVERY_KIND_ROSTER .filter(kind => causeDisposition(causeForRecoveryKind(kind)) === "resend-is-futile"); - expect(futile).toEqual([]); + expect(futile).toEqual(["oauth-account-402"]); }); }); diff --git a/tests/responses/responses-account-label.test.ts b/tests/responses/responses-account-label.test.ts index c127c0ae2c0..a88745c707c 100644 --- a/tests/responses/responses-account-label.test.ts +++ b/tests/responses/responses-account-label.test.ts @@ -11,7 +11,8 @@ import { clearThreadAccountMap, getCodexUpstreamHealth, } from "../../src/codex/routing"; -import type { RequestLogContext } from "../../src/server/request-log"; +import { addFinalRequestLog, type RequestLogContext } from "../../src/server/request-log"; +import { createRequestMetricsOwner } from "../../src/server/request-metrics"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -336,6 +337,37 @@ describe("Responses account usage attribution", () => { }); }); + test("a pre-stream 402 move is recorded and counted as a quota recovery, not a rate limit", async () => { + await withPoolHome(async () => { + takeSpendHome(); + const config = poolConfig(["pool-a", "pool-b"]); + for (const id of ["pool-a", "pool-b"]) { + savePoolCredential(id); + updateAccountQuota(id, id === "pool-a" ? 10 : 20); + } + let sends = 0; + globalThis.fetch = (async () => ++sends === 1 + ? Response.json({ error: { message: "payment required" } }, { status: 402 }) + : completedResponse("pool-b-response")) as typeof fetch; + + const metrics = createRequestMetricsOwner(123); + const logCtx: RequestLogContext = { model: "", provider: "", inboundProtocol: "responses", requestMetricsRecorder: metrics }; + const response = await handleResponses(request(), config, logCtx, {}); + expect(response.status).toBe(200); + expect(logCtx.attempts?.map(attempt => [attempt.status, attempt.recoveryKinds])).toEqual([ + [402, []], + [0, ["oauth-account-402"]], + ]); + + addFinalRequestLog("codex-402-move", Date.now() - 1_000, logCtx, 200, { + terminalStatus: "completed", closeReason: "terminal", + }, () => {}); + const recoveries = metrics.snapshot().split("\n").filter(line => line.startsWith("opencodex_recoveries_total{")); + expect(recoveries.find(line => line.includes('recovery="quota"'))?.endsWith(" 1")).toBe(true); + expect(recoveries.find(line => line.includes('recovery="rate_limit"'))?.endsWith(" 0") ?? true).toBe(true); + }); + }); + test("a quota message wrapped in HTTP 502 cools the account and retries an alternate", async () => { await withPoolHome(async () => { takeSpendHome(); From 9ea4be7a6a40d60c0c9ce2c040ab7c12dc0d5ad9 Mon Sep 17 00:00:00 2001 From: Vadym O Date: Fri, 16 Oct 2026 17:01:29 +0800 Subject: [PATCH 2/2] docs(responses): split Codex 402 quota from 429 rate-limit recovery Co-Authored-By: Claude Code --- structure/transports/responses.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/structure/transports/responses.md b/structure/transports/responses.md index b1e586d3e89..81419262319 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -434,7 +434,7 @@ The continuation-note and continuation-wire tests establish prompt and wire cont Reusable helpers live in `core-auth.ts`, `core-codex-account.ts`, `core-combo.ts`, `core-combo-failure.ts`, `core-combo-native.ts`, `combo-requested-effort.ts`, `core-errors.ts`, `core-lifetime.ts`, `core-normalize.ts`, `core-opaque-recovery.ts` and `core-replay.ts`. `core-options.ts` owns the public option types -and small composition contracts, including [finite model-refusal evidence](../providers-and-adapters.md#combo-model-refusal-evidence). Existing public helper names are re-exported by `core.ts`. After an admitted Anthropic OAuth pool rotation, `adapter-dispatch.ts` rebinds reasoning replay to the committed credential snapshot; `adapter-continuation.ts` rebinds both its continuation clone and the outer request that persists the response. The replacement send strips the previous account's private state, while persisted ownership and the next turn follow the serving account. Codex alternate-account sends in `core-codex-account.ts` record `oauth-account-429` for quota recovery and `transient-5xx` for transient recovery, preserving the first response's wire status when sealing its attempt; gated-model moves keep their existing unattributed kind. +and small composition contracts, including [finite model-refusal evidence](../providers-and-adapters.md#combo-model-refusal-evidence). Existing public helper names are re-exported by `core.ts`. After an admitted Anthropic OAuth pool rotation, `adapter-dispatch.ts` rebinds reasoning replay to the committed credential snapshot; `adapter-continuation.ts` rebinds both its continuation clone and the outer request that persists the response. The replacement send strips the previous account's private state, while persisted ownership and the next turn follow the serving account. Codex alternate-account sends in `core-codex-account.ts` record `oauth-account-429` for rate-limit recovery, `oauth-account-402` for quota recovery and `transient-5xx` for transient recovery, preserving the first response's wire status when sealing its attempt; gated-model moves keep their existing unattributed kind. Adapter construction remains with the existing registry; `fetch-helpers.ts` remains a leaf. For Kiro OAuth with load settings, `request-transport.ts` acquires a lease on the admitted account and transfers it before a reactive replacement send; cancellation permanently fences the request holder so recovery cannot install a late lease after abort cleanup. `core.ts` and `core-lifetime.ts` release the lease on returned-body completion, error, or cancellation, outside the inner admission `finally`. Mutable values are not copied across phases. A phase exposes only the values consumed by later