Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions gui/src/i18n/de.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1148,6 +1148,7 @@ export const de: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "API-Schlüssel erneut authentifiziert",
"logs.detail.attempt.recovery.oauthAccount403": "Kontovalidierung erforderlich (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Konto rate-limitiert (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Kontingent des Kontos erschöpft (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "Veralteter verschlüsselter Zustand verworfen",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Reasoning-Aufwand reduziert",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Schnellmodus abgelehnt, erneut mit Standardtempo versucht",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/en.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1206,6 +1206,7 @@ export const en = {
"logs.detail.attempt.recovery.key401": "API key re-authentication",
"logs.detail.attempt.recovery.oauthAccount403": "Account validation required (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Account rate-limited (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Account quota exhausted (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "Stale encrypted state dropped",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Reasoning effort downgraded",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Fast mode refused, retried at standard speed",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/fr.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1182,6 +1182,7 @@ export const fr: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "Ré-authentification de la clé API",
"logs.detail.attempt.recovery.oauthAccount403": "Validation du compte requise (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Compte limité (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Quota du compte épuisé (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "État chiffré obsolète abandonné",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Effort de raisonnement réduit",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Mode rapide refusé, nouvel essai à vitesse standard",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/ja.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1089,6 +1089,7 @@ export const ja: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "API キーの再認証",
"logs.detail.attempt.recovery.oauthAccount403": "アカウントの検証が必要 (403)",
"logs.detail.attempt.recovery.oauthAccount429": "アカウントのレート制限 (429)",
"logs.detail.attempt.recovery.oauthAccount402": "アカウントのクォータ枯渇 (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "古い暗号化状態を破棄",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "推論強度を下げて再試行",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "高速モードが拒否され、標準速度で再試行",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/ko.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1187,6 +1187,7 @@ export const ko: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "API 키 재인증",
"logs.detail.attempt.recovery.oauthAccount403": "계정 확인 필요 (403)",
"logs.detail.attempt.recovery.oauthAccount429": "계정 속도 제한 (429)",
"logs.detail.attempt.recovery.oauthAccount402": "계정 할당량 소진 (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "만료된 암호화 상태 제거",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "추론 강도 하향",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "빠른 모드 거부, 표준 속도로 재시도",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/pt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1201,6 +1201,7 @@ export const pt: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "Reautenticação da chave de API",
"logs.detail.attempt.recovery.oauthAccount403": "Validação da conta necessária (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Conta com limite de taxa (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Cota da conta esgotada (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "Estado criptografado obsoleto descartado",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Esforço de raciocínio reduzido",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Modo rápido recusado, repetido em velocidade padrão",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/ru.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1165,6 +1165,7 @@ export const ru: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "Повторная аутентификация API-ключа",
"logs.detail.attempt.recovery.oauthAccount403": "Требуется проверка аккаунта (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Ограничение частоты для аккаунта (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Квота аккаунта исчерпана (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "Устаревшее зашифрованное состояние отброшено",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Уровень рассуждения понижен",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Быстрый режим отклонён, повтор со стандартной скоростью",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/tr.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1192,6 +1192,7 @@ export const tr: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "API anahtarı yeniden doğrulandı",
"logs.detail.attempt.recovery.oauthAccount403": "Hesap doğrulaması gerekli (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Hesap hız sınırına takıldı (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Hesap kotası tükendi (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "Eski şifreli durum bırakıldı",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Akıl yürütme düzeyi düşürüldü",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Hızlı mod reddedildi, standart hızda yeniden denendi",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/vi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1186,6 +1186,7 @@ export const vi: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "Xác thực lại khóa API",
"logs.detail.attempt.recovery.oauthAccount403": "Cần xác minh tài khoản (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Tài khoản bị giới hạn tần suất (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Tài khoản đã hết hạn mức (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "Đã bỏ trạng thái mã hóa cũ",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Đã giảm mức suy luận",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Chế độ nhanh bị từ chối, thử lại ở tốc độ chuẩn",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/zh-TW.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3002,6 +3002,7 @@ export const zhTW: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "API 金鑰重新驗證",
"logs.detail.attempt.recovery.oauthAccount403": "需要驗證帳號 (403)",
"logs.detail.attempt.recovery.oauthAccount429": "帳號速率受限 (429)",
"logs.detail.attempt.recovery.oauthAccount402": "帳號配額已用盡 (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "已捨棄過期加密狀態",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "已降低推理強度",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "快速模式遭拒,改以標準速度重試",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/zh.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1160,6 +1160,7 @@ export const zh: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "API 密钥重新认证",
"logs.detail.attempt.recovery.oauthAccount403": "需要验证账号 (403)",
"logs.detail.attempt.recovery.oauthAccount429": "账号速率受限 (429)",
"logs.detail.attempt.recovery.oauthAccount402": "账号配额已用尽 (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "已丢弃过期加密状态",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "已降低推理强度",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "快速模式被拒,按标准速度重试",
Expand Down
1 change: 1 addition & 0 deletions gui/src/pages/Logs.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,7 @@ const RECOVERY_KIND_KEYS = {
"rate-limit-429": "logs.detail.attempt.recovery.rateLimit429",
"anthropic-oauth-429": "logs.detail.attempt.recovery.anthropicOauth429",
"oauth-account-429": "logs.detail.attempt.recovery.oauthAccount429",
"oauth-account-402": "logs.detail.attempt.recovery.oauthAccount402",
"image-413": "logs.detail.attempt.recovery.image413",
"empty-completion": "logs.detail.attempt.recovery.emptyCompletion",
"console-go-upload-retry": "logs.detail.attempt.recovery.consoleGoUpload",
Expand Down
2 changes: 2 additions & 0 deletions src/lib/request-failure-model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,8 @@ const RECOVERY_KIND_CAUSE = {
"rate-limit-429": "rate-limit",
"anthropic-oauth-429": "rate-limit",
"oauth-account-429": "rate-limit",
// A 402 is the account's quota, not a transient limit; only another account can serve it.
"oauth-account-402": "quota-exhausted",
"image-413": "payload-too-large",
// The gateway rejects a body it accepts seconds later and the replay is byte-identical, so
// nothing about the payload was wrong; the origin declined to take it at that moment.
Expand Down
1 change: 1 addition & 0 deletions src/routing/analytics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ const COOLDOWN_RECOVERY_KINDS = new Set([
"oauth-account-403",
"anthropic-oauth-429",
"oauth-account-429",
"oauth-account-402",
]);

function percentile(sorted: number[], p: number): number | undefined {
Expand Down
4 changes: 2 additions & 2 deletions src/server/responses/core-codex-account.ts
Original file line number Diff line number Diff line change
Expand Up @@ -975,8 +975,8 @@ export async function retryCodexPoolOnAlternateAccount(
const maxRetrySends = retrySameConfirmedAccount ? 7 : 1;
let retrySendCount = 0;
// The gated-model 400 has no roster kind; it stays unattributed as before.
const moveRecovery: AttemptRecoveryKind | undefined = outcomeStatus === 429 || outcomeStatus === 402
? "oauth-account-429"
const moveRecovery: AttemptRecoveryKind | undefined = outcomeStatus === 429 ? "oauth-account-429"
: outcomeStatus === 402 ? "oauth-account-402"
: isTransientUpstreamStatus(outcomeStatus) ? "transient-5xx" : undefined;
let upstreamResponse: Response;
try {
Expand Down
1 change: 1 addition & 0 deletions src/usage/telemetry-contract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ export const ATTEMPT_RECOVERY_KIND_ROSTER = Object.freeze([
"rate-limit-429",
"anthropic-oauth-429",
"oauth-account-429",
"oauth-account-402",
"image-413",
"console-go-upload-retry",
"opaque-blob-rejection",
Expand Down
2 changes: 1 addition & 1 deletion structure/transports/responses.md
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,7 @@ The continuation-note and continuation-wire tests establish prompt and wire cont
Reusable helpers live in `core-auth.ts`, `core-codex-account.ts`, `core-combo.ts`,
`core-combo-failure.ts`, `core-combo-native.ts`, `combo-requested-effort.ts`, `core-errors.ts`, `core-lifetime.ts`, `core-normalize.ts`,
`core-opaque-recovery.ts` and `core-replay.ts`. `core-options.ts` owns the public option types
and small composition contracts, including [finite model-refusal evidence](../providers-and-adapters.md#combo-model-refusal-evidence). Existing public helper names are re-exported by `core.ts`. After an admitted Anthropic OAuth pool rotation, `adapter-dispatch.ts` rebinds reasoning replay to the committed credential snapshot; `adapter-continuation.ts` rebinds both its continuation clone and the outer request that persists the response. The replacement send strips the previous account's private state, while persisted ownership and the next turn follow the serving account. Codex alternate-account sends in `core-codex-account.ts` record `oauth-account-429` for quota recovery and `transient-5xx` for transient recovery, preserving the first response's wire status when sealing its attempt; gated-model moves keep their existing unattributed kind.
and small composition contracts, including [finite model-refusal evidence](../providers-and-adapters.md#combo-model-refusal-evidence). Existing public helper names are re-exported by `core.ts`. After an admitted Anthropic OAuth pool rotation, `adapter-dispatch.ts` rebinds reasoning replay to the committed credential snapshot; `adapter-continuation.ts` rebinds both its continuation clone and the outer request that persists the response. The replacement send strips the previous account's private state, while persisted ownership and the next turn follow the serving account. Codex alternate-account sends in `core-codex-account.ts` record `oauth-account-429` for rate-limit recovery, `oauth-account-402` for quota recovery and `transient-5xx` for transient recovery, preserving the first response's wire status when sealing its attempt; gated-model moves keep their existing unattributed kind.
Adapter construction remains with the existing registry; `fetch-helpers.ts` remains a leaf. For Kiro OAuth with load settings, `request-transport.ts` acquires a lease on the admitted account and transfers it before a reactive replacement send; cancellation permanently fences the request holder so recovery cannot install a late lease after abort cleanup. `core.ts` and `core-lifetime.ts` release the lease on returned-body completion, error, or cancellation, outside the inner admission `finally`.

Mutable values are not copied across phases. A phase exposes only the values consumed by later
Expand Down
4 changes: 3 additions & 1 deletion tests/lib/failure-stage-model.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -199,8 +199,10 @@ describe("recovery kinds speak the shared dictionary", () => {
* the durable log were forbidden and unfunded.
*/
test("no recorded recovery classifies as a resend that cannot help", () => {
// Except the Codex 402 account move: re-asking the exhausted account is futile, and the move
// to another account is a route decision this table explicitly does not make.
const futile = ATTEMPT_RECOVERY_KIND_ROSTER
.filter(kind => causeDisposition(causeForRecoveryKind(kind)) === "resend-is-futile");
expect(futile).toEqual([]);
expect(futile).toEqual(["oauth-account-402"]);
});
});
34 changes: 33 additions & 1 deletion tests/responses/responses-account-label.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ import {
clearThreadAccountMap,
getCodexUpstreamHealth,
} from "../../src/codex/routing";
import type { RequestLogContext } from "../../src/server/request-log";
import { addFinalRequestLog, type RequestLogContext } from "../../src/server/request-log";
import { createRequestMetricsOwner } from "../../src/server/request-metrics";
import { handleResponses } from "../../src/server/responses";
import type { OcxConfig } from "../../src/types";
import { removeTreeWithRetry } from "../helpers/remove-tree";
Expand Down Expand Up @@ -336,6 +337,37 @@ describe("Responses account usage attribution", () => {
});
});

test("a pre-stream 402 move is recorded and counted as a quota recovery, not a rate limit", async () => {
await withPoolHome(async () => {
takeSpendHome();
const config = poolConfig(["pool-a", "pool-b"]);
for (const id of ["pool-a", "pool-b"]) {
savePoolCredential(id);
updateAccountQuota(id, id === "pool-a" ? 10 : 20);
}
let sends = 0;
globalThis.fetch = (async () => ++sends === 1
? Response.json({ error: { message: "payment required" } }, { status: 402 })
: completedResponse("pool-b-response")) as typeof fetch;

const metrics = createRequestMetricsOwner(123);
const logCtx: RequestLogContext = { model: "", provider: "", inboundProtocol: "responses", requestMetricsRecorder: metrics };
const response = await handleResponses(request(), config, logCtx, {});
expect(response.status).toBe(200);
expect(logCtx.attempts?.map(attempt => [attempt.status, attempt.recoveryKinds])).toEqual([
[402, []],
[0, ["oauth-account-402"]],
]);

addFinalRequestLog("codex-402-move", Date.now() - 1_000, logCtx, 200, {
terminalStatus: "completed", closeReason: "terminal",
}, () => {});
const recoveries = metrics.snapshot().split("\n").filter(line => line.startsWith("opencodex_recoveries_total{"));
expect(recoveries.find(line => line.includes('recovery="quota"'))?.endsWith(" 1")).toBe(true);
expect(recoveries.find(line => line.includes('recovery="rate_limit"'))?.endsWith(" 0") ?? true).toBe(true);
});
});

test("a quota message wrapped in HTTP 502 cools the account and retries an alternate", async () => {
await withPoolHome(async () => {
takeSpendHome();
Expand Down
Loading