From ebcc5ee15baccd7af5bf25f918b6a1f520704e67 Mon Sep 17 00:00:00 2001 From: Jai Khurana <64320051+jaikhuranna@users.noreply.github.com> Date: Sun, 18 Oct 2026 09:17:27 +0530 Subject: [PATCH] fix(server): Codex stops keep their reset time when the last usage update reads 99% Codex reports whole percentages, so a turn can be refused while the window that ran out still reads 99%. The reset helper only accepted windows at 100%, so those stops lost a reset the server already had and clients offered manual retry instead of auto-resume. When no window reads 100% and exactly one reads 99%, use that window's reset. If several read 99% it is unclear which ran out, so the stop waits for an update at 100% as before. Windows whose reset has passed no longer count. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/CodexAdapterV2.test.ts | 73 ++++++++++++++- .../Adapters/CodexAdapterV2.ts | 15 ++- .../src/provider/codexUsageLimits.test.ts | 91 +++++++++++++++++-- apps/server/src/provider/codexUsageLimits.ts | 19 +++- 4 files changed, 179 insertions(+), 19 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 6e4a6494fb78f..77f6de47c4f8e 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -6592,6 +6592,24 @@ describe("CodexAdapterV2 post-settle continuation", () => { notification: false, expectedClass: "usage_limit", }, + { + name: "rounded-reset", + code: "usageLimitExceeded", + notification: false, + expectedClass: "usage_limit", + }, + { + name: "late-rounded-reset", + code: "usageLimitExceeded", + notification: false, + expectedClass: "usage_limit", + }, + { + name: "ambiguous-late-reset", + code: "usageLimitExceeded", + notification: false, + expectedClass: "usage_limit", + }, { name: "late-reset", code: "usageLimitExceeded", @@ -6626,7 +6644,11 @@ describe("CodexAdapterV2 post-settle continuation", () => { params: { rateLimits: { limitId: "codex", - primary: { usedPercent: 100, resetsAt: 2000100000, windowDurationMins: 300 }, + primary: { + usedPercent: scenario.name.includes("rounded") ? 99 : 100, + resetsAt: 2000100000, + windowDurationMins: 300, + }, }, }, }, @@ -6635,9 +6657,37 @@ describe("CodexAdapterV2 post-settle continuation", () => { scenario: `codex-limit-${scenario.name}`, entries: [ ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Continue." }), - ...(scenario.name === "known-reset" || scenario.name === "deferred-reset" + ...(scenario.name === "known-reset" || + scenario.name === "deferred-reset" || + scenario.name === "rounded-reset" ? [snapshot] : []), + // Both windows read 99%, so the stop cannot tell which one ran out. + ...(scenario.name === "ambiguous-late-reset" + ? [ + { + ...snapshot, + frame: { + method: "account/rateLimits/updated", + params: { + rateLimits: { + limitId: "codex", + primary: { + usedPercent: 99, + resetsAt: 2000100000, + windowDurationMins: 300, + }, + secondary: { + usedPercent: 99, + resetsAt: 2000200000, + windowDurationMins: 10080, + }, + }, + }, + }, + }, + ] + : []), ...(scenario.name === "deferred-reset" ? [ { @@ -6717,7 +6767,11 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }, }, - ...(scenario.name === "late-reset" ? [snapshot] : []), + ...(scenario.name === "late-reset" || + scenario.name === "late-rounded-reset" || + scenario.name === "ambiguous-late-reset" + ? [snapshot] + : []), ...(scenario.name === "deferred-reset" ? [ { @@ -6777,11 +6831,20 @@ describe("CodexAdapterV2 post-settle continuation", () => { if (terminal?.status !== "failed") return; assert.equal(terminal.failure.class, scenario.expectedClass); assert.equal(terminal.threadDisposition, "reusable"); - if (scenario.name === "known-reset" || scenario.name === "deferred-reset") + if ( + scenario.name === "known-reset" || + scenario.name === "deferred-reset" || + scenario.name === "rounded-reset" + ) assert.equal(terminal.failure.resetAt, resetAt); if (scenario.name === "matching-details") assert.equal(terminal.failure.message, "Detailed provider allowance explanation."); - if (scenario.name === "late-reset") { + if (scenario.name === "ambiguous-late-reset") assert.isNull(terminal.failure.resetAt); + if ( + scenario.name === "late-reset" || + scenario.name === "late-rounded-reset" || + scenario.name === "ambiguous-late-reset" + ) { yield* Deferred.await(resetReceipt); const item = harness.events.find( (event) => diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 19d1e7b7ffe29..30651f6e7e137 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -4312,7 +4312,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* Ref.update(rateLimitSnapshot, (previous) => mergeCodexRateLimits(previous, payload.rateLimits), ); - const resetAt = codexUsageLimitResetAt(yield* Ref.get(rateLimitSnapshot)); + const resetAt = codexUsageLimitResetAt( + yield* Ref.get(rateLimitSnapshot), + DateTime.formatIso(yield* DateTime.now), + ); for (const item of (yield* Ref.get(limitedTurnItems)).values()) { if (resetAt === null || item.failure.resetAt != null) continue; const updated = { @@ -5554,7 +5557,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( failure.class === "usage_limit" ? { ...failure, - resetAt: codexUsageLimitResetAt(yield* Ref.get(rateLimitSnapshot)), + resetAt: codexUsageLimitResetAt( + yield* Ref.get(rateLimitSnapshot), + DateTime.formatIso(yield* DateTime.now), + ), } : failure, ...(input.providerRetry === undefined @@ -5591,7 +5597,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ...event.failure, resetAt: event.failure.resetAt ?? - codexUsageLimitResetAt(yield* Ref.get(rateLimitSnapshot)), + codexUsageLimitResetAt( + yield* Ref.get(rateLimitSnapshot), + DateTime.formatIso(yield* DateTime.now), + ), }, } : event; diff --git a/apps/server/src/provider/codexUsageLimits.test.ts b/apps/server/src/provider/codexUsageLimits.test.ts index 8d310a9fff730..9d25c19855937 100644 --- a/apps/server/src/provider/codexUsageLimits.test.ts +++ b/apps/server/src/provider/codexUsageLimits.test.ts @@ -7,6 +7,7 @@ import { codexRateLimitsToUpdate, codexResetCreditsToContract, codexUsageLimitMessage, + codexUsageLimitResetAt, mergeCodexRateLimits, } from "./codexUsageLimits.ts"; @@ -298,19 +299,93 @@ describe("mergeCodexRateLimits", () => { }); }); -import { codexUsageLimitResetAt } from "./codexUsageLimits.ts"; - describe("codexUsageLimitResetAt", () => { + const at = "2026-10-17T21:52:06.961Z"; + const resetAt = "2026-10-18T01:42:09.000Z"; + + it("keeps the reset when the last whole-percent update before refusal reads 99%", () => { + const snapshot = mergeCodexRateLimits( + { + limitId: "codex", + primary: { usedPercent: 99, resetsAt: 1792287729, windowDurationMins: 300 }, + secondary: { usedPercent: 16, resetsAt: 1792874529, windowDurationMins: 10080 }, + }, + { limitId: "premium", primary: null, secondary: null }, + ); + expect(codexUsageLimitResetAt(snapshot, at)).toBe(resetAt); + }); + + it("does not guess when more than one window reads 99%", () => { + expect( + codexUsageLimitResetAt( + { + primary: { usedPercent: 99, resetsAt: 1792287729 }, + secondary: { usedPercent: 99, resetsAt: 1792874529 }, + }, + at, + ), + ).toBeNull(); + }); + + it("prefers an explicitly exhausted window over a rounded near-full window", () => { + expect( + codexUsageLimitResetAt( + { + primary: { usedPercent: 100, resetsAt: 1792287729 }, + secondary: { usedPercent: 99, resetsAt: 1792874529 }, + }, + at, + ), + ).toBe(resetAt); + }); + it("waits for every exhausted window and never invents an unknown reset", () => { expect( - codexUsageLimitResetAt({ - primary: { usedPercent: 100, resetsAt: 2000000000 }, - secondary: { usedPercent: 100, resetsAt: 2000100000 }, - }), + codexUsageLimitResetAt( + { + primary: { usedPercent: 100, resetsAt: 2000000000 }, + secondary: { usedPercent: 100, resetsAt: 2000100000 }, + }, + at, + ), ).toBe("2033-05-19T07:20:00.000Z"); - expect(codexUsageLimitResetAt({ primary: { usedPercent: 100 } })).toBeNull(); + expect(codexUsageLimitResetAt({ primary: { usedPercent: 100 } }, at)).toBeNull(); + expect(codexUsageLimitResetAt({ primary: { usedPercent: 99 } }, at)).toBeNull(); expect( - codexUsageLimitResetAt({ primary: { usedPercent: 50, resetsAt: 2000000000 } }), + codexUsageLimitResetAt( + { primary: { usedPercent: 100, resetsAt: 1792287729 }, secondary: { usedPercent: 100 } }, + at, + ), ).toBeNull(); + expect( + codexUsageLimitResetAt( + { primary: { usedPercent: 100, resetsAt: 1792287729 }, secondary: { usedPercent: 99 } }, + at, + ), + ).toBe(resetAt); + expect( + codexUsageLimitResetAt({ primary: { usedPercent: 98, resetsAt: 2000000000 } }, at), + ).toBeNull(); + expect(codexUsageLimitResetAt(undefined, at)).toBeNull(); + }); + + it("ignores renewed windows and invalid observation times", () => { + for (const usedPercent of [99, 100]) { + expect( + codexUsageLimitResetAt({ primary: { usedPercent, resetsAt: 1792287729 } }, resetAt), + ).toBeNull(); + expect( + codexUsageLimitResetAt({ primary: { usedPercent, resetsAt: 1792287729 } }, "invalid"), + ).toBeNull(); + } + expect( + codexUsageLimitResetAt( + { + primary: { usedPercent: 100, resetsAt: 1792270000 }, + secondary: { usedPercent: 99, resetsAt: 1792874529 }, + }, + at, + ), + ).toBe("2026-10-24T20:42:09.000Z"); }); }); diff --git a/apps/server/src/provider/codexUsageLimits.ts b/apps/server/src/provider/codexUsageLimits.ts index 31d48535cf948..75b4921c24ed6 100644 --- a/apps/server/src/provider/codexUsageLimits.ts +++ b/apps/server/src/provider/codexUsageLimits.ts @@ -210,12 +210,25 @@ export function codexUsageLimitMessage( return `Codex usage limit reached.${reset}${codexUsageLimitNextStep(snapshot?.rateLimitReachedType)}`; } -/** All exhausted windows must reset before a continuation can run. */ +/** + * Called after a confirmed usage-limit failure. All exhausted windows must reset + * before a continuation can run. Codex reports whole percentages, so a refusal + * can arrive while the window that ran out still reads 99%; that is only clear + * when no window reads 100% and exactly one reads 99%. + */ export function codexUsageLimitResetAt( snapshot: CodexRateLimitSnapshot | undefined, + atIso: string, ): string | null { - if (!snapshot) return null; - const windows = codexRateLimitsToWindows(snapshot).filter((window) => window.usedPercent >= 100); + const atMs = Date.parse(atIso); + if (!snapshot || !Number.isFinite(atMs)) return null; + // A window whose reset has passed has renewed since it was observed. + const activeWindows = codexRateLimitsToWindows(snapshot).filter( + (window) => !window.resetsAt || Date.parse(window.resetsAt) > atMs, + ); + const exhausted = activeWindows.filter((window) => window.usedPercent >= 100); + const nearFull = activeWindows.filter((window) => window.usedPercent >= 99); + const windows = exhausted.length > 0 ? exhausted : nearFull.length === 1 ? nearFull : []; if (windows.length === 0 || windows.some((window) => !window.resetsAt)) return null; return windows.reduce( (latest, window) =>