diff --git a/packages/adapters/codex-usage/src/usage-statistics.ts b/packages/adapters/codex-usage/src/usage-statistics.ts index 2e23d936a..c25c30ab9 100644 --- a/packages/adapters/codex-usage/src/usage-statistics.ts +++ b/packages/adapters/codex-usage/src/usage-statistics.ts @@ -36,6 +36,9 @@ const ROW_KINDS = new Set([ ]); const HEAD_TYPE = /"type"\s*:\s*"([^"]+)"/u; const TAIL_TYPE = /"type"\s*:\s*"([^"]+)"\s*\}\s*$/u; +const USER_ROLE = /"role"\s*:\s*"user"/u; +const EVENT_OF_INTEREST = + /"type"\s*:\s*"(?:token_count|task_started|thread_settings_applied|user_message)"/u; /** * Cheap top-level row type guess, confirmed by JSON.parse later. Most rows lead with `type`, @@ -47,6 +50,11 @@ function rowKind(line: string): string | undefined { if (head && ROW_KINDS.has(head)) return head; return TAIL_TYPE.exec(line.slice(-1024))?.[1] ?? head; } + +/** Whether a marker sits in the head, or at the tail once the row kind is already known. */ +function markerIn(pattern: RegExp, line: string): boolean { + return pattern.test(line.slice(0, 1024)) || pattern.test(line.slice(-1024)); +} async function stamp(file: string): Promise { const s = await lstat(file, { bigint: true }); if (!s.isFile()) throw new Error("Codex rollout is not a regular file"); @@ -129,14 +137,9 @@ export async function readCodexRollouts( if (!kind && line.trim()) options.onIncomplete?.(); if (!kind || !ROW_KINDS.has(kind)) continue; // Avoid decoding large tool outputs or conversation bodies which cannot contain usage. - if (kind === "response_item" && !/"role"\s*:\s*"user"/u.test(line.slice(0, 1024))) continue; - if ( - kind === "event_msg" && - !/"type"\s*:\s*"(?:token_count|task_started|thread_settings_applied|user_message)"/u.test( - line.slice(0, 1024), - ) - ) - continue; + // The row kind is already known, so a marker pushed past 1 KiB can still match the tail. + if (kind === "response_item" && !markerIn(USER_ROLE, line)) continue; + if (kind === "event_msg" && !markerIn(EVENT_OF_INTEREST, line)) continue; let row: Record | null; try { row = object(JSON.parse(line)); @@ -190,8 +193,12 @@ export async function readCodexRollouts( break; } case "response_item": - counters.boundary(); - firstPrompt ??= userItemPrompt(p); + // Nested content can contain `"role":"user"` without this row being a user message. + // Clearing the pending pair on those rows drops the following token count. + if (p.type === "message" && p.role === "user") { + counters.boundary(); + firstPrompt ??= userItemPrompt(p); + } break; case "event_msg": if (p.type === "task_started") { diff --git a/packages/adapters/codex-usage/test/usage-statistics.test.ts b/packages/adapters/codex-usage/test/usage-statistics.test.ts index ef734c765..2a4d85ffb 100644 --- a/packages/adapters/codex-usage/test/usage-statistics.test.ts +++ b/packages/adapters/codex-usage/test/usage-statistics.test.ts @@ -106,6 +106,58 @@ describe("Codex native counters", () => { ]); }); + it("reads a token count whose payload type sits past the first kibibyte", async () => { + const entries = await read([ + meta(), + context(), + { + timestamp: new Date(BASE + 1000).toISOString(), + type: "event_msg", + payload: { + info: { total_token_usage: tokens(100, 10), last_token_usage: tokens(100, 10) }, + note: "n".repeat(2048), + type: "token_count", + }, + }, + ]); + expect(entries.map((e) => [e.inputTokens, e.outputTokens])).toEqual([[100, 10]]); + }); + + it("reads a user message title when the role sits past the first kibibyte", async () => { + const entries = await read([ + meta(), + context(), + count(tokens(100, 10), tokens(100, 10)), + { + timestamp: new Date(BASE + 2000).toISOString(), + type: "response_item", + payload: { + type: "message", + content: [{ type: "input_text", text: "ship the usage fix" }], + padding: "p".repeat(2048), + role: "user", + }, + }, + ]); + expect(entries[0]).toMatchObject({ sessionTitle: "ship the usage fix" }); + }); + + it("does not split a pending pair when nested content only quotes a user role", async () => { + const amount = tokens(100, 10); + const entries = await read([ + meta(), + context(), + record("response", amount, tokens(1100, 110)), + row("response_item", { + type: "function_call_output", + output: `${"x".repeat(1100)}\n"role": "user"\n`, + }), + count(tokens(900, 90), amount), + ]); + expect(entries).toHaveLength(1); + expect(entries[0]).toMatchObject({ id: `${THREAD}:response`, inputTokens: 100 }); + }); + it("uses cumulative deltas, ignores repeated checkpoints, and counts runtime resets", async () => { const a = tokens(100, 10, 50), total = tokens(250, 30, 100),