From 45f78c07d8f31356e951120f9800e5c2861876ef Mon Sep 17 00:00:00 2001 From: Max Sapelov <13932412+msapelov@users.noreply.github.com> Date: Tue, 8 Sep 2026 21:28:18 +0200 Subject: [PATCH 1/6] fix(sdk): prevent transcript memory leaks in scan tracking --- sdk/typescript/src/cost.ts | 104 +++++++++----- sdk/typescript/tests-ts/cost.test.ts | 205 +++++++++++++++++++++++++-- 2 files changed, 256 insertions(+), 53 deletions(-) diff --git a/sdk/typescript/src/cost.ts b/sdk/typescript/src/cost.ts index 81247c0728..1b5df23695 100644 --- a/sdk/typescript/src/cost.ts +++ b/sdk/typescript/src/cost.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { open, readdir } from "node:fs/promises"; import { join } from "node:path"; import { @@ -37,6 +38,7 @@ interface SessionReasoning { } interface SessionUsage { + tracked: boolean; offset: number; pendingLine: Buffer[]; pendingLineBytes: number; @@ -50,7 +52,7 @@ interface SessionUsage { usage: ScanTokenUsage | null; calls: Map; activities: ScanActivity[]; - progress: ScanProgress[]; + progress?: ScanProgress[]; filesCompleted: number; filesTotal: number | null; prose: Set; @@ -83,6 +85,7 @@ const SESSION_READ_SIZE = 64 * 1_024; function createSessionUsage(): SessionUsage { return { + tracked: false, offset: 0, pendingLine: [], pendingLineBytes: 0, @@ -96,7 +99,6 @@ function createSessionUsage(): SessionUsage { usage: null, calls: new Map(), activities: [], - progress: [], filesCompleted: 0, filesTotal: null, prose: new Set(), @@ -199,7 +201,6 @@ export class ScanCostTracker { async #readSessions(): Promise { if (this.#threadId === null) return; - const unreadable: Array<{ session: SessionUsage; error: unknown }> = []; for await (const path of sessionFiles( join(this.#options.codexHome, "sessions"), )) { @@ -208,11 +209,10 @@ export class ScanCostTracker { session = createSessionUsage(); this.#sessions.set(path, session); } - try { - await readSessionUsage(path, session, this.#options.repository); - } catch (error) { - if (session.threadId === null) throw error; - unreadable.push({ session, error }); + if (session.threadId === null) { + // Index ownership before reading a transcript. Unrelated sessions need + // only their metadata, including parents discovered by a later poll. + await readSessionUsage(path, session, undefined, true); } } @@ -258,30 +258,34 @@ export class ScanCostTracker { } } } - for (const { session, error } of unreadable) { - if (included.has(session.threadId!)) throw error; - } - const usages = new Map(this.#receipts); for (const [path, tracked] of this.#sessions) { const threadId = tracked.threadId; if (threadId === null || !included.has(threadId)) continue; let session = tracked; - if ( - this.#options.onSessionEvent !== undefined && - session.events === undefined - ) { - // Replay only newly associated sessions, including their early events. - session = createSessionUsage(); - session.events = []; - await readSessionUsage(path, session, this.#options.repository); - this.#sessions.set(path, session); - } let worker: number | undefined; if (threadId !== this.#threadId) { worker = this.#workers.get(threadId) ?? this.#workers.size + 1; this.#workers.set(threadId, worker); } + if (!session.tracked) { + // Replay newly associated sessions from the start for every observer, + // not just raw session events. Their early usage and activity matter. + session = createSessionUsage(); + session.tracked = true; + if (this.#options.onSessionEvent !== undefined) session.events = []; + if (worker !== undefined && this.#options.onProgress !== undefined) { + session.progress = []; + } + this.#sessions.set(path, session); + } + await readSessionUsage( + path, + session, + worker !== undefined && this.#options.onActivity !== undefined + ? this.#options.repository + : undefined, + ); for (const event of session.events?.splice(0) ?? []) { this.#options.onSessionEvent?.({ threadId, @@ -325,7 +329,7 @@ export class ScanCostTracker { if (this.#options.onProgress === undefined || session.threadId === null) { return; } - for (const progress of session.progress.splice(0)) { + for (const progress of session.progress?.splice(0) ?? []) { const expectedFilesTotal = this.#expectedFilesTotal; if ( (expectedFilesTotal !== undefined && @@ -390,6 +394,7 @@ async function readSessionUsage( path: string, session: SessionUsage, repository?: string, + metadataOnly = false, ): Promise { if (session.unreadable) return; let file; @@ -411,13 +416,19 @@ async function readSessionUsage( if (bytesRead === 0) return; session.offset += bytesRead; try { - readSessionChunk(buffer.subarray(0, bytesRead), session, repository); + readSessionChunk( + buffer.subarray(0, bytesRead), + session, + repository, + metadataOnly, + ); } catch (error) { session.unreadable = true; session.pendingLine = []; session.pendingLineBytes = 0; throw error; } + if (metadataOnly && session.threadId !== null) return; } } finally { await file.close(); @@ -428,6 +439,7 @@ function readSessionChunk( contents: Buffer, session: SessionUsage, repository?: string, + metadataOnly = false, ): void { let lineStart = 0; while (lineStart < contents.length) { @@ -445,17 +457,24 @@ function readSessionChunk( } if (session.pendingLineBytes === 0) { - readSessionEvent(fragment.toString("utf8"), session, repository); + readSessionEvent( + fragment.toString("utf8"), + session, + repository, + metadataOnly, + ); } else { if (fragment.length > 0) session.pendingLine.push(Buffer.from(fragment)); readSessionEvent( Buffer.concat(session.pendingLine, lineBytes).toString("utf8"), session, repository, + metadataOnly, ); session.pendingLine = []; session.pendingLineBytes = 0; } + if (metadataOnly && session.threadId !== null) return; lineStart = newline + 1; } } @@ -464,6 +483,7 @@ function readSessionEvent( line: string, session: SessionUsage, repository?: string, + metadataOnly = false, ): void { if (line.length === 0) return; let event: unknown; @@ -491,6 +511,7 @@ function readSessionEvent( session.events?.push(event); return; } + if (metadataOnly) return; if (session.replaying) { if (event["type"] !== "event_msg") return; if (payload["type"] === "token_count" && isRecord(payload["info"])) { @@ -516,7 +537,7 @@ function readSessionEvent( } session.events?.push(event); if (event["type"] === "response_item") { - session.progress.push(...sessionProgressUpdates(payload)); + session.progress?.push(...sessionProgressUpdates(payload)); if (repository === undefined) return; if ( payload["type"] === "reasoning" && @@ -537,10 +558,7 @@ function readSessionEvent( }, repository, ); - if ( - activity === null || - session.prose.has(`${activity.kind}:${activity.description}`) - ) { + if (activity === null || session.prose.has(proseKey(activity))) { continue; } session.reasoning = { @@ -575,12 +593,12 @@ function readSessionEvent( session.reasoning = null; if ( activity.kind === "message" && - session.prose.has(`${activity.kind}:${activity.description}`) + session.prose.has(proseKey(activity)) ) { return; } if (activity.kind === "message") { - session.prose.add(`${activity.kind}:${activity.description}`); + session.prose.add(proseKey(activity)); } if (activity.status === "running") { session.calls.set(activity.id, activity); @@ -616,7 +634,7 @@ function readSessionEvent( payload["type"] === "agent_message" && typeof payload["message"] === "string" ) { - session.progress.push( + session.progress?.push( ...scanProgressUpdatesFromEvent({ type: "item.completed", item: { type: "agent_message", text: payload["message"] }, @@ -630,11 +648,8 @@ function readSessionEvent( } session.reasoning = null; const activity = scanActivityFromSessionEvent(event, repository); - if ( - activity !== null && - !session.prose.has(`${activity.kind}:${activity.description}`) - ) { - session.prose.add(`${activity.kind}:${activity.description}`); + if (activity !== null && !session.prose.has(proseKey(activity))) { + session.prose.add(proseKey(activity)); session.activities.push(activity); } return; @@ -731,10 +746,21 @@ function recordReasoningActivity( return; } reasoning.activity = activity; - session.prose.add(`${activity.kind}:${activity.description}`); + session.prose.add(proseKey(activity)); session.activities.push(activity); } +function proseKey(activity: ScanActivity): string { + // Deduplication needs an identity, not a retained copy of every transcript + // message and every expanding reasoning prefix. + // Hash UTF-16 code units to keep distinct lone surrogates distinct too. + return createHash("sha256") + .update(activity.kind) + .update(":") + .update(activity.description, "utf16le") + .digest("hex"); +} + function sessionProgressUpdates( payload: Readonly>, ): ScanProgress[] { diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index c04c6fa07b..f7ab8507d7 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -1,4 +1,5 @@ import { spawnSync } from "node:child_process"; +import * as filesystem from "node:fs/promises"; import { appendFile, mkdir, @@ -9,7 +10,7 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, parse, sep } from "node:path"; -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, spyOn, test } from "bun:test"; import { estimateScanCost, ScanCostTracker, @@ -382,6 +383,152 @@ describe("scan cost", () => { }); describe("live scan cost tracking", () => { + test("reads only metadata from unrelated sessions and stops reopening them", async () => { + const home = await codexHome(); + const usage = { input_tokens: 100, output_tokens: 10 }; + await writeSession(home, "scan-thread", usage); + const unrelated = await writeSession(home, "unrelated-thread", usage); + await appendSessionItem(unrelated, { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "x".repeat(256 * 1_024) }], + }); + const originalOpen = filesystem.open; + let opens = 0; + let bytesRead = 0; + const restores: Array<() => void> = []; + const opening = spyOn(filesystem, "open").mockImplementation( + async (...args) => { + const file = await originalOpen(...args); + if (String(args[0]) === unrelated) { + opens += 1; + const originalRead = file.read.bind(file); + const reading = spyOn(file, "read").mockImplementation((async ( + ...readArgs + ) => { + const result = await Reflect.apply(originalRead, file, readArgs); + bytesRead += result.bytesRead; + return result; + }) as typeof file.read); + restores.push(() => reading.mockRestore()); + } + return file; + }, + ); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + repository: home, + }); + tracker.start("scan-thread"); + try { + expect((await tracker.refresh()).cost?.inputTokens).toBe(100); + expect(bytesRead).toBeLessThan(256 * 1_024); + expect(opens).toBe(1); + await appendSessionItem(unrelated, { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "More unrelated output." }], + }); + await tracker.refresh(); + await tracker.stop(); + expect(opens).toBe(1); + } finally { + opening.mockRestore(); + for (const restore of restores) restore(); + } + }); + + test("replays a worker after its metadata arrives across multiple reads", async () => { + const home = await codexHome(); + const usage = { input_tokens: 100, output_tokens: 10 }; + await writeSession(home, "scan-thread", usage); + const worker = join(home, "sessions", "partial-worker.jsonl"); + const metadata = JSON.stringify({ + type: "session_meta", + payload: { + padding: "x".repeat(128 * 1_024), + id: "worker-thread", + parent_thread_id: "scan-thread", + }, + }); + await writeFile(worker, metadata.slice(0, -2)); + const events: ScanSessionEvent[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + onSessionEvent: (event) => events.push(event), + }); + tracker.start("scan-thread"); + try { + await tracker.refresh(); + expect(events.some((event) => event.threadId === "worker-thread")).toBe( + false, + ); + await appendFile(worker, `${metadata.slice(-2)}\n`); + await appendSessionItem(worker, { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "Early worker output." }], + }); + await tracker.refresh(); + await tracker.refresh(); + expect( + events + .filter((event) => event.threadId === "worker-thread") + .map(({ event }) => event["type"]), + ).toEqual(["session_meta", "response_item"]); + } finally { + await tracker.stop(); + } + }); + + test("deduplicates long worker messages without dropping distinct suffixes", async () => { + const home = await codexHome(); + const usage = { input_tokens: 100, output_tokens: 10 }; + await writeSession(home, "scan-thread", usage); + const worker = await writeSession( + home, + "worker-thread", + usage, + "scan-thread", + ); + const first = `${"x".repeat(4_096)} first`; + const second = `${"x".repeat(4_096)} second`; + const distinct = [ + first, + second, + `${first}\ud800`, + `${first}\ud801`, + `${first}\ufffd`, + ]; + const activities: ScanActivity[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + repository: home, + onActivity: (activity) => activities.push(activity), + }); + tracker.start("scan-thread"); + try { + for (const message of [...distinct, first]) { + await appendFile( + worker, + `${JSON.stringify({ + type: "event_msg", + payload: { type: "agent_message", message }, + })}\n`, + ); + await tracker.refresh(); + } + expect(activities.map((activity) => activity.description)).toEqual( + distinct, + ); + } finally { + await tracker.stop(); + } + }); + test("coalesces overlapping polling ticks and bounds final work", async () => { const home = await codexHome(); await writeSession(home, "scan-thread", { @@ -566,9 +713,14 @@ describe("live scan cost tracking", () => { expect(events).toHaveLength(6); }); - test.each(["parent", "main"] as const)( - "replays early worker events when the %s session arrives later", - async (missing) => { + test.each([ + ["parent", true], + ["parent", false], + ["main", true], + ["main", false], + ] as const)( + "replays early worker output when the %s session arrives later (raw events: %s)", + async (missing, rawEvents) => { const home = await codexHome(); const scanDirectory = join(home, "scan"); const usage = { input_tokens: 10, output_tokens: 1 }; @@ -598,19 +750,28 @@ describe("live scan cost tracking", () => { "2026-07-26T12:01:00Z", ); const message = (text: string) => ({ + id: text, type: "message", role: "assistant", content: [{ type: "output_text", text }], }); await appendSessionItem(worker, message("Early worker output.")); + await appendSessionItem(worker, progressMessage(2)); const unrelated = await writeSession(home, "unrelated-thread", usage); await appendSessionItem(unrelated, message("Unrelated output.")); const events: ScanSessionEvent[] = []; + const activities: ScanActivity[] = []; + const progress: ScanProgress[] = []; const tracker = new ScanCostTracker({ codexHome: home, scanDirectory, model: "gpt-5.6-sol", - onSessionEvent: (event) => events.push(event), + repository: home, + onActivity: (activity) => activities.push(activity), + onProgress: (update) => progress.push(update), + ...(rawEvents + ? { onSessionEvent: (event: ScanSessionEvent) => events.push(event) } + : {}), }); tracker.start("scan-thread"); await tracker.refresh(); @@ -626,20 +787,36 @@ describe("live scan cost tracking", () => { await appendSessionItem(worker, message("Late worker output.")); await tracker.refresh(); await tracker.refresh(); - await tracker.stop(); + const snapshot = await tracker.stop(); + expect(snapshot.usage).toMatchObject({ + input_tokens: missing === "parent" ? 30 : 20, + output_tokens: missing === "parent" ? 3 : 2, + }); const workerEvents = events.filter( (event) => event.threadId === "worker-thread", ); - expect(workerEvents.map((event) => event.event)).toEqual([ - expect.objectContaining({ type: "session_meta" }), - expect.objectContaining({ type: "event_msg" }), - { type: "response_item", payload: message("Early worker output.") }, - { type: "response_item", payload: message("Late worker output.") }, + if (rawEvents) { + expect(workerEvents.map((event) => event.event)).toEqual([ + expect.objectContaining({ type: "session_meta" }), + expect.objectContaining({ type: "event_msg" }), + { type: "response_item", payload: message("Early worker output.") }, + { type: "response_item", payload: progressMessage(2) }, + { type: "response_item", payload: message("Late worker output.") }, + ]); + expect(new Set(workerEvents.map((event) => event.worker))).toEqual( + new Set([1]), + ); + } else { + expect(events).toEqual([]); + } + expect(activities.map((activity) => activity.description)).toEqual([ + "Early worker output.", + "Late worker output.", + ]); + expect(progress).toEqual([ + { phase: "discovery", filesCompleted: 2, filesTotal: 8 }, ]); - expect(new Set(workerEvents.map((event) => event.worker))).toEqual( - new Set([1]), - ); expect( events.some((event) => event.threadId === "unrelated-thread"), ).toBe(false); From a767478ca179af011e22184daebe402d0a34f4be Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Mon, 5 Oct 2026 09:08:11 +0100 Subject: [PATCH 2/6] fix(server): cancel failed embedding response bodies (#1232) * fix(server): cancel failed embedding response bodies Signed-off-by: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> * test(server): reuse embedding fixtures for response cleanup --------- Signed-off-by: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Co-authored-by: Michael D'Angelo --- sdk/typescript/src/server/embeddings.ts | 1 + .../tests-ts/finding-embeddings.test.ts | 47 +++++++++++++++---- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/sdk/typescript/src/server/embeddings.ts b/sdk/typescript/src/server/embeddings.ts index 474bd9ed5d..31b539f544 100644 --- a/sdk/typescript/src/server/embeddings.ts +++ b/sdk/typescript/src/server/embeddings.ts @@ -104,6 +104,7 @@ export class OpenAiFindingEmbedder implements FindingEmbedder { ); } if (!response.ok) { + void response.body?.cancel().catch(() => undefined); throw new FindingsError( "embedding_failed", `Embedding provider returned HTTP ${response.status}.`, diff --git a/sdk/typescript/tests-ts/finding-embeddings.test.ts b/sdk/typescript/tests-ts/finding-embeddings.test.ts index cc5b13fd6f..4fd911a998 100644 --- a/sdk/typescript/tests-ts/finding-embeddings.test.ts +++ b/sdk/typescript/tests-ts/finding-embeddings.test.ts @@ -157,16 +157,43 @@ test("does not call the provider for empty input or missing credentials", async expect(observeCalls).toHaveBeenCalledTimes(0); }); -test("reports provider failures without echoing response bodies or credentials", async () => { - const embedder = new OpenAiFindingEmbedder( - "synthetic-key", - async () => new Response("synthetic private body", { status: 429 }), - ); - await expect(embedder.embed([example])).rejects.toMatchObject({ - code: "embedding_failed", - message: "Embedding provider returned HTTP 429.", - }); -}); +test.each(["synthetic private body", null])( + "preserves the HTTP error with response body %p", + async (body) => { + const embedder = new OpenAiFindingEmbedder( + "synthetic-key", + async () => new Response(body, { status: 429 }), + ); + await expect(embedder.embed([example])).rejects.toMatchObject({ + code: "embedding_failed", + message: "Embedding provider returned HTTP 429.", + }); + }, +); + +test.each(["complete", "reject", "pending"])( + "cancels a rejected embedding response without replacing its HTTP error (%s)", + async (cleanup) => { + let cancelled = false; + const body = new ReadableStream({ + cancel() { + cancelled = true; + if (cleanup === "reject") + return Promise.reject(new Error("cleanup failed")); + if (cleanup === "pending") return new Promise(() => {}); + }, + }); + const embedder = new OpenAiFindingEmbedder( + "synthetic-key", + async () => new Response(body, { status: 503 }), + ); + await expect(embedder.embed([example])).rejects.toMatchObject({ + code: "embedding_failed", + message: "Embedding provider returned HTTP 503.", + }); + expect(cancelled).toBe(true); + }, +); test("rejects malformed vectors instead of misaligning stored findings", async () => { for (const data of [ From 86275c46833b4b66f3f5305aa42c0cc202009506 Mon Sep 17 00:00:00 2001 From: 12 <153109197+LittleYier@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:14:32 +0900 Subject: [PATCH 3/6] fix(plugin): recognize PHP heredoc terminators inside expressions (#1236) Recognize PHP heredoc and nowdoc closing identifiers inside expressions while preserving following code and PHP identifier-continuation semantics. Add regression coverage for quoting, indentation, comma continuations, same-line braces, and identifier prefixes. --- .../codex-security/scripts/rank_preview.py | 14 ++--- .../codex-security/tests/test_rank_preview.py | 57 +++++++++++++++++++ 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/plugins/codex-security/scripts/rank_preview.py b/plugins/codex-security/scripts/rank_preview.py index 95e54e4b6e..2988d31254 100644 --- a/plugins/codex-security/scripts/rank_preview.py +++ b/plugins/codex-security/scripts/rank_preview.py @@ -303,13 +303,13 @@ def mask_c_style_source(text: str, suffix: str) -> str: continue if heredoc_terminator: if index == 0 or text[index - 1] == "\n": - line_end = text.find("\n", index) - if line_end < 0: - line_end = len(text) - candidate = text[index:line_end].strip().removesuffix(";") - if candidate == heredoc_terminator: - masked.extend(" " * (line_end - index)) - index = line_end + # PHP identifiers allow every non-ASCII byte, including non-word characters. + terminator_match = re.compile( + rf"[ \t]*{heredoc_terminator}(?![A-Za-z0-9_\x80-\U0010ffff])" + ).match(text, index) + if terminator_match: + masked.extend(" " * (terminator_match.end() - index)) + index = terminator_match.end() heredoc_terminator = "" continue masked.append(" ") diff --git a/plugins/codex-security/tests/test_rank_preview.py b/plugins/codex-security/tests/test_rank_preview.py index 8070f14faa..70557f4f42 100644 --- a/plugins/codex-security/tests/test_rank_preview.py +++ b/plugins/codex-security/tests/test_rank_preview.py @@ -551,6 +551,63 @@ def test_javascript_preview_lists_class_field_arrow_handlers(tmp_path: Path) -> assert "method Controller.validate" in preview +@pytest.mark.parametrize("opener", ["TXT", '"TXT"', "'TXT'"]) +@pytest.mark.parametrize("indent", ["", " "]) +def test_php_heredoc_terminator_can_continue_expression( + tmp_path: Path, opener: str, indent: str +) -> None: + source = f""" None: + source = f""" None: + source = f""" None: source = """ Date: Mon, 5 Oct 2026 17:39:47 +0900 Subject: [PATCH 4/6] fix(plugin): preserve Go declarations after raw strings ending in backslashes (#1235) Handle Go backtick strings through the existing raw-string terminator so literal trailing backslashes do not hide later declarations from ranking previews. Add a regression test covering declarations before and after the raw string. --- plugins/codex-security/scripts/rank_preview.py | 5 +++++ .../codex-security/tests/test_rank_preview.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/plugins/codex-security/scripts/rank_preview.py b/plugins/codex-security/scripts/rank_preview.py index 2988d31254..6b87179fd3 100644 --- a/plugins/codex-security/scripts/rank_preview.py +++ b/plugins/codex-security/scripts/rank_preview.py @@ -379,6 +379,11 @@ def mask_c_style_source(text: str, suffix: str) -> str: quote = '@"' index += 2 continue + if suffix == ".go" and char == "`": + masked.append(" ") + raw_terminator = "`" + index += 1 + continue triple_quote = text[index : index + 3] if triple_quote in {'"""', "'''"}: masked.extend((" ", " ", " ")) diff --git a/plugins/codex-security/tests/test_rank_preview.py b/plugins/codex-security/tests/test_rank_preview.py index 70557f4f42..b1856c495d 100644 --- a/plugins/codex-security/tests/test_rank_preview.py +++ b/plugins/codex-security/tests/test_rank_preview.py @@ -626,6 +626,22 @@ class Service { assert "method Service.visible" in preview +def test_go_raw_string_backslash_does_not_hide_following_function(tmp_path: Path) -> None: + source = r"""package sample + +func Before() {} + +const Root = `C:\` + +func After() {} +""" + + preview = generate_preview(tmp_path, "sample.go", source) + + assert "function Before" in preview + assert "function After" in preview + + def test_malformed_python_uses_sampled_source_fallback(tmp_path: Path) -> None: source = """import package broken = ( From 9e1b68a462b9393c5e1d30eb810eae9b303ac170 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Mon, 5 Oct 2026 12:14:35 +0000 Subject: [PATCH 5/6] fix(sdk): report measured usage without model pricing --- .../scripts/workbench_scan_usage.py | 2 + .../tests/test_workbench_scan_usage.py | 28 ++++++++++ sdk/typescript/README.md | 5 ++ sdk/typescript/src/api.ts | 14 +++++ sdk/typescript/src/cli.ts | 1 + sdk/typescript/src/component-scan.ts | 4 +- sdk/typescript/src/cost.ts | 24 ++++++++- sdk/typescript/src/index.ts | 2 +- sdk/typescript/src/scan-dashboard.ts | 16 +++++- sdk/typescript/tests-ts/api.test.ts | 38 ++++++++++++++ sdk/typescript/tests-ts/cli-show-cost.test.ts | 23 ++++++++ .../tests-ts/component-scan.test.ts | 9 ++++ sdk/typescript/tests-ts/cost.test.ts | 52 ++++++++++++++++--- .../tests-ts/scan-dashboard.test.ts | 51 ++++++++++++++++++ 14 files changed, 257 insertions(+), 12 deletions(-) diff --git a/plugins/codex-security/scripts/workbench_scan_usage.py b/plugins/codex-security/scripts/workbench_scan_usage.py index 26d971236f..406208a3a1 100644 --- a/plugins/codex-security/scripts/workbench_scan_usage.py +++ b/plugins/codex-security/scripts/workbench_scan_usage.py @@ -456,6 +456,8 @@ def _read_rollout_usage( continue if event.get("type") != "event_msg" or payload.get("type") != "token_count": continue + if "info" in payload and payload["info"] is None: + continue timestamp = _timestamp(event.get("timestamp")) snapshot = _token_snapshot(payload) if timestamp is None or snapshot is None: diff --git a/plugins/codex-security/tests/test_workbench_scan_usage.py b/plugins/codex-security/tests/test_workbench_scan_usage.py index 06b7ea4250..a7a544ddee 100644 --- a/plugins/codex-security/tests/test_workbench_scan_usage.py +++ b/plugins/codex-security/tests/test_workbench_scan_usage.py @@ -323,6 +323,34 @@ def test_completion_counts_only_scan_owned_parent_and_descendants( assert json.loads(stored[0]) == {"usage": expected} +@pytest.mark.parametrize("measurement", ["unknown", "malformed", "missing"]) +def test_completion_distinguishes_unknown_usage_from_invalid_measurements( + tmp_path: Path, measurement: str +) -> None: + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + payload: dict[str, Any] = {"type": "token_count"} + if measurement == "unknown": + payload["info"] = None + payload["rate_limits"] = {"primary": {"used_percent": 5}} + elif measurement == "malformed": + payload["info"] = {"total_token_usage": {"input_tokens": "invalid"}} + parent = _rollout( + tmp_path, + "scan-parent", + [ + _token_event(counted, 10, 2), + _event(counted, "event_msg", payload), + _token_event(counted, 20, 4), + ], + ) + _state_graph(fixture.environment, {"scan-parent": parent}, []) + usage = _complete_scan(fixture)["scan"]["usage"] + assert usage["totalTokens"] == 24 + assert usage["coverage"] == ("complete" if measurement == "unknown" else "partial") + assert ("token_record_invalid" in usage.get("warnings", [])) is (measurement != "unknown") + + @pytest.mark.parametrize("cache_write_field", ["cache_write_input_tokens", "cache_write_tokens"]) def test_scan_usage_helper_preserves_cached_token_totals( cache_write_field: str, diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 517fc57824..7283ec834c 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -1163,6 +1163,11 @@ answers are ignored. Invalid increases or failures to save them leave the existing limit in place and report a warning. `onCost(cost, maxCostUsd)` reports the current limit, including after an approved increase. +`onUsage(usage)` reports measured scan token totals as `ScanTokenUsage`, +independently of model pricing. The live dashboard displays these totals even +when a cost estimate is unavailable. Component scan observers receive the same +totals in `usage` events. Missing prices do not become zero-dollar estimates. + These amounts estimate API-equivalent model usage, not ChatGPT subscription allowance. Post-scan prompts run after scan cost tracking ends and are outside this limit. diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index 7e10813a59..505580d5b7 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -72,6 +72,7 @@ import { estimateScanCost, ScanCostTracker, type ScanCost, + type ScanTokenUsage, type ScanSessionEvent, } from "./cost.js"; import { @@ -293,6 +294,8 @@ export interface ScanOptions extends ScanSettings { parentScanId?: string; expectedPluginVersion?: string; onCost?: (cost: Readonly, maxCostUsd?: number) => void; + /** Measured scan token usage, including models without known prices. */ + onUsage?: (usage: Readonly) => void; onBudgetApproaching?: ( budget: ScanBudget, ) => number | undefined | Promise; @@ -391,6 +394,7 @@ export interface ScanBudget { type ScanObserverName = | "onAuthentication" | "onCost" + | "onUsage" | "onOutputArchived" | "onOutputDirReady" | "onScanStarted" @@ -1478,6 +1482,16 @@ export class CodexSecurity { ), onProgress: options.onProgress === undefined ? undefined : reportProgress, + onUsage: + options.onUsage === undefined + ? undefined + : (usage) => + notifyObserver( + "onUsage", + options.onUsage, + options.onObserverError, + usage, + ), onCost: options.onCost === undefined && options.maxCostUsd === undefined ? undefined diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 5135264318..578c1b6f76 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -8314,6 +8314,7 @@ async function executeScan( archiveExisting: arguments_.archiveExisting, parentScanId: arguments_.parentScanId, expectedPluginVersion: arguments_.expectedPluginVersion, + onUsage: (usage) => dashboard?.setUsage(usage), onCost: (cost, limit = maxCostUsd) => { if (limit !== maxCostUsd && limit !== undefined) { dashboard?.note(`Total cost limit increased to ${formatUsd(limit)}.`); diff --git a/sdk/typescript/src/component-scan.ts b/sdk/typescript/src/component-scan.ts index ddcc4fa947..27fc6ff057 100644 --- a/sdk/typescript/src/component-scan.ts +++ b/sdk/typescript/src/component-scan.ts @@ -18,7 +18,7 @@ import { scanModelProvider, type CodexSecurityConfig, } from "./config.js"; -import type { ScanCost, ScanSessionEvent } from "./cost.js"; +import type { ScanCost, ScanSessionEvent, ScanTokenUsage } from "./cost.js"; import { readThreatModelPath } from "./artifact-export.js"; import { errorMessage } from "./errors.js"; import type { CoverageCompleteness, Finding } from "./models.js"; @@ -79,6 +79,7 @@ type ComponentScanUpdate = | { type: "activity"; value: ScanActivity } | { type: "session"; value: ScanSessionEvent } | { type: "cost"; value: Readonly } + | { type: "usage"; value: Readonly } | { type: "workers"; value: ScanWorkerStatus } | { type: "warning"; value: string }; @@ -207,6 +208,7 @@ export async function runComponentScans( onActivity: (value) => emit({ type: "activity", value }), onSessionEvent: (value) => emit({ type: "session", value }), onCost: (value) => emit({ type: "cost", value }), + onUsage: (value) => emit({ type: "usage", value }), onWorkerStatus: (value) => emit({ type: "workers", value }), onWarning: (value) => emit({ type: "warning", value }), }; diff --git a/sdk/typescript/src/cost.ts b/sdk/typescript/src/cost.ts index 9606f52b70..a97b310051 100644 --- a/sdk/typescript/src/cost.ts +++ b/sdk/typescript/src/cost.ts @@ -22,7 +22,12 @@ import { type ScanProgress, } from "./worker-progress.js"; -export { estimateScanCost, formatUsd, type ScanCost } from "./cost-model.js"; +export { + estimateScanCost, + formatUsd, + type ScanCost, + type ScanTokenUsage, +} from "./cost-model.js"; export interface ScanSessionEvent { threadId: string; @@ -70,6 +75,7 @@ interface ScanCostTrackerOptions { maxCostUsd?: number; expectedFilesTotal?: number; onCost?: (cost: Readonly) => void; + onUsage?: (usage: Readonly) => void; onActivity?: (activity: ScanActivity) => void; onProgress?: (progress: ScanProgress) => void; onSessionEvent?: (event: ScanSessionEvent) => void; @@ -120,6 +126,7 @@ export class ScanCostTracker { #pending: Promise = Promise.resolve(); #snapshot: ScanCostSnapshot = { usage: null, cost: null }; #lastCost: string | null = null; + #lastUsage: string | null = null; #highestFilesCompleted = 0; #expectedFilesTotal: number | undefined; @@ -145,6 +152,7 @@ export class ScanCostTracker { if ( this.#options.maxCostUsd === undefined && this.#options.onCost === undefined && + this.#options.onUsage === undefined && this.#options.onActivity === undefined && this.#options.onProgress === undefined && this.#options.onSessionEvent === undefined @@ -194,6 +202,7 @@ export class ScanCostTracker { return this.#snapshot; const cost = estimateScanCost(this.#options.model, fallbackUsage); this.#snapshot = { usage: fallbackUsage ?? null, cost }; + this.#reportUsage(tokenUsage(fallbackUsage)); this.#reportCost(cost); return this.#snapshot; } @@ -323,6 +332,7 @@ export class ScanCostTracker { if (usage === null) return; const cost = estimateScanCost(this.#options.model, usage); this.#snapshot = { usage, cost }; + this.#reportUsage(usage); this.#reportCost(cost); } @@ -365,6 +375,14 @@ export class ScanCostTracker { } } + #reportUsage(usage: ScanTokenUsage | null): void { + if (usage === null) return; + const signature = JSON.stringify(usage); + if (signature === this.#lastUsage) return; + this.#lastUsage = signature; + this.#options.onUsage?.(usage); + } + #reportCost(cost: ScanCost | null): void { if (cost === null) return; const signature = JSON.stringify(cost); @@ -636,7 +654,9 @@ function readSessionEvent( payload["type"] === "agent_message" && typeof payload["message"] === "string" ) { - session.progress?.push(...scanProgressUpdatesFromText(payload["message"])); + session.progress?.push( + ...scanProgressUpdatesFromText(payload["message"]), + ); } if (repository === undefined) return; if (payload["type"] !== "agent_message") { diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index e75a9cb766..1c8ee68125 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -51,7 +51,7 @@ export type { ComponentPlanningOptions, } from "./component-plan.js"; export { estimateScanCost } from "./cost.js"; -export type { ScanCost, ScanSessionEvent } from "./cost.js"; +export type { ScanCost, ScanSessionEvent, ScanTokenUsage } from "./cost.js"; export type { DeepScanProgress } from "./deep-progress.js"; export type { CustomValidationResult } from "./custom-validation.js"; export type { ScanActivity, ScanActivityStatus } from "./scan-activity.js"; diff --git a/sdk/typescript/src/scan-dashboard.ts b/sdk/typescript/src/scan-dashboard.ts index b6edbf7ef1..b4e7394b49 100644 --- a/sdk/typescript/src/scan-dashboard.ts +++ b/sdk/typescript/src/scan-dashboard.ts @@ -15,6 +15,8 @@ import { formatScanCost, formatScanCosts, formatScanCostTokens, + formatTokenUsage, + type ScanTokenUsage, } from "./cost-model.js"; import type { ScanActivity } from "./scan-activity.js"; import type { ScanMode } from "./targets.js"; @@ -136,6 +138,7 @@ export class ScanDashboard { #files: ScanProgress | null = null; #publicationProgress: { completed: number; total: number } | null = null; #cost: Readonly | null = null; + #usage: Readonly | null = null; #budget: { request: ScanBudget; input: string; @@ -374,6 +377,9 @@ export class ScanDashboard { case "session": dashboard.recordDetails(event.value); break; + case "usage": + dashboard.setUsage(event.value); + break; case "cost": dashboard.setCost(event.value); break; @@ -414,6 +420,11 @@ export class ScanDashboard { this.#refresh(); } + public setUsage(usage: Readonly): void { + this.#usage = usage; + this.#refresh(); + } + public setCost( cost: Readonly, maxCostUsd = this.#options.maxCostUsd, @@ -831,9 +842,10 @@ export class ScanDashboard { #tokenLines(): string[] { const tokens = - this.#cost === null + formatTokenUsage(this.#usage) ?? + (this.#cost === null ? "waiting for usage" - : formatScanCostTokens(this.#cost); + : formatScanCostTokens(this.#cost)); return wrapActivity(" TOKENS ", tokens, this.#width()); } diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts index 717d4735a7..075b1718ae 100644 --- a/sdk/typescript/tests-ts/api.test.ts +++ b/sdk/typescript/tests-ts/api.test.ts @@ -3858,6 +3858,44 @@ describe("CodexSecurity orchestration", () => { } }); + test.each([false, true])( + "reports unpriced usage with observer failure %p", + async (throwObserver) => { + const { root, repository, codexHome, scanDir } = await scanDirectories(); + const usages: unknown[] = []; + const observerErrors: Array<[ScanObserverName, string]> = []; + const client = new TestClient( + { codexOverrides: { model: "synthetic-unpriced-model" } }, + { + ...scanRuntimeDependencies(codexHome, scanDir), + createCodex: completedCodex(root, copyCompletedScan), + }, + ); + try { + const result = await client.run(repository, { + onUsage: (usage) => { + usages.push(usage); + if (throwObserver) throw new Error("usage observer exploded"); + }, + onObserverError: collectObserverErrors(observerErrors), + }); + expect(result.cost).toBeNull(); + expect(usages).toEqual([ + expect.objectContaining({ + input_tokens: 10, + output_tokens: 3, + total_tokens: 13, + }), + ]); + expect(observerErrors).toEqual( + throwObserver ? [["onUsage", "usage observer exploded"]] : [], + ); + } finally { + await client.close(); + } + }, + ); + const pricedModels = [ "gpt-5.5", "gpt-6-astra", diff --git a/sdk/typescript/tests-ts/cli-show-cost.test.ts b/sdk/typescript/tests-ts/cli-show-cost.test.ts index 4ce8bdb468..12f0d4e0ae 100644 --- a/sdk/typescript/tests-ts/cli-show-cost.test.ts +++ b/sdk/typescript/tests-ts/cli-show-cost.test.ts @@ -9,6 +9,29 @@ const result = fakeResult([], "complete", { output_tokens: 30, }); +test("shows measured usage in the live dashboard when model pricing is unavailable", async () => { + const { stderr, runCli } = createCliTest(main, { stderr: true }); + const usage = { + input_tokens: 1250, + cached_input_tokens: 200, + cache_write_input_tokens: 0, + output_tokens: 30, + reasoning_output_tokens: 0, + total_tokens: 1280, + }; + expect( + await runCli( + ["scan", ".", "--model", "synthetic-unpriced-model", "--show-cost"], + dependencies({ + onTurn: (_repository, options) => options.onUsage?.(usage), + }), + ), + ).toBe(0); + const progress = stderr.text().split("REPORT")[0]!; + expect(progress).toContain("1,280 total"); + expect(progress).toContain("unavailable (model pricing missing)"); +}); + test.each([false, true])("scan cost visibility with TTY %p", async (tty) => { for (const costFlags of [ [], diff --git a/sdk/typescript/tests-ts/component-scan.test.ts b/sdk/typescript/tests-ts/component-scan.test.ts index 597b689b12..dfe83c3689 100644 --- a/sdk/typescript/tests-ts/component-scan.test.ts +++ b/sdk/typescript/tests-ts/component-scan.test.ts @@ -498,6 +498,14 @@ test("forwards scan events with their component identity without letting observe parentThreadId: null, event: {}, }); + options.onUsage?.({ + input_tokens: 100, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 10, + reasoning_output_tokens: 0, + total_tokens: 110, + }); options.onCost?.(fakeResult([], "complete", { input_tokens: 100 }).cost!); options.onWorkerStatus?.({ kind: "dispatch", @@ -522,6 +530,7 @@ test("forwards scan events with their component identity without letting observe "progress", "activity", "session", + "usage", "cost", "workers", "warning", diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index b19cd55153..f18980cb66 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -432,13 +432,50 @@ describe("live scan cost tracking", () => { ); }); + test.each([false, true])( + "reports measured unpriced usage with cost observer %p", + async (observeCost) => { + const home = await codexHome(); + const usage = { input_tokens: 100, output_tokens: 10 }; + await writeSession(home, "scan-thread", usage); + await writeSession(home, "worker-thread", usage, { + parent: "scan-thread", + }); + const usages: unknown[] = []; + const costs: unknown[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "synthetic-unpriced-model", + onUsage: (usage) => usages.push(usage), + onCost: observeCost ? (cost) => costs.push(cost) : undefined, + }); + tracker.start("scan-thread"); + try { + await waitFor(() => usages.length > 0); + const snapshot = await tracker.refresh(); + await tracker.refresh(); + expect(snapshot.cost).toBeNull(); + expect(snapshot.usage).toMatchObject({ + input_tokens: 200, + total_tokens: 220, + }); + expect(usages).toEqual([snapshot.usage]); + expect(costs).toEqual([]); + } finally { + await tracker.stop(); + } + }, + ); + test("reports newly available cache writes even when the dollar amount is unchanged", async () => { const home = await codexHome(); const updates: unknown[] = []; + const usages: unknown[] = []; const tracker = new ScanCostTracker({ codexHome: home, model: "gpt-5.5", onCost: (cost) => updates.push(cost), + onUsage: (usage) => usages.push(usage), }); tracker.start("scan-thread"); tracker.recordUsage( @@ -451,6 +488,12 @@ describe("live scan cost tracking", () => { "scan-thread", ); await tracker.stop(); + expect(usages).toHaveLength(2); + expect(usages[0]).toHaveProperty( + "cache_write_input_tokens_reported", + false, + ); + expect(usages[1]).not.toHaveProperty("cache_write_input_tokens_reported"); expect(updates).toHaveLength(2); expect(updates[0]).toHaveProperty("cacheWriteInputTokensReported", false); expect(updates[1]).not.toHaveProperty("cacheWriteInputTokensReported"); @@ -559,12 +602,9 @@ describe("live scan cost tracking", () => { const home = await codexHome(); const usage = { input_tokens: 100, output_tokens: 10 }; await writeSession(home, "scan-thread", usage); - const worker = await writeSession( - home, - "worker-thread", - usage, - { parent: "scan-thread" }, - ); + const worker = await writeSession(home, "worker-thread", usage, { + parent: "scan-thread", + }); const first = `${"x".repeat(4_096)} first`; const second = `${"x".repeat(4_096)} second`; const distinct = [ diff --git a/sdk/typescript/tests-ts/scan-dashboard.test.ts b/sdk/typescript/tests-ts/scan-dashboard.test.ts index eb6dccf67b..a0114cb865 100644 --- a/sdk/typescript/tests-ts/scan-dashboard.test.ts +++ b/sdk/typescript/tests-ts/scan-dashboard.test.ts @@ -53,6 +53,57 @@ class DashboardTestInput extends EventEmitter { } describe("live scan dashboard", () => { + test.each([false, true])( + "shows unpriced token updates for component view %p", + (component) => { + const stderr = capture(true); + const input = new DashboardTestInput(); + const dashboard = createDashboard(stderr.stream, { + model: { model: "synthetic-unpriced-model", reasoningEffort: "high" }, + input, + showCost: true, + ...(component ? { presentation: "components" as const } : {}), + }); + const frame = () => + stripVTControlCharacters(stderr.text().split("\u001B[H").at(-1)!); + dashboard.start(); + if (component) { + dashboard.setComponents([ + { + id: "component-1", + name: "Synthetic", + paths: ["src"], + status: "started", + outputDir: "/synthetic/output", + }, + ]); + input.emit("data", "\r"); + } + expect(frame()).toContain("waiting for usage"); + for (const count of [100, 200]) { + const usage = { + input_tokens: count, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 10, + reasoning_output_tokens: 0, + total_tokens: count + 10, + }; + if (component) + dashboard.recordComponentEvent({ + componentId: "component-1", + type: "usage", + value: usage, + }); + else dashboard.setUsage(usage); + expect(frame()).toContain(`${count + 10} total`); + expect(frame()).not.toContain("waiting for usage"); + expect(frame()).toContain("unavailable (model pricing missing)"); + } + dashboard.stop(); + }, + ); + test("keeps cost bounds and assumptions readable on a narrow terminal", () => { const stderr = capture(true); const dashboard = createDashboard( From e73753cb8b21711a58296e07b57a89465b667e8a Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Mon, 5 Oct 2026 13:19:04 +0000 Subject: [PATCH 6/6] fix(usage): retain indexed state until replay succeeds --- sdk/typescript/src/cost.ts | 2 +- sdk/typescript/tests-ts/cost.test.ts | 38 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/sdk/typescript/src/cost.ts b/sdk/typescript/src/cost.ts index b156c9a440..057c774c2f 100644 --- a/sdk/typescript/src/cost.ts +++ b/sdk/typescript/src/cost.ts @@ -278,7 +278,6 @@ export class ScanCostTracker { if (worker !== undefined && this.#options.onProgress !== undefined) { session.progress = []; } - this.#sessions.set(path, session); } await readSessionUsage( path, @@ -287,6 +286,7 @@ export class ScanCostTracker { ? this.#options.repository : undefined, ); + this.#sessions.set(path, session); for (const event of session.events?.splice(0) ?? []) { this.#options.onSessionEvent?.({ threadId, diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index 906f4e88fc..d4627a7263 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -182,6 +182,44 @@ describe("scan cost", () => { } }); + test("retries an interrupted initial replay without losing usage or events", async () => { + const home = await codexHome(); + await writeSession(home, "scan-thread", { + input_tokens: 100, + output_tokens: 10, + }); + const events: ScanSessionEvent[] = []; + const errors: unknown[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + onSessionEvent: (event) => events.push(event), + onError: (error) => errors.push(error), + }); + const originalOpen = fs.open; + let opens = 0; + const failure = Object.assign( + new Error("Synthetic transient read failure"), + { code: "EIO" }, + ); + const opening = spyOn(fs, "open").mockImplementation(async (...args) => { + if (++opens === 2) throw failure; + return originalOpen(...args); + }); + try { + tracker.start("scan-thread"); + const final = await tracker.stop(); + expect(errors).toEqual([failure]); + expect(final.cost?.inputTokens).toBe(100); + expect(events.map(({ event }) => event["type"])).toEqual([ + "session_meta", + "event_msg", + ]); + } finally { + opening.mockRestore(); + } + }); + test("replays a worker after its metadata arrives across multiple reads", async () => { const home = await codexHome(); const usage = { input_tokens: 100, output_tokens: 10 };