diff --git a/scripts/analyze-prompt-cache-usage.ts b/scripts/analyze-prompt-cache-usage.ts index 09d400d77..21b70ff82 100644 --- a/scripts/analyze-prompt-cache-usage.ts +++ b/scripts/analyze-prompt-cache-usage.ts @@ -85,6 +85,7 @@ interface ShapeSummary extends CohortDimensions { cacheWriteRatio: number; } +/** Describe the analyzer CLI options and their defaults. */ function usageText(): string { return [ "Usage: bun scripts/analyze-prompt-cache-usage.ts [usage.jsonl|-] [options]", @@ -98,36 +99,44 @@ function usageText(): string { ].join("\n"); } +/** Narrow parsed JSON values to non-null, non-array objects. */ function isRecord(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } +/** Check for an explicitly stored field without consulting the prototype chain. */ function hasOwn(value: Record, key: string): boolean { return Object.prototype.hasOwnProperty.call(value, key); } +/** Return finite, non-negative numeric usage values, or undefined for invalid data. */ function nonNegativeNumber(value: unknown): number | undefined { return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; } +/** Trim a string dimension, returning undefined when it is absent or blank. */ function stringValue(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } +/** Accept a supported analysis window; throw for an unknown range. */ function parseRange(value: string): RangeName { if (value === "7d" || value === "30d" || value === "all") return value; throw new Error(`invalid --range value: ${value}`); } +/** Parse a decimal result limit from 1 through 200; throw for invalid input. */ function parseTop(value: string): number { if (!/^\d+$/u.test(value)) throw new Error(`invalid --top value: ${value}`); const parsed = Number.parseInt(value, 10); - if (parsed < 1 || parsed > 200) throw new Error("--top must be between 1 and 200"); + if (parsed < 1 || parsed > 200) + throw new Error("--top must be between 1 and 200"); return parsed; } +/** Parse a non-negative epoch-millisecond or date-string anchor; throw if invalid. */ function parseNow(value: string): number { if (/^\d+$/u.test(value)) { const numeric = Number(value); @@ -138,6 +147,7 @@ function parseNow(value: string): number { throw new Error(`invalid --now value: ${value}`); } +/** Resolve CLI options and the usage-log path; throw on invalid or extra arguments. */ function parseArgs(args: string[]): AnalyzerOptions { const defaultSource = join( process.env.OPENCODEX_HOME?.trim() || join(homedir(), ".opencodex"), @@ -173,7 +183,8 @@ function parseArgs(args: string[]): AnalyzerOptions { continue; } if (arg.startsWith("--")) throw new Error(`unknown option: ${arg}`); - if (positionalSeen) throw new Error("only one usage-log path may be supplied"); + if (positionalSeen) + throw new Error("only one usage-log path may be supplied"); source = arg; positionalSeen = true; } @@ -181,8 +192,17 @@ function parseArgs(args: string[]): AnalyzerOptions { return { source, range, top, nowMs, jsonMode, help }; } +/** + * Read cache totals only from successful, provider-reported usage rows. + * Prefer explicit reads over legacy cached totals, subtract legacy writes when present, + * and reject malformed reads or cache totals exceeding input tokens. + */ function cacheUsage(row: Record): CacheUsage | undefined { - if (row.status !== 200 || row.usageStatus !== "reported" || !isRecord(row.usage)) { + if ( + row.status !== 200 || + row.usageStatus !== "reported" || + !isRecord(row.usage) + ) { return undefined; } const input = nonNegativeNumber(row.usage.inputTokens); @@ -195,28 +215,34 @@ function cacheUsage(row: Record): CacheUsage | undefined { read = explicitRead; } else { const legacyCached = nonNegativeNumber(row.usage.cachedInputTokens); - read = (legacyCached !== undefined && row.usage.cacheCreationInputTokens !== undefined - ? Math.max(0, legacyCached - write) - : legacyCached) ?? 0; + read = + (legacyCached !== undefined && + row.usage.cacheCreationInputTokens !== undefined + ? Math.max(0, legacyCached - write) + : legacyCached) ?? 0; } if (read + write > input) return undefined; return { input, read, write }; } +/** Compute the inclusive window start in milliseconds, using epoch zero for all history. */ function sinceFor(range: RangeName, nowMs: number): number { if (range === "all") return 0; return nowMs - (range === "7d" ? 7 : 30) * 86_400_000; } +/** Extract cohort dimensions, preferring the resolved model and marking missing values unknown. */ function dimensionsFrom(row: Record): CohortDimensions { return { adapter: stringValue(row.adapter) ?? "unknown", provider: stringValue(row.provider) ?? "unknown", - model: stringValue(row.resolvedModel) ?? stringValue(row.model) ?? "unknown", + model: + stringValue(row.resolvedModel) ?? stringValue(row.model) ?? "unknown", surface: stringValue(row.surface) ?? "unknown", }; } +/** Encode adapter, provider, model, and surface as an unambiguous cohort key. */ function dimensionsKey(dimensions: CohortDimensions): string { return JSON.stringify([ dimensions.adapter, @@ -226,6 +252,7 @@ function dimensionsKey(dimensions: CohortDimensions): string { ]); } +/** Initialize usage and conversation counters for a cohort. */ function emptyCohort(dimensions: CohortDimensions): Cohort { return { ...dimensions, @@ -241,10 +268,12 @@ function emptyCohort(dimensions: CohortDimensions): Cohort { }; } +/** Divide by a positive denominator, returning zero when no measurable denominator exists. */ function ratio(numerator: number, denominator: number): number { return denominator > 0 ? numerator / denominator : 0; } +/** Classify measured reuse using cache-read, cache-write, and input-volume thresholds. */ function signal(cohort: Cohort): CohortSignal { const readRatio = ratio(cohort.cacheRead, cohort.input); const writeRatio = ratio(cohort.cacheWrite, cohort.input); @@ -254,6 +283,7 @@ function signal(cohort: Cohort): CohortSignal { return "mixed"; } +/** Convert cohort counters into token totals, ratios, and conversation counts. */ function summarizeCohort(cohort: Cohort): CohortSummary { let multiTurnConversations = 0; for (const count of cohort.conversations.values()) { @@ -280,6 +310,7 @@ function summarizeCohort(cohort: Cohort): CohortSummary { }; } +/** Encode the cache settings and fingerprints used to group comparable request shapes. */ function cacheShapeKey(observation: PromptCacheRequestObservation): string { return JSON.stringify([ observation.mode, @@ -293,6 +324,7 @@ function cacheShapeKey(observation: PromptCacheRequestObservation): string { ]); } +/** Summarize a shape bucket, measuring tokens only from valid reported-success rows. */ function summarizeShape(bucket: ShapeBucket): ShapeSummary { const pc = bucket.observation; let input = 0; @@ -328,6 +360,7 @@ function summarizeShape(bucket: ShapeBucket): ShapeSummary { }; } +/** Render a fractional ratio as a percentage with one decimal place. */ function formatPercent(value: number): string { return (value * 100).toFixed(1) + "%"; } @@ -350,12 +383,14 @@ interface AnalysisState { totals: AnalysisTotals; } +/** Read UTF-8 usage JSONL from the selected file or stdin; propagate read errors. */ function readUsageSource(options: AnalyzerOptions): string { return options.source === "-" ? readFileSync(0, "utf8") : readFileSync(options.source, "utf8"); } +/** Parse JSONL objects, skipping blank lines and counting malformed or non-object rows. */ function parseUsageRows(text: string): ParsedUsageRows { const rows: Record[] = []; let invalidLines = 0; @@ -372,18 +407,23 @@ function parseUsageRows(text: string): ParsedUsageRows { return { rows, invalidLines }; } +/** Select rows with valid timestamps inside the inclusive analysis window. */ function rowsInWindow( rows: readonly Record[], since: number, nowMs: number, ): Record[] { - return rows.filter(row => { + return rows.filter((row) => { const timestamp = nonNegativeNumber(row.timestamp); return timestamp !== undefined && timestamp >= since && timestamp <= nowMs; }); } -function recordConversation(cohort: Cohort, row: Record): void { +/** Increment a cohort conversation count when the row has a non-blank identifier. */ +function recordConversation( + cohort: Cohort, + row: Record, +): void { const conversationId = stringValue(row.conversationId); if (!conversationId) return; cohort.conversations.set( @@ -392,6 +432,7 @@ function recordConversation(cohort: Cohort, row: Record): void ); } +/** Accumulate validated usage into cohort and global counters, ignoring absent measurements. */ function recordCacheUsage( cohort: Cohort, usage: CacheUsage | undefined, @@ -411,6 +452,7 @@ function recordCacheUsage( totals.write += usage.write; } +/** Validate a row observation and add it to the matching cohort and cache-shape bucket. */ function recordCacheShape( buckets: Map, key: string, @@ -428,6 +470,7 @@ function recordCacheShape( buckets.set(shapeKey, { dimensions, observation, rows: [row] }); } +/** Group selected rows into cohorts and shapes while accumulating valid usage totals. */ function analyzeRows(rows: readonly Record[]): AnalysisState { const cohorts = new Map(); const shapeBuckets = new Map(); @@ -452,25 +495,31 @@ function analyzeRows(rows: readonly Record[]): AnalysisState { return { cohorts, shapeBuckets, totals }; } -function topCohorts(cohorts: Map, top: number): CohortSummary[] { +/** Select cohorts with measured successes, ordered by descending input tokens and capped at top. */ +function topCohorts( + cohorts: Map, + top: number, +): CohortSummary[] { return [...cohorts.values()] - .filter(cohort => cohort.reportedSuccess > 0) + .filter((cohort) => cohort.reportedSuccess > 0) .sort((a, b) => b.input - a.input) .slice(0, top) .map(summarizeCohort); } +/** Select cache shapes with measured successes, ordered by descending input tokens and capped at top. */ function topShapes( buckets: Map, top: number, ): ShapeSummary[] { return [...buckets.values()] .map(summarizeShape) - .filter(shape => shape.reportedSuccess > 0) + .filter((shape) => shape.reportedSuccess > 0) .sort((a, b) => b.inputTokens - a.inputTokens) .slice(0, top); } +/** Assemble window metadata, the measurement boundary, totals, and ranked summaries. */ function buildOutput( options: AnalyzerOptions, since: number, @@ -507,66 +556,109 @@ function buildOutput( type AnalyzerOutput = ReturnType; +/** Print the analysis window, aggregate usage, ranked cohorts, and cache shapes. */ function printHumanOutput(output: AnalyzerOutput): void { console.log("Prompt cache usage (" + output.range + ")"); console.log("window: " + output.windowStart + " .. " + output.windowEnd); console.log("proof: " + output.proofBoundary); console.log( - "reported-success=" + output.summary.reportedSuccess - + " input=" + Math.round(output.summary.inputTokens) - + " read=" + Math.round(output.summary.cacheReadTokens) - + " (" + formatPercent(output.summary.cacheReadRatio) + ")" - + " write=" + Math.round(output.summary.cacheWriteTokens) - + " (" + formatPercent(output.summary.cacheWriteRatio) + ")", + "reported-success=" + + output.summary.reportedSuccess + + " input=" + + Math.round(output.summary.inputTokens) + + " read=" + + Math.round(output.summary.cacheReadTokens) + + " (" + + formatPercent(output.summary.cacheReadRatio) + + ")" + + " write=" + + Math.round(output.summary.cacheWriteTokens) + + " (" + + formatPercent(output.summary.cacheWriteRatio) + + ")", ); console.log(""); console.log("Top cohorts by measured input:"); for (const item of output.cohorts) { console.log( - item.signal.padEnd(11) - + " " + item.adapter + "/" + item.provider + "/" + item.model - + " surface=" + item.surface - + " input=" + item.inputTokens - + " read=" + formatPercent(item.cacheReadRatio) - + " write=" + formatPercent(item.cacheWriteRatio) - + " hits=" + formatPercent(item.cacheHitRequestRatio) - + " n=" + item.reportedSuccess + "/" + item.requests, + item.signal.padEnd(11) + + " " + + item.adapter + + "/" + + item.provider + + "/" + + item.model + + " surface=" + + item.surface + + " input=" + + item.inputTokens + + " read=" + + formatPercent(item.cacheReadRatio) + + " write=" + + formatPercent(item.cacheWriteRatio) + + " hits=" + + formatPercent(item.cacheHitRequestRatio) + + " n=" + + item.reportedSuccess + + "/" + + item.requests, ); } printCacheShapes(output.cacheShapes); } +/** Print measured cache shapes or explain that the window contains no observations. */ function printCacheShapes(shapes: readonly ShapeSummary[]): void { console.log(""); if (shapes.length === 0) { - console.log("No promptCache observations in this window (historical rows predate instrumentation)."); + console.log( + "No promptCache observations in this window (historical rows predate instrumentation).", + ); return; } console.log("Observed outbound cache shapes:"); for (const item of shapes) { console.log( - item.adapter + "/" + item.provider + "/" + item.model - + " surface=" + item.surface - + " mode=" + item.mode - + " tools=" + item.toolCount - + " bp=" + item.breakpointCount - + " input=" + item.inputTokens - + " read=" + formatPercent(item.cacheReadRatio) - + " write=" + formatPercent(item.cacheWriteRatio) - + " n=" + item.reportedSuccess + "/" + item.requests - + " prefix=" + (item.stablePrefixFingerprint ?? "-") - + " toolsHash=" + (item.toolsFingerprint ?? "-"), + item.adapter + + "/" + + item.provider + + "/" + + item.model + + " surface=" + + item.surface + + " mode=" + + item.mode + + " tools=" + + item.toolCount + + " bp=" + + item.breakpointCount + + " input=" + + item.inputTokens + + " read=" + + formatPercent(item.cacheReadRatio) + + " write=" + + formatPercent(item.cacheWriteRatio) + + " n=" + + item.reportedSuccess + + "/" + + item.requests + + " prefix=" + + (item.stablePrefixFingerprint ?? "-") + + " toolsHash=" + + (item.toolsFingerprint ?? "-"), ); } } +/** Run the analyzer and return zero on success or help, or two for invalid CLI arguments. */ function main(): number { let options: AnalyzerOptions; try { options = parseArgs(Bun.argv.slice(2)); } catch (error) { - const message = error instanceof Error ? error.message : "invalid arguments"; + const message = + error instanceof Error ? error.message : "invalid arguments"; console.error(message); console.error(usageText()); return 2; diff --git a/src/prompt-cache/observability.ts b/src/prompt-cache/observability.ts index 13d8cd153..edbec6a21 100644 --- a/src/prompt-cache/observability.ts +++ b/src/prompt-cache/observability.ts @@ -22,6 +22,7 @@ export interface PromptCacheRequestObservation { verbosity?: PromptCacheVerbosity; } +/** Narrow a value to a non-null, non-array object before reading request fields. */ function isRecord(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } @@ -33,7 +34,7 @@ function isRecord(value: unknown): value is Record { function canonicalJson(value: unknown): string | undefined { if (value === null) return "null"; if (Array.isArray(value)) { - const items = value.map(item => canonicalJson(item) ?? "null"); + const items = value.map((item) => canonicalJson(item) ?? "null"); return "[" + items.join(",") + "]"; } if (isRecord(value)) { @@ -47,24 +48,29 @@ function canonicalJson(value: unknown): string | undefined { } if (typeof value === "number" && !Number.isFinite(value)) return "null"; if ( - typeof value === "string" - || typeof value === "number" - || typeof value === "boolean" + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" ) { return JSON.stringify(value); } return undefined; } +/** Hash canonical JSON to 24 lowercase hex characters, or return undefined for unsupported values. */ function fingerprint(value: unknown): string | undefined { const serialized = canonicalJson(value); if (serialized === undefined) return undefined; return createHash("sha256").update(serialized).digest("hex").slice(0, 24); } +/** Recursively count explicit prompt-cache breakpoints without descending into breakpoint metadata. */ function countExplicitBreakpoints(value: unknown): number { if (Array.isArray(value)) { - return value.reduce((total, item) => total + countExplicitBreakpoints(item), 0); + return value.reduce( + (total, item) => total + countExplicitBreakpoints(item), + 0, + ); } if (!isRecord(value)) return 0; const breakpoint = value.prompt_cache_breakpoint; @@ -76,6 +82,7 @@ function countExplicitBreakpoints(value: unknown): number { return count; } +/** Collect instructions and consecutive leading developer/system messages for fingerprinting. */ function stablePrefix(body: Record): unknown[] { const prefix: unknown[] = []; if (body.instructions !== undefined) { @@ -90,37 +97,52 @@ function stablePrefix(body: Record): unknown[] { initialDeveloperItems.push(item); } if (initialDeveloperItems.length > 0) { - prefix.push({ kind: "initial_developer_messages", value: initialDeveloperItems }); + prefix.push({ + kind: "initial_developer_messages", + value: initialDeveloperItems, + }); } return prefix; } -function promptCacheVerbosity(value: unknown): PromptCacheVerbosity | undefined { +/** Accept only supported verbosity labels so arbitrary caller text is omitted from diagnostics. */ +function promptCacheVerbosity( + value: unknown, +): PromptCacheVerbosity | undefined { if (value === "low" || value === "medium" || value === "high") return value; return undefined; } +/** Check for non-whitespace string content without retaining the string value. */ function hasNonEmptyString(value: unknown): boolean { return typeof value === "string" && value.trim().length > 0; } -function promptCacheMode(options: Record | undefined): PromptCacheMode { +/** Read a recognized outbound cache mode, falling back to default for absent or unknown values. */ +function promptCacheMode( + options: Record | undefined, +): PromptCacheMode { if (options?.mode === "explicit") return "explicit"; if (options?.mode === "implicit") return "implicit"; return "default"; } +/** Extract the supported 30-minute TTL from outbound cache options. */ function promptCacheTtl( options: Record | undefined, ): PromptCacheRequestObservation["ttl"] { return options?.ttl === "30m" ? "30m" : undefined; } -function promptCacheLegacyRetention(value: unknown): PromptCacheLegacyRetention | undefined { +/** Accept only the supported legacy retention labels, omitting unknown values. */ +function promptCacheLegacyRetention( + value: unknown, +): PromptCacheLegacyRetention | undefined { if (value === "in_memory" || value === "24h") return value; return undefined; } +/** Count input-array items, treating absent input as zero and other input values as one. */ function promptCacheInputItemCount(input: unknown): number { if (Array.isArray(input)) return input.length; return input === undefined ? 0 : 1; @@ -133,6 +155,7 @@ interface PromptCacheObservationExtras { verbosity?: PromptCacheVerbosity; } +/** Fingerprint present tools, stable prefix, and text format, and include supported verbosity. */ function promptCacheObservationExtras(input: { tools: unknown[]; prefix: unknown[]; @@ -140,8 +163,10 @@ function promptCacheObservationExtras(input: { verbosity: PromptCacheVerbosity | undefined; }): PromptCacheObservationExtras { const extras: PromptCacheObservationExtras = {}; - if (input.tools.length > 0) extras.toolsFingerprint = fingerprint(input.tools); - if (input.prefix.length > 0) extras.stablePrefixFingerprint = fingerprint(input.prefix); + if (input.tools.length > 0) + extras.toolsFingerprint = fingerprint(input.tools); + if (input.prefix.length > 0) + extras.stablePrefixFingerprint = fingerprint(input.prefix); if (input.textFormat !== undefined) { extras.textFormatFingerprint = fingerprint(input.textFormat); } @@ -149,6 +174,11 @@ function promptCacheObservationExtras(input: { return extras; } +/** + * Describe cache settings and request structure from the final outbound Responses body. + * Retain fingerprints and presence flags rather than raw content or cache keys. + * Return undefined for non-object input without changing the supplied body. + */ export function observeOpenAiResponsesPromptCache( value: unknown, ): PromptCacheRequestObservation | undefined { @@ -182,15 +212,19 @@ export function observeOpenAiResponsesPromptCache( }; } +/** Check that a persisted fingerprint contains exactly 24 lowercase hexadecimal characters. */ function isFingerprint(value: unknown): value is string { return typeof value === "string" && /^[0-9a-f]{24}$/u.test(value); } +/** Accept persisted integer counts from zero through one million. */ function isBoundedCount(value: unknown): value is number { - return typeof value === "number" - && Number.isInteger(value) - && value >= 0 - && value <= 1_000_000; + return ( + typeof value === "number" && + Number.isInteger(value) && + value >= 0 && + value <= 1_000_000 + ); } interface PromptCacheObservationCore { @@ -203,27 +237,38 @@ interface PromptCacheObservationCore { toolCount: number; } +/** Validate the required boolean flags and bounded counts of a persisted observation. */ function hasPromptCacheObservationCore( value: Record, ): value is Record & PromptCacheObservationCore { - return typeof value.keyPresent === "boolean" - && typeof value.prewarm === "boolean" - && typeof value.comparisonRequested === "boolean" - && typeof value.previousResponseIdPresent === "boolean" - && isBoundedCount(value.breakpointCount) - && isBoundedCount(value.inputItemCount) - && isBoundedCount(value.toolCount); + return ( + typeof value.keyPresent === "boolean" && + typeof value.prewarm === "boolean" && + typeof value.comparisonRequested === "boolean" && + typeof value.previousResponseIdPresent === "boolean" && + isBoundedCount(value.breakpointCount) && + isBoundedCount(value.inputItemCount) && + isBoundedCount(value.toolCount) + ); } -function normalizedPromptCacheMode(value: unknown): PromptCacheMode | undefined { - if (value === "default" || value === "implicit" || value === "explicit") return value; +/** Accept a persisted cache-mode label, returning undefined for unrecognized values. */ +function normalizedPromptCacheMode( + value: unknown, +): PromptCacheMode | undefined { + if (value === "default" || value === "implicit" || value === "explicit") + return value; return undefined; } -function normalizedPromptCacheTtl(value: unknown): PromptCacheRequestObservation["ttl"] { +/** Retain only the supported 30-minute TTL from a persisted observation. */ +function normalizedPromptCacheTtl( + value: unknown, +): PromptCacheRequestObservation["ttl"] { return value === "30m" ? "30m" : undefined; } +/** Copy only well-formed fingerprints and supported verbosity from persisted metadata. */ function normalizedPromptCacheExtras( value: Record, ): PromptCacheObservationExtras { @@ -242,6 +287,10 @@ function normalizedPromptCacheExtras( return extras; } +/** + * Validate untrusted persisted metadata and rebuild an allowlisted version-1 observation. + * Reject invalid versions or required fields; omit unknown fields and invalid optional values. + */ export function normalizePromptCacheRequestObservation( value: unknown, ): PromptCacheRequestObservation | undefined { diff --git a/tests/openai-responses-passthrough.test.ts b/tests/openai-responses-passthrough.test.ts index 4f9742449..b8d33a812 100644 --- a/tests/openai-responses-passthrough.test.ts +++ b/tests/openai-responses-passthrough.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { createResponsesPassthroughAdapter } from "../src/adapters/openai-responses"; import { sanitizeEncryptedContentInPlace } from "../src/server/responses"; +import { observeOpenAiResponsesPromptCache } from "../src/prompt-cache/observability"; import { configuredReasoningEfforts } from "../src/reasoning-effort"; const provider = { @@ -17,13 +18,16 @@ function buildKeyAuthUrl(baseUrl: string, responsesPath?: string): string { apiKey: "sk-test", ...(responsesPath === undefined ? {} : { responsesPath }), }); - return adapter.buildRequest({ - modelId: "test-model", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { model: "test-model", input: "ping" }, - }, { headers: new Headers() }).url; + return adapter.buildRequest( + { + modelId: "test-model", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { model: "test-model", input: "ping" }, + }, + { headers: new Headers() }, + ).url; } describe("Responses noReasoningModels raw-body boundary", () => { @@ -34,25 +38,44 @@ describe("Responses noReasoningModels raw-body boundary", () => { noReasoningModels: ["Kimi-K2.6"], }; for (const stream of [true, false]) { - for (const reasoning of [{ effort: "low" }, { effort: "low", summary: "auto" }]) { + for (const reasoning of [ + { effort: "low" }, + { effort: "low", summary: "auto" }, + ]) { test(`strips only configured effort, stream=${stream}, summary=${reasoning.summary ?? "absent"}`, async () => { const rawBody = Object.freeze({ - model: "Kimi-K2.6", input: "ping", stream, + model: "Kimi-K2.6", + input: "ping", + stream, reasoning: Object.freeze(reasoning), metadata: Object.freeze({ label: "preserve" }), }); const original = JSON.stringify(rawBody); - const request = await createResponsesPassthroughAdapter(keyProvider).buildRequest({ - modelId: "Kimi-K2.6", context: { messages: [] }, stream, - options: { reasoning: "low" }, _rawBody: rawBody, + const request = await createResponsesPassthroughAdapter( + keyProvider, + ).buildRequest({ + modelId: "Kimi-K2.6", + context: { messages: [] }, + stream, + options: { reasoning: "low" }, + _rawBody: rawBody, }); const body = JSON.parse(request.body); - expect(body).toEqual(reasoning.summary - ? { ...rawBody, reasoning: { summary: "auto" } } - : { model: rawBody.model, input: "ping", stream, metadata: rawBody.metadata }); + expect(body).toEqual( + reasoning.summary + ? { ...rawBody, reasoning: { summary: "auto" } } + : { + model: rawBody.model, + input: "ping", + stream, + metadata: rawBody.metadata, + }, + ); expect(JSON.stringify(rawBody)).toBe(original); expect(request.reasoningLog).toBeUndefined(); - expect(configuredReasoningEfforts(keyProvider, "Kimi-K2.6")).toEqual([]); + expect(configuredReasoningEfforts(keyProvider, "Kimi-K2.6")).toEqual( + [], + ); }); } } @@ -64,10 +87,23 @@ describe("Responses noReasoningModels raw-body boundary", () => { ["Kimi-K2.6", { effort: "low" }, undefined], ["Kimi-K2.6", { effort: "low" }, []], ] as const) { - const rawBody = { model: modelId, input: "ping", ...(reasoning ? { reasoning } : {}) }; + const rawBody = { + model: modelId, + input: "ping", + ...(reasoning ? { reasoning } : {}), + }; const request = await createResponsesPassthroughAdapter({ - ...keyProvider, noReasoningModels: noReasoningModels ? [...noReasoningModels] : undefined, - }).buildRequest({ modelId, context: { messages: [] }, stream: false, options: {}, _rawBody: rawBody }); + ...keyProvider, + noReasoningModels: noReasoningModels + ? [...noReasoningModels] + : undefined, + }).buildRequest({ + modelId, + context: { messages: [] }, + stream: false, + options: {}, + _rawBody: rawBody, + }); expect(JSON.parse(request.body)).toEqual(rawBody); } }); @@ -77,23 +113,32 @@ describe("OpenAI Responses key-auth URL construction", () => { test("BUG-R289 preserves legacy /v1/responses URL when responsesPath is absent", () => { for (const [baseUrl, expectedUrl] of [ ["https://api.openai.example", "https://api.openai.example/v1/responses"], - ["https://api.openai.example/v1", "https://api.openai.example/v1/responses"], - ["https://api.openai.example/v1/", "https://api.openai.example/v1/responses"], + [ + "https://api.openai.example/v1", + "https://api.openai.example/v1/responses", + ], + [ + "https://api.openai.example/v1/", + "https://api.openai.example/v1/responses", + ], ] as const) { expect(buildKeyAuthUrl(baseUrl)).toBe(expectedUrl); } }); test("BUG-R289 appends responsesPath to a baseUrl with one trailing slash", () => { - expect(buildKeyAuthUrl("https://gateway.example/api/v3/", "/responses")) - .toBe("https://gateway.example/api/v3/responses"); + expect( + buildKeyAuthUrl("https://gateway.example/api/v3/", "/responses"), + ).toBe("https://gateway.example/api/v3/responses"); }); test("BUG-R289 routes Volcengine Ark Agent Plan to /api/plan/v3/responses", () => { - expect(buildKeyAuthUrl( - "https://ark.cn-beijing.volces.com/api/plan/v3", - "/responses", - )).toBe("https://ark.cn-beijing.volces.com/api/plan/v3/responses"); + expect( + buildKeyAuthUrl( + "https://ark.cn-beijing.volces.com/api/plan/v3", + "/responses", + ), + ).toBe("https://ark.cn-beijing.volces.com/api/plan/v3/responses"); }); }); @@ -106,26 +151,32 @@ describe("OpenAI Responses passthrough sanitization", () => { apiKey: "sk-test", modelSupportsReasoningSummaries: { "strict-summary-model": false }, }); - const request = adapter.buildRequest({ - modelId: "strict-summary-model", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "strict-summary-model", - input: [], - stream_options: { - include_usage: true, - reasoning_summary_delivery: "sequential_cutoff", - }, - reasoning: { - effort: "high", - summary: "auto", - generate_summary: true, + const request = adapter.buildRequest( + { + modelId: "strict-summary-model", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "strict-summary-model", + input: [], + stream_options: { + include_usage: true, + reasoning_summary_delivery: "sequential_cutoff", + }, + reasoning: { + effort: "high", + summary: "auto", + generate_summary: true, + }, }, }, - }, { headers: new Headers() }); - const body = JSON.parse(request.body) as Record>; + { headers: new Headers() }, + ); + const body = JSON.parse(request.body) as Record< + string, + Record + >; expect(body.stream_options).toEqual({ include_usage: true }); expect(body.reasoning).toEqual({ effort: "high" }); @@ -139,22 +190,28 @@ describe("OpenAI Responses passthrough sanitization", () => { apiKey: "sk-test", modelReasoningSummaryDelivery: { "summary-model": "sequential" }, }); - const request = adapter.buildRequest({ - modelId: "summary-model", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "summary-model", - input: [], - stream_options: { - include_usage: true, - reasoning_summary_delivery: "sequential_cutoff", + const request = adapter.buildRequest( + { + modelId: "summary-model", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "summary-model", + input: [], + stream_options: { + include_usage: true, + reasoning_summary_delivery: "sequential_cutoff", + }, + reasoning: { effort: "high", summary: "auto" }, }, - reasoning: { effort: "high", summary: "auto" }, }, - }, { headers: new Headers() }); - const body = JSON.parse(request.body) as Record>; + { headers: new Headers() }, + ); + const body = JSON.parse(request.body) as Record< + string, + Record + >; expect(body.stream_options).toEqual({ include_usage: true, @@ -171,18 +228,24 @@ describe("OpenAI Responses passthrough sanitization", () => { apiKey: "sk-test", modelReasoningSummaryDelivery: { "summary-model": "concurrent" }, }); - const request = adapter.buildRequest({ - modelId: "summary-model", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "summary-model", - input: [], - stream_options: { include_usage: true }, + const request = adapter.buildRequest( + { + modelId: "summary-model", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "summary-model", + input: [], + stream_options: { include_usage: true }, + }, }, - }, { headers: new Headers() }); - const body = JSON.parse(request.body) as Record>; + { headers: new Headers() }, + ); + const body = JSON.parse(request.body) as Record< + string, + Record + >; expect(body.stream_options).toEqual({ include_usage: true }); }); @@ -194,30 +257,42 @@ describe("OpenAI Responses passthrough sanitization", () => { authMode: "key", apiKey: "sk-test", }); - const request = adapter.buildRequest({ - modelId: "normal-model", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "normal-model", - input: [], - stream_options: { reasoning_summary_delivery: "sequential_cutoff" }, + const request = adapter.buildRequest( + { + modelId: "normal-model", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "normal-model", + input: [], + stream_options: { reasoning_summary_delivery: "sequential_cutoff" }, + }, }, - }, { headers: new Headers() }); - const body = JSON.parse(request.body) as Record>; + { headers: new Headers() }, + ); + const body = JSON.parse(request.body) as Record< + string, + Record + >; - expect(body.stream_options).toEqual({ reasoning_summary_delivery: "sequential_cutoff" }); + expect(body.stream_options).toEqual({ + reasoning_summary_delivery: "sequential_cutoff", + }); }); test("agent_message conversion removes its non-OpenAI item id", () => { - const input = [{ - type: "agent_message", - id: "019f5e7f-ac31-7610-b69c-43ae41759fce", - author: "/root", - recipient: "/root/worker", - content: [{ type: "encrypted_content", encrypted_content: "delegated task" }], - }]; + const input = [ + { + type: "agent_message", + id: "019f5e7f-ac31-7610-b69c-43ae41759fce", + author: "/root", + recipient: "/root/worker", + content: [ + { type: "encrypted_content", encrypted_content: "delegated task" }, + ], + }, + ]; expect(sanitizeEncryptedContentInPlace(input)).toBe(1); expect(input[0]).toEqual({ @@ -232,31 +307,126 @@ describe("OpenAI Responses passthrough sanitization", () => { const adapter = createResponsesPassthroughAdapter(provider); const encryptedContent = "opaque-openai-encrypted-content"; const cases = [ - { item: { type: "message", id: "019f5e7f-ac31-7610-b69c-43ae41759fce", role: "user", content: "first" }, expectedId: undefined }, - { item: { type: "message", id: "msg_abc", role: "assistant", content: "second" }, expectedId: "msg_abc" }, - { item: { type: "custom_tool_call", id: "fc_old", call_id: "call_1", name: "patch", input: "old" }, expectedId: undefined }, - { item: { type: "custom_tool_call", id: "ctc_1", call_id: "call_2", name: "patch", input: "new" }, expectedId: "ctc_1" }, - { item: { type: "function_call", id: "fc_1", call_id: "call_3", name: "ping", arguments: "{}" }, expectedId: "fc_1" }, - { item: { type: "reasoning", id: "rs_1", summary: [], encrypted_content: encryptedContent }, expectedId: "rs_1" }, - { item: { type: "tool_search_call", id: "fc_old_search", call_id: "call_4", execution: "client", arguments: {} }, expectedId: undefined }, - { item: { type: "tool_search_call", id: "tsc_1", call_id: "call_5", execution: "client", arguments: {} }, expectedId: "tsc_1" }, - { item: { type: "web_search_call", id: "fc_wrong", status: "completed" }, expectedId: undefined }, - { item: { type: "web_search_call", id: "ws_valid", status: "completed" }, expectedId: "ws_valid" }, - { item: { type: "agent_message", id: "msg_wrong-dialect", content: [{ type: "output_text", text: "routed reply" }] }, expectedId: undefined }, - { item: { type: "agent_message", id: "amsg_1", content: [{ type: "output_text", text: "routed reply" }] }, expectedId: "amsg_1" }, + { + item: { + type: "message", + id: "019f5e7f-ac31-7610-b69c-43ae41759fce", + role: "user", + content: "first", + }, + expectedId: undefined, + }, + { + item: { + type: "message", + id: "msg_abc", + role: "assistant", + content: "second", + }, + expectedId: "msg_abc", + }, + { + item: { + type: "custom_tool_call", + id: "fc_old", + call_id: "call_1", + name: "patch", + input: "old", + }, + expectedId: undefined, + }, + { + item: { + type: "custom_tool_call", + id: "ctc_1", + call_id: "call_2", + name: "patch", + input: "new", + }, + expectedId: "ctc_1", + }, + { + item: { + type: "function_call", + id: "fc_1", + call_id: "call_3", + name: "ping", + arguments: "{}", + }, + expectedId: "fc_1", + }, + { + item: { + type: "reasoning", + id: "rs_1", + summary: [], + encrypted_content: encryptedContent, + }, + expectedId: "rs_1", + }, + { + item: { + type: "tool_search_call", + id: "fc_old_search", + call_id: "call_4", + execution: "client", + arguments: {}, + }, + expectedId: undefined, + }, + { + item: { + type: "tool_search_call", + id: "tsc_1", + call_id: "call_5", + execution: "client", + arguments: {}, + }, + expectedId: "tsc_1", + }, + { + item: { type: "web_search_call", id: "fc_wrong", status: "completed" }, + expectedId: undefined, + }, + { + item: { type: "web_search_call", id: "ws_valid", status: "completed" }, + expectedId: "ws_valid", + }, + { + item: { + type: "agent_message", + id: "msg_wrong-dialect", + content: [{ type: "output_text", text: "routed reply" }], + }, + expectedId: undefined, + }, + { + item: { + type: "agent_message", + id: "amsg_1", + content: [{ type: "output_text", text: "routed reply" }], + }, + expectedId: "amsg_1", + }, ]; const input = cases.map(({ item }) => item); - const request = adapter.buildRequest({ - modelId: "gpt-5.5", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { model: "gpt-5.5", input }, - }, { headers: new Headers({ authorization: "Bearer token" }) }); - const body = JSON.parse(request.body) as { input: Record[] }; + const request = adapter.buildRequest( + { + modelId: "gpt-5.5", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { model: "gpt-5.5", input }, + }, + { headers: new Headers({ authorization: "Bearer token" }) }, + ); + const body = JSON.parse(request.body) as { + input: Record[]; + }; cases.forEach(({ expectedId }, index) => { - if (expectedId === undefined) expect(body.input[index]).not.toHaveProperty("id"); + if (expectedId === undefined) + expect(body.input[index]).not.toHaveProperty("id"); else expect(body.input[index].id).toBe(expectedId); }); expect(body.input[5]).toEqual(input[5]); @@ -266,64 +436,100 @@ describe("OpenAI Responses passthrough sanitization", () => { const adapter = createResponsesPassthroughAdapter(provider); const input = [ { type: "message", id: "msg_abc", role: "assistant", content: "hello" }, - { type: "function_call", id: "fc_xyz", call_id: "call_1", name: "ping", arguments: "{}" }, + { + type: "function_call", + id: "fc_xyz", + call_id: "call_1", + name: "ping", + arguments: "{}", + }, { type: "reasoning", id: "rs_123", summary: [] }, ]; - const unstoredBody = JSON.parse(adapter.buildRequest({ - modelId: "gpt-5.5", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { model: "gpt-5.5", store: false, input }, - }, { headers: new Headers({ authorization: "Bearer token" }) }).body) as { input: Record[] }; + const unstoredBody = JSON.parse( + adapter.buildRequest( + { + modelId: "gpt-5.5", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { model: "gpt-5.5", store: false, input }, + }, + { headers: new Headers({ authorization: "Bearer token" }) }, + ).body, + ) as { input: Record[] }; - unstoredBody.input.forEach(item => expect(item).not.toHaveProperty("id")); + unstoredBody.input.forEach((item) => expect(item).not.toHaveProperty("id")); expect(unstoredBody.input[1].call_id).toBe("call_1"); - const omittedStoreBody = JSON.parse(adapter.buildRequest({ - modelId: "gpt-5.5", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { model: "gpt-5.5", input }, - }, { headers: new Headers({ authorization: "Bearer token" }) }).body) as { input: Record[] }; - const storedBody = JSON.parse(adapter.buildRequest({ - modelId: "gpt-5.5", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { model: "gpt-5.5", store: true, input }, - }, { headers: new Headers({ authorization: "Bearer token" }) }).body) as { input: Record[] }; - - expect(omittedStoreBody.input.map(item => item.id)).toEqual(["msg_abc", "fc_xyz", "rs_123"]); - expect(storedBody.input.map(item => item.id)).toEqual(["msg_abc", "fc_xyz", "rs_123"]); + const omittedStoreBody = JSON.parse( + adapter.buildRequest( + { + modelId: "gpt-5.5", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { model: "gpt-5.5", input }, + }, + { headers: new Headers({ authorization: "Bearer token" }) }, + ).body, + ) as { input: Record[] }; + const storedBody = JSON.parse( + adapter.buildRequest( + { + modelId: "gpt-5.5", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { model: "gpt-5.5", store: true, input }, + }, + { headers: new Headers({ authorization: "Bearer token" }) }, + ).body, + ) as { input: Record[] }; + + expect(omittedStoreBody.input.map((item) => item.id)).toEqual([ + "msg_abc", + "fc_xyz", + "rs_123", + ]); + expect(storedBody.input.map((item) => item.id)).toEqual([ + "msg_abc", + "fc_xyz", + "rs_123", + ]); }); test("drops raw reasoning input content before native GPT passthrough", () => { const adapter = createResponsesPassthroughAdapter(provider); - const request = adapter.buildRequest({ - modelId: "gpt-5.5", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.5", - input: [ - { - type: "reasoning", - id: "rs_1", - summary: [], - content: [{ type: "reasoning_text", text: "raw routed reasoning" }], - }, - { - type: "message", - role: "user", - content: [{ type: "input_text", text: "hi" }], - }, - ], + const request = adapter.buildRequest( + { + modelId: "gpt-5.5", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.5", + input: [ + { + type: "reasoning", + id: "rs_1", + summary: [], + content: [ + { type: "reasoning_text", text: "raw routed reasoning" }, + ], + }, + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "hi" }], + }, + ], + }, }, - }, { headers: new Headers({ authorization: "Bearer token" }) }); - const body = JSON.parse(request.body) as { input: Record[] }; + { headers: new Headers({ authorization: "Bearer token" }) }, + ); + const body = JSON.parse(request.body) as { + input: Record[]; + }; expect(body.input[0]).toMatchObject({ type: "reasoning", @@ -340,40 +546,46 @@ describe("OpenAI Responses passthrough sanitization", () => { test("strips image_generation hosted tool for codex-spark passthrough", () => { const adapter = createResponsesPassthroughAdapter(provider); - const request = adapter.buildRequest({ - modelId: "gpt-5.3-codex-spark", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.3-codex-spark", - input: [], - tools: [ - { type: "function", name: "shell", parameters: {} }, - { type: "image_generation" }, - ], + const request = adapter.buildRequest( + { + modelId: "gpt-5.3-codex-spark", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.3-codex-spark", + input: [], + tools: [ + { type: "function", name: "shell", parameters: {} }, + { type: "image_generation" }, + ], + }, }, - }, { headers: new Headers({ authorization: "Bearer token" }) }); + { headers: new Headers({ authorization: "Bearer token" }) }, + ); const body = JSON.parse(request.body) as { tools: { type: string }[] }; expect(body.tools).toHaveLength(1); expect(body.tools[0]).toMatchObject({ type: "function", name: "shell" }); - expect(body.tools.some(t => t.type === "image_generation")).toBe(false); + expect(body.tools.some((t) => t.type === "image_generation")).toBe(false); }); test("keeps image_generation hosted tool for supported native slugs", () => { const adapter = createResponsesPassthroughAdapter(provider); - const request = adapter.buildRequest({ - modelId: "gpt-5.5", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.5", - input: [], - tools: [{ type: "image_generation" }], + const request = adapter.buildRequest( + { + modelId: "gpt-5.5", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.5", + input: [], + tools: [{ type: "image_generation" }], + }, }, - }, { headers: new Headers({ authorization: "Bearer token" }) }); + { headers: new Headers({ authorization: "Bearer token" }) }, + ); const body = JSON.parse(request.body) as { tools: { type: string }[] }; expect(body.tools).toHaveLength(1); @@ -382,17 +594,20 @@ describe("OpenAI Responses passthrough sanitization", () => { test("preserves prompt_cache_key in the raw Responses passthrough body", () => { const adapter = createResponsesPassthroughAdapter(provider); - const request = adapter.buildRequest({ - modelId: "gpt-5.5", - context: { messages: [] }, - stream: true, - options: { promptCacheKey: "project-cache-v1" }, - _rawBody: { - model: "gpt-5.5", - input: "hi", - prompt_cache_key: "project-cache-v1", + const request = adapter.buildRequest( + { + modelId: "gpt-5.5", + context: { messages: [] }, + stream: true, + options: { promptCacheKey: "project-cache-v1" }, + _rawBody: { + model: "gpt-5.5", + input: "hi", + prompt_cache_key: "project-cache-v1", + }, }, - }, { headers: new Headers({ authorization: "Bearer token" }) }); + { headers: new Headers({ authorization: "Bearer token" }) }, + ); const body = JSON.parse(request.body) as { prompt_cache_key?: string }; expect(body.prompt_cache_key).toBe("project-cache-v1"); @@ -400,49 +615,64 @@ describe("OpenAI Responses passthrough sanitization", () => { test("preserves prompt_cache_retention in the raw Responses passthrough body", () => { const adapter = createResponsesPassthroughAdapter(provider); - const request = adapter.buildRequest({ - modelId: "gpt-5.5", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.5", - input: "hi", - prompt_cache_retention: "24h", + const request = adapter.buildRequest( + { + modelId: "gpt-5.5", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.5", + input: "hi", + prompt_cache_retention: "24h", + }, }, - }, { headers: new Headers({ authorization: "Bearer token" }) }); - const body = JSON.parse(request.body) as { prompt_cache_retention?: string }; + { headers: new Headers({ authorization: "Bearer token" }) }, + ); + const body = JSON.parse(request.body) as { + prompt_cache_retention?: string; + }; expect(body.prompt_cache_retention).toBe("24h"); }); test("records structural prompt-cache metadata from the final outbound body", () => { const adapter = createResponsesPassthroughAdapter(provider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-terra", - context: { messages: [] }, - stream: true, - options: { promptCacheKey: "private-cache-key" }, - _rawBody: { - model: "gpt-5.6-terra", - instructions: "private fixed instruction", - prompt_cache_key: "private-cache-key", - prompt_cache_options: { mode: "implicit", ttl: "30m" }, - text: { verbosity: "medium", format: { type: "text" } }, - tools: [{ type: "function", name: "shell", parameters: { type: "object" } }], - input: [{ - role: "developer", - content: [{ - type: "input_text", - text: "private developer text", - prompt_cache_breakpoint: { mode: "explicit" }, - }], - }, { - role: "user", - content: [{ type: "input_text", text: "private user text" }], - }], + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-terra", + context: { messages: [] }, + stream: true, + options: { promptCacheKey: "private-cache-key" }, + _rawBody: { + model: "gpt-5.6-terra", + instructions: "private fixed instruction", + prompt_cache_key: "private-cache-key", + prompt_cache_options: { mode: "implicit", ttl: "30m" }, + text: { verbosity: "medium", format: { type: "text" } }, + tools: [ + { type: "function", name: "shell", parameters: { type: "object" } }, + ], + input: [ + { + role: "developer", + content: [ + { + type: "input_text", + text: "private developer text", + prompt_cache_breakpoint: { mode: "explicit" }, + }, + ], + }, + { + role: "user", + content: [{ type: "input_text", text: "private user text" }], + }, + ], + }, }, - }, { headers: new Headers({ authorization: "Bearer token" }) }); + { headers: new Headers({ authorization: "Bearer token" }) }, + ); expect(request.promptCacheLog).toMatchObject({ version: 1, @@ -455,9 +685,15 @@ describe("OpenAI Responses passthrough sanitization", () => { verbosity: "medium", }); expect(request.promptCacheLog?.toolsFingerprint).toMatch(/^[0-9a-f]{24}$/); - expect(request.promptCacheLog?.stablePrefixFingerprint).toMatch(/^[0-9a-f]{24}$/); - expect(JSON.stringify(request.promptCacheLog)).not.toContain("private-cache-key"); - expect(JSON.stringify(request.promptCacheLog)).not.toContain("private developer text"); + expect(request.promptCacheLog?.stablePrefixFingerprint).toMatch( + /^[0-9a-f]{24}$/, + ); + expect(JSON.stringify(request.promptCacheLog)).not.toContain( + "private-cache-key", + ); + expect(JSON.stringify(request.promptCacheLog)).not.toContain( + "private developer text", + ); expect(JSON.parse(request.body)).toMatchObject({ prompt_cache_key: "private-cache-key", prompt_cache_options: { mode: "implicit", ttl: "30m" }, @@ -469,13 +705,19 @@ describe("OpenAI Responses passthrough sanitization", () => { previous_response_id: "resp_1", input: [ { role: "user", content: "first" }, - { type: "message", role: "assistant", content: [{ type: "output_text", text: "ok" }] }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "ok" }], + }, { type: "function_call_output", call_id: "call_1", output: "done" }, ], }; const deltaRawBody = { ...expandedRawBody, - input: [{ type: "function_call_output", call_id: "call_1", output: "done" }], + input: [ + { type: "function_call_output", call_id: "call_1", output: "done" }, + ], }; const parsedBase = { modelId: "gpt-5.5", @@ -489,20 +731,30 @@ describe("OpenAI Responses passthrough sanitization", () => { test("forward mode always drops previous_response_id (ChatGPT backend rejects it)", () => { const adapter = createResponsesPassthroughAdapter(provider); - const expandedBody = JSON.parse(adapter.buildRequest({ - ...parsedBase, - _previousResponseInputExpanded: true, - _rawBody: expandedRawBody, - }, meta).body) as { previous_response_id?: string; input: unknown[] }; + const expandedBody = JSON.parse( + adapter.buildRequest( + { + ...parsedBase, + _previousResponseInputExpanded: true, + _rawBody: expandedRawBody, + }, + meta, + ).body, + ) as { previous_response_id?: string; input: unknown[] }; expect(expandedBody.previous_response_id).toBeUndefined(); expect(expandedBody.input).toHaveLength(3); // Unexpanded miss (proxy restart, TTL, prior passthrough turn): the field must STILL be // stripped — the Codex REST backend 400s on it ({"detail":"Unsupported parameter: ..."}). - const rawDeltaBody = JSON.parse(adapter.buildRequest({ - ...parsedBase, - _rawBody: deltaRawBody, - }, meta).body) as { previous_response_id?: string; input: unknown[] }; + const rawDeltaBody = JSON.parse( + adapter.buildRequest( + { + ...parsedBase, + _rawBody: deltaRawBody, + }, + meta, + ).body, + ) as { previous_response_id?: string; input: unknown[] }; expect(rawDeltaBody.previous_response_id).toBeUndefined(); expect(rawDeltaBody.input).toHaveLength(1); }); @@ -515,38 +767,61 @@ describe("OpenAI Responses passthrough sanitization", () => { apiKey: "sk-test", }); - const expandedBody = JSON.parse(adapter.buildRequest({ - ...parsedBase, - _previousResponseInputExpanded: true, - _rawBody: expandedRawBody, - }, meta).body) as { previous_response_id?: string; input: unknown[] }; + const expandedBody = JSON.parse( + adapter.buildRequest( + { + ...parsedBase, + _previousResponseInputExpanded: true, + _rawBody: expandedRawBody, + }, + meta, + ).body, + ) as { previous_response_id?: string; input: unknown[] }; expect(expandedBody.previous_response_id).toBeUndefined(); expect(expandedBody.input).toHaveLength(3); // Platform /v1/responses supports server-side storage; an unexpanded id stays intact. - const rawDeltaBody = JSON.parse(adapter.buildRequest({ - ...parsedBase, - _rawBody: deltaRawBody, - }, meta).body) as { previous_response_id?: string; input: unknown[] }; + const rawDeltaBody = JSON.parse( + adapter.buildRequest( + { + ...parsedBase, + _rawBody: deltaRawBody, + }, + meta, + ).body, + ) as { previous_response_id?: string; input: unknown[] }; expect(rawDeltaBody.previous_response_id).toBe("resp_1"); expect(rawDeltaBody.input).toHaveLength(1); }); test("forward unexpanded miss converts orphan tool outputs and drops reasoning", () => { const adapter = createResponsesPassthroughAdapter(provider); - const body = JSON.parse(adapter.buildRequest({ - ...parsedBase, - _rawBody: { - model: "gpt-5.5", - previous_response_id: "resp_gone", - input: [ - { type: "reasoning", id: "rs_1", summary: [] }, - { type: "function_call_output", call_id: "call_orphan", output: "tool said hi" }, - { type: "custom_tool_call_output", call_id: "call_custom", output: [{ type: "output_text", text: "custom out" }] }, - { role: "user", content: "next question" }, - ], - }, - }, meta).body) as { previous_response_id?: string; input: Record[] }; + const body = JSON.parse( + adapter.buildRequest( + { + ...parsedBase, + _rawBody: { + model: "gpt-5.5", + previous_response_id: "resp_gone", + input: [ + { type: "reasoning", id: "rs_1", summary: [] }, + { + type: "function_call_output", + call_id: "call_orphan", + output: "tool said hi", + }, + { + type: "custom_tool_call_output", + call_id: "call_custom", + output: [{ type: "output_text", text: "custom out" }], + }, + { role: "user", content: "next question" }, + ], + }, + }, + meta, + ).body, + ) as { previous_response_id?: string; input: Record[] }; expect(body.previous_response_id).toBeUndefined(); // reasoning dropped, both orphan outputs converted to user messages, user message intact @@ -554,32 +829,59 @@ describe("OpenAI Responses passthrough sanitization", () => { expect(body.input[0]).toMatchObject({ type: "message", role: "user", - content: [{ type: "input_text", text: "[tool output for call_orphan]\ntool said hi" }], + content: [ + { + type: "input_text", + text: "[tool output for call_orphan]\ntool said hi", + }, + ], }); expect(body.input[1]).toMatchObject({ type: "message", role: "user", - content: [{ type: "input_text", text: "[tool output for call_custom]\ncustom out" }], + content: [ + { + type: "input_text", + text: "[tool output for call_custom]\ncustom out", + }, + ], + }); + expect(body.input[2]).toMatchObject({ + role: "user", + content: "next question", }); - expect(body.input[2]).toMatchObject({ role: "user", content: "next question" }); }); test("forward mode keeps paired tool outputs and local_shell_call pairs intact", () => { const adapter = createResponsesPassthroughAdapter(provider); const input = [ - { type: "function_call", call_id: "call_fn", name: "ping", arguments: "{}" }, + { + type: "function_call", + call_id: "call_fn", + name: "ping", + arguments: "{}", + }, { type: "function_call_output", call_id: "call_fn", output: "pong" }, - { type: "local_shell_call", call_id: "call_sh", action: { type: "exec", command: ["ls"] } }, + { + type: "local_shell_call", + call_id: "call_sh", + action: { type: "exec", command: ["ls"] }, + }, { type: "function_call_output", call_id: "call_sh", output: "files" }, { role: "user", content: "go on" }, ]; - const body = JSON.parse(adapter.buildRequest({ - modelId: "gpt-5.5", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { model: "gpt-5.5", input }, - }, meta).body) as { input: Record[] }; + const body = JSON.parse( + adapter.buildRequest( + { + modelId: "gpt-5.5", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { model: "gpt-5.5", input }, + }, + meta, + ).body, + ) as { input: Record[] }; expect(body.input).toEqual(input); }); @@ -588,19 +890,38 @@ describe("OpenAI Responses passthrough sanitization", () => { const adapter = createResponsesPassthroughAdapter(provider); const oversizedCallId = `call_${"x".repeat(80)}`; const input = [ - { type: "function_call", call_id: oversizedCallId, name: "ping", arguments: "{}" }, - { type: "function_call_output", call_id: oversizedCallId, output: "pong" }, - { type: "function_call", call_id: "call_short", name: "keep", arguments: "{}" }, + { + type: "function_call", + call_id: oversizedCallId, + name: "ping", + arguments: "{}", + }, + { + type: "function_call_output", + call_id: oversizedCallId, + output: "pong", + }, + { + type: "function_call", + call_id: "call_short", + name: "keep", + arguments: "{}", + }, { type: "function_call_output", call_id: "call_short", output: "kept" }, ]; - const body = JSON.parse(adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { model: "gpt-5.6-sol", input }, - }, meta).body) as { input: Record[] }; + const body = JSON.parse( + adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { model: "gpt-5.6-sol", input }, + }, + meta, + ).body, + ) as { input: Record[] }; const repairedCallId = body.input[0].call_id as string; expect(repairedCallId).toStartWith("call_ocx_"); @@ -617,19 +938,39 @@ describe("OpenAI Responses passthrough sanitization", () => { const customCallId = `call_custom_${"a".repeat(80)}`; const searchCallId = `call_search_${"b".repeat(80)}`; const input = [ - { type: "custom_tool_call", call_id: customCallId, name: "apply_patch", input: "patch" }, - { type: "custom_tool_call_output", call_id: customCallId, output: "done" }, - { type: "tool_search_call", call_id: searchCallId, execution: "client", arguments: {} }, + { + type: "custom_tool_call", + call_id: customCallId, + name: "apply_patch", + input: "patch", + }, + { + type: "custom_tool_call_output", + call_id: customCallId, + output: "done", + }, + { + type: "tool_search_call", + call_id: searchCallId, + execution: "client", + arguments: {}, + }, { type: "tool_search_output", call_id: searchCallId, tools: [] }, ]; - const build = () => JSON.parse(adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { model: "gpt-5.6-sol", input }, - }, meta).body) as { input: Record[] }; + const build = () => + JSON.parse( + adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { model: "gpt-5.6-sol", input }, + }, + meta, + ).body, + ) as { input: Record[] }; const first = build().input; const second = build().input; @@ -638,7 +979,9 @@ describe("OpenAI Responses passthrough sanitization", () => { expect(first[0].call_id).not.toBe(first[2].call_id); expect((first[0].call_id as string).length).toBeLessThanOrEqual(64); expect((first[2].call_id as string).length).toBeLessThanOrEqual(64); - expect(second.map(item => item.call_id)).toEqual(first.map(item => item.call_id)); + expect(second.map((item) => item.call_id)).toEqual( + first.map((item) => item.call_id), + ); }); test("api-key mode preserves oversized call ids that may reference upstream stored state", () => { @@ -650,17 +993,26 @@ describe("OpenAI Responses passthrough sanitization", () => { }); const oversizedCallId = `call_${"stored".repeat(14)}`; const input = [ - { type: "function_call_output", call_id: oversizedCallId, output: "pong" }, + { + type: "function_call_output", + call_id: oversizedCallId, + output: "pong", + }, ]; - const body = JSON.parse(adapter.buildRequest({ - ...parsedBase, - _rawBody: { - model: "gpt-5.5", - previous_response_id: "resp_stored", - input, - }, - }, meta).body) as { previous_response_id: string; input: Array<{ call_id: string }> }; + const body = JSON.parse( + adapter.buildRequest( + { + ...parsedBase, + _rawBody: { + model: "gpt-5.5", + previous_response_id: "resp_stored", + input, + }, + }, + meta, + ).body, + ) as { previous_response_id: string; input: Array<{ call_id: string }> }; expect(body.previous_response_id).toBe("resp_stored"); expect(body.input[0]?.call_id).toBe(oversizedCallId); @@ -675,18 +1027,32 @@ describe("OpenAI Responses passthrough sanitization", () => { }); const oversizedCallId = `call_${"expanded".repeat(12)}`; - const body = JSON.parse(adapter.buildRequest({ - ...parsedBase, - _previousResponseInputExpanded: true, - _rawBody: { - model: "gpt-5.5", - previous_response_id: "resp_expanded", - input: [ - { type: "function_call", call_id: oversizedCallId, name: "ping", arguments: "{}" }, - { type: "function_call_output", call_id: oversizedCallId, output: "pong" }, - ], - }, - }, meta).body) as { previous_response_id?: string; input: Array<{ call_id: string }> }; + const body = JSON.parse( + adapter.buildRequest( + { + ...parsedBase, + _previousResponseInputExpanded: true, + _rawBody: { + model: "gpt-5.5", + previous_response_id: "resp_expanded", + input: [ + { + type: "function_call", + call_id: oversizedCallId, + name: "ping", + arguments: "{}", + }, + { + type: "function_call_output", + call_id: oversizedCallId, + output: "pong", + }, + ], + }, + }, + meta, + ).body, + ) as { previous_response_id?: string; input: Array<{ call_id: string }> }; expect(body.previous_response_id).toBeUndefined(); expect(body.input[0]?.call_id).toStartWith("call_ocx_"); @@ -696,19 +1062,28 @@ describe("OpenAI Responses passthrough sanitization", () => { test("forward expanded replay keeps reasoning items (chain is intact)", () => { const adapter = createResponsesPassthroughAdapter(provider); - const body = JSON.parse(adapter.buildRequest({ - ...parsedBase, - _previousResponseInputExpanded: true, - _rawBody: { - model: "gpt-5.5", - previous_response_id: "resp_1", - input: [ - { type: "reasoning", id: "rs_1", summary: [] }, - { type: "message", role: "assistant", content: [{ type: "output_text", text: "prior" }] }, - { role: "user", content: "next" }, - ], - }, - }, meta).body) as { input: Record[] }; + const body = JSON.parse( + adapter.buildRequest( + { + ...parsedBase, + _previousResponseInputExpanded: true, + _rawBody: { + model: "gpt-5.5", + previous_response_id: "resp_1", + input: [ + { type: "reasoning", id: "rs_1", summary: [] }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "prior" }], + }, + { role: "user", content: "next" }, + ], + }, + }, + meta, + ).body, + ) as { input: Record[] }; expect(body.input).toHaveLength(3); expect(body.input[0]).toMatchObject({ type: "reasoning", id: "rs_1" }); @@ -726,119 +1101,164 @@ describe("OpenAI Responses hosted-tool name conflicts", () => { test("keyed platform replaces a dotted image_gen function with a safe alias", () => { const adapter = createResponsesPassthroughAdapter(keyedProvider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.6-sol", - input: [], - tools: [ - { type: "function", name: "image_gen.imagegen", parameters: {} }, - { type: "image_generation" }, - { type: "web_search" }, - ], + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.6-sol", + input: [], + tools: [ + { type: "function", name: "image_gen.imagegen", parameters: {} }, + { type: "image_generation" }, + { type: "web_search" }, + ], + }, }, - }, meta); - const body = JSON.parse(request.body) as { tools: { type: string; name?: string }[] }; + meta, + ); + const body = JSON.parse(request.body) as { + tools: { type: string; name?: string }[]; + }; // Hosted image_generation dropped; the declared client tool wins and unrelated hosted tools stay. expect(body.tools).toHaveLength(2); - expect(body.tools.some(t => t.type === "image_generation")).toBe(false); - expect(body.tools.some(t => t.type === "function" && t.name === "image_gen__imagegen")).toBe(true); - expect(body.tools.some(t => t.type === "web_search")).toBe(true); + expect(body.tools.some((t) => t.type === "image_generation")).toBe(false); + expect( + body.tools.some( + (t) => t.type === "function" && t.name === "image_gen__imagegen", + ), + ).toBe(true); + expect(body.tools.some((t) => t.type === "web_search")).toBe(true); }); test("keyed platform flattens an image_gen namespace and removes the hosted duplicate", () => { const adapter = createResponsesPassthroughAdapter(keyedProvider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.6-sol", - input: [], - tools: [ - { - type: "namespace", - name: "image_gen", - description: "Client image tools", - tools: [{ - type: "function", - name: "imagegen", - description: "Generate or edit an image", - parameters: { type: "object", properties: { prompt: { type: "string" } } }, - strict: true, - }], - }, - { type: "image_generation" }, - ], + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.6-sol", + input: [], + tools: [ + { + type: "namespace", + name: "image_gen", + description: "Client image tools", + tools: [ + { + type: "function", + name: "imagegen", + description: "Generate or edit an image", + parameters: { + type: "object", + properties: { prompt: { type: "string" } }, + }, + strict: true, + }, + ], + }, + { type: "image_generation" }, + ], + }, }, - }, meta); - const body = JSON.parse(request.body) as { tools: Array> }; + meta, + ); + const body = JSON.parse(request.body) as { + tools: Array>; + }; expect(body.tools).toHaveLength(1); expect(body.tools[0]).toEqual({ type: "function", name: "image_gen__imagegen", description: "Generate or edit an image", - parameters: { type: "object", properties: { prompt: { type: "string" } } }, + parameters: { + type: "object", + properties: { prompt: { type: "string" } }, + }, strict: true, }); }); test("keyed platform rewrites a forced image-gen tool choice with its declared alias", () => { const adapter = createResponsesPassthroughAdapter(keyedProvider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.6-sol", - input: [], - tools: [{ - type: "namespace", - name: "image_gen", - tools: [{ type: "function", name: "imagegen", parameters: {} }], - }], - tool_choice: { type: "function", name: "image_gen.imagegen" }, + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.6-sol", + input: [], + tools: [ + { + type: "namespace", + name: "image_gen", + tools: [{ type: "function", name: "imagegen", parameters: {} }], + }, + ], + tool_choice: { type: "function", name: "image_gen.imagegen" }, + }, }, - }, meta); + meta, + ); const body = JSON.parse(request.body) as { tool_choice: { type: string; name: string }; }; - expect(body.tool_choice).toEqual({ type: "function", name: "image_gen__imagegen" }); + expect(body.tool_choice).toEqual({ + type: "function", + name: "image_gen__imagegen", + }); }); test("keyed platform rewrites image-gen entries in an allowed-tools choice", () => { const adapter = createResponsesPassthroughAdapter(keyedProvider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.6-sol", - input: [{ - type: "additional_tools", - tools: [{ type: "function", name: "image_gen.imagegen", parameters: {} }], - }], - tool_choice: { - type: "allowed_tools", - mode: "required", - tools: [ - { type: "function", name: "image_gen.imagegen" }, - { type: "function", name: "exec_command" }, + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.6-sol", + input: [ + { + type: "additional_tools", + tools: [ + { + type: "function", + name: "image_gen.imagegen", + parameters: {}, + }, + ], + }, ], + tool_choice: { + type: "allowed_tools", + mode: "required", + tools: [ + { type: "function", name: "image_gen.imagegen" }, + { type: "function", name: "exec_command" }, + ], + }, }, }, - }, meta); + meta, + ); const body = JSON.parse(request.body) as { - tool_choice: { type: string; mode: string; tools: Array<{ type: string; name: string }> }; + tool_choice: { + type: string; + mode: string; + tools: Array<{ type: string; name: string }>; + }; }; expect(body.tool_choice).toEqual({ @@ -853,188 +1273,275 @@ describe("OpenAI Responses hosted-tool name conflicts", () => { test("keyed responses-lite flattens a nested namespace without requiring a hosted tool", () => { const adapter = createResponsesPassthroughAdapter(keyedProvider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.6-sol", - input: [ - { - type: "additional_tools", - role: "developer", - tools: [ - { - type: "namespace", - name: "image_gen", - tools: [{ type: "function", name: "imagegen", parameters: { type: "object" } }], - }, - { type: "web_search" }, - ], - }, - { type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }, - ], + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.6-sol", + input: [ + { + type: "additional_tools", + role: "developer", + tools: [ + { + type: "namespace", + name: "image_gen", + tools: [ + { + type: "function", + name: "imagegen", + parameters: { type: "object" }, + }, + ], + }, + { type: "web_search" }, + ], + }, + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "hi" }], + }, + ], + }, }, - }, meta); + meta, + ); const body = JSON.parse(request.body) as { - input: Array<{ type: string; role?: string; tools?: Array<{ type: string; name?: string }> }>; + input: Array<{ + type: string; + role?: string; + tools?: Array<{ type: string; name?: string }>; + }>; }; - const additionalTools = body.input.find(item => item.type === "additional_tools"); + const additionalTools = body.input.find( + (item) => item.type === "additional_tools", + ); // Preserve the input entry and unrelated tools while lowering the private namespace. expect(additionalTools).toBeDefined(); expect(additionalTools?.role).toBe("developer"); - expect(additionalTools?.tools?.some(t => t.type === "namespace")).toBe(false); - expect(additionalTools?.tools?.some(t => - t.type === "function" && t.name === "image_gen__imagegen" - )).toBe(true); - expect(additionalTools?.tools?.some(t => t.type === "web_search")).toBe(true); - expect(body.input.some(item => item.type === "message")).toBe(true); + expect(additionalTools?.tools?.some((t) => t.type === "namespace")).toBe( + false, + ); + expect( + additionalTools?.tools?.some( + (t) => t.type === "function" && t.name === "image_gen__imagegen", + ), + ).toBe(true); + expect(additionalTools?.tools?.some((t) => t.type === "web_search")).toBe( + true, + ); + expect(body.input.some((item) => item.type === "message")).toBe(true); }); test("keyed responses-lite detects image_gen conflicts across top-level and nested tool groups", () => { const adapter = createResponsesPassthroughAdapter(keyedProvider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.6-sol", - tools: [ - { type: "image_generation" }, - { type: "web_search" }, - ], - input: [ - { - type: "additional_tools", - role: "developer", - tools: [{ type: "function", name: "image_gen.imagegen", parameters: {} }], - }, - ], + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.6-sol", + tools: [{ type: "image_generation" }, { type: "web_search" }], + input: [ + { + type: "additional_tools", + role: "developer", + tools: [ + { + type: "function", + name: "image_gen.imagegen", + parameters: {}, + }, + ], + }, + ], + }, }, - }, meta); + meta, + ); const body = JSON.parse(request.body) as { tools: Array<{ type: string }>; - input: Array<{ type: string; tools?: Array<{ type: string; name?: string }> }>; + input: Array<{ + type: string; + tools?: Array<{ type: string; name?: string }>; + }>; }; - const additionalTools = body.input.find(item => item.type === "additional_tools"); + const additionalTools = body.input.find( + (item) => item.type === "additional_tools", + ); // The platform validates one merged namespace even when declarations use different groups. - expect(body.tools.some(t => t.type === "image_generation")).toBe(false); - expect(body.tools.some(t => t.type === "web_search")).toBe(true); - expect(additionalTools?.tools?.some(t => t.name === "image_gen__imagegen")).toBe(true); + expect(body.tools.some((t) => t.type === "image_generation")).toBe(false); + expect(body.tools.some((t) => t.type === "web_search")).toBe(true); + expect( + additionalTools?.tools?.some((t) => t.name === "image_gen__imagegen"), + ).toBe(true); }); test("keyed platform encodes native and legacy image-gen calls for upstream replay", () => { const adapter = createResponsesPassthroughAdapter(keyedProvider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.6-sol", - tools: [{ - type: "namespace", - name: "image_gen", - tools: [{ type: "function", name: "imagegen", parameters: {} }], - }], - input: [ - { - type: "function_call", - namespace: "image_gen", - name: "imagegen", - call_id: "call_native", - arguments: "{}", - }, - { - type: "function_call", - name: "image_gen.imagegen", - call_id: "call_legacy", - arguments: "{}", - }, - { type: "function_call", name: "exec_command", call_id: "call_other", arguments: "{}" }, - ], + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.6-sol", + tools: [ + { + type: "namespace", + name: "image_gen", + tools: [{ type: "function", name: "imagegen", parameters: {} }], + }, + ], + input: [ + { + type: "function_call", + namespace: "image_gen", + name: "imagegen", + call_id: "call_native", + arguments: "{}", + }, + { + type: "function_call", + name: "image_gen.imagegen", + call_id: "call_legacy", + arguments: "{}", + }, + { + type: "function_call", + name: "exec_command", + call_id: "call_other", + arguments: "{}", + }, + ], + }, }, - }, meta); + meta, + ); const body = JSON.parse(request.body) as { input: Array<{ name?: string; namespace?: string }>; }; - expect(body.input[0]).toMatchObject({ name: "image_gen__imagegen", call_id: "call_native" }); + expect(body.input[0]).toMatchObject({ + name: "image_gen__imagegen", + call_id: "call_native", + }); expect(body.input[0]).not.toHaveProperty("namespace"); - expect(body.input[1]).toMatchObject({ name: "image_gen__imagegen", call_id: "call_legacy" }); - expect(body.input[2]).toMatchObject({ name: "exec_command", call_id: "call_other" }); + expect(body.input[1]).toMatchObject({ + name: "image_gen__imagegen", + call_id: "call_legacy", + }); + expect(body.input[2]).toMatchObject({ + name: "exec_command", + call_id: "call_other", + }); }); test("keyed responses normalization is idempotent and deduplicates image-gen aliases", () => { const adapter = createResponsesPassthroughAdapter(keyedProvider); - const firstRequest = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.6-sol", - tools: [{ - type: "namespace", - name: "image_gen", - tools: [{ type: "function", name: "imagegen", parameters: { type: "object" } }], - }], - input: [{ - type: "additional_tools", - role: "developer", + const firstRequest = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.6-sol", tools: [ - { type: "function", name: "image_gen.imagegen", parameters: { type: "object" } }, - { type: "web_search" }, + { + type: "namespace", + name: "image_gen", + tools: [ + { + type: "function", + name: "imagegen", + parameters: { type: "object" }, + }, + ], + }, + ], + input: [ + { + type: "additional_tools", + role: "developer", + tools: [ + { + type: "function", + name: "image_gen.imagegen", + parameters: { type: "object" }, + }, + { type: "web_search" }, + ], + }, ], - }], + }, }, - }, meta); + meta, + ); const firstBody = JSON.parse(firstRequest.body) as { tools: Array<{ type: string; name?: string }>; - input: Array<{ type: string; tools?: Array<{ type: string; name?: string }> }>; + input: Array<{ + type: string; + tools?: Array<{ type: string; name?: string }>; + }>; }; expect(firstBody.tools).toEqual([ - { type: "function", name: "image_gen__imagegen", parameters: { type: "object" } }, + { + type: "function", + name: "image_gen__imagegen", + parameters: { type: "object" }, + }, ]); expect(firstBody.input[0]?.tools).toEqual([{ type: "web_search" }]); - const secondRequest = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: firstBody, - }, meta); + const secondRequest = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: firstBody, + }, + meta, + ); expect(JSON.parse(secondRequest.body)).toEqual(firstBody); }); test("keyed platform preserves unrelated and malformed namespaces", () => { const adapter = createResponsesPassthroughAdapter(keyedProvider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.6-sol", - input: [], - tools: [ - { type: "namespace", name: "image_gen", tools: [] }, - { - type: "namespace", - name: "web", - tools: [{ type: "function", name: "run", parameters: {} }], - }, - { type: "image_generation" }, - ], - tool_choice: { type: "function", name: "image_gen.imagegen" }, + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.6-sol", + input: [], + tools: [ + { type: "namespace", name: "image_gen", tools: [] }, + { + type: "namespace", + name: "web", + tools: [{ type: "function", name: "run", parameters: {} }], + }, + { type: "image_generation" }, + ], + tool_choice: { type: "function", name: "image_gen.imagegen" }, + }, }, - }, meta); + meta, + ); const body = JSON.parse(request.body) as { tools: Array>; tool_choice: { type: string; name: string }; @@ -1049,28 +1556,36 @@ describe("OpenAI Responses hosted-tool name conflicts", () => { }, { type: "image_generation" }, ]); - expect(body.tool_choice).toEqual({ type: "function", name: "image_gen.imagegen" }); + expect(body.tool_choice).toEqual({ + type: "function", + name: "image_gen.imagegen", + }); }); test("keyed platform preserves hosted image_generation for replay-only image-gen calls", () => { const adapter = createResponsesPassthroughAdapter(keyedProvider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.6-sol", - tools: [{ type: "image_generation" }], - input: [{ - type: "function_call", - namespace: "image_gen", - name: "imagegen", - call_id: "call_replay", - arguments: "{}", - }], + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.6-sol", + tools: [{ type: "image_generation" }], + input: [ + { + type: "function_call", + namespace: "image_gen", + name: "imagegen", + call_id: "call_replay", + arguments: "{}", + }, + ], + }, }, - }, meta); + meta, + ); const body = JSON.parse(request.body) as { tools: Array<{ type: string }>; input: Array<{ name?: string; namespace?: string }>; @@ -1087,21 +1602,26 @@ describe("OpenAI Responses hosted-tool name conflicts", () => { test("keyed platform preserves hosted image_generation for a bare image_gen function", () => { const adapter = createResponsesPassthroughAdapter(keyedProvider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.6-sol", - input: [], - tools: [ - { type: "function", name: "image_gen", parameters: {} }, - { type: "image_generation" }, - ], + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.6-sol", + input: [], + tools: [ + { type: "function", name: "image_gen", parameters: {} }, + { type: "image_generation" }, + ], + }, }, - }, meta); - const body = JSON.parse(request.body) as { tools: Array> }; + meta, + ); + const body = JSON.parse(request.body) as { + tools: Array>; + }; expect(body.tools).toEqual([ { type: "function", name: "image_gen", parameters: {} }, @@ -1111,61 +1631,77 @@ describe("OpenAI Responses hosted-tool name conflicts", () => { test("keyed platform keeps hosted image_generation when no conflicting tool is declared", () => { const adapter = createResponsesPassthroughAdapter(keyedProvider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.6-sol", - input: [], - tools: [ - { type: "function", name: "shell", parameters: {} }, - { type: "image_generation" }, - ], + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.6-sol", + input: [], + tools: [ + { type: "function", name: "shell", parameters: {} }, + { type: "image_generation" }, + ], + }, }, - }, meta); + meta, + ); const body = JSON.parse(request.body) as { tools: { type: string }[] }; expect(body.tools).toHaveLength(2); - expect(body.tools.some(t => t.type === "image_generation")).toBe(true); + expect(body.tools.some((t) => t.type === "image_generation")).toBe(true); }); test("forward backend preserves the private image_gen namespace and hosted tool", () => { // The ChatGPT backend understands the private namespace; lowering it would change native behavior. const adapter = createResponsesPassthroughAdapter(provider); - const request = adapter.buildRequest({ - modelId: "gpt-5.5", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { - model: "gpt-5.5", - input: [], - tools: [ - { - type: "namespace", - name: "image_gen", - tools: [{ type: "function", name: "imagegen", parameters: {} }], - }, - { type: "image_generation" }, - ], - tool_choice: { type: "function", name: "image_gen.imagegen" }, + const request = adapter.buildRequest( + { + modelId: "gpt-5.5", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { + model: "gpt-5.5", + input: [], + tools: [ + { + type: "namespace", + name: "image_gen", + tools: [{ type: "function", name: "imagegen", parameters: {} }], + }, + { type: "image_generation" }, + ], + tool_choice: { type: "function", name: "image_gen.imagegen" }, + }, }, - }, meta); + meta, + ); const body = JSON.parse(request.body) as { - tools: Array<{ type: string; name?: string; tools?: Array<{ name?: string }> }>; + tools: Array<{ + type: string; + name?: string; + tools?: Array<{ name?: string }>; + }>; tool_choice: { type: string; name: string }; }; expect(body.tools).toHaveLength(2); - expect(body.tools.some(t => t.type === "image_generation")).toBe(true); - expect(body.tools.some(t => - t.type === "namespace" - && t.name === "image_gen" - && t.tools?.some(inner => inner.name === "imagegen") - )).toBe(true); - expect(body.tool_choice).toEqual({ type: "function", name: "image_gen.imagegen" }); + expect(body.tools.some((t) => t.type === "image_generation")).toBe(true); + expect( + body.tools.some( + (t) => + t.type === "namespace" && + t.name === "image_gen" && + t.tools?.some((inner) => inner.name === "imagegen"), + ), + ).toBe(true); + expect(body.tool_choice).toEqual({ + type: "function", + name: "image_gen.imagegen", + }); }); }); @@ -1183,13 +1719,16 @@ describe("OpenAI Responses forward-mode unsupported param stripping", () => { test("forward mode strips max_output_tokens and metadata", () => { const adapter = createResponsesPassthroughAdapter(provider); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { ...rawBody }, - }, meta); + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { ...rawBody }, + }, + meta, + ); const body = JSON.parse(request.body) as Record; expect(body).not.toHaveProperty("max_output_tokens"); @@ -1201,13 +1740,16 @@ describe("OpenAI Responses forward-mode unsupported param stripping", () => { test("forward mode is a no-op when neither field is present", () => { const adapter = createResponsesPassthroughAdapter(provider); const { max_output_tokens: _m, metadata: _d, ...codexBody } = rawBody; - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { ...codexBody }, - }, meta); + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { ...codexBody }, + }, + meta, + ); const body = JSON.parse(request.body) as Record; expect(body.reasoning).toEqual({ effort: "low" }); @@ -1221,16 +1763,68 @@ describe("OpenAI Responses forward-mode unsupported param stripping", () => { authMode: "key", apiKey: "sk-test", }); - const request = adapter.buildRequest({ - modelId: "gpt-5.6-sol", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { ...rawBody }, - }, { headers: new Headers() }); + const request = adapter.buildRequest( + { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { ...rawBody }, + }, + { headers: new Headers() }, + ); const body = JSON.parse(request.body) as Record; expect(body.max_output_tokens).toBe(32000); expect(body.metadata).toEqual({ user_id: "u-1" }); }); }); + +test("cache diagnostics describe sanitized Spark tools and input, leaving the caller body intact", () => { + const rawBody = { + model: "gpt-5.3-codex-spark", + tools: [ + { + type: "function", + name: "lookup", + parameters: { type: "object" }, + defer_loading: true, + }, + { type: "tool_search" }, + ], + input: [ + { role: "user", content: "hello" }, + { + type: "tool_search_call", + prompt_cache_breakpoint: { mode: "explicit" }, + }, + ], + }; + const original = JSON.stringify(rawBody); + const request = createResponsesPassthroughAdapter(provider).buildRequest( + { + modelId: rawBody.model, + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: rawBody, + }, + { headers: new Headers() }, + ); + const finalBody = JSON.parse(request.body); + expect(finalBody.tools).toHaveLength(1); + expect(finalBody.tools[0]).not.toHaveProperty("defer_loading"); + expect(finalBody.input).toHaveLength(1); + expect(request.promptCacheLog).toEqual( + observeOpenAiResponsesPromptCache(finalBody), + ); + expect(request.promptCacheLog).toMatchObject({ + toolCount: 1, + inputItemCount: 1, + breakpointCount: 0, + }); + expect(request.promptCacheLog?.toolsFingerprint).not.toBe( + observeOpenAiResponsesPromptCache(rawBody)?.toolsFingerprint, + ); + expect(JSON.stringify(rawBody)).toBe(original); +}); diff --git a/tests/prompt-cache-analyzer.test.ts b/tests/prompt-cache-analyzer.test.ts index e97f67f02..472a57f4c 100644 --- a/tests/prompt-cache-analyzer.test.ts +++ b/tests/prompt-cache-analyzer.test.ts @@ -3,14 +3,20 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -const script = join(import.meta.dir, "../scripts/analyze-prompt-cache-usage.ts"); +const script = join( + import.meta.dir, + "../scripts/analyze-prompt-cache-usage.ts", +); const tempDirs: string[] = []; function tempUsageFile(lines: unknown[]): string { const dir = mkdtempSync(join(tmpdir(), "ocx-cache-analyzer-")); tempDirs.push(dir); const path = join(dir, "usage.jsonl"); - writeFileSync(path, lines.map(line => JSON.stringify(line)).join("\n") + "\n"); + writeFileSync( + path, + lines.map((line) => JSON.stringify(line)).join("\n") + "\n", + ); return path; } @@ -27,6 +33,41 @@ afterEach(() => { } }); +const now = Date.parse("2026-10-03T12:00:00Z"); +const observation = { + version: 1, + mode: "default", + keyPresent: false, + prewarm: false, + comparisonRequested: false, + previousResponseIdPresent: false, + breakpointCount: 0, + inputItemCount: 1, + toolCount: 0, +}; + +function row(overrides: Record = {}) { + return { + timestamp: now, + adapter: "openai-responses", + provider: "openai", + model: "test-model", + surface: "codex", + status: 200, + usageStatus: "reported", + usage: { inputTokens: 100, cacheReadInputTokens: 80 }, + promptCache: observation, + ...overrides, + }; +} + +function analyze(rows: unknown[], ...args: string[]) { + const result = run(tempUsageFile(rows), `--now=${now}`, "--json", ...args); + expect(result.exitCode).toBe(0); + expect(result.stderr.toString()).toBe(""); + return JSON.parse(result.stdout.toString()); +} + describe("prompt-cache usage analyzer", () => { test("is reproducible with --now and keeps cache-shape cohort dimensions", () => { const path = tempUsageFile([ @@ -184,10 +225,303 @@ describe("prompt-cache usage analyzer", () => { const path = tempUsageFile([]); const invalidRange = run(path, "--range=week"); expect(invalidRange.exitCode).toBe(2); - expect(invalidRange.stderr.toString()).toContain("invalid --range value: week"); + expect(invalidRange.stderr.toString()).toContain( + "invalid --range value: week", + ); const unknown = run(path, "--bogus"); expect(unknown.exitCode).toBe(2); expect(unknown.stderr.toString()).toContain("unknown option: --bogus"); }); }); + +describe("prompt-cache analyzer boundaries", () => { + test("counts requests but excludes unproven or inconsistent usage from all measured totals", () => { + const invalid = [ + row({ status: 201 }), + row({ status: "200" }), + row({ status: 500 }), + ...["estimated", "unsupported", "unreported", undefined].map( + (usageStatus) => row({ usageStatus }), + ), + ...[ + null, + [], + {}, + { inputTokens: -1 }, + { inputTokens: "100" }, + { inputTokens: 100, cacheReadInputTokens: -1 }, + { inputTokens: 100, cacheReadInputTokens: null, cachedInputTokens: 80 }, + { + inputTokens: 100, + cacheReadInputTokens: 80, + cacheCreationInputTokens: 21, + }, + ].map((usage) => row({ usage })), + ]; + const output = analyze([row(), ...invalid]); + expect(output.rows).toBe(invalid.length + 1); + expect(output.summary).toEqual({ + reportedSuccess: 1, + inputTokens: 100, + cacheReadTokens: 80, + cacheWriteTokens: 0, + uncachedInputTokens: 20, + cacheReadRatio: 0.8, + cacheWriteRatio: 0, + }); + for (const bucket of [output.cohorts[0], output.cacheShapes[0]]) { + expect(bucket).toMatchObject({ + requests: invalid.length + 1, + reportedSuccess: 1, + inputTokens: 100, + cacheReadTokens: 80, + cacheReadRatio: 0.8, + }); + } + expect(output.cohorts[0].cacheHitRequestRatio).toBe(1); + }); + + test.each([ + { + usage: { + inputTokens: 100, + cachedInputTokens: 80, + cacheCreationInputTokens: 30, + }, + read: 50, + write: 30, + }, + { + usage: { + inputTokens: 100, + cachedInputTokens: 10, + cacheCreationInputTokens: 30, + }, + read: 0, + write: 30, + }, + { usage: { inputTokens: 100, cachedInputTokens: 80 }, read: 80, write: 0 }, + { + usage: { + inputTokens: 100, + cachedInputTokens: 90, + cacheReadInputTokens: 0, + }, + read: 0, + write: 0, + }, + { + usage: { + inputTokens: 100, + cacheReadInputTokens: 70, + cacheCreationInputTokens: 30, + }, + read: 70, + write: 30, + }, + ])( + "accounts for legacy and explicit usage: $usage", + ({ usage, read, write }) => { + const output = analyze([row({ usage })]); + expect(output.summary).toMatchObject({ + reportedSuccess: 1, + inputTokens: 100, + cacheReadTokens: read, + cacheWriteTokens: write, + uncachedInputTokens: 100 - read - write, + }); + expect(output.cohorts[0]).toMatchObject({ + cacheHitRequestRatio: read > 0 ? 1 : 0, + cacheWriteRequestRatio: write > 0 ? 1 : 0, + }); + }, + ); + + test.each(["7d", "30d", "all"])( + "includes both window endpoints for %s", + (range) => { + const since = + range === "all" ? 0 : now - (range === "7d" ? 7 : 30) * 86_400_000; + const output = analyze( + [ + ...[since - 1, since, now, now + 1, -1, null, "timestamp"].map( + (timestamp) => row({ timestamp }), + ), + row({ timestamp: undefined }), + ], + `--range=${range}`, + ); + expect(output.windowStartMs).toBe(since); + expect(output.windowEndMs).toBe(now); + expect(output.rows).toBe(2); + expect(output.summary.inputTokens).toBe(200); + }, + ); + + test("reads stdin, counts corrupt lines, and ignores blank lines", () => { + const result = Bun.spawnSync( + [process.execPath, script, "-", `--now=${now}`, "--json"], + { + stdin: Buffer.from( + [ + "", + " ", + JSON.stringify(row()), + "{broken", + "null", + "[]", + "42", + "", + ].join("\r\n"), + ), + stdout: "pipe", + stderr: "pipe", + }, + ); + expect(result.exitCode).toBe(0); + expect(JSON.parse(result.stdout.toString())).toMatchObject({ + source: "stdin", + invalidLines: 4, + rows: 1, + summary: { reportedSuccess: 1 }, + }); + }); + + test("keeps dimensions and cache shapes separate, while top limits leave global totals intact", () => { + const output = analyze([ + row({ + provider: " other ", + resolvedModel: " resolved ", + usage: { inputTokens: 300 }, + }), + row({ + promptCache: { ...observation, mode: "explicit" }, + usage: { inputTokens: 200 }, + }), + row(), + row({ adapter: "openai-chat", usage: { inputTokens: 50 } }), + row({ surface: "claude", usage: { inputTokens: 25 } }), + row({ + status: 500, + provider: "failed-only", + usage: { inputTokens: 999 }, + }), + ]); + expect(output.cohorts).toHaveLength(4); + expect(output.cacheShapes).toHaveLength(5); + expect(output.cohorts[0]).toMatchObject({ + provider: "other", + model: "resolved", + inputTokens: 300, + }); + expect(output.cohorts[1]).toMatchObject({ + provider: "openai", + requests: 2, + inputTokens: 300, + }); + const limited = analyze( + [row({ provider: "largest", usage: { inputTokens: 300 } }), row()], + "--top=1", + ); + expect(limited.cohorts).toHaveLength(1); + expect(limited.cacheShapes).toHaveLength(1); + expect(limited.cohorts[0].provider).toBe("largest"); + expect(limited.summary.inputTokens).toBe(400); + expect(limited.summary.reportedSuccess).toBe(2); + }); + + test("historical and malformed observations keep usage without creating cache shapes", () => { + const output = analyze([ + row({ + adapter: undefined, + promptCache: undefined, + conversationId: " session ", + }), + row({ + adapter: " ", + promptCache: { ...observation, toolCount: -1 }, + conversationId: "session", + }), + row({ + adapter: undefined, + promptCache: { ...observation, version: 2 }, + conversationId: "other", + }), + ]); + expect(output.cacheShapes).toEqual([]); + expect(output.cohorts).toEqual([ + expect.objectContaining({ + adapter: "unknown", + reportedSuccess: 3, + inputTokens: 300, + conversations: 2, + multiTurnConversations: 1, + }), + ]); + }); + + test.each([ + { input: 100, read: 9, write: 50, signal: "write-heavy" }, + { input: 100, read: 10, write: 50, signal: "mixed" }, + { input: 999_999, read: 0, write: 0, signal: "mixed" }, + { input: 1_000_000, read: 249_999, write: 0, signal: "low-reuse" }, + { input: 1_000_000, read: 250_000, write: 0, signal: "mixed" }, + { input: 100, read: 75, write: 0, signal: "healthy" }, + { input: 100, read: 74, write: 0, signal: "mixed" }, + { input: 0, read: 0, write: 0, signal: "mixed" }, + ])( + "classifies signal at threshold $input/$read/$write", + ({ input, read, write, signal }) => { + const output = analyze([ + row({ + usage: { + inputTokens: input, + cacheReadInputTokens: read, + cacheCreationInputTokens: write, + }, + }), + ]); + expect(output.cohorts[0].signal).toBe(signal); + expect(output.summary.cacheReadRatio).toBe(input ? read / input : 0); + expect(output.summary.cacheWriteRatio).toBe(input ? write / input : 0); + }, + ); + + test("empty windows produce zero ratios and no cohorts", () => { + const output = analyze([]); + expect(output.cohorts).toEqual([]); + expect(output.cacheShapes).toEqual([]); + expect(output.summary).toEqual({ + reportedSuccess: 0, + inputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + uncachedInputTokens: 0, + cacheReadRatio: 0, + cacheWriteRatio: 0, + }); + }); + + test.each([ + "--top=0", + "--top=201", + "--top=1.5", + "--top=abc", + "--now=invalid", + "second.jsonl", + ])("rejects invalid argument %s", (arg) => { + const result = run(tempUsageFile([]), arg); + expect(result.exitCode).toBe(2); + expect(result.stdout.toString()).toBe(""); + expect(result.stderr.toString()).toContain("Usage:"); + }); + + test("help does not try to open its input file", () => { + const path = tempUsageFile([]); + const result = run(join(path, "missing"), "--help"); + expect(result.exitCode).toBe(0); + expect(result.stdout.toString()).toContain("Usage:"); + expect(result.stderr.toString()).toBe(""); + }); +}); diff --git a/tests/prompt-cache-observability.test.ts b/tests/prompt-cache-observability.test.ts index a0d246dda..18f84e5a1 100644 --- a/tests/prompt-cache-observability.test.ts +++ b/tests/prompt-cache-observability.test.ts @@ -37,22 +37,32 @@ describe("prompt cache observability", () => { verbosity: "medium", format: { type: "json_schema", name: "private-format" }, }, - tools: [{ - type: "function", - name: "lookup_private_tool", - parameters: { type: "object", properties: { secret: { type: "string" } } }, - }], - input: [{ - role: "developer", - content: [{ - type: "input_text", - text: "private developer prefix", - prompt_cache_breakpoint: { mode: "explicit" }, - }], - }, { - role: "user", - content: [{ type: "input_text", text: "private user content" }], - }], + tools: [ + { + type: "function", + name: "lookup_private_tool", + parameters: { + type: "object", + properties: { secret: { type: "string" } }, + }, + }, + ], + input: [ + { + role: "developer", + content: [ + { + type: "input_text", + text: "private developer prefix", + prompt_cache_breakpoint: { mode: "explicit" }, + }, + ], + }, + { + role: "user", + content: [{ type: "input_text", text: "private user content" }], + }, + ], }); expect(observation).toMatchObject({ @@ -81,37 +91,53 @@ describe("prompt cache observability", () => { }); test("only persists supported Responses verbosity values", () => { - expect(observeOpenAiResponsesPromptCache({ - text: { verbosity: "low" }, - })?.verbosity).toBe("low"); - expect(observeOpenAiResponsesPromptCache({ - text: { verbosity: "medium" }, - })?.verbosity).toBe("medium"); - expect(observeOpenAiResponsesPromptCache({ - text: { verbosity: "high" }, - })?.verbosity).toBe("high"); + expect( + observeOpenAiResponsesPromptCache({ + text: { verbosity: "low" }, + })?.verbosity, + ).toBe("low"); + expect( + observeOpenAiResponsesPromptCache({ + text: { verbosity: "medium" }, + })?.verbosity, + ).toBe("medium"); + expect( + observeOpenAiResponsesPromptCache({ + text: { verbosity: "high" }, + })?.verbosity, + ).toBe("high"); const unsupported = observeOpenAiResponsesPromptCache({ text: { verbosity: "private caller-controlled marker" }, }); expect(unsupported).not.toHaveProperty("verbosity"); - expect(normalizePromptCacheRequestObservation({ - ...unsupported, - verbosity: "private persisted marker", - })).not.toHaveProperty("verbosity"); + expect( + normalizePromptCacheRequestObservation({ + ...unsupported, + verbosity: "private persisted marker", + }), + ).not.toHaveProperty("verbosity"); }); test("fingerprints are stable for object-key order but preserve tool array order", () => { const a = observeOpenAiResponsesPromptCache({ instructions: "stable", tools: [ - { type: "function", name: "a", parameters: { type: "object", properties: { x: { type: "string" } } } }, + { + type: "function", + name: "a", + parameters: { type: "object", properties: { x: { type: "string" } } }, + }, { type: "function", name: "b", parameters: { type: "object" } }, ], }); const same = observeOpenAiResponsesPromptCache({ tools: [ - { name: "a", parameters: { properties: { x: { type: "string" } }, type: "object" }, type: "function" }, + { + name: "a", + parameters: { properties: { x: { type: "string" } }, type: "object" }, + type: "function", + }, { parameters: { type: "object" }, name: "b", type: "function" }, ], instructions: "stable", @@ -120,7 +146,11 @@ describe("prompt cache observability", () => { instructions: "stable", tools: [ { type: "function", name: "b", parameters: { type: "object" } }, - { type: "function", name: "a", parameters: { type: "object", properties: { x: { type: "string" } } } }, + { + type: "function", + name: "a", + parameters: { type: "object", properties: { x: { type: "string" } } }, + }, ], }); @@ -150,22 +180,286 @@ describe("prompt cache observability", () => { prompt_cache_options: { mode: "explicit", ttl: "30m" }, input: [{ content: [{ prompt_cache_breakpoint: { mode: "explicit" } }] }], }); - expect(normalizePromptCacheRequestObservation(observation)).toEqual(observation); - expect(normalizePromptCacheRequestObservation({ - ...observation, - breakpointCount: -1, - })).toBeUndefined(); - expect(normalizePromptCacheRequestObservation({ + expect(normalizePromptCacheRequestObservation(observation)).toEqual( + observation, + ); + expect( + normalizePromptCacheRequestObservation({ + ...observation, + breakpointCount: -1, + }), + ).toBeUndefined(); + expect( + normalizePromptCacheRequestObservation({ + version: 1, + keyPresent: true, + mode: "invalid", + prewarm: false, + comparisonRequested: false, + previousResponseIdPresent: false, + breakpointCount: 0, + inputItemCount: 0, + toolCount: 0, + }), + ).toBeUndefined(); + }); + + test.each( + [undefined, null, [], "prompt", 42, true].map((value) => ({ value })), + )("ignores non-object request and persisted values: $value", ({ value }) => { + expect(observeOpenAiResponsesPromptCache(value)).toBeUndefined(); + expect(normalizePromptCacheRequestObservation(value)).toBeUndefined(); + }); + + test("empty requests have only structural defaults", () => { + expect(observeOpenAiResponsesPromptCache({})).toEqual({ version: 1, - keyPresent: true, - mode: "invalid", + mode: "default", + keyPresent: false, prewarm: false, comparisonRequested: false, previousResponseIdPresent: false, breakpointCount: 0, inputItemCount: 0, toolCount: 0, - })).toBeUndefined(); + }); + }); + + test("does not coerce unsupported cache options or presence flags", () => { + const observation = observeOpenAiResponsesPromptCache({ + prompt_cache_key: " \t", + previous_response_id: 1, + prompt_cache_retention: "forever", + prompt_cache_options: { + mode: "IMPLICIT", + ttl: "60m", + prewarm: "true", + comparison_response_id: [], + }, + tools: { name: "not-an-array" }, + text: { verbosity: "HIGH" }, + }); + expect(observation).toEqual(observeOpenAiResponsesPromptCache({})); + expect( + observeOpenAiResponsesPromptCache({ prompt_cache_options: [] }), + ).toEqual(observeOpenAiResponsesPromptCache({})); + }); + + test.each(["in_memory", "24h"])( + "preserves supported legacy retention %s", + (retention) => { + const observation = observeOpenAiResponsesPromptCache({ + prompt_cache_retention: retention, + }); + expect(observation?.legacyRetention).toBe(retention); + expect(normalizePromptCacheRequestObservation(observation)).toEqual( + observation, + ); + }, + ); + + test.each([ + { input: undefined, count: 0 }, + { input: [], count: 0 }, + { input: "hello", count: 1 }, + { input: null, count: 1 }, + { input: [{ role: "user" }, { role: "assistant" }], count: 2 }, + ])("counts input items for $input", ({ input, count }) => { + expect(observeOpenAiResponsesPromptCache({ input })?.inputItemCount).toBe( + count, + ); + }); + + test("counts only explicit breakpoints within input, excluding breakpoint metadata", () => { + const observation = observeOpenAiResponsesPromptCache({ + tools: [{ prompt_cache_breakpoint: { mode: "explicit" } }], + input: [ + { + prompt_cache_breakpoint: { + mode: "explicit", + metadata: { prompt_cache_breakpoint: { mode: "explicit" } }, + }, + content: [ + { prompt_cache_breakpoint: { mode: "explicit" } }, + { prompt_cache_breakpoint: { mode: "implicit" } }, + { prompt_cache_breakpoint: "explicit" }, + { prompt_cache_breakpoint: [{ mode: "explicit" }] }, + null, + [{ nested: { prompt_cache_breakpoint: { mode: "explicit" } } }], + ], + }, + ], + }); + expect(observation?.breakpointCount).toBe(3); + }); + + test("stable prefix ends at the first non-system/developer item", () => { + const prefix = [ + { role: "system", content: "fixed system" }, + { role: "developer", content: "fixed developer" }, + ]; + const body = { instructions: "fixed instructions", input: prefix }; + const baseline = + observeOpenAiResponsesPromptCache(body)?.stablePrefixFingerprint; + expect(baseline).toMatch(/^[0-9a-f]{24}$/); + for (const boundary of [ + { role: "user", content: "variable" }, + { type: "function_call" }, + null, + ]) { + expect( + observeOpenAiResponsesPromptCache({ + ...body, + input: [ + ...prefix, + boundary, + { role: "developer", content: "later instructions" }, + ], + })?.stablePrefixFingerprint, + ).toBe(baseline); + } + expect( + observeOpenAiResponsesPromptCache({ + ...body, + instructions: "changed instructions", + })?.stablePrefixFingerprint, + ).not.toBe(baseline); + expect( + observeOpenAiResponsesPromptCache({ + ...body, + input: [...prefix].reverse(), + })?.stablePrefixFingerprint, + ).not.toBe(baseline); + expect( + observeOpenAiResponsesPromptCache({ + input: [{ role: "user", content: "hello" }, ...prefix], + }), + ).not.toHaveProperty("stablePrefixFingerprint"); + }); + + test("format fingerprints ignore object-key order but retain nested array order and values", () => { + const fingerprint = (format: unknown) => + observeOpenAiResponsesPromptCache({ text: { format } }) + ?.textFormatFingerprint; + const first = fingerprint({ + type: "json_schema", + schema: { enum: ["a", "b"], title: "result" }, + }); + expect(first).toMatch(/^[0-9a-f]{24}$/); + expect( + fingerprint({ + schema: { title: "result", enum: ["a", "b"] }, + type: "json_schema", + }), + ).toBe(first); + expect( + fingerprint({ + type: "json_schema", + schema: { enum: ["b", "a"], title: "result" }, + }), + ).not.toBe(first); + expect( + fingerprint({ + type: "json_schema", + schema: { enum: ["a", "c"], title: "result" }, + }), + ).not.toBe(first); + }); + + for (const field of [ + "breakpointCount", + "inputItemCount", + "toolCount", + ] as const) { + test.each([-1, 0.5, 1_000_001, NaN, Infinity, "1", null, undefined])( + `rejects invalid persisted ${field}: %j`, + (value) => { + expect( + normalizePromptCacheRequestObservation({ + ...observeOpenAiResponsesPromptCache({}), + [field]: value, + }), + ).toBeUndefined(); + }, + ); + test.each([0, 1_000_000])( + `accepts persisted ${field} boundary %i`, + (value) => { + const observation = { + ...observeOpenAiResponsesPromptCache({}), + [field]: value, + }; + expect(normalizePromptCacheRequestObservation(observation)).toEqual( + observation, + ); + }, + ); + } + + for (const field of [ + "keyPresent", + "prewarm", + "comparisonRequested", + "previousResponseIdPresent", + ]) { + test.each([undefined, null, 0, "false"])( + `rejects non-boolean ${field}: %j`, + (value) => { + expect( + normalizePromptCacheRequestObservation({ + ...observeOpenAiResponsesPromptCache({}), + [field]: value, + }), + ).toBeUndefined(); + }, + ); + } + + test.each([undefined, 0, 2, "1"])( + "rejects unsupported persisted version %j", + (version) => { + expect( + normalizePromptCacheRequestObservation({ + ...observeOpenAiResponsesPromptCache({}), + version, + }), + ).toBeUndefined(); + }, + ); + + test("normalization strips unknown fields and invalid optional metadata without losing the core", () => { + const core = observeOpenAiResponsesPromptCache({}); + for (const invalid of [ + "a".repeat(23), + "a".repeat(25), + "A".repeat(24), + "g".repeat(24), + 123, + null, + ]) { + expect( + normalizePromptCacheRequestObservation({ + ...core, + toolsFingerprint: invalid, + stablePrefixFingerprint: invalid, + textFormatFingerprint: invalid, + ttl: "24h", + legacyRetention: "forever", + verbosity: "private verbosity", + prompt_cache_key: "private key", + instructions: "private text", + }), + ).toEqual(core); + } + const valid = { + ...core, + toolsFingerprint: "a".repeat(24), + stablePrefixFingerprint: "0".repeat(24), + textFormatFingerprint: "0123456789abcdef01234567", + }; + const normalized = normalizePromptCacheRequestObservation(valid); + expect(normalized).toEqual(valid); + expect(normalized).not.toBe(valid); }); test("flows the final outbound observation through request logging into usage.jsonl", () => { @@ -192,7 +486,9 @@ describe("prompt cache observability", () => { model: "gpt-5.6-terra", instructions: "private fixed prefix", input: [{ role: "user", content: "hello" }], - tools: [{ type: "function", name: "shell", parameters: { type: "object" } }], + tools: [ + { type: "function", name: "shell", parameters: { type: "object" } }, + ], prompt_cache_key: "private-cache-key", prompt_cache_options: { mode: "implicit", ttl: "30m" }, }, @@ -228,7 +524,7 @@ describe("prompt cache observability", () => { const persisted = readFileSync(usageLogPath(), "utf8"); expect(persisted).not.toContain("private-cache-key"); expect(persisted).not.toContain("private fixed prefix"); - expect(persisted).not.toContain("\"shell\""); + expect(persisted).not.toContain('"shell"'); } finally { clearRequestLogsForTests(); resetUsageReadCacheForTests(); diff --git a/tests/request-log.test.ts b/tests/request-log.test.ts index 04d562e71..1f7750425 100644 --- a/tests/request-log.test.ts +++ b/tests/request-log.test.ts @@ -28,9 +28,14 @@ import { bindLogProviderAccount, type RequestLogContext, } from "../src/server/request-log"; -import { clearProviderAccountRuntimeState, getProviderAccountOccupancy, releaseRequestProviderAccount } from "../src/providers/account-runtime-state"; +import { + clearProviderAccountRuntimeState, + getProviderAccountOccupancy, + releaseRequestProviderAccount, +} from "../src/providers/account-runtime-state"; import { findKeyPoolEntryId } from "../src/providers/api-keys"; import { fallbackCodexAccountLogLabel } from "../src/codex/account-label"; +import { observeOpenAiResponsesPromptCache } from "../src/prompt-cache/observability"; import { bridgeToResponsesSSE } from "../src/bridge"; import type { AdapterEvent, OcxUsage } from "../src/types"; import { @@ -44,7 +49,9 @@ import { mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -async function* replayAdapterEvents(events: AdapterEvent[]): AsyncGenerator { +async function* replayAdapterEvents( + events: AdapterEvent[], +): AsyncGenerator { for (const event of events) yield event; } @@ -63,48 +70,80 @@ function log(overrides: Partial): RequestLogEntry { describe("request telemetry privacy", () => { test("strips account-scoped provider labels before external telemetry", () => { - const telemetry = spyOn(posthogTelemetry, "captureRequestTelemetry").mockImplementation(() => {}); + const telemetry = spyOn( + posthogTelemetry, + "captureRequestTelemetry", + ).mockImplementation(() => {}); clearRequestLogsForTests(); const entry = log({ provider: "openai-pabcdef" }); addRequestLog(entry); - expect(telemetry).toHaveBeenCalledWith(expect.objectContaining({ provider: "openai" })); + expect(telemetry).toHaveBeenCalledWith( + expect.objectContaining({ provider: "openai" }), + ); expect(getRequestLogEntries()[0]?.provider).toBe("openai-pabcdef"); telemetry.mockRestore(); clearRequestLogsForTests(); }); test("forwards only pricing-safe combo attempt fields to external generation telemetry", () => { - const telemetry = spyOn(posthogTelemetry, "captureRequestTelemetry").mockImplementation(() => {}); + const telemetry = spyOn( + posthogTelemetry, + "captureRequestTelemetry", + ).mockImplementation(() => {}); clearRequestLogsForTests(); - addRequestLog(log({ - provider: "combo", - attempts: [{ - ordinal: 1, provider: "openai-pabcdef", model: "gpt-5.6-sol", adapter: "openai-chat", - status: 200, durationMs: 10, sendCount: 1, recoveryKinds: [], usageStatus: "reported", - usage: { inputTokens: 10, outputTokens: 2 }, providerAccountId: "must-not-leave-local-log", - }], - })); - - expect(telemetry).toHaveBeenCalledWith(expect.objectContaining({ - attempts: [{ - ordinal: 1, provider: "openai", model: "gpt-5.6-sol", usageStatus: "reported", - usage: { inputTokens: 10, outputTokens: 2 }, - }], - })); + addRequestLog( + log({ + provider: "combo", + attempts: [ + { + ordinal: 1, + provider: "openai-pabcdef", + model: "gpt-5.6-sol", + adapter: "openai-chat", + status: 200, + durationMs: 10, + sendCount: 1, + recoveryKinds: [], + usageStatus: "reported", + usage: { inputTokens: 10, outputTokens: 2 }, + providerAccountId: "must-not-leave-local-log", + }, + ], + }), + ); + + expect(telemetry).toHaveBeenCalledWith( + expect.objectContaining({ + attempts: [ + { + ordinal: 1, + provider: "openai", + model: "gpt-5.6-sol", + usageStatus: "reported", + usage: { inputTokens: 10, outputTokens: 2 }, + }, + ], + }), + ); telemetry.mockRestore(); clearRequestLogsForTests(); }); test("forwards client streaming state to external generation telemetry", () => { - const telemetry = spyOn(posthogTelemetry, "captureRequestTelemetry").mockImplementation(() => {}); + const telemetry = spyOn( + posthogTelemetry, + "captureRequestTelemetry", + ).mockImplementation(() => {}); clearRequestLogsForTests(); addRequestLog(log({ stream: true })); - expect(telemetry).toHaveBeenCalledWith(expect.objectContaining({ stream: true })); + expect(telemetry).toHaveBeenCalledWith( + expect.objectContaining({ stream: true }), + ); telemetry.mockRestore(); clearRequestLogsForTests(); }); @@ -219,13 +258,25 @@ describe("request log metadata", () => { test("malformed adapter reasoning metadata never interrupts request logging", () => { const malformed = [ { effectiveEffort: 123, wireField: "reasoning_effort", wireValue: 123 }, - { effectiveEffort: null, wireField: "reasoning_effort", wireValue: "high" }, + { + effectiveEffort: null, + wireField: "reasoning_effort", + wireValue: "high", + }, { effectiveEffort: {}, wireField: "reasoning_effort", wireValue: "high" }, { effectiveEffort: "high", wireField: "unknown", wireValue: "high" }, { effectiveEffort: "high", wireField: "reasoning_effort", wireValue: "" }, - { effectiveEffort: "high", wireField: "thinking_budget", wireValue: null }, + { + effectiveEffort: "high", + wireField: "thinking_budget", + wireValue: null, + }, { effectiveEffort: "high", wireField: "thinking_budget", wireValue: {} }, - { effectiveEffort: "high", wireField: "thinking_budget", wireValue: Number.NaN }, + { + effectiveEffort: "high", + wireField: "thinking_budget", + wireValue: Number.NaN, + }, { effectiveEffort: "high", wireField: "thinking_budget", wireValue: -1 }, ]; @@ -246,13 +297,15 @@ describe("request log metadata", () => { reasoningWireValue: "stale", }); - expect(() => recordAdapterRequestMetadata(logCtx, { - url: "https://provider.test/v1/chat/completions", - method: "POST", - headers: {}, - body: "{}", - reasoningLog: reasoningLog as never, - })).not.toThrow(); + expect(() => + recordAdapterRequestMetadata(logCtx, { + url: "https://provider.test/v1/chat/completions", + method: "POST", + headers: {}, + body: "{}", + reasoningLog: reasoningLog as never, + }), + ).not.toThrow(); expect(logCtx.effectiveEffort).toBeUndefined(); expect(logCtx.reasoningWireField).toBeUndefined(); expect(logCtx.reasoningWireValue).toBeUndefined(); @@ -272,8 +325,8 @@ describe("request log metadata", () => { activeAttemptStartedAt: 1_000, }; recordFirstOutput(logCtx, 500, 1_250); - expect(logCtx.firstOutputMs).toBe(750); // request-relative - expect(attempt.firstOutputMs).toBe(250); // attempt-relative + expect(logCtx.firstOutputMs).toBe(750); // request-relative + expect(attempt.firstOutputMs).toBe(250); // attempt-relative // second call is a no-op recordFirstOutput(logCtx, 500, 9_999); expect(logCtx.firstOutputMs).toBe(750); @@ -286,16 +339,37 @@ describe("request log metadata", () => { test("addFinalRequestLog preserves client stream state", () => { const entries: RequestLogEntry[] = []; - addFinalRequestLog("ocx-stream", 0, { model: "m", provider: "p", stream: true }, 200, undefined, entry => entries.push(entry)); + addFinalRequestLog( + "ocx-stream", + 0, + { model: "m", provider: "p", stream: true }, + 200, + undefined, + (entry) => entries.push(entry), + ); expect(entries[0]?.stream).toBe(true); }); test("addFinalRequestLog preserves firstOutputMs; unset stays absent", () => { const captured: RequestLogEntry[] = []; - addFinalRequestLog("ocx-ttft", 0, { model: "m", provider: "p", firstOutputMs: 12 }, 200, undefined, entry => captured.push(entry)); + addFinalRequestLog( + "ocx-ttft", + 0, + { model: "m", provider: "p", firstOutputMs: 12 }, + 200, + undefined, + (entry) => captured.push(entry), + ); expect(captured[0]?.firstOutputMs).toBe(12); const captured2: RequestLogEntry[] = []; - addFinalRequestLog("ocx-nostream", 0, { model: "m", provider: "p" }, 200, undefined, entry => captured2.push(entry)); + addFinalRequestLog( + "ocx-nostream", + 0, + { model: "m", provider: "p" }, + 200, + undefined, + (entry) => captured2.push(entry), + ); expect(captured2[0]).not.toHaveProperty("firstOutputMs"); }); @@ -329,7 +403,12 @@ describe("request log metadata", () => { totalTokens: 120, errorCode: "server_is_overloaded", }); - expect(b).toMatchObject({ status: 200, sendCount: 1, usageStatus: "reported", totalTokens: 12 }); + expect(b).toMatchObject({ + status: 200, + sendCount: 1, + usageStatus: "reported", + totalTokens: 12, + }); expect(aggregateAttemptUsage([a, b])).toEqual({ status: "estimated", @@ -363,35 +442,39 @@ describe("request log metadata", () => { totalTokens: 5, }); const unsupportedA = { ...unreported, usageStatus: "unsupported" as const }; - const unsupportedB = { ...unreported, ordinal: 3, usageStatus: "unsupported" as const }; - expect(aggregateAttemptUsage([unsupportedA, unsupportedB])).toEqual({ status: "unsupported" }); + const unsupportedB = { + ...unreported, + ordinal: 3, + usageStatus: "unsupported" as const, + }; + expect(aggregateAttemptUsage([unsupportedA, unsupportedB])).toEqual({ + status: "unsupported", + }); }); test("final combo logging keeps one logical row and finalizes its active attempt", () => { const entries: RequestLogEntry[] = []; const a = beginRequestAttempt(1, "a", "model-a", "openai-chat"); - recordAdapterRequestMetadata({ - model: "model-a", - provider: "a", - requestedEffort: "minimal", - activeAttempt: a, - }, { - url: "https://provider-a.test/v1/chat/completions", - method: "POST", - headers: {}, - body: "{}", - reasoningLog: { - effectiveEffort: "low", - wireField: "thinking_budget", - wireValue: 0, + recordAdapterRequestMetadata( + { + model: "model-a", + provider: "a", + requestedEffort: "minimal", + activeAttempt: a, + }, + { + url: "https://provider-a.test/v1/chat/completions", + method: "POST", + headers: {}, + body: "{}", + reasoningLog: { + effectiveEffort: "low", + wireField: "thinking_budget", + wireValue: 0, + }, }, - }); - finishRequestAttempt( - a, - 503, - 3, - { inputTokens: 4, outputTokens: 1 }, ); + finishRequestAttempt(a, 503, 3, { inputTokens: 4, outputTokens: 1 }); const b = beginRequestAttempt(2, "b", "model-b", "openai-chat"); noteAttemptSend(b, undefined); const start = Date.now(); @@ -419,7 +502,9 @@ describe("request log metadata", () => { wireValue: "high", }, }); - addFinalRequestLog("combo-parent", start, logCtx, 200, undefined, entry => entries.push(entry)); + addFinalRequestLog("combo-parent", start, logCtx, 200, undefined, (entry) => + entries.push(entry), + ); expect(entries).toHaveLength(1); expect(entries[0]).toMatchObject({ @@ -454,10 +539,23 @@ describe("request log metadata", () => { test("combo finalization does not backfill winner account id onto other providers", () => { const entries: RequestLogEntry[] = []; - const openaiAttempt = beginRequestAttempt(1, "openai", "gpt", "openai-chat"); - const anthropicAttempt = beginRequestAttempt(2, "anthropic", "claude", "anthropic"); + const openaiAttempt = beginRequestAttempt( + 1, + "openai", + "gpt", + "openai-chat", + ); + const anthropicAttempt = beginRequestAttempt( + 2, + "anthropic", + "claude", + "anthropic", + ); finishRequestAttempt(openaiAttempt, 503, 1); - finishRequestAttempt(anthropicAttempt, 200, 2, { inputTokens: 1, outputTokens: 1 }); + finishRequestAttempt(anthropicAttempt, 200, 2, { + inputTokens: 1, + outputTokens: 1, + }); const logCtx: RequestLogContext = { model: "combo/free", provider: "anthropic", @@ -467,10 +565,19 @@ describe("request log metadata", () => { attempts: [openaiAttempt, anthropicAttempt], activeAttempt: anthropicAttempt, }; - addFinalRequestLog("combo-xprov", Date.now(), logCtx, 200, undefined, entry => entries.push(entry)); + addFinalRequestLog( + "combo-xprov", + Date.now(), + logCtx, + 200, + undefined, + (entry) => entries.push(entry), + ); expect(entries[0]?.providerAccountId).toBe("anthropic-seat-x"); expect(entries[0]?.attempts?.[0]?.providerAccountId).toBeUndefined(); - expect(entries[0]?.attempts?.[1]?.providerAccountId).toBe("anthropic-seat-x"); + expect(entries[0]?.attempts?.[1]?.providerAccountId).toBe( + "anthropic-seat-x", + ); }); test("streaming terminal usage updates only the committed final attempt", async () => { @@ -486,7 +593,9 @@ describe("request log metadata", () => { }, }); const response = responseWithDeferredRequestLog( - new Response(`data: ${payload}\n\n`, { headers: { "content-type": "text/event-stream" } }), + new Response(`data: ${payload}\n\n`, { + headers: { "content-type": "text/event-stream" }, + }), "combo-stream", Date.now(), { @@ -497,7 +606,7 @@ describe("request log metadata", () => { activeAttempt: attempt, activeAttemptStartedAt: Date.now(), }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); await response.text(); expect(entries).toHaveLength(1); @@ -516,20 +625,26 @@ describe("request log metadata", () => { provider: "combo", model: "combo/free", status: 200, - attempts: [{ - ordinal: 1, - provider: "a", - model: "m1", - adapter: "openai-chat", - status: 503, - durationMs: 2, - sendCount: 1, - recoveryKinds: [], - usageStatus: "unreported", - }], + attempts: [ + { + ordinal: 1, + provider: "a", + model: "m1", + adapter: "openai-chat", + status: 503, + durationMs: 2, + sendCount: 1, + recoveryKinds: [], + usageStatus: "unreported", + }, + ], }); - expect(filterRequestLogs([combo], new URLSearchParams("provider=a"))).toEqual([combo]); - expect(filterRequestLogs([combo], new URLSearchParams("provider=a&status=503"))).toEqual([]); + expect( + filterRequestLogs([combo], new URLSearchParams("provider=a")), + ).toEqual([combo]); + expect( + filterRequestLogs([combo], new URLSearchParams("provider=a&status=503")), + ).toEqual([]); }); test("records the Claude surface on the final log entry", () => { @@ -540,7 +655,7 @@ describe("request log metadata", () => { { model: "claude-sonnet-4-5", provider: "openai", surface: "claude" }, 200, { closeReason: "non_stream" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); expect(entries).toHaveLength(1); @@ -562,11 +677,15 @@ describe("request log metadata", () => { }, 200, { closeReason: "terminal" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); expect(entries).toHaveLength(1); expect(entries[0]!.usageStatus).toBe("estimated"); - expect(entries[0]!.usage).toMatchObject({ inputTokens: 44000, outputTokens: 98, estimated: true }); + expect(entries[0]!.usage).toMatchObject({ + inputTokens: 44000, + outputTokens: 98, + estimated: true, + }); }); test("accurate providers stay untouched when no input estimate is stashed", () => { @@ -579,20 +698,33 @@ describe("request log metadata", () => { provider: "anthropic-pb51d9b", providerAdapter: "anthropic", surface: "claude", - usage: { inputTokens: 353000, outputTokens: 2033, cachedInputTokens: 350000, cacheReadInputTokens: 350000, cacheCreationInputTokens: 1200 }, + usage: { + inputTokens: 353000, + outputTokens: 2033, + cachedInputTokens: 350000, + cacheReadInputTokens: 350000, + cacheCreationInputTokens: 1200, + }, }, 200, { closeReason: "terminal" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); expect(entries[0]!.usageStatus).toBe("reported"); - expect(entries[0]!.usage).toMatchObject({ inputTokens: 353000, cacheReadInputTokens: 350000 }); + expect(entries[0]!.usage).toMatchObject({ + inputTokens: 353000, + cacheReadInputTokens: 350000, + }); expect(entries[0]!.usage!.estimated).toBeUndefined(); }); test("generates compact request ids", () => { - expect(nextRequestLogId(1_700_000_000_000)).toMatch(/^ocx-[a-z0-9]+-[a-z0-9]+$/); - expect(nextRequestLogId(1_700_000_000_000)).not.toBe(nextRequestLogId(1_700_000_000_000)); + expect(nextRequestLogId(1_700_000_000_000)).toMatch( + /^ocx-[a-z0-9]+-[a-z0-9]+$/, + ); + expect(nextRequestLogId(1_700_000_000_000)).not.toBe( + nextRequestLogId(1_700_000_000_000), + ); }); test("classifies status codes with optional upstream error context", () => { @@ -600,18 +732,26 @@ describe("request log metadata", () => { expect(requestLogErrorCode(400)).toBe("invalid_request_error"); expect(requestLogErrorCode(401)).toBe("invalid_api_key"); expect(requestLogErrorCode(403)).toBe("permission_denied"); - expect(requestLogErrorCode(403, "Provider error 403")).toBe("permission_denied"); - expect(requestLogErrorCode( - 403, - "Provider error 403: this model requires a subscription, upgrade for access: https://ollama.com/upgrade", - )).toBe("subscription_required"); - expect(requestLogErrorCode( - 401, - "Provider error 401: this model requires a subscription, upgrade for access", - )).toBe("invalid_api_key"); + expect(requestLogErrorCode(403, "Provider error 403")).toBe( + "permission_denied", + ); + expect( + requestLogErrorCode( + 403, + "Provider error 403: this model requires a subscription, upgrade for access: https://ollama.com/upgrade", + ), + ).toBe("subscription_required"); + expect( + requestLogErrorCode( + 401, + "Provider error 401: this model requires a subscription, upgrade for access", + ), + ).toBe("invalid_api_key"); expect(requestLogErrorCode(429)).toBe("rate_limit_exceeded"); expect(requestLogErrorCode(499)).toBe("client_closed_request"); - expect(requestLogErrorCode(502, "client closed request during web-search")).toBe("client_closed_request"); + expect( + requestLogErrorCode(502, "client closed request during web-search"), + ).toBe("client_closed_request"); expect(requestLogErrorCode(503)).toBe("server_is_overloaded"); expect(requestLogErrorCode(502)).toBe("upstream_server_error"); expect(requestLogErrorCode(404)).toBe("http_404"); @@ -630,7 +770,7 @@ describe("request log metadata", () => { }, 403, { closeReason: "non_stream" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); expect(entries[0]).toMatchObject({ status: 403, @@ -645,11 +785,12 @@ describe("request log metadata", () => { { model: "kimi-k2.7-code", provider: "ollama-cloud", - upstreamError: "Provider error 403: this model requires a subscription, upgrade for access: https://ollama.com/upgrade", + upstreamError: + "Provider error 403: this model requires a subscription, upgrade for access: https://ollama.com/upgrade", }, 403, { closeReason: "non_stream" }, - entry => subEntries.push(entry), + (entry) => subEntries.push(entry), ); expect(subEntries[0]).toMatchObject({ status: 403, @@ -669,17 +810,42 @@ describe("request log metadata", () => { const logs = [ log({ requestId: "a", provider: "openai", status: 200 }), log({ requestId: "b", provider: "umans", status: 429 }), - log({ requestId: "c", provider: "umans", status: 502, requestedServiceTier: "priority", requestedSpeedLabel: "fast" }), + log({ + requestId: "c", + provider: "umans", + status: 502, + requestedServiceTier: "priority", + requestedSpeedLabel: "fast", + }), log({ requestId: "d", provider: "opencode-go", status: 500 }), ]; - expect(filterRequestLogs(logs, new URLSearchParams("provider=umans")).map(entry => entry.requestId)).toEqual(["b", "c"]); - expect(filterRequestLogs(logs, new URLSearchParams("status=5xx")).map(entry => entry.requestId)).toEqual(["c", "d"]); - expect(filterRequestLogs(logs, new URLSearchParams("status=429")).map(entry => entry.requestId)).toEqual(["b"]); - expect(filterRequestLogs(logs, new URLSearchParams("tail=2")).map(entry => entry.requestId)).toEqual(["c", "d"]); - - const combined = filterRequestLogs(logs, new URLSearchParams("provider=umans&status=5xx&tail=1")); - expect(combined.map(entry => entry.requestId)).toEqual(["c"]); + expect( + filterRequestLogs(logs, new URLSearchParams("provider=umans")).map( + (entry) => entry.requestId, + ), + ).toEqual(["b", "c"]); + expect( + filterRequestLogs(logs, new URLSearchParams("status=5xx")).map( + (entry) => entry.requestId, + ), + ).toEqual(["c", "d"]); + expect( + filterRequestLogs(logs, new URLSearchParams("status=429")).map( + (entry) => entry.requestId, + ), + ).toEqual(["b"]); + expect( + filterRequestLogs(logs, new URLSearchParams("tail=2")).map( + (entry) => entry.requestId, + ), + ).toEqual(["c", "d"]); + + const combined = filterRequestLogs( + logs, + new URLSearchParams("provider=umans&status=5xx&tail=1"), + ); + expect(combined.map((entry) => entry.requestId)).toEqual(["c"]); }); test("deferred JSON logging preserves response service tier before final log", async () => { @@ -699,18 +865,24 @@ describe("request log metadata", () => { modelSupportsServiceTier: true, }; const response = responseWithDeferredRequestLog( - new Response(JSON.stringify({ - model: "gpt-5.5", - service_tier: "auto", - status: "completed", - }), { status: 200, headers: { "content-type": "application/json" } }), + new Response( + JSON.stringify({ + model: "gpt-5.5", + service_tier: "auto", + status: "completed", + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), "ocx-test-json", Date.now(), logCtx, - entry => entries.push(entry), + (entry) => entries.push(entry), ); - expect(await response.json()).toMatchObject({ model: "gpt-5.5", service_tier: "auto" }); + expect(await response.json()).toMatchObject({ + model: "gpt-5.5", + service_tier: "auto", + }); expect(entries).toHaveLength(1); expect(entries[0]).toMatchObject({ requestedModel: "gpt-5.5", @@ -732,20 +904,23 @@ describe("request log metadata", () => { test("deferred JSON logging captures reported usage", async () => { const entries: RequestLogEntry[] = []; const response = responseWithDeferredRequestLog( - new Response(JSON.stringify({ - model: "gpt-5.5", - status: "completed", - usage: { - input_tokens: 100, - output_tokens: 23, - input_tokens_details: { cached_tokens: 7, cache_write_tokens: 3 }, - output_tokens_details: { reasoning_tokens: 5 }, - }, - }), { status: 200, headers: { "content-type": "application/json" } }), + new Response( + JSON.stringify({ + model: "gpt-5.5", + status: "completed", + usage: { + input_tokens: 100, + output_tokens: 23, + input_tokens_details: { cached_tokens: 7, cache_write_tokens: 3 }, + output_tokens_details: { reasoning_tokens: 5 }, + }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), "ocx-test-json-usage", Date.now(), { model: "gpt-5.5", provider: "openai" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); await response.text(); @@ -768,14 +943,17 @@ describe("request log metadata", () => { test("deferred JSON logging accepts ChatCompletions-shape usage", async () => { const entries: RequestLogEntry[] = []; const response = responseWithDeferredRequestLog( - new Response(JSON.stringify({ - model: "gpt-5.5", - usage: { prompt_tokens: 42, completion_tokens: 7 }, - }), { status: 200, headers: { "content-type": "application/json" } }), + new Response( + JSON.stringify({ + model: "gpt-5.5", + usage: { prompt_tokens: 42, completion_tokens: 7 }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), "ocx-test-json-chat-completions", Date.now(), { model: "gpt-5.5", provider: "chatgpt" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); await response.text(); expect(entries).toHaveLength(1); @@ -790,18 +968,23 @@ describe("request log metadata", () => { const entries: RequestLogEntry[] = []; const body = new ReadableStream({ start(controller) { - controller.enqueue(new TextEncoder().encode( - "data: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"model\":\"gpt-5.5\",\"usage\":{\"input_tokens\":9,\"output_tokens\":4}}}\n\n", - )); + controller.enqueue( + new TextEncoder().encode( + 'data: {"type":"response.completed","response":{"status":"completed","model":"gpt-5.5","usage":{"input_tokens":9,"output_tokens":4}}}\n\n', + ), + ); controller.close(); }, }); const response = responseWithDeferredRequestLog( - new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }), + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), "ocx-test-sse-usage", Date.now(), { model: "gpt-5.5", provider: "openai" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); await response.text(); @@ -816,7 +999,8 @@ describe("request log metadata", () => { test("deferred SSE logging marks Kiro usage as estimated without changing SSE payload", async () => { const entries: RequestLogEntry[] = []; - const payload = "{\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"model\":\"kiro/claude-sonnet-4.5\",\"usage\":{\"input_tokens\":9,\"output_tokens\":4}}}"; + const payload = + '{"type":"response.completed","response":{"status":"completed","model":"kiro/claude-sonnet-4.5","usage":{"input_tokens":9,"output_tokens":4}}}'; const body = new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode(`data: ${payload}\n\n`)); @@ -824,15 +1008,18 @@ describe("request log metadata", () => { }, }); const response = responseWithDeferredRequestLog( - new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }), + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), "ocx-test-kiro-sse-usage", Date.now(), { model: "kiro/claude-sonnet-4.5", provider: "kiro-p9d8524" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); const text = await response.text(); - expect(text).toContain("\"usage\":{\"input_tokens\":9,\"output_tokens\":4}"); + expect(text).toContain('"usage":{"input_tokens":9,"output_tokens":4}'); expect(text).not.toContain("estimated"); expect(entries).toHaveLength(1); expect(entries[0]).toMatchObject({ @@ -845,26 +1032,40 @@ describe("request log metadata", () => { test("deferred SSE logging captures the granular upstream reason from response.failed", async () => { const entries: RequestLogEntry[] = []; - const cursorMessage = "Cursor rate limit exceeded: Cursor Connect error resource_exhausted: too many requests"; + const cursorMessage = + "Cursor rate limit exceeded: Cursor Connect error resource_exhausted: too many requests"; const failedPayload = JSON.stringify({ type: "response.failed", response: { - error: { type: "rate_limit_error", code: "rate_limit_exceeded", message: cursorMessage }, - last_error: { type: "rate_limit_error", code: "rate_limit_exceeded", message: cursorMessage }, + error: { + type: "rate_limit_error", + code: "rate_limit_exceeded", + message: cursorMessage, + }, + last_error: { + type: "rate_limit_error", + code: "rate_limit_exceeded", + message: cursorMessage, + }, }, }); const body = new ReadableStream({ start(controller) { - controller.enqueue(new TextEncoder().encode(`data: ${failedPayload}\n\n`)); + controller.enqueue( + new TextEncoder().encode(`data: ${failedPayload}\n\n`), + ); controller.close(); }, }); const response = responseWithDeferredRequestLog( - new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }), + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), "ocx-test-cursor-rate-limit", Date.now(), { model: "cursor/gpt-5", provider: "cursor" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); await response.text(); @@ -883,22 +1084,35 @@ describe("request log metadata", () => { const failedPayload = JSON.stringify({ type: "response.failed", response: { - error: { type: "invalid_request_error", code: "client_closed_request", message }, - last_error: { type: "invalid_request_error", code: "client_closed_request", message }, + error: { + type: "invalid_request_error", + code: "client_closed_request", + message, + }, + last_error: { + type: "invalid_request_error", + code: "client_closed_request", + message, + }, }, }); const body = new ReadableStream({ start(controller) { - controller.enqueue(new TextEncoder().encode(`data: ${failedPayload}\n\n`)); + controller.enqueue( + new TextEncoder().encode(`data: ${failedPayload}\n\n`), + ); controller.close(); }, }); const response = responseWithDeferredRequestLog( - new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }), + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), "ocx-test-web-search-client-close", Date.now(), { model: "k3", provider: "kimi" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); await response.text(); @@ -921,7 +1135,8 @@ describe("request log metadata", () => { model: "cline-sonnet", provider: "cline-pass", providerConfigKey: "cline-pass", - upstreamError: 'Error 429: {"code":"INFERENCE_CAP_ERROR","message":"weekly Clinepass limit. The limit resets in 1d 22h"}', + upstreamError: + 'Error 429: {"code":"INFERENCE_CAP_ERROR","message":"weekly Clinepass limit. The limit resets in 1d 22h"}', }, 429, { closeReason: "non_stream" }, @@ -943,7 +1158,7 @@ describe("request log metadata", () => { }, 502, { terminalStatus: "failed", closeReason: "terminal" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); expect(entries[0]).toMatchObject({ status: 499, @@ -966,7 +1181,7 @@ describe("request log metadata", () => { }, 400, { closeReason: "non_stream" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); expect(entries[0]).toMatchObject({ model: "tencent/hy3:free", @@ -983,7 +1198,7 @@ describe("request log metadata", () => { { model: "gpt-5.6-sol", provider: "openai-p104398" }, 200, undefined, - entry => entries.push(entry), + (entry) => entries.push(entry), ); expect(entries[0]).toMatchObject({ provider: "openai-p104398", @@ -1000,7 +1215,12 @@ describe("request log metadata", () => { // Placeholder token shape is constrained by scripts/privacy-scan.ts's tests/ allowlist. const secret = "sk-test-000111222333444"; try { - const attempt = beginRequestAttempt(1, "openai", "gpt-5.5", "openai-chat"); + const attempt = beginRequestAttempt( + 1, + "openai", + "gpt-5.5", + "openai-chat", + ); const logCtx: RequestLogContext = { model: "gpt-5.5", provider: "openai", @@ -1016,9 +1236,13 @@ describe("request log metadata", () => { keyPool: { provider: "openai", accountId: resolvedId! }, }); bindLogProviderAccount(logCtx, "key-pool", "openai", "k2b3c4d5"); - expect(getProviderAccountOccupancy("key-pool", "openai", "k2b3c4d5").inFlight).toBe(1); + expect( + getProviderAccountOccupancy("key-pool", "openai", "k2b3c4d5").inFlight, + ).toBe(1); addFinalRequestLog("ocx-key-pool", 1, logCtx, 200); - expect(getProviderAccountOccupancy("key-pool", "openai", "k2b3c4d5").inFlight).toBe(0); + expect( + getProviderAccountOccupancy("key-pool", "openai", "k2b3c4d5").inFlight, + ).toBe(0); expect(logCtx.providerAccountId).toBeUndefined(); const raw = readFileSync(usageLogPath(), "utf-8"); expect(raw).not.toContain(secret); @@ -1039,8 +1263,12 @@ describe("request log metadata", () => { test("codex bind without config hashes providerAccountId for usage", () => { const logCtx: RequestLogContext = { model: "gpt-5", provider: "codex" }; - bindLogFromSelectCandidate(logCtx, { authCtx: { kind: "pool", accountId: "raw-store-id-12345" } }); - expect(logCtx.providerAccountId).toBe(fallbackCodexAccountLogLabel("raw-store-id-12345")); + bindLogFromSelectCandidate(logCtx, { + authCtx: { kind: "pool", accountId: "raw-store-id-12345" }, + }); + expect(logCtx.providerAccountId).toBe( + fallbackCodexAccountLogLabel("raw-store-id-12345"), + ); expect(logCtx.providerAccountId).not.toBe("raw-store-id-12345"); releaseRequestProviderAccount(logCtx); }); @@ -1053,56 +1281,84 @@ describe("request log metadata", () => { activeAttempt: beginRequestAttempt(1, "anthropic", "claude", "anthropic"), }; bindLogProviderAccount(logCtx, "oauth", "anthropic", "aaaa1111"); - expect(getProviderAccountOccupancy("oauth", "anthropic", "aaaa1111").inFlight).toBe(1); - expect(() => addFinalRequestLog("throw-log", Date.now(), logCtx, 500, undefined, () => { - throw new Error("persist failed"); - })).toThrow("persist failed"); - expect(getProviderAccountOccupancy("oauth", "anthropic", "aaaa1111").inFlight).toBe(0); + expect( + getProviderAccountOccupancy("oauth", "anthropic", "aaaa1111").inFlight, + ).toBe(1); + expect(() => + addFinalRequestLog( + "throw-log", + Date.now(), + logCtx, + 500, + undefined, + () => { + throw new Error("persist failed"); + }, + ), + ).toThrow("persist failed"); + expect( + getProviderAccountOccupancy("oauth", "anthropic", "aaaa1111").inFlight, + ).toBe(0); }); test("httpStatusFromTerminalError maps Cursor tool catalog limits to 400", () => { - expect(httpStatusFromTerminalError({ - type: "invalid_request_error", - code: "tool_catalog_too_large", - message: "Cursor resource limit exceeded: tool catalog too large", - })).toBe(400); + expect( + httpStatusFromTerminalError({ + type: "invalid_request_error", + code: "tool_catalog_too_large", + message: "Cursor resource limit exceeded: tool catalog too large", + }), + ).toBe(400); }); test("httpStatusFromTerminalError maps Cursor quota-style resource exhaustion to 429", () => { - expect(httpStatusFromTerminalError({ - type: "rate_limit_error", - code: "rate_limit_exceeded", - message: "Cursor rate limit exceeded: Cursor Connect error resource limit exceeded: Error", - })).toBe(429); + expect( + httpStatusFromTerminalError({ + type: "rate_limit_error", + code: "rate_limit_exceeded", + message: + "Cursor rate limit exceeded: Cursor Connect error resource limit exceeded: Error", + }), + ).toBe(429); }); test("httpStatusFromTerminalError maps client-closed web-search aborts to 499", () => { - expect(httpStatusFromTerminalError({ - type: "invalid_request_error", - code: "client_closed_request", - message: "client closed request during web-search", - })).toBe(499); - expect(httpStatusFromTerminalError({ - message: "client closed request during web-search", - })).toBe(499); + expect( + httpStatusFromTerminalError({ + type: "invalid_request_error", + code: "client_closed_request", + message: "client closed request during web-search", + }), + ).toBe(499); + expect( + httpStatusFromTerminalError({ + message: "client closed request during web-search", + }), + ).toBe(499); }); test("httpStatusFromTerminalError preserves auth precedence and permission status", () => { - expect(httpStatusFromTerminalError({ - type: "authentication_error", - code: "invalid_api_key", - message: "upgrade your subscription", - })).toBe(401); - expect(httpStatusFromTerminalError({ - type: "permission_error", - code: "permission_denied", - message: "Access denied", - })).toBe(403); - expect(httpStatusFromTerminalError({ - type: "permission_error", - code: "subscription_required", - message: "this model requires a subscription", - })).toBe(403); + expect( + httpStatusFromTerminalError({ + type: "authentication_error", + code: "invalid_api_key", + message: "upgrade your subscription", + }), + ).toBe(401); + expect( + httpStatusFromTerminalError({ + type: "permission_error", + code: "permission_denied", + message: "Access denied", + }), + ).toBe(403); + expect( + httpStatusFromTerminalError({ + type: "permission_error", + code: "subscription_required", + message: "this model requires a subscription", + }), + ).toBe(403); }); test("upstream reason capture redacts secret-shaped error messages", async () => { @@ -1113,20 +1369,27 @@ describe("request log metadata", () => { }); const body = new ReadableStream({ start(controller) { - controller.enqueue(new TextEncoder().encode(`data: ${failedPayload}\n\n`)); + controller.enqueue( + new TextEncoder().encode(`data: ${failedPayload}\n\n`), + ); controller.close(); }, }); const response = responseWithDeferredRequestLog( - new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }), + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), "ocx-test-cursor-redact", Date.now(), { model: "cursor/gpt-5", provider: "cursor" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); const text = await response.text(); - expect(text).toContain("\"message\":\"unauthorized: Bearer secret-leak-abc123\""); + expect(text).toContain( + '"message":"unauthorized: Bearer secret-leak-abc123"', + ); expect(entries).toHaveLength(1); expect(entries[0].upstreamError).not.toContain("secret-leak-abc123"); expect(entries[0].upstreamError).toContain("[REDACTED]"); @@ -1135,11 +1398,17 @@ describe("request log metadata", () => { test("plain-text upstream errors are captured in deferred logging", async () => { const entries: RequestLogEntry[] = []; const response = responseWithDeferredRequestLog( - new Response("provider says nope", { status: 400, headers: { "content-type": "text/plain" } }), + new Response("provider says nope", { + status: 400, + headers: { "content-type": "text/plain" }, + }), "ocx-test-plain-upstream-error", Date.now(), - { model: "opencode-free/deepseek-v4-flash-free", provider: "opencode-free" }, - entry => entries.push(entry), + { + model: "opencode-free/deepseek-v4-flash-free", + provider: "opencode-free", + }, + (entry) => entries.push(entry), ); const text = await response.text(); @@ -1150,7 +1419,8 @@ describe("request log metadata", () => { test("deferred SSE logging uses adapter-provided Kiro log input tokens", async () => { const entries: RequestLogEntry[] = []; - const payload = "{\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"model\":\"kiro/claude-sonnet-4.5\",\"usage\":{\"input_tokens\":9,\"output_tokens\":4}}}"; + const payload = + '{"type":"response.completed","response":{"status":"completed","model":"kiro/claude-sonnet-4.5","usage":{"input_tokens":9,"output_tokens":4}}}'; const body = new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode(`data: ${payload}\n\n`)); @@ -1158,15 +1428,22 @@ describe("request log metadata", () => { }, }); const response = responseWithDeferredRequestLog( - new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }), + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), "ocx-test-kiro-sse-log-usage", Date.now(), - { model: "kiro/claude-sonnet-4.5", provider: "kiro-p9d8524", usageLogInputTokens: 240_000 }, - entry => entries.push(entry), + { + model: "kiro/claude-sonnet-4.5", + provider: "kiro-p9d8524", + usageLogInputTokens: 240_000, + }, + (entry) => entries.push(entry), ); const text = await response.text(); - expect(text).toContain("\"input_tokens\":9"); + expect(text).toContain('"input_tokens":9'); expect(entries).toHaveLength(1); expect(entries[0]).toMatchObject({ usageStatus: "estimated", @@ -1177,21 +1454,33 @@ describe("request log metadata", () => { test("deferred logging preserves a bridged Kiro absolute context checkpoint", async () => { const entries: RequestLogEntry[] = []; - const body = bridgeToResponsesSSE(replayAdapterEvents([{ - type: "done", - usage: { - inputTokens: 58, - outputTokens: 100, - contextTotalTokens: 50_000, - estimated: true, - }, - }]), "kiro/claude-opus-5"); + const body = bridgeToResponsesSSE( + replayAdapterEvents([ + { + type: "done", + usage: { + inputTokens: 58, + outputTokens: 100, + contextTotalTokens: 50_000, + estimated: true, + }, + }, + ]), + "kiro/claude-opus-5", + ); const response = responseWithDeferredRequestLog( - new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }), + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), "ocx-test-kiro-context-checkpoint", Date.now(), - { model: "kiro/claude-opus-5", provider: "kiro-p9d8524", usageLogInputTokens: 200 }, - entry => entries.push(entry), + { + model: "kiro/claude-opus-5", + provider: "kiro-p9d8524", + usageLogInputTokens: 200, + }, + (entry) => entries.push(entry), ); const text = await response.text(); @@ -1201,7 +1490,12 @@ describe("request log metadata", () => { expect(entries[0]).toMatchObject({ usageStatus: "estimated", totalTokens: 50_000, - usage: { inputTokens: 49_900, outputTokens: 100, totalTokens: 50_000, estimated: true }, + usage: { + inputTokens: 49_900, + outputTokens: 100, + totalTokens: 50_000, + estimated: true, + }, }); }); @@ -1221,15 +1515,17 @@ describe("request log metadata", () => { usageLogInputTokens: 200, }; const body = bridgeToResponsesSSE( - replayAdapterEvents([{ - type: "done", - usage: { - inputTokens: 58, - outputTokens: 100, - contextTotalTokens: 50_000, - estimated: true, + replayAdapterEvents([ + { + type: "done", + usage: { + inputTokens: 58, + outputTokens: 100, + contextTotalTokens: 50_000, + estimated: true, + }, }, - }]), + ]), "kiro/claude-opus-5", undefined, undefined, @@ -1237,7 +1533,7 @@ describe("request log metadata", () => { undefined, undefined, { - onUsage: usage => { + onUsage: (usage) => { // Mirror responses/core.ts: store RAW adapter usage and mark provenance so the // deferred logger does not re-parse the wire. reportedRaw = usage; @@ -1247,16 +1543,22 @@ describe("request log metadata", () => { }, ); const response = responseWithDeferredRequestLog( - new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }), + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), "ocx-test-kiro-raw-usage-checkpoint", Date.now(), logCtx, - entry => entries.push(entry), + (entry) => entries.push(entry), ); await response.text(); // The bridge hands the logger the RAW adapter usage, not the projected wire shape. - expect(reportedRaw).toMatchObject({ inputTokens: 58, contextTotalTokens: 50_000 }); + expect(reportedRaw).toMatchObject({ + inputTokens: 58, + contextTotalTokens: 50_000, + }); expect(entries).toHaveLength(1); const logged = entries[0]?.usage; expect(logged?.contextTotalTokens).toBe(50_000); @@ -1299,8 +1601,12 @@ describe("request log metadata", () => { new Response(null, { status: 200 }), "ocx-test-kiro-fallback-log-usage", Date.now(), - { model: "kiro/claude-opus-4.8", provider: "kiro-p442fff", usageLogInputTokens: 133_900 }, - entry => entries.push(entry), + { + model: "kiro/claude-opus-4.8", + provider: "kiro-p442fff", + usageLogInputTokens: 133_900, + }, + (entry) => entries.push(entry), ); await response.text(); @@ -1323,16 +1629,21 @@ describe("request log metadata", () => { }); const body = new ReadableStream({ start(controller) { - controller.enqueue(new TextEncoder().encode(`data: ${incompletePayload}\n\n`)); + controller.enqueue( + new TextEncoder().encode(`data: ${incompletePayload}\n\n`), + ); controller.close(); }, }); const response = responseWithDeferredRequestLog( - new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }), + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), "ocx-test-stall-timeout", Date.now(), { model: "cursor/kimi-k2.7-code", provider: "cursor" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); await response.text(); @@ -1358,16 +1669,21 @@ describe("request log metadata", () => { }); const body = new ReadableStream({ start(controller) { - controller.enqueue(new TextEncoder().encode(`data: ${incompletePayload}\n\n`)); + controller.enqueue( + new TextEncoder().encode(`data: ${incompletePayload}\n\n`), + ); controller.close(); }, }); const response = responseWithDeferredRequestLog( - new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }), + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), "ocx-test-requested-output-limit", Date.now(), { model: "anthropic/claude-sonnet-5", provider: "anthropic" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); await response.text(); @@ -1377,7 +1693,8 @@ describe("request log metadata", () => { status: 200, usageStatus: "reported", usage: { inputTokens: 9, outputTokens: 64 }, - upstreamError: "Output reached the requested token limit (max_output_tokens)", + upstreamError: + "Output reached the requested token limit (max_output_tokens)", }); expect(entries[0]).not.toHaveProperty("errorCode"); }); @@ -1393,16 +1710,21 @@ describe("request log metadata", () => { }); const body = new ReadableStream({ start(controller) { - controller.enqueue(new TextEncoder().encode(`data: ${incompletePayload}\n\n`)); + controller.enqueue( + new TextEncoder().encode(`data: ${incompletePayload}\n\n`), + ); controller.close(); }, }); const response = responseWithDeferredRequestLog( - new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }), + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), "ocx-test-adapter-eof", Date.now(), { model: "cursor/kimi-k2.7-code", provider: "cursor" }, - entry => entries.push(entry), + (entry) => entries.push(entry), ); await response.text(); @@ -1500,7 +1822,10 @@ describe("request log restart hydrate", () => { ]; expect(hydrateRequestLogsFromDisk(() => persisted)).toBe(2); - expect(getRequestLogEntries().map(e => e.requestId)).toEqual(["ocx-old", "ocx-sticky-502"]); + expect(getRequestLogEntries().map((e) => e.requestId)).toEqual([ + "ocx-old", + "ocx-sticky-502", + ]); expect(getRequestLogEntries()[1]).toMatchObject({ requestId: "ocx-sticky-502", status: 502, @@ -1516,17 +1841,20 @@ describe("request log restart hydrate", () => { test("hydrate keeps only the newest MAX_LOG_SIZE rows from a long usage.jsonl", () => { clearRequestLogsForTests(); - const persisted: PersistedUsageEntry[] = Array.from({ length: 205 }, (_, i) => ({ - requestId: `ocx-${i}`, - timestamp: i, - provider: "openai", - model: "gpt", - status: 200, - durationMs: 1, - usageStatus: "unreported" as const, - })); + const persisted: PersistedUsageEntry[] = Array.from( + { length: 205 }, + (_, i) => ({ + requestId: `ocx-${i}`, + timestamp: i, + provider: "openai", + model: "gpt", + status: 200, + durationMs: 1, + usageStatus: "unreported" as const, + }), + ); expect(hydrateRequestLogsFromDisk(() => persisted)).toBe(200); - const ids = getRequestLogEntries().map(e => e.requestId); + const ids = getRequestLogEntries().map((e) => e.requestId); expect(ids[0]).toBe("ocx-5"); expect(ids.at(-1)).toBe("ocx-204"); }); @@ -1535,17 +1863,134 @@ describe("request log restart hydrate", () => { clearRequestLogsForTests(); const warn = spyOn(console, "warn").mockImplementation(() => {}); try { - expect(hydrateRequestLogsFromDisk(() => { - throw new Error("EISDIR: illegal operation on a directory"); - })).toBe(0); + expect( + hydrateRequestLogsFromDisk(() => { + throw new Error("EISDIR: illegal operation on a directory"); + }), + ).toBe(0); expect(getRequestLogEntries()).toHaveLength(0); expect(warn).toHaveBeenCalled(); // Still idempotent after the failed attempt. - expect(hydrateRequestLogsFromDisk(() => { - throw new Error("should not run"); - })).toBe(0); + expect( + hydrateRequestLogsFromDisk(() => { + throw new Error("should not run"); + }), + ).toBe(0); } finally { warn.mockRestore(); } }); }); + +describe("prompt-cache request log lifecycle", () => { + test("snapshots adapter metadata at recording, finalization, and hydration", () => { + const observation = observeOpenAiResponsesPromptCache({ + tools: [{ type: "function", name: "lookup" }], + })!; + const context: RequestLogContext = { + provider: "openai", + model: "test-model", + providerAdapter: "openai-responses", + }; + recordAdapterRequestMetadata(context, { + url: "https://provider.test/v1/responses", + method: "POST", + headers: {}, + body: "{}", + promptCacheLog: observation, + }); + expect(context.promptCache).toEqual(observation); + expect(context.promptCache).not.toBe(observation); + observation.toolCount = 99; + expect(context.promptCache?.toolCount).toBe(1); + + const entries: RequestLogEntry[] = []; + addFinalRequestLog( + "cache-snapshot", + Date.now(), + context, + 200, + undefined, + (entry) => entries.push(entry), + ); + expect(entries).toHaveLength(1); + expect(entries[0]).toMatchObject({ + adapter: "openai-responses", + promptCache: { toolCount: 1 }, + }); + expect(entries[0].promptCache).not.toBe(context.promptCache); + context.promptCache!.toolCount = 50; + expect(entries[0].promptCache?.toolCount).toBe(1); + + const restored = requestLogEntryFromPersistedUsage(entries[0]); + expect(restored.adapter).toBe("openai-responses"); + expect(restored.promptCache).toEqual(entries[0].promptCache); + expect(restored.promptCache).not.toBe(entries[0].promptCache); + restored.promptCache!.toolCount = 25; + expect(entries[0].promptCache?.toolCount).toBe(1); + }); + + test.each([true, false])( + "combo winner determines adapter and cache metadata (observed=%s)", + (observed) => { + const context: RequestLogContext = { + provider: "winner", + model: "combo/test", + comboId: "test", + providerAdapter: "openai-responses", + attempts: [ + finishRequestAttempt( + beginRequestAttempt(1, "first", "a", "openai-responses"), + 503, + 1, + ), + finishRequestAttempt( + beginRequestAttempt(2, "winner", "b", "openai-chat"), + 200, + 1, + ), + ], + }; + const request = { + url: "https://provider.test/v1/responses", + method: "POST" as const, + headers: {}, + body: "{}", + }; + recordAdapterRequestMetadata(context, { + ...request, + promptCacheLog: observeOpenAiResponsesPromptCache({ + prompt_cache_key: "initial", + }), + }); + const finalObservation = observed + ? observeOpenAiResponsesPromptCache({ + prompt_cache_options: { mode: "explicit" }, + }) + : undefined; + recordAdapterRequestMetadata(context, { + ...request, + promptCacheLog: finalObservation, + }); + const entries: RequestLogEntry[] = []; + addFinalRequestLog( + "cache-failover", + Date.now(), + context, + 200, + undefined, + (entry) => entries.push(entry), + ); + expect(entries).toHaveLength(1); + expect(entries[0].adapter).toBe("openai-chat"); + if (observed) expect(entries[0].promptCache).toEqual(finalObservation); + else expect(entries[0]).not.toHaveProperty("promptCache"); + }, + ); + + test("historical usage rows hydrate without inventing adapter or cache metadata", () => { + const restored = requestLogEntryFromPersistedUsage(log({})); + expect(restored).not.toHaveProperty("adapter"); + expect(restored).not.toHaveProperty("promptCache"); + }); +}); diff --git a/tests/usage-log.test.ts b/tests/usage-log.test.ts index a3929b7a4..539cdb628 100644 --- a/tests/usage-log.test.ts +++ b/tests/usage-log.test.ts @@ -1,5 +1,12 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -18,6 +25,8 @@ import { usageLogRevisionKey, } from "../src/usage/log"; +import { observeOpenAiResponsesPromptCache } from "../src/prompt-cache/observability"; + let testDir = ""; let previousHome: string | undefined; @@ -35,24 +44,28 @@ afterEach(() => { }); describe("usage log", () => { - const persistedLine = (requestId: string) => JSON.stringify({ - requestId, - timestamp: 1, - provider: "openai", - model: "gpt-5.5", - status: 200, - durationMs: 1, - usageStatus: "reported", - usage: { inputTokens: 1, outputTokens: 1 }, - totalTokens: 2, - }); + const persistedLine = (requestId: string) => + JSON.stringify({ + requestId, + timestamp: 1, + provider: "openai", + model: "gpt-5.5", + status: 200, + durationMs: 1, + usageStatus: "reported", + usage: { inputTokens: 1, outputTokens: 1 }, + totalTokens: 2, + }); test("file revisions change after append and in-place rewrite", () => { - writeFileSync(usageLogPath(), `${persistedLine("a")}\n${persistedLine("b")}\n`); + writeFileSync( + usageLogPath(), + `${persistedLine("a")}\n${persistedLine("b")}\n`, + ); const first = usageLogRevisionKey(currentUsageLogRevision()); writeFileSync(usageLogPath(), `${persistedLine("new")}\n`); expect(usageLogRevisionKey(currentUsageLogRevision())).not.toBe(first); - expect(readUsageEntries().map(entry => entry.requestId)).toEqual(["new"]); + expect(readUsageEntries().map((entry) => entry.requestId)).toEqual(["new"]); }); test("management full reads yield while parsing a large existing log", async () => { @@ -61,11 +74,17 @@ describe("usage log", () => { `${Array.from({ length: 2_100 }, (_, index) => persistedLine(`row-${index}`)).join("\n")}\n`, ); let timerRan = false; - setTimeout(() => { timerRan = true; }, 0); + setTimeout(() => { + timerRan = true; + }, 0); const entries = await readUsageEntriesForManagement(); expect(entries).toHaveLength(2_100); expect(timerRan).toBe(true); - expect(usageReadCacheStatsForTests()).toEqual({ fullReads: 1, tailReads: 0, parsedLines: 2_100 }); + expect(usageReadCacheStatsForTests()).toEqual({ + fullReads: 1, + tailReads: 0, + parsedLines: 2_100, + }); }); test("a replacement does not join an in-flight read for the previous file revision", async () => { @@ -74,13 +93,17 @@ describe("usage log", () => { `${Array.from({ length: 2_100 }, (_, index) => persistedLine(`old-${index}`)).join("\n")}\n`, ); const oldRead = readUsageSnapshotForManagement(); - await new Promise(resolve => setTimeout(resolve, 0)); + await new Promise((resolve) => setTimeout(resolve, 0)); writeFileSync(usageLogPath(), `${persistedLine("replacement")}\n`); const newRead = readUsageSnapshotForManagement(); const [oldSnapshot, newSnapshot] = await Promise.all([oldRead, newRead]); expect(oldSnapshot.entries).toHaveLength(2_100); - expect(newSnapshot.entries.map(entry => entry.requestId)).toEqual(["replacement"]); - expect(usageLogRevisionKey(newSnapshot.revision)).not.toBe(usageLogRevisionKey(oldSnapshot.revision)); + expect(newSnapshot.entries.map((entry) => entry.requestId)).toEqual([ + "replacement", + ]); + expect(usageLogRevisionKey(newSnapshot.revision)).not.toBe( + usageLogRevisionKey(oldSnapshot.revision), + ); }); test("persists conversationId for Logs session correlation", () => { @@ -96,10 +119,12 @@ describe("usage log", () => { usage: { inputTokens: 1, outputTokens: 1 }, totalTokens: 2, }); - expect(readUsageEntries()).toEqual([expect.objectContaining({ - requestId: "ocx-conversation", - conversationId: "thread-abc", - })]); + expect(readUsageEntries()).toEqual([ + expect.objectContaining({ + requestId: "ocx-conversation", + conversationId: "thread-abc", + }), + ]); }); test("persists adapter and bounded prompt-cache observations", () => { @@ -129,33 +154,90 @@ describe("usage log", () => { usage: { inputTokens: 100, outputTokens: 1, cacheReadInputTokens: 80 }, totalTokens: 101, }); - expect(readUsageEntries()).toEqual([expect.objectContaining({ - adapter: "openai-responses", - promptCache: expect.objectContaining({ - mode: "implicit", - ttl: "30m", - toolCount: 1, - toolsFingerprint: "0123456789abcdef01234567", + expect(readUsageEntries()).toEqual([ + expect.objectContaining({ + adapter: "openai-responses", + promptCache: expect.objectContaining({ + mode: "implicit", + ttl: "30m", + toolCount: 1, + toolsFingerprint: "0123456789abcdef01234567", + }), }), - })]); + ]); }); test("drops malformed prompt-cache observations from hand-edited logs", () => { - writeFileSync(usageLogPath(), JSON.stringify({ - requestId: "bad-cache-shape", + writeFileSync( + usageLogPath(), + JSON.stringify({ + requestId: "bad-cache-shape", + timestamp: 1, + provider: "openai", + model: "gpt-5.6-terra", + adapter: "openai-responses", + promptCache: { version: 1, mode: "implicit", keyPresent: true }, + status: 200, + durationMs: 1, + usageStatus: "reported", + usage: { inputTokens: 1, outputTokens: 1 }, + }) + "\n", + ); + expect(readUsageEntries()[0]).not.toHaveProperty("promptCache"); + }); + + test("sanitizes prompt-cache fields before writing bytes and after reading hand-edited logs", () => { + const core = observeOpenAiResponsesPromptCache({}); + const entry = { + requestId: "cache-boundary", timestamp: 1, provider: "openai", - model: "gpt-5.6-terra", - adapter: "openai-responses", - promptCache: { version: 1, mode: "implicit", keyPresent: true }, + model: "test-model", + adapter: " openai-responses ", status: 200, durationMs: 1, - usageStatus: "reported", - usage: { inputTokens: 1, outputTokens: 1 }, - }) + "\n"); - expect(readUsageEntries()[0]).not.toHaveProperty("promptCache"); + usageStatus: "reported" as const, + promptCache: { + ...core!, + toolsFingerprint: "private fingerprint", + ttl: "30m" as const, + prompt_cache_key: "private cache key", + instructions: "private instructions", + }, + }; + appendUsageEntry(entry); + const expected = { + ...entry, + adapter: "openai-responses", + promptCache: { ...core, ttl: "30m" }, + }; + expect(JSON.parse(readFileSync(usageLogPath(), "utf8"))).toEqual(expected); + expect(readUsageEntries()).toEqual([expected]); + + writeFileSync(usageLogPath(), JSON.stringify(entry) + "\n"); + resetUsageReadCacheForTests(); + expect(readUsageEntries()).toEqual([expected]); }); + test.each(["", " \t", null, 42, "x".repeat(65)])( + "bounds persisted adapter identity: %j", + (adapter) => { + writeFileSync( + usageLogPath(), + JSON.stringify({ + ...JSON.parse(persistedLine("adapter-boundary")), + adapter, + }) + "\n", + ); + const entry = readUsageEntries()[0]; + if (typeof adapter === "string" && adapter.trim()) { + expect(entry.adapter).toBe("x".repeat(64)); + } else { + expect(entry).not.toHaveProperty("adapter"); + } + }, + ); + test("persists providerAccountId separately from the display account label", () => { appendUsageEntry({ requestId: "ocx-account-id", @@ -167,25 +249,29 @@ describe("usage log", () => { usageStatus: "reported", account: "pab12cd", providerAccountId: "aaaa1111", - attempts: [{ - ordinal: 1, - provider: "anthropic", - model: "claude-opus-4", - adapter: "anthropic", - status: 200, - durationMs: 1, - sendCount: 1, - recoveryKinds: [], - usageStatus: "reported", - providerAccountId: "aaaa1111", - }], + attempts: [ + { + ordinal: 1, + provider: "anthropic", + model: "claude-opus-4", + adapter: "anthropic", + status: 200, + durationMs: 1, + sendCount: 1, + recoveryKinds: [], + usageStatus: "reported", + providerAccountId: "aaaa1111", + }, + ], }); - expect(readUsageEntries()).toEqual([expect.objectContaining({ - requestId: "ocx-account-id", - account: "pab12cd", - providerAccountId: "aaaa1111", - attempts: [expect.objectContaining({ providerAccountId: "aaaa1111" })], - })]); + expect(readUsageEntries()).toEqual([ + expect.objectContaining({ + requestId: "ocx-account-id", + account: "pab12cd", + providerAccountId: "aaaa1111", + attempts: [expect.objectContaining({ providerAccountId: "aaaa1111" })], + }), + ]); }); test("persists an absolute context checkpoint for stateful providers", () => { @@ -200,20 +286,27 @@ describe("usage log", () => { status: 200, durationMs: 10, usageStatus: "estimated", - usage: { inputTokens: 220, outputTokens: 252, contextTotalTokens: 127_000, estimated: true }, - totalTokens: 472, - }); - expect(readUsageEntries()).toEqual([expect.objectContaining({ - requestId: "ocx-context-checkpoint", - usage: expect.objectContaining({ + usage: { inputTokens: 220, outputTokens: 252, contextTotalTokens: 127_000, estimated: true, - }), - // The checkpoint must NOT be folded into the per-request total. + }, totalTokens: 472, - })]); + }); + expect(readUsageEntries()).toEqual([ + expect.objectContaining({ + requestId: "ocx-context-checkpoint", + usage: expect.objectContaining({ + inputTokens: 220, + outputTokens: 252, + contextTotalTokens: 127_000, + estimated: true, + }), + // The checkpoint must NOT be folded into the per-request total. + totalTokens: 472, + }), + ]); }); test("never invents a context checkpoint when the adapter reported none", () => { @@ -246,7 +339,56 @@ describe("usage log", () => { usageStatus: "estimated", usage: { inputTokens: 15, outputTokens: 2, estimated: true }, totalTokens: 17, - attempts: [{ + attempts: [ + { + ordinal: 1, + provider: "a", + model: "m1", + adapter: "openai-chat", + status: 503, + durationMs: 4, + sendCount: 2, + recoveryKinds: ["transient-5xx", "transient-5xx", "oauth-401"], + usageStatus: "estimated", + inputTokenEstimate: 5, + usage: { inputTokens: 5, outputTokens: 0, estimated: true }, + totalTokens: 5, + requestedEffort: "max", + effectiveEffort: "high", + reasoningWireField: "reasoning_effort", + reasoningWireValue: "high", + headers: { authorization: "Bearer attempt-token" }, + body: "attempt body secret", + messages: ["attempt message secret"], + accessToken: "attempt-access", + refreshToken: "attempt-refresh", + error: "raw attempt error", + } as never, + ], + headers: { authorization: "Bearer parent-token" }, + body: "parent body secret", + messages: ["parent message secret"], + } as unknown as Parameters[0]); + + const raw = readFileSync(usageLogPath(), "utf-8"); + for (const forbidden of [ + "attempt-token", + "attempt body secret", + "attempt message secret", + "attempt-access", + "attempt-refresh", + "raw attempt error", + "parent-token", + "parent body secret", + "parent message secret", + "authorization", + "headers", + "messages", + "refreshToken", + ]) + expect(raw).not.toContain(forbidden); + expect(readUsageEntries()[0]?.attempts).toEqual([ + { ordinal: 1, provider: "a", model: "m1", @@ -254,7 +396,7 @@ describe("usage log", () => { status: 503, durationMs: 4, sendCount: 2, - recoveryKinds: ["transient-5xx", "transient-5xx", "oauth-401"], + recoveryKinds: ["transient-5xx", "oauth-401"], usageStatus: "estimated", inputTokenEstimate: 5, usage: { inputTokens: 5, outputTokens: 0, estimated: true }, @@ -263,43 +405,8 @@ describe("usage log", () => { effectiveEffort: "high", reasoningWireField: "reasoning_effort", reasoningWireValue: "high", - headers: { authorization: "Bearer attempt-token" }, - body: "attempt body secret", - messages: ["attempt message secret"], - accessToken: "attempt-access", - refreshToken: "attempt-refresh", - error: "raw attempt error", - } as never], - headers: { authorization: "Bearer parent-token" }, - body: "parent body secret", - messages: ["parent message secret"], - } as unknown as Parameters[0]); - - const raw = readFileSync(usageLogPath(), "utf-8"); - for (const forbidden of [ - "attempt-token", "attempt body secret", "attempt message secret", - "attempt-access", "attempt-refresh", "raw attempt error", - "parent-token", "parent body secret", "parent message secret", - "authorization", "headers", "messages", "refreshToken", - ]) expect(raw).not.toContain(forbidden); - expect(readUsageEntries()[0]?.attempts).toEqual([{ - ordinal: 1, - provider: "a", - model: "m1", - adapter: "openai-chat", - status: 503, - durationMs: 4, - sendCount: 2, - recoveryKinds: ["transient-5xx", "oauth-401"], - usageStatus: "estimated", - inputTokenEstimate: 5, - usage: { inputTokens: 5, outputTokens: 0, estimated: true }, - totalTokens: 5, - requestedEffort: "max", - effectiveEffort: "high", - reasoningWireField: "reasoning_effort", - reasoningWireValue: "high", - }]); + }, + ]); }); test("omits malformed optional attempt reasoning metadata without dropping the attempt", () => { @@ -311,21 +418,23 @@ describe("usage log", () => { status: 200, durationMs: 4, usageStatus: "unreported", - attempts: [{ - ordinal: 1, - provider: "a", - model: "m1", - adapter: "openai-chat", - status: 200, - durationMs: 3, - sendCount: 1, - recoveryKinds: [], - usageStatus: "unreported", - requestedEffort: 123, - effectiveEffort: null, - reasoningWireField: {}, - reasoningWireValue: -1, - } as never], + attempts: [ + { + ordinal: 1, + provider: "a", + model: "m1", + adapter: "openai-chat", + status: 200, + durationMs: 3, + sendCount: 1, + recoveryKinds: [], + usageStatus: "unreported", + requestedEffort: 123, + effectiveEffort: null, + reasoningWireField: {}, + reasoningWireValue: -1, + } as never, + ], }); const attempt = readUsageEntries()[0]?.attempts?.[0]; @@ -363,21 +472,26 @@ describe("usage log", () => { { ...valid(2), usage: { inputTokens: 2, outputTokens: "1" } }, ]; for (const middle of malformed) { - writeFileSync(usageLogPath(), `${JSON.stringify({ - requestId: "parent", - timestamp: 1, - provider: "combo", - model: "combo/free", - status: 200, - durationMs: 3, - usageStatus: "reported", - usage: { inputTokens: 4, outputTokens: 2 }, - totalTokens: 6, - attempts: [valid(1), middle, valid(3)], - })}\n`); + writeFileSync( + usageLogPath(), + `${JSON.stringify({ + requestId: "parent", + timestamp: 1, + provider: "combo", + model: "combo/free", + status: 200, + durationMs: 3, + usageStatus: "reported", + usage: { inputTokens: 4, outputTokens: 2 }, + totalTokens: 6, + attempts: [valid(1), middle, valid(3)], + })}\n`, + ); const [entry] = readUsageEntries(); expect(entry?.requestId).toBe("parent"); - expect(entry?.attempts?.map(attempt => attempt.ordinal)).toEqual([1, 3]); + expect(entry?.attempts?.map((attempt) => attempt.ordinal)).toEqual([ + 1, 3, + ]); } }); @@ -393,20 +507,22 @@ describe("usage log", () => { usageStatus: "reported", usage: { inputTokens: 10, outputTokens: 5 }, totalTokens: 15, - attempts: [{ - ordinal: 1, - provider: "a", - model: "m1", - adapter: "openai-chat", - status: 200, - durationMs: 18, - firstOutputMs: 3, - sendCount: 1, - recoveryKinds: [], - usageStatus: "reported", - usage: { inputTokens: 10, outputTokens: 5 }, - totalTokens: 15, - }], + attempts: [ + { + ordinal: 1, + provider: "a", + model: "m1", + adapter: "openai-chat", + status: 200, + durationMs: 18, + firstOutputMs: 3, + sendCount: 1, + recoveryKinds: [], + usageStatus: "reported", + usage: { inputTokens: 10, outputTokens: 5 }, + totalTokens: 15, + }, + ], }); const [entry] = readUsageEntries(); expect(entry?.firstOutputMs).toBe(7); @@ -436,45 +552,51 @@ describe("usage log", () => { }); test("legacy lines without firstOutputMs stay readable and unset", () => { - writeFileSync(usageLogPath(), `${JSON.stringify({ - requestId: "legacy", - timestamp: 1, - provider: "a", - model: "m1", - status: 200, - durationMs: 5, - usageStatus: "reported", - usage: { inputTokens: 1, outputTokens: 1 }, - })}\n`); + writeFileSync( + usageLogPath(), + `${JSON.stringify({ + requestId: "legacy", + timestamp: 1, + provider: "a", + model: "m1", + status: 200, + durationMs: 5, + usageStatus: "reported", + usage: { inputTokens: 1, outputTokens: 1 }, + })}\n`, + ); const [entry] = readUsageEntries(); expect(entry?.requestId).toBe("legacy"); expect(entry).not.toHaveProperty("firstOutputMs"); }); test("ignores malformed attempt arrays and keeps legacy parents readable", () => { - writeFileSync(usageLogPath(), [ - JSON.stringify({ - requestId: "bad-attempt-array", - timestamp: 1, - provider: "combo", - model: "combo/free", - status: 200, - durationMs: 1, - usageStatus: "unreported", - attempts: { ordinal: 1 }, - }), - JSON.stringify({ - requestId: "legacy", - timestamp: 2, - provider: "openai", - model: "gpt-5.5", - status: 200, - durationMs: 1, - usageStatus: "reported", - usage: { inputTokens: 1, outputTokens: 2 }, - totalTokens: 3, - }), - ].join("\n")); + writeFileSync( + usageLogPath(), + [ + JSON.stringify({ + requestId: "bad-attempt-array", + timestamp: 1, + provider: "combo", + model: "combo/free", + status: 200, + durationMs: 1, + usageStatus: "unreported", + attempts: { ordinal: 1 }, + }), + JSON.stringify({ + requestId: "legacy", + timestamp: 2, + provider: "openai", + model: "gpt-5.5", + status: 200, + durationMs: 1, + usageStatus: "reported", + usage: { inputTokens: 1, outputTokens: 2 }, + totalTokens: 3, + }), + ].join("\n"), + ); const entries = readUsageEntries(); expect(entries).toHaveLength(2); expect(entries[0]).not.toHaveProperty("attempts"); @@ -514,24 +636,26 @@ describe("usage log", () => { expect(existsSync(usageLogPath())).toBe(true); const raw = readFileSync(usageLogPath(), "utf-8"); - expect(raw).toContain("\"requestId\":\"ocx-1\""); + expect(raw).toContain('"requestId":"ocx-1"'); expect(raw).not.toContain("prompt"); expect(raw).not.toContain("authorization"); - expect(readUsageEntries()).toEqual([{ - requestId: "ocx-1", - timestamp: 1, - provider: "openai", - model: "gpt-5.5", - surface: "claude", - requestedModel: "openai-apikey/gpt-5.5", - resolvedModel: "gpt-5.5", - stream: true, - status: 200, - durationMs: 42, - usageStatus: "reported", - usage: { inputTokens: 10, outputTokens: 3, cachedInputTokens: 2 }, - totalTokens: 13, - }]); + expect(readUsageEntries()).toEqual([ + { + requestId: "ocx-1", + timestamp: 1, + provider: "openai", + model: "gpt-5.5", + surface: "claude", + requestedModel: "openai-apikey/gpt-5.5", + resolvedModel: "gpt-5.5", + stream: true, + status: 200, + durationMs: 42, + usageStatus: "reported", + usage: { inputTokens: 10, outputTokens: 3, cachedInputTokens: 2 }, + totalTokens: 13, + }, + ]); if (process.platform !== "win32") { expect((statSync(usageLogPath()).mode & 0o777).toString(8)).toBe("600"); } @@ -576,51 +700,102 @@ describe("usage log", () => { ]) { expect(raw).not.toContain(leaked); } - expect(readUsageEntries()).toEqual([{ - requestId: "ocx-extra", - timestamp: 2, - provider: "openai", - model: "gpt-5.5", - surface: "codex", - status: 200, - durationMs: 12, - usageStatus: "reported", - usage: { inputTokens: 1, outputTokens: 2, estimated: true }, - totalTokens: 3, - }]); + expect(readUsageEntries()).toEqual([ + { + requestId: "ocx-extra", + timestamp: 2, + provider: "openai", + model: "gpt-5.5", + surface: "codex", + status: 200, + durationMs: 12, + usageStatus: "reported", + usage: { inputTokens: 1, outputTokens: 2, estimated: true }, + totalTokens: 3, + }, + ]); }); test("skips malformed JSONL lines while keeping valid entries", () => { - writeFileSync(usageLogPath(), [ - "{\"requestId\":\"a\",\"timestamp\":1,\"provider\":\"p\",\"model\":\"m\",\"status\":200,\"durationMs\":1,\"usageStatus\":\"unreported\"}", - "{not-json", - "{\"requestId\":\"b\",\"timestamp\":2,\"provider\":\"p\",\"model\":\"m\",\"status\":200,\"durationMs\":1,\"usageStatus\":\"reported\",\"usage\":{\"inputTokens\":1,\"outputTokens\":2},\"totalTokens\":3}", - ].join("\n")); + writeFileSync( + usageLogPath(), + [ + '{"requestId":"a","timestamp":1,"provider":"p","model":"m","status":200,"durationMs":1,"usageStatus":"unreported"}', + "{not-json", + '{"requestId":"b","timestamp":2,"provider":"p","model":"m","status":200,"durationMs":1,"usageStatus":"reported","usage":{"inputTokens":1,"outputTokens":2},"totalTokens":3}', + ].join("\n"), + ); - expect(readUsageEntries().map(entry => entry.requestId)).toEqual(["a", "b"]); + expect(readUsageEntries().map((entry) => entry.requestId)).toEqual([ + "a", + "b", + ]); }); test("keeps missing usage distinct from zero usage", () => { expect(usageStatusForFinalLog(undefined)).toBe("unreported"); - expect(usageStatusForFinalLog({ inputTokens: 0, outputTokens: 0 })).toBe("reported"); - expect(usageStatusForFinalLog({ inputTokens: 0, outputTokens: 0, estimated: true })).toBe("estimated"); + expect(usageStatusForFinalLog({ inputTokens: 0, outputTokens: 0 })).toBe( + "reported", + ); + expect( + usageStatusForFinalLog({ + inputTokens: 0, + outputTokens: 0, + estimated: true, + }), + ).toBe("estimated"); expect(usageTotalTokens(undefined)).toBeUndefined(); - expect(usageTotalTokens({ inputTokens: 4, outputTokens: 6, cachedInputTokens: 2 })).toBe(10); + expect( + usageTotalTokens({ + inputTokens: 4, + outputTokens: 6, + cachedInputTokens: 2, + }), + ).toBe(10); // inputTokens is inclusive of cache detail — the total never re-adds it - expect(usageTotalTokens({ inputTokens: 4, outputTokens: 6, cachedInputTokens: 2, cacheReadInputTokens: 1, cacheCreationInputTokens: 1 })).toBe(10); - expect(usageTotalTokens({ inputTokens: 4, outputTokens: 6, totalTokens: 50_000 })).toBe(50_000); + expect( + usageTotalTokens({ + inputTokens: 4, + outputTokens: 6, + cachedInputTokens: 2, + cacheReadInputTokens: 1, + cacheCreationInputTokens: 1, + }), + ).toBe(10); + expect( + usageTotalTokens({ + inputTokens: 4, + outputTokens: 6, + totalTokens: 50_000, + }), + ).toBe(50_000); }); test("marks Kiro final log usage as estimated without changing other providers", () => { const usage = { inputTokens: 4, outputTokens: 6 }; - expect(usageForFinalLog("kiro", usage)).toEqual({ ...usage, estimated: true }); - expect(usageForFinalLog("kiro-p9d8524", usage)).toEqual({ ...usage, estimated: true }); + expect(usageForFinalLog("kiro", usage)).toEqual({ + ...usage, + estimated: true, + }); + expect(usageForFinalLog("kiro-p9d8524", usage)).toEqual({ + ...usage, + estimated: true, + }); // cursor: adapter name AND configured-provider-name prefixes both count (devlog 130 B2 — // "cursor-pb51d9b" rows previously logged as accurately "reported"). - expect(usageForFinalLog("cursor", usage)).toEqual({ ...usage, estimated: true }); - expect(usageForFinalLog("cursor-pb51d9b", usage)).toEqual({ ...usage, estimated: true }); + expect(usageForFinalLog("cursor", usage)).toEqual({ + ...usage, + estimated: true, + }); + expect(usageForFinalLog("cursor-pb51d9b", usage)).toEqual({ + ...usage, + estimated: true, + }); expect(usageForFinalLog("openai", usage)).toEqual(usage); - expect(usageForFinalLog("openai", { ...usage, estimated: true })).toEqual({ ...usage, estimated: true }); + expect(usageForFinalLog("openai", { ...usage, estimated: true })).toEqual({ + ...usage, + estimated: true, + }); }); test("preserves cached token counts alongside estimated status", () => { @@ -709,7 +884,7 @@ describe("usage log", () => { usageStatus: "unreported", }); } - expect(readRecentUsageEntries(5).map(e => e.requestId)).toEqual([ + expect(readRecentUsageEntries(5).map((e) => e.requestId)).toEqual([ "ocx-tail-7", "ocx-tail-8", "ocx-tail-9",