From 85443b67b2afde5f89af0fb8256cb3e916346e0a Mon Sep 17 00:00:00 2001 From: chefgroep Date: Sat, 3 Oct 2026 23:37:04 +0200 Subject: [PATCH 1/5] feat: add prompt cache observability --- scripts/analyze-prompt-cache-usage.ts | 362 +++++++++++++++++++++ src/adapters/base.ts | 3 + src/adapters/openai-responses.ts | 13 +- src/prompt-cache/observability.ts | 188 +++++++++++ src/server/request-log.ts | 42 +++ src/server/responses/core.ts | 16 +- src/usage/log.ts | 13 + tests/openai-responses-passthrough.test.ts | 48 +++ tests/prompt-cache-observability.test.ts | 117 +++++++ tests/request-log.test.ts | 54 +++ tests/usage-log.test.ts | 54 +++ 11 files changed, 897 insertions(+), 13 deletions(-) create mode 100644 scripts/analyze-prompt-cache-usage.ts create mode 100644 src/prompt-cache/observability.ts create mode 100644 tests/prompt-cache-observability.test.ts diff --git a/scripts/analyze-prompt-cache-usage.ts b/scripts/analyze-prompt-cache-usage.ts new file mode 100644 index 000000000..ceafa8590 --- /dev/null +++ b/scripts/analyze-prompt-cache-usage.ts @@ -0,0 +1,362 @@ +#!/usr/bin/env bun +import { readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +type RangeName = "7d" | "30d" | "all"; + +interface CacheUsage { + input: number; + read: number; + write: number; +} + +interface Cohort { + adapter: string; + provider: string; + model: string; + surface: string; + requests: number; + reportedSuccess: number; + input: number; + cacheRead: number; + cacheWrite: number; + uncachedInput: number; + hitRequests: number; + writeRequests: number; + conversations: Map; +} + +interface ShapeSummary { + shape: string; + mode: string; + ttl: string | null; + keyPresent: boolean; + breakpointCount: number; + toolCount: number; + toolsFingerprint: string | null; + stablePrefixFingerprint: string | null; + textFormatFingerprint: string | null; + verbosity: string | null; + requests: number; + reportedSuccess: number; + inputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + cacheReadRatio: number; + cacheWriteRatio: number; +} + +function isRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +function nonNegativeNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? value + : undefined; +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function cacheUsage(row: Record): CacheUsage | undefined { + if (row.status !== 200 || row.usageStatus !== "reported" || !isRecord(row.usage)) { + return undefined; + } + const input = nonNegativeNumber(row.usage.inputTokens); + if (input === undefined) return undefined; + const write = nonNegativeNumber(row.usage.cacheCreationInputTokens) ?? 0; + const explicitRead = nonNegativeNumber(row.usage.cacheReadInputTokens); + const legacyCached = nonNegativeNumber(row.usage.cachedInputTokens); + const read = explicitRead + ?? (legacyCached !== undefined && row.usage.cacheCreationInputTokens !== undefined + ? Math.max(0, legacyCached - write) + : legacyCached) + ?? 0; + if (read + write > input) return undefined; + return { input, read, write }; +} + +function parseRange(value: string | undefined): RangeName { + return value === "7d" || value === "30d" || value === "all" ? value : "7d"; +} + +function sinceFor(range: RangeName, now: number): number { + if (range === "all") return 0; + return now - (range === "7d" ? 7 : 30) * 86_400_000; +} + +function keyParts(row: Record): [string, string, string, string] { + const adapter = stringValue(row.adapter) ?? "unknown"; + const provider = stringValue(row.provider) ?? "unknown"; + const model = stringValue(row.resolvedModel) ?? stringValue(row.model) ?? "unknown"; + const surface = stringValue(row.surface) ?? "unknown"; + return [adapter, provider, model, surface]; +} + +function emptyCohort(parts: [string, string, string, string]): Cohort { + return { + adapter: parts[0], + provider: parts[1], + model: parts[2], + surface: parts[3], + requests: 0, + reportedSuccess: 0, + input: 0, + cacheRead: 0, + cacheWrite: 0, + uncachedInput: 0, + hitRequests: 0, + writeRequests: 0, + conversations: new Map(), + }; +} + +function ratio(numerator: number, denominator: number): number { + return denominator > 0 ? numerator / denominator : 0; +} + +function signal(cohort: Cohort): string { + const readRatio = ratio(cohort.cacheRead, cohort.input); + const writeRatio = ratio(cohort.cacheWrite, cohort.input); + if (writeRatio >= 0.5 && readRatio < 0.1) return "write-heavy"; + if (cohort.input >= 1_000_000 && readRatio < 0.25) return "low-reuse"; + if (readRatio >= 0.75) return "healthy"; + return "mixed"; +} + +function publicCohort(cohort: Cohort): Record { + let multiTurnConversations = 0; + for (const count of cohort.conversations.values()) { + if (count > 1) multiTurnConversations += 1; + } + return { + adapter: cohort.adapter, + provider: cohort.provider, + model: cohort.model, + surface: cohort.surface, + signal: signal(cohort), + requests: cohort.requests, + reportedSuccess: cohort.reportedSuccess, + inputTokens: cohort.input, + cacheReadTokens: cohort.cacheRead, + cacheWriteTokens: cohort.cacheWrite, + uncachedInputTokens: cohort.uncachedInput, + cacheReadRatio: ratio(cohort.cacheRead, cohort.input), + cacheWriteRatio: ratio(cohort.cacheWrite, cohort.input), + cacheHitRequestRatio: ratio(cohort.hitRequests, cohort.reportedSuccess), + cacheWriteRequestRatio: ratio(cohort.writeRequests, cohort.reportedSuccess), + conversations: cohort.conversations.size, + multiTurnConversations, + }; +} + +function cacheShapeKey(row: Record): string | undefined { + if (!isRecord(row.promptCache)) return undefined; + const pc = row.promptCache; + const parts = [ + stringValue(pc.mode) ?? "default", + stringValue(pc.ttl) ?? "-", + pc.keyPresent === true ? "key" : "nokey", + String(nonNegativeNumber(pc.breakpointCount) ?? 0), + stringValue(pc.toolsFingerprint) ?? "-", + stringValue(pc.stablePrefixFingerprint) ?? "-", + stringValue(pc.textFormatFingerprint) ?? "-", + stringValue(pc.verbosity) ?? "-", + ]; + return parts.join("|"); +} + +function shapeSummary(key: string, rows: Record[]): ShapeSummary { + const first = rows[0]?.promptCache; + const pc = isRecord(first) ? first : {}; + let input = 0; + let read = 0; + let write = 0; + let reportedSuccess = 0; + for (const row of rows) { + const usage = cacheUsage(row); + if (!usage) continue; + reportedSuccess += 1; + input += usage.input; + read += usage.read; + write += usage.write; + } + return { + shape: key, + mode: stringValue(pc.mode) ?? "default", + ttl: stringValue(pc.ttl) ?? null, + keyPresent: pc.keyPresent === true, + breakpointCount: nonNegativeNumber(pc.breakpointCount) ?? 0, + toolCount: nonNegativeNumber(pc.toolCount) ?? 0, + toolsFingerprint: stringValue(pc.toolsFingerprint) ?? null, + stablePrefixFingerprint: stringValue(pc.stablePrefixFingerprint) ?? null, + textFormatFingerprint: stringValue(pc.textFormatFingerprint) ?? null, + verbosity: stringValue(pc.verbosity) ?? null, + requests: rows.length, + reportedSuccess, + inputTokens: input, + cacheReadTokens: read, + cacheWriteTokens: write, + cacheReadRatio: ratio(read, input), + cacheWriteRatio: ratio(write, input), + }; +} + +function formatPercent(value: unknown): string { + return typeof value === "number" ? (value * 100).toFixed(1) + "%" : "-"; +} + +const args = Bun.argv.slice(2); +const source = args.find(arg => !arg.startsWith("--")) + ?? join(process.env.OPENCODEX_HOME?.trim() || join(homedir(), ".opencodex"), "usage.jsonl"); +const rangeArg = args.find(arg => arg.startsWith("--range="))?.slice("--range=".length); +const topArg = args.find(arg => arg.startsWith("--top="))?.slice("--top=".length); +const range = parseRange(rangeArg); +const top = Math.max(1, Math.min(200, Number.parseInt(topArg ?? "20", 10) || 20)); +const jsonMode = args.includes("--json"); +const now = Date.now(); +const since = sinceFor(range, now); +const text = source === "-" ? readFileSync(0, "utf8") : readFileSync(source, "utf8"); + +const rows: Record[] = []; +let invalidLines = 0; +for (const line of text.split(/\r?\n/u)) { + if (!line.trim()) continue; + try { + const parsed: unknown = JSON.parse(line); + if (isRecord(parsed)) rows.push(parsed); + else invalidLines += 1; + } catch { + invalidLines += 1; + } +} + +const selected = rows.filter(row => { + const timestamp = nonNegativeNumber(row.timestamp); + return timestamp !== undefined && timestamp >= since; +}); +const cohorts = new Map(); +const shapeRows = new Map[]>(); +let reportedSuccess = 0; +let input = 0; +let read = 0; +let write = 0; + +for (const row of selected) { + const parts = keyParts(row); + const key = parts.join("\0"); + const cohort = cohorts.get(key) ?? emptyCohort(parts); + cohorts.set(key, cohort); + cohort.requests += 1; + const conversationId = stringValue(row.conversationId); + if (conversationId) { + cohort.conversations.set( + conversationId, + (cohort.conversations.get(conversationId) ?? 0) + 1, + ); + } + + const usage = cacheUsage(row); + if (usage) { + cohort.reportedSuccess += 1; + cohort.input += usage.input; + cohort.cacheRead += usage.read; + cohort.cacheWrite += usage.write; + cohort.uncachedInput += Math.max(0, usage.input - usage.read - usage.write); + if (usage.read > 0) cohort.hitRequests += 1; + if (usage.write > 0) cohort.writeRequests += 1; + reportedSuccess += 1; + input += usage.input; + read += usage.read; + write += usage.write; + } + + const shape = cacheShapeKey(row); + if (shape) { + const compositeShape = [...parts, shape].join("\0"); + const existing = shapeRows.get(compositeShape); + if (existing) existing.push(row); + else shapeRows.set(compositeShape, [row]); + } +} + +const cohortOutput = [...cohorts.values()] + .filter(cohort => cohort.reportedSuccess > 0) + .sort((a, b) => b.input - a.input) + .slice(0, top) + .map(publicCohort); +const shapeOutput = [...shapeRows.entries()] + .map(([key, value]) => shapeSummary(key.split("\0").slice(4).join("|"), value)) + .filter(shape => shape.reportedSuccess > 0) + .sort((a, b) => b.inputTokens - a.inputTokens) + .slice(0, top); + +const output = { + source: source === "-" ? "stdin" : source, + range, + rows: selected.length, + invalidLines, + proofBoundary: "status=200 AND usageStatus=reported", + summary: { + reportedSuccess, + inputTokens: input, + cacheReadTokens: read, + cacheWriteTokens: write, + uncachedInputTokens: Math.max(0, input - read - write), + cacheReadRatio: ratio(read, input), + cacheWriteRatio: ratio(write, input), + }, + cohorts: cohortOutput, + cacheShapes: shapeOutput, +}; + +if (jsonMode) { + process.stdout.write(JSON.stringify(output, null, 2) + "\n"); + process.exit(0); +} + +console.log("Prompt cache usage (" + range + ")"); +console.log("proof: " + output.proofBoundary); +console.log( + "reported-success=" + reportedSuccess + + " input=" + Math.round(input) + + " read=" + Math.round(read) + " (" + formatPercent(output.summary.cacheReadRatio) + ")" + + " write=" + Math.round(write) + " (" + formatPercent(output.summary.cacheWriteRatio) + ")", +); +console.log(""); +console.log("Top cohorts by measured input:"); +for (const item of cohortOutput) { + console.log( + String(item.signal).padEnd(11) + + " " + String(item.adapter) + "/" + String(item.provider) + "/" + String(item.model) + + " surface=" + String(item.surface) + + " input=" + String(item.inputTokens) + + " read=" + formatPercent(item.cacheReadRatio) + + " write=" + formatPercent(item.cacheWriteRatio) + + " hits=" + formatPercent(item.cacheHitRequestRatio) + + " n=" + String(item.reportedSuccess) + "/" + String(item.requests), + ); +} +if (shapeOutput.length > 0) { + console.log(""); + console.log("Observed outbound cache shapes:"); + for (const item of shapeOutput) { + console.log( + item.mode.padEnd(8) + + " tools=" + item.toolCount + + " bp=" + item.breakpointCount + + " input=" + item.inputTokens + + " read=" + formatPercent(item.cacheReadRatio) + + " write=" + formatPercent(item.cacheWriteRatio) + + " n=" + item.reportedSuccess + "/" + item.requests + + " prefix=" + (item.stablePrefixFingerprint ?? "-") + + " toolsHash=" + (item.toolsFingerprint ?? "-"), + ); + } +} else { + console.log(""); + console.log("No promptCache observations in this window (historical rows predate instrumentation)."); +} diff --git a/src/adapters/base.ts b/src/adapters/base.ts index 36fde6942..8cb49968e 100644 --- a/src/adapters/base.ts +++ b/src/adapters/base.ts @@ -1,5 +1,6 @@ import type { AdapterEvent, OcxParsedRequest } from "../types"; import type { UpstreamAttemptBudget } from "../lib/upstream-attempt-budget"; +import type { PromptCacheRequestObservation } from "../prompt-cache/observability"; /** Metadata about the caller's incoming request, for auth-forwarding adapters. */ export interface IncomingMeta { @@ -105,6 +106,8 @@ export interface AdapterRequest { inputTokens?: number; estimated?: boolean; }; + /** Structural prompt-cache diagnostics derived from the exact outbound body. */ + promptCacheLog?: PromptCacheRequestObservation; } export interface AdapterFetchContext { diff --git a/src/adapters/openai-responses.ts b/src/adapters/openai-responses.ts index 06506651e..b044bf415 100644 --- a/src/adapters/openai-responses.ts +++ b/src/adapters/openai-responses.ts @@ -9,6 +9,7 @@ import { decodeServerSentEvents } from "../lib/sse-decoder"; import { supportsNativeRemoteCompactionV2 } from "../providers/openai-tiers"; import { OCX_REASONING_PREFIX } from "../responses/reasoning-envelope"; import { modelRecordValue } from "../reasoning-effort"; +import { observeOpenAiResponsesPromptCache } from "../prompt-cache/observability"; // Headers relayed verbatim from the caller in OAuth-passthrough ("forward") mode. // Exported so the web-search sidecar reuses the exact same forwarded-auth set for its ChatGPT call. @@ -954,15 +955,17 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig): outBody = buildRoutedCompactionBody(outBody); } const sanitizedBody = stripSparkCompatibility(stripUnsupportedReasoningParams(stripItemIdsWhenUnstored(stripInvalidItemIds(stripUnsupportedHostedTools(sanitizeReasoningInputContent(scrubOcxCompactionItems(outBody))))))); + const finalBody = stripDisabledReasoningSummaries( + normalizeConfiguredReasoningSummaryDelivery(sanitizedBody, provider, parsed.modelId), + provider, + parsed.modelId, + ); return { url, method: "POST", headers, - body: JSON.stringify(stripDisabledReasoningSummaries( - normalizeConfiguredReasoningSummaryDelivery(sanitizedBody, provider, parsed.modelId), - provider, - parsed.modelId, - )), + body: JSON.stringify(finalBody), + promptCacheLog: observeOpenAiResponsesPromptCache(finalBody), }; }, diff --git a/src/prompt-cache/observability.ts b/src/prompt-cache/observability.ts new file mode 100644 index 000000000..7c655f4a4 --- /dev/null +++ b/src/prompt-cache/observability.ts @@ -0,0 +1,188 @@ +import { createHash } from "node:crypto"; + +export type PromptCacheMode = "default" | "implicit" | "explicit"; +export type PromptCacheLegacyRetention = "in_memory" | "24h"; + +export interface PromptCacheRequestObservation { + version: 1; + keyPresent: boolean; + mode: PromptCacheMode; + ttl?: "30m"; + legacyRetention?: PromptCacheLegacyRetention; + prewarm: boolean; + comparisonRequested: boolean; + previousResponseIdPresent: boolean; + breakpointCount: number; + inputItemCount: number; + toolCount: number; + toolsFingerprint?: string; + stablePrefixFingerprint?: string; + textFormatFingerprint?: string; + verbosity?: string; +} + +function isRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +function canonicalValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalValue); + if (!isRecord(value)) return value; + const out: Record = {}; + for (const key of Object.keys(value).sort()) { + out[key] = canonicalValue(value[key]); + } + return out; +} + +function fingerprint(value: unknown): string | undefined { + const serialized = JSON.stringify(canonicalValue(value)); + if (typeof serialized !== "string") return undefined; + return createHash("sha256").update(serialized).digest("hex").slice(0, 24); +} + +function countExplicitBreakpoints(value: unknown): number { + if (Array.isArray(value)) { + return value.reduce((total, item) => total + countExplicitBreakpoints(item), 0); + } + if (!isRecord(value)) return 0; + const breakpoint = value.prompt_cache_breakpoint; + let count = isRecord(breakpoint) && breakpoint.mode === "explicit" ? 1 : 0; + for (const [key, child] of Object.entries(value)) { + if (key === "prompt_cache_breakpoint") continue; + count += countExplicitBreakpoints(child); + } + return count; +} + +function stablePrefix(body: Record): unknown[] { + const prefix: unknown[] = []; + if (body.instructions !== undefined) { + prefix.push({ kind: "instructions", value: body.instructions }); + } + if (!Array.isArray(body.input)) return prefix; + + const initialDeveloperItems: unknown[] = []; + for (const item of body.input) { + if (!isRecord(item)) break; + if (item.role !== "developer" && item.role !== "system") break; + initialDeveloperItems.push(item); + } + if (initialDeveloperItems.length > 0) { + prefix.push({ kind: "initial_developer_messages", value: initialDeveloperItems }); + } + return prefix; +} + +function boundedString(value: unknown, maxLength = 32): string | undefined { + if (typeof value !== "string") return undefined; + const trimmed = value.trim(); + if (!trimmed) return undefined; + return trimmed.slice(0, maxLength); +} + +export function observeOpenAiResponsesPromptCache( + value: unknown, +): PromptCacheRequestObservation | undefined { + if (!isRecord(value)) return undefined; + + const options = isRecord(value.prompt_cache_options) + ? value.prompt_cache_options + : undefined; + const mode: PromptCacheMode = options?.mode === "explicit" + ? "explicit" + : options?.mode === "implicit" + ? "implicit" + : "default"; + const ttl = options?.ttl === "30m" ? "30m" as const : undefined; + const retention = value.prompt_cache_retention === "in_memory" + || value.prompt_cache_retention === "24h" + ? value.prompt_cache_retention + : undefined; + const tools = Array.isArray(value.tools) ? value.tools : []; + const input = value.input; + const prefix = stablePrefix(value); + const text = isRecord(value.text) ? value.text : undefined; + const textFormat = text?.format; + const verbosity = boundedString(text?.verbosity); + + return { + version: 1, + keyPresent: typeof value.prompt_cache_key === "string" + && value.prompt_cache_key.trim().length > 0, + mode, + ...(ttl ? { ttl } : {}), + ...(retention ? { legacyRetention: retention } : {}), + prewarm: options?.prewarm === true, + comparisonRequested: typeof options?.comparison_response_id === "string" + && options.comparison_response_id.trim().length > 0, + previousResponseIdPresent: typeof value.previous_response_id === "string" + && value.previous_response_id.trim().length > 0, + breakpointCount: countExplicitBreakpoints(input), + inputItemCount: Array.isArray(input) ? input.length : input === undefined ? 0 : 1, + toolCount: tools.length, + ...(tools.length > 0 ? { toolsFingerprint: fingerprint(tools) } : {}), + ...(prefix.length > 0 ? { stablePrefixFingerprint: fingerprint(prefix) } : {}), + ...(textFormat !== undefined ? { textFormatFingerprint: fingerprint(textFormat) } : {}), + ...(verbosity ? { verbosity } : {}), + }; +} + +function isFingerprint(value: unknown): value is string { + return typeof value === "string" && /^[0-9a-f]{24}$/u.test(value); +} + +function isBoundedCount(value: unknown): value is number { + return typeof value === "number" + && Number.isInteger(value) + && value >= 0 + && value <= 1_000_000; +} + +export function normalizePromptCacheRequestObservation( + value: unknown, +): PromptCacheRequestObservation | undefined { + if (!isRecord(value) || value.version !== 1) return undefined; + if (typeof value.keyPresent !== "boolean" + || typeof value.prewarm !== "boolean" + || typeof value.comparisonRequested !== "boolean" + || typeof value.previousResponseIdPresent !== "boolean" + || !isBoundedCount(value.breakpointCount) + || !isBoundedCount(value.inputItemCount) + || !isBoundedCount(value.toolCount)) { + return undefined; + } + const mode = value.mode === "default" + || value.mode === "implicit" + || value.mode === "explicit" + ? value.mode + : undefined; + if (!mode) return undefined; + + const ttl = value.ttl === "30m" ? value.ttl : undefined; + const legacyRetention = value.legacyRetention === "in_memory" + || value.legacyRetention === "24h" + ? value.legacyRetention + : undefined; + const verbosity = boundedString(value.verbosity); + return { + version: 1, + keyPresent: value.keyPresent, + mode, + ...(ttl ? { ttl } : {}), + ...(legacyRetention ? { legacyRetention } : {}), + prewarm: value.prewarm, + comparisonRequested: value.comparisonRequested, + previousResponseIdPresent: value.previousResponseIdPresent, + breakpointCount: value.breakpointCount, + inputItemCount: value.inputItemCount, + toolCount: value.toolCount, + ...(isFingerprint(value.toolsFingerprint) + ? { toolsFingerprint: value.toolsFingerprint } : {}), + ...(isFingerprint(value.stablePrefixFingerprint) + ? { stablePrefixFingerprint: value.stablePrefixFingerprint } : {}), + ...(isFingerprint(value.textFormatFingerprint) + ? { textFormatFingerprint: value.textFormatFingerprint } : {}), + ...(verbosity ? { verbosity } : {}), + }; +} diff --git a/src/server/request-log.ts b/src/server/request-log.ts index 7693769ba..ef50790bc 100644 --- a/src/server/request-log.ts +++ b/src/server/request-log.ts @@ -39,6 +39,10 @@ import { } from "../usage/debug"; import { matchesLogConversationId } from "./request-log-conversation"; import { captureRequestTelemetry } from "../telemetry/posthog-server"; +import { + normalizePromptCacheRequestObservation, + type PromptCacheRequestObservation, +} from "../prompt-cache/observability"; export interface RequestLogContext { model: string; @@ -83,6 +87,8 @@ export interface RequestLogContext { /** Route adapter type ("cursor"/"kiro"/"anthropic"/…): drives estimated-usage detection * independent of the user-chosen provider NAME (devlog 130 B2). */ providerAdapter?: string; + /** Structural prompt-cache metadata captured from the exact outbound adapter body. */ + promptCache?: PromptCacheRequestObservation; /** * Stable store id for the OAuth account or key-pool entry that served this request. * Distinct from the display `account` label persisted on the usage row. @@ -113,6 +119,10 @@ export interface RequestLogEntry { timestamp: number; model: string; provider: string; + /** Adapter that produced the final upstream wire request. */ + adapter?: string; + /** Structural prompt-cache metadata captured from the exact outbound adapter body. */ + promptCache?: PromptCacheRequestObservation; /** Whether the client requested a streamed generation. */ stream?: boolean; /** TTFT: ms from request start to the first non-empty model output delta; unset for non-streaming/tool-only. */ @@ -277,6 +287,8 @@ export function requestLogEntryFromPersistedUsage(entry: PersistedUsageEntry): R timestamp: entry.timestamp, model: entry.model, provider: entry.provider, + ...(entry.adapter ? { adapter: entry.adapter } : {}), + ...(entry.promptCache ? { promptCache: { ...entry.promptCache } } : {}), ...(entry.firstOutputMs !== undefined ? { firstOutputMs: entry.firstOutputMs } : {}), ...(isKnownUsageSurface(entry.surface) ? { surface: entry.surface } : {}), ...(entry.conversationId ? { conversationId: entry.conversationId } : {}), @@ -394,6 +406,8 @@ export function addRequestLog(entry: RequestLogEntry) { timestamp: entry.timestamp, provider: entry.provider, model: entry.model, + ...(entry.adapter ? { adapter: entry.adapter } : {}), + ...(entry.promptCache ? { promptCache: { ...entry.promptCache } } : {}), ...(isKnownUsageSurface(entry.surface) ? { surface: entry.surface } : {}), ...(entry.conversationId ? { conversationId: entry.conversationId } : {}), ...(account ? { account } : {}), @@ -517,6 +531,29 @@ export function recordAdapterReasoning( } } +/** Copy prompt-cache diagnostics from the exact outbound adapter request. */ +export function recordAdapterPromptCache( + logCtx: RequestLogContext, + request: AdapterRequest, +): void { + delete logCtx.promptCache; + try { + const observation = normalizePromptCacheRequestObservation(request.promptCacheLog); + if (observation) logCtx.promptCache = observation; + } catch { + // Request logging is best-effort and must not affect request delivery. + } +} + +/** Record all adapter-derived request diagnostics at one lifecycle seam. */ +export function recordAdapterRequestMetadata( + logCtx: RequestLogContext, + request: AdapterRequest, +): void { + recordAdapterReasoning(logCtx, request); + recordAdapterPromptCache(logCtx, request); +} + export function requestLogErrorCode(status: number, upstreamError?: string): string | undefined { if (status >= 200 && status < 400) return undefined; // Defense in depth: mid-stream web-search aborts used to land as 502 with this message. @@ -874,6 +911,9 @@ export function addFinalRequestLog( recoveryKinds: [...attempt.recoveryKinds], ...(attempt.usage ? { usage: { ...attempt.usage } } : {}), })); + const adapter = isCombo + ? attempts?.at(-1)?.adapter ?? logCtx.providerAdapter + : logCtx.providerAdapter; const aggregate = isCombo ? aggregateAttemptUsage(attempts ?? []) : null; const loggedUsage = aggregate?.usage ?? existing.usage; const usageStatus = aggregate?.status ?? existing.status; @@ -883,6 +923,8 @@ export function addFinalRequestLog( timestamp: start, model, provider, + ...(adapter ? { adapter } : {}), + ...(logCtx.promptCache ? { promptCache: { ...logCtx.promptCache } } : {}), ...(account ? { account } : {}), ...(providerAccountId ? { providerAccountId } : {}), ...(logCtx.surface ? { surface: logCtx.surface } : {}), diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 0321ecbc7..9f4a9f699 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -147,7 +147,7 @@ import { inspectResponseLogJson, noteAttemptSend, readConfiguredCodexServiceTier, - recordAdapterReasoning, + recordAdapterRequestMetadata, recordAttemptRequestedEffort, requestLogSpeedLabel, sealRequestAttemptIdentity, @@ -398,7 +398,7 @@ async function retryCodexPoolOnAlternateAccount( } catch (err) { return { kind: "build-request-failed", response: resolveAdapterBuildRequestError(err, options.abortSignal) }; } - recordAdapterReasoning(logCtx, request); + recordAdapterRequestMetadata(logCtx, request); await firstResponse.body?.cancel().catch(() => undefined); options.onCodexAuthContextResolved?.(retryAuthCtx); @@ -1687,7 +1687,7 @@ export async function handleResponses( } catch (err) { return resolveAdapterBuildRequestError(err, options.abortSignal); } - recordAdapterReasoning(logCtx, request); + recordAdapterRequestMetadata(logCtx, request); const passthroughEstimate = typeof request.usageLog?.inputTokens === "number" ? request.usageLog.inputTokens : undefined; @@ -2125,7 +2125,7 @@ export async function handleResponses( connectTimeoutMs: config.connectTimeoutMs ?? 200_000, stallTimeoutSec: config.stallTimeoutSec, fetchImpl: providerFetch(route.provider), - onRequestBuilt: request => recordAdapterReasoning(logCtx, request), + onRequestBuilt: request => recordAdapterRequestMetadata(logCtx, request), ...(vidPlan?.timeoutMs ? { videoTimeoutMs: vidPlan.timeoutMs } : {}), onUsage: usage => { // Cursor may assign _cursorConversationId inside the image loop's first runTurn; @@ -2208,7 +2208,7 @@ export async function handleResponses( forceEmptyResponseId: true, abortSignal: options.abortSignal, ...(options.onFirstOutput ? { onFirstOutput: options.onFirstOutput } : {}), - onRequestBuilt: request => recordAdapterReasoning(logCtx, request), + onRequestBuilt: request => recordAdapterRequestMetadata(logCtx, request), onUsage: usage => { logCtx.usageFromBridge = true; if (usage) { @@ -2487,7 +2487,7 @@ export async function handleResponses( cleanupUpstreamAbort(); return resolveAdapterBuildRequestError(err, options.abortSignal); } - recordAdapterReasoning(logCtx, request); + recordAdapterRequestMetadata(logCtx, request); const inputTokenEstimate = typeof request.usageLog?.inputTokens === "number" ? request.usageLog.inputTokens : undefined; @@ -2563,7 +2563,7 @@ export async function handleResponses( cleanupUpstreamAbort(); return { failed: resolveAdapterBuildRequestError(err, options.abortSignal) }; } - recordAdapterReasoning(logCtx, retryRequest); + recordAdapterRequestMetadata(logCtx, retryRequest); const retryEstimate = typeof retryRequest.usageLog?.inputTokens === "number" ? retryRequest.usageLog.inputTokens : undefined; @@ -2907,7 +2907,7 @@ export async function handleResponses( }; return; } - recordAdapterReasoning(logCtx, continuationRequest); + recordAdapterRequestMetadata(logCtx, continuationRequest); const continuationEstimate = typeof continuationRequest.usageLog?.inputTokens === "number" ? continuationRequest.usageLog.inputTokens : undefined; diff --git a/src/usage/log.ts b/src/usage/log.ts index f37166115..57e720be1 100644 --- a/src/usage/log.ts +++ b/src/usage/log.ts @@ -4,6 +4,10 @@ import { getConfigDir } from "../config"; import { recordOwnedConfigPath } from "../lib/config-ownership"; import { usageDisplayTotalTokens } from "./totals"; import type { OcxUsage } from "../types"; +import { + normalizePromptCacheRequestObservation, + type PromptCacheRequestObservation, +} from "../prompt-cache/observability"; export type UsageStatus = "reported" | "unreported" | "unsupported" | "estimated"; @@ -50,6 +54,10 @@ export interface PersistedUsageEntry { timestamp: number; provider: string; model: string; + /** Adapter that produced the final upstream wire request. */ + adapter?: string; + /** Structural prompt-cache metadata from the final outbound request. */ + promptCache?: PromptCacheRequestObservation; surface?: "claude" | "claude-desktop" | "codex" | "grok"; /** Best-effort chat/session correlation for Logs grouping (#330). */ conversationId?: string; @@ -297,11 +305,16 @@ function capMetadataString(s: string): string { */ function normalizeUsageEntry(entry: PersistedUsageEntry): PersistedUsageEntry { const attempts = normalizedAttempts(entry.attempts); + const promptCache = normalizePromptCacheRequestObservation(entry.promptCache); return { requestId: entry.requestId, timestamp: entry.timestamp, provider: entry.provider, model: entry.model, + ...(typeof entry.adapter === "string" && entry.adapter.trim() + ? { adapter: capMetadataString(entry.adapter.trim()) } + : {}), + ...(promptCache ? { promptCache } : {}), ...(isKnownUsageSurface(entry.surface) ? { surface: entry.surface } : {}), ...(typeof entry.conversationId === "string" && entry.conversationId.trim() ? { conversationId: entry.conversationId.trim().slice(0, 128) } diff --git a/tests/openai-responses-passthrough.test.ts b/tests/openai-responses-passthrough.test.ts index fe7128bf5..4f9742449 100644 --- a/tests/openai-responses-passthrough.test.ts +++ b/tests/openai-responses-passthrough.test.ts @@ -416,6 +416,54 @@ describe("OpenAI Responses passthrough sanitization", () => { expect(body.prompt_cache_retention).toBe("24h"); }); + test("records structural prompt-cache metadata from the final outbound body", () => { + const adapter = createResponsesPassthroughAdapter(provider); + const request = adapter.buildRequest({ + modelId: "gpt-5.6-terra", + context: { messages: [] }, + stream: true, + options: { promptCacheKey: "private-cache-key" }, + _rawBody: { + model: "gpt-5.6-terra", + instructions: "private fixed instruction", + prompt_cache_key: "private-cache-key", + prompt_cache_options: { mode: "implicit", ttl: "30m" }, + text: { verbosity: "medium", format: { type: "text" } }, + tools: [{ type: "function", name: "shell", parameters: { type: "object" } }], + input: [{ + role: "developer", + content: [{ + type: "input_text", + text: "private developer text", + prompt_cache_breakpoint: { mode: "explicit" }, + }], + }, { + role: "user", + content: [{ type: "input_text", text: "private user text" }], + }], + }, + }, { headers: new Headers({ authorization: "Bearer token" }) }); + + expect(request.promptCacheLog).toMatchObject({ + version: 1, + keyPresent: true, + mode: "implicit", + ttl: "30m", + breakpointCount: 1, + inputItemCount: 2, + toolCount: 1, + verbosity: "medium", + }); + expect(request.promptCacheLog?.toolsFingerprint).toMatch(/^[0-9a-f]{24}$/); + expect(request.promptCacheLog?.stablePrefixFingerprint).toMatch(/^[0-9a-f]{24}$/); + expect(JSON.stringify(request.promptCacheLog)).not.toContain("private-cache-key"); + expect(JSON.stringify(request.promptCacheLog)).not.toContain("private developer text"); + expect(JSON.parse(request.body)).toMatchObject({ + prompt_cache_key: "private-cache-key", + prompt_cache_options: { mode: "implicit", ttl: "30m" }, + }); + }); + const expandedRawBody = { model: "gpt-5.5", previous_response_id: "resp_1", diff --git a/tests/prompt-cache-observability.test.ts b/tests/prompt-cache-observability.test.ts new file mode 100644 index 000000000..821d2e1ad --- /dev/null +++ b/tests/prompt-cache-observability.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, test } from "bun:test"; +import { + normalizePromptCacheRequestObservation, + observeOpenAiResponsesPromptCache, +} from "../src/prompt-cache/observability"; + +describe("prompt cache observability", () => { + test("captures structural cache dimensions without retaining request content", () => { + const observation = observeOpenAiResponsesPromptCache({ + model: "gpt-5.6-terra", + instructions: "private instruction marker", + prompt_cache_key: "private-cache-key", + prompt_cache_options: { + mode: "implicit", + ttl: "30m", + prewarm: true, + comparison_response_id: "resp_private", + }, + previous_response_id: "resp_previous", + text: { + verbosity: "medium", + format: { type: "json_schema", name: "private-format" }, + }, + tools: [{ + type: "function", + name: "lookup_private_tool", + parameters: { type: "object", properties: { secret: { type: "string" } } }, + }], + input: [{ + role: "developer", + content: [{ + type: "input_text", + text: "private developer prefix", + prompt_cache_breakpoint: { mode: "explicit" }, + }], + }, { + role: "user", + content: [{ type: "input_text", text: "private user content" }], + }], + }); + + expect(observation).toMatchObject({ + version: 1, + keyPresent: true, + mode: "implicit", + ttl: "30m", + prewarm: true, + comparisonRequested: true, + previousResponseIdPresent: true, + breakpointCount: 1, + inputItemCount: 2, + toolCount: 1, + verbosity: "medium", + }); + expect(observation?.toolsFingerprint).toMatch(/^[0-9a-f]{24}$/); + expect(observation?.stablePrefixFingerprint).toMatch(/^[0-9a-f]{24}$/); + expect(observation?.textFormatFingerprint).toMatch(/^[0-9a-f]{24}$/); + const serialized = JSON.stringify(observation); + expect(serialized).not.toContain("private-cache-key"); + expect(serialized).not.toContain("private instruction marker"); + expect(serialized).not.toContain("private developer prefix"); + expect(serialized).not.toContain("private user content"); + expect(serialized).not.toContain("lookup_private_tool"); + expect(serialized).not.toContain("private-format"); + }); + + test("fingerprints are stable for object-key order but preserve tool array order", () => { + const a = observeOpenAiResponsesPromptCache({ + instructions: "stable", + tools: [ + { type: "function", name: "a", parameters: { type: "object", properties: { x: { type: "string" } } } }, + { type: "function", name: "b", parameters: { type: "object" } }, + ], + }); + const same = observeOpenAiResponsesPromptCache({ + tools: [ + { name: "a", parameters: { properties: { x: { type: "string" } }, type: "object" }, type: "function" }, + { parameters: { type: "object" }, name: "b", type: "function" }, + ], + instructions: "stable", + }); + const reordered = observeOpenAiResponsesPromptCache({ + instructions: "stable", + tools: [ + { type: "function", name: "b", parameters: { type: "object" } }, + { type: "function", name: "a", parameters: { type: "object", properties: { x: { type: "string" } } } }, + ], + }); + + expect(a?.toolsFingerprint).toBe(same?.toolsFingerprint); + expect(a?.stablePrefixFingerprint).toBe(same?.stablePrefixFingerprint); + expect(a?.toolsFingerprint).not.toBe(reordered?.toolsFingerprint); + }); + + test("normalizer accepts generated observations and rejects malformed persisted data", () => { + const observation = observeOpenAiResponsesPromptCache({ + prompt_cache_options: { mode: "explicit", ttl: "30m" }, + input: [{ content: [{ prompt_cache_breakpoint: { mode: "explicit" } }] }], + }); + expect(normalizePromptCacheRequestObservation(observation)).toEqual(observation); + expect(normalizePromptCacheRequestObservation({ + ...observation, + breakpointCount: -1, + })).toBeUndefined(); + expect(normalizePromptCacheRequestObservation({ + version: 1, + keyPresent: true, + mode: "invalid", + prewarm: false, + comparisonRequested: false, + previousResponseIdPresent: false, + breakpointCount: 0, + inputItemCount: 0, + toolCount: 0, + })).toBeUndefined(); + }); +}); diff --git a/tests/request-log.test.ts b/tests/request-log.test.ts index 90fcb5179..bb99f1802 100644 --- a/tests/request-log.test.ts +++ b/tests/request-log.test.ts @@ -21,6 +21,7 @@ import { hydrateRequestLogsFromDisk, noteAttemptSend, recordAdapterReasoning, + recordAdapterRequestMetadata, recordFirstOutput, requestLogEntryFromPersistedUsage, sealRequestAttemptIdentity, @@ -163,6 +164,59 @@ describe("request log metadata", () => { expect(attempt.reasoningWireValue).not.toContain("redaction-fixture"); }); + test("records and replaces outbound prompt-cache metadata at the adapter seam", () => { + const logCtx: RequestLogContext = { + model: "gpt-5.6-terra", + provider: "openai", + promptCache: { + version: 1, + keyPresent: false, + mode: "default", + prewarm: false, + comparisonRequested: false, + previousResponseIdPresent: false, + breakpointCount: 0, + inputItemCount: 0, + toolCount: 0, + }, + }; + + recordAdapterRequestMetadata(logCtx, { + url: "https://api.openai.com/v1/responses", + method: "POST", + headers: {}, + body: "{}", + promptCacheLog: { + version: 1, + keyPresent: true, + mode: "implicit", + ttl: "30m", + prewarm: false, + comparisonRequested: false, + previousResponseIdPresent: true, + breakpointCount: 1, + inputItemCount: 3, + toolCount: 2, + toolsFingerprint: "0123456789abcdef01234567", + }, + }); + expect(logCtx.promptCache).toMatchObject({ + keyPresent: true, + mode: "implicit", + previousResponseIdPresent: true, + breakpointCount: 1, + toolCount: 2, + }); + + recordAdapterRequestMetadata(logCtx, { + url: "https://strict-provider.test/v1/responses", + method: "POST", + headers: {}, + body: "{}", + }); + expect(logCtx.promptCache).toBeUndefined(); + }); + test("malformed adapter reasoning metadata never interrupts request logging", () => { const malformed = [ { effectiveEffort: 123, wireField: "reasoning_effort", wireValue: 123 }, diff --git a/tests/usage-log.test.ts b/tests/usage-log.test.ts index 487ad2d02..a3929b7a4 100644 --- a/tests/usage-log.test.ts +++ b/tests/usage-log.test.ts @@ -102,6 +102,60 @@ describe("usage log", () => { })]); }); + test("persists adapter and bounded prompt-cache observations", () => { + appendUsageEntry({ + requestId: "ocx-cache-observation", + timestamp: 1, + provider: "openai", + model: "gpt-5.6-terra", + adapter: "openai-responses", + promptCache: { + version: 1, + keyPresent: true, + mode: "implicit", + ttl: "30m", + prewarm: false, + comparisonRequested: false, + previousResponseIdPresent: false, + breakpointCount: 0, + inputItemCount: 2, + toolCount: 1, + toolsFingerprint: "0123456789abcdef01234567", + stablePrefixFingerprint: "89abcdef0123456789abcdef", + }, + status: 200, + durationMs: 1, + usageStatus: "reported", + usage: { inputTokens: 100, outputTokens: 1, cacheReadInputTokens: 80 }, + totalTokens: 101, + }); + expect(readUsageEntries()).toEqual([expect.objectContaining({ + adapter: "openai-responses", + promptCache: expect.objectContaining({ + mode: "implicit", + ttl: "30m", + toolCount: 1, + toolsFingerprint: "0123456789abcdef01234567", + }), + })]); + }); + + test("drops malformed prompt-cache observations from hand-edited logs", () => { + writeFileSync(usageLogPath(), JSON.stringify({ + requestId: "bad-cache-shape", + timestamp: 1, + provider: "openai", + model: "gpt-5.6-terra", + adapter: "openai-responses", + promptCache: { version: 1, mode: "implicit", keyPresent: true }, + status: 200, + durationMs: 1, + usageStatus: "reported", + usage: { inputTokens: 1, outputTokens: 1 }, + }) + "\n"); + expect(readUsageEntries()[0]).not.toHaveProperty("promptCache"); + }); + test("persists providerAccountId separately from the display account label", () => { appendUsageEntry({ requestId: "ocx-account-id", From 8e13e35dc5c9b49f12a50c8042409362c65f2230 Mon Sep 17 00:00:00 2001 From: chefgroep Date: Sat, 3 Oct 2026 23:50:44 +0200 Subject: [PATCH 2/5] refactor: harden prompt cache evidence path --- scripts/analyze-prompt-cache-usage.ts | 505 ++++++++++++++--------- src/prompt-cache/observability.ts | 41 +- src/server/request-log.ts | 26 +- tests/prompt-cache-analyzer.test.ts | 145 +++++++ tests/prompt-cache-observability.test.ts | 102 +++++ tests/request-log.test.ts | 11 +- 6 files changed, 609 insertions(+), 221 deletions(-) create mode 100644 tests/prompt-cache-analyzer.test.ts diff --git a/scripts/analyze-prompt-cache-usage.ts b/scripts/analyze-prompt-cache-usage.ts index ceafa8590..3dd142da5 100644 --- a/scripts/analyze-prompt-cache-usage.ts +++ b/scripts/analyze-prompt-cache-usage.ts @@ -2,8 +2,22 @@ import { readFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; +import { + normalizePromptCacheRequestObservation, + type PromptCacheRequestObservation, +} from "../src/prompt-cache/observability"; type RangeName = "7d" | "30d" | "all"; +type CohortSignal = "write-heavy" | "low-reuse" | "healthy" | "mixed"; + +interface AnalyzerOptions { + source: string; + range: RangeName; + top: number; + nowMs: number; + jsonMode: boolean; + help: boolean; +} interface CacheUsage { input: number; @@ -11,11 +25,14 @@ interface CacheUsage { write: number; } -interface Cohort { +interface CohortDimensions { adapter: string; provider: string; model: string; surface: string; +} + +interface Cohort extends CohortDimensions { requests: number; reportedSuccess: number; input: number; @@ -27,10 +44,31 @@ interface Cohort { conversations: Map; } -interface ShapeSummary { - shape: string; - mode: string; - ttl: string | null; +interface CohortSummary extends CohortDimensions { + signal: CohortSignal; + requests: number; + reportedSuccess: number; + inputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + uncachedInputTokens: number; + cacheReadRatio: number; + cacheWriteRatio: number; + cacheHitRequestRatio: number; + cacheWriteRequestRatio: number; + conversations: number; + multiTurnConversations: number; +} + +interface ShapeBucket { + dimensions: CohortDimensions; + observation: PromptCacheRequestObservation; + rows: Record[]; +} + +interface ShapeSummary extends CohortDimensions { + mode: PromptCacheRequestObservation["mode"]; + ttl: PromptCacheRequestObservation["ttl"] | null; keyPresent: boolean; breakpointCount: number; toolCount: number; @@ -47,6 +85,19 @@ interface ShapeSummary { cacheWriteRatio: number; } +function usageText(): string { + return [ + "Usage: bun scripts/analyze-prompt-cache-usage.ts [usage.jsonl|-] [options]", + "", + "Options:", + " --range=7d|30d|all Analysis window (default: 7d)", + " --top=N Maximum cohorts/shapes to print, 1..200 (default: 20)", + " --now=ISO|epoch-ms Anchor relative windows for reproducible analysis", + " --json Emit JSON", + " --help Show this help", + ].join("\n"); +} + function isRecord(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } @@ -61,6 +112,71 @@ function stringValue(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } +function parseRange(value: string): RangeName { + if (value === "7d" || value === "30d" || value === "all") return value; + throw new Error(`invalid --range value: ${value}`); +} + +function parseTop(value: string): number { + if (!/^\d+$/u.test(value)) throw new Error(`invalid --top value: ${value}`); + const parsed = Number.parseInt(value, 10); + if (parsed < 1 || parsed > 200) throw new Error("--top must be between 1 and 200"); + return parsed; +} + +function parseNow(value: string): number { + if (/^\d+$/u.test(value)) { + const numeric = Number(value); + if (Number.isSafeInteger(numeric) && numeric >= 0) return numeric; + } + const parsed = Date.parse(value); + if (Number.isFinite(parsed) && parsed >= 0) return parsed; + throw new Error(`invalid --now value: ${value}`); +} + +function parseArgs(args: string[]): AnalyzerOptions { + const defaultSource = join( + process.env.OPENCODEX_HOME?.trim() || join(homedir(), ".opencodex"), + "usage.jsonl", + ); + let source = defaultSource; + let positionalSeen = false; + let range: RangeName = "7d"; + let top = 20; + let nowMs = Date.now(); + let jsonMode = false; + let help = false; + + for (const arg of args) { + if (arg === "--json") { + jsonMode = true; + continue; + } + if (arg === "--help") { + help = true; + continue; + } + if (arg.startsWith("--range=")) { + range = parseRange(arg.slice("--range=".length)); + continue; + } + if (arg.startsWith("--top=")) { + top = parseTop(arg.slice("--top=".length)); + continue; + } + if (arg.startsWith("--now=")) { + nowMs = parseNow(arg.slice("--now=".length)); + continue; + } + if (arg.startsWith("--")) throw new Error(`unknown option: ${arg}`); + if (positionalSeen) throw new Error("only one usage-log path may be supplied"); + source = arg; + positionalSeen = true; + } + + return { source, range, top, nowMs, jsonMode, help }; +} + function cacheUsage(row: Record): CacheUsage | undefined { if (row.status !== 200 || row.usageStatus !== "reported" || !isRecord(row.usage)) { return undefined; @@ -79,29 +195,32 @@ function cacheUsage(row: Record): CacheUsage | undefined { return { input, read, write }; } -function parseRange(value: string | undefined): RangeName { - return value === "7d" || value === "30d" || value === "all" ? value : "7d"; +function sinceFor(range: RangeName, nowMs: number): number { + if (range === "all") return 0; + return nowMs - (range === "7d" ? 7 : 30) * 86_400_000; } -function sinceFor(range: RangeName, now: number): number { - if (range === "all") return 0; - return now - (range === "7d" ? 7 : 30) * 86_400_000; +function dimensionsFrom(row: Record): CohortDimensions { + return { + adapter: stringValue(row.adapter) ?? "unknown", + provider: stringValue(row.provider) ?? "unknown", + model: stringValue(row.resolvedModel) ?? stringValue(row.model) ?? "unknown", + surface: stringValue(row.surface) ?? "unknown", + }; } -function keyParts(row: Record): [string, string, string, string] { - const adapter = stringValue(row.adapter) ?? "unknown"; - const provider = stringValue(row.provider) ?? "unknown"; - const model = stringValue(row.resolvedModel) ?? stringValue(row.model) ?? "unknown"; - const surface = stringValue(row.surface) ?? "unknown"; - return [adapter, provider, model, surface]; +function dimensionsKey(dimensions: CohortDimensions): string { + return JSON.stringify([ + dimensions.adapter, + dimensions.provider, + dimensions.model, + dimensions.surface, + ]); } -function emptyCohort(parts: [string, string, string, string]): Cohort { +function emptyCohort(dimensions: CohortDimensions): Cohort { return { - adapter: parts[0], - provider: parts[1], - model: parts[2], - surface: parts[3], + ...dimensions, requests: 0, reportedSuccess: 0, input: 0, @@ -118,7 +237,7 @@ function ratio(numerator: number, denominator: number): number { return denominator > 0 ? numerator / denominator : 0; } -function signal(cohort: Cohort): string { +function signal(cohort: Cohort): CohortSignal { const readRatio = ratio(cohort.cacheRead, cohort.input); const writeRatio = ratio(cohort.cacheWrite, cohort.input); if (writeRatio >= 0.5 && readRatio < 0.1) return "write-heavy"; @@ -127,7 +246,7 @@ function signal(cohort: Cohort): string { return "mixed"; } -function publicCohort(cohort: Cohort): Record { +function summarizeCohort(cohort: Cohort): CohortSummary { let multiTurnConversations = 0; for (const count of cohort.conversations.values()) { if (count > 1) multiTurnConversations += 1; @@ -153,30 +272,26 @@ function publicCohort(cohort: Cohort): Record { }; } -function cacheShapeKey(row: Record): string | undefined { - if (!isRecord(row.promptCache)) return undefined; - const pc = row.promptCache; - const parts = [ - stringValue(pc.mode) ?? "default", - stringValue(pc.ttl) ?? "-", - pc.keyPresent === true ? "key" : "nokey", - String(nonNegativeNumber(pc.breakpointCount) ?? 0), - stringValue(pc.toolsFingerprint) ?? "-", - stringValue(pc.stablePrefixFingerprint) ?? "-", - stringValue(pc.textFormatFingerprint) ?? "-", - stringValue(pc.verbosity) ?? "-", - ]; - return parts.join("|"); +function cacheShapeKey(observation: PromptCacheRequestObservation): string { + return JSON.stringify([ + observation.mode, + observation.ttl ?? null, + observation.keyPresent, + observation.breakpointCount, + observation.toolsFingerprint ?? null, + observation.stablePrefixFingerprint ?? null, + observation.textFormatFingerprint ?? null, + observation.verbosity ?? null, + ]); } -function shapeSummary(key: string, rows: Record[]): ShapeSummary { - const first = rows[0]?.promptCache; - const pc = isRecord(first) ? first : {}; +function summarizeShape(bucket: ShapeBucket): ShapeSummary { + const pc = bucket.observation; let input = 0; let read = 0; let write = 0; let reportedSuccess = 0; - for (const row of rows) { + for (const row of bucket.rows) { const usage = cacheUsage(row); if (!usage) continue; reportedSuccess += 1; @@ -185,17 +300,17 @@ function shapeSummary(key: string, rows: Record[]): ShapeSummar write += usage.write; } return { - shape: key, - mode: stringValue(pc.mode) ?? "default", - ttl: stringValue(pc.ttl) ?? null, - keyPresent: pc.keyPresent === true, - breakpointCount: nonNegativeNumber(pc.breakpointCount) ?? 0, - toolCount: nonNegativeNumber(pc.toolCount) ?? 0, - toolsFingerprint: stringValue(pc.toolsFingerprint) ?? null, - stablePrefixFingerprint: stringValue(pc.stablePrefixFingerprint) ?? null, - textFormatFingerprint: stringValue(pc.textFormatFingerprint) ?? null, - verbosity: stringValue(pc.verbosity) ?? null, - requests: rows.length, + ...bucket.dimensions, + mode: pc.mode, + ttl: pc.ttl ?? null, + keyPresent: pc.keyPresent, + breakpointCount: pc.breakpointCount, + toolCount: pc.toolCount, + toolsFingerprint: pc.toolsFingerprint ?? null, + stablePrefixFingerprint: pc.stablePrefixFingerprint ?? null, + textFormatFingerprint: pc.textFormatFingerprint ?? null, + verbosity: pc.verbosity ?? null, + requests: bucket.rows.length, reportedSuccess, inputTokens: input, cacheReadTokens: read, @@ -205,158 +320,178 @@ function shapeSummary(key: string, rows: Record[]): ShapeSummar }; } -function formatPercent(value: unknown): string { - return typeof value === "number" ? (value * 100).toFixed(1) + "%" : "-"; +function formatPercent(value: number): string { + return (value * 100).toFixed(1) + "%"; } -const args = Bun.argv.slice(2); -const source = args.find(arg => !arg.startsWith("--")) - ?? join(process.env.OPENCODEX_HOME?.trim() || join(homedir(), ".opencodex"), "usage.jsonl"); -const rangeArg = args.find(arg => arg.startsWith("--range="))?.slice("--range=".length); -const topArg = args.find(arg => arg.startsWith("--top="))?.slice("--top=".length); -const range = parseRange(rangeArg); -const top = Math.max(1, Math.min(200, Number.parseInt(topArg ?? "20", 10) || 20)); -const jsonMode = args.includes("--json"); -const now = Date.now(); -const since = sinceFor(range, now); -const text = source === "-" ? readFileSync(0, "utf8") : readFileSync(source, "utf8"); - -const rows: Record[] = []; -let invalidLines = 0; -for (const line of text.split(/\r?\n/u)) { - if (!line.trim()) continue; +function main(): number { + let options: AnalyzerOptions; try { - const parsed: unknown = JSON.parse(line); - if (isRecord(parsed)) rows.push(parsed); - else invalidLines += 1; - } catch { - invalidLines += 1; + options = parseArgs(Bun.argv.slice(2)); + } catch (error) { + const message = error instanceof Error ? error.message : "invalid arguments"; + console.error(message); + console.error(usageText()); + return 2; } -} -const selected = rows.filter(row => { - const timestamp = nonNegativeNumber(row.timestamp); - return timestamp !== undefined && timestamp >= since; -}); -const cohorts = new Map(); -const shapeRows = new Map[]>(); -let reportedSuccess = 0; -let input = 0; -let read = 0; -let write = 0; - -for (const row of selected) { - const parts = keyParts(row); - const key = parts.join("\0"); - const cohort = cohorts.get(key) ?? emptyCohort(parts); - cohorts.set(key, cohort); - cohort.requests += 1; - const conversationId = stringValue(row.conversationId); - if (conversationId) { - cohort.conversations.set( - conversationId, - (cohort.conversations.get(conversationId) ?? 0) + 1, - ); + if (options.help) { + console.log(usageText()); + return 0; } - const usage = cacheUsage(row); - if (usage) { - cohort.reportedSuccess += 1; - cohort.input += usage.input; - cohort.cacheRead += usage.read; - cohort.cacheWrite += usage.write; - cohort.uncachedInput += Math.max(0, usage.input - usage.read - usage.write); - if (usage.read > 0) cohort.hitRequests += 1; - if (usage.write > 0) cohort.writeRequests += 1; - reportedSuccess += 1; - input += usage.input; - read += usage.read; - write += usage.write; + const since = sinceFor(options.range, options.nowMs); + const text = options.source === "-" + ? readFileSync(0, "utf8") + : readFileSync(options.source, "utf8"); + + const rows: Record[] = []; + let invalidLines = 0; + for (const line of text.split(/\r?\n/u)) { + if (!line.trim()) continue; + try { + const parsed: unknown = JSON.parse(line); + if (isRecord(parsed)) rows.push(parsed); + else invalidLines += 1; + } catch { + invalidLines += 1; + } } - const shape = cacheShapeKey(row); - if (shape) { - const compositeShape = [...parts, shape].join("\0"); - const existing = shapeRows.get(compositeShape); - if (existing) existing.push(row); - else shapeRows.set(compositeShape, [row]); + const selected = rows.filter(row => { + const timestamp = nonNegativeNumber(row.timestamp); + return timestamp !== undefined && timestamp >= since && timestamp <= options.nowMs; + }); + const cohorts = new Map(); + const shapeBuckets = new Map(); + let reportedSuccess = 0; + let input = 0; + let read = 0; + let write = 0; + + for (const row of selected) { + const dimensions = dimensionsFrom(row); + const key = dimensionsKey(dimensions); + const cohort = cohorts.get(key) ?? emptyCohort(dimensions); + cohorts.set(key, cohort); + cohort.requests += 1; + const conversationId = stringValue(row.conversationId); + if (conversationId) { + cohort.conversations.set( + conversationId, + (cohort.conversations.get(conversationId) ?? 0) + 1, + ); + } + + const usage = cacheUsage(row); + if (usage) { + cohort.reportedSuccess += 1; + cohort.input += usage.input; + cohort.cacheRead += usage.read; + cohort.cacheWrite += usage.write; + cohort.uncachedInput += Math.max(0, usage.input - usage.read - usage.write); + if (usage.read > 0) cohort.hitRequests += 1; + if (usage.write > 0) cohort.writeRequests += 1; + reportedSuccess += 1; + input += usage.input; + read += usage.read; + write += usage.write; + } + + const observation = normalizePromptCacheRequestObservation(row.promptCache); + if (observation) { + const shapeKey = JSON.stringify([key, cacheShapeKey(observation)]); + const existing = shapeBuckets.get(shapeKey); + if (existing) existing.rows.push(row); + else shapeBuckets.set(shapeKey, { dimensions, observation, rows: [row] }); + } } -} -const cohortOutput = [...cohorts.values()] - .filter(cohort => cohort.reportedSuccess > 0) - .sort((a, b) => b.input - a.input) - .slice(0, top) - .map(publicCohort); -const shapeOutput = [...shapeRows.entries()] - .map(([key, value]) => shapeSummary(key.split("\0").slice(4).join("|"), value)) - .filter(shape => shape.reportedSuccess > 0) - .sort((a, b) => b.inputTokens - a.inputTokens) - .slice(0, top); - -const output = { - source: source === "-" ? "stdin" : source, - range, - rows: selected.length, - invalidLines, - proofBoundary: "status=200 AND usageStatus=reported", - summary: { - reportedSuccess, - inputTokens: input, - cacheReadTokens: read, - cacheWriteTokens: write, - uncachedInputTokens: Math.max(0, input - read - write), - cacheReadRatio: ratio(read, input), - cacheWriteRatio: ratio(write, input), - }, - cohorts: cohortOutput, - cacheShapes: shapeOutput, -}; - -if (jsonMode) { - process.stdout.write(JSON.stringify(output, null, 2) + "\n"); - process.exit(0); -} + const cohortOutput = [...cohorts.values()] + .filter(cohort => cohort.reportedSuccess > 0) + .sort((a, b) => b.input - a.input) + .slice(0, options.top) + .map(summarizeCohort); + const shapeOutput = [...shapeBuckets.values()] + .map(summarizeShape) + .filter(shape => shape.reportedSuccess > 0) + .sort((a, b) => b.inputTokens - a.inputTokens) + .slice(0, options.top); -console.log("Prompt cache usage (" + range + ")"); -console.log("proof: " + output.proofBoundary); -console.log( - "reported-success=" + reportedSuccess - + " input=" + Math.round(input) - + " read=" + Math.round(read) + " (" + formatPercent(output.summary.cacheReadRatio) + ")" - + " write=" + Math.round(write) + " (" + formatPercent(output.summary.cacheWriteRatio) + ")", -); -console.log(""); -console.log("Top cohorts by measured input:"); -for (const item of cohortOutput) { + const output = { + source: options.source === "-" ? "stdin" : options.source, + range: options.range, + windowStartMs: since, + windowEndMs: options.nowMs, + windowStart: new Date(since).toISOString(), + windowEnd: new Date(options.nowMs).toISOString(), + rows: selected.length, + invalidLines, + proofBoundary: "status=200 AND usageStatus=reported", + summary: { + reportedSuccess, + inputTokens: input, + cacheReadTokens: read, + cacheWriteTokens: write, + uncachedInputTokens: Math.max(0, input - read - write), + cacheReadRatio: ratio(read, input), + cacheWriteRatio: ratio(write, input), + }, + cohorts: cohortOutput, + cacheShapes: shapeOutput, + }; + + if (options.jsonMode) { + process.stdout.write(JSON.stringify(output, null, 2) + "\n"); + return 0; + } + + console.log("Prompt cache usage (" + options.range + ")"); + console.log("window: " + output.windowStart + " .. " + output.windowEnd); + console.log("proof: " + output.proofBoundary); console.log( - String(item.signal).padEnd(11) - + " " + String(item.adapter) + "/" + String(item.provider) + "/" + String(item.model) - + " surface=" + String(item.surface) - + " input=" + String(item.inputTokens) - + " read=" + formatPercent(item.cacheReadRatio) - + " write=" + formatPercent(item.cacheWriteRatio) - + " hits=" + formatPercent(item.cacheHitRequestRatio) - + " n=" + String(item.reportedSuccess) + "/" + String(item.requests), + "reported-success=" + reportedSuccess + + " input=" + Math.round(input) + + " read=" + Math.round(read) + " (" + formatPercent(output.summary.cacheReadRatio) + ")" + + " write=" + Math.round(write) + " (" + formatPercent(output.summary.cacheWriteRatio) + ")", ); -} -if (shapeOutput.length > 0) { console.log(""); - console.log("Observed outbound cache shapes:"); - for (const item of shapeOutput) { + console.log("Top cohorts by measured input:"); + for (const item of cohortOutput) { console.log( - item.mode.padEnd(8) - + " tools=" + item.toolCount - + " bp=" + item.breakpointCount + item.signal.padEnd(11) + + " " + item.adapter + "/" + item.provider + "/" + item.model + + " surface=" + item.surface + " input=" + item.inputTokens + " read=" + formatPercent(item.cacheReadRatio) + " write=" + formatPercent(item.cacheWriteRatio) - + " n=" + item.reportedSuccess + "/" + item.requests - + " prefix=" + (item.stablePrefixFingerprint ?? "-") - + " toolsHash=" + (item.toolsFingerprint ?? "-"), + + " hits=" + formatPercent(item.cacheHitRequestRatio) + + " n=" + item.reportedSuccess + "/" + item.requests, ); } -} else { - console.log(""); - console.log("No promptCache observations in this window (historical rows predate instrumentation)."); + if (shapeOutput.length > 0) { + console.log(""); + console.log("Observed outbound cache shapes:"); + for (const item of shapeOutput) { + console.log( + item.adapter + "/" + item.provider + "/" + item.model + + " surface=" + item.surface + + " mode=" + item.mode + + " tools=" + item.toolCount + + " bp=" + item.breakpointCount + + " input=" + item.inputTokens + + " read=" + formatPercent(item.cacheReadRatio) + + " write=" + formatPercent(item.cacheWriteRatio) + + " n=" + item.reportedSuccess + "/" + item.requests + + " prefix=" + (item.stablePrefixFingerprint ?? "-") + + " toolsHash=" + (item.toolsFingerprint ?? "-"), + ); + } + } else { + console.log(""); + console.log("No promptCache observations in this window (historical rows predate instrumentation)."); + } + return 0; } + +process.exitCode = main(); diff --git a/src/prompt-cache/observability.ts b/src/prompt-cache/observability.ts index 7c655f4a4..0b217fb58 100644 --- a/src/prompt-cache/observability.ts +++ b/src/prompt-cache/observability.ts @@ -25,19 +25,39 @@ function isRecord(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } -function canonicalValue(value: unknown): unknown { - if (Array.isArray(value)) return value.map(canonicalValue); - if (!isRecord(value)) return value; - const out: Record = {}; - for (const key of Object.keys(value).sort()) { - out[key] = canonicalValue(value[key]); +/** + * Serialize JSON-compatible input with stable object-key ordering. + * Arrays retain their wire order and object keys such as "__proto__" remain ordinary data. + */ +function canonicalJson(value: unknown): string | undefined { + if (value === null) return "null"; + if (Array.isArray(value)) { + const items = value.map(item => canonicalJson(item) ?? "null"); + return "[" + items.join(",") + "]"; + } + if (isRecord(value)) { + const fields: string[] = []; + for (const key of Object.keys(value).sort()) { + const child = canonicalJson(value[key]); + if (child === undefined) continue; + fields.push(JSON.stringify(key) + ":" + child); + } + return "{" + fields.join(",") + "}"; + } + if (typeof value === "number" && !Number.isFinite(value)) return "null"; + if ( + typeof value === "string" + || typeof value === "number" + || typeof value === "boolean" + ) { + return JSON.stringify(value); } - return out; + return undefined; } function fingerprint(value: unknown): string | undefined { - const serialized = JSON.stringify(canonicalValue(value)); - if (typeof serialized !== "string") return undefined; + const serialized = canonicalJson(value); + if (serialized === undefined) return undefined; return createHash("sha256").update(serialized).digest("hex").slice(0, 24); } @@ -94,7 +114,8 @@ export function observeOpenAiResponsesPromptCache( : options?.mode === "implicit" ? "implicit" : "default"; - const ttl = options?.ttl === "30m" ? "30m" as const : undefined; + const ttl: PromptCacheRequestObservation["ttl"] = + options?.ttl === "30m" ? "30m" : undefined; const retention = value.prompt_cache_retention === "in_memory" || value.prompt_cache_retention === "24h" ? value.prompt_cache_retention diff --git a/src/server/request-log.ts b/src/server/request-log.ts index ef50790bc..37db9d4da 100644 --- a/src/server/request-log.ts +++ b/src/server/request-log.ts @@ -39,10 +39,7 @@ import { } from "../usage/debug"; import { matchesLogConversationId } from "./request-log-conversation"; import { captureRequestTelemetry } from "../telemetry/posthog-server"; -import { - normalizePromptCacheRequestObservation, - type PromptCacheRequestObservation, -} from "../prompt-cache/observability"; +import type { PromptCacheRequestObservation } from "../prompt-cache/observability"; export interface RequestLogContext { model: string; @@ -481,7 +478,7 @@ export function recordAttemptRequestedEffort(logCtx: RequestLogContext): void { } /** Copy the adapter's exact outbound reasoning parameter into the durable request log. */ -export function recordAdapterReasoning( +function recordAdapterReasoning( logCtx: RequestLogContext, request: AdapterRequest, ): void { @@ -531,27 +528,16 @@ export function recordAdapterReasoning( } } -/** Copy prompt-cache diagnostics from the exact outbound adapter request. */ -export function recordAdapterPromptCache( - logCtx: RequestLogContext, - request: AdapterRequest, -): void { - delete logCtx.promptCache; - try { - const observation = normalizePromptCacheRequestObservation(request.promptCacheLog); - if (observation) logCtx.promptCache = observation; - } catch { - // Request logging is best-effort and must not affect request delivery. - } -} - /** Record all adapter-derived request diagnostics at one lifecycle seam. */ export function recordAdapterRequestMetadata( logCtx: RequestLogContext, request: AdapterRequest, ): void { recordAdapterReasoning(logCtx, request); - recordAdapterPromptCache(logCtx, request); + delete logCtx.promptCache; + if (request.promptCacheLog) { + logCtx.promptCache = { ...request.promptCacheLog }; + } } export function requestLogErrorCode(status: number, upstreamError?: string): string | undefined { diff --git a/tests/prompt-cache-analyzer.test.ts b/tests/prompt-cache-analyzer.test.ts new file mode 100644 index 000000000..65c33917e --- /dev/null +++ b/tests/prompt-cache-analyzer.test.ts @@ -0,0 +1,145 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const script = join(import.meta.dir, "../scripts/analyze-prompt-cache-usage.ts"); +const tempDirs: string[] = []; + +function tempUsageFile(lines: unknown[]): string { + const dir = mkdtempSync(join(tmpdir(), "ocx-cache-analyzer-")); + tempDirs.push(dir); + const path = join(dir, "usage.jsonl"); + writeFileSync(path, lines.map(line => JSON.stringify(line)).join("\n") + "\n"); + return path; +} + +function run(path: string, ...args: string[]) { + return Bun.spawnSync([process.execPath, script, path, ...args], { + stdout: "pipe", + stderr: "pipe", + }); +} + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("prompt-cache usage analyzer", () => { + test("is reproducible with --now and keeps cache-shape cohort dimensions", () => { + const path = tempUsageFile([ + { + requestId: "r1", + timestamp: Date.parse("2026-10-02T10:00:00Z"), + provider: "openai", + adapter: "openai-responses", + model: "gpt-5.6-terra", + resolvedModel: "gpt-5.6-terra", + surface: "codex", + status: 200, + usageStatus: "reported", + usage: { + inputTokens: 100, + outputTokens: 1, + cacheReadInputTokens: 80, + }, + promptCache: { + version: 1, + keyPresent: false, + mode: "implicit", + ttl: "30m", + prewarm: false, + comparisonRequested: false, + previousResponseIdPresent: false, + breakpointCount: 0, + inputItemCount: 2, + toolCount: 1, + toolsFingerprint: "0123456789abcdef01234567", + stablePrefixFingerprint: "89abcdef0123456789abcdef", + }, + }, + { + requestId: "r2", + timestamp: Date.parse("2026-10-02T11:00:00Z"), + provider: "openai", + adapter: "openai-responses", + model: "gpt-5.6-terra", + resolvedModel: "gpt-5.6-terra", + surface: "codex", + status: 200, + usageStatus: "reported", + usage: { + inputTokens: 200, + outputTokens: 1, + cacheReadInputTokens: 160, + }, + promptCache: { + version: 1, + keyPresent: false, + mode: "implicit", + ttl: "30m", + prewarm: false, + comparisonRequested: false, + previousResponseIdPresent: false, + breakpointCount: 0, + inputItemCount: 3, + toolCount: 1, + toolsFingerprint: "0123456789abcdef01234567", + stablePrefixFingerprint: "89abcdef0123456789abcdef", + }, + }, + { + requestId: "old", + timestamp: Date.parse("2026-09-20T10:00:00Z"), + provider: "openai", + adapter: "openai-responses", + model: "gpt-5.6-terra", + status: 200, + usageStatus: "reported", + usage: { inputTokens: 999, outputTokens: 1, cacheReadInputTokens: 999 }, + }, + ]); + + const args = ["--range=7d", "--now=2026-10-03T12:00:00Z", "--json"]; + const first = run(path, ...args); + const second = run(path, ...args); + + expect(first.exitCode).toBe(0); + expect(second.exitCode).toBe(0); + expect(first.stdout.toString()).toBe(second.stdout.toString()); + + const output = JSON.parse(first.stdout.toString()); + expect(output.windowEnd).toBe("2026-10-03T12:00:00.000Z"); + expect(output.summary).toMatchObject({ + reportedSuccess: 2, + inputTokens: 300, + cacheReadTokens: 240, + cacheReadRatio: 0.8, + }); + expect(output.cacheShapes).toEqual([ + expect.objectContaining({ + adapter: "openai-responses", + provider: "openai", + model: "gpt-5.6-terra", + surface: "codex", + mode: "implicit", + ttl: "30m", + requests: 2, + reportedSuccess: 2, + }), + ]); + }); + + test("rejects invalid CLI arguments instead of silently changing the analysis", () => { + const path = tempUsageFile([]); + const invalidRange = run(path, "--range=week"); + expect(invalidRange.exitCode).toBe(2); + expect(invalidRange.stderr.toString()).toContain("invalid --range value: week"); + + const unknown = run(path, "--bogus"); + expect(unknown.exitCode).toBe(2); + expect(unknown.stderr.toString()).toContain("unknown option: --bogus"); + }); +}); diff --git a/tests/prompt-cache-observability.test.ts b/tests/prompt-cache-observability.test.ts index 821d2e1ad..18805b733 100644 --- a/tests/prompt-cache-observability.test.ts +++ b/tests/prompt-cache-observability.test.ts @@ -1,8 +1,24 @@ import { describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createResponsesPassthroughAdapter } from "../src/adapters/openai-responses"; import { normalizePromptCacheRequestObservation, observeOpenAiResponsesPromptCache, } from "../src/prompt-cache/observability"; +import { + addFinalRequestLog, + clearRequestLogsForTests, + recordAdapterRequestMetadata, + type RequestLogContext, +} from "../src/server/request-log"; +import { + readUsageEntries, + resetUsageReadCacheForTests, + usageLogPath, +} from "../src/usage/log"; +import type { OcxProviderConfig } from "../src/types"; describe("prompt cache observability", () => { test("captures structural cache dimensions without retaining request content", () => { @@ -92,6 +108,22 @@ describe("prompt cache observability", () => { expect(a?.toolsFingerprint).not.toBe(reordered?.toolsFingerprint); }); + test("preserves __proto__ as ordinary schema data when fingerprinting", () => { + const firstTool: unknown = JSON.parse( + '{"type":"function","name":"x","parameters":{"type":"object","properties":{"__proto__":{"const":"a"}}}}', + ); + const secondTool: unknown = JSON.parse( + '{"type":"function","name":"x","parameters":{"type":"object","properties":{"__proto__":{"const":"b"}}}}', + ); + + const first = observeOpenAiResponsesPromptCache({ tools: [firstTool] }); + const second = observeOpenAiResponsesPromptCache({ tools: [secondTool] }); + + expect(first?.toolsFingerprint).toMatch(/^[0-9a-f]{24}$/); + expect(second?.toolsFingerprint).toMatch(/^[0-9a-f]{24}$/); + expect(first?.toolsFingerprint).not.toBe(second?.toolsFingerprint); + }); + test("normalizer accepts generated observations and rejects malformed persisted data", () => { const observation = observeOpenAiResponsesPromptCache({ prompt_cache_options: { mode: "explicit", ttl: "30m" }, @@ -114,4 +146,74 @@ describe("prompt cache observability", () => { toolCount: 0, })).toBeUndefined(); }); + + test("flows the final outbound observation through request logging into usage.jsonl", () => { + const dir = mkdtempSync(join(tmpdir(), "ocx-cache-observation-")); + const previousHome = process.env.OPENCODEX_HOME; + process.env.OPENCODEX_HOME = dir; + resetUsageReadCacheForTests(); + clearRequestLogsForTests(); + + try { + const provider: OcxProviderConfig = { + adapter: "openai-responses", + baseUrl: "https://api.openai.com/v1", + authMode: "key", + apiKey: "test-key", + }; + const adapter = createResponsesPassthroughAdapter(provider); + const request = adapter.buildRequest({ + modelId: "gpt-5.6-terra", + context: { messages: [] }, + stream: true, + options: { promptCacheKey: "private-cache-key" }, + _rawBody: { + model: "gpt-5.6-terra", + instructions: "private fixed prefix", + input: [{ role: "user", content: "hello" }], + tools: [{ type: "function", name: "shell", parameters: { type: "object" } }], + prompt_cache_key: "private-cache-key", + prompt_cache_options: { mode: "implicit", ttl: "30m" }, + }, + }); + const logCtx: RequestLogContext = { + model: "gpt-5.6-terra", + provider: "openai-apikey", + providerAdapter: adapter.name, + usage: { + inputTokens: 100, + outputTokens: 5, + cacheReadInputTokens: 80, + }, + }; + recordAdapterRequestMetadata(logCtx, request); + addFinalRequestLog("ocx-cache-e2e", 1_000, logCtx, 200); + + const entries = readUsageEntries(); + expect(entries).toHaveLength(1); + expect(entries[0]).toMatchObject({ + requestId: "ocx-cache-e2e", + adapter: "openai-responses", + usageStatus: "reported", + usage: { inputTokens: 100, cacheReadInputTokens: 80 }, + promptCache: { + version: 1, + keyPresent: true, + mode: "implicit", + ttl: "30m", + toolCount: 1, + }, + }); + const persisted = readFileSync(usageLogPath(), "utf8"); + expect(persisted).not.toContain("private-cache-key"); + expect(persisted).not.toContain("private fixed prefix"); + expect(persisted).not.toContain("\"shell\""); + } finally { + clearRequestLogsForTests(); + resetUsageReadCacheForTests(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/tests/request-log.test.ts b/tests/request-log.test.ts index bb99f1802..04d562e71 100644 --- a/tests/request-log.test.ts +++ b/tests/request-log.test.ts @@ -20,7 +20,6 @@ import { getRequestLogEntries, hydrateRequestLogsFromDisk, noteAttemptSend, - recordAdapterReasoning, recordAdapterRequestMetadata, recordFirstOutput, requestLogEntryFromPersistedUsage, @@ -121,7 +120,7 @@ describe("request log metadata", () => { activeAttempt: attempt, }; - recordAdapterReasoning(logCtx, { + recordAdapterRequestMetadata(logCtx, { url: "https://api.x.ai/v1/chat/completions", method: "POST", headers: {}, @@ -147,7 +146,7 @@ describe("request log metadata", () => { }); const sensitiveAlias = ["sk", "proj", "redaction-fixture"].join("-"); - recordAdapterReasoning(logCtx, { + recordAdapterRequestMetadata(logCtx, { url: "https://provider.test/v1/chat/completions", method: "POST", headers: {}, @@ -247,7 +246,7 @@ describe("request log metadata", () => { reasoningWireValue: "stale", }); - expect(() => recordAdapterReasoning(logCtx, { + expect(() => recordAdapterRequestMetadata(logCtx, { url: "https://provider.test/v1/chat/completions", method: "POST", headers: {}, @@ -371,7 +370,7 @@ describe("request log metadata", () => { test("final combo logging keeps one logical row and finalizes its active attempt", () => { const entries: RequestLogEntry[] = []; const a = beginRequestAttempt(1, "a", "model-a", "openai-chat"); - recordAdapterReasoning({ + recordAdapterRequestMetadata({ model: "model-a", provider: "a", requestedEffort: "minimal", @@ -409,7 +408,7 @@ describe("request log metadata", () => { activeAttempt: b, activeAttemptStartedAt: start, }; - recordAdapterReasoning(logCtx, { + recordAdapterRequestMetadata(logCtx, { url: "https://provider.test/v1/chat/completions", method: "POST", headers: {}, From e468b0eae7a25b672137f81592d46d0244e7e183 Mon Sep 17 00:00:00 2001 From: chefgroep Date: Sun, 4 Oct 2026 00:20:18 +0200 Subject: [PATCH 3/5] refactor: reduce prompt cache analysis complexity --- scripts/analyze-prompt-cache-usage.ts | 277 +++++++++++++++++--------- src/prompt-cache/observability.ts | 161 ++++++++++----- 2 files changed, 297 insertions(+), 141 deletions(-) diff --git a/scripts/analyze-prompt-cache-usage.ts b/scripts/analyze-prompt-cache-usage.ts index 3dd142da5..2f92b5357 100644 --- a/scripts/analyze-prompt-cache-usage.ts +++ b/scripts/analyze-prompt-cache-usage.ts @@ -324,27 +324,31 @@ function formatPercent(value: number): string { return (value * 100).toFixed(1) + "%"; } -function main(): number { - let options: AnalyzerOptions; - try { - options = parseArgs(Bun.argv.slice(2)); - } catch (error) { - const message = error instanceof Error ? error.message : "invalid arguments"; - console.error(message); - console.error(usageText()); - return 2; - } +interface ParsedUsageRows { + rows: Record[]; + invalidLines: number; +} - if (options.help) { - console.log(usageText()); - return 0; - } +interface AnalysisTotals { + reportedSuccess: number; + input: number; + read: number; + write: number; +} - const since = sinceFor(options.range, options.nowMs); - const text = options.source === "-" +interface AnalysisState { + cohorts: Map; + shapeBuckets: Map; + totals: AnalysisTotals; +} + +function readUsageSource(options: AnalyzerOptions): string { + return options.source === "-" ? readFileSync(0, "utf8") : readFileSync(options.source, "utf8"); +} +function parseUsageRows(text: string): ParsedUsageRows { const rows: Record[] = []; let invalidLines = 0; for (const line of text.split(/\r?\n/u)) { @@ -357,75 +361,126 @@ function main(): number { invalidLines += 1; } } + return { rows, invalidLines }; +} - const selected = rows.filter(row => { +function rowsInWindow( + rows: readonly Record[], + since: number, + nowMs: number, +): Record[] { + return rows.filter(row => { const timestamp = nonNegativeNumber(row.timestamp); - return timestamp !== undefined && timestamp >= since && timestamp <= options.nowMs; + return timestamp !== undefined && timestamp >= since && timestamp <= nowMs; }); +} + +function recordConversation(cohort: Cohort, row: Record): void { + const conversationId = stringValue(row.conversationId); + if (!conversationId) return; + cohort.conversations.set( + conversationId, + (cohort.conversations.get(conversationId) ?? 0) + 1, + ); +} + +function recordCacheUsage( + cohort: Cohort, + usage: CacheUsage | undefined, + totals: AnalysisTotals, +): void { + if (!usage) return; + cohort.reportedSuccess += 1; + cohort.input += usage.input; + cohort.cacheRead += usage.read; + cohort.cacheWrite += usage.write; + cohort.uncachedInput += Math.max(0, usage.input - usage.read - usage.write); + if (usage.read > 0) cohort.hitRequests += 1; + if (usage.write > 0) cohort.writeRequests += 1; + totals.reportedSuccess += 1; + totals.input += usage.input; + totals.read += usage.read; + totals.write += usage.write; +} + +function recordCacheShape( + buckets: Map, + key: string, + dimensions: CohortDimensions, + row: Record, +): void { + const observation = normalizePromptCacheRequestObservation(row.promptCache); + if (!observation) return; + const shapeKey = JSON.stringify([key, cacheShapeKey(observation)]); + const existing = buckets.get(shapeKey); + if (existing) { + existing.rows.push(row); + return; + } + buckets.set(shapeKey, { dimensions, observation, rows: [row] }); +} + +function analyzeRows(rows: readonly Record[]): AnalysisState { const cohorts = new Map(); const shapeBuckets = new Map(); - let reportedSuccess = 0; - let input = 0; - let read = 0; - let write = 0; + const totals: AnalysisTotals = { + reportedSuccess: 0, + input: 0, + read: 0, + write: 0, + }; - for (const row of selected) { + for (const row of rows) { const dimensions = dimensionsFrom(row); const key = dimensionsKey(dimensions); const cohort = cohorts.get(key) ?? emptyCohort(dimensions); cohorts.set(key, cohort); cohort.requests += 1; - const conversationId = stringValue(row.conversationId); - if (conversationId) { - cohort.conversations.set( - conversationId, - (cohort.conversations.get(conversationId) ?? 0) + 1, - ); - } - - const usage = cacheUsage(row); - if (usage) { - cohort.reportedSuccess += 1; - cohort.input += usage.input; - cohort.cacheRead += usage.read; - cohort.cacheWrite += usage.write; - cohort.uncachedInput += Math.max(0, usage.input - usage.read - usage.write); - if (usage.read > 0) cohort.hitRequests += 1; - if (usage.write > 0) cohort.writeRequests += 1; - reportedSuccess += 1; - input += usage.input; - read += usage.read; - write += usage.write; - } - - const observation = normalizePromptCacheRequestObservation(row.promptCache); - if (observation) { - const shapeKey = JSON.stringify([key, cacheShapeKey(observation)]); - const existing = shapeBuckets.get(shapeKey); - if (existing) existing.rows.push(row); - else shapeBuckets.set(shapeKey, { dimensions, observation, rows: [row] }); - } + recordConversation(cohort, row); + recordCacheUsage(cohort, cacheUsage(row), totals); + recordCacheShape(shapeBuckets, key, dimensions, row); } - const cohortOutput = [...cohorts.values()] + return { cohorts, shapeBuckets, totals }; +} + +function topCohorts(cohorts: Map, top: number): CohortSummary[] { + return [...cohorts.values()] .filter(cohort => cohort.reportedSuccess > 0) .sort((a, b) => b.input - a.input) - .slice(0, options.top) + .slice(0, top) .map(summarizeCohort); - const shapeOutput = [...shapeBuckets.values()] +} + +function topShapes( + buckets: Map, + top: number, +): ShapeSummary[] { + return [...buckets.values()] .map(summarizeShape) .filter(shape => shape.reportedSuccess > 0) .sort((a, b) => b.inputTokens - a.inputTokens) - .slice(0, options.top); + .slice(0, top); +} - const output = { +function buildOutput( + options: AnalyzerOptions, + since: number, + selectedRows: number, + invalidLines: number, + analysis: AnalysisState, +) { + const cohorts = topCohorts(analysis.cohorts, options.top); + const cacheShapes = topShapes(analysis.shapeBuckets, options.top); + const { reportedSuccess, input, read, write } = analysis.totals; + return { source: options.source === "-" ? "stdin" : options.source, range: options.range, windowStartMs: since, windowEndMs: options.nowMs, windowStart: new Date(since).toISOString(), windowEnd: new Date(options.nowMs).toISOString(), - rows: selected.length, + rows: selectedRows, invalidLines, proofBoundary: "status=200 AND usageStatus=reported", summary: { @@ -437,27 +492,28 @@ function main(): number { cacheReadRatio: ratio(read, input), cacheWriteRatio: ratio(write, input), }, - cohorts: cohortOutput, - cacheShapes: shapeOutput, + cohorts, + cacheShapes, }; +} - if (options.jsonMode) { - process.stdout.write(JSON.stringify(output, null, 2) + "\n"); - return 0; - } +type AnalyzerOutput = ReturnType; - console.log("Prompt cache usage (" + options.range + ")"); +function printHumanOutput(output: AnalyzerOutput): void { + console.log("Prompt cache usage (" + output.range + ")"); console.log("window: " + output.windowStart + " .. " + output.windowEnd); console.log("proof: " + output.proofBoundary); console.log( - "reported-success=" + reportedSuccess - + " input=" + Math.round(input) - + " read=" + Math.round(read) + " (" + formatPercent(output.summary.cacheReadRatio) + ")" - + " write=" + Math.round(write) + " (" + formatPercent(output.summary.cacheWriteRatio) + ")", + "reported-success=" + output.summary.reportedSuccess + + " input=" + Math.round(output.summary.inputTokens) + + " read=" + Math.round(output.summary.cacheReadTokens) + + " (" + formatPercent(output.summary.cacheReadRatio) + ")" + + " write=" + Math.round(output.summary.cacheWriteTokens) + + " (" + formatPercent(output.summary.cacheWriteRatio) + ")", ); console.log(""); console.log("Top cohorts by measured input:"); - for (const item of cohortOutput) { + for (const item of output.cohorts) { console.log( item.signal.padEnd(11) + " " + item.adapter + "/" + item.provider + "/" + item.model @@ -469,28 +525,67 @@ function main(): number { + " n=" + item.reportedSuccess + "/" + item.requests, ); } - if (shapeOutput.length > 0) { - console.log(""); - console.log("Observed outbound cache shapes:"); - for (const item of shapeOutput) { - console.log( - item.adapter + "/" + item.provider + "/" + item.model - + " surface=" + item.surface - + " mode=" + item.mode - + " tools=" + item.toolCount - + " bp=" + item.breakpointCount - + " input=" + item.inputTokens - + " read=" + formatPercent(item.cacheReadRatio) - + " write=" + formatPercent(item.cacheWriteRatio) - + " n=" + item.reportedSuccess + "/" + item.requests - + " prefix=" + (item.stablePrefixFingerprint ?? "-") - + " toolsHash=" + (item.toolsFingerprint ?? "-"), - ); - } - } else { - console.log(""); + printCacheShapes(output.cacheShapes); +} + +function printCacheShapes(shapes: readonly ShapeSummary[]): void { + console.log(""); + if (shapes.length === 0) { console.log("No promptCache observations in this window (historical rows predate instrumentation)."); + return; + } + + console.log("Observed outbound cache shapes:"); + for (const item of shapes) { + console.log( + item.adapter + "/" + item.provider + "/" + item.model + + " surface=" + item.surface + + " mode=" + item.mode + + " tools=" + item.toolCount + + " bp=" + item.breakpointCount + + " input=" + item.inputTokens + + " read=" + formatPercent(item.cacheReadRatio) + + " write=" + formatPercent(item.cacheWriteRatio) + + " n=" + item.reportedSuccess + "/" + item.requests + + " prefix=" + (item.stablePrefixFingerprint ?? "-") + + " toolsHash=" + (item.toolsFingerprint ?? "-"), + ); } +} + +function main(): number { + let options: AnalyzerOptions; + try { + options = parseArgs(Bun.argv.slice(2)); + } catch (error) { + const message = error instanceof Error ? error.message : "invalid arguments"; + console.error(message); + console.error(usageText()); + return 2; + } + + if (options.help) { + console.log(usageText()); + return 0; + } + + const since = sinceFor(options.range, options.nowMs); + const parsed = parseUsageRows(readUsageSource(options)); + const selected = rowsInWindow(parsed.rows, since, options.nowMs); + const output = buildOutput( + options, + since, + selected.length, + parsed.invalidLines, + analyzeRows(selected), + ); + + if (options.jsonMode) { + process.stdout.write(JSON.stringify(output, null, 2) + "\n"); + return 0; + } + + printHumanOutput(output); return 0; } diff --git a/src/prompt-cache/observability.ts b/src/prompt-cache/observability.ts index 0b217fb58..1a9b8f20f 100644 --- a/src/prompt-cache/observability.ts +++ b/src/prompt-cache/observability.ts @@ -101,6 +101,55 @@ function boundedString(value: unknown, maxLength = 32): string | undefined { return trimmed.slice(0, maxLength); } +function hasNonEmptyString(value: unknown): boolean { + return typeof value === "string" && value.trim().length > 0; +} + +function promptCacheMode(options: Record | undefined): PromptCacheMode { + if (options?.mode === "explicit") return "explicit"; + if (options?.mode === "implicit") return "implicit"; + return "default"; +} + +function promptCacheTtl( + options: Record | undefined, +): PromptCacheRequestObservation["ttl"] { + return options?.ttl === "30m" ? "30m" : undefined; +} + +function promptCacheLegacyRetention(value: unknown): PromptCacheLegacyRetention | undefined { + if (value === "in_memory" || value === "24h") return value; + return undefined; +} + +function promptCacheInputItemCount(input: unknown): number { + if (Array.isArray(input)) return input.length; + return input === undefined ? 0 : 1; +} + +interface PromptCacheObservationExtras { + toolsFingerprint?: string; + stablePrefixFingerprint?: string; + textFormatFingerprint?: string; + verbosity?: string; +} + +function promptCacheObservationExtras(input: { + tools: unknown[]; + prefix: unknown[]; + textFormat: unknown; + verbosity: string | undefined; +}): PromptCacheObservationExtras { + const extras: PromptCacheObservationExtras = {}; + if (input.tools.length > 0) extras.toolsFingerprint = fingerprint(input.tools); + if (input.prefix.length > 0) extras.stablePrefixFingerprint = fingerprint(input.prefix); + if (input.textFormat !== undefined) { + extras.textFormatFingerprint = fingerprint(input.textFormat); + } + if (input.verbosity) extras.verbosity = input.verbosity; + return extras; +} + export function observeOpenAiResponsesPromptCache( value: unknown, ): PromptCacheRequestObservation | undefined { @@ -109,17 +158,8 @@ export function observeOpenAiResponsesPromptCache( const options = isRecord(value.prompt_cache_options) ? value.prompt_cache_options : undefined; - const mode: PromptCacheMode = options?.mode === "explicit" - ? "explicit" - : options?.mode === "implicit" - ? "implicit" - : "default"; - const ttl: PromptCacheRequestObservation["ttl"] = - options?.ttl === "30m" ? "30m" : undefined; - const retention = value.prompt_cache_retention === "in_memory" - || value.prompt_cache_retention === "24h" - ? value.prompt_cache_retention - : undefined; + const ttl = promptCacheTtl(options); + const retention = promptCacheLegacyRetention(value.prompt_cache_retention); const tools = Array.isArray(value.tools) ? value.tools : []; const input = value.input; const prefix = stablePrefix(value); @@ -129,23 +169,17 @@ export function observeOpenAiResponsesPromptCache( return { version: 1, - keyPresent: typeof value.prompt_cache_key === "string" - && value.prompt_cache_key.trim().length > 0, - mode, + keyPresent: hasNonEmptyString(value.prompt_cache_key), + mode: promptCacheMode(options), ...(ttl ? { ttl } : {}), ...(retention ? { legacyRetention: retention } : {}), prewarm: options?.prewarm === true, - comparisonRequested: typeof options?.comparison_response_id === "string" - && options.comparison_response_id.trim().length > 0, - previousResponseIdPresent: typeof value.previous_response_id === "string" - && value.previous_response_id.trim().length > 0, + comparisonRequested: hasNonEmptyString(options?.comparison_response_id), + previousResponseIdPresent: hasNonEmptyString(value.previous_response_id), breakpointCount: countExplicitBreakpoints(input), - inputItemCount: Array.isArray(input) ? input.length : input === undefined ? 0 : 1, + inputItemCount: promptCacheInputItemCount(input), toolCount: tools.length, - ...(tools.length > 0 ? { toolsFingerprint: fingerprint(tools) } : {}), - ...(prefix.length > 0 ? { stablePrefixFingerprint: fingerprint(prefix) } : {}), - ...(textFormat !== undefined ? { textFormatFingerprint: fingerprint(textFormat) } : {}), - ...(verbosity ? { verbosity } : {}), + ...promptCacheObservationExtras({ tools, prefix, textFormat, verbosity }), }; } @@ -160,32 +194,65 @@ function isBoundedCount(value: unknown): value is number { && value <= 1_000_000; } +interface PromptCacheObservationCore { + keyPresent: boolean; + prewarm: boolean; + comparisonRequested: boolean; + previousResponseIdPresent: boolean; + breakpointCount: number; + inputItemCount: number; + toolCount: number; +} + +function hasPromptCacheObservationCore( + value: Record, +): value is Record & PromptCacheObservationCore { + return typeof value.keyPresent === "boolean" + && typeof value.prewarm === "boolean" + && typeof value.comparisonRequested === "boolean" + && typeof value.previousResponseIdPresent === "boolean" + && isBoundedCount(value.breakpointCount) + && isBoundedCount(value.inputItemCount) + && isBoundedCount(value.toolCount); +} + +function normalizedPromptCacheMode(value: unknown): PromptCacheMode | undefined { + if (value === "default" || value === "implicit" || value === "explicit") return value; + return undefined; +} + +function normalizedPromptCacheTtl(value: unknown): PromptCacheRequestObservation["ttl"] { + return value === "30m" ? "30m" : undefined; +} + +function normalizedPromptCacheExtras( + value: Record, +): PromptCacheObservationExtras { + const extras: PromptCacheObservationExtras = {}; + if (isFingerprint(value.toolsFingerprint)) { + extras.toolsFingerprint = value.toolsFingerprint; + } + if (isFingerprint(value.stablePrefixFingerprint)) { + extras.stablePrefixFingerprint = value.stablePrefixFingerprint; + } + if (isFingerprint(value.textFormatFingerprint)) { + extras.textFormatFingerprint = value.textFormatFingerprint; + } + const verbosity = boundedString(value.verbosity); + if (verbosity) extras.verbosity = verbosity; + return extras; +} + export function normalizePromptCacheRequestObservation( value: unknown, ): PromptCacheRequestObservation | undefined { if (!isRecord(value) || value.version !== 1) return undefined; - if (typeof value.keyPresent !== "boolean" - || typeof value.prewarm !== "boolean" - || typeof value.comparisonRequested !== "boolean" - || typeof value.previousResponseIdPresent !== "boolean" - || !isBoundedCount(value.breakpointCount) - || !isBoundedCount(value.inputItemCount) - || !isBoundedCount(value.toolCount)) { - return undefined; - } - const mode = value.mode === "default" - || value.mode === "implicit" - || value.mode === "explicit" - ? value.mode - : undefined; + if (!hasPromptCacheObservationCore(value)) return undefined; + const mode = normalizedPromptCacheMode(value.mode); if (!mode) return undefined; - const ttl = value.ttl === "30m" ? value.ttl : undefined; - const legacyRetention = value.legacyRetention === "in_memory" - || value.legacyRetention === "24h" - ? value.legacyRetention - : undefined; - const verbosity = boundedString(value.verbosity); + const ttl = normalizedPromptCacheTtl(value.ttl); + const legacyRetention = promptCacheLegacyRetention(value.legacyRetention); return { version: 1, keyPresent: value.keyPresent, @@ -198,12 +265,6 @@ export function normalizePromptCacheRequestObservation( breakpointCount: value.breakpointCount, inputItemCount: value.inputItemCount, toolCount: value.toolCount, - ...(isFingerprint(value.toolsFingerprint) - ? { toolsFingerprint: value.toolsFingerprint } : {}), - ...(isFingerprint(value.stablePrefixFingerprint) - ? { stablePrefixFingerprint: value.stablePrefixFingerprint } : {}), - ...(isFingerprint(value.textFormatFingerprint) - ? { textFormatFingerprint: value.textFormatFingerprint } : {}), - ...(verbosity ? { verbosity } : {}), + ...normalizedPromptCacheExtras(value), }; } From 54e565df91a0721106da3f36476daa14465a3a4b Mon Sep 17 00:00:00 2001 From: chefgroep Date: Sun, 4 Oct 2026 01:04:39 +0200 Subject: [PATCH 4/5] fix: harden prompt cache evidence parsing --- scripts/analyze-prompt-cache-usage.ts | 20 +++++++--- src/prompt-cache/observability.ts | 19 +++++----- tests/prompt-cache-analyzer.test.ts | 48 ++++++++++++++++++++++++ tests/prompt-cache-observability.test.ts | 21 +++++++++++ 4 files changed, 92 insertions(+), 16 deletions(-) diff --git a/scripts/analyze-prompt-cache-usage.ts b/scripts/analyze-prompt-cache-usage.ts index 2f92b5357..09d400d77 100644 --- a/scripts/analyze-prompt-cache-usage.ts +++ b/scripts/analyze-prompt-cache-usage.ts @@ -102,6 +102,10 @@ function isRecord(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } +function hasOwn(value: Record, key: string): boolean { + return Object.prototype.hasOwnProperty.call(value, key); +} + function nonNegativeNumber(value: unknown): number | undefined { return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value @@ -184,13 +188,17 @@ function cacheUsage(row: Record): CacheUsage | undefined { const input = nonNegativeNumber(row.usage.inputTokens); if (input === undefined) return undefined; const write = nonNegativeNumber(row.usage.cacheCreationInputTokens) ?? 0; - const explicitRead = nonNegativeNumber(row.usage.cacheReadInputTokens); - const legacyCached = nonNegativeNumber(row.usage.cachedInputTokens); - const read = explicitRead - ?? (legacyCached !== undefined && row.usage.cacheCreationInputTokens !== undefined + let read: number; + if (hasOwn(row.usage, "cacheReadInputTokens")) { + const explicitRead = nonNegativeNumber(row.usage.cacheReadInputTokens); + if (explicitRead === undefined) return undefined; + read = explicitRead; + } else { + const legacyCached = nonNegativeNumber(row.usage.cachedInputTokens); + read = (legacyCached !== undefined && row.usage.cacheCreationInputTokens !== undefined ? Math.max(0, legacyCached - write) - : legacyCached) - ?? 0; + : legacyCached) ?? 0; + } if (read + write > input) return undefined; return { input, read, write }; } diff --git a/src/prompt-cache/observability.ts b/src/prompt-cache/observability.ts index 1a9b8f20f..13d8cd153 100644 --- a/src/prompt-cache/observability.ts +++ b/src/prompt-cache/observability.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; export type PromptCacheMode = "default" | "implicit" | "explicit"; export type PromptCacheLegacyRetention = "in_memory" | "24h"; +export type PromptCacheVerbosity = "low" | "medium" | "high"; export interface PromptCacheRequestObservation { version: 1; @@ -18,7 +19,7 @@ export interface PromptCacheRequestObservation { toolsFingerprint?: string; stablePrefixFingerprint?: string; textFormatFingerprint?: string; - verbosity?: string; + verbosity?: PromptCacheVerbosity; } function isRecord(value: unknown): value is Record { @@ -94,11 +95,9 @@ function stablePrefix(body: Record): unknown[] { return prefix; } -function boundedString(value: unknown, maxLength = 32): string | undefined { - if (typeof value !== "string") return undefined; - const trimmed = value.trim(); - if (!trimmed) return undefined; - return trimmed.slice(0, maxLength); +function promptCacheVerbosity(value: unknown): PromptCacheVerbosity | undefined { + if (value === "low" || value === "medium" || value === "high") return value; + return undefined; } function hasNonEmptyString(value: unknown): boolean { @@ -131,14 +130,14 @@ interface PromptCacheObservationExtras { toolsFingerprint?: string; stablePrefixFingerprint?: string; textFormatFingerprint?: string; - verbosity?: string; + verbosity?: PromptCacheVerbosity; } function promptCacheObservationExtras(input: { tools: unknown[]; prefix: unknown[]; textFormat: unknown; - verbosity: string | undefined; + verbosity: PromptCacheVerbosity | undefined; }): PromptCacheObservationExtras { const extras: PromptCacheObservationExtras = {}; if (input.tools.length > 0) extras.toolsFingerprint = fingerprint(input.tools); @@ -165,7 +164,7 @@ export function observeOpenAiResponsesPromptCache( const prefix = stablePrefix(value); const text = isRecord(value.text) ? value.text : undefined; const textFormat = text?.format; - const verbosity = boundedString(text?.verbosity); + const verbosity = promptCacheVerbosity(text?.verbosity); return { version: 1, @@ -238,7 +237,7 @@ function normalizedPromptCacheExtras( if (isFingerprint(value.textFormatFingerprint)) { extras.textFormatFingerprint = value.textFormatFingerprint; } - const verbosity = boundedString(value.verbosity); + const verbosity = promptCacheVerbosity(value.verbosity); if (verbosity) extras.verbosity = verbosity; return extras; } diff --git a/tests/prompt-cache-analyzer.test.ts b/tests/prompt-cache-analyzer.test.ts index 65c33917e..e97f67f02 100644 --- a/tests/prompt-cache-analyzer.test.ts +++ b/tests/prompt-cache-analyzer.test.ts @@ -132,6 +132,54 @@ describe("prompt-cache usage analyzer", () => { ]); }); + test("does not treat malformed explicit cache-read usage as legacy cache data", () => { + const path = tempUsageFile([ + { + requestId: "malformed-explicit", + timestamp: Date.parse("2026-10-02T10:00:00Z"), + provider: "openai", + model: "gpt-5.6-terra", + status: 200, + usageStatus: "reported", + usage: { + inputTokens: 100, + outputTokens: 1, + cacheReadInputTokens: "not-a-number", + cachedInputTokens: 90, + }, + }, + { + requestId: "legacy-valid", + timestamp: Date.parse("2026-10-02T11:00:00Z"), + provider: "openai", + model: "gpt-5.6-terra", + status: 200, + usageStatus: "reported", + usage: { + inputTokens: 100, + outputTokens: 1, + cachedInputTokens: 60, + }, + }, + ]); + + const result = run( + path, + "--range=7d", + "--now=2026-10-03T12:00:00Z", + "--json", + ); + expect(result.exitCode).toBe(0); + const output = JSON.parse(result.stdout.toString()); + expect(output.rows).toBe(2); + expect(output.summary).toMatchObject({ + reportedSuccess: 1, + inputTokens: 100, + cacheReadTokens: 60, + cacheReadRatio: 0.6, + }); + }); + test("rejects invalid CLI arguments instead of silently changing the analysis", () => { const path = tempUsageFile([]); const invalidRange = run(path, "--range=week"); diff --git a/tests/prompt-cache-observability.test.ts b/tests/prompt-cache-observability.test.ts index 18805b733..a0d246dda 100644 --- a/tests/prompt-cache-observability.test.ts +++ b/tests/prompt-cache-observability.test.ts @@ -80,6 +80,27 @@ describe("prompt cache observability", () => { expect(serialized).not.toContain("private-format"); }); + test("only persists supported Responses verbosity values", () => { + expect(observeOpenAiResponsesPromptCache({ + text: { verbosity: "low" }, + })?.verbosity).toBe("low"); + expect(observeOpenAiResponsesPromptCache({ + text: { verbosity: "medium" }, + })?.verbosity).toBe("medium"); + expect(observeOpenAiResponsesPromptCache({ + text: { verbosity: "high" }, + })?.verbosity).toBe("high"); + + const unsupported = observeOpenAiResponsesPromptCache({ + text: { verbosity: "private caller-controlled marker" }, + }); + expect(unsupported).not.toHaveProperty("verbosity"); + expect(normalizePromptCacheRequestObservation({ + ...unsupported, + verbosity: "private persisted marker", + })).not.toHaveProperty("verbosity"); + }); + test("fingerprints are stable for object-key order but preserve tool array order", () => { const a = observeOpenAiResponsesPromptCache({ instructions: "stable", From e9ae3a4b0c8d7d75af4ba44fec80705e24eca575 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:56:54 +0000 Subject: [PATCH 5/5] docs: document prompt cache observability and usage analyzer helpers --- scripts/analyze-prompt-cache-usage.ts | 170 ++++++++++++++++++++------ src/prompt-cache/observability.ts | 99 +++++++++++---- 2 files changed, 205 insertions(+), 64 deletions(-) diff --git a/scripts/analyze-prompt-cache-usage.ts b/scripts/analyze-prompt-cache-usage.ts index 09d400d77..21b70ff82 100644 --- a/scripts/analyze-prompt-cache-usage.ts +++ b/scripts/analyze-prompt-cache-usage.ts @@ -85,6 +85,7 @@ interface ShapeSummary extends CohortDimensions { cacheWriteRatio: number; } +/** Describe the analyzer CLI options and their defaults. */ function usageText(): string { return [ "Usage: bun scripts/analyze-prompt-cache-usage.ts [usage.jsonl|-] [options]", @@ -98,36 +99,44 @@ function usageText(): string { ].join("\n"); } +/** Narrow parsed JSON values to non-null, non-array objects. */ function isRecord(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } +/** Check for an explicitly stored field without consulting the prototype chain. */ function hasOwn(value: Record, key: string): boolean { return Object.prototype.hasOwnProperty.call(value, key); } +/** Return finite, non-negative numeric usage values, or undefined for invalid data. */ function nonNegativeNumber(value: unknown): number | undefined { return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; } +/** Trim a string dimension, returning undefined when it is absent or blank. */ function stringValue(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } +/** Accept a supported analysis window; throw for an unknown range. */ function parseRange(value: string): RangeName { if (value === "7d" || value === "30d" || value === "all") return value; throw new Error(`invalid --range value: ${value}`); } +/** Parse a decimal result limit from 1 through 200; throw for invalid input. */ function parseTop(value: string): number { if (!/^\d+$/u.test(value)) throw new Error(`invalid --top value: ${value}`); const parsed = Number.parseInt(value, 10); - if (parsed < 1 || parsed > 200) throw new Error("--top must be between 1 and 200"); + if (parsed < 1 || parsed > 200) + throw new Error("--top must be between 1 and 200"); return parsed; } +/** Parse a non-negative epoch-millisecond or date-string anchor; throw if invalid. */ function parseNow(value: string): number { if (/^\d+$/u.test(value)) { const numeric = Number(value); @@ -138,6 +147,7 @@ function parseNow(value: string): number { throw new Error(`invalid --now value: ${value}`); } +/** Resolve CLI options and the usage-log path; throw on invalid or extra arguments. */ function parseArgs(args: string[]): AnalyzerOptions { const defaultSource = join( process.env.OPENCODEX_HOME?.trim() || join(homedir(), ".opencodex"), @@ -173,7 +183,8 @@ function parseArgs(args: string[]): AnalyzerOptions { continue; } if (arg.startsWith("--")) throw new Error(`unknown option: ${arg}`); - if (positionalSeen) throw new Error("only one usage-log path may be supplied"); + if (positionalSeen) + throw new Error("only one usage-log path may be supplied"); source = arg; positionalSeen = true; } @@ -181,8 +192,17 @@ function parseArgs(args: string[]): AnalyzerOptions { return { source, range, top, nowMs, jsonMode, help }; } +/** + * Read cache totals only from successful, provider-reported usage rows. + * Prefer explicit reads over legacy cached totals, subtract legacy writes when present, + * and reject malformed reads or cache totals exceeding input tokens. + */ function cacheUsage(row: Record): CacheUsage | undefined { - if (row.status !== 200 || row.usageStatus !== "reported" || !isRecord(row.usage)) { + if ( + row.status !== 200 || + row.usageStatus !== "reported" || + !isRecord(row.usage) + ) { return undefined; } const input = nonNegativeNumber(row.usage.inputTokens); @@ -195,28 +215,34 @@ function cacheUsage(row: Record): CacheUsage | undefined { read = explicitRead; } else { const legacyCached = nonNegativeNumber(row.usage.cachedInputTokens); - read = (legacyCached !== undefined && row.usage.cacheCreationInputTokens !== undefined - ? Math.max(0, legacyCached - write) - : legacyCached) ?? 0; + read = + (legacyCached !== undefined && + row.usage.cacheCreationInputTokens !== undefined + ? Math.max(0, legacyCached - write) + : legacyCached) ?? 0; } if (read + write > input) return undefined; return { input, read, write }; } +/** Compute the inclusive window start in milliseconds, using epoch zero for all history. */ function sinceFor(range: RangeName, nowMs: number): number { if (range === "all") return 0; return nowMs - (range === "7d" ? 7 : 30) * 86_400_000; } +/** Extract cohort dimensions, preferring the resolved model and marking missing values unknown. */ function dimensionsFrom(row: Record): CohortDimensions { return { adapter: stringValue(row.adapter) ?? "unknown", provider: stringValue(row.provider) ?? "unknown", - model: stringValue(row.resolvedModel) ?? stringValue(row.model) ?? "unknown", + model: + stringValue(row.resolvedModel) ?? stringValue(row.model) ?? "unknown", surface: stringValue(row.surface) ?? "unknown", }; } +/** Encode adapter, provider, model, and surface as an unambiguous cohort key. */ function dimensionsKey(dimensions: CohortDimensions): string { return JSON.stringify([ dimensions.adapter, @@ -226,6 +252,7 @@ function dimensionsKey(dimensions: CohortDimensions): string { ]); } +/** Initialize usage and conversation counters for a cohort. */ function emptyCohort(dimensions: CohortDimensions): Cohort { return { ...dimensions, @@ -241,10 +268,12 @@ function emptyCohort(dimensions: CohortDimensions): Cohort { }; } +/** Divide by a positive denominator, returning zero when no measurable denominator exists. */ function ratio(numerator: number, denominator: number): number { return denominator > 0 ? numerator / denominator : 0; } +/** Classify measured reuse using cache-read, cache-write, and input-volume thresholds. */ function signal(cohort: Cohort): CohortSignal { const readRatio = ratio(cohort.cacheRead, cohort.input); const writeRatio = ratio(cohort.cacheWrite, cohort.input); @@ -254,6 +283,7 @@ function signal(cohort: Cohort): CohortSignal { return "mixed"; } +/** Convert cohort counters into token totals, ratios, and conversation counts. */ function summarizeCohort(cohort: Cohort): CohortSummary { let multiTurnConversations = 0; for (const count of cohort.conversations.values()) { @@ -280,6 +310,7 @@ function summarizeCohort(cohort: Cohort): CohortSummary { }; } +/** Encode the cache settings and fingerprints used to group comparable request shapes. */ function cacheShapeKey(observation: PromptCacheRequestObservation): string { return JSON.stringify([ observation.mode, @@ -293,6 +324,7 @@ function cacheShapeKey(observation: PromptCacheRequestObservation): string { ]); } +/** Summarize a shape bucket, measuring tokens only from valid reported-success rows. */ function summarizeShape(bucket: ShapeBucket): ShapeSummary { const pc = bucket.observation; let input = 0; @@ -328,6 +360,7 @@ function summarizeShape(bucket: ShapeBucket): ShapeSummary { }; } +/** Render a fractional ratio as a percentage with one decimal place. */ function formatPercent(value: number): string { return (value * 100).toFixed(1) + "%"; } @@ -350,12 +383,14 @@ interface AnalysisState { totals: AnalysisTotals; } +/** Read UTF-8 usage JSONL from the selected file or stdin; propagate read errors. */ function readUsageSource(options: AnalyzerOptions): string { return options.source === "-" ? readFileSync(0, "utf8") : readFileSync(options.source, "utf8"); } +/** Parse JSONL objects, skipping blank lines and counting malformed or non-object rows. */ function parseUsageRows(text: string): ParsedUsageRows { const rows: Record[] = []; let invalidLines = 0; @@ -372,18 +407,23 @@ function parseUsageRows(text: string): ParsedUsageRows { return { rows, invalidLines }; } +/** Select rows with valid timestamps inside the inclusive analysis window. */ function rowsInWindow( rows: readonly Record[], since: number, nowMs: number, ): Record[] { - return rows.filter(row => { + return rows.filter((row) => { const timestamp = nonNegativeNumber(row.timestamp); return timestamp !== undefined && timestamp >= since && timestamp <= nowMs; }); } -function recordConversation(cohort: Cohort, row: Record): void { +/** Increment a cohort conversation count when the row has a non-blank identifier. */ +function recordConversation( + cohort: Cohort, + row: Record, +): void { const conversationId = stringValue(row.conversationId); if (!conversationId) return; cohort.conversations.set( @@ -392,6 +432,7 @@ function recordConversation(cohort: Cohort, row: Record): void ); } +/** Accumulate validated usage into cohort and global counters, ignoring absent measurements. */ function recordCacheUsage( cohort: Cohort, usage: CacheUsage | undefined, @@ -411,6 +452,7 @@ function recordCacheUsage( totals.write += usage.write; } +/** Validate a row observation and add it to the matching cohort and cache-shape bucket. */ function recordCacheShape( buckets: Map, key: string, @@ -428,6 +470,7 @@ function recordCacheShape( buckets.set(shapeKey, { dimensions, observation, rows: [row] }); } +/** Group selected rows into cohorts and shapes while accumulating valid usage totals. */ function analyzeRows(rows: readonly Record[]): AnalysisState { const cohorts = new Map(); const shapeBuckets = new Map(); @@ -452,25 +495,31 @@ function analyzeRows(rows: readonly Record[]): AnalysisState { return { cohorts, shapeBuckets, totals }; } -function topCohorts(cohorts: Map, top: number): CohortSummary[] { +/** Select cohorts with measured successes, ordered by descending input tokens and capped at top. */ +function topCohorts( + cohorts: Map, + top: number, +): CohortSummary[] { return [...cohorts.values()] - .filter(cohort => cohort.reportedSuccess > 0) + .filter((cohort) => cohort.reportedSuccess > 0) .sort((a, b) => b.input - a.input) .slice(0, top) .map(summarizeCohort); } +/** Select cache shapes with measured successes, ordered by descending input tokens and capped at top. */ function topShapes( buckets: Map, top: number, ): ShapeSummary[] { return [...buckets.values()] .map(summarizeShape) - .filter(shape => shape.reportedSuccess > 0) + .filter((shape) => shape.reportedSuccess > 0) .sort((a, b) => b.inputTokens - a.inputTokens) .slice(0, top); } +/** Assemble window metadata, the measurement boundary, totals, and ranked summaries. */ function buildOutput( options: AnalyzerOptions, since: number, @@ -507,66 +556,109 @@ function buildOutput( type AnalyzerOutput = ReturnType; +/** Print the analysis window, aggregate usage, ranked cohorts, and cache shapes. */ function printHumanOutput(output: AnalyzerOutput): void { console.log("Prompt cache usage (" + output.range + ")"); console.log("window: " + output.windowStart + " .. " + output.windowEnd); console.log("proof: " + output.proofBoundary); console.log( - "reported-success=" + output.summary.reportedSuccess - + " input=" + Math.round(output.summary.inputTokens) - + " read=" + Math.round(output.summary.cacheReadTokens) - + " (" + formatPercent(output.summary.cacheReadRatio) + ")" - + " write=" + Math.round(output.summary.cacheWriteTokens) - + " (" + formatPercent(output.summary.cacheWriteRatio) + ")", + "reported-success=" + + output.summary.reportedSuccess + + " input=" + + Math.round(output.summary.inputTokens) + + " read=" + + Math.round(output.summary.cacheReadTokens) + + " (" + + formatPercent(output.summary.cacheReadRatio) + + ")" + + " write=" + + Math.round(output.summary.cacheWriteTokens) + + " (" + + formatPercent(output.summary.cacheWriteRatio) + + ")", ); console.log(""); console.log("Top cohorts by measured input:"); for (const item of output.cohorts) { console.log( - item.signal.padEnd(11) - + " " + item.adapter + "/" + item.provider + "/" + item.model - + " surface=" + item.surface - + " input=" + item.inputTokens - + " read=" + formatPercent(item.cacheReadRatio) - + " write=" + formatPercent(item.cacheWriteRatio) - + " hits=" + formatPercent(item.cacheHitRequestRatio) - + " n=" + item.reportedSuccess + "/" + item.requests, + item.signal.padEnd(11) + + " " + + item.adapter + + "/" + + item.provider + + "/" + + item.model + + " surface=" + + item.surface + + " input=" + + item.inputTokens + + " read=" + + formatPercent(item.cacheReadRatio) + + " write=" + + formatPercent(item.cacheWriteRatio) + + " hits=" + + formatPercent(item.cacheHitRequestRatio) + + " n=" + + item.reportedSuccess + + "/" + + item.requests, ); } printCacheShapes(output.cacheShapes); } +/** Print measured cache shapes or explain that the window contains no observations. */ function printCacheShapes(shapes: readonly ShapeSummary[]): void { console.log(""); if (shapes.length === 0) { - console.log("No promptCache observations in this window (historical rows predate instrumentation)."); + console.log( + "No promptCache observations in this window (historical rows predate instrumentation).", + ); return; } console.log("Observed outbound cache shapes:"); for (const item of shapes) { console.log( - item.adapter + "/" + item.provider + "/" + item.model - + " surface=" + item.surface - + " mode=" + item.mode - + " tools=" + item.toolCount - + " bp=" + item.breakpointCount - + " input=" + item.inputTokens - + " read=" + formatPercent(item.cacheReadRatio) - + " write=" + formatPercent(item.cacheWriteRatio) - + " n=" + item.reportedSuccess + "/" + item.requests - + " prefix=" + (item.stablePrefixFingerprint ?? "-") - + " toolsHash=" + (item.toolsFingerprint ?? "-"), + item.adapter + + "/" + + item.provider + + "/" + + item.model + + " surface=" + + item.surface + + " mode=" + + item.mode + + " tools=" + + item.toolCount + + " bp=" + + item.breakpointCount + + " input=" + + item.inputTokens + + " read=" + + formatPercent(item.cacheReadRatio) + + " write=" + + formatPercent(item.cacheWriteRatio) + + " n=" + + item.reportedSuccess + + "/" + + item.requests + + " prefix=" + + (item.stablePrefixFingerprint ?? "-") + + " toolsHash=" + + (item.toolsFingerprint ?? "-"), ); } } +/** Run the analyzer and return zero on success or help, or two for invalid CLI arguments. */ function main(): number { let options: AnalyzerOptions; try { options = parseArgs(Bun.argv.slice(2)); } catch (error) { - const message = error instanceof Error ? error.message : "invalid arguments"; + const message = + error instanceof Error ? error.message : "invalid arguments"; console.error(message); console.error(usageText()); return 2; diff --git a/src/prompt-cache/observability.ts b/src/prompt-cache/observability.ts index 13d8cd153..edbec6a21 100644 --- a/src/prompt-cache/observability.ts +++ b/src/prompt-cache/observability.ts @@ -22,6 +22,7 @@ export interface PromptCacheRequestObservation { verbosity?: PromptCacheVerbosity; } +/** Narrow a value to a non-null, non-array object before reading request fields. */ function isRecord(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } @@ -33,7 +34,7 @@ function isRecord(value: unknown): value is Record { function canonicalJson(value: unknown): string | undefined { if (value === null) return "null"; if (Array.isArray(value)) { - const items = value.map(item => canonicalJson(item) ?? "null"); + const items = value.map((item) => canonicalJson(item) ?? "null"); return "[" + items.join(",") + "]"; } if (isRecord(value)) { @@ -47,24 +48,29 @@ function canonicalJson(value: unknown): string | undefined { } if (typeof value === "number" && !Number.isFinite(value)) return "null"; if ( - typeof value === "string" - || typeof value === "number" - || typeof value === "boolean" + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" ) { return JSON.stringify(value); } return undefined; } +/** Hash canonical JSON to 24 lowercase hex characters, or return undefined for unsupported values. */ function fingerprint(value: unknown): string | undefined { const serialized = canonicalJson(value); if (serialized === undefined) return undefined; return createHash("sha256").update(serialized).digest("hex").slice(0, 24); } +/** Recursively count explicit prompt-cache breakpoints without descending into breakpoint metadata. */ function countExplicitBreakpoints(value: unknown): number { if (Array.isArray(value)) { - return value.reduce((total, item) => total + countExplicitBreakpoints(item), 0); + return value.reduce( + (total, item) => total + countExplicitBreakpoints(item), + 0, + ); } if (!isRecord(value)) return 0; const breakpoint = value.prompt_cache_breakpoint; @@ -76,6 +82,7 @@ function countExplicitBreakpoints(value: unknown): number { return count; } +/** Collect instructions and consecutive leading developer/system messages for fingerprinting. */ function stablePrefix(body: Record): unknown[] { const prefix: unknown[] = []; if (body.instructions !== undefined) { @@ -90,37 +97,52 @@ function stablePrefix(body: Record): unknown[] { initialDeveloperItems.push(item); } if (initialDeveloperItems.length > 0) { - prefix.push({ kind: "initial_developer_messages", value: initialDeveloperItems }); + prefix.push({ + kind: "initial_developer_messages", + value: initialDeveloperItems, + }); } return prefix; } -function promptCacheVerbosity(value: unknown): PromptCacheVerbosity | undefined { +/** Accept only supported verbosity labels so arbitrary caller text is omitted from diagnostics. */ +function promptCacheVerbosity( + value: unknown, +): PromptCacheVerbosity | undefined { if (value === "low" || value === "medium" || value === "high") return value; return undefined; } +/** Check for non-whitespace string content without retaining the string value. */ function hasNonEmptyString(value: unknown): boolean { return typeof value === "string" && value.trim().length > 0; } -function promptCacheMode(options: Record | undefined): PromptCacheMode { +/** Read a recognized outbound cache mode, falling back to default for absent or unknown values. */ +function promptCacheMode( + options: Record | undefined, +): PromptCacheMode { if (options?.mode === "explicit") return "explicit"; if (options?.mode === "implicit") return "implicit"; return "default"; } +/** Extract the supported 30-minute TTL from outbound cache options. */ function promptCacheTtl( options: Record | undefined, ): PromptCacheRequestObservation["ttl"] { return options?.ttl === "30m" ? "30m" : undefined; } -function promptCacheLegacyRetention(value: unknown): PromptCacheLegacyRetention | undefined { +/** Accept only the supported legacy retention labels, omitting unknown values. */ +function promptCacheLegacyRetention( + value: unknown, +): PromptCacheLegacyRetention | undefined { if (value === "in_memory" || value === "24h") return value; return undefined; } +/** Count input-array items, treating absent input as zero and other input values as one. */ function promptCacheInputItemCount(input: unknown): number { if (Array.isArray(input)) return input.length; return input === undefined ? 0 : 1; @@ -133,6 +155,7 @@ interface PromptCacheObservationExtras { verbosity?: PromptCacheVerbosity; } +/** Fingerprint present tools, stable prefix, and text format, and include supported verbosity. */ function promptCacheObservationExtras(input: { tools: unknown[]; prefix: unknown[]; @@ -140,8 +163,10 @@ function promptCacheObservationExtras(input: { verbosity: PromptCacheVerbosity | undefined; }): PromptCacheObservationExtras { const extras: PromptCacheObservationExtras = {}; - if (input.tools.length > 0) extras.toolsFingerprint = fingerprint(input.tools); - if (input.prefix.length > 0) extras.stablePrefixFingerprint = fingerprint(input.prefix); + if (input.tools.length > 0) + extras.toolsFingerprint = fingerprint(input.tools); + if (input.prefix.length > 0) + extras.stablePrefixFingerprint = fingerprint(input.prefix); if (input.textFormat !== undefined) { extras.textFormatFingerprint = fingerprint(input.textFormat); } @@ -149,6 +174,11 @@ function promptCacheObservationExtras(input: { return extras; } +/** + * Describe cache settings and request structure from the final outbound Responses body. + * Retain fingerprints and presence flags rather than raw content or cache keys. + * Return undefined for non-object input without changing the supplied body. + */ export function observeOpenAiResponsesPromptCache( value: unknown, ): PromptCacheRequestObservation | undefined { @@ -182,15 +212,19 @@ export function observeOpenAiResponsesPromptCache( }; } +/** Check that a persisted fingerprint contains exactly 24 lowercase hexadecimal characters. */ function isFingerprint(value: unknown): value is string { return typeof value === "string" && /^[0-9a-f]{24}$/u.test(value); } +/** Accept persisted integer counts from zero through one million. */ function isBoundedCount(value: unknown): value is number { - return typeof value === "number" - && Number.isInteger(value) - && value >= 0 - && value <= 1_000_000; + return ( + typeof value === "number" && + Number.isInteger(value) && + value >= 0 && + value <= 1_000_000 + ); } interface PromptCacheObservationCore { @@ -203,27 +237,38 @@ interface PromptCacheObservationCore { toolCount: number; } +/** Validate the required boolean flags and bounded counts of a persisted observation. */ function hasPromptCacheObservationCore( value: Record, ): value is Record & PromptCacheObservationCore { - return typeof value.keyPresent === "boolean" - && typeof value.prewarm === "boolean" - && typeof value.comparisonRequested === "boolean" - && typeof value.previousResponseIdPresent === "boolean" - && isBoundedCount(value.breakpointCount) - && isBoundedCount(value.inputItemCount) - && isBoundedCount(value.toolCount); + return ( + typeof value.keyPresent === "boolean" && + typeof value.prewarm === "boolean" && + typeof value.comparisonRequested === "boolean" && + typeof value.previousResponseIdPresent === "boolean" && + isBoundedCount(value.breakpointCount) && + isBoundedCount(value.inputItemCount) && + isBoundedCount(value.toolCount) + ); } -function normalizedPromptCacheMode(value: unknown): PromptCacheMode | undefined { - if (value === "default" || value === "implicit" || value === "explicit") return value; +/** Accept a persisted cache-mode label, returning undefined for unrecognized values. */ +function normalizedPromptCacheMode( + value: unknown, +): PromptCacheMode | undefined { + if (value === "default" || value === "implicit" || value === "explicit") + return value; return undefined; } -function normalizedPromptCacheTtl(value: unknown): PromptCacheRequestObservation["ttl"] { +/** Retain only the supported 30-minute TTL from a persisted observation. */ +function normalizedPromptCacheTtl( + value: unknown, +): PromptCacheRequestObservation["ttl"] { return value === "30m" ? "30m" : undefined; } +/** Copy only well-formed fingerprints and supported verbosity from persisted metadata. */ function normalizedPromptCacheExtras( value: Record, ): PromptCacheObservationExtras { @@ -242,6 +287,10 @@ function normalizedPromptCacheExtras( return extras; } +/** + * Validate untrusted persisted metadata and rebuild an allowlisted version-1 observation. + * Reject invalid versions or required fields; omit unknown fields and invalid optional values. + */ export function normalizePromptCacheRequestObservation( value: unknown, ): PromptCacheRequestObservation | undefined {