[stack 4/8] feat(trace): opt-in trace store linked to usage.jsonl

docs-site/src/content/docs/reference/cli.md

@@ -511,6 +511,32 @@ With no scope, `ocx debug` prints usage and, when the proxy is stopped, the next
 defaults. Provider debug defaults from `OCX_DEBUG=1` (legacy `OCX_DEBUG_FRAMES=1` also works); usage
 debug defaults from `OPENCODEX_USAGE_DEBUG=1`.
 
+## Request traces
+
+`usage.jsonl` stays a compact telemetry log. Full prompts and responses go to a separate local store,
+`trace.sqlite` in the opencodex config directory, linked to usage rows by `traceId` (the request id).
+Tracing is **off by default** and is configured by environment variable at proxy start:
+
+| Variable                   | Default  | Meaning                                                                 |
+| -------------------------- | -------- | ----------------------------------------------------------------------- |
+| `OCX_TRACE`                | `off`    | `off`, `metadata`, `redacted`, or `full`.                               |
+| `OCX_TRACE_TTL_HOURS`      | `24`     | Retention for stored traces (1–720).                                    |
+| `OCX_TRACE_MAX_BODY_BYTES` | `524288` | Cap per stored body. Hashes and byte counts always cover the full body. |
+| `OCX_TRACE_MAX_DB_MB`      | `256`    | Total store cap; oldest traces are evicted first.                       |
+| `OCX_TRACE_SAMPLE`         | `1`      | Fraction of requests whose bodies are stored in `redacted`/`full` mode. |
+
+- `metadata` stores nothing but adds a payload-free `trace` object to each usage row: byte sizes,
+  message/tool-call/attachment counts, request/outbound/response hashes, and `systemHash`, `toolsHash`
+  and `prefixHash` of the final provider wire body. Comparing these between two turns of one conversation
+  shows which section broke a provider prompt-cache prefix.
+- `redacted` also stores the inbound request, the final provider body, and the response, with credentials
+  and token-shaped values removed. Free-text secrets in prompts that do not match a known pattern are not
+  detectable, so treat the store as sensitive.
+- `full` stores bodies verbatim. Use it deliberately and briefly.
+
+Outbound capture covers adapters that send through the shared upstream fetch helper; the inbound request
+and response are captured for `/v1/responses`, `/v1/messages`, and `/v1/chat/completions`.
+
 ## Updating
 
 ### `ocx update`

src/server/index.ts

@@ -1,4 +1,5 @@
 import { markActivity } from "../lib/sidecar-tracker";
+import { beginTrace, runWithTrace } from "../trace/capture";
 import { initServerSentry } from "../telemetry/sentry-server";
 import {
   buildWarmupCompletionFrames,
@@ -1199,7 +1200,8 @@ export function startServer(port?: number) {
         const responsesWorkReq = responsesCacheProbe?.request ?? req;
         const start = Date.now();
         const requestId = nextRequestLogId(start);
-        const logCtx = { model: "unknown", provider: "unknown" };
+        const logCtx: RequestLogContext = { model: "unknown", provider: "unknown" };
+        await beginTrace(logCtx, responsesWorkReq);
         let logged = false;
         const finalizeNativePassthroughLog = (
           status: number,
@@ -1212,7 +1214,7 @@ export function startServer(port?: number) {
           logged = true;
           addFinalRequestLog(requestId, start, logCtx, status, meta);
         };
-        const response = await handleResponses(
+        const response = await runWithTrace(logCtx, () => handleResponses(
           responsesWorkReq,
           config,
           logCtx,
@@ -1234,7 +1236,7 @@ export function startServer(port?: number) {
               });
             },
           },
-        );
+        ));
         responsesCacheProbe?.store(response);
         return withCors(
           responseWithDeferredRequestLog(response, requestId, start, logCtx),
@@ -1342,15 +1344,16 @@ export function startServer(port?: number) {
           model: "unknown",
           provider: "unknown",
         };
+        await beginTrace(logCtx, messagesWorkReq);
         // Logging is finalized inside handleClaudeMessages (Responses-vocab tap on the
         // pre-translation stream + native passthrough callbacks) — do not re-wrap the
         // translated Anthropic stream here.
-        const response = await handleClaudeMessages(
+        const response = await runWithTrace(logCtx, () => handleClaudeMessages(
           messagesWorkReq,
           config,
           logCtx,
           { requestId, start },
-        );
+        ));
         messagesCacheProbe?.store(response);
         return withCors(response, req, config);
       }
@@ -1397,12 +1400,13 @@ export function startServer(port?: number) {
           model: "unknown",
           provider: "unknown",
         };
-        const response = await handleChatCompletions(
+        await beginTrace(logCtx, chatWorkReq);
+        const response = await runWithTrace(logCtx, () => handleChatCompletions(
           chatWorkReq,
           config,
           logCtx,
           { requestId, start },
-        );
+        ));
         chatCacheProbe?.store(response);
         return withCors(response, req, config);
       }

src/server/request-log.ts

@@ -11,6 +11,8 @@ import { readCodexCatalogPath } from "../codex/catalog";
 import type { OcxConfig, OcxUsage } from "../types";
 import type { AdapterRequest } from "../adapters/base";
 import { redactSecretString } from "../lib/redact";
+import { appendTraceResponse, finalizeTrace, type TraceCapture } from "../trace/capture";
+import type { UsageTraceMeta } from "../trace/types";
 import { providerAccountLabel, baseProviderLabel } from "../providers/label";
 import { getAccountSet } from "../oauth/store";
 import {
@@ -109,6 +111,8 @@ export interface RequestLogContext {
   affinity?: "reused" | "new_bind" | "rebound" | "cleared";
   transportPhase?: "pre_headers" | "mid_stream" | "terminal_sse";
   terminalSource?: "upstream" | "synthetic";
+  /** Internal per-request trace capture (src/trace); never persisted or serialized. */
+  trace?: TraceCapture;
 }
 
 export interface RequestLogEntry {
@@ -160,6 +164,10 @@ export interface RequestLogEntry {
   transportPhase?: "pre_headers" | "mid_stream" | "terminal_sse";
   /** Whether the terminal came from a real upstream SSE event or a proxy synthetic tail. */
   terminalSource?: "upstream" | "synthetic";
+  /** Key into trace.sqlite when this request's bodies were stored. */
+  traceId?: string;
+  /** Payload-free trace summary (sizes, counts, hashes). */
+  trace?: UsageTraceMeta;
 }
 
 const requestLog: RequestLogEntry[] = [];
@@ -316,6 +324,8 @@ export function requestLogEntryFromPersistedUsage(entry: PersistedUsageEntry): R
     ...(entry.usage ? { usage: entry.usage } : {}),
     ...(entry.totalTokens !== undefined ? { totalTokens: entry.totalTokens } : {}),
     ...(entry.attempts?.length ? { attempts: entry.attempts } : {}),
+    ...(entry.traceId ? { traceId: entry.traceId } : {}),
+    ...(entry.trace ? { trace: entry.trace } : {}),
   };
 }
 
@@ -432,6 +442,8 @@ export function addRequestLog(entry: RequestLogEntry) {
       ...(entry.totalTokens !== undefined ? { totalTokens: entry.totalTokens } : {}),
       ...(entry.attempts?.length ? { attempts: entry.attempts } : {}),
       ...failureDiagnostics,
+      ...(entry.traceId ? { traceId: entry.traceId } : {}),
+      ...(entry.trace ? { trace: entry.trace } : {}),
     });
   } catch {
     /* request logging must never fail a user request */
@@ -679,6 +691,7 @@ export function inspectResponseLogJson(logCtx: RequestLogContext, text: string):
     /* body may not be JSON; request log metadata is best-effort only */
   }
   captureUpstreamError(logCtx, text);
+  appendTraceResponse(logCtx.trace, text);
   if (isUsageDebugEnabled() && logCtx.usageDebugBodyKind === undefined) {
     logCtx.usageDebugBodyKind = "json";
     logCtx.usageDebugBodySample = truncateForDebug(text);
@@ -695,6 +708,7 @@ export function inspectResponseLogSsePayload(logCtx: RequestLogContext, payload:
     /* SSE block payload may not be JSON; metadata inspection is best-effort */
   }
   captureUpstreamError(logCtx, payload);
+  appendTraceResponse(logCtx.trace, payload);
   if (debugEnabled) {
     if (!sseAlreadyMarked) {
       logCtx.usageDebugBodyKind = "sse";
@@ -904,6 +918,13 @@ export function addFinalRequestLog(
     const loggedUsage = aggregate?.usage ?? existing.usage;
     const usageStatus = aggregate?.status ?? existing.status;
     const totalTokens = aggregate?.totalTokens ?? existing.totalTokens;
+    const traced = finalizeTrace(requestId, logCtx, {
+      ...(logCtx.conversationId ? { conversationId: logCtx.conversationId } : {}),
+      provider,
+      model,
+      status: effectiveStatus,
+      timestamp: start,
+    });
     addLog({
       requestId,
       timestamp: start,
@@ -942,6 +963,8 @@ export function addFinalRequestLog(
       ...(logCtx.affinity ? { affinity: logCtx.affinity } : {}),
       ...(logCtx.transportPhase ? { transportPhase: logCtx.transportPhase } : {}),
       ...(logCtx.terminalSource ? { terminalSource: logCtx.terminalSource } : {}),
+      ...(traced.traceId ? { traceId: traced.traceId } : {}),
+      ...(traced.trace ? { trace: traced.trace } : {}),
     });
     if (isUsageDebugEnabled()) {
       appendUsageDebug({

src/server/responses/fetch-helpers.ts

@@ -1,6 +1,7 @@
 import type { Server } from "bun";
 import type { OcxProviderConfig } from "../../types";
 import type { WsData } from "../ws-bridge";
+import { noteOutboundRequestBody } from "../../trace/capture";
 
 
 export function disableResponsesRequestTimeout(req: Request, server: Pick<Server<WsData>, "timeout"> | undefined): boolean {
@@ -44,6 +45,7 @@ export async function fetchWithHeaderTimeout(
     if (!timeout.signal.aborted) timeout.abort(new DOMException("Timeout elapsed", "TimeoutError"));
   }, timeoutMs);
   const headers = new Headers(init.headers);
+  noteOutboundRequestBody(init.body);
   // Compressed SSE can be held until the decompressor has a complete block. Streaming calls
   // default to identity for low-latency frame delivery, while an explicit caller choice wins.
   if (preferIdentityEncoding && !headers.has("accept-encoding")) {