From f6350491d5fc51be33c3a1c6140fc8d9adebfdc3 Mon Sep 17 00:00:00 2001 From: yzxcj797 Date: Fri, 21 Aug 2026 12:42:42 +0800 Subject: [PATCH 1/2] feat(metrics): surface turn-level spend on turn_metrics and /v1/admin/metrics The budget now debits the LlmCallUsage the harnesses meter, but an operator planning which models to run qm on still had to join spend out of session_llm_requests by hand: turn_metrics carried the cache columns and no cost, no output tokens. Harness turn results gain costUsage {outputTokens, costUsd} alongside cacheUsage -- pi sums its per-call stats, claude reports the SDK's running cost total with the fallback branch flagging unknown cost, and the mock harness reports its deterministic usage. The orchestrator lands both on TurnMetricSample, turn_metrics grows output_tokens and cost_usd (ALTER TABLE ADD COLUMN IF NOT EXISTS, so existing deployments migrate in place), and /v1/admin/metrics returns a spend block: samples, turnsWithKnownCost, costUsdTotal, outputTokensTotal. Follow-up to #586 --- src/admin/metrics-sink.ts | 2 + src/admin/postgres-metrics-sink.ts | 2 + src/api/routes/admin/observability.ts | 13 ++++++ src/core/orchestrator.ts | 3 ++ src/harness/claude-harness.ts | 9 +++- src/harness/harness.ts | 1 + src/harness/mock-harness.ts | 6 ++- src/harness/pi-harness.ts | 21 ++++++++- test/turn-metrics-cost.test.ts | 66 +++++++++++++++++++++++++++ 9 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 test/turn-metrics-cost.test.ts diff --git a/src/admin/metrics-sink.ts b/src/admin/metrics-sink.ts index df9cb1743..8dfba6411 100644 --- a/src/admin/metrics-sink.ts +++ b/src/admin/metrics-sink.ts @@ -36,6 +36,8 @@ export interface TurnMetricSample { cacheRead?: number; cacheWrite?: number; uncachedInput?: number; + outputTokens?: number; + costUsd?: number; } export function cacheHitRatio(s: { cacheRead?: number; cacheWrite?: number; uncachedInput?: number }): number | null { diff --git a/src/admin/postgres-metrics-sink.ts b/src/admin/postgres-metrics-sink.ts index 29187f828..57bb48161 100644 --- a/src/admin/postgres-metrics-sink.ts +++ b/src/admin/postgres-metrics-sink.ts @@ -37,6 +37,8 @@ const COLUMNS: readonly EventColumn[] = [ ["cache_read", "cacheRead", "BIGINT", "number"], ["cache_write", "cacheWrite", "BIGINT", "number"], ["uncached_input", "uncachedInput", "BIGINT", "number"], + ["output_tokens", "outputTokens", "BIGINT", "number"], + ["cost_usd", "costUsd", "DOUBLE PRECISION", "number"], ]; const EXTRA_SCHEMA_STATEMENTS = [ diff --git a/src/api/routes/admin/observability.ts b/src/api/routes/admin/observability.ts index aee13d633..e856cec04 100644 --- a/src/api/routes/admin/observability.ts +++ b/src/api/routes/admin/observability.ts @@ -150,6 +150,18 @@ export async function metrics(ctx: ApiCtx): Promise { uncachedInputTotal, }; + // Turn-level spend, straight off the usage the harnesses already meter: + // output tokens and provider-computed cost land on turn_metrics next to + // the cache telemetry, so an operator no longer joins session_llm_requests + // by hand to know what a scope or model spends. + const costSamples = samples.filter((s) => s.costUsd !== undefined || s.outputTokens !== undefined); + const spend = { + samples: costSamples.length, + turnsWithKnownCost: samples.filter((s) => s.costUsd !== undefined).length, + costUsdTotal: samples.reduce((n, s) => n + (s.costUsd ?? 0), 0), + outputTokensTotal: samples.reduce((n, s) => n + (s.outputTokens ?? 0), 0), + }; + const phaseFields: [string, (s: TurnMetricSample) => number | undefined][] = [ ["total", (s) => s.totalMs], ["ttft", (s) => s.ttftMs], @@ -225,6 +237,7 @@ export async function metrics(ctx: ApiCtx): Promise { series, anatomy, cache, + spend, phases, }); } diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts index 3a17db7ab..a5e86ba4f 100644 --- a/src/core/orchestrator.ts +++ b/src/core/orchestrator.ts @@ -2796,6 +2796,9 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { uncachedInput: result.cacheUsage.uncachedInput, } : {}), + ...(result.costUsage + ? { outputTokens: result.costUsage.outputTokens, costUsd: result.costUsage.costUsd } + : {}), }); const onTurnEnd = memoryStrategy.onTurnEnd?.bind(memoryStrategy); if (!pausing && useMemory && memoryPolicy.capture !== "off" && onTurnEnd) { diff --git a/src/harness/claude-harness.ts b/src/harness/claude-harness.ts index 2edbb26eb..8d09c77b3 100644 --- a/src/harness/claude-harness.ts +++ b/src/harness/claude-harness.ts @@ -764,11 +764,12 @@ export function createClaudeHarness(opts: ClaudeHarnessOptions = {}): Harness { const usageTotals = [...callUsage.values()].reduce( (acc, usage) => { acc.input += usage.input; + acc.output += usage.output; acc.cacheRead += usage.cacheRead; acc.cacheWrite += usage.cacheWrite; return acc; }, - { input: 0, cacheRead: 0, cacheWrite: 0 }, + { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, ); return { reply, @@ -793,6 +794,12 @@ export function createClaudeHarness(opts: ClaudeHarnessOptions = {}): Harness { finalResult.usage.cache_creation_input_tokens, ), }, + // The SDK result carries no cost total, so spend is surfaced only + // when the per-message accumulator saw cost deltas; the fallback + // branch keeps output tokens but reports no known cost. + costUsage: callUsage.size + ? { outputTokens: usageTotals.output, costUsd: lastTotalCostUsd } + : { outputTokens: finalResult.usage.output_tokens ?? 0, costUsd: 0 }, ...(tapeWriteFailed ? { tapeWriteFailed: true } : {}), }; } finally { diff --git a/src/harness/harness.ts b/src/harness/harness.ts index 77940e80a..018e0b6d0 100644 --- a/src/harness/harness.ts +++ b/src/harness/harness.ts @@ -109,6 +109,7 @@ export interface HarnessTurnResult { pausedOnApproval?: boolean; modelCalls?: number; cacheUsage?: { cacheRead: number; cacheWrite: number; uncachedInput: number }; + costUsage?: { outputTokens: number; costUsd: number }; compileMs?: number; tapeWriteFailed?: boolean; } diff --git a/src/harness/mock-harness.ts b/src/harness/mock-harness.ts index 319732612..36ca80574 100644 --- a/src/harness/mock-harness.ts +++ b/src/harness/mock-harness.ts @@ -682,6 +682,10 @@ export function createMockHarness(): Harness { }), { cacheRead: 0, cacheWrite: 0, uncachedInput: 0 }, ); + const costUsage = steps.reduce( + (acc, u) => ({ outputTokens: acc.outputTokens + u.output, costUsd: acc.costUsd + u.costUsd }), + { outputTokens: 0, costUsd: 0 }, + ); const base = collected.length ? { reply, @@ -691,7 +695,7 @@ export function createMockHarness(): Harness { } : { reply, modelCalls }; if (muteReply) base.reply = ""; - return { ...base, ...(silent ? { silent: true as const } : {}), cacheUsage }; + return { ...base, ...(silent ? { silent: true as const } : {}), cacheUsage, costUsage }; }, shouldRespond(detect: HarnessDetectInput): Promise { diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts index 73ff2ba74..6d86e5758 100644 --- a/src/harness/pi-harness.ts +++ b/src/harness/pi-harness.ts @@ -574,6 +574,21 @@ function piUsageToCallUsage(u: PiUsageShape | undefined): LlmCallUsage | null { }; } +function sumCostUsage( + stats: ReadonlyArray<{ usage: LlmCallUsage | null }>, +): { outputTokens: number; costUsd: number } | null { + let saw = false; + let outputTokens = 0; + let costUsd = 0; + for (const s of stats) { + if (!s.usage) continue; + saw = true; + outputTokens += s.usage.output; + costUsd += s.usage.costUsd; + } + return saw ? { outputTokens, costUsd } : null; +} + function sumCacheUsage( stats: ReadonlyArray<{ usage: LlmCallUsage | null }>, ): { cacheRead: number; cacheWrite: number; uncachedInput: number } | null { @@ -1989,6 +2004,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness { }); await checkpointSubturn(finalEntry.seq); const cacheUsage = sumCacheUsage(callStats); + const costUsage = sumCostUsage(callStats); const base = { reply, stopped: true as const, @@ -1996,7 +2012,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness { compileMs, ...(tapeWriteFailed ? { tapeWriteFailed: true } : {}), }; - return cacheUsage ? { ...base, cacheUsage } : base; + return { ...base, ...(cacheUsage ? { cacheUsage } : {}), ...(costUsage ? { costUsage } : {}) }; } const closingText = recoveryDead ? "" : (piLastAssistantTextOrThrow(entry.agentSession) ?? ""); const closingTextWithWaiver = [closingText, grindWaiverNote].filter(Boolean).join("\n\n"); @@ -2011,6 +2027,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness { const pendingApprovals = entry.ref.pendingApprovals ?? []; const modelCalls = entry.ref.modelCalls ?? 0; const cacheUsage = sumCacheUsage(callStats); + const costUsage = sumCostUsage(callStats); const silent = entry.ref.silentRequested ? { silent: true as const } : {}; const base = pendingApprovals.length ? { @@ -2023,7 +2040,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness { compileMs, } : { reply, ...(tapeWriteFailed ? { tapeWriteFailed: true } : {}), modelCalls, ...silent, compileMs }; - return cacheUsage ? { ...base, cacheUsage } : base; + return { ...base, ...(cacheUsage ? { cacheUsage } : {}), ...(costUsage ? { costUsage } : {}) }; } finally { removeIsolatedDirs(entry); } diff --git a/test/turn-metrics-cost.test.ts b/test/turn-metrics-cost.test.ts new file mode 100644 index 000000000..ed15a9035 --- /dev/null +++ b/test/turn-metrics-cost.test.ts @@ -0,0 +1,66 @@ +import "./support/auto-fake-sprites.ts"; + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AddressInfo } from "node:net"; +import { createInsecureTestServer } from "../src/api/server.ts"; +import { buildApp } from "../src/wiring.ts"; +import type { TurnRequest } from "../src/types.ts"; +import { testConfig } from "./support/test-config.ts"; + +function start() { + const built = buildApp(testConfig({ dataDir: mkdtempSync(join(tmpdir(), "cost-obs-")) })); + const server = createInsecureTestServer(built.app, { + admin: built.admin, + sessions: built.sessions, + auditLog: built.auditLog, + metrics: built.metrics, + runs: built.runs, + }); + server.listen(0); + const base = `http://localhost:${(server.address() as AddressInfo).port}`; + return { base, built, close: () => new Promise((r) => server.close(() => r())) }; +} + +const ALICE = { "x-admin-actor": "admin-alice@default-org" }; +const getJson = async (base: string, path: string, headers: Record = ALICE): Promise => + (await fetch(base + path, { headers })).json(); + +test("metrics: turn spend (output tokens + cost) is surfaced on the metrics endpoint", async () => { + const s = start(); + try { + const turn: TurnRequest = { + surface: "test", + actor: { externalId: "U1" }, + conversation: { kind: "dm", threadRef: "dm:U1:c1" }, + text: "hello there", + }; + assert.equal((await s.built.app.turn(turn)).status, "ok"); + + const m = await getJson(s.base, "/v1/admin/metrics?scope=org:default-org"); + assert.ok(m.spend, "the metrics response carries a spend block"); + assert.ok(m.spend.samples >= 1, "the turn carried spend telemetry"); + assert.ok(m.spend.turnsWithKnownCost >= 1, "the mock harness reports known (if zero) cost"); + assert.ok(m.spend.outputTokensTotal > 0, "output tokens are summed per turn"); + assert.equal(typeof m.spend.costUsdTotal, "number", "cost total is numeric even when the mock reports 0"); + } finally { + await s.close(); + } +}); + +test("metrics: an empty scope yields a present-but-empty spend block", async () => { + const s = start(); + try { + const m = await getJson(s.base, "/v1/admin/metrics?scope=personal:nobody"); + assert.ok(m.spend, "spend block is always present"); + assert.equal(m.spend.samples, 0); + assert.equal(m.spend.turnsWithKnownCost, 0); + assert.equal(m.spend.outputTokensTotal, 0); + assert.equal(m.spend.costUsdTotal, 0); + } finally { + await s.close(); + } +}); From 6511dddbf50261d03025ed24b4406690e9127266 Mon Sep 17 00:00:00 2001 From: yzxcj797 Date: Fri, 21 Aug 2026 13:12:06 +0800 Subject: [PATCH 2/2] feat(admin): show turn spend beside cache health in the metrics viewer The metrics endpoint grew a spend block, but the viewer still showed only the cache line -- an operator opening the page still could not see what the scope spends. renderSpend adds one statline under cache health: model spend as currency, turns with known cost, and summed output tokens, using the same fmtTokens/fmtPct vocabulary as its neighbour. --- plugins/admin/public/index.html | 15 +++++++++++++++ test/turn-metrics-cost.test.ts | 13 +++++++++++++ 2 files changed, 28 insertions(+) diff --git a/plugins/admin/public/index.html b/plugins/admin/public/index.html index becbb6f75..84a71a317 100644 --- a/plugins/admin/public/index.html +++ b/plugins/admin/public/index.html @@ -8516,6 +8516,7 @@

Confirm governance change

const phaseSel = urlToState().phase; if (phaseSel) return renderPhaseDrill(root, d, phaseSel); renderCacheHealth(root, d); + renderSpend(root, d); const phases = d.phases || []; root.appendChild( openableTable( @@ -8560,6 +8561,20 @@

Confirm governance change

]), ); } + function renderSpend(root, d) { + const sp = d.spend; + if (!sp || !(sp.samples || 0)) return; + root.appendChild( + statline([ + fmtUsd(sp.costUsdTotal) + " model spend", + (sp.turnsWithKnownCost || 0) + " turns with known cost", + fmtTokens(sp.outputTokensTotal || 0) + " output tokens", + (sp.samples || 0) + " turns with spend data", + ]), + ); + } + const fmtUsd = (n) => + n == null ? "—" : "$" + (Math.round((n || 0) * 100) / 100).toFixed(2); const fmtX = (n) => (n == null ? "—" : String(Math.round(n * 10) / 10)); function renderTurnMix(root, d) { const an = d.anatomy; diff --git a/test/turn-metrics-cost.test.ts b/test/turn-metrics-cost.test.ts index ed15a9035..215e29ea3 100644 --- a/test/turn-metrics-cost.test.ts +++ b/test/turn-metrics-cost.test.ts @@ -10,6 +10,8 @@ import { createInsecureTestServer } from "../src/api/server.ts"; import { buildApp } from "../src/wiring.ts"; import type { TurnRequest } from "../src/types.ts"; import { testConfig } from "./support/test-config.ts"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; function start() { const built = buildApp(testConfig({ dataDir: mkdtempSync(join(tmpdir(), "cost-obs-")) })); @@ -64,3 +66,14 @@ test("metrics: an empty scope yields a present-but-empty spend block", async () await s.close(); } }); + +test("metrics viewer: the spend line renders cost and output totals beside cache health", () => { + const html = readFileSync(fileURLToPath(new URL("../plugins/admin/public/index.html", import.meta.url)), "utf8").split(String.fromCharCode(13, 10)).join(String.fromCharCode(10)); + assert.ok(html.includes("function renderSpend(root, d)"), "viewer has a spend renderer"); + assert.ok( + html.includes("renderCacheHealth(root, d);" + String.fromCharCode(10) + " renderSpend(root, d);"), + "the spend line renders directly beside cache health on the metrics page", + ); + assert.ok(html.includes("fmtUsd(sp.costUsdTotal)"), "cost total is formatted as currency"); + assert.ok(html.includes("fmtTokens(sp.outputTokensTotal || 0)"), "output tokens are formatted"); +});