From 940c53d5a28e6420862ce194c8a0100663852bd5 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 04:45:06 -0700 Subject: [PATCH 1/3] Meter the judge's calls apart from the worker's Every ask made around the work (drafting and its critic, the reference check, the review, the post-work audit, the arbiter) now reports its input, output, cache read and cache write tokens to a judge meter. Each call is priced from the worker's price table keyed on the judge's own model id; a model with no known price shows its tokens and $ unknown, never $0, and a subscription judge is paid by its plan. The judge's spend is reported apart from the worker's: judge on job_end (every call since the previous job, so pre-turn drafting counts with its turn), judge_usage journal entries, a judge line and a worker + judge total in the CLI, the TUI and desktop job lines, the ACP prompt response, mission handoffs, and the receipt (index row updated when the job ends, since the review runs after the receipt is written). --budget and the per-turn token and dollar ceilings now count worker and judge spend together. sessionTokens and the step lines stay the worker's, so the lean-budget measurements are unchanged. --- src/acp-server.ts | 4 + src/app.tsx | 6 +- src/ask.ts | 81 +++++++++- src/cli.tsx | 51 +++++- src/criteria.ts | 7 + src/engine.ts | 101 +++++++++++- src/format.ts | 23 +++ src/journal.ts | 19 +++ src/judge-meter.ts | 241 ++++++++++++++++++++++++++++ src/mission.ts | 5 + src/post-audit.ts | 3 + src/receipts.ts | 35 +++- src/stream.ts | 1 + src/types.ts | 10 ++ test/judge-meter.test.ts | 333 +++++++++++++++++++++++++++++++++++++++ ui/app.ts | 6 +- 16 files changed, 907 insertions(+), 19 deletions(-) create mode 100644 src/judge-meter.ts create mode 100644 test/judge-meter.test.ts diff --git a/src/acp-server.ts b/src/acp-server.ts index 2a00975..126cb95 100644 --- a/src/acp-server.ts +++ b/src/acp-server.ts @@ -896,6 +896,7 @@ export class AcpServer { let truncated = false; let lastError: string | undefined; let spend: Spend | undefined; + let judge: import("./judge-meter.js").JudgeSpend | undefined; let exhaustedSeen = false; /** A message ended; the next text starts a new paragraph. */ let paragraph = false; @@ -1133,6 +1134,7 @@ export class AcpServer { case "job_end": record.outcome = ev.outcome; spend = ev.spend; + judge = ev.judge; break; default: break; @@ -1199,6 +1201,8 @@ export class AcpServer { } : {}), ...(spend?.costUsd !== undefined ? { costUsd: spend.costUsd, costEstimated: spend.estimated } : {}), + // The judge's spend, apart from the worker's `usage` (judge-meter.ts). + ...(judge ? { judge } : {}), }, }, }; diff --git a/src/app.tsx b/src/app.tsx index fd4521c..616e331 100644 --- a/src/app.tsx +++ b/src/app.tsx @@ -11,7 +11,7 @@ import { useCallback, useEffect, useRef, useState, type ReactNode } from "react" import { Box, Static, Text, render, useApp, useInput, useStdout } from "ink"; import type { RenderOptions } from "ink"; import { Banner, fmtCost, fmtDuration, fmtTokens } from "./banner.js"; -import { stepDid } from "./format.js"; +import { judgeSpendLine, stepDid } from "./format.js"; import { COMMANDS, COMMAND_COL, @@ -984,7 +984,9 @@ export function App({ ); note( `▪ job ${ev.job} ${ev.outcome} · ${ev.steps} step(s) · ` + - `${spendText(ev.spend)} · ${fmtDuration(ev.durationMs)}`, + `${spendText(ev.spend)} · ${fmtDuration(ev.durationMs)}` + + // The judge's share, apart from the worker's (judge-meter.ts). + (ev.judge ? ` · ${judgeSpendLine(ev.judge)}` : ""), ); break; case "request": diff --git a/src/ask.ts b/src/ask.ts index f49e998..1969210 100644 --- a/src/ask.ts +++ b/src/ask.ts @@ -17,9 +17,11 @@ import { isOpencode, opencodeAsk } from "./opencode.js"; import { removedSubscriptionProblem } from "./endpoint.js"; import { errorText } from "./format.js"; import { authHeaders, isSelfHosted, selfHostedThinking, openRouterProvider } from "./providers.js"; -import { LONG_RATE_LIMIT_MS, longQuotaText, providerErrorText, rateLimitResetAt, readStream, transientProviderError, untilText, type ProviderError } from "./stream.js"; +import { LONG_RATE_LIMIT_MS, longQuotaText, providerErrorText, rateLimitResetAt, readStream, transientProviderError, untilText, type ProviderError, type Usage } from "./stream.js"; import { askError, askTimeoutMs, localSpeed, probeSignal } from "./watchdog.js"; import { takeTurn } from "./localgate.js"; +import type { AskMeter } from "./judge-meter.js"; +import { estTokens } from "./types.js"; /** * Pauses before asking again when the provider said it is overloaded or @@ -64,6 +66,13 @@ export type AskOptions = { * three before the first step. Never recorded for a self-hosted server. */ latency?: { record(w: { firstProgressMs: number | undefined; maxGapMs: number }): void }; + /** + * Where each answered ask reports what it used (src/judge-meter.ts): input, + * output, cache read and write tokens, and the provider's own dollar figure + * when it sends one. Every ask that returns an answer is recorded, retries + * included; an ask the provider refused is not, since nothing was billed. + */ + meter?: AskMeter; }; export type Asked = { ok: true; text: string; cutOff: boolean } | { ok: false; error: string; transient?: true }; @@ -201,7 +210,10 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { prompt: opts.prompt, ...(opts.cliRun ? { run: opts.cliRun } : {}), }); - if (asked.ok) return { ok: true, text: asked.text, cutOff: false }; + if (asked.ok) { + meterEstimate(opts, asked.text); + return { ok: true, text: asked.text, cutOff: false }; + } return { ok: false, error: longQuotaText(asked.error) ?? asked.error, ...(asked.transient ? { transient: true as const } : {}) }; } @@ -216,7 +228,10 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { cwd: opts.cwd, ...(opts.acpSpawn ? { spawnFn: opts.acpSpawn } : {}), }); - if (asked.ok) return { ok: true, text: asked.text, cutOff: false }; + if (asked.ok) { + meterEstimate(opts, asked.text); + return { ok: true, text: asked.text, cutOff: false }; + } return { ok: false, error: longQuotaText(asked.error) ?? asked.error }; } @@ -276,10 +291,12 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { type Reply = { choices?: { message?: { content?: string | null }; finish_reason?: string | null }[]; error?: ProviderError; + usage?: Usage; }; let content: string; let finish: string | null | undefined; let error: ProviderError | undefined; + let used: Usage | undefined; // A server that was sent `stream: true` and answers with plain JSON (a // proxy that buffers, or an error body) is read as JSON, as it always was. if (stream && res.body && /text\/event-stream/i.test(res.headers.get("content-type") ?? "")) { @@ -287,12 +304,25 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { content = done.message.content ?? ""; finish = done.finishReason; error = done.error; + if (done.promptTokens !== undefined || done.completionTokens !== undefined) { + used = { + ...(done.promptTokens !== undefined ? { prompt_tokens: done.promptTokens } : {}), + ...(done.completionTokens !== undefined ? { completion_tokens: done.completionTokens } : {}), + ...(done.cachedTokens !== undefined ? { cache_read_input_tokens: done.cachedTokens } : {}), + ...(done.cacheWriteTokens !== undefined ? { cache_creation_input_tokens: done.cacheWriteTokens } : {}), + ...(done.costUsd !== undefined ? { cost: done.costUsd } : {}), + }; + } } else { const json = (await res.json()) as Reply; content = json.choices?.[0]?.message?.content ?? ""; finish = json.choices?.[0]?.finish_reason; error = json.error && typeof json.error === "object" ? json.error : undefined; + used = json.usage && typeof json.usage === "object" ? json.usage : undefined; } + // Before the error check: a provider can bill a request whose reply then + // carries an error, and a usage block is the provider saying it did. + if (!error || used) meterUsage(opts, used, content); if (error) { const resetAt = rateLimitResetAt({ error }); if (resetAt !== undefined && resetAt - Date.now() > LONG_RATE_LIMIT_MS) { @@ -315,6 +345,51 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { } } +/** A finite, non-negative count, or undefined. */ +function count(v: unknown): number | undefined { + return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined; +} + +/** + * Report one answered HTTP ask to the meter: the provider's usage block when + * it sent one, Maat's estimate (marked) when it did not. + */ +function meterUsage(opts: AskOptions, u: Usage | undefined, reply: string): void { + if (!opts.meter) return; + const prompt = count(u?.prompt_tokens); + const completion = count(u?.completion_tokens); + if (prompt === undefined && completion === undefined) { + meterEstimate(opts, reply); + return; + } + const read = count(u?.prompt_tokens_details?.cached_tokens) ?? count(u?.cache_read_input_tokens); + const write = count(u?.cache_creation_input_tokens); + const billed = count(u?.cost); + opts.meter.record({ + baseUrl: opts.baseUrl, + model: opts.model, + ...(opts.what ? { what: opts.what } : {}), + promptTokens: prompt ?? estTokens(opts.system + opts.prompt), + completionTokens: completion ?? estTokens(reply), + ...(read !== undefined ? { cacheReadTokens: read } : {}), + ...(write !== undefined ? { cacheWriteTokens: write } : {}), + ...(billed !== undefined ? { billedUsd: billed } : {}), + estimated: prompt === undefined || completion === undefined, + }); +} + +/** An answered ask whose transport reports no usage (ACP, a CLI, a bare reply): estimated. */ +function meterEstimate(opts: AskOptions, reply: string): void { + opts.meter?.record({ + baseUrl: opts.baseUrl, + model: opts.model, + ...(opts.what ? { what: opts.what } : {}), + promptTokens: estTokens(opts.system + opts.prompt), + completionTokens: estTokens(reply), + estimated: true, + }); +} + /** * The JSON object in a reply, or null. * diff --git a/src/cli.tsx b/src/cli.tsx index 126a1e3..fe68c74 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -16,6 +16,7 @@ import { resolve } from "node:path"; import { Archive } from "./archive.js"; import { isAutonomy, type Autonomy } from "./autonomy.js"; import { fmtCost, fmtDuration } from "./banner.js"; +import { judgeSpendLine } from "./format.js"; import { stepDid } from "./format.js"; import { BarError, hasBar, loadBar, selectChecks, writeDefaultBar } from "./bar.js"; import { Engine, type FileAccess } from "./engine.js"; @@ -148,7 +149,8 @@ options --verbose show every call, argument, and result (press v in the TUI) --provider label shown in the status line --cwd project directory (default: current) - --budget hard token ceiling for the session + --budget hard token ceiling for the session: the worker's tokens plus + the judge's (drafting, review, audit), counted together --review [n] after a verified claim, n independent reviews (default 3) read the task and the receipt; a majority-backed violation quoted from the task labels the work "passed its checks, @@ -925,6 +927,7 @@ function engineFor(args: Args, session = false, extra: { files?: FileAccess } = * nothing, no cost is shown at all. */ async function priceEngine(engine: Engine, args: Args): Promise { + await priceJudge(engine, args); if (!needsPriceLookup(args.model, engine.pricing(), storedEndpoint())) return; const p = await fetchPricing(args.url, args.model, keyForUrl(args.url, args.key)); if (!p) { @@ -938,6 +941,31 @@ async function priceEngine(engine: Engine, args: Args): Promise { savePricing(args.model, p); } +/** + * Give the judge's meter a price for each model asked around the work that is + * not the worker's own: the judge (MAAT_JUDGE_MODEL) and the arbiter. Keyed on + * that model's id, from the same lookup the worker uses (fetchPricing). A + * model that publishes nothing stays unpriced: its tokens are reported with + * "$ unknown", never $0. The worker's own model needs nothing here; the meter + * prices it with the worker's prices, hand-set ones included. + */ +async function priceJudge(engine: Engine, args: Args): Promise { + const key = keyForUrl(args.url, args.key); + const judge = judgeTarget({ baseUrl: args.url, apiKey: key, model: args.model }); + const arb = engine.cfg.dispute; + const others = [ + judge, + ...(arb?.model ? [{ baseUrl: arb.baseUrl || args.url, apiKey: arb.apiKey ?? key, model: arb.model }] : []), + ].filter((t) => !(t.model === args.model && t.baseUrl === args.url)); + await Promise.all( + others.map(async (t) => { + if (planFor(t.baseUrl) || engine.judgeMeter.hasPricing(t.baseUrl, t.model)) return; + const p = await fetchPricing(t.baseUrl, t.model, t.apiKey).catch(() => null); + engine.judgeMeter.setPricing(t.baseUrl, t.model, p); + }), + ); +} + /** * `from` decides which explanation a work-landed failure gets. * @@ -1344,6 +1372,8 @@ async function autoDraftFrom( ...(preWork ? { preWorkDir: preWork.dir, preWorkIntact: preWork.intact } : {}), reasoningEffort: judgeEffort(args.reasoningChecks ?? args.reasoning), latency: engine.askLatency, + // The drafter and the critic are the judge: metered as the judge. + meter: engine.judgeMeter, deadlineAt, // What is ready when a time budget stops the wait (RunOptions.criteriaSoFar). onProgress: (d) => { @@ -1390,7 +1420,7 @@ async function autoDraftFrom( * there is no python3, the project is too large to copy, or no reference * applies. */ -function startReference(args: Args, deadlineAt?: number): Promise<{ check: Check; note: Record } | null> | undefined { +function startReference(args: Args, deadlineAt?: number, engine?: Engine): Promise<{ check: Check; note: Record } | null> | undefined { const here = commandsHere(args.cwd); if (!here.present.includes("python3")) { process.stderr.write("maat: no reference check — python3 is not installed here\n"); @@ -1410,6 +1440,8 @@ function startReference(args: Args, deadlineAt?: number): Promise<{ check: Check cwd: args.cwd, reasoningEffort: args.reasoningChecks ?? args.reasoning, ...(deadlineAt !== undefined ? { deadlineAt } : {}), + // Asked around the work, like the drafter: on the judge's meter. + ...(engine ? { meter: engine.judgeMeter } : {}), }).then((r) => { if (!r.ok) { process.stderr.write(`maat: no reference check — ${r.why}\n`); @@ -1557,6 +1589,8 @@ async function cmdRun(args: Args, ask = false): Promise { (sp.costUsd === undefined ? "" : ` · ${sp.estimated ? "~" : ""}${fmtCost(sp.costUsd)}`) + "\n", ); + // The judge's share of the job, on its own line (judge-meter.ts). + if (ev.judge) process.stdout.write(` ${judgeSpendLine(ev.judge)}\n`); break; } case "request": @@ -1661,7 +1695,7 @@ async function cmdRun(args: Args, ask = false): Promise { return soFar.draft?.checks.length ? sealDraft(soFar.draft, args) : Promise.resolve({ taskChecks: [], taskNotes: [] }); } : undefined; - const referenceCheck = args.reference && !ask ? startReference(args, runDeadlineAt) : undefined; + const referenceCheck = args.reference && !ask ? startReference(args, runDeadlineAt, engine) : undefined; let undetermined = false; /** The turn's own verdict, from job_end. */ let outcome: string | undefined; @@ -1700,6 +1734,17 @@ async function cmdRun(args: Args, ask = false): Promise { "\n", ); } + // The judge's spend, apart from the worker's line above, and the two added + // up when both are priced. An unpriced judge says so; it is never $0. + if (b.judge) { + process.stdout.write(`${judgeSpendLine(b.judge)}\n`); + if (b.costUsd !== undefined && b.judge.costUsd !== undefined) { + process.stdout.write( + `total ${b.costEstimated || b.judge.estimated ? "~" : ""}${fmtCost(b.costUsd + b.judge.costUsd)} ` + + `(worker ${fmtCost(b.costUsd)} + judge ${fmtCost(b.judge.costUsd)})\n`, + ); + } + } // An unverified answer is not a success. Neither is no answer at all. // diff --git a/src/criteria.ts b/src/criteria.ts index 51a6b30..bf2473b 100644 --- a/src/criteria.ts +++ b/src/criteria.ts @@ -793,6 +793,7 @@ async function edgeProbes(opts: Parameters[0], view: strin deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, + meter: opts.meter, }).catch(() => null); if (!asked?.ok) return ""; let probes: { input?: unknown; readingA?: unknown; outputA?: unknown; readingB?: unknown; outputB?: unknown; quote?: unknown }[] = []; @@ -838,6 +839,8 @@ export async function draftCriteria(opts: { reasoningEffort?: string; /** Learns from completed asks (AskOptions.latency). */ latency?: AskOptions["latency"]; + /** Where each ask reports its tokens (AskOptions.meter; the judge's meter). */ + meter?: AskOptions["meter"]; /** What is installed, so a check never calls a command that is not (see commandsHere). */ commands?: { present: string[]; missing: string[] }; /** @@ -904,6 +907,7 @@ export async function draftCriteria(opts: { deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, + meter: opts.meter, }); if (!asked.ok) return asked; const first = drafted(asked.text, asked.cutOff); @@ -925,6 +929,7 @@ export async function draftCriteria(opts: { deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, + meter: opts.meter, }); if (!again.ok) return first; const second = drafted(again.text, again.cutOff); @@ -950,6 +955,7 @@ export async function draftCriteria(opts: { deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, + meter: opts.meter, }); if (!more.ok) break; const d = drafted(more.text, more.cutOff); @@ -1345,6 +1351,7 @@ async function critiqued( deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, + meter: opts.meter, }); return asked.ok ? applyCritique(d, asked.text, opts.task) : null; }; diff --git a/src/engine.ts b/src/engine.ts index 513109a..37f4a44 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -25,6 +25,7 @@ import { describeStart, listBackground, startBackground, stopBackground } from " import { objectionLine, readsTheWork, reviewClaim, type ExecutableReview, type ObjectionRun, type Review } from "./review.js"; import { credentialFreeEnv } from "./credenv.js"; import { judgeEffort, judgeTarget } from "./judge.js"; +import { JudgeMeter, judgeTokens, type JudgeSpend } from "./judge-meter.js"; import { auditClaim, authorKey, authorWords, claimLabel, contradictions, independentOf, tierOf, withAuthor, type Tier } from "./tiers.js"; import { recordGoldens, valueUnproven } from "./golden.js"; import { discountedChecks } from "./control.js"; @@ -2415,6 +2416,31 @@ export class Engine { get askLatency(): LatencyLearner | undefined { return isSelfHosted(this.cfg.baseUrl) ? undefined : this.latency; } + /** + * The judge's meter (src/judge-meter.ts): every ask made around the work — + * drafting, the critic, the reference check, the review, the post-work + * audit, the arbiter — with its tokens and price, kept apart from the + * worker's. A judge that is the worker's own model is priced as the worker is. + */ + readonly judgeMeter: JudgeMeter = new JudgeMeter(() => ({ + baseUrl: this.cfg.baseUrl, + model: this.cfg.model, + pricing: + this.cfg.priceInPerMtok !== undefined && this.cfg.priceOutPerMtok !== undefined + ? { + in: this.cfg.priceInPerMtok, + out: this.cfg.priceOutPerMtok, + ...(this.cfg.priceCachedInPerMtok !== undefined ? { cached: this.cfg.priceCachedInPerMtok } : {}), + source: this.cfg.priceSource ?? "the worker's prices", + } + : null, + })); + /** + * Judge calls before this mark were reported in an earlier job_end. A job + * reports every call since the last one, so the drafting that runs before a + * turn starts is counted in the turn it was for. + */ + private judgeReported = 0; /** Why the independent review was skipped this turn (see reviewSkipReason), if it was. */ private reviewSkipped: string | undefined; /** Hidden checks whose commands were shown to the model this turn. */ @@ -2469,6 +2495,23 @@ export class Engine { // Under --worker-user, one line in the journal saying what isolation was // actually in effect (src/privsep.ts), e.g. // "isolation: worker uid 1001, check uid 1002, pid namespace on". + // Every judge call, as it returns: what it used and what it cost, beside + // the worker's `response` entries (judge-meter.ts). + this.judgeMeter.onRecord((c) => { + this.cfg.journal?.append("judge_usage", { + model: c.model, + ...(c.what ? { what: c.what } : {}), + promptTokens: c.promptTokens, + completionTokens: c.completionTokens, + ...(c.cacheReadTokens !== undefined ? { cacheReadTokens: c.cacheReadTokens } : {}), + ...(c.cacheWriteTokens !== undefined ? { cacheWriteTokens: c.cacheWriteTokens } : {}), + // null is "no price known", never 0. + costUsd: c.costUsd ?? null, + billed: typeof c.billedUsd === "number", + estimated: c.estimated, + ...(c.plan ? { plan: c.plan } : {}), + }); + }); const isolation = isolationLine(); if (isolation && cfg.journal && !isolationNoted.has(cfg.journal)) { isolationNoted.add(cfg.journal); @@ -3187,6 +3230,7 @@ export class Engine { costUsd: this.costUsd(), costEstimated: this.costEstimated, budgetTokens: this.budgetTokens, + ...(this.judgeMeter.mark() ? { judge: this.judgeMeter.total() } : {}), }; } @@ -3501,8 +3545,35 @@ export class Engine { return existsSync(p) ? readFileSync(p, "utf8") : undefined; } + /** + * A budget counts everything spent: the worker's tokens and the judge's. + * The judge's asks are not the worker's requests (the step lines and + * sessionTokens stay the worker's), but they are billed all the same. + */ private overBudget(): boolean { - return this.budgetTokens !== undefined && this.sessionTokens >= this.budgetTokens; + return this.budgetTokens !== undefined && this.spentTokens >= this.budgetTokens; + } + + /** Worker and judge tokens together: what a token budget counts. */ + get spentTokens(): number { + return this.sessionTokens + judgeTokens(this.judgeMeter.total()); + } + + /** + * Worker and judge USD together: what a money budget counts. Undefined when + * the worker has no price. A judge with no price adds nothing here (its + * tokens still count toward a token budget, and every surface says its $ is + * unknown); a judge on a subscription plan costs no money. + */ + totalCostUsd(): number | undefined { + const worker = this.costUsd(); + if (worker === undefined) return undefined; + return worker + (this.judgeMeter.total().costUsd ?? 0); + } + + /** The whole session's judge spend (judge-meter.ts). */ + judgeSpend(): JudgeSpend { + return this.judgeMeter.total(); } /** @@ -4109,6 +4180,7 @@ export class Engine { fetchFn: this.cfg.fetchFn, acpSpawn: this.cfg.acpSpawn, reasoningEffort: judgeEffort(this.cfg.reasoningEffort), + meter: this.judgeMeter, deadlineAt, signal: this.running.signal, // Masked like every hidden check while it runs, released below. @@ -4561,6 +4633,7 @@ export class Engine { provider: this.provider, sessionTokens: this.sessionTokens, shedBatches: this.transcript.shedCount, + ...(this.judgeMeter.mark() ? { judge: this.judgeMeter.total() } : {}), ...(endedBy ? { endedBy } : {}), }); log?.append("receipt", { verdict: "exhausted", file: receipt.path, attempt: attempts }); @@ -5001,6 +5074,7 @@ export class Engine { reasoningEffort: judgeEffort(this.cfg.review.reasoningEffort ?? this.cfg.reasoningEffort), fetchFn: this.cfg.fetchFn, acpSpawn: this.cfg.acpSpawn, + meter: this.judgeMeter, // Never longer than the turn has left: a review that outlives the clock ends it without a verdict. ...(this.timeLeftMs() !== undefined ? { timeoutMs: Math.max(1_000, this.timeLeftMs()!) } : {}), // And every retry and pause inside it, not just one ask. @@ -5135,11 +5209,18 @@ export class Engine { this.cfg.journal?.append("note", { text: "tier: verified-audit", claim, by: [audit.judge], accepted: audit.accepted.map((n) => `audit:${n}`) }); } } + // The judge's calls since the last job reported, the drafting before this + // turn included; and the receipt's row brought up to date with them, since + // the review and the audit run after the receipt was written. + const judge = this.judgeMeter.since(this.judgeReported); + this.judgeReported = this.judgeMeter.mark(); + if (receiptPath && judge.calls) this.cfg.receipts?.amendJudge(basename(receiptPath), this.judgeMeter.total()); yield { kind: "job_end", job, steps, spend: this.spendSince(before), + ...(judge.calls ? { judge } : {}), durationMs: Date.now() - startedAt, outcome, ...(tier ? { tier, ...(tierReason ? { tierReason } : {}) } : {}), @@ -6449,6 +6530,7 @@ export class Engine { claim, result, attempt: proofAttempts, verdict, head, model: self.modelOfRecord(), provider: self.provider, sessionTokens: self.sessionTokens, session: self.cfg.journal?.sessionId, costUsd: self.costUsd(), costEstimated: self.costEstimated, + ...(self.judgeMeter.mark() ? { judge: self.judgeMeter.total() } : {}), shedBatches: self.transcript.shedCount, endedBy: why, changed: self.turnLedger().map((e) => ({ path: e.path, before: e.before, after: e.after, @@ -6495,8 +6577,9 @@ export class Engine { } } - const turnStartTokens = this.sessionTokens; - const turnStartCost = this.costUsd(); + // Both ceilings count the judge's spend too (overBudget). + const turnStartTokens = this.spentTokens; + const turnStartCost = this.totalCostUsd(); let warned = 0; // The step guard is the last way out of a turn, and it had the same fault // the spending ceiling had: it stopped dead. A reported run reached it with @@ -6611,7 +6694,10 @@ export class Engine { if (this.overBudget()) { yield { kind: "error", - text: `budget hit (${this.budgetTokens} tokens) — loop stopped. /budget to raise.`, + text: + `budget hit (${this.budgetTokens} tokens` + + (this.spentTokens > this.sessionTokens ? `, ${this.spentTokens - this.sessionTokens} of them the judge's` : "") + + `) — loop stopped. /budget to raise.`, ceiling: "budget", }; yield* this.salvage(`You have reached the token budget for this session.`, fetchFn, log); @@ -6622,9 +6708,9 @@ export class Engine { // before the request rather than after, so the limit is what molt // refuses to spend rather than what it noticed spending. // Money where a price is known, tokens only where it is not. - const spentThisTurn = this.sessionTokens - turnStartTokens; + const spentThisTurn = this.spentTokens - turnStartTokens; const usdThisTurn = - turnStartCost === undefined ? undefined : (this.costUsd() ?? 0) - turnStartCost; + turnStartCost === undefined ? undefined : (this.totalCostUsd() ?? 0) - turnStartCost; // Zero unless someone set one. The self-hosted exception that used to // live here — no default ceiling on hardware you own — is gone with the // default itself: there is nothing left to make an exception to. @@ -8175,6 +8261,7 @@ export class Engine { reasoningEffort: this.cfg.review?.reasoningEffort ?? this.cfg.reasoningEffort, fetchFn: this.cfg.fetchFn, acpSpawn: this.cfg.acpSpawn, + meter: this.judgeMeter, deadlineAt: this.deadlineAt(), }, }).catch(() => null); @@ -8448,6 +8535,7 @@ export class Engine { session: this.cfg.journal?.sessionId, costUsd: this.costUsd(), costEstimated: this.costEstimated, + ...(this.judgeMeter.mark() ? { judge: this.judgeMeter.total() } : {}), shedBatches: this.transcript.shedCount, // A question the bar could not refuse is recorded as one, so stats // never count an answer as a verified change. @@ -8510,6 +8598,7 @@ export class Engine { reasoningEffort: judgeEffort(this.cfg.review.reasoningEffort ?? this.cfg.reasoningEffort), fetchFn: this.cfg.fetchFn, acpSpawn: this.cfg.acpSpawn, + meter: this.judgeMeter, ...(this.timeLeftMs() !== undefined ? { timeoutMs: Math.max(1_000, this.timeLeftMs()!) } : {}), deadlineAt: this.deadlineAt(), }, diff --git a/src/format.ts b/src/format.ts index 8f63f0c..79c4296 100644 --- a/src/format.ts +++ b/src/format.ts @@ -10,6 +10,7 @@ * Pure, and re-exported from `banner.tsx` so nothing that already imported * them has to move. */ +import type { JudgeSpend } from "./judge-meter.js"; export function fmtTokens(n: number): string { if (n < 1000) return `${n}`; @@ -119,3 +120,25 @@ export function spendLine(sp: { (sp.costUsd === undefined ? "" : ` · ${sp.estimated ? "~" : ""}${fmtCost(sp.costUsd)}`) ); } + +/** + * A judge spend in one line, for the CLI and the receipt: + * `judge 3 calls · 1200 in (400 cached) · 300 out · $0.004`, with `$ unknown` + * when a model had no price and the plan's name when a plan paid. + */ +export function judgeSpendLine(s: JudgeSpend, fmt: (usd: number) => string = fmtCost): string { + const cache = + s.cacheReadTokens > 0 || s.cacheWriteTokens > 0 + ? ` (${[s.cacheReadTokens > 0 ? `${s.cacheReadTokens} cached` : "", s.cacheWriteTokens > 0 ? `${s.cacheWriteTokens} cache write` : ""].filter(Boolean).join(", ")})` + : ""; + const e = s.estimated ? "~" : ""; + const money = + s.costUsd !== undefined + ? `${s.billed ? "" : e}${fmt(s.costUsd)}${s.plan ? ` + ${s.plan} plan` : ""}` + : s.unpricedCalls > 0 + ? `$ unknown (no price for ${(s.unpricedModels ?? s.models).join(", ")})` + : s.plan + ? `your ${s.plan} plan, not metered` + : "$ unknown"; + return `judge ${s.calls} call${s.calls === 1 ? "" : "s"} · ${e}${s.promptTokens} in${cache} · ${e}${s.completionTokens} out · ${money}`; +} diff --git a/src/journal.ts b/src/journal.ts index 89f1853..7b4864b 100644 --- a/src/journal.ts +++ b/src/journal.ts @@ -86,6 +86,12 @@ export type JournalKind = */ | "loop_stop" | "salvage" + /** + * One answered ask to the judge (drafting, critic, reference, review, + * audit, arbiter): its tokens and cost, apart from the worker's `response` + * entries. `costUsd: null` means no price is known, never $0. + */ + | "judge_usage" /** The independent review of a verified claim: a label, never a gate. */ | "review" | "dispute" @@ -387,6 +393,19 @@ export class Journal { ); break; } + case "judge_usage": { + const e = d.estimated ? "~" : ""; + const cached = Number(d.cacheReadTokens ?? 0) > 0 ? ` (${d.cacheReadTokens} cached)` : ""; + const cost = d.plan + ? ` · ${d.plan} plan` + : d.costUsd === null || d.costUsd === undefined + ? " · $ unknown" + : ` · ${d.billed ? "" : e}$${Number(d.costUsd).toFixed(6)}`; + out.push( + `${t} judge ${d.model}${d.what ? ` (${d.what})` : ""} · ${e}${d.promptTokens} in${cached} / ${e}${d.completionTokens} out${cost}`, + ); + break; + } case "tool_call": out.push(`${t} tool ${d.name}: ${d.detail}`); break; diff --git a/src/judge-meter.ts b/src/judge-meter.ts new file mode 100644 index 0000000..12feb94 --- /dev/null +++ b/src/judge-meter.ts @@ -0,0 +1,241 @@ +/** + * The judge's meter: every out-of-loop ask, counted and priced. + * + * The worker's own requests have always been metered step by step. The asks + * made around the work were not: drafting the hidden checks, the critic that + * reads the draft, the reference check, the independent review, the post-work + * audit and the dispute arbiter. With a separate judge model (MAAT_JUDGE_MODEL) + * those asks go to a different model, often on a different provider, and a + * bench run could only estimate what they cost; the exact figure was visible + * only in the provider's console. Every token spent should be visible here. + * + * Each ask that returns records what the provider reported (input, output, + * cache read and cache write tokens, and a billed dollar figure when the + * provider sends one). It is priced with the same table as the worker + * (providers.ts), keyed on the judge's own model id. A model with no known + * price reports its tokens and "$ unknown", never $0: an unpriced judge + * looking free is the confidently wrong meter the worker's pricing already + * refuses to be. A subscription backend (Grok Build, OpenCode) is paid by its + * plan and gets no dollar figure, as on the worker side. + * + * Kept apart from the worker's meter on purpose. The worker's tokens are what + * the lean-budget suite measures and what the step lines report; the judge's + * are a separate line on every surface, and both count toward a budget. + */ +import { anthropicPricing, planFor, type Pricing } from "./providers.js"; + +/** What one ask used, as the ask path read it from the reply. */ +export type AskUsage = { + baseUrl: string; + model: string; + /** What was being asked ("drafting criteria", "reviewing the claim"). */ + what?: string; + promptTokens: number; + completionTokens: number; + /** Prompt tokens served from the provider's cache, when it itemises them. */ + cacheReadTokens?: number; + /** Prompt tokens written to the provider's cache, when it reports them. */ + cacheWriteTokens?: number; + /** USD, when the provider reported the figure itself (OpenRouter's `usage.cost`). */ + billedUsd?: number; + /** True when the provider reported no usage and the counts are Maat's estimate. */ + estimated: boolean; +}; + +/** Anything an ask can report its usage to. */ +export interface AskMeter { + record(u: AskUsage): void; +} + +/** The judge's spend over some span of calls. */ +export type JudgeSpend = { + calls: number; + promptTokens: number; + completionTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + /** + * USD, or undefined when it is not known: some call's model has no price. + * Never 0 for unknown. Calls paid by a subscription plan add nothing. + */ + costUsd?: number; + /** Calls whose model had no price and no billed figure. */ + unpricedCalls: number; + /** The models those calls went to. */ + unpricedModels?: string[]; + /** Calls paid for by a subscription plan, by the plan's name. */ + plan?: string; + /** True when any call's counts are Maat's estimate. */ + estimated: boolean; + /** True when every priced call's dollar figure came from the provider. */ + billed: boolean; + /** The models asked, in the order first seen. */ + models: string[]; +}; + +/** The worker's pricing, for a judge that is the worker's own model. */ +export type WorkerPricing = { baseUrl: string; model: string; pricing: Pricing | null; plan?: string }; + +/** A call as recorded, with what it cost when it was recorded. */ +export type JudgeCall = AskUsage & { costUsd?: number; plan?: string }; + +function key(baseUrl: string, model: string): string { + return `${baseUrl.replace(/\/+$/, "")} ${model}`; +} + +function anthropicHost(baseUrl: string): boolean { + try { + return /anthropic\.com$/.test(new URL(baseUrl).hostname); + } catch { + return false; + } +} + +/** USD for these tokens at this price; cached tokens are inside `prompt`. */ +export function priceTokens(p: Pricing, u: Pick): number { + // As the worker's meter: cached prompt tokens at the cache rate when one is + // published, the rest of the prompt (cache writes included) at the base rate. + const cached = p.cached === undefined ? 0 : Math.min(u.cacheReadTokens ?? 0, u.promptTokens); + const fresh = u.promptTokens - cached; + return (fresh / 1e6) * p.in + (cached / 1e6) * (p.cached ?? p.in) + (u.completionTokens / 1e6) * p.out; +} + +export class JudgeMeter implements AskMeter { + private readonly calls: JudgeCall[] = []; + private readonly prices = new Map(); + private readonly listeners: ((c: JudgeCall) => void)[] = []; + + /** + * @param worker The worker's endpoint, model and prices, read when needed: a + * judge that is the worker's own model is priced exactly as the worker is, + * hand-set prices included. + */ + constructor(private readonly worker?: () => WorkerPricing) {} + + /** Set the price of a judge model, as resolved from its provider (or null: none published). */ + setPricing(baseUrl: string, model: string, p: Pricing | null): void { + this.prices.set(key(baseUrl, model), p); + } + + /** Has a price lookup already been made for this model? */ + hasPricing(baseUrl: string, model: string): boolean { + return this.prices.has(key(baseUrl, model)); + } + + /** + * The price of one judge model, or null when none is known. + * + * A price set for this endpoint and model wins. Then the worker's own prices, + * when the judge is the worker's model on the worker's endpoint. Then the + * published Anthropic rates, on Anthropic's API (the table fetchPricing reads + * there). Nothing else is guessed. + */ + pricingFor(baseUrl: string, model: string): Pricing | null { + const set = this.prices.get(key(baseUrl, model)); + if (set !== undefined) return set; + const w = this.worker?.(); + if (w && key(w.baseUrl, w.model) === key(baseUrl, model)) return w.pricing; + if (anthropicHost(baseUrl)) return anthropicPricing(model); + return null; + } + + /** The subscription plan that pays for asks on this endpoint, if any. */ + private planOf(baseUrl: string): string | undefined { + return planFor(baseUrl); + } + + /** Called with every call as it is recorded (the journal's `judge_usage`). */ + onRecord(fn: (c: JudgeCall) => void): void { + this.listeners.push(fn); + } + + record(u: AskUsage): void { + const c = this.priced(u); + this.calls.push(c); + for (const fn of this.listeners) { + try { + fn(c); + } catch { + /* a listener that throws must not lose the record */ + } + } + } + + private priced(u: AskUsage): JudgeCall { + const plan = this.planOf(u.baseUrl); + if (plan) return { ...u, plan }; + if (typeof u.billedUsd === "number") return { ...u, costUsd: u.billedUsd }; + const p = this.pricingFor(u.baseUrl, u.model); + return p ? { ...u, costUsd: priceTokens(p, u) } : { ...u }; + } + + /** How many calls have been recorded: a mark to take `since` from. */ + mark(): number { + return this.calls.length; + } + + /** Every call recorded, in order. */ + all(): readonly JudgeCall[] { + return this.calls; + } + + /** The whole session's judge spend. */ + total(): JudgeSpend { + return this.since(0); + } + + /** + * The judge's spend since a mark. Priced now, not when recorded, so a price + * that arrived after the first ask (a lookup still in flight, a /price) is + * applied to every call, as the worker's meter does. + */ + since(mark: number): JudgeSpend { + const span = this.calls.slice(mark).map((c) => this.priced(c)); + const models: string[] = []; + let cost = 0; + let unpriced = 0; + const unpricedModels: string[] = []; + let billedAll = true; + let pricedAny = false; + let plan: string | undefined; + for (const c of span) { + if (!models.includes(c.model)) models.push(c.model); + if (c.plan) { + plan = c.plan; + continue; + } + if (c.costUsd === undefined) { + unpriced += 1; + if (!unpricedModels.includes(c.model)) unpricedModels.push(c.model); + continue; + } + pricedAny = true; + cost += c.costUsd; + if (typeof c.billedUsd !== "number") billedAll = false; + } + return { + calls: span.length, + promptTokens: span.reduce((n, c) => n + c.promptTokens, 0), + completionTokens: span.reduce((n, c) => n + c.completionTokens, 0), + cacheReadTokens: span.reduce((n, c) => n + (c.cacheReadTokens ?? 0), 0), + cacheWriteTokens: span.reduce((n, c) => n + (c.cacheWriteTokens ?? 0), 0), + // Unknown if any call is unknown: a partial sum would read as the whole. + // Plan-paid calls cost no money, so a span of only those has no figure. + ...(unpriced === 0 && pricedAny ? { costUsd: cost } : {}), + unpricedCalls: unpriced, + ...(unpricedModels.length ? { unpricedModels } : {}), + ...(plan ? { plan } : {}), + estimated: span.some((c) => c.estimated), + billed: pricedAny && billedAll, + models, + }; + } +} + +/** Tokens a judge spend counts toward a budget: everything sent and received. */ +export function judgeTokens(s: JudgeSpend | undefined): number { + return s ? s.promptTokens + s.completionTokens : 0; +} + +/** Said in one line by format.ts, which the renderer can import. */ +export { judgeSpendLine } from "./format.js"; diff --git a/src/mission.ts b/src/mission.ts index 7175ea2..5084e53 100644 --- a/src/mission.ts +++ b/src/mission.ts @@ -499,6 +499,8 @@ export type Handoff = { failed: string[]; receipt?: string; spend?: Spend; + /** The judge's spend on this attempt, apart from the worker's (judge-meter.ts). */ + judge?: import("./judge-meter.js").JudgeSpend; durationMs: number; at: string; }; @@ -640,6 +642,7 @@ export async function runMission(opts: MissionRunOptions): Promise `$${usd.toFixed(4)}`)} so far`] : []), `- shed batches archived: ${args.shedBatches}`, `- bar duration: ${args.result.durationMs}ms`, "", @@ -775,6 +789,7 @@ export class Receipts { ...(args.session ? { session: args.session } : {}), ...(args.costUsd === undefined ? {} : { costUsd: args.costUsd }), ...(args.costEstimated ? { costEstimated: true } : {}), + ...(args.judge ? { judge: args.judge } : {}), ...(args.ask ? { ask: true } : {}), ...(args.revealed?.length ? { revealed: [...args.revealed] } : {}), ...(args.endedBy ? { endedBy: args.endedBy } : {}), @@ -847,7 +862,16 @@ export class Receipts { * other write while hidden commands are withheld. The receipt file itself, * hash-bound when it was written, is never rewritten. */ - amendReview(file: string, objections: readonly Objection[]): boolean { + /** + * Bring a row's judge spend up to date at the end of its job: the review and + * the post-work audit ask the judge after the receipt is written. + */ + amendJudge(file: string, judge: JudgeSpend): boolean { + return this.amendRow(file, (r) => ({ ...r, judge })); + } + + /** Rewrite one index row, by receipt file name. False when there is none. */ + private amendRow(file: string, change: (r: ReceiptRecord) => ReceiptRecord): boolean { try { if (!existsSync(this.indexPath)) return false; let hit = false; @@ -859,7 +883,7 @@ export class Receipts { const r = JSON.parse(l) as ReceiptRecord; if (r.file !== file) return l; hit = true; - return redact(JSON.stringify({ ...r, objections: maskDeep([...objections], this.withheld) }), this.secrets); + return redact(JSON.stringify(change(r)), this.secrets); } catch { return l; } @@ -871,6 +895,11 @@ export class Receipts { } } + amendReview(file: string, objections: readonly Objection[]): boolean { + return this.amendRow(file, (r) => ({ ...r, objections: maskDeep([...objections], this.withheld) })); + } + + /** * Reconcile the index against the files on disk. * diff --git a/src/stream.ts b/src/stream.ts index 3ac0990..bc6810a 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -395,6 +395,7 @@ export class StreamAccumulator { promptTokens: this.promptTokens, completionTokens: this.completionTokens, cachedTokens: this.cachedTokens, + ...(this.cacheWriteTokens !== undefined ? { cacheWriteTokens: this.cacheWriteTokens } : {}), reasoningTokens: this.reasoningTokens, costUsd: this.costUsd, finishReason: this.finishReason, diff --git a/src/types.ts b/src/types.ts index 67745f8..1163c48 100644 --- a/src/types.ts +++ b/src/types.ts @@ -77,6 +77,8 @@ export type Bom = { /** True when any part of the cost rests on molt's own token estimate. */ costEstimated?: boolean; budgetTokens?: number; + /** The judge's session spend, apart from the worker's above (judge-meter.ts). Absent before its first call. */ + judge?: import("./judge-meter.js").JudgeSpend; }; /** @@ -638,7 +640,15 @@ export type EngineEvent = kind: "job_end"; job: number; steps: number; + /** The worker's spend: its own requests, step by step. */ spend: Spend; + /** + * The judge's spend on this job, apart from the worker's: every ask made + * around the work (drafting, critic, reference, review, audit, arbiter) + * since the last job_end. `costUsd` absent means the price is unknown, + * never $0. Absent when the judge was not asked. + */ + judge?: import("./judge-meter.js").JudgeSpend; durationMs: number; outcome: JobOutcome; /** diff --git a/test/judge-meter.test.ts b/test/judge-meter.test.ts new file mode 100644 index 0000000..652a2fc --- /dev/null +++ b/test/judge-meter.test.ts @@ -0,0 +1,333 @@ +/** + * The judge's meter: every ask made around the work (drafting, critic, + * reference, review, audit, arbiter) is counted and priced apart from the + * worker's requests, reported on job_end, the receipt, the journal and the + * CLI line, and counted by a budget. + * + * A bench run on 2026-10-08 could only estimate the judge's cost; the exact + * figure was visible only in the provider's console. + */ +import assert from "node:assert/strict"; +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, it } from "node:test"; +import { askModel } from "../src/ask.js"; +import { Engine } from "../src/engine.js"; +import { judgeSpendLine } from "../src/format.js"; +import { Journal } from "../src/journal.js"; +import { JudgeMeter, priceTokens } from "../src/judge-meter.js"; +import { Receipts } from "../src/receipts.js"; +import type { Check, EngineEvent } from "../src/types.js"; +import { allowAll, drain, workspace } from "./helpers.js"; + +const URL = "http://provider.test/v1"; +const JUDGE = "judge-j"; +const PRICE = { in: 2, out: 10, cached: 0.2, source: "test" }; +/** What the scripted judge reports for every ask. */ +const JUDGE_USAGE = { prompt_tokens: 1000, completion_tokens: 200, prompt_tokens_details: { cached_tokens: 400 }, cache_creation_input_tokens: 50 }; +/** 600 fresh x $2 + 400 cached x $0.2 + 200 out x $10, per 1M. */ +const PER_CALL = (600 * 2 + 400 * 0.2 + 200 * 10) / 1e6; + +const reply = (body: unknown) => + ({ ok: true, status: 200, json: async () => body, text: async () => JSON.stringify(body), headers: new Headers() }) as unknown as Response; + +/** + * One endpoint serving two models: the worker ("m") writes out.txt and says + * done; the judge answers every review ask with no violations and JUDGE_USAGE. + */ +function twoModels() { + const sent: { model: string }[] = []; + let worker = 0; + const fetchFn = (async (_url: string, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { model: string }; + sent.push(body); + if (body.model === JUDGE) { + return reply({ choices: [{ message: { role: "assistant", content: '{"violations":[]}' }, finish_reason: "stop" }], usage: JUDGE_USAGE }); + } + worker += 1; + const message = + worker === 1 + ? { role: "assistant", content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "write_file", arguments: JSON.stringify({ path: "out.txt", content: "hello\n" }) } }] } + : { role: "assistant", content: "Done." }; + return reply({ choices: [{ message, finish_reason: worker === 1 ? "tool_calls" : "stop" }], usage: { prompt_tokens: 100, completion_tokens: 20 } }); + }) as unknown as typeof fetch; + return { fetchFn, sent, judgeCalls: () => sent.filter((b) => b.model === JUDGE).length }; +} + +const greet: Check = { + name: "greeting", kind: "command", run: "grep -qx hello out.txt", timeoutMs: 5_000, expectExit: 0, tags: ["task", "value", "exact"], hidden: true, + author: { kind: "judge", model: JUDGE }, +}; + +function jobEnd(events: EngineEvent[]) { + const e = events.find((x) => x.kind === "job_end"); + assert.ok(e && e.kind === "job_end"); + return e; +} + +describe("the judge's meter on a reviewed turn", () => { + let saved: string | undefined; + beforeEach(() => { + saved = process.env.MAAT_JUDGE_MODEL; + process.env.MAAT_JUDGE_MODEL = JUDGE; + }); + afterEach(() => { + if (saved === undefined) delete process.env.MAAT_JUDGE_MODEL; + else process.env.MAAT_JUDGE_MODEL = saved; + }); + + const run = async (priced: boolean) => { + const ws = workspace(); + const p = twoModels(); + const journal = new Journal(ws.dir); + const engine = new Engine({ + baseUrl: URL, model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false, autonomy: "high", + review: { votes: 3 }, receipts: new Receipts(ws.dir), journal, + priceInPerMtok: 1, priceOutPerMtok: 5, + }); + if (priced) engine.judgeMeter.setPricing(URL, JUDGE, PRICE); + // The drafting the CLI does before the turn, on the same meter. + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "drafting criteria", promptTokens: 1000, completionTokens: 200, cacheReadTokens: 400, cacheWriteTokens: 50, estimated: false }); + const events = await drain(engine.run("write out.txt", allowAll, { taskChecks: [greet] })); + return { ws, engine, events, p, journal }; + }; + + it("reports the judge's tokens and $ on job_end, apart from the worker's", async () => { + const { ws, events, p, engine } = await run(true); + try { + const e = jobEnd(events); + assert.equal(p.judgeCalls(), 3, "three review votes went to the judge"); + assert.ok(e.judge, "job_end carries the judge's spend"); + // The drafting before the turn and the three review votes. + assert.equal(e.judge.calls, 4); + assert.equal(e.judge.promptTokens, 4000); + assert.equal(e.judge.completionTokens, 800); + assert.equal(e.judge.cacheReadTokens, 1600); + assert.equal(e.judge.cacheWriteTokens, 200); + assert.ok(Math.abs(e.judge.costUsd! - 4 * PER_CALL) < 1e-12, `judge $ ${e.judge.costUsd}`); + assert.deepEqual(e.judge.models, [JUDGE]); + // The worker's spend is its own two requests, untouched by the judge's. + assert.equal(e.spend.promptTokens, 200); + assert.equal(e.spend.completionTokens, 40); + assert.equal(engine.sessionTokens, 240, "sessionTokens stays the worker's"); + assert.equal(engine.spentTokens, 240 + 4800, "a budget counts both"); + } finally { + ws.cleanup(); + } + }); + + it("writes the judge's spend to the receipt row, the receipt and the journal", async () => { + const { ws, journal } = await run(true); + try { + const rows = new Receipts(ws.dir).records(); + const row = rows.find((r) => r.verdict === "accepted"); + assert.ok(row?.judge, "the receipt row carries the judge's spend"); + // Brought up to date at the job's end: the review ran after the receipt was written. + assert.equal(row.judge.calls, 4); + assert.ok(Math.abs(row.judge.costUsd! - 4 * PER_CALL) < 1e-12); + assert.equal(row.costUsd, (200 * 1 + 40 * 5) / 1e6, "the worker's cost stays the worker's"); + // The receipt file, written at the claim, says what the judge had spent by then. + const dir = join(ws.dir, ".maat", "receipts"); + const file = readdirSync(dir).find((f) => f.endsWith("-accepted.md")); + assert.ok(file); + const body = readFileSync(join(dir, file), "utf8"); + assert.match(body, /- session cost: \$0\.0004 \(worker\)/); + assert.match(body, /- judge 1 call · 1000 in \(400 cached, 50 cache write\) · 200 out · \$0\.0033 so far/); + const entries = Journal.read(journal.path).filter((x) => x.kind === "judge_usage"); + assert.equal(entries.length, 4); + const d = entries[1]!.data as Record; + assert.equal(d.model, JUDGE); + assert.equal(d.what, "reviewing the claim"); + assert.equal(d.promptTokens, 1000); + assert.equal(d.cacheReadTokens, 400); + assert.equal(d.cacheWriteTokens, 50); + assert.ok(Math.abs((d.costUsd as number) - PER_CALL) < 1e-12); + assert.ok(Journal.summarize(entries.slice(1)).every((l) => /judge judge-j \(reviewing the claim\) · 1000 in \(400 cached\) \/ 200 out · \$0\.0032/.test(l))); + } finally { + ws.cleanup(); + } + }); + + it("an unpriced judge model reports its tokens and $ unknown, never $0", async () => { + const { ws, events, journal } = await run(false); + try { + const e = jobEnd(events); + assert.ok(e.judge); + assert.equal(e.judge.promptTokens, 4000); + assert.equal(e.judge.costUsd, undefined, "no price means no dollar figure"); + assert.equal(e.judge.unpricedCalls, 4); + assert.deepEqual(e.judge.unpricedModels, [JUDGE]); + assert.match(judgeSpendLine(e.judge), /^judge 4 calls · 4000 in \(1600 cached, 200 cache write\) · 800 out · \$ unknown \(no price for judge-j\)$/); + const row = new Receipts(ws.dir).records().find((r) => r.verdict === "accepted"); + assert.equal(row?.judge?.costUsd, undefined); + const entries = Journal.read(journal.path).filter((x) => x.kind === "judge_usage"); + assert.equal((entries[0]!.data as Record).costUsd, null, "null, not 0"); + assert.ok(Journal.summarize(entries).every((l) => /\$ unknown$/.test(l))); + // The receipt file says so in words. + const dir = join(ws.dir, ".maat", "receipts"); + const file = readdirSync(dir).find((f) => f.endsWith("-accepted.md")); + assert.ok(file); + const body = readFileSync(join(dir, file), "utf8"); + assert.match(body, /- judge 1 call · 1000 in \(400 cached, 50 cache write\) · 200 out · \$ unknown \(no price for judge-j\) so far/); + assert.doesNotMatch(body, /judge \d+ calls? · .*\$0\.0000/); + } finally { + ws.cleanup(); + } + }); +}); + +describe("a budget counts the judge's spend", () => { + it("a token budget the judge has used up stops the worker before its first step", async () => { + const ws = workspace(); + try { + let asked = 0; + const fetchFn = (async () => { + asked += 1; + return reply({ choices: [{ message: { role: "assistant", content: "Done." }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 5 } }); + }) as unknown as typeof fetch; + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high" }); + engine.setBudget(5_000); + // The drafting before the turn: on the judge's meter, not the worker's. + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "drafting criteria", promptTokens: 4_800, completionTokens: 400, estimated: false }); + assert.equal(engine.sessionTokens, 0); + const events = await drain(engine.run("anything", allowAll)); + const hit = events.find((x) => x.kind === "error" && x.ceiling === "budget"); + assert.ok(hit && hit.kind === "error", "the budget was hit"); + assert.match(hit.text, /budget hit \(5000 tokens, 5200 of them the judge's\)/); + assert.equal(asked, 1, "only the closing summary was asked for; no work step ran"); + const e = jobEnd(events); + assert.equal(e.judge?.promptTokens, 4_800, "the pre-turn drafting is reported with the turn it was for"); + } finally { + ws.cleanup(); + } + }); + + it("a money ceiling counts the judge's dollars", async () => { + const ws = workspace(); + try { + const fetchFn = (async () => + reply({ choices: [{ message: { role: "assistant", content: null, tool_calls: [{ id: "c", type: "function", function: { name: "list_files", arguments: "{}" } }] }, finish_reason: "tool_calls" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })) as unknown as typeof fetch; + const engine = new Engine({ + baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high", + priceInPerMtok: 1, priceOutPerMtok: 1, maxTurnUsd: 0.01, maxSteps: 3, + }); + engine.judgeMeter.setPricing(URL, JUDGE, { in: 1000, out: 1000, source: "test" }); + const gen = engine.run("anything", allowAll); + const events: EngineEvent[] = []; + for await (const ev of gen) { + events.push(ev); + // The judge spends $0.02 mid-turn: past the $0.01 ceiling on its own. + if (ev.kind === "usage" && events.filter((x) => x.kind === "usage").length === 1) { + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, promptTokens: 10, completionTokens: 10, estimated: false }); + } + } + assert.equal(engine.totalCostUsd()! > 0.02, true); + assert.ok( + events.some((x) => (x.kind === "error" || x.kind === "info") && /\$0\.0\d+ of \$0\.01/.test(x.text)), + "the turn's money ceiling saw the judge's spend", + ); + } finally { + ws.cleanup(); + } + }); +}); + +describe("JudgeMeter", () => { + const u = { baseUrl: URL, model: JUDGE, promptTokens: 1000, completionTokens: 200, cacheReadTokens: 400, estimated: false }; + + it("prices a call with a set price, keyed on the judge's model id", () => { + const m = new JudgeMeter(); + m.setPricing(URL, JUDGE, PRICE); + m.record(u); + assert.ok(Math.abs(m.total().costUsd! - PER_CALL) < 1e-12); + assert.equal(priceTokens(PRICE, u), m.total().costUsd); + }); + + it("prices a call again once a price arrives after it", () => { + const m = new JudgeMeter(); + m.record(u); + assert.equal(m.total().costUsd, undefined); + m.setPricing(URL, JUDGE, PRICE); + assert.ok(Math.abs(m.total().costUsd! - PER_CALL) < 1e-12); + }); + + it("uses the provider's billed figure when it sends one", () => { + const m = new JudgeMeter(); + m.record({ ...u, billedUsd: 0.5 }); + assert.equal(m.total().costUsd, 0.5); + assert.equal(m.total().billed, true); + }); + + it("is unknown, not a partial sum, when any call is unpriced", () => { + const m = new JudgeMeter(); + m.setPricing(URL, JUDGE, PRICE); + m.record(u); + m.record({ ...u, model: "other" }); + const s = m.total(); + assert.equal(s.costUsd, undefined); + assert.equal(s.unpricedCalls, 1); + assert.deepEqual(s.unpricedModels, ["other"]); + }); + + it("a judge that is the worker's own model is priced as the worker is", () => { + const m = new JudgeMeter(() => ({ baseUrl: URL, model: "m", pricing: PRICE })); + m.record({ ...u, model: "m" }); + assert.ok(Math.abs(m.total().costUsd! - PER_CALL) < 1e-12); + }); + + it("knows Anthropic's published rates on Anthropic's API", () => { + const m = new JudgeMeter(); + m.record({ ...u, baseUrl: "https://api.anthropic.com/v1", model: "claude-haiku-4-5", cacheReadTokens: 0 }); + // Haiku 4.5: $1 in, $5 out. + assert.ok(Math.abs(m.total().costUsd! - (1000 * 1 + 200 * 5) / 1e6) < 1e-12); + }); + + it("a subscription judge is paid by its plan: no dollar figure, said as the plan", () => { + const m = new JudgeMeter(); + m.record({ ...u, baseUrl: "grok-build://subscription", estimated: true }); + const s = m.total(); + assert.equal(s.costUsd, undefined); + assert.equal(s.unpricedCalls, 0); + assert.match(judgeSpendLine(s), /your .+ plan, not metered$/); + }); + + it("since() reports only the calls after a mark", () => { + const m = new JudgeMeter(); + m.record(u); + const at = m.mark(); + m.record({ ...u, promptTokens: 7 }); + assert.equal(m.since(at).calls, 1); + assert.equal(m.since(at).promptTokens, 7); + }); +}); + +describe("askModel reports usage to its meter", () => { + it("reads the provider's usage block, cache read and write included", async () => { + const m = new JudgeMeter(); + const fetchFn = (async () => reply({ choices: [{ message: { content: "ok" }, finish_reason: "stop" }], usage: { ...JUDGE_USAGE, cost: 0.004 } })) as unknown as typeof fetch; + const r = await askModel({ baseUrl: URL, model: JUDGE, system: "S", prompt: "P", fetchFn, meter: m, what: "drafting criteria" }); + assert.equal(r.ok, true); + const [c] = m.all(); + assert.deepEqual( + { p: c!.promptTokens, c: c!.completionTokens, r: c!.cacheReadTokens, w: c!.cacheWriteTokens, b: c!.billedUsd, e: c!.estimated, what: c!.what }, + { p: 1000, c: 200, r: 400, w: 50, b: 0.004, e: false, what: "drafting criteria" }, + ); + }); + + it("estimates, and says so, when the provider reports no usage", async () => { + const m = new JudgeMeter(); + const fetchFn = (async () => reply({ choices: [{ message: { content: "abcdefgh" }, finish_reason: "stop" }] })) as unknown as typeof fetch; + await askModel({ baseUrl: URL, model: JUDGE, system: "SSSS", prompt: "PPPP", fetchFn, meter: m }); + const [c] = m.all(); + assert.equal(c!.estimated, true); + assert.equal(c!.promptTokens, 2); + assert.equal(c!.completionTokens, 2); + }); + + it("records nothing for a refused ask", async () => { + const m = new JudgeMeter(); + const fetchFn = (async () => ({ ok: false, status: 500, text: async () => "", json: async () => ({}) }) as unknown as Response) as unknown as typeof fetch; + await askModel({ baseUrl: URL, model: JUDGE, system: "S", prompt: "P", fetchFn, meter: m, overloadBackoffMs: [] }); + assert.equal(m.mark(), 0); + }); +}); diff --git a/ui/app.ts b/ui/app.ts index dbb764d..6d66247 100644 --- a/ui/app.ts +++ b/ui/app.ts @@ -15,7 +15,7 @@ import { renderMarkdown } from "./markdown.js"; import { nextWaitWord } from "./wait-words.js"; import { playMaatSplash } from "./maat-splash.js"; -import { fmtCost, spendLine, stepDid } from "../src/format.js"; +import { fmtCost, judgeSpendLine, spendLine, stepDid } from "../src/format.js"; import { jobEndWords, type JobEndWords } from "../src/tiers.js"; import { expandEndpointShorthand, typedEndpointProblem } from "../src/endpoint.js"; import { matchCommands } from "../src/commands.js"; @@ -1143,7 +1143,9 @@ molt.onEvent((ev) => { const said = jobEndWords(ev as JobEndWords); say( "", - `job ${said} · ${ev.steps} step(s) · ${spendLine(ev.spend)} · ${fmtMs(Number(ev.durationMs) || 0)}`, + `job ${said} · ${ev.steps} step(s) · ${spendLine(ev.spend)} · ${fmtMs(Number(ev.durationMs) || 0)}` + + // The judge's share, apart from the worker's (judge-meter.ts). + (ev.judge ? ` · ${judgeSpendLine(ev.judge)}` : ""), "step", ); void refreshStats(); From 41136f85746ff6396ede355222519e8d118b6fc8 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 04:45:06 -0700 Subject: [PATCH 2/3] Judge spend columns in bench rows; document judge metering bench/local/run.py adds judge_calls, judge_tokens_in/out, judge_cache_read/write and judge_cost_usd to each result row when job_end carries the judge's spend (null cost for an unpriced judge, never 0). docs/transparency.md describes the judge's meter and that a budget counts both; docs/lean.md names the new columns. --- CHANGELOG.md | 11 +++++++ bench/local/run.py | 27 ++++++++++++++-- bench/local/tests/test_judge_columns.py | 33 +++++++++++++++++++ docs/lean.md | 7 ++++- docs/transparency.md | 42 +++++++++++++++++++++++++ 5 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 bench/local/tests/test_judge_columns.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7023c6a..1b2f9f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ ### Added +- **The judge's spend is metered.** Every ask made around the work (drafting + and its critic, the reference check, `--review`, the post-work audit, the + arbiter) records its input, output, cache read and cache write tokens, priced + from the worker's price table keyed on the judge's own model id; an unpriced + model shows `$ unknown`, never `$0`. Reported apart from the worker's spend + as `judge` on `job_end`, `judge_usage` journal entries, a `judge` line and a + worker + judge total in the CLI, and `judge` on the receipt (row updated at + the job's end). `--budget` and the per-turn ceilings now count worker and + judge spend together. `bench/local/run.py` adds `judge_*` token and cost + columns to each row. See `docs/transparency.md`. + - **`--review-executable`** (experimental, also `MAAT_REVIEW_EXECUTABLE=1`): every `--review` objection must carry a read-only command that demonstrates it. Maat runs it on a throwaway copy of the tree, with no API keys, tokens or diff --git a/bench/local/run.py b/bench/local/run.py index a12d726..444f237 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -627,7 +627,7 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: write_private(log, out) # Maat's own notices (criteria not drafted, dropped checks, refusals) go to stderr. write_private(log.with_suffix(".err"), err or "") - steps = 0; per = []; outcome = None; spend = {}; review = None; disagree = []; extra = {} + steps = 0; per = []; outcome = None; spend = {}; judge = None; review = None; disagree = []; extra = {} for line in out.splitlines(): if not line.startswith("{"): continue @@ -641,6 +641,9 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: elif ev.get("kind") == "job_end": outcome = claim_of(ev) spend = ev.get("spend") or {} + # The judge's asks (drafting, critic, review, audit), metered apart from + # the worker's. Absent on builds before judge metering, or when no judge ran. + judge = ev.get("judge") review = ev.get("review") disagree = ev.get("checksDisagree") or [] # Only what job_end carried; absent keys stay absent (older builds). @@ -651,7 +654,9 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: return { "provider_capped": capped, "secs": round(secs), "turns": steps, "calls": sum(per), "multi": sum(1 for x in per if x > 1), - "tokens_in": spend.get("promptTokens"), "cost_usd": spend.get("costUsd"), "claim": outcome, "timed_out": timed_out, + "tokens_in": spend.get("promptTokens"), "cost_usd": spend.get("costUsd"), + **judge_columns(judge), + "claim": outcome, "timed_out": timed_out, "said_done": (outcome or "").startswith("verified"), # The reviewer's label: a verified claim it did not confirm. "review": review, @@ -662,6 +667,24 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: } +def judge_columns(judge: dict | None) -> dict: + """The judge's tokens and $, next to the worker's `tokens_in` / `cost_usd`. + + Only when the run output carried them (job_end.judge, Maat's judge meter): None + otherwise, so an older build is never read as a free judge. `judge_cost_usd` is + None when the judge's model had no price ("$ unknown"), never 0.""" + if not isinstance(judge, dict): + return {} + return { + "judge_calls": judge.get("calls"), + "judge_tokens_in": judge.get("promptTokens"), + "judge_tokens_out": judge.get("completionTokens"), + "judge_cache_read": judge.get("cacheReadTokens"), + "judge_cache_write": judge.get("cacheWriteTokens"), + "judge_cost_usd": judge.get("costUsd"), + } + + # Per-run cost alarm (reports/loop-charter.md, non-regression rule 1): a run that # costs more than max(5x the lane's running median cost, $0.10), or sends more # than 1.5M prompt tokens, stops the lane before the bill grows. 2026-10-07: one diff --git a/bench/local/tests/test_judge_columns.py b/bench/local/tests/test_judge_columns.py new file mode 100644 index 0000000..09d90fa --- /dev/null +++ b/bench/local/tests/test_judge_columns.py @@ -0,0 +1,33 @@ +"""The judge's tokens and $ in each result row, next to the worker's. + +Maat meters the judge's asks (drafting, critic, review, audit) apart from the +worker's requests and reports them on job_end as `judge`. A row carries them only +when the run output did, and an unpriced judge is None, never 0. +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import run # noqa: E402 + + +def test_columns_from_job_end_judge(): + cols = run.judge_columns({ + "calls": 4, "promptTokens": 12000, "completionTokens": 900, + "cacheReadTokens": 3000, "cacheWriteTokens": 500, "costUsd": 0.0123, + }) + assert cols == { + "judge_calls": 4, "judge_tokens_in": 12000, "judge_tokens_out": 900, + "judge_cache_read": 3000, "judge_cache_write": 500, "judge_cost_usd": 0.0123, + } + + +def test_unpriced_judge_is_none_not_zero(): + cols = run.judge_columns({"calls": 2, "promptTokens": 800, "completionTokens": 100, + "cacheReadTokens": 0, "cacheWriteTokens": 0, "unpricedCalls": 2}) + assert cols["judge_tokens_in"] == 800 + assert cols["judge_cost_usd"] is None + + +def test_no_judge_in_the_output_adds_no_columns(): + assert run.judge_columns(None) == {} diff --git a/docs/lean.md b/docs/lean.md index 7246761..8094f5d 100644 --- a/docs/lean.md +++ b/docs/lean.md @@ -83,7 +83,12 @@ part of this. After each run, run.py reads the run's cost (`spend.costUsd`) and prompt tokens (`spend.promptTokens`) from Maat's `job_end` event, and records the cost -in the row as `cost_usd`. The run trips the alarm when: +in the row as `cost_usd`. The judge's spend, when `job_end` carries it +(`judge`, see transparency.md), goes in the same row beside the worker's: +`judge_calls`, `judge_tokens_in`, `judge_tokens_out`, `judge_cache_read`, +`judge_cache_write` and `judge_cost_usd` (null when the judge's model has no +price, never 0). The alarm below is on the worker's cost and tokens. The run +trips the alarm when: - its cost exceeds **max(5x the lane's running median cost, $0.10)**. The median is over the lane's earlier runs that reported a cost (resumed rows diff --git a/docs/transparency.md b/docs/transparency.md index ca122d5..8d06a6d 100644 --- a/docs/transparency.md +++ b/docs/transparency.md @@ -152,6 +152,48 @@ The bottom line is the session meter and only ever climbs. **Per-job** figures — what one question cost — live in the view, measured as a delta against the session meter rather than by resetting it. +### The judge's spend + +The worker's requests are not the only ones a run pays for. Around the work, +Maat asks the judge: it drafts the hidden checks, a critic reads the draft, a +reference check is written (`--reference`), the independent review votes +(`--review`), the post-work audit drafts its checks, and the arbiter rules on a +dispute. With `--judge` (`MAAT_JUDGE_MODEL`) those asks go to another model, +often on another provider. Each one is metered on the **judge's meter** +(`src/judge-meter.ts`), apart from the worker's: + +- **Tokens** — input, output, cache read and cache write — come from the + judge's own usage block, per call. A reply with no usage is counted from its + length and marked `~`, as on the worker side. +- **Prices** come from the same table as the worker's (`providers.ts`), keyed + on the judge's model id: its provider's published list (xAI, OpenRouter), + Anthropic's rates on Anthropic's API, or the worker's own prices when the + judge is the worker's model. A provider-reported dollar figure is used as + is. A model with no known price shows its tokens and **`$ unknown`**, never + `$0`. A judge on a subscription (Grok Build, OpenCode) is paid by the plan + and gets no dollar figure. +- **Where it shows:** a `judge_usage` journal entry per call; `judge` on + `job_end` (every judge call since the previous job, so the drafting before a + turn is counted with that turn); a `judge ...` line under the CLI's job line + and under the session totals, followed by `total $x (worker $a + judge $b)` + when both are priced; and the receipt, which carries the judge's spend so far + when the claim is made, with the index row (`judge`) brought up to date when + the job ends, since the review and the audit run after the receipt is written. + +The worker's figures (`spend`, `sessionTokens`, the step lines, `costUsd` on a +receipt) stay the worker's: the lean-budget suite measures the worker's +requests, and the judge's are not among them. + +**A budget counts both.** `--budget` / `/budget ` is checked against +the worker's tokens plus the judge's, and the per-turn ceiling in tokens or +dollars counts the judge's spend in that turn too: a budget is what you are +willing to spend, and the judge's tokens are billed like any others. When a +budget is hit with judge spend in it, the message says how much was the +judge's. A judge with no known price adds its tokens to a token budget and +nothing to a dollar ceiling (its spend there is unknown, and said so). The +budget is checked before every worker step; a judge ask already under way is +not cut off. + ## Work that goes nowhere A tool call that returns exactly what it returned before has taught the model From 44bb43dcc5cb0c54b8f4c8896a96abe60aa0327c Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 04:52:05 -0700 Subject: [PATCH 3/3] Bench cost alarm judges worker plus judge spend The per-run alarm now compares the run's total cost (worker + judge) and total prompt tokens against its limits, and the lane median is over the same totals, so a runaway judge stops the lane too. When the run had a judge the detail shows the split, and the STOPPED row carries the worker and judge figures beside the totals. --- CHANGELOG.md | 4 +- bench/local/run.py | 55 ++++++++++++++++++------ bench/local/tests/test_cost_alarm.py | 64 ++++++++++++++++++++++++++++ docs/lean.md | 10 ++++- 4 files changed, 118 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b2f9f4..b71cbf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,9 @@ worker + judge total in the CLI, and `judge` on the receipt (row updated at the job's end). `--budget` and the per-turn ceilings now count worker and judge spend together. `bench/local/run.py` adds `judge_*` token and cost - columns to each row. See `docs/transparency.md`. + columns to each row, and its cost alarm now judges worker + judge cost and + prompt tokens (the detail shows the split), so a runaway judge stops the lane + too. See `docs/transparency.md` and `docs/lean.md`. - **`--review-executable`** (experimental, also `MAAT_REVIEW_EXECUTABLE=1`): every `--review` objection must carry a read-only command that demonstrates diff --git a/bench/local/run.py b/bench/local/run.py index 444f237..0d6e954 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -701,24 +701,55 @@ def alarm_limits() -> dict: } +def _sum_known(*xs): + """The sum of the values that are known, or None when none is.""" + known = [x for x in xs if x is not None] + return sum(known) if known else None + + +def run_cost(row: dict) -> float | None: + """A run's whole cost: the worker's plus the judge's (judge_columns). None when + neither is known; an unpriced judge adds nothing it cannot price.""" + return _sum_known(row.get("cost_usd"), row.get("judge_cost_usd")) + + +def run_tokens(row: dict) -> int | None: + """A run's prompt tokens, the worker's plus the judge's.""" + return _sum_known(row.get("tokens_in"), row.get("judge_tokens_in")) + + def cost_alarm(row: dict, prior_costs: list[float]) -> dict | None: - """Why this run trips the alarm, or None. `prior_costs` are the lane's earlier runs - (this run excluded): the median is over runs that reported a cost. A run with no - cost (an unpriced model) is judged on its prompt tokens only.""" + """Why this run trips the alarm, or None. `prior_costs` are the lane's earlier runs' + total costs (this run excluded): the median is over runs that reported a cost. A + run with no cost (an unpriced model) is judged on its prompt tokens only. + + Cost and tokens are the worker's plus the judge's: a runaway judge trips it too. + When the run had a judge, the detail says how the total splits.""" lim = alarm_limits() med = statistics.median(prior_costs) if prior_costs else None limit_usd = max(lim["x"] * med, lim["usd"]) if med is not None else lim["usd"] - cost, toks = row.get("cost_usd"), row.get("tokens_in") + cost, toks = run_cost(row), run_tokens(row) + judged = row.get("judge_calls") is not None + + def usd(x): + return "$ unknown" if x is None else f"${x:.4f}" + why = [] if cost is not None and cost > limit_usd: - why.append(f"cost ${cost:.4f} > ${limit_usd:.4f} (max({lim['x']:g}x lane median " + split = f" (worker {usd(row.get('cost_usd'))} + judge {usd(row.get('judge_cost_usd'))})" if judged else "" + why.append(f"cost ${cost:.4f}{split} > ${limit_usd:.4f} (max({lim['x']:g}x lane median " f"{'n/a' if med is None else f'${med:.4f}'}, ${lim['usd']:g}))") if toks is not None and toks > lim["tokens"]: - why.append(f"prompt tokens {toks:,} > {lim['tokens']:,}") + split = (f" (worker {row.get('tokens_in') or 0:,} + judge {row.get('judge_tokens_in') or 0:,})" + if judged else "") + why.append(f"prompt tokens {toks:,}{split} > {lim['tokens']:,}") if not why: return None - return {"detail": "; ".join(why), "cost_usd": cost, "tokens_in": toks, "lane_median_usd": med, - "limit_usd": round(limit_usd, 6), "limit_tokens": lim["tokens"]} + return {"detail": "; ".join(why), "cost_usd": cost, "tokens_in": toks, + **({"worker_cost_usd": row.get("cost_usd"), "judge_cost_usd": row.get("judge_cost_usd"), + "worker_tokens_in": row.get("tokens_in"), "judge_tokens_in": row.get("judge_tokens_in")} + if judged else {}), + "lane_median_usd": med, "limit_usd": round(limit_usd, 6), "limit_tokens": lim["tokens"]} def claims_done(text: str) -> bool: @@ -878,8 +909,8 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: if r.get("stopped"): # a STOPPED marker, not a run continue done.add((r["task"], r["agent"], r["rep"], r.get("arm"))) - if r.get("cost_usd") is not None: - lane_costs.setdefault(r.get("arm"), []).append(r["cost_usd"]) + if run_cost(r) is not None: + lane_costs.setdefault(r.get("arm"), []).append(run_cost(r)) global SCRUBBER SCRUBBER = make_scrubber() for rep in range(repeats): @@ -937,8 +968,8 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: alarm = cost_alarm(r, costs) if alarm: stop_lane(out, tag, EXPORT / f"{tag}.log", alarm, arm, lid) - if r.get("cost_usd") is not None: - costs.append(r["cost_usd"]) + if run_cost(r) is not None: + costs.append(run_cost(r)) def stop_lane(out: Path, tag: str, log: Path, alarm: dict, arm: str | None, lid: str | None = None) -> None: diff --git a/bench/local/tests/test_cost_alarm.py b/bench/local/tests/test_cost_alarm.py index 341fe88..e315a19 100644 --- a/bench/local/tests/test_cost_alarm.py +++ b/bench/local/tests/test_cost_alarm.py @@ -129,3 +129,67 @@ def test_resume_skips_the_marker_and_keeps_the_median(monkeypatch, tmp_path): # Earlier run's $0.10 is the median: limit max(0.50, 0.10), so $0.40 passes. run.main("molt", 1, None) assert json.loads(out.read_text().splitlines()[-1])["task"] == "t0" + + +def test_a_runaway_judge_trips_the_alarm_and_the_detail_shows_the_split(): + # The worker alone is well under the floor; the judge pushes the run over it. + row = {"cost_usd": 0.03, "tokens_in": 40_000, "judge_calls": 9, "judge_cost_usd": 0.25, "judge_tokens_in": 300_000} + a = run.cost_alarm(row, []) + assert a, "worker + judge is what the alarm judges" + assert "cost $0.2800 (worker $0.0300 + judge $0.2500) > $0.1000" in a["detail"] + assert a["cost_usd"] == pytest.approx(0.28) + assert a["worker_cost_usd"] == 0.03 and a["judge_cost_usd"] == 0.25 + assert a["tokens_in"] == 340_000 and a["judge_tokens_in"] == 300_000 + + +def test_judge_tokens_count_toward_the_token_alarm(): + row = {"cost_usd": None, "tokens_in": 900_000, "judge_calls": 5, "judge_cost_usd": None, "judge_tokens_in": 700_000} + a = run.cost_alarm(row, []) + assert a and "prompt tokens 1,600,000 (worker 900,000 + judge 700,000) > 1,500,000" in a["detail"] + + +def test_an_unpriced_judge_is_said_as_unknown_in_the_split(): + row = {"cost_usd": 0.12, "judge_calls": 2, "judge_cost_usd": None, "judge_tokens_in": 1_000} + a = run.cost_alarm(row, []) + assert a and "cost $0.1200 (worker $0.1200 + judge $ unknown)" in a["detail"] + + +def test_a_run_with_no_judge_reads_as_before(): + a = run.cost_alarm({"cost_usd": 0.11, "tokens_in": 1000}, []) + assert a and "(worker" not in a["detail"] and "judge_cost_usd" not in a + + +def test_the_lane_median_is_over_worker_plus_judge(monkeypatch, tmp_path): + out = _lane(monkeypatch, tmp_path, [(0.02, 50_000), (0.03, 60_000), (0.02, 50_000)]) + judge = iter([0.08, 0.07, 0.30]) + inner = run.run_molt + + def with_judge(d, prompt, log): + r = inner(d, prompt, log) + return {**r, "judge_calls": 4, "judge_cost_usd": next(judge), "judge_tokens_in": 10_000} + + monkeypatch.setattr(run, "run_molt", with_judge) + # Totals 0.10, 0.10, 0.32: median 0.10 -> limit max(0.50, 0.10); 0.32 passes. + run.main("molt", 1, None) + rows = [json.loads(x) for x in out.read_text().splitlines()] + assert len(rows) == 3 and not any(r.get("stopped") for r in rows) + assert [r["judge_cost_usd"] for r in rows] == [0.08, 0.07, 0.30] + + +def test_a_lane_stops_on_a_runaway_judge(monkeypatch, tmp_path): + out = _lane(monkeypatch, tmp_path, [(0.02, 50_000), (0.02, 50_000), (0.02, 50_000), (0.02, 50_000)]) + judge = iter([0.01, 0.01, 0.90, 0.01]) + inner = run.run_molt + + def with_judge(d, prompt, log): + r = inner(d, prompt, log) + return {**r, "judge_calls": 4, "judge_cost_usd": next(judge), "judge_tokens_in": 10_000} + + monkeypatch.setattr(run, "run_molt", with_judge) + with pytest.raises(SystemExit) as e: + run.main("molt", 1, None) + assert e.value.code == run.COST_ALARM_EXIT + stop = [json.loads(x) for x in out.read_text().splitlines()][-1] + assert stop["stopped"] is True and stop["run"] == "t2-molt-0" + assert stop["worker_cost_usd"] == 0.02 and stop["judge_cost_usd"] == 0.90 + assert "(worker $0.0200 + judge $0.9000)" in stop["detail"] diff --git a/docs/lean.md b/docs/lean.md index 8094f5d..586cdf1 100644 --- a/docs/lean.md +++ b/docs/lean.md @@ -87,8 +87,14 @@ in the row as `cost_usd`. The judge's spend, when `job_end` carries it (`judge`, see transparency.md), goes in the same row beside the worker's: `judge_calls`, `judge_tokens_in`, `judge_tokens_out`, `judge_cache_read`, `judge_cache_write` and `judge_cost_usd` (null when the judge's model has no -price, never 0). The alarm below is on the worker's cost and tokens. The run -trips the alarm when: +price, never 0). The alarm below is on the run's whole spend: the worker's +cost plus the judge's, and the worker's prompt tokens plus the judge's, so a +runaway judge trips it too (an unpriced judge adds tokens, not dollars). When +the run had a judge, the detail shows the split, e.g. `cost $0.2800 (worker +$0.0300 + judge $0.2500) > $0.1000 ...`, and the `STOPPED` row adds +`worker_cost_usd`, `judge_cost_usd`, `worker_tokens_in` and `judge_tokens_in`; +`cost_usd` and `tokens_in` there are the totals. The lane median is over the +same totals. The run trips the alarm when: - its cost exceeds **max(5x the lane's running median cost, $0.10)**. The median is over the lane's earlier runs that reported a cost (resumed rows