From 87aa6de8469590b50ad2a32a017ca11edaa3e7a2 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 05:22:12 -0700 Subject: [PATCH 1/3] Judge meter follow-ups from the #56 review - The per-turn dollar ceiling prices the turn as worker plus judge whenever either is priced, so an unpriced (local or subscription) worker with a paid judge is still held to a dollar ceiling. - The bench lane median leaves out runs whose judge ran unpriced: their total is a lower bound and would pull the limit down. The run's own alarm still uses the lower bound. - JudgeMeter.since() drops the figure stored at record time before re-pricing, so a cleared price is reflected; the journal keeps the figure as it stood. - The budget-hit message says when the judge's drafting used the budget before the worker's first step. - The judge's and the worker's price lookups run in parallel. - amendReview gets its doc comment back; one import from format.js. --- CHANGELOG.md | 3 ++ bench/local/run.py | 21 +++++++-- bench/local/tests/test_cost_alarm.py | 26 +++++++++++ docs/lean.md | 4 +- docs/transparency.md | 15 ++++-- src/cli.tsx | 10 ++-- src/engine.ts | 34 ++++++++++---- src/format.ts | 7 +-- src/judge-meter.ts | 6 ++- src/receipts.ts | 16 ++++--- test/judge-meter.test.ts | 70 ++++++++++++++++++++++++++++ 11 files changed, 179 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b71cbf0..1605070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ columns to each row, and its cost alarm now judges worker + judge cost and prompt tokens (the detail shows the split), so a runaway judge stops the lane too. See `docs/transparency.md` and `docs/lean.md`. + The per-turn dollar ceiling counts a priced judge even when the worker is + unpriced; the lane median leaves out runs whose judge was unpriced; a budget + used up by drafting before the first step says so. - **`--review-executable`** (experimental, also `MAAT_REVIEW_EXECUTABLE=1`): every `--review` objection must carry a read-only command that demonstrates diff --git a/bench/local/run.py b/bench/local/run.py index 0d6e954..47664ad 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -713,6 +713,19 @@ def run_cost(row: dict) -> float | None: return _sum_known(row.get("cost_usd"), row.get("judge_cost_usd")) +def median_cost(row: dict) -> float | None: + """A run's cost as the lane median counts it, or None to leave it out. + + A run whose judge ran but had no price is left out: its total is the worker's + cost alone, a lower bound, and averaged with runs whose judge was priced it would + pull the median (and so the limit) down without anyone seeing why. The alarm on + that run itself still uses the lower bound (run_cost): under-counting there only + makes the alarm late, never wrong.""" + if row.get("judge_calls") is not None and row.get("judge_cost_usd") is None: + return None + return run_cost(row) + + def run_tokens(row: dict) -> int | None: """A run's prompt tokens, the worker's plus the judge's.""" return _sum_known(row.get("tokens_in"), row.get("judge_tokens_in")) @@ -909,8 +922,8 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: if r.get("stopped"): # a STOPPED marker, not a run continue done.add((r["task"], r["agent"], r["rep"], r.get("arm"))) - if run_cost(r) is not None: - lane_costs.setdefault(r.get("arm"), []).append(run_cost(r)) + if median_cost(r) is not None: + lane_costs.setdefault(r.get("arm"), []).append(median_cost(r)) global SCRUBBER SCRUBBER = make_scrubber() for rep in range(repeats): @@ -968,8 +981,8 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: alarm = cost_alarm(r, costs) if alarm: stop_lane(out, tag, EXPORT / f"{tag}.log", alarm, arm, lid) - if run_cost(r) is not None: - costs.append(run_cost(r)) + if median_cost(r) is not None: + costs.append(median_cost(r)) def stop_lane(out: Path, tag: str, log: Path, alarm: dict, arm: str | None, lid: str | None = None) -> None: diff --git a/bench/local/tests/test_cost_alarm.py b/bench/local/tests/test_cost_alarm.py index e315a19..be9a688 100644 --- a/bench/local/tests/test_cost_alarm.py +++ b/bench/local/tests/test_cost_alarm.py @@ -193,3 +193,29 @@ def with_judge(d, prompt, log): assert stop["stopped"] is True and stop["run"] == "t2-molt-0" assert stop["worker_cost_usd"] == 0.02 and stop["judge_cost_usd"] == 0.90 assert "(worker $0.0200 + judge $0.9000)" in stop["detail"] + + +def test_a_run_with_an_unpriced_judge_is_left_out_of_the_median(): + assert run.median_cost({"cost_usd": 0.02, "judge_calls": 3, "judge_cost_usd": None}) is None + assert run.median_cost({"cost_usd": 0.02, "judge_calls": 3, "judge_cost_usd": 0.01}) == pytest.approx(0.03) + assert run.median_cost({"cost_usd": 0.02}) == 0.02 # no judge: the worker's cost, as before + # Its own alarm still uses the lower bound. + assert run.cost_alarm({"cost_usd": 0.12, "judge_calls": 3, "judge_cost_usd": None}, []) + + +def test_the_lane_median_skips_unpriced_judge_runs(monkeypatch, tmp_path): + # Priced-judge runs total $0.10; two runs with an unpriced judge cost $0.01 (worker + # only). Counted, they would drag the median to $0.01 and the limit to the $0.10 + # floor, stopping the fifth run ($0.30). Left out, the median is $0.10, limit $0.50. + out = _lane(monkeypatch, tmp_path, [(0.02, 1), (0.01, 1), (0.01, 1), (0.02, 1), (0.02, 1)]) + judge = iter([0.08, None, None, 0.08, 0.28]) + inner = run.run_molt + + def with_judge(d, prompt, log): + r = inner(d, prompt, log) + return {**r, "judge_calls": 2, "judge_cost_usd": next(judge), "judge_tokens_in": 1} + + monkeypatch.setattr(run, "run_molt", with_judge) + run.main("molt", 1, None) + rows = [json.loads(x) for x in out.read_text().splitlines()] + assert len(rows) == 5 and not any(r.get("stopped") for r in rows) diff --git a/docs/lean.md b/docs/lean.md index 586cdf1..07e8470 100644 --- a/docs/lean.md +++ b/docs/lean.md @@ -94,7 +94,9 @@ the run had a judge, the detail shows the split, e.g. `cost $0.2800 (worker $0.0300 + judge $0.2500) > $0.1000 ...`, and the `STOPPED` row adds `worker_cost_usd`, `judge_cost_usd`, `worker_tokens_in` and `judge_tokens_in`; `cost_usd` and `tokens_in` there are the totals. The lane median is over the -same totals. The run trips the alarm when: +same totals, leaving out runs whose judge ran unpriced: their total is only a +lower bound, and mixed with priced runs it would pull the limit down unseen. +Such a run's own alarm still uses that lower bound. The run trips the alarm when: - its cost exceeds **max(5x the lane's running median cost, $0.10)**. The median is over the lane's earlier runs that reported a cost (resumed rows diff --git a/docs/transparency.md b/docs/transparency.md index 8d06a6d..a9207f7 100644 --- a/docs/transparency.md +++ b/docs/transparency.md @@ -189,10 +189,17 @@ the worker's tokens plus the judge's, and the per-turn ceiling in tokens or dollars counts the judge's spend in that turn too: a budget is what you are willing to spend, and the judge's tokens are billed like any others. When a budget is hit with judge spend in it, the message says how much was the -judge's. A judge with no known price adds its tokens to a token budget and -nothing to a dollar ceiling (its spend there is unknown, and said so). The -budget is checked before every worker step; a judge ask already under way is -not cut off. +judge's; when the judge's drafting used it before the worker's first step, +the message says that too. The turn is priced in dollars when either side is +priced: a local or subscription worker with a paid judge is still held to a +dollar ceiling by the judge's dollars. A side with no known price adds its +tokens to a token budget and nothing to a dollar ceiling (its spend there is +unknown, and said so). The budget is checked before every worker step; a judge +ask already under way is not cut off. + +Judge calls are priced when they are read, not when they were made: a price +that arrives later applies to every call, and a price that is cleared stops +applying (the journal keeps each call's figure as it stood at the time). ## Work that goes nowhere diff --git a/src/cli.tsx b/src/cli.tsx index fe68c74..28cfa4b 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -16,8 +16,7 @@ import { resolve } from "node:path"; import { Archive } from "./archive.js"; import { isAutonomy, type Autonomy } from "./autonomy.js"; import { fmtCost, fmtDuration } from "./banner.js"; -import { judgeSpendLine } from "./format.js"; -import { stepDid } from "./format.js"; +import { judgeSpendLine, stepDid } from "./format.js"; import { BarError, hasBar, loadBar, selectChecks, writeDefaultBar } from "./bar.js"; import { Engine, type FileAccess } from "./engine.js"; import { describeDrift, driftSince } from "./git.js"; @@ -927,7 +926,12 @@ function engineFor(args: Args, session = false, extra: { files?: FileAccess } = * nothing, no cost is shown at all. */ async function priceEngine(engine: Engine, args: Args): Promise { - await priceJudge(engine, args); + // Side by side: each lookup is bounded, and neither needs the other's answer. + await Promise.all([priceJudge(engine, args), priceWorker(engine, args)]); +} + +/** The worker's own price lookup (see priceEngine). */ +async function priceWorker(engine: Engine, args: Args): Promise { if (!needsPriceLookup(args.model, engine.pricing(), storedEndpoint())) return; const p = await fetchPricing(args.url, args.model, keyForUrl(args.url, args.key)); if (!p) { diff --git a/src/engine.ts b/src/engine.ts index 37f4a44..b60b6c0 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1873,6 +1873,12 @@ function fmtUsd(usd: number): string { return usd >= 0.1 ? `$${usd.toFixed(2)}` : usd >= 0.001 ? `$${usd.toFixed(3)}` : "<$0.001"; } +/** The sum of the values that are known, or undefined when none is. */ +function sumKnown(...xs: (number | undefined)[]): number | undefined { + const known = xs.filter((x): x is number => x !== undefined); + return known.length ? known.reduce((a, b) => a + b, 0) : undefined; +} + /** A tool argument that should be a non-empty string, or nothing. */ function str(v: unknown): string | undefined { return typeof v === "string" && v.trim() !== "" ? v : undefined; @@ -3560,15 +3566,14 @@ export class Engine { } /** - * Worker and judge USD together: what a money budget counts. Undefined when - * the worker has no price. A judge with no price adds nothing here (its + * Worker and judge USD together: what a money budget counts. Defined when + * either is priced, so a local or subscription worker with a paid judge is + * still bounded by a dollar ceiling. An unpriced side adds nothing here (its * tokens still count toward a token budget, and every surface says its $ is * unknown); a judge on a subscription plan costs no money. */ totalCostUsd(): number | undefined { - const worker = this.costUsd(); - if (worker === undefined) return undefined; - return worker + (this.judgeMeter.total().costUsd ?? 0); + return sumKnown(this.costUsd(), this.judgeMeter.total().costUsd); } /** The whole session's judge spend (judge-meter.ts). */ @@ -6579,7 +6584,10 @@ export class Engine { // Both ceilings count the judge's spend too (overBudget). const turnStartTokens = this.spentTokens; - const turnStartCost = this.totalCostUsd(); + // Priced per side: the worker's delta when the worker is priced, plus the + // judge's calls this turn when they are. Either one makes the turn priced. + const turnStartCost = this.costUsd(); + const turnJudgeMark = this.judgeMeter.mark(); let warned = 0; // The step guard is the last way out of a turn, and it had the same fault // the spending ceiling had: it stopped dead. A reported run reached it with @@ -6697,7 +6705,13 @@ export class Engine { text: `budget hit (${this.budgetTokens} tokens` + (this.spentTokens > this.sessionTokens ? `, ${this.spentTokens - this.sessionTokens} of them the judge's` : "") + - `) — loop stopped. /budget to raise.`, + `) — loop stopped. /budget to raise.` + + // Hit before the worker took a step, with the judge asked for this + // turn already (its drafting): say what used it up. + (step === 0 && this.judgeMeter.mark() > this.judgeReported + ? ` The judge drafting this turn's checks used the budget before the worker's first step; ` + + `raise --budget to leave room for both.` + : ""), ceiling: "budget", }; yield* this.salvage(`You have reached the token budget for this session.`, fetchFn, log); @@ -6709,8 +6723,10 @@ export class Engine { // refuses to spend rather than what it noticed spending. // Money where a price is known, tokens only where it is not. const spentThisTurn = this.spentTokens - turnStartTokens; - const usdThisTurn = - turnStartCost === undefined ? undefined : (this.totalCostUsd() ?? 0) - turnStartCost; + const usdThisTurn = sumKnown( + turnStartCost === undefined ? undefined : (this.costUsd() ?? 0) - turnStartCost, + this.judgeMeter.since(turnJudgeMark).costUsd, + ); // Zero unless someone set one. The self-hosted exception that used to // live here — no default ceiling on hardware you own — is gone with the // default itself: there is nothing left to make an exception to. diff --git a/src/format.ts b/src/format.ts index 79c4296..f5af84c 100644 --- a/src/format.ts +++ b/src/format.ts @@ -122,11 +122,12 @@ export function spendLine(sp: { } /** - * A judge spend in one line, for the CLI and the receipt: + * A judge spend in one line, for the CLI and the receipt (`label` names whose + * asks they were; a worker's own planning asks use the same shape): * `judge 3 calls · 1200 in (400 cached) · 300 out · $0.004`, with `$ unknown` * when a model had no price and the plan's name when a plan paid. */ -export function judgeSpendLine(s: JudgeSpend, fmt: (usd: number) => string = fmtCost): string { +export function judgeSpendLine(s: JudgeSpend, fmt: (usd: number) => string = fmtCost, label = "judge"): string { const cache = s.cacheReadTokens > 0 || s.cacheWriteTokens > 0 ? ` (${[s.cacheReadTokens > 0 ? `${s.cacheReadTokens} cached` : "", s.cacheWriteTokens > 0 ? `${s.cacheWriteTokens} cache write` : ""].filter(Boolean).join(", ")})` @@ -140,5 +141,5 @@ export function judgeSpendLine(s: JudgeSpend, fmt: (usd: number) => string = fmt : s.plan ? `your ${s.plan} plan, not metered` : "$ unknown"; - return `judge ${s.calls} call${s.calls === 1 ? "" : "s"} · ${e}${s.promptTokens} in${cache} · ${e}${s.completionTokens} out · ${money}`; + return `${label} ${s.calls} call${s.calls === 1 ? "" : "s"} · ${e}${s.promptTokens} in${cache} · ${e}${s.completionTokens} out · ${money}`; } diff --git a/src/judge-meter.ts b/src/judge-meter.ts index 12feb94..a89025f 100644 --- a/src/judge-meter.ts +++ b/src/judge-meter.ts @@ -187,10 +187,12 @@ export class JudgeMeter implements AskMeter { /** * The judge's spend since a mark. Priced now, not when recorded, so a price * that arrived after the first ask (a lookup still in flight, a /price) is - * applied to every call, as the worker's meter does. + * applied to every call, and a price that was cleared is no longer applied + * to any: the figure stored at record time (the journal's) is dropped before + * re-pricing. The worker's meter does the same with its session totals. */ since(mark: number): JudgeSpend { - const span = this.calls.slice(mark).map((c) => this.priced(c)); + const span = this.calls.slice(mark).map(({ costUsd: _recorded, plan: _plan, ...u }) => this.priced(u)); const models: string[] = []; let cost = 0; let unpriced = 0; diff --git a/src/receipts.ts b/src/receipts.ts index e5d847f..78ad362 100644 --- a/src/receipts.ts +++ b/src/receipts.ts @@ -856,15 +856,11 @@ export class Receipts { } } - /** - * Record the independent review's objections on a receipt's index row: each - * with its command and result (`--review-executable`). Masked like every - * other write while hidden commands are withheld. The receipt file itself, - * hash-bound when it was written, is never rewritten. - */ /** * Bring a row's judge spend up to date at the end of its job: the review and - * the post-work audit ask the judge after the receipt is written. + * the post-work audit ask the judge after the receipt is written. Only the + * index row changes; the receipt file, hash-bound when it was written, keeps + * the "so far" figure it was written with. */ amendJudge(file: string, judge: JudgeSpend): boolean { return this.amendRow(file, (r) => ({ ...r, judge })); @@ -895,6 +891,12 @@ export class Receipts { } } + /** + * Record the independent review's objections on a receipt's index row: each + * with its command and result (`--review-executable`). Masked like every + * other write while hidden commands are withheld. The receipt file itself, + * hash-bound when it was written, is never rewritten. + */ amendReview(file: string, objections: readonly Objection[]): boolean { return this.amendRow(file, (r) => ({ ...r, objections: maskDeep([...objections], this.withheld) })); } diff --git a/test/judge-meter.test.ts b/test/judge-meter.test.ts index 652a2fc..2eed200 100644 --- a/test/judge-meter.test.ts +++ b/test/judge-meter.test.ts @@ -232,6 +232,66 @@ describe("a budget counts the judge's spend", () => { }); }); +describe("the turn's money ceiling with an unpriced worker", () => { + it("still counts a priced judge's dollars", async () => { + const ws = workspace(); + try { + const fetchFn = (async () => + reply({ choices: [{ message: { role: "assistant", content: null, tool_calls: [{ id: "c", type: "function", function: { name: "list_files", arguments: "{}" } }] }, finish_reason: "tool_calls" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })) as unknown as typeof fetch; + // No worker price at all: a local or subscription worker with a paid judge. + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high", maxTurnUsd: 0.01, maxSteps: 4 }); + engine.judgeMeter.setPricing(URL, JUDGE, { in: 1000, out: 1000, source: "test" }); + assert.equal(engine.costUsd(), undefined); + const events: EngineEvent[] = []; + for await (const ev of engine.run("anything", allowAll)) { + events.push(ev); + if (ev.kind === "usage" && events.filter((x) => x.kind === "usage").length === 1) { + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, promptTokens: 10, completionTokens: 10, estimated: false }); + } + } + assert.ok(Math.abs(engine.totalCostUsd()! - 0.02) < 1e-12, "the judge's $0.02 is the turn's priced spend"); + assert.ok( + events.some((x) => (x.kind === "error" || x.kind === "info") && /\$0\.020 of \$0\.010/.test(x.text)), + "the dollar ceiling saw the judge's spend", + ); + } finally { + ws.cleanup(); + } + }); +}); + +describe("the budget-hit message", () => { + it("says the drafting used the budget when it was hit before the worker's first step", async () => { + const ws = workspace(); + try { + const fetchFn = (async () => reply({ choices: [{ message: { role: "assistant", content: "Done." }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })) as unknown as typeof fetch; + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high" }); + engine.setBudget(1_000); + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "drafting criteria", promptTokens: 1_200, completionTokens: 0, estimated: false }); + const hit = (await drain(engine.run("anything", allowAll))).find((x) => x.kind === "error" && x.ceiling === "budget"); + assert.ok(hit && hit.kind === "error"); + assert.match(hit.text, /drafting this turn's checks used the budget before the worker's first step; raise --budget to leave room for both/); + } finally { + ws.cleanup(); + } + }); + + it("gives no drafting hint when the worker spent it", async () => { + const ws = workspace(); + try { + const fetchFn = (async () => + reply({ choices: [{ message: { role: "assistant", content: null, tool_calls: [{ id: "c", type: "function", function: { name: "list_files", arguments: "{}" } }] }, finish_reason: "tool_calls" }], usage: { prompt_tokens: 600, completion_tokens: 0 } })) as unknown as typeof fetch; + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high", maxSteps: 5 }); + engine.setBudget(1_000); + const hit = (await drain(engine.run("anything", allowAll))).find((x) => x.kind === "error" && x.ceiling === "budget"); + assert.ok(hit && hit.kind === "error"); + assert.doesNotMatch(hit.text, /drafting/); + } finally { + ws.cleanup(); + } + }); +}); + describe("JudgeMeter", () => { const u = { baseUrl: URL, model: JUDGE, promptTokens: 1000, completionTokens: 200, cacheReadTokens: 400, estimated: false }; @@ -251,6 +311,16 @@ describe("JudgeMeter", () => { assert.ok(Math.abs(m.total().costUsd! - PER_CALL) < 1e-12); }); + it("a price cleared after a call is no longer applied to it", () => { + const m = new JudgeMeter(); + m.setPricing(URL, JUDGE, PRICE); + m.record(u); + assert.ok(m.total().costUsd! > 0); + m.setPricing(URL, JUDGE, null); + assert.equal(m.total().costUsd, undefined, "re-priced now: no price, $ unknown"); + assert.equal(m.all()[0]!.costUsd, PER_CALL, "the figure recorded at the time (the journal's) is kept on the call"); + }); + it("uses the provider's billed figure when it sends one", () => { const m = new JudgeMeter(); m.record({ ...u, billedUsd: 0.5 }); From 63aff3c84d53eb4a1120e85b9937aab7356e4061 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 05:24:42 -0700 Subject: [PATCH 2/3] Meter interview and mission-planning asks as worker spend Both ask the worker's own model outside a turn, and their tokens reached no meter. The interview's ask now reports to Engine.workerAskMeter, which adds it to the session meter like a step (cost priced as the worker's) and journals it as worker_ask; the TUI and the desktop pass it. maat mission plan meters its ask at the worker's prices and prints a 'worker (planning)' line. askModel's usage reading is shared as recordUsage for the interview's own HTTP path. --- CHANGELOG.md | 4 +++ docs/transparency.md | 5 +++ electron/main.ts | 2 ++ src/app.tsx | 2 ++ src/ask.ts | 58 ++++++++++++++++---------------- src/cli.tsx | 18 ++++++++-- src/engine.ts | 28 +++++++++++++++- src/interview.ts | 13 +++++++- src/journal.ts | 10 ++++++ test/worker-asks.test.ts | 72 ++++++++++++++++++++++++++++++++++++++++ 10 files changed, 180 insertions(+), 32 deletions(-) create mode 100644 test/worker-asks.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1605070..65ca2fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ### Added +- **Interview and mission-planning asks are metered as worker spend.** The + interview's question is added to the session meter (journal: `worker_ask`); + `maat mission plan` prints the plan's tokens and cost at the worker's prices. + - **The judge's spend is metered.** Every ask made around the work (drafting and its critic, the reference check, `--review`, the post-work audit, the arbiter) records its input, output, cache read and cache write tokens, priced diff --git a/docs/transparency.md b/docs/transparency.md index a9207f7..e17cb46 100644 --- a/docs/transparency.md +++ b/docs/transparency.md @@ -197,6 +197,11 @@ tokens to a token budget and nothing to a dollar ceiling (its spend there is unknown, and said so). The budget is checked before every worker step; a judge ask already under way is not cut off. +The worker's own asks outside a turn are worker spend, not the judge's: the +interview's question is added to the session meter like a step (and journalled +as `worker_ask`), and `maat mission plan` prints a `worker (planning) ...` line +with the plan's tokens and cost at the worker's prices. + Judge calls are priced when they are read, not when they were made: a price that arrives later applies to every call, and a price that is cleared stops applying (the journal keeps each call's figure as it stood at the time). diff --git a/electron/main.ts b/electron/main.ts index 070f421..9d716b7 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -1550,6 +1550,8 @@ ipcMain.handle( // The workspace, not the app's own directory: on the ACP // backend this is where the CLI that answers is run. cwd: session.cwd, + // The worker's own model: worker spend on the session meter. + meter: session.engine.workerAskMeter, }); }, ); diff --git a/src/app.tsx b/src/app.tsx index 616e331..aad2a7b 100644 --- a/src/app.tsx +++ b/src/app.tsx @@ -702,6 +702,8 @@ export function App({ model: engine.model, cwd: engine.cwd, fetchFn: engine.cfg.fetchFn, + // The worker's own model: worker spend on the session meter. + meter: engine.workerAskMeter, }); if (seq !== interviewSeq.current) return; if (r.kind === "error") { diff --git a/src/ask.ts b/src/ask.ts index 1969210..db8e83d 100644 --- a/src/ask.ts +++ b/src/ask.ts @@ -350,46 +350,48 @@ function count(v: unknown): number | undefined { return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined; } +/** Report one answered HTTP ask to its meter (see recordUsage). */ +function meterUsage(opts: AskOptions, u: Usage | undefined, reply: string): void { + recordUsage(opts.meter, { baseUrl: opts.baseUrl, model: opts.model, what: opts.what, sent: opts.system + opts.prompt }, u, reply); +} + +/** An answered ask whose transport reports no usage (ACP, a CLI): estimated. */ +function meterEstimate(opts: AskOptions, reply: string): void { + meterUsage(opts, undefined, reply); +} + /** - * Report one answered HTTP ask to the meter: the provider's usage block when - * it sent one, Maat's estimate (marked) when it did not. + * Report one answered ask to a meter: the provider's usage block when it sent + * one (input, output, cache read and write, its own dollar figure), Maat's + * estimate from `sent` and the reply, marked, when it did not. Exported for the + * asks that do not go through askModel (the interview). */ -function meterUsage(opts: AskOptions, u: Usage | undefined, reply: string): void { - if (!opts.meter) return; +export function recordUsage( + meter: AskMeter | undefined, + ctx: { baseUrl: string; model: string; what?: string; sent: string }, + u: Usage | undefined, + reply: string, +): void { + if (!meter) return; const prompt = count(u?.prompt_tokens); const completion = count(u?.completion_tokens); - if (prompt === undefined && completion === undefined) { - meterEstimate(opts, reply); - return; - } const read = count(u?.prompt_tokens_details?.cached_tokens) ?? count(u?.cache_read_input_tokens); const write = count(u?.cache_creation_input_tokens); const billed = count(u?.cost); - opts.meter.record({ - baseUrl: opts.baseUrl, - model: opts.model, - ...(opts.what ? { what: opts.what } : {}), - promptTokens: prompt ?? estTokens(opts.system + opts.prompt), + const reported = prompt !== undefined || completion !== undefined; + meter.record({ + baseUrl: ctx.baseUrl, + model: ctx.model, + ...(ctx.what ? { what: ctx.what } : {}), + promptTokens: prompt ?? estTokens(ctx.sent), completionTokens: completion ?? estTokens(reply), - ...(read !== undefined ? { cacheReadTokens: read } : {}), - ...(write !== undefined ? { cacheWriteTokens: write } : {}), - ...(billed !== undefined ? { billedUsd: billed } : {}), + ...(reported && read !== undefined ? { cacheReadTokens: read } : {}), + ...(reported && write !== undefined ? { cacheWriteTokens: write } : {}), + ...(reported && billed !== undefined ? { billedUsd: billed } : {}), estimated: prompt === undefined || completion === undefined, }); } -/** An answered ask whose transport reports no usage (ACP, a CLI, a bare reply): estimated. */ -function meterEstimate(opts: AskOptions, reply: string): void { - opts.meter?.record({ - baseUrl: opts.baseUrl, - model: opts.model, - ...(opts.what ? { what: opts.what } : {}), - promptTokens: estTokens(opts.system + opts.prompt), - completionTokens: estTokens(reply), - estimated: true, - }); -} - /** * The JSON object in a reply, or null. * diff --git a/src/cli.tsx b/src/cli.tsx index 28cfa4b..7e1eef7 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -30,6 +30,7 @@ import { draftMission, missionStatus, runMission, writePlan, type MissionSummary import { parseDuration } from "./session-commands.js"; import { jobEndWords } from "./tiers.js"; import { judgeEffort, judgeTarget } from "./judge.js"; +import { JudgeMeter } from "./judge-meter.js"; import { commandsHere, draftCriteriaCritiqued, drafterInputsHash, drafterSnapshot, preflightCriteria, taskChecksFrom, type Draft, type DrafterInputs } from "./criteria.js"; import { listProject, removeNew } from "./leftovers.js"; import { projectScripts } from "./interview.js"; @@ -1202,13 +1203,26 @@ async function cmdMission(args: Args): Promise { process.stderr.write("maat: no model selected — pass --model or set MOLT_MODEL\n"); return 2; } - const brief = await buildBrief({ cwd: args.cwd }).catch(() => ({ text: "" })); + // The plan is asked of the worker's own model, so it is worker spend: + // metered with the worker's prices (hand-set, or the provider's list, + // looked up beside the brief) and said as the worker's below. + const key = keyForUrl(args.url, args.key); + const byHand = args.priceIn !== undefined && args.priceOut !== undefined; + const [brief, listed] = await Promise.all([ + buildBrief({ cwd: args.cwd }).catch(() => ({ text: "" })), + byHand ? Promise.resolve(null) : fetchPricing(args.url, args.model, key).catch(() => null), + ]); + const pricing = byHand + ? { in: args.priceIn!, out: args.priceOut!, ...(args.priceCachedIn !== undefined ? { cached: args.priceCachedIn } : {}), source: "set by hand" } + : listed; + const planMeter = new JudgeMeter(() => ({ baseUrl: args.url, model: args.model, pricing })); const r = await draftMission({ goal, context: brief.text, scripts: projectScripts(args.cwd), - ask: { baseUrl: args.url, apiKey: keyForUrl(args.url, args.key), model: args.model, cwd: args.cwd, reasoningEffort: args.reasoningChecks ?? args.reasoning }, + ask: { baseUrl: args.url, apiKey: key, model: args.model, cwd: args.cwd, reasoningEffort: args.reasoningChecks ?? args.reasoning, meter: planMeter }, }); + if (planMeter.mark()) process.stdout.write(`${judgeSpendLine(planMeter.total(), fmtCost, "worker (planning)")}\n`); if (!r.ok) { process.stderr.write(`maat: ${r.error}\n`); return 1; diff --git a/src/engine.ts b/src/engine.ts index b60b6c0..1edf8f9 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -25,7 +25,7 @@ import { describeStart, listBackground, startBackground, stopBackground } from " import { objectionLine, readsTheWork, reviewClaim, type ExecutableReview, type ObjectionRun, type Review } from "./review.js"; import { credentialFreeEnv } from "./credenv.js"; import { judgeEffort, judgeTarget } from "./judge.js"; -import { JudgeMeter, judgeTokens, type JudgeSpend } from "./judge-meter.js"; +import { JudgeMeter, judgeTokens, type AskMeter, type AskUsage, type JudgeSpend } from "./judge-meter.js"; import { auditClaim, authorKey, authorWords, claimLabel, contradictions, independentOf, tierOf, withAuthor, type Tier } from "./tiers.js"; import { recordGoldens, valueUnproven } from "./golden.js"; import { discountedChecks } from "./control.js"; @@ -2441,6 +2441,32 @@ export class Engine { } : null, })); + /** + * The worker's meter for asks on its own model outside a turn (the + * interview): counted into the session meter as worker spend, like a step, + * and journalled as `worker_ask`. Not the judge's: it is the worker's model. + */ + readonly workerAskMeter: AskMeter = { record: (u) => this.recordWorkerAsk(u) }; + + private recordWorkerAsk(u: AskUsage): void { + this.sessionPrompt += u.promptTokens; + this.sessionCompletion += u.completionTokens; + this.sessionCached += u.cacheReadTokens ?? 0; + if (u.estimated) this.estimatedSteps += 1; + if (typeof u.billedUsd === "number") this.sessionBilled += u.billedUsd; + else this.unbilledSteps += 1; + this.cfg.journal?.append("worker_ask", { + model: u.model, + ...(u.what ? { what: u.what } : {}), + promptTokens: u.promptTokens, + completionTokens: u.completionTokens, + ...(u.cacheReadTokens !== undefined ? { cacheReadTokens: u.cacheReadTokens } : {}), + ...(u.cacheWriteTokens !== undefined ? { cacheWriteTokens: u.cacheWriteTokens } : {}), + ...(typeof u.billedUsd === "number" ? { billedUsd: u.billedUsd } : {}), + estimated: u.estimated, + }); + } + /** * Judge calls before this mark were reported in an earlier job_end. A job * reports every call since the last one, so the drafting that runs before a diff --git a/src/interview.ts b/src/interview.ts index 1b8028e..477b5ca 100644 --- a/src/interview.ts +++ b/src/interview.ts @@ -17,6 +17,9 @@ import { removedSubscriptionProblem } from "./endpoint.js"; import { errorText } from "./format.js"; import { askError, askTimeoutMs, probeSignal } from "./watchdog.js"; import { authHeaders } from "./providers.js"; +import { recordUsage } from "./ask.js"; +import type { AskMeter } from "./judge-meter.js"; +import type { Usage } from "./stream.js"; import { loadBar, parseBar, barPath } from "./bar.js"; import type { Bar, Check } from "./types.js"; import { @@ -277,6 +280,12 @@ export async function interviewTurn(opts: { acpSpawn?: typeof import("node:child_process").spawn; /** How long the HTTP question may wait for its answer; see askTimeoutMs. Tests only. */ timeoutMs?: number; + /** + * Where the ask reports what it used. The interview asks the worker's own + * model, so callers pass the engine's worker meter (Engine.workerAskMeter): + * its tokens and cost are the worker's spend. + */ + meter?: AskMeter; }): Promise { const f = opts.fetchFn ?? fetch; const base = opts.baseUrl.replace(/\/$/, ""); @@ -320,6 +329,7 @@ export async function interviewTurn(opts: { ...(opts.acpSpawn ? { spawnFn: opts.acpSpawn } : {}), }); if (!asked.ok) return { kind: "error", error: asked.error }; + recordUsage(opts.meter, { baseUrl: opts.baseUrl, model: opts.model, what: "interviewing", sent: SYSTEM + context }, undefined, asked.text); return parseInterviewReply(asked.text, opts.round); } @@ -340,8 +350,9 @@ export async function interviewTurn(opts: { }), }); if (!res.ok) return { kind: "error", error: `HTTP ${res.status} interviewing` }; - const json = (await res.json()) as { choices?: { message?: { content?: string } }[] }; + const json = (await res.json()) as { choices?: { message?: { content?: string } }[]; usage?: Usage }; const text = json.choices?.[0]?.message?.content ?? ""; + recordUsage(opts.meter, { baseUrl: opts.baseUrl, model: opts.model, what: "interviewing", sent: SYSTEM + context }, json.usage, text); /** * No last-round guard here, because there is nothing left to guard. * diff --git a/src/journal.ts b/src/journal.ts index 7b4864b..19308a2 100644 --- a/src/journal.ts +++ b/src/journal.ts @@ -92,6 +92,11 @@ export type JournalKind = * entries. `costUsd: null` means no price is known, never $0. */ | "judge_usage" + /** + * One answered ask on the worker's own model outside its turn (the + * interview): worker spend, counted into the session meter like a step. + */ + | "worker_ask" /** The independent review of a verified claim: a label, never a gate. */ | "review" | "dispute" @@ -406,6 +411,11 @@ export class Journal { ); break; } + case "worker_ask": { + const e = d.estimated ? "~" : ""; + out.push(`${t} ask ${d.model}${d.what ? ` (${d.what})` : ""} · ${e}${d.promptTokens} in / ${e}${d.completionTokens} out`); + break; + } case "tool_call": out.push(`${t} tool ${d.name}: ${d.detail}`); break; diff --git a/test/worker-asks.test.ts b/test/worker-asks.test.ts new file mode 100644 index 0000000..021cf38 --- /dev/null +++ b/test/worker-asks.test.ts @@ -0,0 +1,72 @@ +/** + * Asks on the worker's own model outside a turn — the interview and mission + * planning — are metered as worker spend. Before this they used tokens no + * meter saw (the judge's asks were metered in #56; these are not the judge's). + */ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { Engine } from "../src/engine.js"; +import { judgeSpendLine } from "../src/format.js"; +import { Journal } from "../src/journal.js"; +import { JudgeMeter } from "../src/judge-meter.js"; +import { interviewTurn } from "../src/interview.js"; +import { draftMission } from "../src/mission.js"; +import { workspace } from "./helpers.js"; + +const URL = "http://provider.test/v1"; +const reply = (body: unknown) => + (async () => ({ ok: true, status: 200, json: async () => body, text: async () => JSON.stringify(body), headers: new Headers() }) as unknown as Response) as unknown as typeof fetch; + +describe("the interview's ask is worker spend", () => { + it("adds its reported tokens to the session meter and the journal", async () => { + const ws = workspace(); + try { + const journal = new Journal(ws.dir); + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, bar: null, journal, priceInPerMtok: 2, priceOutPerMtok: 10 }); + const fetchFn = reply({ + choices: [{ message: { content: JSON.stringify({ questions: [{ id: "q1", prompt: "What counts as done?", options: ["a", "b"] }] }) } }], + usage: { prompt_tokens: 900, completion_tokens: 100, prompt_tokens_details: { cached_tokens: 300 } }, + }); + const r = await interviewTurn({ task: "t", scripts: [], barChecks: [], history: [], round: 1, baseUrl: URL, model: "m", fetchFn, meter: engine.workerAskMeter }); + assert.notEqual(r.kind, "error"); + const b = engine.bom(); + assert.equal(b.sessionPromptTokens, 900); + assert.equal(b.sessionCompletionTokens, 100); + assert.equal(b.sessionCachedTokens, 300); + assert.ok(Math.abs(b.costUsd! - (900 * 2 + 100 * 10) / 1e6) < 1e-12, "priced as the worker"); + assert.equal(b.judge, undefined, "not the judge's"); + const e = Journal.read(journal.path).filter((x) => x.kind === "worker_ask"); + assert.equal(e.length, 1); + assert.equal((e[0]!.data as Record).what, "interviewing"); + assert.match(Journal.summarize(e)[0]!, /ask m \(interviewing\) · 900 in \/ 100 out/); + } finally { + ws.cleanup(); + } + }); + + it("estimates, and marks it, when the provider reports no usage", async () => { + const ws = workspace(); + try { + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, bar: null }); + const fetchFn = reply({ choices: [{ message: { content: JSON.stringify({ proposal: { criteria: [] } }) } }] }); + await interviewTurn({ task: "t", scripts: [], barChecks: [], history: [], round: 3, baseUrl: URL, model: "m", fetchFn, meter: engine.workerAskMeter }); + assert.ok(engine.sessionTokens > 0); + assert.equal(engine.costEstimated, true); + } finally { + ws.cleanup(); + } + }); +}); + +describe("mission planning is worker spend", () => { + it("reports the plan's tokens and cost on a meter priced as the worker", async () => { + const meter = new JudgeMeter(() => ({ baseUrl: URL, model: "m", pricing: { in: 2, out: 10, source: "test" } })); + const fetchFn = reply({ choices: [{ message: { content: "not a plan" }, finish_reason: "stop" }], usage: { prompt_tokens: 5000, completion_tokens: 1000 } }); + await draftMission({ goal: "a CLI", ask: { baseUrl: URL, model: "m", fetchFn, meter } }); + const s = meter.total(); + assert.equal(s.calls, 1); + assert.equal(s.promptTokens, 5000); + assert.ok(Math.abs(s.costUsd! - (5000 * 2 + 1000 * 10) / 1e6) < 1e-12); + assert.equal(judgeSpendLine(s, undefined, "worker (planning)"), "worker (planning) 1 call · 5000 in · 1000 out · $0.020"); + }); +}); From 5c8082662e4fe502c6242252a789706e70c67307 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Thu, 8 Oct 2026 06:53:46 -0700 Subject: [PATCH 3/3] Address the #57 review: plan judges in the bench, both ceilings, neutral names - bench: judge_columns carries the judge's plan (judge_plan). A run whose judge was plan-paid counts in the lane median at the worker's cost; only a judge with no price and no plan is left out. The alarm split reads 'judge (OpenCode plan)'. - An unpriced worker with a priced judge is held by both per-turn ceilings, dollars for the judge's part and tokens for the whole turn, and stops at whichever it reaches first. - The drafting hint keys on a mark and token count taken when drafting starts (Engine.draftingStarted), and is given only when the spend before the drafting was under the budget. - mission plan resolves the worker's price through the same helper as run (workerPricing): hand-set, stored for the model, or the provider's list. - Neutral names: SpendMeter, AskSpend, askSpendLine, with JudgeMeter, JudgeSpend and judgeSpendLine kept as aliases. - docs: interview asks are in the session total but in no job's spend. --- CHANGELOG.md | 6 ++- bench/local/run.py | 17 ++++-- bench/local/tests/test_cost_alarm.py | 24 +++++++++ bench/local/tests/test_judge_columns.py | 7 +++ docs/lean.md | 4 +- docs/transparency.md | 12 +++-- src/cli.tsx | 50 ++++++++++++----- src/engine.ts | 39 +++++++++++--- src/format.ts | 7 ++- src/judge-meter.ts | 25 ++++++--- test/judge-meter.test.ts | 72 +++++++++++++++++++++++++ 11 files changed, 224 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65ca2fd..cdf666a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,10 @@ prompt tokens (the detail shows the split), so a runaway judge stops the lane too. See `docs/transparency.md` and `docs/lean.md`. The per-turn dollar ceiling counts a priced judge even when the worker is - unpriced; the lane median leaves out runs whose judge was unpriced; a budget - used up by drafting before the first step says so. + unpriced, with the token ceiling still bounding the whole turn; the lane + median leaves out runs whose judge was unpriced (a plan-paid judge, + `judge_plan`, still counts); a budget used up by this turn's drafting before + the first step says so. - **`--review-executable`** (experimental, also `MAAT_REVIEW_EXECUTABLE=1`): every `--review` objection must carry a read-only command that demonstrates diff --git a/bench/local/run.py b/bench/local/run.py index 47664ad..07d17cc 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -682,6 +682,9 @@ def judge_columns(judge: dict | None) -> dict: "judge_cache_read": judge.get("cacheReadTokens"), "judge_cache_write": judge.get("cacheWriteTokens"), "judge_cost_usd": judge.get("costUsd"), + # The plan that paid for the judge's calls (e.g. "OpenCode"): no money, so a + # missing judge_cost_usd with a plan is exact, not unknown. + "judge_plan": judge.get("plan"), } @@ -716,13 +719,14 @@ def run_cost(row: dict) -> float | None: def median_cost(row: dict) -> float | None: """A run's cost as the lane median counts it, or None to leave it out. - A run whose judge ran but had no price is left out: its total is the worker's + A run whose judge ran but had no price (and no plan paying for it) is left out: its total is the worker's cost alone, a lower bound, and averaged with runs whose judge was priced it would pull the median (and so the limit) down without anyone seeing why. The alarm on that run itself still uses the lower bound (run_cost): under-counting there only makes the alarm late, never wrong.""" - if row.get("judge_calls") is not None and row.get("judge_cost_usd") is None: + if row.get("judge_calls") is not None and row.get("judge_cost_usd") is None and not row.get("judge_plan"): return None + # A plan-paid judge cost no money: the total is the worker's, and exact. return run_cost(row) @@ -747,9 +751,15 @@ def cost_alarm(row: dict, prior_costs: list[float]) -> dict | None: def usd(x): return "$ unknown" if x is None else f"${x:.4f}" + def judge_usd(): + plan, cost = row.get("judge_plan"), row.get("judge_cost_usd") + if plan and cost is None: + return f"({plan} plan)" + return usd(cost) + (f" + {plan} plan" if plan else "") + why = [] if cost is not None and cost > limit_usd: - split = f" (worker {usd(row.get('cost_usd'))} + judge {usd(row.get('judge_cost_usd'))})" if judged else "" + split = f" (worker {usd(row.get('cost_usd'))} + judge {judge_usd()})" if judged else "" why.append(f"cost ${cost:.4f}{split} > ${limit_usd:.4f} (max({lim['x']:g}x lane median " f"{'n/a' if med is None else f'${med:.4f}'}, ${lim['usd']:g}))") if toks is not None and toks > lim["tokens"]: @@ -760,6 +770,7 @@ def usd(x): return None return {"detail": "; ".join(why), "cost_usd": cost, "tokens_in": toks, **({"worker_cost_usd": row.get("cost_usd"), "judge_cost_usd": row.get("judge_cost_usd"), + **({"judge_plan": row["judge_plan"]} if row.get("judge_plan") else {}), "worker_tokens_in": row.get("tokens_in"), "judge_tokens_in": row.get("judge_tokens_in")} if judged else {}), "lane_median_usd": med, "limit_usd": round(limit_usd, 6), "limit_tokens": lim["tokens"]} diff --git a/bench/local/tests/test_cost_alarm.py b/bench/local/tests/test_cost_alarm.py index be9a688..346c749 100644 --- a/bench/local/tests/test_cost_alarm.py +++ b/bench/local/tests/test_cost_alarm.py @@ -219,3 +219,27 @@ def with_judge(d, prompt, log): run.main("molt", 1, None) rows = [json.loads(x) for x in out.read_text().splitlines()] assert len(rows) == 5 and not any(r.get("stopped") for r in rows) + + +def test_a_plan_paid_judge_counts_in_the_median_at_the_workers_cost(): + row = {"cost_usd": 0.15, "judge_calls": 3, "judge_cost_usd": None, "judge_plan": "OpenCode"} + assert run.median_cost(row) == 0.15, "a plan-paid judge cost no money: exact, not unknown" + a = run.cost_alarm(row, []) + assert a and "(worker $0.1500 + judge (OpenCode plan))" in a["detail"] and a["judge_plan"] == "OpenCode" + + +def test_a_lane_with_a_plan_paid_judge_forms_its_median(monkeypatch, tmp_path): + # The documented OpenCode-judge arm: every judge is plan-paid. The median must + # form from the worker's costs (0.10), so a $0.15 run (limit max(0.50, 0.10)) passes; + # with the plan rows left out, the limit stayed at the $0.10 floor and stopped it. + out = _lane(monkeypatch, tmp_path, [(0.10, 1), (0.10, 1), (0.10, 1), (0.15, 1)]) + inner = run.run_molt + + def plan_judge(d, prompt, log): + return {**inner(d, prompt, log), "judge_calls": 3, "judge_cost_usd": None, "judge_plan": "OpenCode", "judge_tokens_in": 1} + + monkeypatch.setattr(run, "run_molt", plan_judge) + run.main("molt", 1, None) + rows = [json.loads(x) for x in out.read_text().splitlines()] + assert len(rows) == 4 and not any(r.get("stopped") for r in rows) + assert all(r["judge_plan"] == "OpenCode" for r in rows) diff --git a/bench/local/tests/test_judge_columns.py b/bench/local/tests/test_judge_columns.py index 09d90fa..7c2c910 100644 --- a/bench/local/tests/test_judge_columns.py +++ b/bench/local/tests/test_judge_columns.py @@ -19,9 +19,16 @@ def test_columns_from_job_end_judge(): assert cols == { "judge_calls": 4, "judge_tokens_in": 12000, "judge_tokens_out": 900, "judge_cache_read": 3000, "judge_cache_write": 500, "judge_cost_usd": 0.0123, + "judge_plan": None, } +def test_a_plan_paid_judge_carries_its_plan(): + cols = run.judge_columns({"calls": 3, "promptTokens": 10, "completionTokens": 2, + "cacheReadTokens": 0, "cacheWriteTokens": 0, "plan": "OpenCode"}) + assert cols["judge_plan"] == "OpenCode" and cols["judge_cost_usd"] is None + + def test_unpriced_judge_is_none_not_zero(): cols = run.judge_columns({"calls": 2, "promptTokens": 800, "completionTokens": 100, "cacheReadTokens": 0, "cacheWriteTokens": 0, "unpricedCalls": 2}) diff --git a/docs/lean.md b/docs/lean.md index 07e8470..8735f29 100644 --- a/docs/lean.md +++ b/docs/lean.md @@ -94,7 +94,9 @@ the run had a judge, the detail shows the split, e.g. `cost $0.2800 (worker $0.0300 + judge $0.2500) > $0.1000 ...`, and the `STOPPED` row adds `worker_cost_usd`, `judge_cost_usd`, `worker_tokens_in` and `judge_tokens_in`; `cost_usd` and `tokens_in` there are the totals. The lane median is over the -same totals, leaving out runs whose judge ran unpriced: their total is only a +same totals, leaving out runs whose judge ran unpriced (a judge paid by a plan, +`judge_plan`, cost no money and counts at the worker's cost; the split reads +`judge (OpenCode plan)`): their total is only a lower bound, and mixed with priced runs it would pull the limit down unseen. Such a run's own alarm still uses that lower bound. The run trips the alarm when: diff --git a/docs/transparency.md b/docs/transparency.md index e17cb46..55d7a39 100644 --- a/docs/transparency.md +++ b/docs/transparency.md @@ -190,16 +190,20 @@ dollars counts the judge's spend in that turn too: a budget is what you are willing to spend, and the judge's tokens are billed like any others. When a budget is hit with judge spend in it, the message says how much was the judge's; when the judge's drafting used it before the worker's first step, -the message says that too. The turn is priced in dollars when either side is -priced: a local or subscription worker with a paid judge is still held to a -dollar ceiling by the judge's dollars. A side with no known price adds its +the message says that too (only when the spend before the drafting was still +under the budget). The turn is priced in dollars when either side is priced: a +local or subscription worker with a paid judge is held to a dollar ceiling by +the judge's dollars and to the token ceiling by the whole turn's tokens, and +stops at whichever it reaches first. A side with no known price adds its tokens to a token budget and nothing to a dollar ceiling (its spend there is unknown, and said so). The budget is checked before every worker step; a judge ask already under way is not cut off. The worker's own asks outside a turn are worker spend, not the judge's: the interview's question is added to the session meter like a step (and journalled -as `worker_ask`), and `maat mission plan` prints a `worker (planning) ...` line +as `worker_ask`). It is asked before the turn starts, so it is in the session +total but in no job's `job_end` spend or receipt: a session total higher than +the sum of its jobs is the interview, not a gap. And `maat mission plan` prints a `worker (planning) ...` line with the plan's tokens and cost at the worker's prices. Judge calls are priced when they are read, not when they were made: a price diff --git a/src/cli.tsx b/src/cli.tsx index 7e1eef7..cea5fcf 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -16,7 +16,7 @@ import { resolve } from "node:path"; import { Archive } from "./archive.js"; import { isAutonomy, type Autonomy } from "./autonomy.js"; import { fmtCost, fmtDuration } from "./banner.js"; -import { judgeSpendLine, stepDid } from "./format.js"; +import { judgeSpendLine, askSpendLine, stepDid } from "./format.js"; import { BarError, hasBar, loadBar, selectChecks, writeDefaultBar } from "./bar.js"; import { Engine, type FileAccess } from "./engine.js"; import { describeDrift, driftSince } from "./git.js"; @@ -30,7 +30,7 @@ import { draftMission, missionStatus, runMission, writePlan, type MissionSummary import { parseDuration } from "./session-commands.js"; import { jobEndWords } from "./tiers.js"; import { judgeEffort, judgeTarget } from "./judge.js"; -import { JudgeMeter } from "./judge-meter.js"; +import { SpendMeter } from "./judge-meter.js"; import { commandsHere, draftCriteriaCritiqued, drafterInputsHash, drafterSnapshot, preflightCriteria, taskChecksFrom, type Draft, type DrafterInputs } from "./criteria.js"; import { listProject, removeNew } from "./leftovers.js"; import { projectScripts } from "./interview.js"; @@ -933,17 +933,35 @@ async function priceEngine(engine: Engine, args: Args): Promise { /** The worker's own price lookup (see priceEngine). */ async function priceWorker(engine: Engine, args: Args): Promise { - if (!needsPriceLookup(args.model, engine.pricing(), storedEndpoint())) return; + engine.setPricing(await workerPricing(args, engine.pricing())); +} + +/** + * The worker's price, resolved one way for every command that spends on the + * worker's model (`run`, `mission plan`): a hand-set or stored price for this + * model stands; otherwise the provider's list is asked and the answer saved. + * `current` is what is in force now (args' prices, seeded from the stored one + * by parseArgs). Returns the prices to use, `{}` for none. + */ +async function workerPricing( + args: Args, + current: { in?: number; out?: number; cached?: number; source?: string } = { + in: args.priceIn, + out: args.priceOut, + cached: args.priceCachedIn, + source: args.priceSource, + }, +): Promise<{ in?: number; out?: number; cached?: number; source?: string }> { + if (!needsPriceLookup(args.model, current, storedEndpoint())) return current; const p = await fetchPricing(args.url, args.model, keyForUrl(args.url, args.key)); if (!p) { // Nothing published. A price only stands if it was recorded for THIS // model; inheriting the last one is how a Claude session gets billed at // grok's rates. - if (storedEndpoint().priceModel !== args.model) engine.setPricing({}); - return; + return storedEndpoint().priceModel !== args.model ? {} : current; } - engine.setPricing({ in: p.in, out: p.out, cached: p.cached, source: p.source }); savePricing(args.model, p); + return { in: p.in, out: p.out, cached: p.cached, source: p.source }; } /** @@ -1207,22 +1225,24 @@ async function cmdMission(args: Args): Promise { // metered with the worker's prices (hand-set, or the provider's list, // looked up beside the brief) and said as the worker's below. const key = keyForUrl(args.url, args.key); - const byHand = args.priceIn !== undefined && args.priceOut !== undefined; - const [brief, listed] = await Promise.all([ + const [brief, prices] = await Promise.all([ buildBrief({ cwd: args.cwd }).catch(() => ({ text: "" })), - byHand ? Promise.resolve(null) : fetchPricing(args.url, args.model, key).catch(() => null), + // Resolved exactly as `run` resolves it (workerPricing): hand-set, + // stored for this model, or the provider's list. + workerPricing(args).catch(() => ({}) as { in?: number; out?: number; cached?: number; source?: string }), ]); - const pricing = byHand - ? { in: args.priceIn!, out: args.priceOut!, ...(args.priceCachedIn !== undefined ? { cached: args.priceCachedIn } : {}), source: "set by hand" } - : listed; - const planMeter = new JudgeMeter(() => ({ baseUrl: args.url, model: args.model, pricing })); + const pricing = + prices.in !== undefined && prices.out !== undefined + ? { in: prices.in, out: prices.out, ...(prices.cached !== undefined ? { cached: prices.cached } : {}), source: prices.source ?? "stored" } + : null; + const planMeter = new SpendMeter(() => ({ baseUrl: args.url, model: args.model, pricing })); const r = await draftMission({ goal, context: brief.text, scripts: projectScripts(args.cwd), ask: { baseUrl: args.url, apiKey: key, model: args.model, cwd: args.cwd, reasoningEffort: args.reasoningChecks ?? args.reasoning, meter: planMeter }, }); - if (planMeter.mark()) process.stdout.write(`${judgeSpendLine(planMeter.total(), fmtCost, "worker (planning)")}\n`); + if (planMeter.mark()) process.stdout.write(`${askSpendLine(planMeter.total(), fmtCost, "worker (planning)")}\n`); if (!r.ok) { process.stderr.write(`maat: ${r.error}\n`); return 1; @@ -1376,6 +1396,8 @@ async function autoDraftFrom( // Drafted, then read cold by a critic against the task text: a check that // invents or guesses is dropped (with a task quote), and a draft where // nothing runs the deliverable is asked for once more. See criteria.ts. + // Where the drafting began, for the budget-hit message (Engine.draftStart). + engine.draftingStarted(); const draftOnce = () => draftCriteriaCritiqued({ snapshot, diff --git a/src/engine.ts b/src/engine.ts index 1edf8f9..fef31c8 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -2473,6 +2473,18 @@ export class Engine { * turn starts is counted in the turn it was for. */ private judgeReported = 0; + /** + * Where the judge's drafting for the next turn began: the meter's mark and + * what had been spent by then (draftingStarted). The budget-hit message says + * the drafting used the budget only when this turn's drafting is what took + * it over, not when earlier spend had already nearly used it up. + */ + private draftStart: { mark: number; tokens: number } | undefined; + + /** The caller is about to draft this turn's checks on the judge (cli.tsx). */ + draftingStarted(): void { + this.draftStart ??= { mark: this.judgeMeter.mark(), tokens: this.spentTokens }; + } /** Why the independent review was skipped this turn (see reviewSkipReason), if it was. */ private reviewSkipped: string | undefined; /** Hidden checks whose commands were shown to the model this turn. */ @@ -5245,6 +5257,7 @@ export class Engine { // the review and the audit run after the receipt was written. const judge = this.judgeMeter.since(this.judgeReported); this.judgeReported = this.judgeMeter.mark(); + this.draftStart = undefined; if (receiptPath && judge.calls) this.cfg.receipts?.amendJudge(basename(receiptPath), this.judgeMeter.total()); yield { kind: "job_end", @@ -6732,9 +6745,13 @@ export class Engine { `budget hit (${this.budgetTokens} tokens` + (this.spentTokens > this.sessionTokens ? `, ${this.spentTokens - this.sessionTokens} of them the judge's` : "") + `) — loop stopped. /budget to raise.` + - // Hit before the worker took a step, with the judge asked for this - // turn already (its drafting): say what used it up. - (step === 0 && this.judgeMeter.mark() > this.judgeReported + // Hit before the worker took a step, by this turn's drafting: the + // judge drafted since draftingStarted, and the spend before it was + // still under the budget. + (step === 0 && + this.draftStart !== undefined && + this.judgeMeter.mark() > this.draftStart.mark && + this.draftStart.tokens < (this.budgetTokens ?? 0) ? ` The judge drafting this turn's checks used the budget before the worker's first step; ` + `raise --budget to leave room for both.` : ""), @@ -6758,14 +6775,24 @@ export class Engine { // default itself: there is nothing left to make an exception to. const usdCeiling = this.cfg.maxTurnUsd ?? 0; const tokenCeiling = this.cfg.maxTurnTokens ?? 0; - const priced = usdThisTurn !== undefined && usdCeiling > 0; - const used = priced ? usdThisTurn : spentThisTurn; + // A priced worker is held in dollars, as it always was. An unpriced + // worker with a priced judge is held by both: dollars bound the part that + // has a price (the judge's), tokens bound the whole turn, and whichever + // is nearer its ceiling is the one that speaks and stops. Without the + // token side, one priced judge call flipped the turn to dollars and the + // worker's unpriced tokens ran on unbounded. + const usdShare = usdThisTurn !== undefined && usdCeiling > 0 ? usdThisTurn / usdCeiling : undefined; + const tokenShare = tokenCeiling > 0 ? spentThisTurn / tokenCeiling : undefined; + const priced = + usdShare !== undefined && + (turnStartCost !== undefined || tokenShare === undefined || usdShare >= tokenShare); + const used = priced ? usdThisTurn! : spentThisTurn; const ceiling = priced ? usdCeiling : tokenCeiling; // Named for what it is, and not `shown` — which is the read-coverage map // a few lines down, and which this quietly shadowed until the compiler // said so. const ceilingLine = priced - ? `${fmtUsd(usdThisTurn)} of ${fmtUsd(usdCeiling)}` + ? `${fmtUsd(usdThisTurn!)} of ${fmtUsd(usdCeiling)}` : `${spentThisTurn} of ${tokenCeiling} tokens`; // Said on the way up, not only on arrival. A limit that speaks for the diff --git a/src/format.ts b/src/format.ts index f5af84c..5245550 100644 --- a/src/format.ts +++ b/src/format.ts @@ -10,7 +10,7 @@ * Pure, and re-exported from `banner.tsx` so nothing that already imported * them has to move. */ -import type { JudgeSpend } from "./judge-meter.js"; +import type { AskSpend } from "./judge-meter.js"; export function fmtTokens(n: number): string { if (n < 1000) return `${n}`; @@ -127,7 +127,7 @@ export function spendLine(sp: { * `judge 3 calls · 1200 in (400 cached) · 300 out · $0.004`, with `$ unknown` * when a model had no price and the plan's name when a plan paid. */ -export function judgeSpendLine(s: JudgeSpend, fmt: (usd: number) => string = fmtCost, label = "judge"): string { +export function askSpendLine(s: AskSpend, fmt: (usd: number) => string = fmtCost, label = "judge"): string { const cache = s.cacheReadTokens > 0 || s.cacheWriteTokens > 0 ? ` (${[s.cacheReadTokens > 0 ? `${s.cacheReadTokens} cached` : "", s.cacheWriteTokens > 0 ? `${s.cacheWriteTokens} cache write` : ""].filter(Boolean).join(", ")})` @@ -143,3 +143,6 @@ export function judgeSpendLine(s: JudgeSpend, fmt: (usd: number) => string = fmt : "$ unknown"; return `${label} ${s.calls} call${s.calls === 1 ? "" : "s"} · ${e}${s.promptTokens} in${cache} · ${e}${s.completionTokens} out · ${money}`; } + +/** The judge's spend line: askSpendLine under its original name. */ +export const judgeSpendLine = askSpendLine; diff --git a/src/judge-meter.ts b/src/judge-meter.ts index a89025f..d0edf5a 100644 --- a/src/judge-meter.ts +++ b/src/judge-meter.ts @@ -47,8 +47,8 @@ export interface AskMeter { record(u: AskUsage): void; } -/** The judge's spend over some span of calls. */ -export type JudgeSpend = { +/** The spend of a span of asks (the judge's, or the worker's planning ask). */ +export type AskSpend = { calls: number; promptTokens: number; completionTokens: number; @@ -100,7 +100,12 @@ export function priceTokens(p: Pricing, u: Pick(); private readonly listeners: ((c: JudgeCall) => void)[] = []; @@ -180,7 +185,7 @@ export class JudgeMeter implements AskMeter { } /** The whole session's judge spend. */ - total(): JudgeSpend { + total(): AskSpend { return this.since(0); } @@ -191,7 +196,7 @@ export class JudgeMeter implements AskMeter { * to any: the figure stored at record time (the journal's) is dropped before * re-pricing. The worker's meter does the same with its session totals. */ - since(mark: number): JudgeSpend { + since(mark: number): AskSpend { const span = this.calls.slice(mark).map(({ costUsd: _recorded, plan: _plan, ...u }) => this.priced(u)); const models: string[] = []; let cost = 0; @@ -235,9 +240,15 @@ export class JudgeMeter implements AskMeter { } /** Tokens a judge spend counts toward a budget: everything sent and received. */ -export function judgeTokens(s: JudgeSpend | undefined): number { +export function judgeTokens(s: AskSpend | undefined): number { return s ? s.promptTokens + s.completionTokens : 0; } +/** The judge's spend: an AskSpend over the judge's calls. */ +export type JudgeSpend = AskSpend; +/** The engine's meter of the judge's calls: a SpendMeter. */ +export const JudgeMeter = SpendMeter; +export type JudgeMeter = SpendMeter; + /** Said in one line by format.ts, which the renderer can import. */ -export { judgeSpendLine } from "./format.js"; +export { askSpendLine, judgeSpendLine } from "./format.js"; diff --git a/test/judge-meter.test.ts b/test/judge-meter.test.ts index 2eed200..f326a8a 100644 --- a/test/judge-meter.test.ts +++ b/test/judge-meter.test.ts @@ -260,6 +260,44 @@ describe("the turn's money ceiling with an unpriced worker", () => { }); }); +describe("an unpriced worker with a priced judge is held by both ceilings", () => { + // Grok's reproducer (#57 review, N1): one priced judge call flipped the turn + // to dollars and the worker's tokens ran to the step guard (700k). + it("stops at the token ceiling, not at the step guard", async () => { + const ws = workspace(); + try { + let asked = 0; + const fetchFn = (async () => { + asked += 1; + return reply({ choices: [{ message: { role: "assistant", content: null, tool_calls: [{ id: `c${asked}`, type: "function", function: { name: "list_files", arguments: JSON.stringify({ path: `d${asked}` }) } }] }, finish_reason: "tool_calls" }], usage: { prompt_tokens: 100_000, completion_tokens: 0 } }); + }) as unknown as typeof fetch; + const engine = new Engine({ + baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high", + maxTurnUsd: 1, maxTurnTokens: 150_000, maxSteps: 7, + }); + engine.judgeMeter.setPricing(URL, JUDGE, { in: 1, out: 1, source: "test" }); + const events: EngineEvent[] = []; + let judged = false; + for await (const ev of engine.run("anything", allowAll)) { + events.push(ev); + if (ev.kind === "usage" && !judged) { + judged = true; + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, promptTokens: 10, completionTokens: 10, estimated: false }); + } + } + // Stopped after two steps at 200,020 (the third request is the closing + // summary every stopped turn asks for), not run on to the step guard. + assert.ok( + events.some((x) => x.kind === "error" && /stopped: this turn has spent 200020 of 150000 tokens/.test(x.text)), + "the token ceiling stopped the turn at ~200k", + ); + assert.equal(asked, 3, "two work steps and the closing summary, not seven steps"); + } finally { + ws.cleanup(); + } + }); +}); + describe("the budget-hit message", () => { it("says the drafting used the budget when it was hit before the worker's first step", async () => { const ws = workspace(); @@ -267,6 +305,7 @@ describe("the budget-hit message", () => { const fetchFn = (async () => reply({ choices: [{ message: { role: "assistant", content: "Done." }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })) as unknown as typeof fetch; const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high" }); engine.setBudget(1_000); + engine.draftingStarted(); engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "drafting criteria", promptTokens: 1_200, completionTokens: 0, estimated: false }); const hit = (await drain(engine.run("anything", allowAll))).find((x) => x.kind === "error" && x.ceiling === "budget"); assert.ok(hit && hit.kind === "error"); @@ -276,6 +315,39 @@ describe("the budget-hit message", () => { } }); + it("gives no drafting hint when the budget was already used up before the drafting began", async () => { + const ws = workspace(); + try { + const fetchFn = (async () => reply({ choices: [{ message: { role: "assistant", content: "Done." }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })) as unknown as typeof fetch; + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high" }); + engine.setBudget(1_000); + // A judge ask outside the drafting (an arbiter, say) already spent the budget. + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "ruling on a disputed check", promptTokens: 1_100, completionTokens: 0, estimated: false }); + engine.draftingStarted(); + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "drafting criteria", promptTokens: 50, completionTokens: 0, estimated: false }); + const hit = (await drain(engine.run("anything", allowAll))).find((x) => x.kind === "error" && x.ceiling === "budget"); + assert.ok(hit && hit.kind === "error"); + assert.doesNotMatch(hit.text, /drafting/); + } finally { + ws.cleanup(); + } + }); + + it("gives no drafting hint for judge spend outside any drafting", async () => { + const ws = workspace(); + try { + const fetchFn = (async () => reply({ choices: [{ message: { role: "assistant", content: "Done." }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })) as unknown as typeof fetch; + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high" }); + engine.setBudget(1_000); + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "reviewing the claim", promptTokens: 1_200, completionTokens: 0, estimated: false }); + const hit = (await drain(engine.run("anything", allowAll))).find((x) => x.kind === "error" && x.ceiling === "budget"); + assert.ok(hit && hit.kind === "error"); + assert.doesNotMatch(hit.text, /drafting/); + } finally { + ws.cleanup(); + } + }); + it("gives no drafting hint when the worker spent it", async () => { const ws = workspace(); try {