diff --git a/CHANGELOG.md b/CHANGELOG.md index 7023c6a..b71cbf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ ### Added +- **The judge's spend is metered.** Every ask made around the work (drafting + and its critic, the reference check, `--review`, the post-work audit, the + arbiter) records its input, output, cache read and cache write tokens, priced + from the worker's price table keyed on the judge's own model id; an unpriced + model shows `$ unknown`, never `$0`. Reported apart from the worker's spend + as `judge` on `job_end`, `judge_usage` journal entries, a `judge` line and a + worker + judge total in the CLI, and `judge` on the receipt (row updated at + the job's end). `--budget` and the per-turn ceilings now count worker and + judge spend together. `bench/local/run.py` adds `judge_*` token and cost + columns to each row, and its cost alarm now judges worker + judge cost and + prompt tokens (the detail shows the split), so a runaway judge stops the lane + too. See `docs/transparency.md` and `docs/lean.md`. + - **`--review-executable`** (experimental, also `MAAT_REVIEW_EXECUTABLE=1`): every `--review` objection must carry a read-only command that demonstrates it. Maat runs it on a throwaway copy of the tree, with no API keys, tokens or diff --git a/bench/local/run.py b/bench/local/run.py index a12d726..0d6e954 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -627,7 +627,7 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: write_private(log, out) # Maat's own notices (criteria not drafted, dropped checks, refusals) go to stderr. write_private(log.with_suffix(".err"), err or "") - steps = 0; per = []; outcome = None; spend = {}; review = None; disagree = []; extra = {} + steps = 0; per = []; outcome = None; spend = {}; judge = None; review = None; disagree = []; extra = {} for line in out.splitlines(): if not line.startswith("{"): continue @@ -641,6 +641,9 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: elif ev.get("kind") == "job_end": outcome = claim_of(ev) spend = ev.get("spend") or {} + # The judge's asks (drafting, critic, review, audit), metered apart from + # the worker's. Absent on builds before judge metering, or when no judge ran. + judge = ev.get("judge") review = ev.get("review") disagree = ev.get("checksDisagree") or [] # Only what job_end carried; absent keys stay absent (older builds). @@ -651,7 +654,9 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: return { "provider_capped": capped, "secs": round(secs), "turns": steps, "calls": sum(per), "multi": sum(1 for x in per if x > 1), - "tokens_in": spend.get("promptTokens"), "cost_usd": spend.get("costUsd"), "claim": outcome, "timed_out": timed_out, + "tokens_in": spend.get("promptTokens"), "cost_usd": spend.get("costUsd"), + **judge_columns(judge), + "claim": outcome, "timed_out": timed_out, "said_done": (outcome or "").startswith("verified"), # The reviewer's label: a verified claim it did not confirm. "review": review, @@ -662,6 +667,24 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: } +def judge_columns(judge: dict | None) -> dict: + """The judge's tokens and $, next to the worker's `tokens_in` / `cost_usd`. + + Only when the run output carried them (job_end.judge, Maat's judge meter): None + otherwise, so an older build is never read as a free judge. `judge_cost_usd` is + None when the judge's model had no price ("$ unknown"), never 0.""" + if not isinstance(judge, dict): + return {} + return { + "judge_calls": judge.get("calls"), + "judge_tokens_in": judge.get("promptTokens"), + "judge_tokens_out": judge.get("completionTokens"), + "judge_cache_read": judge.get("cacheReadTokens"), + "judge_cache_write": judge.get("cacheWriteTokens"), + "judge_cost_usd": judge.get("costUsd"), + } + + # Per-run cost alarm (reports/loop-charter.md, non-regression rule 1): a run that # costs more than max(5x the lane's running median cost, $0.10), or sends more # than 1.5M prompt tokens, stops the lane before the bill grows. 2026-10-07: one @@ -678,24 +701,55 @@ def alarm_limits() -> dict: } +def _sum_known(*xs): + """The sum of the values that are known, or None when none is.""" + known = [x for x in xs if x is not None] + return sum(known) if known else None + + +def run_cost(row: dict) -> float | None: + """A run's whole cost: the worker's plus the judge's (judge_columns). None when + neither is known; an unpriced judge adds nothing it cannot price.""" + return _sum_known(row.get("cost_usd"), row.get("judge_cost_usd")) + + +def run_tokens(row: dict) -> int | None: + """A run's prompt tokens, the worker's plus the judge's.""" + return _sum_known(row.get("tokens_in"), row.get("judge_tokens_in")) + + def cost_alarm(row: dict, prior_costs: list[float]) -> dict | None: - """Why this run trips the alarm, or None. `prior_costs` are the lane's earlier runs - (this run excluded): the median is over runs that reported a cost. A run with no - cost (an unpriced model) is judged on its prompt tokens only.""" + """Why this run trips the alarm, or None. `prior_costs` are the lane's earlier runs' + total costs (this run excluded): the median is over runs that reported a cost. A + run with no cost (an unpriced model) is judged on its prompt tokens only. + + Cost and tokens are the worker's plus the judge's: a runaway judge trips it too. + When the run had a judge, the detail says how the total splits.""" lim = alarm_limits() med = statistics.median(prior_costs) if prior_costs else None limit_usd = max(lim["x"] * med, lim["usd"]) if med is not None else lim["usd"] - cost, toks = row.get("cost_usd"), row.get("tokens_in") + cost, toks = run_cost(row), run_tokens(row) + judged = row.get("judge_calls") is not None + + def usd(x): + return "$ unknown" if x is None else f"${x:.4f}" + why = [] if cost is not None and cost > limit_usd: - why.append(f"cost ${cost:.4f} > ${limit_usd:.4f} (max({lim['x']:g}x lane median " + split = f" (worker {usd(row.get('cost_usd'))} + judge {usd(row.get('judge_cost_usd'))})" if judged else "" + why.append(f"cost ${cost:.4f}{split} > ${limit_usd:.4f} (max({lim['x']:g}x lane median " f"{'n/a' if med is None else f'${med:.4f}'}, ${lim['usd']:g}))") if toks is not None and toks > lim["tokens"]: - why.append(f"prompt tokens {toks:,} > {lim['tokens']:,}") + split = (f" (worker {row.get('tokens_in') or 0:,} + judge {row.get('judge_tokens_in') or 0:,})" + if judged else "") + why.append(f"prompt tokens {toks:,}{split} > {lim['tokens']:,}") if not why: return None - return {"detail": "; ".join(why), "cost_usd": cost, "tokens_in": toks, "lane_median_usd": med, - "limit_usd": round(limit_usd, 6), "limit_tokens": lim["tokens"]} + return {"detail": "; ".join(why), "cost_usd": cost, "tokens_in": toks, + **({"worker_cost_usd": row.get("cost_usd"), "judge_cost_usd": row.get("judge_cost_usd"), + "worker_tokens_in": row.get("tokens_in"), "judge_tokens_in": row.get("judge_tokens_in")} + if judged else {}), + "lane_median_usd": med, "limit_usd": round(limit_usd, 6), "limit_tokens": lim["tokens"]} def claims_done(text: str) -> bool: @@ -855,8 +909,8 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: if r.get("stopped"): # a STOPPED marker, not a run continue done.add((r["task"], r["agent"], r["rep"], r.get("arm"))) - if r.get("cost_usd") is not None: - lane_costs.setdefault(r.get("arm"), []).append(r["cost_usd"]) + if run_cost(r) is not None: + lane_costs.setdefault(r.get("arm"), []).append(run_cost(r)) global SCRUBBER SCRUBBER = make_scrubber() for rep in range(repeats): @@ -914,8 +968,8 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: alarm = cost_alarm(r, costs) if alarm: stop_lane(out, tag, EXPORT / f"{tag}.log", alarm, arm, lid) - if r.get("cost_usd") is not None: - costs.append(r["cost_usd"]) + if run_cost(r) is not None: + costs.append(run_cost(r)) def stop_lane(out: Path, tag: str, log: Path, alarm: dict, arm: str | None, lid: str | None = None) -> None: diff --git a/bench/local/tests/test_cost_alarm.py b/bench/local/tests/test_cost_alarm.py index 341fe88..e315a19 100644 --- a/bench/local/tests/test_cost_alarm.py +++ b/bench/local/tests/test_cost_alarm.py @@ -129,3 +129,67 @@ def test_resume_skips_the_marker_and_keeps_the_median(monkeypatch, tmp_path): # Earlier run's $0.10 is the median: limit max(0.50, 0.10), so $0.40 passes. run.main("molt", 1, None) assert json.loads(out.read_text().splitlines()[-1])["task"] == "t0" + + +def test_a_runaway_judge_trips_the_alarm_and_the_detail_shows_the_split(): + # The worker alone is well under the floor; the judge pushes the run over it. + row = {"cost_usd": 0.03, "tokens_in": 40_000, "judge_calls": 9, "judge_cost_usd": 0.25, "judge_tokens_in": 300_000} + a = run.cost_alarm(row, []) + assert a, "worker + judge is what the alarm judges" + assert "cost $0.2800 (worker $0.0300 + judge $0.2500) > $0.1000" in a["detail"] + assert a["cost_usd"] == pytest.approx(0.28) + assert a["worker_cost_usd"] == 0.03 and a["judge_cost_usd"] == 0.25 + assert a["tokens_in"] == 340_000 and a["judge_tokens_in"] == 300_000 + + +def test_judge_tokens_count_toward_the_token_alarm(): + row = {"cost_usd": None, "tokens_in": 900_000, "judge_calls": 5, "judge_cost_usd": None, "judge_tokens_in": 700_000} + a = run.cost_alarm(row, []) + assert a and "prompt tokens 1,600,000 (worker 900,000 + judge 700,000) > 1,500,000" in a["detail"] + + +def test_an_unpriced_judge_is_said_as_unknown_in_the_split(): + row = {"cost_usd": 0.12, "judge_calls": 2, "judge_cost_usd": None, "judge_tokens_in": 1_000} + a = run.cost_alarm(row, []) + assert a and "cost $0.1200 (worker $0.1200 + judge $ unknown)" in a["detail"] + + +def test_a_run_with_no_judge_reads_as_before(): + a = run.cost_alarm({"cost_usd": 0.11, "tokens_in": 1000}, []) + assert a and "(worker" not in a["detail"] and "judge_cost_usd" not in a + + +def test_the_lane_median_is_over_worker_plus_judge(monkeypatch, tmp_path): + out = _lane(monkeypatch, tmp_path, [(0.02, 50_000), (0.03, 60_000), (0.02, 50_000)]) + judge = iter([0.08, 0.07, 0.30]) + inner = run.run_molt + + def with_judge(d, prompt, log): + r = inner(d, prompt, log) + return {**r, "judge_calls": 4, "judge_cost_usd": next(judge), "judge_tokens_in": 10_000} + + monkeypatch.setattr(run, "run_molt", with_judge) + # Totals 0.10, 0.10, 0.32: median 0.10 -> limit max(0.50, 0.10); 0.32 passes. + run.main("molt", 1, None) + rows = [json.loads(x) for x in out.read_text().splitlines()] + assert len(rows) == 3 and not any(r.get("stopped") for r in rows) + assert [r["judge_cost_usd"] for r in rows] == [0.08, 0.07, 0.30] + + +def test_a_lane_stops_on_a_runaway_judge(monkeypatch, tmp_path): + out = _lane(monkeypatch, tmp_path, [(0.02, 50_000), (0.02, 50_000), (0.02, 50_000), (0.02, 50_000)]) + judge = iter([0.01, 0.01, 0.90, 0.01]) + inner = run.run_molt + + def with_judge(d, prompt, log): + r = inner(d, prompt, log) + return {**r, "judge_calls": 4, "judge_cost_usd": next(judge), "judge_tokens_in": 10_000} + + monkeypatch.setattr(run, "run_molt", with_judge) + with pytest.raises(SystemExit) as e: + run.main("molt", 1, None) + assert e.value.code == run.COST_ALARM_EXIT + stop = [json.loads(x) for x in out.read_text().splitlines()][-1] + assert stop["stopped"] is True and stop["run"] == "t2-molt-0" + assert stop["worker_cost_usd"] == 0.02 and stop["judge_cost_usd"] == 0.90 + assert "(worker $0.0200 + judge $0.9000)" in stop["detail"] diff --git a/bench/local/tests/test_judge_columns.py b/bench/local/tests/test_judge_columns.py new file mode 100644 index 0000000..09d90fa --- /dev/null +++ b/bench/local/tests/test_judge_columns.py @@ -0,0 +1,33 @@ +"""The judge's tokens and $ in each result row, next to the worker's. + +Maat meters the judge's asks (drafting, critic, review, audit) apart from the +worker's requests and reports them on job_end as `judge`. A row carries them only +when the run output did, and an unpriced judge is None, never 0. +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import run # noqa: E402 + + +def test_columns_from_job_end_judge(): + cols = run.judge_columns({ + "calls": 4, "promptTokens": 12000, "completionTokens": 900, + "cacheReadTokens": 3000, "cacheWriteTokens": 500, "costUsd": 0.0123, + }) + assert cols == { + "judge_calls": 4, "judge_tokens_in": 12000, "judge_tokens_out": 900, + "judge_cache_read": 3000, "judge_cache_write": 500, "judge_cost_usd": 0.0123, + } + + +def test_unpriced_judge_is_none_not_zero(): + cols = run.judge_columns({"calls": 2, "promptTokens": 800, "completionTokens": 100, + "cacheReadTokens": 0, "cacheWriteTokens": 0, "unpricedCalls": 2}) + assert cols["judge_tokens_in"] == 800 + assert cols["judge_cost_usd"] is None + + +def test_no_judge_in_the_output_adds_no_columns(): + assert run.judge_columns(None) == {} diff --git a/docs/lean.md b/docs/lean.md index 7246761..586cdf1 100644 --- a/docs/lean.md +++ b/docs/lean.md @@ -83,7 +83,18 @@ part of this. After each run, run.py reads the run's cost (`spend.costUsd`) and prompt tokens (`spend.promptTokens`) from Maat's `job_end` event, and records the cost -in the row as `cost_usd`. The run trips the alarm when: +in the row as `cost_usd`. The judge's spend, when `job_end` carries it +(`judge`, see transparency.md), goes in the same row beside the worker's: +`judge_calls`, `judge_tokens_in`, `judge_tokens_out`, `judge_cache_read`, +`judge_cache_write` and `judge_cost_usd` (null when the judge's model has no +price, never 0). The alarm below is on the run's whole spend: the worker's +cost plus the judge's, and the worker's prompt tokens plus the judge's, so a +runaway judge trips it too (an unpriced judge adds tokens, not dollars). When +the run had a judge, the detail shows the split, e.g. `cost $0.2800 (worker +$0.0300 + judge $0.2500) > $0.1000 ...`, and the `STOPPED` row adds +`worker_cost_usd`, `judge_cost_usd`, `worker_tokens_in` and `judge_tokens_in`; +`cost_usd` and `tokens_in` there are the totals. The lane median is over the +same totals. The run trips the alarm when: - its cost exceeds **max(5x the lane's running median cost, $0.10)**. The median is over the lane's earlier runs that reported a cost (resumed rows diff --git a/docs/transparency.md b/docs/transparency.md index ca122d5..8d06a6d 100644 --- a/docs/transparency.md +++ b/docs/transparency.md @@ -152,6 +152,48 @@ The bottom line is the session meter and only ever climbs. **Per-job** figures — what one question cost — live in the view, measured as a delta against the session meter rather than by resetting it. +### The judge's spend + +The worker's requests are not the only ones a run pays for. Around the work, +Maat asks the judge: it drafts the hidden checks, a critic reads the draft, a +reference check is written (`--reference`), the independent review votes +(`--review`), the post-work audit drafts its checks, and the arbiter rules on a +dispute. With `--judge` (`MAAT_JUDGE_MODEL`) those asks go to another model, +often on another provider. Each one is metered on the **judge's meter** +(`src/judge-meter.ts`), apart from the worker's: + +- **Tokens** — input, output, cache read and cache write — come from the + judge's own usage block, per call. A reply with no usage is counted from its + length and marked `~`, as on the worker side. +- **Prices** come from the same table as the worker's (`providers.ts`), keyed + on the judge's model id: its provider's published list (xAI, OpenRouter), + Anthropic's rates on Anthropic's API, or the worker's own prices when the + judge is the worker's model. A provider-reported dollar figure is used as + is. A model with no known price shows its tokens and **`$ unknown`**, never + `$0`. A judge on a subscription (Grok Build, OpenCode) is paid by the plan + and gets no dollar figure. +- **Where it shows:** a `judge_usage` journal entry per call; `judge` on + `job_end` (every judge call since the previous job, so the drafting before a + turn is counted with that turn); a `judge ...` line under the CLI's job line + and under the session totals, followed by `total $x (worker $a + judge $b)` + when both are priced; and the receipt, which carries the judge's spend so far + when the claim is made, with the index row (`judge`) brought up to date when + the job ends, since the review and the audit run after the receipt is written. + +The worker's figures (`spend`, `sessionTokens`, the step lines, `costUsd` on a +receipt) stay the worker's: the lean-budget suite measures the worker's +requests, and the judge's are not among them. + +**A budget counts both.** `--budget` / `/budget ` is checked against +the worker's tokens plus the judge's, and the per-turn ceiling in tokens or +dollars counts the judge's spend in that turn too: a budget is what you are +willing to spend, and the judge's tokens are billed like any others. When a +budget is hit with judge spend in it, the message says how much was the +judge's. A judge with no known price adds its tokens to a token budget and +nothing to a dollar ceiling (its spend there is unknown, and said so). The +budget is checked before every worker step; a judge ask already under way is +not cut off. + ## Work that goes nowhere A tool call that returns exactly what it returned before has taught the model diff --git a/src/acp-server.ts b/src/acp-server.ts index 2a00975..126cb95 100644 --- a/src/acp-server.ts +++ b/src/acp-server.ts @@ -896,6 +896,7 @@ export class AcpServer { let truncated = false; let lastError: string | undefined; let spend: Spend | undefined; + let judge: import("./judge-meter.js").JudgeSpend | undefined; let exhaustedSeen = false; /** A message ended; the next text starts a new paragraph. */ let paragraph = false; @@ -1133,6 +1134,7 @@ export class AcpServer { case "job_end": record.outcome = ev.outcome; spend = ev.spend; + judge = ev.judge; break; default: break; @@ -1199,6 +1201,8 @@ export class AcpServer { } : {}), ...(spend?.costUsd !== undefined ? { costUsd: spend.costUsd, costEstimated: spend.estimated } : {}), + // The judge's spend, apart from the worker's `usage` (judge-meter.ts). + ...(judge ? { judge } : {}), }, }, }; diff --git a/src/app.tsx b/src/app.tsx index fd4521c..616e331 100644 --- a/src/app.tsx +++ b/src/app.tsx @@ -11,7 +11,7 @@ import { useCallback, useEffect, useRef, useState, type ReactNode } from "react" import { Box, Static, Text, render, useApp, useInput, useStdout } from "ink"; import type { RenderOptions } from "ink"; import { Banner, fmtCost, fmtDuration, fmtTokens } from "./banner.js"; -import { stepDid } from "./format.js"; +import { judgeSpendLine, stepDid } from "./format.js"; import { COMMANDS, COMMAND_COL, @@ -984,7 +984,9 @@ export function App({ ); note( `▪ job ${ev.job} ${ev.outcome} · ${ev.steps} step(s) · ` + - `${spendText(ev.spend)} · ${fmtDuration(ev.durationMs)}`, + `${spendText(ev.spend)} · ${fmtDuration(ev.durationMs)}` + + // The judge's share, apart from the worker's (judge-meter.ts). + (ev.judge ? ` · ${judgeSpendLine(ev.judge)}` : ""), ); break; case "request": diff --git a/src/ask.ts b/src/ask.ts index f49e998..1969210 100644 --- a/src/ask.ts +++ b/src/ask.ts @@ -17,9 +17,11 @@ import { isOpencode, opencodeAsk } from "./opencode.js"; import { removedSubscriptionProblem } from "./endpoint.js"; import { errorText } from "./format.js"; import { authHeaders, isSelfHosted, selfHostedThinking, openRouterProvider } from "./providers.js"; -import { LONG_RATE_LIMIT_MS, longQuotaText, providerErrorText, rateLimitResetAt, readStream, transientProviderError, untilText, type ProviderError } from "./stream.js"; +import { LONG_RATE_LIMIT_MS, longQuotaText, providerErrorText, rateLimitResetAt, readStream, transientProviderError, untilText, type ProviderError, type Usage } from "./stream.js"; import { askError, askTimeoutMs, localSpeed, probeSignal } from "./watchdog.js"; import { takeTurn } from "./localgate.js"; +import type { AskMeter } from "./judge-meter.js"; +import { estTokens } from "./types.js"; /** * Pauses before asking again when the provider said it is overloaded or @@ -64,6 +66,13 @@ export type AskOptions = { * three before the first step. Never recorded for a self-hosted server. */ latency?: { record(w: { firstProgressMs: number | undefined; maxGapMs: number }): void }; + /** + * Where each answered ask reports what it used (src/judge-meter.ts): input, + * output, cache read and write tokens, and the provider's own dollar figure + * when it sends one. Every ask that returns an answer is recorded, retries + * included; an ask the provider refused is not, since nothing was billed. + */ + meter?: AskMeter; }; export type Asked = { ok: true; text: string; cutOff: boolean } | { ok: false; error: string; transient?: true }; @@ -201,7 +210,10 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { prompt: opts.prompt, ...(opts.cliRun ? { run: opts.cliRun } : {}), }); - if (asked.ok) return { ok: true, text: asked.text, cutOff: false }; + if (asked.ok) { + meterEstimate(opts, asked.text); + return { ok: true, text: asked.text, cutOff: false }; + } return { ok: false, error: longQuotaText(asked.error) ?? asked.error, ...(asked.transient ? { transient: true as const } : {}) }; } @@ -216,7 +228,10 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { cwd: opts.cwd, ...(opts.acpSpawn ? { spawnFn: opts.acpSpawn } : {}), }); - if (asked.ok) return { ok: true, text: asked.text, cutOff: false }; + if (asked.ok) { + meterEstimate(opts, asked.text); + return { ok: true, text: asked.text, cutOff: false }; + } return { ok: false, error: longQuotaText(asked.error) ?? asked.error }; } @@ -276,10 +291,12 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { type Reply = { choices?: { message?: { content?: string | null }; finish_reason?: string | null }[]; error?: ProviderError; + usage?: Usage; }; let content: string; let finish: string | null | undefined; let error: ProviderError | undefined; + let used: Usage | undefined; // A server that was sent `stream: true` and answers with plain JSON (a // proxy that buffers, or an error body) is read as JSON, as it always was. if (stream && res.body && /text\/event-stream/i.test(res.headers.get("content-type") ?? "")) { @@ -287,12 +304,25 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { content = done.message.content ?? ""; finish = done.finishReason; error = done.error; + if (done.promptTokens !== undefined || done.completionTokens !== undefined) { + used = { + ...(done.promptTokens !== undefined ? { prompt_tokens: done.promptTokens } : {}), + ...(done.completionTokens !== undefined ? { completion_tokens: done.completionTokens } : {}), + ...(done.cachedTokens !== undefined ? { cache_read_input_tokens: done.cachedTokens } : {}), + ...(done.cacheWriteTokens !== undefined ? { cache_creation_input_tokens: done.cacheWriteTokens } : {}), + ...(done.costUsd !== undefined ? { cost: done.costUsd } : {}), + }; + } } else { const json = (await res.json()) as Reply; content = json.choices?.[0]?.message?.content ?? ""; finish = json.choices?.[0]?.finish_reason; error = json.error && typeof json.error === "object" ? json.error : undefined; + used = json.usage && typeof json.usage === "object" ? json.usage : undefined; } + // Before the error check: a provider can bill a request whose reply then + // carries an error, and a usage block is the provider saying it did. + if (!error || used) meterUsage(opts, used, content); if (error) { const resetAt = rateLimitResetAt({ error }); if (resetAt !== undefined && resetAt - Date.now() > LONG_RATE_LIMIT_MS) { @@ -315,6 +345,51 @@ async function askOnce(opts: AskOptions, maxTokens: number): Promise { } } +/** A finite, non-negative count, or undefined. */ +function count(v: unknown): number | undefined { + return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined; +} + +/** + * Report one answered HTTP ask to the meter: the provider's usage block when + * it sent one, Maat's estimate (marked) when it did not. + */ +function meterUsage(opts: AskOptions, u: Usage | undefined, reply: string): void { + if (!opts.meter) return; + const prompt = count(u?.prompt_tokens); + const completion = count(u?.completion_tokens); + if (prompt === undefined && completion === undefined) { + meterEstimate(opts, reply); + return; + } + const read = count(u?.prompt_tokens_details?.cached_tokens) ?? count(u?.cache_read_input_tokens); + const write = count(u?.cache_creation_input_tokens); + const billed = count(u?.cost); + opts.meter.record({ + baseUrl: opts.baseUrl, + model: opts.model, + ...(opts.what ? { what: opts.what } : {}), + promptTokens: prompt ?? estTokens(opts.system + opts.prompt), + completionTokens: completion ?? estTokens(reply), + ...(read !== undefined ? { cacheReadTokens: read } : {}), + ...(write !== undefined ? { cacheWriteTokens: write } : {}), + ...(billed !== undefined ? { billedUsd: billed } : {}), + estimated: prompt === undefined || completion === undefined, + }); +} + +/** An answered ask whose transport reports no usage (ACP, a CLI, a bare reply): estimated. */ +function meterEstimate(opts: AskOptions, reply: string): void { + opts.meter?.record({ + baseUrl: opts.baseUrl, + model: opts.model, + ...(opts.what ? { what: opts.what } : {}), + promptTokens: estTokens(opts.system + opts.prompt), + completionTokens: estTokens(reply), + estimated: true, + }); +} + /** * The JSON object in a reply, or null. * diff --git a/src/cli.tsx b/src/cli.tsx index 126a1e3..fe68c74 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -16,6 +16,7 @@ import { resolve } from "node:path"; import { Archive } from "./archive.js"; import { isAutonomy, type Autonomy } from "./autonomy.js"; import { fmtCost, fmtDuration } from "./banner.js"; +import { judgeSpendLine } from "./format.js"; import { stepDid } from "./format.js"; import { BarError, hasBar, loadBar, selectChecks, writeDefaultBar } from "./bar.js"; import { Engine, type FileAccess } from "./engine.js"; @@ -148,7 +149,8 @@ options --verbose show every call, argument, and result (press v in the TUI) --provider label shown in the status line --cwd project directory (default: current) - --budget hard token ceiling for the session + --budget hard token ceiling for the session: the worker's tokens plus + the judge's (drafting, review, audit), counted together --review [n] after a verified claim, n independent reviews (default 3) read the task and the receipt; a majority-backed violation quoted from the task labels the work "passed its checks, @@ -925,6 +927,7 @@ function engineFor(args: Args, session = false, extra: { files?: FileAccess } = * nothing, no cost is shown at all. */ async function priceEngine(engine: Engine, args: Args): Promise { + await priceJudge(engine, args); if (!needsPriceLookup(args.model, engine.pricing(), storedEndpoint())) return; const p = await fetchPricing(args.url, args.model, keyForUrl(args.url, args.key)); if (!p) { @@ -938,6 +941,31 @@ async function priceEngine(engine: Engine, args: Args): Promise { savePricing(args.model, p); } +/** + * Give the judge's meter a price for each model asked around the work that is + * not the worker's own: the judge (MAAT_JUDGE_MODEL) and the arbiter. Keyed on + * that model's id, from the same lookup the worker uses (fetchPricing). A + * model that publishes nothing stays unpriced: its tokens are reported with + * "$ unknown", never $0. The worker's own model needs nothing here; the meter + * prices it with the worker's prices, hand-set ones included. + */ +async function priceJudge(engine: Engine, args: Args): Promise { + const key = keyForUrl(args.url, args.key); + const judge = judgeTarget({ baseUrl: args.url, apiKey: key, model: args.model }); + const arb = engine.cfg.dispute; + const others = [ + judge, + ...(arb?.model ? [{ baseUrl: arb.baseUrl || args.url, apiKey: arb.apiKey ?? key, model: arb.model }] : []), + ].filter((t) => !(t.model === args.model && t.baseUrl === args.url)); + await Promise.all( + others.map(async (t) => { + if (planFor(t.baseUrl) || engine.judgeMeter.hasPricing(t.baseUrl, t.model)) return; + const p = await fetchPricing(t.baseUrl, t.model, t.apiKey).catch(() => null); + engine.judgeMeter.setPricing(t.baseUrl, t.model, p); + }), + ); +} + /** * `from` decides which explanation a work-landed failure gets. * @@ -1344,6 +1372,8 @@ async function autoDraftFrom( ...(preWork ? { preWorkDir: preWork.dir, preWorkIntact: preWork.intact } : {}), reasoningEffort: judgeEffort(args.reasoningChecks ?? args.reasoning), latency: engine.askLatency, + // The drafter and the critic are the judge: metered as the judge. + meter: engine.judgeMeter, deadlineAt, // What is ready when a time budget stops the wait (RunOptions.criteriaSoFar). onProgress: (d) => { @@ -1390,7 +1420,7 @@ async function autoDraftFrom( * there is no python3, the project is too large to copy, or no reference * applies. */ -function startReference(args: Args, deadlineAt?: number): Promise<{ check: Check; note: Record } | null> | undefined { +function startReference(args: Args, deadlineAt?: number, engine?: Engine): Promise<{ check: Check; note: Record } | null> | undefined { const here = commandsHere(args.cwd); if (!here.present.includes("python3")) { process.stderr.write("maat: no reference check — python3 is not installed here\n"); @@ -1410,6 +1440,8 @@ function startReference(args: Args, deadlineAt?: number): Promise<{ check: Check cwd: args.cwd, reasoningEffort: args.reasoningChecks ?? args.reasoning, ...(deadlineAt !== undefined ? { deadlineAt } : {}), + // Asked around the work, like the drafter: on the judge's meter. + ...(engine ? { meter: engine.judgeMeter } : {}), }).then((r) => { if (!r.ok) { process.stderr.write(`maat: no reference check — ${r.why}\n`); @@ -1557,6 +1589,8 @@ async function cmdRun(args: Args, ask = false): Promise { (sp.costUsd === undefined ? "" : ` · ${sp.estimated ? "~" : ""}${fmtCost(sp.costUsd)}`) + "\n", ); + // The judge's share of the job, on its own line (judge-meter.ts). + if (ev.judge) process.stdout.write(` ${judgeSpendLine(ev.judge)}\n`); break; } case "request": @@ -1661,7 +1695,7 @@ async function cmdRun(args: Args, ask = false): Promise { return soFar.draft?.checks.length ? sealDraft(soFar.draft, args) : Promise.resolve({ taskChecks: [], taskNotes: [] }); } : undefined; - const referenceCheck = args.reference && !ask ? startReference(args, runDeadlineAt) : undefined; + const referenceCheck = args.reference && !ask ? startReference(args, runDeadlineAt, engine) : undefined; let undetermined = false; /** The turn's own verdict, from job_end. */ let outcome: string | undefined; @@ -1700,6 +1734,17 @@ async function cmdRun(args: Args, ask = false): Promise { "\n", ); } + // The judge's spend, apart from the worker's line above, and the two added + // up when both are priced. An unpriced judge says so; it is never $0. + if (b.judge) { + process.stdout.write(`${judgeSpendLine(b.judge)}\n`); + if (b.costUsd !== undefined && b.judge.costUsd !== undefined) { + process.stdout.write( + `total ${b.costEstimated || b.judge.estimated ? "~" : ""}${fmtCost(b.costUsd + b.judge.costUsd)} ` + + `(worker ${fmtCost(b.costUsd)} + judge ${fmtCost(b.judge.costUsd)})\n`, + ); + } + } // An unverified answer is not a success. Neither is no answer at all. // diff --git a/src/criteria.ts b/src/criteria.ts index 51a6b30..bf2473b 100644 --- a/src/criteria.ts +++ b/src/criteria.ts @@ -793,6 +793,7 @@ async function edgeProbes(opts: Parameters[0], view: strin deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, + meter: opts.meter, }).catch(() => null); if (!asked?.ok) return ""; let probes: { input?: unknown; readingA?: unknown; outputA?: unknown; readingB?: unknown; outputB?: unknown; quote?: unknown }[] = []; @@ -838,6 +839,8 @@ export async function draftCriteria(opts: { reasoningEffort?: string; /** Learns from completed asks (AskOptions.latency). */ latency?: AskOptions["latency"]; + /** Where each ask reports its tokens (AskOptions.meter; the judge's meter). */ + meter?: AskOptions["meter"]; /** What is installed, so a check never calls a command that is not (see commandsHere). */ commands?: { present: string[]; missing: string[] }; /** @@ -904,6 +907,7 @@ export async function draftCriteria(opts: { deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, + meter: opts.meter, }); if (!asked.ok) return asked; const first = drafted(asked.text, asked.cutOff); @@ -925,6 +929,7 @@ export async function draftCriteria(opts: { deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, + meter: opts.meter, }); if (!again.ok) return first; const second = drafted(again.text, again.cutOff); @@ -950,6 +955,7 @@ export async function draftCriteria(opts: { deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, + meter: opts.meter, }); if (!more.ok) break; const d = drafted(more.text, more.cutOff); @@ -1345,6 +1351,7 @@ async function critiqued( deadlineAt: opts.deadlineAt, reasoningEffort: opts.reasoningEffort, latency: opts.latency, + meter: opts.meter, }); return asked.ok ? applyCritique(d, asked.text, opts.task) : null; }; diff --git a/src/engine.ts b/src/engine.ts index 513109a..37f4a44 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -25,6 +25,7 @@ import { describeStart, listBackground, startBackground, stopBackground } from " import { objectionLine, readsTheWork, reviewClaim, type ExecutableReview, type ObjectionRun, type Review } from "./review.js"; import { credentialFreeEnv } from "./credenv.js"; import { judgeEffort, judgeTarget } from "./judge.js"; +import { JudgeMeter, judgeTokens, type JudgeSpend } from "./judge-meter.js"; import { auditClaim, authorKey, authorWords, claimLabel, contradictions, independentOf, tierOf, withAuthor, type Tier } from "./tiers.js"; import { recordGoldens, valueUnproven } from "./golden.js"; import { discountedChecks } from "./control.js"; @@ -2415,6 +2416,31 @@ export class Engine { get askLatency(): LatencyLearner | undefined { return isSelfHosted(this.cfg.baseUrl) ? undefined : this.latency; } + /** + * The judge's meter (src/judge-meter.ts): every ask made around the work — + * drafting, the critic, the reference check, the review, the post-work + * audit, the arbiter — with its tokens and price, kept apart from the + * worker's. A judge that is the worker's own model is priced as the worker is. + */ + readonly judgeMeter: JudgeMeter = new JudgeMeter(() => ({ + baseUrl: this.cfg.baseUrl, + model: this.cfg.model, + pricing: + this.cfg.priceInPerMtok !== undefined && this.cfg.priceOutPerMtok !== undefined + ? { + in: this.cfg.priceInPerMtok, + out: this.cfg.priceOutPerMtok, + ...(this.cfg.priceCachedInPerMtok !== undefined ? { cached: this.cfg.priceCachedInPerMtok } : {}), + source: this.cfg.priceSource ?? "the worker's prices", + } + : null, + })); + /** + * Judge calls before this mark were reported in an earlier job_end. A job + * reports every call since the last one, so the drafting that runs before a + * turn starts is counted in the turn it was for. + */ + private judgeReported = 0; /** Why the independent review was skipped this turn (see reviewSkipReason), if it was. */ private reviewSkipped: string | undefined; /** Hidden checks whose commands were shown to the model this turn. */ @@ -2469,6 +2495,23 @@ export class Engine { // Under --worker-user, one line in the journal saying what isolation was // actually in effect (src/privsep.ts), e.g. // "isolation: worker uid 1001, check uid 1002, pid namespace on". + // Every judge call, as it returns: what it used and what it cost, beside + // the worker's `response` entries (judge-meter.ts). + this.judgeMeter.onRecord((c) => { + this.cfg.journal?.append("judge_usage", { + model: c.model, + ...(c.what ? { what: c.what } : {}), + promptTokens: c.promptTokens, + completionTokens: c.completionTokens, + ...(c.cacheReadTokens !== undefined ? { cacheReadTokens: c.cacheReadTokens } : {}), + ...(c.cacheWriteTokens !== undefined ? { cacheWriteTokens: c.cacheWriteTokens } : {}), + // null is "no price known", never 0. + costUsd: c.costUsd ?? null, + billed: typeof c.billedUsd === "number", + estimated: c.estimated, + ...(c.plan ? { plan: c.plan } : {}), + }); + }); const isolation = isolationLine(); if (isolation && cfg.journal && !isolationNoted.has(cfg.journal)) { isolationNoted.add(cfg.journal); @@ -3187,6 +3230,7 @@ export class Engine { costUsd: this.costUsd(), costEstimated: this.costEstimated, budgetTokens: this.budgetTokens, + ...(this.judgeMeter.mark() ? { judge: this.judgeMeter.total() } : {}), }; } @@ -3501,8 +3545,35 @@ export class Engine { return existsSync(p) ? readFileSync(p, "utf8") : undefined; } + /** + * A budget counts everything spent: the worker's tokens and the judge's. + * The judge's asks are not the worker's requests (the step lines and + * sessionTokens stay the worker's), but they are billed all the same. + */ private overBudget(): boolean { - return this.budgetTokens !== undefined && this.sessionTokens >= this.budgetTokens; + return this.budgetTokens !== undefined && this.spentTokens >= this.budgetTokens; + } + + /** Worker and judge tokens together: what a token budget counts. */ + get spentTokens(): number { + return this.sessionTokens + judgeTokens(this.judgeMeter.total()); + } + + /** + * Worker and judge USD together: what a money budget counts. Undefined when + * the worker has no price. A judge with no price adds nothing here (its + * tokens still count toward a token budget, and every surface says its $ is + * unknown); a judge on a subscription plan costs no money. + */ + totalCostUsd(): number | undefined { + const worker = this.costUsd(); + if (worker === undefined) return undefined; + return worker + (this.judgeMeter.total().costUsd ?? 0); + } + + /** The whole session's judge spend (judge-meter.ts). */ + judgeSpend(): JudgeSpend { + return this.judgeMeter.total(); } /** @@ -4109,6 +4180,7 @@ export class Engine { fetchFn: this.cfg.fetchFn, acpSpawn: this.cfg.acpSpawn, reasoningEffort: judgeEffort(this.cfg.reasoningEffort), + meter: this.judgeMeter, deadlineAt, signal: this.running.signal, // Masked like every hidden check while it runs, released below. @@ -4561,6 +4633,7 @@ export class Engine { provider: this.provider, sessionTokens: this.sessionTokens, shedBatches: this.transcript.shedCount, + ...(this.judgeMeter.mark() ? { judge: this.judgeMeter.total() } : {}), ...(endedBy ? { endedBy } : {}), }); log?.append("receipt", { verdict: "exhausted", file: receipt.path, attempt: attempts }); @@ -5001,6 +5074,7 @@ export class Engine { reasoningEffort: judgeEffort(this.cfg.review.reasoningEffort ?? this.cfg.reasoningEffort), fetchFn: this.cfg.fetchFn, acpSpawn: this.cfg.acpSpawn, + meter: this.judgeMeter, // Never longer than the turn has left: a review that outlives the clock ends it without a verdict. ...(this.timeLeftMs() !== undefined ? { timeoutMs: Math.max(1_000, this.timeLeftMs()!) } : {}), // And every retry and pause inside it, not just one ask. @@ -5135,11 +5209,18 @@ export class Engine { this.cfg.journal?.append("note", { text: "tier: verified-audit", claim, by: [audit.judge], accepted: audit.accepted.map((n) => `audit:${n}`) }); } } + // The judge's calls since the last job reported, the drafting before this + // turn included; and the receipt's row brought up to date with them, since + // the review and the audit run after the receipt was written. + const judge = this.judgeMeter.since(this.judgeReported); + this.judgeReported = this.judgeMeter.mark(); + if (receiptPath && judge.calls) this.cfg.receipts?.amendJudge(basename(receiptPath), this.judgeMeter.total()); yield { kind: "job_end", job, steps, spend: this.spendSince(before), + ...(judge.calls ? { judge } : {}), durationMs: Date.now() - startedAt, outcome, ...(tier ? { tier, ...(tierReason ? { tierReason } : {}) } : {}), @@ -6449,6 +6530,7 @@ export class Engine { claim, result, attempt: proofAttempts, verdict, head, model: self.modelOfRecord(), provider: self.provider, sessionTokens: self.sessionTokens, session: self.cfg.journal?.sessionId, costUsd: self.costUsd(), costEstimated: self.costEstimated, + ...(self.judgeMeter.mark() ? { judge: self.judgeMeter.total() } : {}), shedBatches: self.transcript.shedCount, endedBy: why, changed: self.turnLedger().map((e) => ({ path: e.path, before: e.before, after: e.after, @@ -6495,8 +6577,9 @@ export class Engine { } } - const turnStartTokens = this.sessionTokens; - const turnStartCost = this.costUsd(); + // Both ceilings count the judge's spend too (overBudget). + const turnStartTokens = this.spentTokens; + const turnStartCost = this.totalCostUsd(); let warned = 0; // The step guard is the last way out of a turn, and it had the same fault // the spending ceiling had: it stopped dead. A reported run reached it with @@ -6611,7 +6694,10 @@ export class Engine { if (this.overBudget()) { yield { kind: "error", - text: `budget hit (${this.budgetTokens} tokens) — loop stopped. /budget to raise.`, + text: + `budget hit (${this.budgetTokens} tokens` + + (this.spentTokens > this.sessionTokens ? `, ${this.spentTokens - this.sessionTokens} of them the judge's` : "") + + `) — loop stopped. /budget to raise.`, ceiling: "budget", }; yield* this.salvage(`You have reached the token budget for this session.`, fetchFn, log); @@ -6622,9 +6708,9 @@ export class Engine { // before the request rather than after, so the limit is what molt // refuses to spend rather than what it noticed spending. // Money where a price is known, tokens only where it is not. - const spentThisTurn = this.sessionTokens - turnStartTokens; + const spentThisTurn = this.spentTokens - turnStartTokens; const usdThisTurn = - turnStartCost === undefined ? undefined : (this.costUsd() ?? 0) - turnStartCost; + turnStartCost === undefined ? undefined : (this.totalCostUsd() ?? 0) - turnStartCost; // Zero unless someone set one. The self-hosted exception that used to // live here — no default ceiling on hardware you own — is gone with the // default itself: there is nothing left to make an exception to. @@ -8175,6 +8261,7 @@ export class Engine { reasoningEffort: this.cfg.review?.reasoningEffort ?? this.cfg.reasoningEffort, fetchFn: this.cfg.fetchFn, acpSpawn: this.cfg.acpSpawn, + meter: this.judgeMeter, deadlineAt: this.deadlineAt(), }, }).catch(() => null); @@ -8448,6 +8535,7 @@ export class Engine { session: this.cfg.journal?.sessionId, costUsd: this.costUsd(), costEstimated: this.costEstimated, + ...(this.judgeMeter.mark() ? { judge: this.judgeMeter.total() } : {}), shedBatches: this.transcript.shedCount, // A question the bar could not refuse is recorded as one, so stats // never count an answer as a verified change. @@ -8510,6 +8598,7 @@ export class Engine { reasoningEffort: judgeEffort(this.cfg.review.reasoningEffort ?? this.cfg.reasoningEffort), fetchFn: this.cfg.fetchFn, acpSpawn: this.cfg.acpSpawn, + meter: this.judgeMeter, ...(this.timeLeftMs() !== undefined ? { timeoutMs: Math.max(1_000, this.timeLeftMs()!) } : {}), deadlineAt: this.deadlineAt(), }, diff --git a/src/format.ts b/src/format.ts index 8f63f0c..79c4296 100644 --- a/src/format.ts +++ b/src/format.ts @@ -10,6 +10,7 @@ * Pure, and re-exported from `banner.tsx` so nothing that already imported * them has to move. */ +import type { JudgeSpend } from "./judge-meter.js"; export function fmtTokens(n: number): string { if (n < 1000) return `${n}`; @@ -119,3 +120,25 @@ export function spendLine(sp: { (sp.costUsd === undefined ? "" : ` · ${sp.estimated ? "~" : ""}${fmtCost(sp.costUsd)}`) ); } + +/** + * A judge spend in one line, for the CLI and the receipt: + * `judge 3 calls · 1200 in (400 cached) · 300 out · $0.004`, with `$ unknown` + * when a model had no price and the plan's name when a plan paid. + */ +export function judgeSpendLine(s: JudgeSpend, fmt: (usd: number) => string = fmtCost): string { + const cache = + s.cacheReadTokens > 0 || s.cacheWriteTokens > 0 + ? ` (${[s.cacheReadTokens > 0 ? `${s.cacheReadTokens} cached` : "", s.cacheWriteTokens > 0 ? `${s.cacheWriteTokens} cache write` : ""].filter(Boolean).join(", ")})` + : ""; + const e = s.estimated ? "~" : ""; + const money = + s.costUsd !== undefined + ? `${s.billed ? "" : e}${fmt(s.costUsd)}${s.plan ? ` + ${s.plan} plan` : ""}` + : s.unpricedCalls > 0 + ? `$ unknown (no price for ${(s.unpricedModels ?? s.models).join(", ")})` + : s.plan + ? `your ${s.plan} plan, not metered` + : "$ unknown"; + return `judge ${s.calls} call${s.calls === 1 ? "" : "s"} · ${e}${s.promptTokens} in${cache} · ${e}${s.completionTokens} out · ${money}`; +} diff --git a/src/journal.ts b/src/journal.ts index 89f1853..7b4864b 100644 --- a/src/journal.ts +++ b/src/journal.ts @@ -86,6 +86,12 @@ export type JournalKind = */ | "loop_stop" | "salvage" + /** + * One answered ask to the judge (drafting, critic, reference, review, + * audit, arbiter): its tokens and cost, apart from the worker's `response` + * entries. `costUsd: null` means no price is known, never $0. + */ + | "judge_usage" /** The independent review of a verified claim: a label, never a gate. */ | "review" | "dispute" @@ -387,6 +393,19 @@ export class Journal { ); break; } + case "judge_usage": { + const e = d.estimated ? "~" : ""; + const cached = Number(d.cacheReadTokens ?? 0) > 0 ? ` (${d.cacheReadTokens} cached)` : ""; + const cost = d.plan + ? ` · ${d.plan} plan` + : d.costUsd === null || d.costUsd === undefined + ? " · $ unknown" + : ` · ${d.billed ? "" : e}$${Number(d.costUsd).toFixed(6)}`; + out.push( + `${t} judge ${d.model}${d.what ? ` (${d.what})` : ""} · ${e}${d.promptTokens} in${cached} / ${e}${d.completionTokens} out${cost}`, + ); + break; + } case "tool_call": out.push(`${t} tool ${d.name}: ${d.detail}`); break; diff --git a/src/judge-meter.ts b/src/judge-meter.ts new file mode 100644 index 0000000..12feb94 --- /dev/null +++ b/src/judge-meter.ts @@ -0,0 +1,241 @@ +/** + * The judge's meter: every out-of-loop ask, counted and priced. + * + * The worker's own requests have always been metered step by step. The asks + * made around the work were not: drafting the hidden checks, the critic that + * reads the draft, the reference check, the independent review, the post-work + * audit and the dispute arbiter. With a separate judge model (MAAT_JUDGE_MODEL) + * those asks go to a different model, often on a different provider, and a + * bench run could only estimate what they cost; the exact figure was visible + * only in the provider's console. Every token spent should be visible here. + * + * Each ask that returns records what the provider reported (input, output, + * cache read and cache write tokens, and a billed dollar figure when the + * provider sends one). It is priced with the same table as the worker + * (providers.ts), keyed on the judge's own model id. A model with no known + * price reports its tokens and "$ unknown", never $0: an unpriced judge + * looking free is the confidently wrong meter the worker's pricing already + * refuses to be. A subscription backend (Grok Build, OpenCode) is paid by its + * plan and gets no dollar figure, as on the worker side. + * + * Kept apart from the worker's meter on purpose. The worker's tokens are what + * the lean-budget suite measures and what the step lines report; the judge's + * are a separate line on every surface, and both count toward a budget. + */ +import { anthropicPricing, planFor, type Pricing } from "./providers.js"; + +/** What one ask used, as the ask path read it from the reply. */ +export type AskUsage = { + baseUrl: string; + model: string; + /** What was being asked ("drafting criteria", "reviewing the claim"). */ + what?: string; + promptTokens: number; + completionTokens: number; + /** Prompt tokens served from the provider's cache, when it itemises them. */ + cacheReadTokens?: number; + /** Prompt tokens written to the provider's cache, when it reports them. */ + cacheWriteTokens?: number; + /** USD, when the provider reported the figure itself (OpenRouter's `usage.cost`). */ + billedUsd?: number; + /** True when the provider reported no usage and the counts are Maat's estimate. */ + estimated: boolean; +}; + +/** Anything an ask can report its usage to. */ +export interface AskMeter { + record(u: AskUsage): void; +} + +/** The judge's spend over some span of calls. */ +export type JudgeSpend = { + calls: number; + promptTokens: number; + completionTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + /** + * USD, or undefined when it is not known: some call's model has no price. + * Never 0 for unknown. Calls paid by a subscription plan add nothing. + */ + costUsd?: number; + /** Calls whose model had no price and no billed figure. */ + unpricedCalls: number; + /** The models those calls went to. */ + unpricedModels?: string[]; + /** Calls paid for by a subscription plan, by the plan's name. */ + plan?: string; + /** True when any call's counts are Maat's estimate. */ + estimated: boolean; + /** True when every priced call's dollar figure came from the provider. */ + billed: boolean; + /** The models asked, in the order first seen. */ + models: string[]; +}; + +/** The worker's pricing, for a judge that is the worker's own model. */ +export type WorkerPricing = { baseUrl: string; model: string; pricing: Pricing | null; plan?: string }; + +/** A call as recorded, with what it cost when it was recorded. */ +export type JudgeCall = AskUsage & { costUsd?: number; plan?: string }; + +function key(baseUrl: string, model: string): string { + return `${baseUrl.replace(/\/+$/, "")} ${model}`; +} + +function anthropicHost(baseUrl: string): boolean { + try { + return /anthropic\.com$/.test(new URL(baseUrl).hostname); + } catch { + return false; + } +} + +/** USD for these tokens at this price; cached tokens are inside `prompt`. */ +export function priceTokens(p: Pricing, u: Pick): number { + // As the worker's meter: cached prompt tokens at the cache rate when one is + // published, the rest of the prompt (cache writes included) at the base rate. + const cached = p.cached === undefined ? 0 : Math.min(u.cacheReadTokens ?? 0, u.promptTokens); + const fresh = u.promptTokens - cached; + return (fresh / 1e6) * p.in + (cached / 1e6) * (p.cached ?? p.in) + (u.completionTokens / 1e6) * p.out; +} + +export class JudgeMeter implements AskMeter { + private readonly calls: JudgeCall[] = []; + private readonly prices = new Map(); + private readonly listeners: ((c: JudgeCall) => void)[] = []; + + /** + * @param worker The worker's endpoint, model and prices, read when needed: a + * judge that is the worker's own model is priced exactly as the worker is, + * hand-set prices included. + */ + constructor(private readonly worker?: () => WorkerPricing) {} + + /** Set the price of a judge model, as resolved from its provider (or null: none published). */ + setPricing(baseUrl: string, model: string, p: Pricing | null): void { + this.prices.set(key(baseUrl, model), p); + } + + /** Has a price lookup already been made for this model? */ + hasPricing(baseUrl: string, model: string): boolean { + return this.prices.has(key(baseUrl, model)); + } + + /** + * The price of one judge model, or null when none is known. + * + * A price set for this endpoint and model wins. Then the worker's own prices, + * when the judge is the worker's model on the worker's endpoint. Then the + * published Anthropic rates, on Anthropic's API (the table fetchPricing reads + * there). Nothing else is guessed. + */ + pricingFor(baseUrl: string, model: string): Pricing | null { + const set = this.prices.get(key(baseUrl, model)); + if (set !== undefined) return set; + const w = this.worker?.(); + if (w && key(w.baseUrl, w.model) === key(baseUrl, model)) return w.pricing; + if (anthropicHost(baseUrl)) return anthropicPricing(model); + return null; + } + + /** The subscription plan that pays for asks on this endpoint, if any. */ + private planOf(baseUrl: string): string | undefined { + return planFor(baseUrl); + } + + /** Called with every call as it is recorded (the journal's `judge_usage`). */ + onRecord(fn: (c: JudgeCall) => void): void { + this.listeners.push(fn); + } + + record(u: AskUsage): void { + const c = this.priced(u); + this.calls.push(c); + for (const fn of this.listeners) { + try { + fn(c); + } catch { + /* a listener that throws must not lose the record */ + } + } + } + + private priced(u: AskUsage): JudgeCall { + const plan = this.planOf(u.baseUrl); + if (plan) return { ...u, plan }; + if (typeof u.billedUsd === "number") return { ...u, costUsd: u.billedUsd }; + const p = this.pricingFor(u.baseUrl, u.model); + return p ? { ...u, costUsd: priceTokens(p, u) } : { ...u }; + } + + /** How many calls have been recorded: a mark to take `since` from. */ + mark(): number { + return this.calls.length; + } + + /** Every call recorded, in order. */ + all(): readonly JudgeCall[] { + return this.calls; + } + + /** The whole session's judge spend. */ + total(): JudgeSpend { + return this.since(0); + } + + /** + * The judge's spend since a mark. Priced now, not when recorded, so a price + * that arrived after the first ask (a lookup still in flight, a /price) is + * applied to every call, as the worker's meter does. + */ + since(mark: number): JudgeSpend { + const span = this.calls.slice(mark).map((c) => this.priced(c)); + const models: string[] = []; + let cost = 0; + let unpriced = 0; + const unpricedModels: string[] = []; + let billedAll = true; + let pricedAny = false; + let plan: string | undefined; + for (const c of span) { + if (!models.includes(c.model)) models.push(c.model); + if (c.plan) { + plan = c.plan; + continue; + } + if (c.costUsd === undefined) { + unpriced += 1; + if (!unpricedModels.includes(c.model)) unpricedModels.push(c.model); + continue; + } + pricedAny = true; + cost += c.costUsd; + if (typeof c.billedUsd !== "number") billedAll = false; + } + return { + calls: span.length, + promptTokens: span.reduce((n, c) => n + c.promptTokens, 0), + completionTokens: span.reduce((n, c) => n + c.completionTokens, 0), + cacheReadTokens: span.reduce((n, c) => n + (c.cacheReadTokens ?? 0), 0), + cacheWriteTokens: span.reduce((n, c) => n + (c.cacheWriteTokens ?? 0), 0), + // Unknown if any call is unknown: a partial sum would read as the whole. + // Plan-paid calls cost no money, so a span of only those has no figure. + ...(unpriced === 0 && pricedAny ? { costUsd: cost } : {}), + unpricedCalls: unpriced, + ...(unpricedModels.length ? { unpricedModels } : {}), + ...(plan ? { plan } : {}), + estimated: span.some((c) => c.estimated), + billed: pricedAny && billedAll, + models, + }; + } +} + +/** Tokens a judge spend counts toward a budget: everything sent and received. */ +export function judgeTokens(s: JudgeSpend | undefined): number { + return s ? s.promptTokens + s.completionTokens : 0; +} + +/** Said in one line by format.ts, which the renderer can import. */ +export { judgeSpendLine } from "./format.js"; diff --git a/src/mission.ts b/src/mission.ts index 7175ea2..5084e53 100644 --- a/src/mission.ts +++ b/src/mission.ts @@ -499,6 +499,8 @@ export type Handoff = { failed: string[]; receipt?: string; spend?: Spend; + /** The judge's spend on this attempt, apart from the worker's (judge-meter.ts). */ + judge?: import("./judge-meter.js").JudgeSpend; durationMs: number; at: string; }; @@ -640,6 +642,7 @@ export async function runMission(opts: MissionRunOptions): Promise `$${usd.toFixed(4)}`)} so far`] : []), `- shed batches archived: ${args.shedBatches}`, `- bar duration: ${args.result.durationMs}ms`, "", @@ -775,6 +789,7 @@ export class Receipts { ...(args.session ? { session: args.session } : {}), ...(args.costUsd === undefined ? {} : { costUsd: args.costUsd }), ...(args.costEstimated ? { costEstimated: true } : {}), + ...(args.judge ? { judge: args.judge } : {}), ...(args.ask ? { ask: true } : {}), ...(args.revealed?.length ? { revealed: [...args.revealed] } : {}), ...(args.endedBy ? { endedBy: args.endedBy } : {}), @@ -847,7 +862,16 @@ export class Receipts { * other write while hidden commands are withheld. The receipt file itself, * hash-bound when it was written, is never rewritten. */ - amendReview(file: string, objections: readonly Objection[]): boolean { + /** + * Bring a row's judge spend up to date at the end of its job: the review and + * the post-work audit ask the judge after the receipt is written. + */ + amendJudge(file: string, judge: JudgeSpend): boolean { + return this.amendRow(file, (r) => ({ ...r, judge })); + } + + /** Rewrite one index row, by receipt file name. False when there is none. */ + private amendRow(file: string, change: (r: ReceiptRecord) => ReceiptRecord): boolean { try { if (!existsSync(this.indexPath)) return false; let hit = false; @@ -859,7 +883,7 @@ export class Receipts { const r = JSON.parse(l) as ReceiptRecord; if (r.file !== file) return l; hit = true; - return redact(JSON.stringify({ ...r, objections: maskDeep([...objections], this.withheld) }), this.secrets); + return redact(JSON.stringify(change(r)), this.secrets); } catch { return l; } @@ -871,6 +895,11 @@ export class Receipts { } } + amendReview(file: string, objections: readonly Objection[]): boolean { + return this.amendRow(file, (r) => ({ ...r, objections: maskDeep([...objections], this.withheld) })); + } + + /** * Reconcile the index against the files on disk. * diff --git a/src/stream.ts b/src/stream.ts index 3ac0990..bc6810a 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -395,6 +395,7 @@ export class StreamAccumulator { promptTokens: this.promptTokens, completionTokens: this.completionTokens, cachedTokens: this.cachedTokens, + ...(this.cacheWriteTokens !== undefined ? { cacheWriteTokens: this.cacheWriteTokens } : {}), reasoningTokens: this.reasoningTokens, costUsd: this.costUsd, finishReason: this.finishReason, diff --git a/src/types.ts b/src/types.ts index 67745f8..1163c48 100644 --- a/src/types.ts +++ b/src/types.ts @@ -77,6 +77,8 @@ export type Bom = { /** True when any part of the cost rests on molt's own token estimate. */ costEstimated?: boolean; budgetTokens?: number; + /** The judge's session spend, apart from the worker's above (judge-meter.ts). Absent before its first call. */ + judge?: import("./judge-meter.js").JudgeSpend; }; /** @@ -638,7 +640,15 @@ export type EngineEvent = kind: "job_end"; job: number; steps: number; + /** The worker's spend: its own requests, step by step. */ spend: Spend; + /** + * The judge's spend on this job, apart from the worker's: every ask made + * around the work (drafting, critic, reference, review, audit, arbiter) + * since the last job_end. `costUsd` absent means the price is unknown, + * never $0. Absent when the judge was not asked. + */ + judge?: import("./judge-meter.js").JudgeSpend; durationMs: number; outcome: JobOutcome; /** diff --git a/test/judge-meter.test.ts b/test/judge-meter.test.ts new file mode 100644 index 0000000..652a2fc --- /dev/null +++ b/test/judge-meter.test.ts @@ -0,0 +1,333 @@ +/** + * The judge's meter: every ask made around the work (drafting, critic, + * reference, review, audit, arbiter) is counted and priced apart from the + * worker's requests, reported on job_end, the receipt, the journal and the + * CLI line, and counted by a budget. + * + * A bench run on 2026-10-08 could only estimate the judge's cost; the exact + * figure was visible only in the provider's console. + */ +import assert from "node:assert/strict"; +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, it } from "node:test"; +import { askModel } from "../src/ask.js"; +import { Engine } from "../src/engine.js"; +import { judgeSpendLine } from "../src/format.js"; +import { Journal } from "../src/journal.js"; +import { JudgeMeter, priceTokens } from "../src/judge-meter.js"; +import { Receipts } from "../src/receipts.js"; +import type { Check, EngineEvent } from "../src/types.js"; +import { allowAll, drain, workspace } from "./helpers.js"; + +const URL = "http://provider.test/v1"; +const JUDGE = "judge-j"; +const PRICE = { in: 2, out: 10, cached: 0.2, source: "test" }; +/** What the scripted judge reports for every ask. */ +const JUDGE_USAGE = { prompt_tokens: 1000, completion_tokens: 200, prompt_tokens_details: { cached_tokens: 400 }, cache_creation_input_tokens: 50 }; +/** 600 fresh x $2 + 400 cached x $0.2 + 200 out x $10, per 1M. */ +const PER_CALL = (600 * 2 + 400 * 0.2 + 200 * 10) / 1e6; + +const reply = (body: unknown) => + ({ ok: true, status: 200, json: async () => body, text: async () => JSON.stringify(body), headers: new Headers() }) as unknown as Response; + +/** + * One endpoint serving two models: the worker ("m") writes out.txt and says + * done; the judge answers every review ask with no violations and JUDGE_USAGE. + */ +function twoModels() { + const sent: { model: string }[] = []; + let worker = 0; + const fetchFn = (async (_url: string, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { model: string }; + sent.push(body); + if (body.model === JUDGE) { + return reply({ choices: [{ message: { role: "assistant", content: '{"violations":[]}' }, finish_reason: "stop" }], usage: JUDGE_USAGE }); + } + worker += 1; + const message = + worker === 1 + ? { role: "assistant", content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "write_file", arguments: JSON.stringify({ path: "out.txt", content: "hello\n" }) } }] } + : { role: "assistant", content: "Done." }; + return reply({ choices: [{ message, finish_reason: worker === 1 ? "tool_calls" : "stop" }], usage: { prompt_tokens: 100, completion_tokens: 20 } }); + }) as unknown as typeof fetch; + return { fetchFn, sent, judgeCalls: () => sent.filter((b) => b.model === JUDGE).length }; +} + +const greet: Check = { + name: "greeting", kind: "command", run: "grep -qx hello out.txt", timeoutMs: 5_000, expectExit: 0, tags: ["task", "value", "exact"], hidden: true, + author: { kind: "judge", model: JUDGE }, +}; + +function jobEnd(events: EngineEvent[]) { + const e = events.find((x) => x.kind === "job_end"); + assert.ok(e && e.kind === "job_end"); + return e; +} + +describe("the judge's meter on a reviewed turn", () => { + let saved: string | undefined; + beforeEach(() => { + saved = process.env.MAAT_JUDGE_MODEL; + process.env.MAAT_JUDGE_MODEL = JUDGE; + }); + afterEach(() => { + if (saved === undefined) delete process.env.MAAT_JUDGE_MODEL; + else process.env.MAAT_JUDGE_MODEL = saved; + }); + + const run = async (priced: boolean) => { + const ws = workspace(); + const p = twoModels(); + const journal = new Journal(ws.dir); + const engine = new Engine({ + baseUrl: URL, model: "m", cwd: ws.dir, fetchFn: p.fetchFn, bar: null, stream: false, autonomy: "high", + review: { votes: 3 }, receipts: new Receipts(ws.dir), journal, + priceInPerMtok: 1, priceOutPerMtok: 5, + }); + if (priced) engine.judgeMeter.setPricing(URL, JUDGE, PRICE); + // The drafting the CLI does before the turn, on the same meter. + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "drafting criteria", promptTokens: 1000, completionTokens: 200, cacheReadTokens: 400, cacheWriteTokens: 50, estimated: false }); + const events = await drain(engine.run("write out.txt", allowAll, { taskChecks: [greet] })); + return { ws, engine, events, p, journal }; + }; + + it("reports the judge's tokens and $ on job_end, apart from the worker's", async () => { + const { ws, events, p, engine } = await run(true); + try { + const e = jobEnd(events); + assert.equal(p.judgeCalls(), 3, "three review votes went to the judge"); + assert.ok(e.judge, "job_end carries the judge's spend"); + // The drafting before the turn and the three review votes. + assert.equal(e.judge.calls, 4); + assert.equal(e.judge.promptTokens, 4000); + assert.equal(e.judge.completionTokens, 800); + assert.equal(e.judge.cacheReadTokens, 1600); + assert.equal(e.judge.cacheWriteTokens, 200); + assert.ok(Math.abs(e.judge.costUsd! - 4 * PER_CALL) < 1e-12, `judge $ ${e.judge.costUsd}`); + assert.deepEqual(e.judge.models, [JUDGE]); + // The worker's spend is its own two requests, untouched by the judge's. + assert.equal(e.spend.promptTokens, 200); + assert.equal(e.spend.completionTokens, 40); + assert.equal(engine.sessionTokens, 240, "sessionTokens stays the worker's"); + assert.equal(engine.spentTokens, 240 + 4800, "a budget counts both"); + } finally { + ws.cleanup(); + } + }); + + it("writes the judge's spend to the receipt row, the receipt and the journal", async () => { + const { ws, journal } = await run(true); + try { + const rows = new Receipts(ws.dir).records(); + const row = rows.find((r) => r.verdict === "accepted"); + assert.ok(row?.judge, "the receipt row carries the judge's spend"); + // Brought up to date at the job's end: the review ran after the receipt was written. + assert.equal(row.judge.calls, 4); + assert.ok(Math.abs(row.judge.costUsd! - 4 * PER_CALL) < 1e-12); + assert.equal(row.costUsd, (200 * 1 + 40 * 5) / 1e6, "the worker's cost stays the worker's"); + // The receipt file, written at the claim, says what the judge had spent by then. + const dir = join(ws.dir, ".maat", "receipts"); + const file = readdirSync(dir).find((f) => f.endsWith("-accepted.md")); + assert.ok(file); + const body = readFileSync(join(dir, file), "utf8"); + assert.match(body, /- session cost: \$0\.0004 \(worker\)/); + assert.match(body, /- judge 1 call · 1000 in \(400 cached, 50 cache write\) · 200 out · \$0\.0033 so far/); + const entries = Journal.read(journal.path).filter((x) => x.kind === "judge_usage"); + assert.equal(entries.length, 4); + const d = entries[1]!.data as Record; + assert.equal(d.model, JUDGE); + assert.equal(d.what, "reviewing the claim"); + assert.equal(d.promptTokens, 1000); + assert.equal(d.cacheReadTokens, 400); + assert.equal(d.cacheWriteTokens, 50); + assert.ok(Math.abs((d.costUsd as number) - PER_CALL) < 1e-12); + assert.ok(Journal.summarize(entries.slice(1)).every((l) => /judge judge-j \(reviewing the claim\) · 1000 in \(400 cached\) \/ 200 out · \$0\.0032/.test(l))); + } finally { + ws.cleanup(); + } + }); + + it("an unpriced judge model reports its tokens and $ unknown, never $0", async () => { + const { ws, events, journal } = await run(false); + try { + const e = jobEnd(events); + assert.ok(e.judge); + assert.equal(e.judge.promptTokens, 4000); + assert.equal(e.judge.costUsd, undefined, "no price means no dollar figure"); + assert.equal(e.judge.unpricedCalls, 4); + assert.deepEqual(e.judge.unpricedModels, [JUDGE]); + assert.match(judgeSpendLine(e.judge), /^judge 4 calls · 4000 in \(1600 cached, 200 cache write\) · 800 out · \$ unknown \(no price for judge-j\)$/); + const row = new Receipts(ws.dir).records().find((r) => r.verdict === "accepted"); + assert.equal(row?.judge?.costUsd, undefined); + const entries = Journal.read(journal.path).filter((x) => x.kind === "judge_usage"); + assert.equal((entries[0]!.data as Record).costUsd, null, "null, not 0"); + assert.ok(Journal.summarize(entries).every((l) => /\$ unknown$/.test(l))); + // The receipt file says so in words. + const dir = join(ws.dir, ".maat", "receipts"); + const file = readdirSync(dir).find((f) => f.endsWith("-accepted.md")); + assert.ok(file); + const body = readFileSync(join(dir, file), "utf8"); + assert.match(body, /- judge 1 call · 1000 in \(400 cached, 50 cache write\) · 200 out · \$ unknown \(no price for judge-j\) so far/); + assert.doesNotMatch(body, /judge \d+ calls? · .*\$0\.0000/); + } finally { + ws.cleanup(); + } + }); +}); + +describe("a budget counts the judge's spend", () => { + it("a token budget the judge has used up stops the worker before its first step", async () => { + const ws = workspace(); + try { + let asked = 0; + const fetchFn = (async () => { + asked += 1; + return reply({ choices: [{ message: { role: "assistant", content: "Done." }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 5 } }); + }) as unknown as typeof fetch; + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high" }); + engine.setBudget(5_000); + // The drafting before the turn: on the judge's meter, not the worker's. + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "drafting criteria", promptTokens: 4_800, completionTokens: 400, estimated: false }); + assert.equal(engine.sessionTokens, 0); + const events = await drain(engine.run("anything", allowAll)); + const hit = events.find((x) => x.kind === "error" && x.ceiling === "budget"); + assert.ok(hit && hit.kind === "error", "the budget was hit"); + assert.match(hit.text, /budget hit \(5000 tokens, 5200 of them the judge's\)/); + assert.equal(asked, 1, "only the closing summary was asked for; no work step ran"); + const e = jobEnd(events); + assert.equal(e.judge?.promptTokens, 4_800, "the pre-turn drafting is reported with the turn it was for"); + } finally { + ws.cleanup(); + } + }); + + it("a money ceiling counts the judge's dollars", async () => { + const ws = workspace(); + try { + const fetchFn = (async () => + reply({ choices: [{ message: { role: "assistant", content: null, tool_calls: [{ id: "c", type: "function", function: { name: "list_files", arguments: "{}" } }] }, finish_reason: "tool_calls" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })) as unknown as typeof fetch; + const engine = new Engine({ + baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high", + priceInPerMtok: 1, priceOutPerMtok: 1, maxTurnUsd: 0.01, maxSteps: 3, + }); + engine.judgeMeter.setPricing(URL, JUDGE, { in: 1000, out: 1000, source: "test" }); + const gen = engine.run("anything", allowAll); + const events: EngineEvent[] = []; + for await (const ev of gen) { + events.push(ev); + // The judge spends $0.02 mid-turn: past the $0.01 ceiling on its own. + if (ev.kind === "usage" && events.filter((x) => x.kind === "usage").length === 1) { + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, promptTokens: 10, completionTokens: 10, estimated: false }); + } + } + assert.equal(engine.totalCostUsd()! > 0.02, true); + assert.ok( + events.some((x) => (x.kind === "error" || x.kind === "info") && /\$0\.0\d+ of \$0\.01/.test(x.text)), + "the turn's money ceiling saw the judge's spend", + ); + } finally { + ws.cleanup(); + } + }); +}); + +describe("JudgeMeter", () => { + const u = { baseUrl: URL, model: JUDGE, promptTokens: 1000, completionTokens: 200, cacheReadTokens: 400, estimated: false }; + + it("prices a call with a set price, keyed on the judge's model id", () => { + const m = new JudgeMeter(); + m.setPricing(URL, JUDGE, PRICE); + m.record(u); + assert.ok(Math.abs(m.total().costUsd! - PER_CALL) < 1e-12); + assert.equal(priceTokens(PRICE, u), m.total().costUsd); + }); + + it("prices a call again once a price arrives after it", () => { + const m = new JudgeMeter(); + m.record(u); + assert.equal(m.total().costUsd, undefined); + m.setPricing(URL, JUDGE, PRICE); + assert.ok(Math.abs(m.total().costUsd! - PER_CALL) < 1e-12); + }); + + it("uses the provider's billed figure when it sends one", () => { + const m = new JudgeMeter(); + m.record({ ...u, billedUsd: 0.5 }); + assert.equal(m.total().costUsd, 0.5); + assert.equal(m.total().billed, true); + }); + + it("is unknown, not a partial sum, when any call is unpriced", () => { + const m = new JudgeMeter(); + m.setPricing(URL, JUDGE, PRICE); + m.record(u); + m.record({ ...u, model: "other" }); + const s = m.total(); + assert.equal(s.costUsd, undefined); + assert.equal(s.unpricedCalls, 1); + assert.deepEqual(s.unpricedModels, ["other"]); + }); + + it("a judge that is the worker's own model is priced as the worker is", () => { + const m = new JudgeMeter(() => ({ baseUrl: URL, model: "m", pricing: PRICE })); + m.record({ ...u, model: "m" }); + assert.ok(Math.abs(m.total().costUsd! - PER_CALL) < 1e-12); + }); + + it("knows Anthropic's published rates on Anthropic's API", () => { + const m = new JudgeMeter(); + m.record({ ...u, baseUrl: "https://api.anthropic.com/v1", model: "claude-haiku-4-5", cacheReadTokens: 0 }); + // Haiku 4.5: $1 in, $5 out. + assert.ok(Math.abs(m.total().costUsd! - (1000 * 1 + 200 * 5) / 1e6) < 1e-12); + }); + + it("a subscription judge is paid by its plan: no dollar figure, said as the plan", () => { + const m = new JudgeMeter(); + m.record({ ...u, baseUrl: "grok-build://subscription", estimated: true }); + const s = m.total(); + assert.equal(s.costUsd, undefined); + assert.equal(s.unpricedCalls, 0); + assert.match(judgeSpendLine(s), /your .+ plan, not metered$/); + }); + + it("since() reports only the calls after a mark", () => { + const m = new JudgeMeter(); + m.record(u); + const at = m.mark(); + m.record({ ...u, promptTokens: 7 }); + assert.equal(m.since(at).calls, 1); + assert.equal(m.since(at).promptTokens, 7); + }); +}); + +describe("askModel reports usage to its meter", () => { + it("reads the provider's usage block, cache read and write included", async () => { + const m = new JudgeMeter(); + const fetchFn = (async () => reply({ choices: [{ message: { content: "ok" }, finish_reason: "stop" }], usage: { ...JUDGE_USAGE, cost: 0.004 } })) as unknown as typeof fetch; + const r = await askModel({ baseUrl: URL, model: JUDGE, system: "S", prompt: "P", fetchFn, meter: m, what: "drafting criteria" }); + assert.equal(r.ok, true); + const [c] = m.all(); + assert.deepEqual( + { p: c!.promptTokens, c: c!.completionTokens, r: c!.cacheReadTokens, w: c!.cacheWriteTokens, b: c!.billedUsd, e: c!.estimated, what: c!.what }, + { p: 1000, c: 200, r: 400, w: 50, b: 0.004, e: false, what: "drafting criteria" }, + ); + }); + + it("estimates, and says so, when the provider reports no usage", async () => { + const m = new JudgeMeter(); + const fetchFn = (async () => reply({ choices: [{ message: { content: "abcdefgh" }, finish_reason: "stop" }] })) as unknown as typeof fetch; + await askModel({ baseUrl: URL, model: JUDGE, system: "SSSS", prompt: "PPPP", fetchFn, meter: m }); + const [c] = m.all(); + assert.equal(c!.estimated, true); + assert.equal(c!.promptTokens, 2); + assert.equal(c!.completionTokens, 2); + }); + + it("records nothing for a refused ask", async () => { + const m = new JudgeMeter(); + const fetchFn = (async () => ({ ok: false, status: 500, text: async () => "", json: async () => ({}) }) as unknown as Response) as unknown as typeof fetch; + await askModel({ baseUrl: URL, model: JUDGE, system: "S", prompt: "P", fetchFn, meter: m, overloadBackoffMs: [] }); + assert.equal(m.mark(), 0); + }); +}); diff --git a/ui/app.ts b/ui/app.ts index dbb764d..6d66247 100644 --- a/ui/app.ts +++ b/ui/app.ts @@ -15,7 +15,7 @@ import { renderMarkdown } from "./markdown.js"; import { nextWaitWord } from "./wait-words.js"; import { playMaatSplash } from "./maat-splash.js"; -import { fmtCost, spendLine, stepDid } from "../src/format.js"; +import { fmtCost, judgeSpendLine, spendLine, stepDid } from "../src/format.js"; import { jobEndWords, type JobEndWords } from "../src/tiers.js"; import { expandEndpointShorthand, typedEndpointProblem } from "../src/endpoint.js"; import { matchCommands } from "../src/commands.js"; @@ -1143,7 +1143,9 @@ molt.onEvent((ev) => { const said = jobEndWords(ev as JobEndWords); say( "", - `job ${said} · ${ev.steps} step(s) · ${spendLine(ev.spend)} · ${fmtMs(Number(ev.durationMs) || 0)}`, + `job ${said} · ${ev.steps} step(s) · ${spendLine(ev.spend)} · ${fmtMs(Number(ev.durationMs) || 0)}` + + // The judge's share, apart from the worker's (judge-meter.ts). + (ev.judge ? ` · ${judgeSpendLine(ev.judge)}` : ""), "step", ); void refreshStats();