Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,19 @@

### Added

- **The judge's spend is metered.** Every ask made around the work (drafting
and its critic, the reference check, `--review`, the post-work audit, the
arbiter) records its input, output, cache read and cache write tokens, priced
from the worker's price table keyed on the judge's own model id; an unpriced
model shows `$ unknown`, never `$0`. Reported apart from the worker's spend
as `judge` on `job_end`, `judge_usage` journal entries, a `judge` line and a
worker + judge total in the CLI, and `judge` on the receipt (row updated at
the job's end). `--budget` and the per-turn ceilings now count worker and
judge spend together. `bench/local/run.py` adds `judge_*` token and cost
columns to each row, and its cost alarm now judges worker + judge cost and
prompt tokens (the detail shows the split), so a runaway judge stops the lane
too. See `docs/transparency.md` and `docs/lean.md`.

- **`--review-executable`** (experimental, also `MAAT_REVIEW_EXECUTABLE=1`):
every `--review` objection must carry a read-only command that demonstrates
it. Maat runs it on a throwaway copy of the tree, with no API keys, tokens or
Expand Down
82 changes: 68 additions & 14 deletions bench/local/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -627,7 +627,7 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict:
write_private(log, out)
# Maat's own notices (criteria not drafted, dropped checks, refusals) go to stderr.
write_private(log.with_suffix(".err"), err or "")
steps = 0; per = []; outcome = None; spend = {}; review = None; disagree = []; extra = {}
steps = 0; per = []; outcome = None; spend = {}; judge = None; review = None; disagree = []; extra = {}
for line in out.splitlines():
if not line.startswith("{"):
continue
Expand All @@ -641,6 +641,9 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict:
elif ev.get("kind") == "job_end":
outcome = claim_of(ev)
spend = ev.get("spend") or {}
# The judge's asks (drafting, critic, review, audit), metered apart from
# the worker's. Absent on builds before judge metering, or when no judge ran.
judge = ev.get("judge")
review = ev.get("review")
disagree = ev.get("checksDisagree") or []
# Only what job_end carried; absent keys stay absent (older builds).
Expand All @@ -651,7 +654,9 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict:
return {
"provider_capped": capped,
"secs": round(secs), "turns": steps, "calls": sum(per), "multi": sum(1 for x in per if x > 1),
"tokens_in": spend.get("promptTokens"), "cost_usd": spend.get("costUsd"), "claim": outcome, "timed_out": timed_out,
"tokens_in": spend.get("promptTokens"), "cost_usd": spend.get("costUsd"),
**judge_columns(judge),
"claim": outcome, "timed_out": timed_out,
"said_done": (outcome or "").startswith("verified"),
# The reviewer's label: a verified claim it did not confirm.
"review": review,
Expand All @@ -662,6 +667,24 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict:
}


def judge_columns(judge: dict | None) -> dict:
"""The judge's tokens and $, next to the worker's `tokens_in` / `cost_usd`.

Only when the run output carried them (job_end.judge, Maat's judge meter): None
otherwise, so an older build is never read as a free judge. `judge_cost_usd` is
None when the judge's model had no price ("$ unknown"), never 0."""
if not isinstance(judge, dict):
return {}
return {
"judge_calls": judge.get("calls"),
"judge_tokens_in": judge.get("promptTokens"),
"judge_tokens_out": judge.get("completionTokens"),
"judge_cache_read": judge.get("cacheReadTokens"),
"judge_cache_write": judge.get("cacheWriteTokens"),
"judge_cost_usd": judge.get("costUsd"),
}


# Per-run cost alarm (reports/loop-charter.md, non-regression rule 1): a run that
# costs more than max(5x the lane's running median cost, $0.10), or sends more
# than 1.5M prompt tokens, stops the lane before the bill grows. 2026-10-07: one
Expand All @@ -678,24 +701,55 @@ def alarm_limits() -> dict:
}


def _sum_known(*xs):
"""The sum of the values that are known, or None when none is."""
known = [x for x in xs if x is not None]
return sum(known) if known else None


def run_cost(row: dict) -> float | None:
"""A run's whole cost: the worker's plus the judge's (judge_columns). None when
neither is known; an unpriced judge adds nothing it cannot price."""
return _sum_known(row.get("cost_usd"), row.get("judge_cost_usd"))


def run_tokens(row: dict) -> int | None:
"""A run's prompt tokens, the worker's plus the judge's."""
return _sum_known(row.get("tokens_in"), row.get("judge_tokens_in"))


def cost_alarm(row: dict, prior_costs: list[float]) -> dict | None:
"""Why this run trips the alarm, or None. `prior_costs` are the lane's earlier runs
(this run excluded): the median is over runs that reported a cost. A run with no
cost (an unpriced model) is judged on its prompt tokens only."""
"""Why this run trips the alarm, or None. `prior_costs` are the lane's earlier runs'
total costs (this run excluded): the median is over runs that reported a cost. A
run with no cost (an unpriced model) is judged on its prompt tokens only.

Cost and tokens are the worker's plus the judge's: a runaway judge trips it too.
When the run had a judge, the detail says how the total splits."""
lim = alarm_limits()
med = statistics.median(prior_costs) if prior_costs else None
limit_usd = max(lim["x"] * med, lim["usd"]) if med is not None else lim["usd"]
cost, toks = row.get("cost_usd"), row.get("tokens_in")
cost, toks = run_cost(row), run_tokens(row)
judged = row.get("judge_calls") is not None

def usd(x):
return "$ unknown" if x is None else f"${x:.4f}"

why = []
if cost is not None and cost > limit_usd:
why.append(f"cost ${cost:.4f} > ${limit_usd:.4f} (max({lim['x']:g}x lane median "
split = f" (worker {usd(row.get('cost_usd'))} + judge {usd(row.get('judge_cost_usd'))})" if judged else ""
why.append(f"cost ${cost:.4f}{split} > ${limit_usd:.4f} (max({lim['x']:g}x lane median "
f"{'n/a' if med is None else f'${med:.4f}'}, ${lim['usd']:g}))")
if toks is not None and toks > lim["tokens"]:
why.append(f"prompt tokens {toks:,} > {lim['tokens']:,}")
split = (f" (worker {row.get('tokens_in') or 0:,} + judge {row.get('judge_tokens_in') or 0:,})"
if judged else "")
why.append(f"prompt tokens {toks:,}{split} > {lim['tokens']:,}")
if not why:
return None
return {"detail": "; ".join(why), "cost_usd": cost, "tokens_in": toks, "lane_median_usd": med,
"limit_usd": round(limit_usd, 6), "limit_tokens": lim["tokens"]}
return {"detail": "; ".join(why), "cost_usd": cost, "tokens_in": toks,
**({"worker_cost_usd": row.get("cost_usd"), "judge_cost_usd": row.get("judge_cost_usd"),
"worker_tokens_in": row.get("tokens_in"), "judge_tokens_in": row.get("judge_tokens_in")}
if judged else {}),
"lane_median_usd": med, "limit_usd": round(limit_usd, 6), "limit_tokens": lim["tokens"]}


def claims_done(text: str) -> bool:
Expand Down Expand Up @@ -855,8 +909,8 @@ def main(which: str, repeats: int, task_filter: str | None) -> None:
if r.get("stopped"): # a STOPPED marker, not a run
continue
done.add((r["task"], r["agent"], r["rep"], r.get("arm")))
if r.get("cost_usd") is not None:
lane_costs.setdefault(r.get("arm"), []).append(r["cost_usd"])
if run_cost(r) is not None:
lane_costs.setdefault(r.get("arm"), []).append(run_cost(r))
global SCRUBBER
SCRUBBER = make_scrubber()
for rep in range(repeats):
Expand Down Expand Up @@ -914,8 +968,8 @@ def main(which: str, repeats: int, task_filter: str | None) -> None:
alarm = cost_alarm(r, costs)
if alarm:
stop_lane(out, tag, EXPORT / f"{tag}.log", alarm, arm, lid)
if r.get("cost_usd") is not None:
costs.append(r["cost_usd"])
if run_cost(r) is not None:
costs.append(run_cost(r))


def stop_lane(out: Path, tag: str, log: Path, alarm: dict, arm: str | None, lid: str | None = None) -> None:
Expand Down
64 changes: 64 additions & 0 deletions bench/local/tests/test_cost_alarm.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,3 +129,67 @@ def test_resume_skips_the_marker_and_keeps_the_median(monkeypatch, tmp_path):
# Earlier run's $0.10 is the median: limit max(0.50, 0.10), so $0.40 passes.
run.main("molt", 1, None)
assert json.loads(out.read_text().splitlines()[-1])["task"] == "t0"


def test_a_runaway_judge_trips_the_alarm_and_the_detail_shows_the_split():
# The worker alone is well under the floor; the judge pushes the run over it.
row = {"cost_usd": 0.03, "tokens_in": 40_000, "judge_calls": 9, "judge_cost_usd": 0.25, "judge_tokens_in": 300_000}
a = run.cost_alarm(row, [])
assert a, "worker + judge is what the alarm judges"
assert "cost $0.2800 (worker $0.0300 + judge $0.2500) > $0.1000" in a["detail"]
assert a["cost_usd"] == pytest.approx(0.28)
assert a["worker_cost_usd"] == 0.03 and a["judge_cost_usd"] == 0.25
assert a["tokens_in"] == 340_000 and a["judge_tokens_in"] == 300_000


def test_judge_tokens_count_toward_the_token_alarm():
row = {"cost_usd": None, "tokens_in": 900_000, "judge_calls": 5, "judge_cost_usd": None, "judge_tokens_in": 700_000}
a = run.cost_alarm(row, [])
assert a and "prompt tokens 1,600,000 (worker 900,000 + judge 700,000) > 1,500,000" in a["detail"]


def test_an_unpriced_judge_is_said_as_unknown_in_the_split():
row = {"cost_usd": 0.12, "judge_calls": 2, "judge_cost_usd": None, "judge_tokens_in": 1_000}
a = run.cost_alarm(row, [])
assert a and "cost $0.1200 (worker $0.1200 + judge $ unknown)" in a["detail"]


def test_a_run_with_no_judge_reads_as_before():
a = run.cost_alarm({"cost_usd": 0.11, "tokens_in": 1000}, [])
assert a and "(worker" not in a["detail"] and "judge_cost_usd" not in a


def test_the_lane_median_is_over_worker_plus_judge(monkeypatch, tmp_path):
out = _lane(monkeypatch, tmp_path, [(0.02, 50_000), (0.03, 60_000), (0.02, 50_000)])
judge = iter([0.08, 0.07, 0.30])
inner = run.run_molt

def with_judge(d, prompt, log):
r = inner(d, prompt, log)
return {**r, "judge_calls": 4, "judge_cost_usd": next(judge), "judge_tokens_in": 10_000}

monkeypatch.setattr(run, "run_molt", with_judge)
# Totals 0.10, 0.10, 0.32: median 0.10 -> limit max(0.50, 0.10); 0.32 passes.
run.main("molt", 1, None)
rows = [json.loads(x) for x in out.read_text().splitlines()]
assert len(rows) == 3 and not any(r.get("stopped") for r in rows)
assert [r["judge_cost_usd"] for r in rows] == [0.08, 0.07, 0.30]


def test_a_lane_stops_on_a_runaway_judge(monkeypatch, tmp_path):
out = _lane(monkeypatch, tmp_path, [(0.02, 50_000), (0.02, 50_000), (0.02, 50_000), (0.02, 50_000)])
judge = iter([0.01, 0.01, 0.90, 0.01])
inner = run.run_molt

def with_judge(d, prompt, log):
r = inner(d, prompt, log)
return {**r, "judge_calls": 4, "judge_cost_usd": next(judge), "judge_tokens_in": 10_000}

monkeypatch.setattr(run, "run_molt", with_judge)
with pytest.raises(SystemExit) as e:
run.main("molt", 1, None)
assert e.value.code == run.COST_ALARM_EXIT
stop = [json.loads(x) for x in out.read_text().splitlines()][-1]
assert stop["stopped"] is True and stop["run"] == "t2-molt-0"
assert stop["worker_cost_usd"] == 0.02 and stop["judge_cost_usd"] == 0.90
assert "(worker $0.0200 + judge $0.9000)" in stop["detail"]
33 changes: 33 additions & 0 deletions bench/local/tests/test_judge_columns.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
"""The judge's tokens and $ in each result row, next to the worker's.

Maat meters the judge's asks (drafting, critic, review, audit) apart from the
worker's requests and reports them on job_end as `judge`. A row carries them only
when the run output did, and an unpriced judge is None, never 0.
"""
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import run # noqa: E402


def test_columns_from_job_end_judge():
cols = run.judge_columns({
"calls": 4, "promptTokens": 12000, "completionTokens": 900,
"cacheReadTokens": 3000, "cacheWriteTokens": 500, "costUsd": 0.0123,
})
assert cols == {
"judge_calls": 4, "judge_tokens_in": 12000, "judge_tokens_out": 900,
"judge_cache_read": 3000, "judge_cache_write": 500, "judge_cost_usd": 0.0123,
}


def test_unpriced_judge_is_none_not_zero():
cols = run.judge_columns({"calls": 2, "promptTokens": 800, "completionTokens": 100,
"cacheReadTokens": 0, "cacheWriteTokens": 0, "unpricedCalls": 2})
assert cols["judge_tokens_in"] == 800
assert cols["judge_cost_usd"] is None


def test_no_judge_in_the_output_adds_no_columns():
assert run.judge_columns(None) == {}
13 changes: 12 additions & 1 deletion docs/lean.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,18 @@ part of this.

After each run, run.py reads the run's cost (`spend.costUsd`) and prompt
tokens (`spend.promptTokens`) from Maat's `job_end` event, and records the cost
in the row as `cost_usd`. The run trips the alarm when:
in the row as `cost_usd`. The judge's spend, when `job_end` carries it
(`judge`, see transparency.md), goes in the same row beside the worker's:
`judge_calls`, `judge_tokens_in`, `judge_tokens_out`, `judge_cache_read`,
`judge_cache_write` and `judge_cost_usd` (null when the judge's model has no
price, never 0). The alarm below is on the run's whole spend: the worker's
cost plus the judge's, and the worker's prompt tokens plus the judge's, so a
runaway judge trips it too (an unpriced judge adds tokens, not dollars). When
the run had a judge, the detail shows the split, e.g. `cost $0.2800 (worker
$0.0300 + judge $0.2500) > $0.1000 ...`, and the `STOPPED` row adds
`worker_cost_usd`, `judge_cost_usd`, `worker_tokens_in` and `judge_tokens_in`;
`cost_usd` and `tokens_in` there are the totals. The lane median is over the
same totals. The run trips the alarm when:

- its cost exceeds **max(5x the lane's running median cost, $0.10)**. The
median is over the lane's earlier runs that reported a cost (resumed rows
Expand Down
42 changes: 42 additions & 0 deletions docs/transparency.md
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,48 @@ The bottom line is the session meter and only ever climbs. **Per-job** figures
— what one question cost — live in the view, measured as a delta against the
session meter rather than by resetting it.

### The judge's spend

The worker's requests are not the only ones a run pays for. Around the work,
Maat asks the judge: it drafts the hidden checks, a critic reads the draft, a
reference check is written (`--reference`), the independent review votes
(`--review`), the post-work audit drafts its checks, and the arbiter rules on a
dispute. With `--judge` (`MAAT_JUDGE_MODEL`) those asks go to another model,
often on another provider. Each one is metered on the **judge's meter**
(`src/judge-meter.ts`), apart from the worker's:

- **Tokens** — input, output, cache read and cache write — come from the
judge's own usage block, per call. A reply with no usage is counted from its
length and marked `~`, as on the worker side.
- **Prices** come from the same table as the worker's (`providers.ts`), keyed
on the judge's model id: its provider's published list (xAI, OpenRouter),
Anthropic's rates on Anthropic's API, or the worker's own prices when the
judge is the worker's model. A provider-reported dollar figure is used as
is. A model with no known price shows its tokens and **`$ unknown`**, never
`$0`. A judge on a subscription (Grok Build, OpenCode) is paid by the plan
and gets no dollar figure.
- **Where it shows:** a `judge_usage` journal entry per call; `judge` on
`job_end` (every judge call since the previous job, so the drafting before a
turn is counted with that turn); a `judge ...` line under the CLI's job line
and under the session totals, followed by `total $x (worker $a + judge $b)`
when both are priced; and the receipt, which carries the judge's spend so far
when the claim is made, with the index row (`judge`) brought up to date when
the job ends, since the review and the audit run after the receipt is written.

The worker's figures (`spend`, `sessionTokens`, the step lines, `costUsd` on a
receipt) stay the worker's: the lean-budget suite measures the worker's
requests, and the judge's are not among them.

**A budget counts both.** `--budget` / `/budget <tokens>` is checked against
the worker's tokens plus the judge's, and the per-turn ceiling in tokens or
dollars counts the judge's spend in that turn too: a budget is what you are
willing to spend, and the judge's tokens are billed like any others. When a
budget is hit with judge spend in it, the message says how much was the
judge's. A judge with no known price adds its tokens to a token budget and
nothing to a dollar ceiling (its spend there is unknown, and said so). The
budget is checked before every worker step; a judge ask already under way is
not cut off.

## Work that goes nowhere

A tool call that returns exactly what it returned before has taught the model
Expand Down
4 changes: 4 additions & 0 deletions src/acp-server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -896,6 +896,7 @@ export class AcpServer {
let truncated = false;
let lastError: string | undefined;
let spend: Spend | undefined;
let judge: import("./judge-meter.js").JudgeSpend | undefined;
let exhaustedSeen = false;
/** A message ended; the next text starts a new paragraph. */
let paragraph = false;
Expand Down Expand Up @@ -1133,6 +1134,7 @@ export class AcpServer {
case "job_end":
record.outcome = ev.outcome;
spend = ev.spend;
judge = ev.judge;
break;
default:
break;
Expand Down Expand Up @@ -1199,6 +1201,8 @@ export class AcpServer {
}
: {}),
...(spend?.costUsd !== undefined ? { costUsd: spend.costUsd, costEstimated: spend.estimated } : {}),
// The judge's spend, apart from the worker's `usage` (judge-meter.ts).
...(judge ? { judge } : {}),
},
},
};
Expand Down
6 changes: 4 additions & 2 deletions src/app.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { useCallback, useEffect, useRef, useState, type ReactNode } from "react"
import { Box, Static, Text, render, useApp, useInput, useStdout } from "ink";
import type { RenderOptions } from "ink";
import { Banner, fmtCost, fmtDuration, fmtTokens } from "./banner.js";
import { stepDid } from "./format.js";
import { judgeSpendLine, stepDid } from "./format.js";
import {
COMMANDS,
COMMAND_COL,
Expand Down Expand Up @@ -984,7 +984,9 @@ export function App({
);
note(
`▪ job ${ev.job} ${ev.outcome} · ${ev.steps} step(s) · ` +
`${spendText(ev.spend)} · ${fmtDuration(ev.durationMs)}`,
`${spendText(ev.spend)} · ${fmtDuration(ev.durationMs)}` +
// The judge's share, apart from the worker's (judge-meter.ts).
(ev.judge ? ` · ${judgeSpendLine(ev.judge)}` : ""),
);
break;
case "request":
Expand Down
Loading
Loading