Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@

### Added

- **Interview and mission-planning asks are metered as worker spend.** The
interview's question is added to the session meter (journal: `worker_ask`);
`maat mission plan` prints the plan's tokens and cost at the worker's prices.

- **The judge's spend is metered.** Every ask made around the work (drafting
and its critic, the reference check, `--review`, the post-work audit, the
arbiter) records its input, output, cache read and cache write tokens, priced
Expand All @@ -16,6 +20,11 @@
columns to each row, and its cost alarm now judges worker + judge cost and
prompt tokens (the detail shows the split), so a runaway judge stops the lane
too. See `docs/transparency.md` and `docs/lean.md`.
The per-turn dollar ceiling counts a priced judge even when the worker is
unpriced, with the token ceiling still bounding the whole turn; the lane
median leaves out runs whose judge was unpriced (a plan-paid judge,
`judge_plan`, still counts); a budget used up by this turn's drafting before
the first step says so.

- **`--review-executable`** (experimental, also `MAAT_REVIEW_EXECUTABLE=1`):
every `--review` objection must carry a read-only command that demonstrates
Expand Down
34 changes: 29 additions & 5 deletions bench/local/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -682,6 +682,9 @@ def judge_columns(judge: dict | None) -> dict:
"judge_cache_read": judge.get("cacheReadTokens"),
"judge_cache_write": judge.get("cacheWriteTokens"),
"judge_cost_usd": judge.get("costUsd"),
# The plan that paid for the judge's calls (e.g. "OpenCode"): no money, so a
# missing judge_cost_usd with a plan is exact, not unknown.
"judge_plan": judge.get("plan"),
}


Expand Down Expand Up @@ -713,6 +716,20 @@ def run_cost(row: dict) -> float | None:
return _sum_known(row.get("cost_usd"), row.get("judge_cost_usd"))


def median_cost(row: dict) -> float | None:
"""A run's cost as the lane median counts it, or None to leave it out.

A run whose judge ran but had no price (and no plan paying for it) is left out: its total is the worker's
cost alone, a lower bound, and averaged with runs whose judge was priced it would
pull the median (and so the limit) down without anyone seeing why. The alarm on
that run itself still uses the lower bound (run_cost): under-counting there only
makes the alarm late, never wrong."""
if row.get("judge_calls") is not None and row.get("judge_cost_usd") is None and not row.get("judge_plan"):
return None
# A plan-paid judge cost no money: the total is the worker's, and exact.
return run_cost(row)


def run_tokens(row: dict) -> int | None:
"""A run's prompt tokens, the worker's plus the judge's."""
return _sum_known(row.get("tokens_in"), row.get("judge_tokens_in"))
Expand All @@ -734,9 +751,15 @@ def cost_alarm(row: dict, prior_costs: list[float]) -> dict | None:
def usd(x):
return "$ unknown" if x is None else f"${x:.4f}"

def judge_usd():
plan, cost = row.get("judge_plan"), row.get("judge_cost_usd")
if plan and cost is None:
return f"({plan} plan)"
return usd(cost) + (f" + {plan} plan" if plan else "")

why = []
if cost is not None and cost > limit_usd:
split = f" (worker {usd(row.get('cost_usd'))} + judge {usd(row.get('judge_cost_usd'))})" if judged else ""
split = f" (worker {usd(row.get('cost_usd'))} + judge {judge_usd()})" if judged else ""
why.append(f"cost ${cost:.4f}{split} > ${limit_usd:.4f} (max({lim['x']:g}x lane median "
f"{'n/a' if med is None else f'${med:.4f}'}, ${lim['usd']:g}))")
if toks is not None and toks > lim["tokens"]:
Expand All @@ -747,6 +770,7 @@ def usd(x):
return None
return {"detail": "; ".join(why), "cost_usd": cost, "tokens_in": toks,
**({"worker_cost_usd": row.get("cost_usd"), "judge_cost_usd": row.get("judge_cost_usd"),
**({"judge_plan": row["judge_plan"]} if row.get("judge_plan") else {}),
"worker_tokens_in": row.get("tokens_in"), "judge_tokens_in": row.get("judge_tokens_in")}
if judged else {}),
"lane_median_usd": med, "limit_usd": round(limit_usd, 6), "limit_tokens": lim["tokens"]}
Expand Down Expand Up @@ -909,8 +933,8 @@ def main(which: str, repeats: int, task_filter: str | None) -> None:
if r.get("stopped"): # a STOPPED marker, not a run
continue
done.add((r["task"], r["agent"], r["rep"], r.get("arm")))
if run_cost(r) is not None:
lane_costs.setdefault(r.get("arm"), []).append(run_cost(r))
if median_cost(r) is not None:
lane_costs.setdefault(r.get("arm"), []).append(median_cost(r))
global SCRUBBER
SCRUBBER = make_scrubber()
for rep in range(repeats):
Expand Down Expand Up @@ -968,8 +992,8 @@ def main(which: str, repeats: int, task_filter: str | None) -> None:
alarm = cost_alarm(r, costs)
if alarm:
stop_lane(out, tag, EXPORT / f"{tag}.log", alarm, arm, lid)
if run_cost(r) is not None:
costs.append(run_cost(r))
if median_cost(r) is not None:
costs.append(median_cost(r))


def stop_lane(out: Path, tag: str, log: Path, alarm: dict, arm: str | None, lid: str | None = None) -> None:
Expand Down
50 changes: 50 additions & 0 deletions bench/local/tests/test_cost_alarm.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,3 +193,53 @@ def with_judge(d, prompt, log):
assert stop["stopped"] is True and stop["run"] == "t2-molt-0"
assert stop["worker_cost_usd"] == 0.02 and stop["judge_cost_usd"] == 0.90
assert "(worker $0.0200 + judge $0.9000)" in stop["detail"]


def test_a_run_with_an_unpriced_judge_is_left_out_of_the_median():
assert run.median_cost({"cost_usd": 0.02, "judge_calls": 3, "judge_cost_usd": None}) is None
assert run.median_cost({"cost_usd": 0.02, "judge_calls": 3, "judge_cost_usd": 0.01}) == pytest.approx(0.03)
assert run.median_cost({"cost_usd": 0.02}) == 0.02 # no judge: the worker's cost, as before
# Its own alarm still uses the lower bound.
assert run.cost_alarm({"cost_usd": 0.12, "judge_calls": 3, "judge_cost_usd": None}, [])


def test_the_lane_median_skips_unpriced_judge_runs(monkeypatch, tmp_path):
# Priced-judge runs total $0.10; two runs with an unpriced judge cost $0.01 (worker
# only). Counted, they would drag the median to $0.01 and the limit to the $0.10
# floor, stopping the fifth run ($0.30). Left out, the median is $0.10, limit $0.50.
out = _lane(monkeypatch, tmp_path, [(0.02, 1), (0.01, 1), (0.01, 1), (0.02, 1), (0.02, 1)])
judge = iter([0.08, None, None, 0.08, 0.28])
inner = run.run_molt

def with_judge(d, prompt, log):
r = inner(d, prompt, log)
return {**r, "judge_calls": 2, "judge_cost_usd": next(judge), "judge_tokens_in": 1}

monkeypatch.setattr(run, "run_molt", with_judge)
run.main("molt", 1, None)
rows = [json.loads(x) for x in out.read_text().splitlines()]
assert len(rows) == 5 and not any(r.get("stopped") for r in rows)


def test_a_plan_paid_judge_counts_in_the_median_at_the_workers_cost():
row = {"cost_usd": 0.15, "judge_calls": 3, "judge_cost_usd": None, "judge_plan": "OpenCode"}
assert run.median_cost(row) == 0.15, "a plan-paid judge cost no money: exact, not unknown"
a = run.cost_alarm(row, [])
assert a and "(worker $0.1500 + judge (OpenCode plan))" in a["detail"] and a["judge_plan"] == "OpenCode"


def test_a_lane_with_a_plan_paid_judge_forms_its_median(monkeypatch, tmp_path):
# The documented OpenCode-judge arm: every judge is plan-paid. The median must
# form from the worker's costs (0.10), so a $0.15 run (limit max(0.50, 0.10)) passes;
# with the plan rows left out, the limit stayed at the $0.10 floor and stopped it.
out = _lane(monkeypatch, tmp_path, [(0.10, 1), (0.10, 1), (0.10, 1), (0.15, 1)])
inner = run.run_molt

def plan_judge(d, prompt, log):
return {**inner(d, prompt, log), "judge_calls": 3, "judge_cost_usd": None, "judge_plan": "OpenCode", "judge_tokens_in": 1}

monkeypatch.setattr(run, "run_molt", plan_judge)
run.main("molt", 1, None)
rows = [json.loads(x) for x in out.read_text().splitlines()]
assert len(rows) == 4 and not any(r.get("stopped") for r in rows)
assert all(r["judge_plan"] == "OpenCode" for r in rows)
7 changes: 7 additions & 0 deletions bench/local/tests/test_judge_columns.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,16 @@ def test_columns_from_job_end_judge():
assert cols == {
"judge_calls": 4, "judge_tokens_in": 12000, "judge_tokens_out": 900,
"judge_cache_read": 3000, "judge_cache_write": 500, "judge_cost_usd": 0.0123,
"judge_plan": None,
}


def test_a_plan_paid_judge_carries_its_plan():
cols = run.judge_columns({"calls": 3, "promptTokens": 10, "completionTokens": 2,
"cacheReadTokens": 0, "cacheWriteTokens": 0, "plan": "OpenCode"})
assert cols["judge_plan"] == "OpenCode" and cols["judge_cost_usd"] is None


def test_unpriced_judge_is_none_not_zero():
cols = run.judge_columns({"calls": 2, "promptTokens": 800, "completionTokens": 100,
"cacheReadTokens": 0, "cacheWriteTokens": 0, "unpricedCalls": 2})
Expand Down
6 changes: 5 additions & 1 deletion docs/lean.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,11 @@ the run had a judge, the detail shows the split, e.g. `cost $0.2800 (worker
$0.0300 + judge $0.2500) > $0.1000 ...`, and the `STOPPED` row adds
`worker_cost_usd`, `judge_cost_usd`, `worker_tokens_in` and `judge_tokens_in`;
`cost_usd` and `tokens_in` there are the totals. The lane median is over the
same totals. The run trips the alarm when:
same totals, leaving out runs whose judge ran unpriced (a judge paid by a plan,
`judge_plan`, cost no money and counts at the worker's cost; the split reads
`judge (OpenCode plan)`): their total is only a
lower bound, and mixed with priced runs it would pull the limit down unseen.
Such a run's own alarm still uses that lower bound. The run trips the alarm when:

- its cost exceeds **max(5x the lane's running median cost, $0.10)**. The
median is over the lane's earlier runs that reported a cost (resumed rows
Expand Down
24 changes: 20 additions & 4 deletions docs/transparency.md
Original file line number Diff line number Diff line change
Expand Up @@ -189,10 +189,26 @@ the worker's tokens plus the judge's, and the per-turn ceiling in tokens or
dollars counts the judge's spend in that turn too: a budget is what you are
willing to spend, and the judge's tokens are billed like any others. When a
budget is hit with judge spend in it, the message says how much was the
judge's. A judge with no known price adds its tokens to a token budget and
nothing to a dollar ceiling (its spend there is unknown, and said so). The
budget is checked before every worker step; a judge ask already under way is
not cut off.
judge's; when the judge's drafting used it before the worker's first step,
the message says that too (only when the spend before the drafting was still
under the budget). The turn is priced in dollars when either side is priced: a
local or subscription worker with a paid judge is held to a dollar ceiling by
the judge's dollars and to the token ceiling by the whole turn's tokens, and
stops at whichever it reaches first. A side with no known price adds its
tokens to a token budget and nothing to a dollar ceiling (its spend there is
unknown, and said so). The budget is checked before every worker step; a judge
ask already under way is not cut off.

The worker's own asks outside a turn are worker spend, not the judge's: the
interview's question is added to the session meter like a step (and journalled
as `worker_ask`). It is asked before the turn starts, so it is in the session
total but in no job's `job_end` spend or receipt: a session total higher than
the sum of its jobs is the interview, not a gap. And `maat mission plan` prints a `worker (planning) ...` line
with the plan's tokens and cost at the worker's prices.

Judge calls are priced when they are read, not when they were made: a price
that arrives later applies to every call, and a price that is cleared stops
applying (the journal keeps each call's figure as it stood at the time).

## Work that goes nowhere

Expand Down
2 changes: 2 additions & 0 deletions electron/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1550,6 +1550,8 @@ ipcMain.handle(
// The workspace, not the app's own directory: on the ACP
// backend this is where the CLI that answers is run.
cwd: session.cwd,
// The worker's own model: worker spend on the session meter.
meter: session.engine.workerAskMeter,
});
},
);
Expand Down
2 changes: 2 additions & 0 deletions src/app.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -702,6 +702,8 @@ export function App({
model: engine.model,
cwd: engine.cwd,
fetchFn: engine.cfg.fetchFn,
// The worker's own model: worker spend on the session meter.
meter: engine.workerAskMeter,
});
if (seq !== interviewSeq.current) return;
if (r.kind === "error") {
Expand Down
58 changes: 30 additions & 28 deletions src/ask.ts
Original file line number Diff line number Diff line change
Expand Up @@ -350,46 +350,48 @@ function count(v: unknown): number | undefined {
return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined;
}

/** Report one answered HTTP ask to its meter (see recordUsage). */
function meterUsage(opts: AskOptions, u: Usage | undefined, reply: string): void {
recordUsage(opts.meter, { baseUrl: opts.baseUrl, model: opts.model, what: opts.what, sent: opts.system + opts.prompt }, u, reply);
}

/** An answered ask whose transport reports no usage (ACP, a CLI): estimated. */
function meterEstimate(opts: AskOptions, reply: string): void {
meterUsage(opts, undefined, reply);
}

/**
* Report one answered HTTP ask to the meter: the provider's usage block when
* it sent one, Maat's estimate (marked) when it did not.
* Report one answered ask to a meter: the provider's usage block when it sent
* one (input, output, cache read and write, its own dollar figure), Maat's
* estimate from `sent` and the reply, marked, when it did not. Exported for the
* asks that do not go through askModel (the interview).
*/
function meterUsage(opts: AskOptions, u: Usage | undefined, reply: string): void {
if (!opts.meter) return;
export function recordUsage(
meter: AskMeter | undefined,
ctx: { baseUrl: string; model: string; what?: string; sent: string },
u: Usage | undefined,
reply: string,
): void {
if (!meter) return;
const prompt = count(u?.prompt_tokens);
const completion = count(u?.completion_tokens);
if (prompt === undefined && completion === undefined) {
meterEstimate(opts, reply);
return;
}
const read = count(u?.prompt_tokens_details?.cached_tokens) ?? count(u?.cache_read_input_tokens);
const write = count(u?.cache_creation_input_tokens);
const billed = count(u?.cost);
opts.meter.record({
baseUrl: opts.baseUrl,
model: opts.model,
...(opts.what ? { what: opts.what } : {}),
promptTokens: prompt ?? estTokens(opts.system + opts.prompt),
const reported = prompt !== undefined || completion !== undefined;
meter.record({
baseUrl: ctx.baseUrl,
model: ctx.model,
...(ctx.what ? { what: ctx.what } : {}),
promptTokens: prompt ?? estTokens(ctx.sent),
completionTokens: completion ?? estTokens(reply),
...(read !== undefined ? { cacheReadTokens: read } : {}),
...(write !== undefined ? { cacheWriteTokens: write } : {}),
...(billed !== undefined ? { billedUsd: billed } : {}),
...(reported && read !== undefined ? { cacheReadTokens: read } : {}),
...(reported && write !== undefined ? { cacheWriteTokens: write } : {}),
...(reported && billed !== undefined ? { billedUsd: billed } : {}),
estimated: prompt === undefined || completion === undefined,
});
}

/** An answered ask whose transport reports no usage (ACP, a CLI, a bare reply): estimated. */
function meterEstimate(opts: AskOptions, reply: string): void {
opts.meter?.record({
baseUrl: opts.baseUrl,
model: opts.model,
...(opts.what ? { what: opts.what } : {}),
promptTokens: estTokens(opts.system + opts.prompt),
completionTokens: estTokens(reply),
estimated: true,
});
}

/**
* The JSON object in a reply, or null.
*
Expand Down
Loading
Loading