diff --git a/CHANGELOG.md b/CHANGELOG.md index b71cbf0..cdf666a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ### Added +- **Interview and mission-planning asks are metered as worker spend.** The + interview's question is added to the session meter (journal: `worker_ask`); + `maat mission plan` prints the plan's tokens and cost at the worker's prices. + - **The judge's spend is metered.** Every ask made around the work (drafting and its critic, the reference check, `--review`, the post-work audit, the arbiter) records its input, output, cache read and cache write tokens, priced @@ -16,6 +20,11 @@ columns to each row, and its cost alarm now judges worker + judge cost and prompt tokens (the detail shows the split), so a runaway judge stops the lane too. See `docs/transparency.md` and `docs/lean.md`. + The per-turn dollar ceiling counts a priced judge even when the worker is + unpriced, with the token ceiling still bounding the whole turn; the lane + median leaves out runs whose judge was unpriced (a plan-paid judge, + `judge_plan`, still counts); a budget used up by this turn's drafting before + the first step says so. - **`--review-executable`** (experimental, also `MAAT_REVIEW_EXECUTABLE=1`): every `--review` objection must carry a read-only command that demonstrates diff --git a/bench/local/run.py b/bench/local/run.py index 0d6e954..07d17cc 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -682,6 +682,9 @@ def judge_columns(judge: dict | None) -> dict: "judge_cache_read": judge.get("cacheReadTokens"), "judge_cache_write": judge.get("cacheWriteTokens"), "judge_cost_usd": judge.get("costUsd"), + # The plan that paid for the judge's calls (e.g. "OpenCode"): no money, so a + # missing judge_cost_usd with a plan is exact, not unknown. + "judge_plan": judge.get("plan"), } @@ -713,6 +716,20 @@ def run_cost(row: dict) -> float | None: return _sum_known(row.get("cost_usd"), row.get("judge_cost_usd")) +def median_cost(row: dict) -> float | None: + """A run's cost as the lane median counts it, or None to leave it out. + + A run whose judge ran but had no price (and no plan paying for it) is left out: its total is the worker's + cost alone, a lower bound, and averaged with runs whose judge was priced it would + pull the median (and so the limit) down without anyone seeing why. The alarm on + that run itself still uses the lower bound (run_cost): under-counting there only + makes the alarm late, never wrong.""" + if row.get("judge_calls") is not None and row.get("judge_cost_usd") is None and not row.get("judge_plan"): + return None + # A plan-paid judge cost no money: the total is the worker's, and exact. + return run_cost(row) + + def run_tokens(row: dict) -> int | None: """A run's prompt tokens, the worker's plus the judge's.""" return _sum_known(row.get("tokens_in"), row.get("judge_tokens_in")) @@ -734,9 +751,15 @@ def cost_alarm(row: dict, prior_costs: list[float]) -> dict | None: def usd(x): return "$ unknown" if x is None else f"${x:.4f}" + def judge_usd(): + plan, cost = row.get("judge_plan"), row.get("judge_cost_usd") + if plan and cost is None: + return f"({plan} plan)" + return usd(cost) + (f" + {plan} plan" if plan else "") + why = [] if cost is not None and cost > limit_usd: - split = f" (worker {usd(row.get('cost_usd'))} + judge {usd(row.get('judge_cost_usd'))})" if judged else "" + split = f" (worker {usd(row.get('cost_usd'))} + judge {judge_usd()})" if judged else "" why.append(f"cost ${cost:.4f}{split} > ${limit_usd:.4f} (max({lim['x']:g}x lane median " f"{'n/a' if med is None else f'${med:.4f}'}, ${lim['usd']:g}))") if toks is not None and toks > lim["tokens"]: @@ -747,6 +770,7 @@ def usd(x): return None return {"detail": "; ".join(why), "cost_usd": cost, "tokens_in": toks, **({"worker_cost_usd": row.get("cost_usd"), "judge_cost_usd": row.get("judge_cost_usd"), + **({"judge_plan": row["judge_plan"]} if row.get("judge_plan") else {}), "worker_tokens_in": row.get("tokens_in"), "judge_tokens_in": row.get("judge_tokens_in")} if judged else {}), "lane_median_usd": med, "limit_usd": round(limit_usd, 6), "limit_tokens": lim["tokens"]} @@ -909,8 +933,8 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: if r.get("stopped"): # a STOPPED marker, not a run continue done.add((r["task"], r["agent"], r["rep"], r.get("arm"))) - if run_cost(r) is not None: - lane_costs.setdefault(r.get("arm"), []).append(run_cost(r)) + if median_cost(r) is not None: + lane_costs.setdefault(r.get("arm"), []).append(median_cost(r)) global SCRUBBER SCRUBBER = make_scrubber() for rep in range(repeats): @@ -968,8 +992,8 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: alarm = cost_alarm(r, costs) if alarm: stop_lane(out, tag, EXPORT / f"{tag}.log", alarm, arm, lid) - if run_cost(r) is not None: - costs.append(run_cost(r)) + if median_cost(r) is not None: + costs.append(median_cost(r)) def stop_lane(out: Path, tag: str, log: Path, alarm: dict, arm: str | None, lid: str | None = None) -> None: diff --git a/bench/local/tests/test_cost_alarm.py b/bench/local/tests/test_cost_alarm.py index e315a19..346c749 100644 --- a/bench/local/tests/test_cost_alarm.py +++ b/bench/local/tests/test_cost_alarm.py @@ -193,3 +193,53 @@ def with_judge(d, prompt, log): assert stop["stopped"] is True and stop["run"] == "t2-molt-0" assert stop["worker_cost_usd"] == 0.02 and stop["judge_cost_usd"] == 0.90 assert "(worker $0.0200 + judge $0.9000)" in stop["detail"] + + +def test_a_run_with_an_unpriced_judge_is_left_out_of_the_median(): + assert run.median_cost({"cost_usd": 0.02, "judge_calls": 3, "judge_cost_usd": None}) is None + assert run.median_cost({"cost_usd": 0.02, "judge_calls": 3, "judge_cost_usd": 0.01}) == pytest.approx(0.03) + assert run.median_cost({"cost_usd": 0.02}) == 0.02 # no judge: the worker's cost, as before + # Its own alarm still uses the lower bound. + assert run.cost_alarm({"cost_usd": 0.12, "judge_calls": 3, "judge_cost_usd": None}, []) + + +def test_the_lane_median_skips_unpriced_judge_runs(monkeypatch, tmp_path): + # Priced-judge runs total $0.10; two runs with an unpriced judge cost $0.01 (worker + # only). Counted, they would drag the median to $0.01 and the limit to the $0.10 + # floor, stopping the fifth run ($0.30). Left out, the median is $0.10, limit $0.50. + out = _lane(monkeypatch, tmp_path, [(0.02, 1), (0.01, 1), (0.01, 1), (0.02, 1), (0.02, 1)]) + judge = iter([0.08, None, None, 0.08, 0.28]) + inner = run.run_molt + + def with_judge(d, prompt, log): + r = inner(d, prompt, log) + return {**r, "judge_calls": 2, "judge_cost_usd": next(judge), "judge_tokens_in": 1} + + monkeypatch.setattr(run, "run_molt", with_judge) + run.main("molt", 1, None) + rows = [json.loads(x) for x in out.read_text().splitlines()] + assert len(rows) == 5 and not any(r.get("stopped") for r in rows) + + +def test_a_plan_paid_judge_counts_in_the_median_at_the_workers_cost(): + row = {"cost_usd": 0.15, "judge_calls": 3, "judge_cost_usd": None, "judge_plan": "OpenCode"} + assert run.median_cost(row) == 0.15, "a plan-paid judge cost no money: exact, not unknown" + a = run.cost_alarm(row, []) + assert a and "(worker $0.1500 + judge (OpenCode plan))" in a["detail"] and a["judge_plan"] == "OpenCode" + + +def test_a_lane_with_a_plan_paid_judge_forms_its_median(monkeypatch, tmp_path): + # The documented OpenCode-judge arm: every judge is plan-paid. The median must + # form from the worker's costs (0.10), so a $0.15 run (limit max(0.50, 0.10)) passes; + # with the plan rows left out, the limit stayed at the $0.10 floor and stopped it. + out = _lane(monkeypatch, tmp_path, [(0.10, 1), (0.10, 1), (0.10, 1), (0.15, 1)]) + inner = run.run_molt + + def plan_judge(d, prompt, log): + return {**inner(d, prompt, log), "judge_calls": 3, "judge_cost_usd": None, "judge_plan": "OpenCode", "judge_tokens_in": 1} + + monkeypatch.setattr(run, "run_molt", plan_judge) + run.main("molt", 1, None) + rows = [json.loads(x) for x in out.read_text().splitlines()] + assert len(rows) == 4 and not any(r.get("stopped") for r in rows) + assert all(r["judge_plan"] == "OpenCode" for r in rows) diff --git a/bench/local/tests/test_judge_columns.py b/bench/local/tests/test_judge_columns.py index 09d90fa..7c2c910 100644 --- a/bench/local/tests/test_judge_columns.py +++ b/bench/local/tests/test_judge_columns.py @@ -19,9 +19,16 @@ def test_columns_from_job_end_judge(): assert cols == { "judge_calls": 4, "judge_tokens_in": 12000, "judge_tokens_out": 900, "judge_cache_read": 3000, "judge_cache_write": 500, "judge_cost_usd": 0.0123, + "judge_plan": None, } +def test_a_plan_paid_judge_carries_its_plan(): + cols = run.judge_columns({"calls": 3, "promptTokens": 10, "completionTokens": 2, + "cacheReadTokens": 0, "cacheWriteTokens": 0, "plan": "OpenCode"}) + assert cols["judge_plan"] == "OpenCode" and cols["judge_cost_usd"] is None + + def test_unpriced_judge_is_none_not_zero(): cols = run.judge_columns({"calls": 2, "promptTokens": 800, "completionTokens": 100, "cacheReadTokens": 0, "cacheWriteTokens": 0, "unpricedCalls": 2}) diff --git a/docs/lean.md b/docs/lean.md index 586cdf1..8735f29 100644 --- a/docs/lean.md +++ b/docs/lean.md @@ -94,7 +94,11 @@ the run had a judge, the detail shows the split, e.g. `cost $0.2800 (worker $0.0300 + judge $0.2500) > $0.1000 ...`, and the `STOPPED` row adds `worker_cost_usd`, `judge_cost_usd`, `worker_tokens_in` and `judge_tokens_in`; `cost_usd` and `tokens_in` there are the totals. The lane median is over the -same totals. The run trips the alarm when: +same totals, leaving out runs whose judge ran unpriced (a judge paid by a plan, +`judge_plan`, cost no money and counts at the worker's cost; the split reads +`judge (OpenCode plan)`): their total is only a +lower bound, and mixed with priced runs it would pull the limit down unseen. +Such a run's own alarm still uses that lower bound. The run trips the alarm when: - its cost exceeds **max(5x the lane's running median cost, $0.10)**. The median is over the lane's earlier runs that reported a cost (resumed rows diff --git a/docs/transparency.md b/docs/transparency.md index 8d06a6d..55d7a39 100644 --- a/docs/transparency.md +++ b/docs/transparency.md @@ -189,10 +189,26 @@ the worker's tokens plus the judge's, and the per-turn ceiling in tokens or dollars counts the judge's spend in that turn too: a budget is what you are willing to spend, and the judge's tokens are billed like any others. When a budget is hit with judge spend in it, the message says how much was the -judge's. A judge with no known price adds its tokens to a token budget and -nothing to a dollar ceiling (its spend there is unknown, and said so). The -budget is checked before every worker step; a judge ask already under way is -not cut off. +judge's; when the judge's drafting used it before the worker's first step, +the message says that too (only when the spend before the drafting was still +under the budget). The turn is priced in dollars when either side is priced: a +local or subscription worker with a paid judge is held to a dollar ceiling by +the judge's dollars and to the token ceiling by the whole turn's tokens, and +stops at whichever it reaches first. A side with no known price adds its +tokens to a token budget and nothing to a dollar ceiling (its spend there is +unknown, and said so). The budget is checked before every worker step; a judge +ask already under way is not cut off. + +The worker's own asks outside a turn are worker spend, not the judge's: the +interview's question is added to the session meter like a step (and journalled +as `worker_ask`). It is asked before the turn starts, so it is in the session +total but in no job's `job_end` spend or receipt: a session total higher than +the sum of its jobs is the interview, not a gap. And `maat mission plan` prints a `worker (planning) ...` line +with the plan's tokens and cost at the worker's prices. + +Judge calls are priced when they are read, not when they were made: a price +that arrives later applies to every call, and a price that is cleared stops +applying (the journal keeps each call's figure as it stood at the time). ## Work that goes nowhere diff --git a/electron/main.ts b/electron/main.ts index 070f421..9d716b7 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -1550,6 +1550,8 @@ ipcMain.handle( // The workspace, not the app's own directory: on the ACP // backend this is where the CLI that answers is run. cwd: session.cwd, + // The worker's own model: worker spend on the session meter. + meter: session.engine.workerAskMeter, }); }, ); diff --git a/src/app.tsx b/src/app.tsx index 616e331..aad2a7b 100644 --- a/src/app.tsx +++ b/src/app.tsx @@ -702,6 +702,8 @@ export function App({ model: engine.model, cwd: engine.cwd, fetchFn: engine.cfg.fetchFn, + // The worker's own model: worker spend on the session meter. + meter: engine.workerAskMeter, }); if (seq !== interviewSeq.current) return; if (r.kind === "error") { diff --git a/src/ask.ts b/src/ask.ts index 1969210..db8e83d 100644 --- a/src/ask.ts +++ b/src/ask.ts @@ -350,46 +350,48 @@ function count(v: unknown): number | undefined { return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined; } +/** Report one answered HTTP ask to its meter (see recordUsage). */ +function meterUsage(opts: AskOptions, u: Usage | undefined, reply: string): void { + recordUsage(opts.meter, { baseUrl: opts.baseUrl, model: opts.model, what: opts.what, sent: opts.system + opts.prompt }, u, reply); +} + +/** An answered ask whose transport reports no usage (ACP, a CLI): estimated. */ +function meterEstimate(opts: AskOptions, reply: string): void { + meterUsage(opts, undefined, reply); +} + /** - * Report one answered HTTP ask to the meter: the provider's usage block when - * it sent one, Maat's estimate (marked) when it did not. + * Report one answered ask to a meter: the provider's usage block when it sent + * one (input, output, cache read and write, its own dollar figure), Maat's + * estimate from `sent` and the reply, marked, when it did not. Exported for the + * asks that do not go through askModel (the interview). */ -function meterUsage(opts: AskOptions, u: Usage | undefined, reply: string): void { - if (!opts.meter) return; +export function recordUsage( + meter: AskMeter | undefined, + ctx: { baseUrl: string; model: string; what?: string; sent: string }, + u: Usage | undefined, + reply: string, +): void { + if (!meter) return; const prompt = count(u?.prompt_tokens); const completion = count(u?.completion_tokens); - if (prompt === undefined && completion === undefined) { - meterEstimate(opts, reply); - return; - } const read = count(u?.prompt_tokens_details?.cached_tokens) ?? count(u?.cache_read_input_tokens); const write = count(u?.cache_creation_input_tokens); const billed = count(u?.cost); - opts.meter.record({ - baseUrl: opts.baseUrl, - model: opts.model, - ...(opts.what ? { what: opts.what } : {}), - promptTokens: prompt ?? estTokens(opts.system + opts.prompt), + const reported = prompt !== undefined || completion !== undefined; + meter.record({ + baseUrl: ctx.baseUrl, + model: ctx.model, + ...(ctx.what ? { what: ctx.what } : {}), + promptTokens: prompt ?? estTokens(ctx.sent), completionTokens: completion ?? estTokens(reply), - ...(read !== undefined ? { cacheReadTokens: read } : {}), - ...(write !== undefined ? { cacheWriteTokens: write } : {}), - ...(billed !== undefined ? { billedUsd: billed } : {}), + ...(reported && read !== undefined ? { cacheReadTokens: read } : {}), + ...(reported && write !== undefined ? { cacheWriteTokens: write } : {}), + ...(reported && billed !== undefined ? { billedUsd: billed } : {}), estimated: prompt === undefined || completion === undefined, }); } -/** An answered ask whose transport reports no usage (ACP, a CLI, a bare reply): estimated. */ -function meterEstimate(opts: AskOptions, reply: string): void { - opts.meter?.record({ - baseUrl: opts.baseUrl, - model: opts.model, - ...(opts.what ? { what: opts.what } : {}), - promptTokens: estTokens(opts.system + opts.prompt), - completionTokens: estTokens(reply), - estimated: true, - }); -} - /** * The JSON object in a reply, or null. * diff --git a/src/cli.tsx b/src/cli.tsx index fe68c74..cea5fcf 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -16,8 +16,7 @@ import { resolve } from "node:path"; import { Archive } from "./archive.js"; import { isAutonomy, type Autonomy } from "./autonomy.js"; import { fmtCost, fmtDuration } from "./banner.js"; -import { judgeSpendLine } from "./format.js"; -import { stepDid } from "./format.js"; +import { judgeSpendLine, askSpendLine, stepDid } from "./format.js"; import { BarError, hasBar, loadBar, selectChecks, writeDefaultBar } from "./bar.js"; import { Engine, type FileAccess } from "./engine.js"; import { describeDrift, driftSince } from "./git.js"; @@ -31,6 +30,7 @@ import { draftMission, missionStatus, runMission, writePlan, type MissionSummary import { parseDuration } from "./session-commands.js"; import { jobEndWords } from "./tiers.js"; import { judgeEffort, judgeTarget } from "./judge.js"; +import { SpendMeter } from "./judge-meter.js"; import { commandsHere, draftCriteriaCritiqued, drafterInputsHash, drafterSnapshot, preflightCriteria, taskChecksFrom, type Draft, type DrafterInputs } from "./criteria.js"; import { listProject, removeNew } from "./leftovers.js"; import { projectScripts } from "./interview.js"; @@ -927,18 +927,41 @@ function engineFor(args: Args, session = false, extra: { files?: FileAccess } = * nothing, no cost is shown at all. */ async function priceEngine(engine: Engine, args: Args): Promise { - await priceJudge(engine, args); - if (!needsPriceLookup(args.model, engine.pricing(), storedEndpoint())) return; + // Side by side: each lookup is bounded, and neither needs the other's answer. + await Promise.all([priceJudge(engine, args), priceWorker(engine, args)]); +} + +/** The worker's own price lookup (see priceEngine). */ +async function priceWorker(engine: Engine, args: Args): Promise { + engine.setPricing(await workerPricing(args, engine.pricing())); +} + +/** + * The worker's price, resolved one way for every command that spends on the + * worker's model (`run`, `mission plan`): a hand-set or stored price for this + * model stands; otherwise the provider's list is asked and the answer saved. + * `current` is what is in force now (args' prices, seeded from the stored one + * by parseArgs). Returns the prices to use, `{}` for none. + */ +async function workerPricing( + args: Args, + current: { in?: number; out?: number; cached?: number; source?: string } = { + in: args.priceIn, + out: args.priceOut, + cached: args.priceCachedIn, + source: args.priceSource, + }, +): Promise<{ in?: number; out?: number; cached?: number; source?: string }> { + if (!needsPriceLookup(args.model, current, storedEndpoint())) return current; const p = await fetchPricing(args.url, args.model, keyForUrl(args.url, args.key)); if (!p) { // Nothing published. A price only stands if it was recorded for THIS // model; inheriting the last one is how a Claude session gets billed at // grok's rates. - if (storedEndpoint().priceModel !== args.model) engine.setPricing({}); - return; + return storedEndpoint().priceModel !== args.model ? {} : current; } - engine.setPricing({ in: p.in, out: p.out, cached: p.cached, source: p.source }); savePricing(args.model, p); + return { in: p.in, out: p.out, cached: p.cached, source: p.source }; } /** @@ -1198,13 +1221,28 @@ async function cmdMission(args: Args): Promise { process.stderr.write("maat: no model selected — pass --model or set MOLT_MODEL\n"); return 2; } - const brief = await buildBrief({ cwd: args.cwd }).catch(() => ({ text: "" })); + // The plan is asked of the worker's own model, so it is worker spend: + // metered with the worker's prices (hand-set, or the provider's list, + // looked up beside the brief) and said as the worker's below. + const key = keyForUrl(args.url, args.key); + const [brief, prices] = await Promise.all([ + buildBrief({ cwd: args.cwd }).catch(() => ({ text: "" })), + // Resolved exactly as `run` resolves it (workerPricing): hand-set, + // stored for this model, or the provider's list. + workerPricing(args).catch(() => ({}) as { in?: number; out?: number; cached?: number; source?: string }), + ]); + const pricing = + prices.in !== undefined && prices.out !== undefined + ? { in: prices.in, out: prices.out, ...(prices.cached !== undefined ? { cached: prices.cached } : {}), source: prices.source ?? "stored" } + : null; + const planMeter = new SpendMeter(() => ({ baseUrl: args.url, model: args.model, pricing })); const r = await draftMission({ goal, context: brief.text, scripts: projectScripts(args.cwd), - ask: { baseUrl: args.url, apiKey: keyForUrl(args.url, args.key), model: args.model, cwd: args.cwd, reasoningEffort: args.reasoningChecks ?? args.reasoning }, + ask: { baseUrl: args.url, apiKey: key, model: args.model, cwd: args.cwd, reasoningEffort: args.reasoningChecks ?? args.reasoning, meter: planMeter }, }); + if (planMeter.mark()) process.stdout.write(`${askSpendLine(planMeter.total(), fmtCost, "worker (planning)")}\n`); if (!r.ok) { process.stderr.write(`maat: ${r.error}\n`); return 1; @@ -1358,6 +1396,8 @@ async function autoDraftFrom( // Drafted, then read cold by a critic against the task text: a check that // invents or guesses is dropped (with a task quote), and a draft where // nothing runs the deliverable is asked for once more. See criteria.ts. + // Where the drafting began, for the budget-hit message (Engine.draftStart). + engine.draftingStarted(); const draftOnce = () => draftCriteriaCritiqued({ snapshot, diff --git a/src/engine.ts b/src/engine.ts index 37f4a44..fef31c8 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -25,7 +25,7 @@ import { describeStart, listBackground, startBackground, stopBackground } from " import { objectionLine, readsTheWork, reviewClaim, type ExecutableReview, type ObjectionRun, type Review } from "./review.js"; import { credentialFreeEnv } from "./credenv.js"; import { judgeEffort, judgeTarget } from "./judge.js"; -import { JudgeMeter, judgeTokens, type JudgeSpend } from "./judge-meter.js"; +import { JudgeMeter, judgeTokens, type AskMeter, type AskUsage, type JudgeSpend } from "./judge-meter.js"; import { auditClaim, authorKey, authorWords, claimLabel, contradictions, independentOf, tierOf, withAuthor, type Tier } from "./tiers.js"; import { recordGoldens, valueUnproven } from "./golden.js"; import { discountedChecks } from "./control.js"; @@ -1873,6 +1873,12 @@ function fmtUsd(usd: number): string { return usd >= 0.1 ? `$${usd.toFixed(2)}` : usd >= 0.001 ? `$${usd.toFixed(3)}` : "<$0.001"; } +/** The sum of the values that are known, or undefined when none is. */ +function sumKnown(...xs: (number | undefined)[]): number | undefined { + const known = xs.filter((x): x is number => x !== undefined); + return known.length ? known.reduce((a, b) => a + b, 0) : undefined; +} + /** A tool argument that should be a non-empty string, or nothing. */ function str(v: unknown): string | undefined { return typeof v === "string" && v.trim() !== "" ? v : undefined; @@ -2435,12 +2441,50 @@ export class Engine { } : null, })); + /** + * The worker's meter for asks on its own model outside a turn (the + * interview): counted into the session meter as worker spend, like a step, + * and journalled as `worker_ask`. Not the judge's: it is the worker's model. + */ + readonly workerAskMeter: AskMeter = { record: (u) => this.recordWorkerAsk(u) }; + + private recordWorkerAsk(u: AskUsage): void { + this.sessionPrompt += u.promptTokens; + this.sessionCompletion += u.completionTokens; + this.sessionCached += u.cacheReadTokens ?? 0; + if (u.estimated) this.estimatedSteps += 1; + if (typeof u.billedUsd === "number") this.sessionBilled += u.billedUsd; + else this.unbilledSteps += 1; + this.cfg.journal?.append("worker_ask", { + model: u.model, + ...(u.what ? { what: u.what } : {}), + promptTokens: u.promptTokens, + completionTokens: u.completionTokens, + ...(u.cacheReadTokens !== undefined ? { cacheReadTokens: u.cacheReadTokens } : {}), + ...(u.cacheWriteTokens !== undefined ? { cacheWriteTokens: u.cacheWriteTokens } : {}), + ...(typeof u.billedUsd === "number" ? { billedUsd: u.billedUsd } : {}), + estimated: u.estimated, + }); + } + /** * Judge calls before this mark were reported in an earlier job_end. A job * reports every call since the last one, so the drafting that runs before a * turn starts is counted in the turn it was for. */ private judgeReported = 0; + /** + * Where the judge's drafting for the next turn began: the meter's mark and + * what had been spent by then (draftingStarted). The budget-hit message says + * the drafting used the budget only when this turn's drafting is what took + * it over, not when earlier spend had already nearly used it up. + */ + private draftStart: { mark: number; tokens: number } | undefined; + + /** The caller is about to draft this turn's checks on the judge (cli.tsx). */ + draftingStarted(): void { + this.draftStart ??= { mark: this.judgeMeter.mark(), tokens: this.spentTokens }; + } /** Why the independent review was skipped this turn (see reviewSkipReason), if it was. */ private reviewSkipped: string | undefined; /** Hidden checks whose commands were shown to the model this turn. */ @@ -3560,15 +3604,14 @@ export class Engine { } /** - * Worker and judge USD together: what a money budget counts. Undefined when - * the worker has no price. A judge with no price adds nothing here (its + * Worker and judge USD together: what a money budget counts. Defined when + * either is priced, so a local or subscription worker with a paid judge is + * still bounded by a dollar ceiling. An unpriced side adds nothing here (its * tokens still count toward a token budget, and every surface says its $ is * unknown); a judge on a subscription plan costs no money. */ totalCostUsd(): number | undefined { - const worker = this.costUsd(); - if (worker === undefined) return undefined; - return worker + (this.judgeMeter.total().costUsd ?? 0); + return sumKnown(this.costUsd(), this.judgeMeter.total().costUsd); } /** The whole session's judge spend (judge-meter.ts). */ @@ -5214,6 +5257,7 @@ export class Engine { // the review and the audit run after the receipt was written. const judge = this.judgeMeter.since(this.judgeReported); this.judgeReported = this.judgeMeter.mark(); + this.draftStart = undefined; if (receiptPath && judge.calls) this.cfg.receipts?.amendJudge(basename(receiptPath), this.judgeMeter.total()); yield { kind: "job_end", @@ -6579,7 +6623,10 @@ export class Engine { // Both ceilings count the judge's spend too (overBudget). const turnStartTokens = this.spentTokens; - const turnStartCost = this.totalCostUsd(); + // Priced per side: the worker's delta when the worker is priced, plus the + // judge's calls this turn when they are. Either one makes the turn priced. + const turnStartCost = this.costUsd(); + const turnJudgeMark = this.judgeMeter.mark(); let warned = 0; // The step guard is the last way out of a turn, and it had the same fault // the spending ceiling had: it stopped dead. A reported run reached it with @@ -6697,7 +6744,17 @@ export class Engine { text: `budget hit (${this.budgetTokens} tokens` + (this.spentTokens > this.sessionTokens ? `, ${this.spentTokens - this.sessionTokens} of them the judge's` : "") + - `) — loop stopped. /budget to raise.`, + `) — loop stopped. /budget to raise.` + + // Hit before the worker took a step, by this turn's drafting: the + // judge drafted since draftingStarted, and the spend before it was + // still under the budget. + (step === 0 && + this.draftStart !== undefined && + this.judgeMeter.mark() > this.draftStart.mark && + this.draftStart.tokens < (this.budgetTokens ?? 0) + ? ` The judge drafting this turn's checks used the budget before the worker's first step; ` + + `raise --budget to leave room for both.` + : ""), ceiling: "budget", }; yield* this.salvage(`You have reached the token budget for this session.`, fetchFn, log); @@ -6709,21 +6766,33 @@ export class Engine { // refuses to spend rather than what it noticed spending. // Money where a price is known, tokens only where it is not. const spentThisTurn = this.spentTokens - turnStartTokens; - const usdThisTurn = - turnStartCost === undefined ? undefined : (this.totalCostUsd() ?? 0) - turnStartCost; + const usdThisTurn = sumKnown( + turnStartCost === undefined ? undefined : (this.costUsd() ?? 0) - turnStartCost, + this.judgeMeter.since(turnJudgeMark).costUsd, + ); // Zero unless someone set one. The self-hosted exception that used to // live here — no default ceiling on hardware you own — is gone with the // default itself: there is nothing left to make an exception to. const usdCeiling = this.cfg.maxTurnUsd ?? 0; const tokenCeiling = this.cfg.maxTurnTokens ?? 0; - const priced = usdThisTurn !== undefined && usdCeiling > 0; - const used = priced ? usdThisTurn : spentThisTurn; + // A priced worker is held in dollars, as it always was. An unpriced + // worker with a priced judge is held by both: dollars bound the part that + // has a price (the judge's), tokens bound the whole turn, and whichever + // is nearer its ceiling is the one that speaks and stops. Without the + // token side, one priced judge call flipped the turn to dollars and the + // worker's unpriced tokens ran on unbounded. + const usdShare = usdThisTurn !== undefined && usdCeiling > 0 ? usdThisTurn / usdCeiling : undefined; + const tokenShare = tokenCeiling > 0 ? spentThisTurn / tokenCeiling : undefined; + const priced = + usdShare !== undefined && + (turnStartCost !== undefined || tokenShare === undefined || usdShare >= tokenShare); + const used = priced ? usdThisTurn! : spentThisTurn; const ceiling = priced ? usdCeiling : tokenCeiling; // Named for what it is, and not `shown` — which is the read-coverage map // a few lines down, and which this quietly shadowed until the compiler // said so. const ceilingLine = priced - ? `${fmtUsd(usdThisTurn)} of ${fmtUsd(usdCeiling)}` + ? `${fmtUsd(usdThisTurn!)} of ${fmtUsd(usdCeiling)}` : `${spentThisTurn} of ${tokenCeiling} tokens`; // Said on the way up, not only on arrival. A limit that speaks for the diff --git a/src/format.ts b/src/format.ts index 79c4296..5245550 100644 --- a/src/format.ts +++ b/src/format.ts @@ -10,7 +10,7 @@ * Pure, and re-exported from `banner.tsx` so nothing that already imported * them has to move. */ -import type { JudgeSpend } from "./judge-meter.js"; +import type { AskSpend } from "./judge-meter.js"; export function fmtTokens(n: number): string { if (n < 1000) return `${n}`; @@ -122,11 +122,12 @@ export function spendLine(sp: { } /** - * A judge spend in one line, for the CLI and the receipt: + * A judge spend in one line, for the CLI and the receipt (`label` names whose + * asks they were; a worker's own planning asks use the same shape): * `judge 3 calls · 1200 in (400 cached) · 300 out · $0.004`, with `$ unknown` * when a model had no price and the plan's name when a plan paid. */ -export function judgeSpendLine(s: JudgeSpend, fmt: (usd: number) => string = fmtCost): string { +export function askSpendLine(s: AskSpend, fmt: (usd: number) => string = fmtCost, label = "judge"): string { const cache = s.cacheReadTokens > 0 || s.cacheWriteTokens > 0 ? ` (${[s.cacheReadTokens > 0 ? `${s.cacheReadTokens} cached` : "", s.cacheWriteTokens > 0 ? `${s.cacheWriteTokens} cache write` : ""].filter(Boolean).join(", ")})` @@ -140,5 +141,8 @@ export function judgeSpendLine(s: JudgeSpend, fmt: (usd: number) => string = fmt : s.plan ? `your ${s.plan} plan, not metered` : "$ unknown"; - return `judge ${s.calls} call${s.calls === 1 ? "" : "s"} · ${e}${s.promptTokens} in${cache} · ${e}${s.completionTokens} out · ${money}`; + return `${label} ${s.calls} call${s.calls === 1 ? "" : "s"} · ${e}${s.promptTokens} in${cache} · ${e}${s.completionTokens} out · ${money}`; } + +/** The judge's spend line: askSpendLine under its original name. */ +export const judgeSpendLine = askSpendLine; diff --git a/src/interview.ts b/src/interview.ts index 1b8028e..477b5ca 100644 --- a/src/interview.ts +++ b/src/interview.ts @@ -17,6 +17,9 @@ import { removedSubscriptionProblem } from "./endpoint.js"; import { errorText } from "./format.js"; import { askError, askTimeoutMs, probeSignal } from "./watchdog.js"; import { authHeaders } from "./providers.js"; +import { recordUsage } from "./ask.js"; +import type { AskMeter } from "./judge-meter.js"; +import type { Usage } from "./stream.js"; import { loadBar, parseBar, barPath } from "./bar.js"; import type { Bar, Check } from "./types.js"; import { @@ -277,6 +280,12 @@ export async function interviewTurn(opts: { acpSpawn?: typeof import("node:child_process").spawn; /** How long the HTTP question may wait for its answer; see askTimeoutMs. Tests only. */ timeoutMs?: number; + /** + * Where the ask reports what it used. The interview asks the worker's own + * model, so callers pass the engine's worker meter (Engine.workerAskMeter): + * its tokens and cost are the worker's spend. + */ + meter?: AskMeter; }): Promise { const f = opts.fetchFn ?? fetch; const base = opts.baseUrl.replace(/\/$/, ""); @@ -320,6 +329,7 @@ export async function interviewTurn(opts: { ...(opts.acpSpawn ? { spawnFn: opts.acpSpawn } : {}), }); if (!asked.ok) return { kind: "error", error: asked.error }; + recordUsage(opts.meter, { baseUrl: opts.baseUrl, model: opts.model, what: "interviewing", sent: SYSTEM + context }, undefined, asked.text); return parseInterviewReply(asked.text, opts.round); } @@ -340,8 +350,9 @@ export async function interviewTurn(opts: { }), }); if (!res.ok) return { kind: "error", error: `HTTP ${res.status} interviewing` }; - const json = (await res.json()) as { choices?: { message?: { content?: string } }[] }; + const json = (await res.json()) as { choices?: { message?: { content?: string } }[]; usage?: Usage }; const text = json.choices?.[0]?.message?.content ?? ""; + recordUsage(opts.meter, { baseUrl: opts.baseUrl, model: opts.model, what: "interviewing", sent: SYSTEM + context }, json.usage, text); /** * No last-round guard here, because there is nothing left to guard. * diff --git a/src/journal.ts b/src/journal.ts index 7b4864b..19308a2 100644 --- a/src/journal.ts +++ b/src/journal.ts @@ -92,6 +92,11 @@ export type JournalKind = * entries. `costUsd: null` means no price is known, never $0. */ | "judge_usage" + /** + * One answered ask on the worker's own model outside its turn (the + * interview): worker spend, counted into the session meter like a step. + */ + | "worker_ask" /** The independent review of a verified claim: a label, never a gate. */ | "review" | "dispute" @@ -406,6 +411,11 @@ export class Journal { ); break; } + case "worker_ask": { + const e = d.estimated ? "~" : ""; + out.push(`${t} ask ${d.model}${d.what ? ` (${d.what})` : ""} · ${e}${d.promptTokens} in / ${e}${d.completionTokens} out`); + break; + } case "tool_call": out.push(`${t} tool ${d.name}: ${d.detail}`); break; diff --git a/src/judge-meter.ts b/src/judge-meter.ts index 12feb94..d0edf5a 100644 --- a/src/judge-meter.ts +++ b/src/judge-meter.ts @@ -47,8 +47,8 @@ export interface AskMeter { record(u: AskUsage): void; } -/** The judge's spend over some span of calls. */ -export type JudgeSpend = { +/** The spend of a span of asks (the judge's, or the worker's planning ask). */ +export type AskSpend = { calls: number; promptTokens: number; completionTokens: number; @@ -100,7 +100,12 @@ export function priceTokens(p: Pricing, u: Pick(); private readonly listeners: ((c: JudgeCall) => void)[] = []; @@ -180,17 +185,19 @@ export class JudgeMeter implements AskMeter { } /** The whole session's judge spend. */ - total(): JudgeSpend { + total(): AskSpend { return this.since(0); } /** * The judge's spend since a mark. Priced now, not when recorded, so a price * that arrived after the first ask (a lookup still in flight, a /price) is - * applied to every call, as the worker's meter does. + * applied to every call, and a price that was cleared is no longer applied + * to any: the figure stored at record time (the journal's) is dropped before + * re-pricing. The worker's meter does the same with its session totals. */ - since(mark: number): JudgeSpend { - const span = this.calls.slice(mark).map((c) => this.priced(c)); + since(mark: number): AskSpend { + const span = this.calls.slice(mark).map(({ costUsd: _recorded, plan: _plan, ...u }) => this.priced(u)); const models: string[] = []; let cost = 0; let unpriced = 0; @@ -233,9 +240,15 @@ export class JudgeMeter implements AskMeter { } /** Tokens a judge spend counts toward a budget: everything sent and received. */ -export function judgeTokens(s: JudgeSpend | undefined): number { +export function judgeTokens(s: AskSpend | undefined): number { return s ? s.promptTokens + s.completionTokens : 0; } +/** The judge's spend: an AskSpend over the judge's calls. */ +export type JudgeSpend = AskSpend; +/** The engine's meter of the judge's calls: a SpendMeter. */ +export const JudgeMeter = SpendMeter; +export type JudgeMeter = SpendMeter; + /** Said in one line by format.ts, which the renderer can import. */ -export { judgeSpendLine } from "./format.js"; +export { askSpendLine, judgeSpendLine } from "./format.js"; diff --git a/src/receipts.ts b/src/receipts.ts index e5d847f..78ad362 100644 --- a/src/receipts.ts +++ b/src/receipts.ts @@ -856,15 +856,11 @@ export class Receipts { } } - /** - * Record the independent review's objections on a receipt's index row: each - * with its command and result (`--review-executable`). Masked like every - * other write while hidden commands are withheld. The receipt file itself, - * hash-bound when it was written, is never rewritten. - */ /** * Bring a row's judge spend up to date at the end of its job: the review and - * the post-work audit ask the judge after the receipt is written. + * the post-work audit ask the judge after the receipt is written. Only the + * index row changes; the receipt file, hash-bound when it was written, keeps + * the "so far" figure it was written with. */ amendJudge(file: string, judge: JudgeSpend): boolean { return this.amendRow(file, (r) => ({ ...r, judge })); @@ -895,6 +891,12 @@ export class Receipts { } } + /** + * Record the independent review's objections on a receipt's index row: each + * with its command and result (`--review-executable`). Masked like every + * other write while hidden commands are withheld. The receipt file itself, + * hash-bound when it was written, is never rewritten. + */ amendReview(file: string, objections: readonly Objection[]): boolean { return this.amendRow(file, (r) => ({ ...r, objections: maskDeep([...objections], this.withheld) })); } diff --git a/test/judge-meter.test.ts b/test/judge-meter.test.ts index 652a2fc..f326a8a 100644 --- a/test/judge-meter.test.ts +++ b/test/judge-meter.test.ts @@ -232,6 +232,138 @@ describe("a budget counts the judge's spend", () => { }); }); +describe("the turn's money ceiling with an unpriced worker", () => { + it("still counts a priced judge's dollars", async () => { + const ws = workspace(); + try { + const fetchFn = (async () => + reply({ choices: [{ message: { role: "assistant", content: null, tool_calls: [{ id: "c", type: "function", function: { name: "list_files", arguments: "{}" } }] }, finish_reason: "tool_calls" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })) as unknown as typeof fetch; + // No worker price at all: a local or subscription worker with a paid judge. + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high", maxTurnUsd: 0.01, maxSteps: 4 }); + engine.judgeMeter.setPricing(URL, JUDGE, { in: 1000, out: 1000, source: "test" }); + assert.equal(engine.costUsd(), undefined); + const events: EngineEvent[] = []; + for await (const ev of engine.run("anything", allowAll)) { + events.push(ev); + if (ev.kind === "usage" && events.filter((x) => x.kind === "usage").length === 1) { + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, promptTokens: 10, completionTokens: 10, estimated: false }); + } + } + assert.ok(Math.abs(engine.totalCostUsd()! - 0.02) < 1e-12, "the judge's $0.02 is the turn's priced spend"); + assert.ok( + events.some((x) => (x.kind === "error" || x.kind === "info") && /\$0\.020 of \$0\.010/.test(x.text)), + "the dollar ceiling saw the judge's spend", + ); + } finally { + ws.cleanup(); + } + }); +}); + +describe("an unpriced worker with a priced judge is held by both ceilings", () => { + // Grok's reproducer (#57 review, N1): one priced judge call flipped the turn + // to dollars and the worker's tokens ran to the step guard (700k). + it("stops at the token ceiling, not at the step guard", async () => { + const ws = workspace(); + try { + let asked = 0; + const fetchFn = (async () => { + asked += 1; + return reply({ choices: [{ message: { role: "assistant", content: null, tool_calls: [{ id: `c${asked}`, type: "function", function: { name: "list_files", arguments: JSON.stringify({ path: `d${asked}` }) } }] }, finish_reason: "tool_calls" }], usage: { prompt_tokens: 100_000, completion_tokens: 0 } }); + }) as unknown as typeof fetch; + const engine = new Engine({ + baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high", + maxTurnUsd: 1, maxTurnTokens: 150_000, maxSteps: 7, + }); + engine.judgeMeter.setPricing(URL, JUDGE, { in: 1, out: 1, source: "test" }); + const events: EngineEvent[] = []; + let judged = false; + for await (const ev of engine.run("anything", allowAll)) { + events.push(ev); + if (ev.kind === "usage" && !judged) { + judged = true; + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, promptTokens: 10, completionTokens: 10, estimated: false }); + } + } + // Stopped after two steps at 200,020 (the third request is the closing + // summary every stopped turn asks for), not run on to the step guard. + assert.ok( + events.some((x) => x.kind === "error" && /stopped: this turn has spent 200020 of 150000 tokens/.test(x.text)), + "the token ceiling stopped the turn at ~200k", + ); + assert.equal(asked, 3, "two work steps and the closing summary, not seven steps"); + } finally { + ws.cleanup(); + } + }); +}); + +describe("the budget-hit message", () => { + it("says the drafting used the budget when it was hit before the worker's first step", async () => { + const ws = workspace(); + try { + const fetchFn = (async () => reply({ choices: [{ message: { role: "assistant", content: "Done." }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })) as unknown as typeof fetch; + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high" }); + engine.setBudget(1_000); + engine.draftingStarted(); + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "drafting criteria", promptTokens: 1_200, completionTokens: 0, estimated: false }); + const hit = (await drain(engine.run("anything", allowAll))).find((x) => x.kind === "error" && x.ceiling === "budget"); + assert.ok(hit && hit.kind === "error"); + assert.match(hit.text, /drafting this turn's checks used the budget before the worker's first step; raise --budget to leave room for both/); + } finally { + ws.cleanup(); + } + }); + + it("gives no drafting hint when the budget was already used up before the drafting began", async () => { + const ws = workspace(); + try { + const fetchFn = (async () => reply({ choices: [{ message: { role: "assistant", content: "Done." }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })) as unknown as typeof fetch; + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high" }); + engine.setBudget(1_000); + // A judge ask outside the drafting (an arbiter, say) already spent the budget. + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "ruling on a disputed check", promptTokens: 1_100, completionTokens: 0, estimated: false }); + engine.draftingStarted(); + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "drafting criteria", promptTokens: 50, completionTokens: 0, estimated: false }); + const hit = (await drain(engine.run("anything", allowAll))).find((x) => x.kind === "error" && x.ceiling === "budget"); + assert.ok(hit && hit.kind === "error"); + assert.doesNotMatch(hit.text, /drafting/); + } finally { + ws.cleanup(); + } + }); + + it("gives no drafting hint for judge spend outside any drafting", async () => { + const ws = workspace(); + try { + const fetchFn = (async () => reply({ choices: [{ message: { role: "assistant", content: "Done." }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })) as unknown as typeof fetch; + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high" }); + engine.setBudget(1_000); + engine.judgeMeter.record({ baseUrl: URL, model: JUDGE, what: "reviewing the claim", promptTokens: 1_200, completionTokens: 0, estimated: false }); + const hit = (await drain(engine.run("anything", allowAll))).find((x) => x.kind === "error" && x.ceiling === "budget"); + assert.ok(hit && hit.kind === "error"); + assert.doesNotMatch(hit.text, /drafting/); + } finally { + ws.cleanup(); + } + }); + + it("gives no drafting hint when the worker spent it", async () => { + const ws = workspace(); + try { + const fetchFn = (async () => + reply({ choices: [{ message: { role: "assistant", content: null, tool_calls: [{ id: "c", type: "function", function: { name: "list_files", arguments: "{}" } }] }, finish_reason: "tool_calls" }], usage: { prompt_tokens: 600, completion_tokens: 0 } })) as unknown as typeof fetch; + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, fetchFn, bar: null, stream: false, autonomy: "high", maxSteps: 5 }); + engine.setBudget(1_000); + const hit = (await drain(engine.run("anything", allowAll))).find((x) => x.kind === "error" && x.ceiling === "budget"); + assert.ok(hit && hit.kind === "error"); + assert.doesNotMatch(hit.text, /drafting/); + } finally { + ws.cleanup(); + } + }); +}); + describe("JudgeMeter", () => { const u = { baseUrl: URL, model: JUDGE, promptTokens: 1000, completionTokens: 200, cacheReadTokens: 400, estimated: false }; @@ -251,6 +383,16 @@ describe("JudgeMeter", () => { assert.ok(Math.abs(m.total().costUsd! - PER_CALL) < 1e-12); }); + it("a price cleared after a call is no longer applied to it", () => { + const m = new JudgeMeter(); + m.setPricing(URL, JUDGE, PRICE); + m.record(u); + assert.ok(m.total().costUsd! > 0); + m.setPricing(URL, JUDGE, null); + assert.equal(m.total().costUsd, undefined, "re-priced now: no price, $ unknown"); + assert.equal(m.all()[0]!.costUsd, PER_CALL, "the figure recorded at the time (the journal's) is kept on the call"); + }); + it("uses the provider's billed figure when it sends one", () => { const m = new JudgeMeter(); m.record({ ...u, billedUsd: 0.5 }); diff --git a/test/worker-asks.test.ts b/test/worker-asks.test.ts new file mode 100644 index 0000000..021cf38 --- /dev/null +++ b/test/worker-asks.test.ts @@ -0,0 +1,72 @@ +/** + * Asks on the worker's own model outside a turn — the interview and mission + * planning — are metered as worker spend. Before this they used tokens no + * meter saw (the judge's asks were metered in #56; these are not the judge's). + */ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { Engine } from "../src/engine.js"; +import { judgeSpendLine } from "../src/format.js"; +import { Journal } from "../src/journal.js"; +import { JudgeMeter } from "../src/judge-meter.js"; +import { interviewTurn } from "../src/interview.js"; +import { draftMission } from "../src/mission.js"; +import { workspace } from "./helpers.js"; + +const URL = "http://provider.test/v1"; +const reply = (body: unknown) => + (async () => ({ ok: true, status: 200, json: async () => body, text: async () => JSON.stringify(body), headers: new Headers() }) as unknown as Response) as unknown as typeof fetch; + +describe("the interview's ask is worker spend", () => { + it("adds its reported tokens to the session meter and the journal", async () => { + const ws = workspace(); + try { + const journal = new Journal(ws.dir); + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, bar: null, journal, priceInPerMtok: 2, priceOutPerMtok: 10 }); + const fetchFn = reply({ + choices: [{ message: { content: JSON.stringify({ questions: [{ id: "q1", prompt: "What counts as done?", options: ["a", "b"] }] }) } }], + usage: { prompt_tokens: 900, completion_tokens: 100, prompt_tokens_details: { cached_tokens: 300 } }, + }); + const r = await interviewTurn({ task: "t", scripts: [], barChecks: [], history: [], round: 1, baseUrl: URL, model: "m", fetchFn, meter: engine.workerAskMeter }); + assert.notEqual(r.kind, "error"); + const b = engine.bom(); + assert.equal(b.sessionPromptTokens, 900); + assert.equal(b.sessionCompletionTokens, 100); + assert.equal(b.sessionCachedTokens, 300); + assert.ok(Math.abs(b.costUsd! - (900 * 2 + 100 * 10) / 1e6) < 1e-12, "priced as the worker"); + assert.equal(b.judge, undefined, "not the judge's"); + const e = Journal.read(journal.path).filter((x) => x.kind === "worker_ask"); + assert.equal(e.length, 1); + assert.equal((e[0]!.data as Record).what, "interviewing"); + assert.match(Journal.summarize(e)[0]!, /ask m \(interviewing\) · 900 in \/ 100 out/); + } finally { + ws.cleanup(); + } + }); + + it("estimates, and marks it, when the provider reports no usage", async () => { + const ws = workspace(); + try { + const engine = new Engine({ baseUrl: URL, model: "m", cwd: ws.dir, bar: null }); + const fetchFn = reply({ choices: [{ message: { content: JSON.stringify({ proposal: { criteria: [] } }) } }] }); + await interviewTurn({ task: "t", scripts: [], barChecks: [], history: [], round: 3, baseUrl: URL, model: "m", fetchFn, meter: engine.workerAskMeter }); + assert.ok(engine.sessionTokens > 0); + assert.equal(engine.costEstimated, true); + } finally { + ws.cleanup(); + } + }); +}); + +describe("mission planning is worker spend", () => { + it("reports the plan's tokens and cost on a meter priced as the worker", async () => { + const meter = new JudgeMeter(() => ({ baseUrl: URL, model: "m", pricing: { in: 2, out: 10, source: "test" } })); + const fetchFn = reply({ choices: [{ message: { content: "not a plan" }, finish_reason: "stop" }], usage: { prompt_tokens: 5000, completion_tokens: 1000 } }); + await draftMission({ goal: "a CLI", ask: { baseUrl: URL, model: "m", fetchFn, meter } }); + const s = meter.total(); + assert.equal(s.calls, 1); + assert.equal(s.promptTokens, 5000); + assert.ok(Math.abs(s.costUsd! - (5000 * 2 + 1000 * 10) / 1e6) < 1e-12); + assert.equal(judgeSpendLine(s, undefined, "worker (planning)"), "worker (planning) 1 call · 5000 in · 1000 out · $0.020"); + }); +});