Repository navigation
Lean guards: request-size ceilings in CI, per-run cost alarm in the bench #48
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
solvyxtech
wants to merge
3
commits into
claude/malformed-cap
Choose a base branch
from
claude/lean-guards
base: claude/malformed-cap
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,6 +17,7 @@ | |
| import json | ||
| import os | ||
| import shutil | ||
| import statistics | ||
| import subprocess | ||
| import sys | ||
| import time | ||
|
|
@@ -148,7 +149,7 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: | |
| return { | ||
| "provider_capped": capped, | ||
| "secs": round(secs), "turns": steps, "calls": sum(per), "multi": sum(1 for x in per if x > 1), | ||
| "tokens_in": spend.get("promptTokens"), "claim": outcome, "timed_out": timed_out, | ||
| "tokens_in": spend.get("promptTokens"), "cost_usd": spend.get("costUsd"), "claim": outcome, "timed_out": timed_out, | ||
| "said_done": (outcome or "").startswith("verified"), | ||
| # The reviewer's label: a verified claim it did not confirm. | ||
| "review": review, | ||
|
|
@@ -159,6 +160,42 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: | |
| } | ||
|
|
||
|
|
||
| # Per-run cost alarm (reports/loop-charter.md, non-regression rule 1): a run that | ||
| # costs more than max(5x the lane's running median cost, $0.10), or sends more | ||
| # than 1.5M prompt tokens, stops the lane before the bill grows. 2026-10-07: one | ||
| # task cost 2.68M tokens / $0.82 because malformed calls were resent in full. | ||
| COST_ALARM_EXIT = 3 | ||
|
|
||
|
|
||
| def alarm_limits() -> dict: | ||
| """The alarm thresholds, each overridable by env.""" | ||
| return { | ||
| "x": float(os.environ.get("BENCH_COST_ALARM_X") or 5), | ||
| "usd": float(os.environ.get("BENCH_COST_ALARM_USD") or 0.10), | ||
| "tokens": int(float(os.environ.get("BENCH_TOKEN_ALARM") or 1_500_000)), | ||
| } | ||
|
|
||
|
|
||
| def cost_alarm(row: dict, prior_costs: list[float]) -> dict | None: | ||
| """Why this run trips the alarm, or None. `prior_costs` are the lane's earlier runs | ||
| (this run excluded): the median is over runs that reported a cost. A run with no | ||
| cost (an unpriced model) is judged on its prompt tokens only.""" | ||
| lim = alarm_limits() | ||
| med = statistics.median(prior_costs) if prior_costs else None | ||
| limit_usd = max(lim["x"] * med, lim["usd"]) if med is not None else lim["usd"] | ||
| cost, toks = row.get("cost_usd"), row.get("tokens_in") | ||
| why = [] | ||
| if cost is not None and cost > limit_usd: | ||
| why.append(f"cost ${cost:.4f} > ${limit_usd:.4f} (max({lim['x']:g}x lane median " | ||
| f"{'n/a' if med is None else f'${med:.4f}'}, ${lim['usd']:g}))") | ||
| if toks is not None and toks > lim["tokens"]: | ||
| why.append(f"prompt tokens {toks:,} > {lim['tokens']:,}") | ||
| if not why: | ||
| return None | ||
| return {"detail": "; ".join(why), "cost_usd": cost, "tokens_in": toks, "lane_median_usd": med, | ||
| "limit_usd": round(limit_usd, 6), "limit_tokens": lim["tokens"]} | ||
|
|
||
|
|
||
| def claims_done(text: str) -> bool: | ||
| """ | ||
| Whether a final message says the work is done, by one fixed rule: a hedge | ||
|
|
@@ -211,10 +248,15 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: | |
| arms = parse_arms(os.environ.get("ARMS")) | ||
| out = Path(os.environ.get("RESULTS_DIR", HERE)) / os.environ.get("RESULTS", f"results-{which}-x{repeats}.jsonl") | ||
| done = set() | ||
| lane_costs: dict = {} # arm -> costs of its runs so far, for the cost alarm's median | ||
| if out.exists(): # resume: skip runs already recorded | ||
| for line in out.read_text().splitlines(): | ||
| r = json.loads(line) | ||
| if r.get("stopped"): # a STOPPED marker, not a run | ||
| continue | ||
| done.add((r["task"], r["agent"], r["rep"], r.get("arm"))) | ||
| if r.get("cost_usd") is not None: | ||
| lane_costs.setdefault(r.get("arm"), []).append(r["cost_usd"]) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Non-blocking, high confidence: on resume this adds the run that tripped the alarm into the median. With |
||
| for rep in range(repeats): | ||
| for T in tasks: | ||
| for arm, arm_env in arms: # paired: arm A then arm B on this task before the next task | ||
|
|
@@ -251,6 +293,23 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: | |
| with out.open("a") as f: | ||
| f.write(json.dumps(r) + "\n") | ||
| print(json.dumps({k: r[k] for k in ("task", "agent", "arm", "rep", "passed", "said_done", "turns", "secs") if k in r}), flush=True) | ||
| costs = lane_costs.setdefault(arm, []) | ||
| alarm = cost_alarm(r, costs) | ||
| if alarm: | ||
| stop_lane(out, tag, WORK / f"{tag}.log", alarm, arm) | ||
| if r.get("cost_usd") is not None: | ||
| costs.append(r["cost_usd"]) | ||
|
|
||
|
|
||
| def stop_lane(out: Path, tag: str, log: Path, alarm: dict, arm: str | None) -> None: | ||
| """Record a STOPPED row (no `passed`: not a run) and end the lane non-zero.""" | ||
| row = {"stopped": True, "reason": "cost alarm", "run": tag, "log": str(log), **alarm} | ||
| if arm: | ||
| row["arm"] = arm | ||
| with out.open("a") as f: | ||
| f.write(json.dumps(row) + "\n") | ||
| print(f"STOPPED: cost alarm on {tag}: {alarm['detail']}\n log: {log}", flush=True) | ||
| sys.exit(COST_ALARM_EXIT) | ||
|
|
||
|
|
||
|
|
||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| """The per-run cost alarm: a run far over the lane's usual cost stops the lane. | ||
|
|
||
| 2026-10-07: one task cost 2.68M tokens / $0.82 because malformed tool calls were | ||
| resent in full every step. These tests pin the thresholds and that the lane | ||
| stops with a STOPPED row and a non-zero exit instead of running on. | ||
| """ | ||
| import json | ||
| import sys | ||
| import types | ||
| from pathlib import Path | ||
|
|
||
| import pytest | ||
|
|
||
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | ||
| import run # noqa: E402 | ||
| import scoreboard # noqa: E402 | ||
|
|
||
|
|
||
| @pytest.fixture(autouse=True) | ||
| def _no_env_overrides(monkeypatch): | ||
| for k in ("BENCH_COST_ALARM_X", "BENCH_COST_ALARM_USD", "BENCH_TOKEN_ALARM", "ARMS", "BENCH_TASKS", "MAAT_BUILD"): | ||
| monkeypatch.delenv(k, raising=False) | ||
|
|
||
|
|
||
| def test_floor_applies_with_no_history(): | ||
| assert run.cost_alarm({"cost_usd": 0.09, "tokens_in": 1000}, []) is None | ||
| a = run.cost_alarm({"cost_usd": 0.11, "tokens_in": 1000}, []) | ||
| assert a and a["limit_usd"] == 0.10 and "cost $0.1100" in a["detail"] | ||
|
|
||
|
|
||
| def test_five_times_the_median_when_that_is_above_the_floor(): | ||
| prior = [0.03, 0.04, 0.05] # median 0.04 -> limit 0.20 | ||
| assert run.cost_alarm({"cost_usd": 0.19}, prior) is None | ||
| a = run.cost_alarm({"cost_usd": 0.21}, prior) | ||
| assert a and a["limit_usd"] == pytest.approx(0.20) and a["lane_median_usd"] == 0.04 | ||
|
|
||
|
|
||
| def test_floor_wins_over_a_tiny_median(): | ||
| # A cheap lane (median $0.002) must not alarm on an ordinary $0.05 run. | ||
| assert run.cost_alarm({"cost_usd": 0.05}, [0.002, 0.002, 0.003]) is None | ||
|
|
||
|
|
||
| def test_the_regression_that_motivated_it_trips_both_alarms(): | ||
| a = run.cost_alarm({"cost_usd": 0.82, "tokens_in": 2_680_000}, [0.02, 0.03, 0.025]) | ||
| assert a and "cost $0.8200" in a["detail"] and "prompt tokens 2,680,000 > 1,500,000" in a["detail"] | ||
|
|
||
|
|
||
| def test_token_alarm_without_a_price(): | ||
| # Free models report no cost: prompt tokens are the only signal. | ||
| assert run.cost_alarm({"cost_usd": None, "tokens_in": 1_400_000}, []) is None | ||
| a = run.cost_alarm({"cost_usd": None, "tokens_in": 1_600_000}, []) | ||
| assert a and a["cost_usd"] is None and a["limit_tokens"] == 1_500_000 | ||
|
|
||
|
|
||
| def test_env_overrides(monkeypatch): | ||
| monkeypatch.setenv("BENCH_COST_ALARM_X", "2") | ||
| monkeypatch.setenv("BENCH_COST_ALARM_USD", "0.01") | ||
| monkeypatch.setenv("BENCH_TOKEN_ALARM", "1000") | ||
| assert run.cost_alarm({"cost_usd": 0.05}, [0.03]) is None # limit max(0.06, 0.01) | ||
| assert run.cost_alarm({"cost_usd": 0.07}, [0.03]) | ||
| assert run.cost_alarm({"tokens_in": 1001}, []) | ||
|
|
||
|
|
||
| class _Task: | ||
| def __init__(self, name): | ||
| self.name = name | ||
| self.PROMPT = "p" | ||
|
|
||
| def setup(self, d): | ||
| pass | ||
|
|
||
| def grade(self, d): | ||
| return True, "ok" | ||
|
|
||
|
|
||
| def _lane(monkeypatch, tmp_path, costs): | ||
| """Run main() over len(costs) fake tasks whose runs cost `costs`, in order.""" | ||
| monkeypatch.setattr(run, "WORK", tmp_path / "work") | ||
| monkeypatch.setattr(run, "TASKS", [_Task(f"t{i}") for i in range(len(costs))]) | ||
| monkeypatch.setitem(sys.modules, "tasks2", types.SimpleNamespace(TASKS2=[])) | ||
| monkeypatch.setitem(sys.modules, "tasks3", types.SimpleNamespace(TASKS3=[])) | ||
| monkeypatch.setenv("RESULTS_DIR", str(tmp_path)) | ||
| monkeypatch.setenv("RESULTS", "lane.jsonl") | ||
| it = iter(costs) | ||
|
|
||
| def fake_molt(d, prompt, log): | ||
| cost, toks = next(it) | ||
| log.write_text("") | ||
| return {"provider_capped": False, "cost_usd": cost, "tokens_in": toks, "turns": 3, "secs": 1} | ||
|
|
||
| monkeypatch.setattr(run, "run_molt", fake_molt) | ||
| return tmp_path / "lane.jsonl" | ||
|
|
||
|
|
||
| def test_lane_stops_with_a_stopped_row_and_nonzero_exit(monkeypatch, tmp_path, capsys): | ||
| out = _lane(monkeypatch, tmp_path, [(0.02, 50_000), (0.03, 60_000), (0.82, 2_680_000), (0.02, 50_000)]) | ||
| with pytest.raises(SystemExit) as e: | ||
| run.main("molt", 1, None) | ||
| assert e.value.code == run.COST_ALARM_EXIT != 0 | ||
| rows = [json.loads(x) for x in out.read_text().splitlines()] | ||
| assert [r.get("task") for r in rows[:3]] == ["t0", "t1", "t2"] # the offending run is recorded | ||
| stop = rows[3] | ||
| assert len(rows) == 4, "the lane ran on after the alarm" | ||
| assert stop["stopped"] is True and stop["reason"] == "cost alarm" and stop["run"] == "t2-molt-0" | ||
| assert stop["cost_usd"] == 0.82 and stop["tokens_in"] == 2_680_000 | ||
| assert stop["log"].endswith("t2-molt-0.log") | ||
| printed = capsys.readouterr().out | ||
| assert "STOPPED: cost alarm on t2-molt-0" in printed and "t2-molt-0.log" in printed | ||
| # The marker is not a run: the scoreboard does not count it. | ||
| assert len(scoreboard.load(out)) == 3 | ||
|
|
||
|
|
||
| def test_an_ordinary_lane_runs_to_the_end(monkeypatch, tmp_path): | ||
| out = _lane(monkeypatch, tmp_path, [(0.02, 50_000), (0.06, 80_000), (0.03, 60_000)]) | ||
| run.main("molt", 1, None) | ||
| rows = [json.loads(x) for x in out.read_text().splitlines()] | ||
| assert len(rows) == 3 and not any(r.get("stopped") for r in rows) | ||
|
|
||
|
|
||
| def test_resume_skips_the_marker_and_keeps_the_median(monkeypatch, tmp_path): | ||
| out = _lane(monkeypatch, tmp_path, [(0.40, 1000)]) | ||
| out.write_text( | ||
| json.dumps({"task": "t9", "agent": "molt", "rep": 0, "cost_usd": 0.10}) + "\n" | ||
| + json.dumps({"stopped": True, "reason": "cost alarm", "run": "t9-molt-0"}) + "\n" | ||
| ) | ||
| # Earlier run's $0.10 is the median: limit max(0.50, 0.10), so $0.40 passes. | ||
| run.main("molt", 1, None) | ||
| assert json.loads(out.read_text().splitlines()[-1])["task"] == "t0" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| # Lean guards | ||
|
|
||
| Maat's first product goal is lean token spend. Two guards keep a cost | ||
| regression from reaching the bill: a CI suite with hard request-size ceilings, | ||
| and a bench alarm that stops a lane on a run that costs far more than usual. | ||
| They exist because of 2026-10-07: malformed tool calls were resent in full on | ||
| every later step, and one task cost 2.68M tokens / $0.82 for a 181-byte file. | ||
| The rules they enforce are the non-regression rules in the loop charter. | ||
|
|
||
| ## The lean-budget suite (`test/lean-budget.test.ts`) | ||
|
|
||
| Each scenario drives the engine with a scripted provider (no model, no | ||
| network) and measures three numbers for the turn: | ||
|
|
||
| - **largest**: the biggest single request body, in characters | ||
| - **total**: all request bodies of the turn added up, in characters | ||
| - **steps**: how many requests the provider received | ||
|
|
||
| | Scenario | Guards against | Measured (largest / total / steps) | Ceiling | | ||
| |---|---|---|---| | ||
| | a. small task: read, write, done | system prompt, tool schemas or per-step framing growing | 7,805 / 22,690 / 3 | 11,700 / 34,000 / 5 | | ||
| | b. one read of a 400 KB file | read_file paging lost; a big file sent whole or carried whole | 41,234 / 130,307 / 4 | 62,000 / 195,000 / 6 | | ||
| | c. the same 40 KB file read 8 times | the repeat pointer lost; every re-read a fresh copy | 40,732 / 115,399 / 9 | 61,000 / 173,000 / 14 | | ||
| | d. 60-step session (default auto-shed) | auto-shed not firing; requests growing every step | 259,597 / 7,262,564 / 61 | 390,000 / 10,900,000 / 92 | | ||
| | e1. 12 malformed calls, 60 KB unparseable arguments | malformed calls resent in full (PR #47); no stop after six | 12,450 / 69,197 / 7 | 18,700 / 104,000 / 11 | | ||
| | e2. 12 malformed `act` calls, 60 KB arguments (batch mode) | the same on the batch path | 9,384 / 48,455 / 7 | 14,100 / 73,000 / 11 | | ||
| | f. bash printing 1 MB | tool-result capping lost; huge output resent each step | 16,537 / 56,108 / 4 | 24,800 / 84,000 / 6 | | ||
|
|
||
| Measured on 2026-10-07 on the PR #47 build. The build before #47 measures e1 | ||
| at 369,044 / 4,070,554 / 13 and e2 at 727,583 / 4,766,054 / 13, so those two | ||
| fail by 20-60x. Each scenario also checks that it really ran (the file, the | ||
| last step, the command's output reached the model), so a scenario that stops | ||
| early cannot pass on small numbers. | ||
|
|
||
| ### Updating ceilings | ||
|
|
||
| Every run prints its measurements: | ||
|
|
||
| lean-budget small largest=7805 total=22690 steps=3 | ||
|
|
||
| To see them: `npx tsc -p tsconfig.test.json && node --test dist-test/test/lean-budget.test.js | grep lean-budget`. | ||
|
|
||
| - A change made a scenario cheaper: set its ceiling to about 1.5x the new | ||
| value, and update the measured comment beside it and this table. | ||
| - A change made a scenario more expensive and fails the suite: that is the | ||
| guard working. Fix the change. Raising a ceiling is allowed only with the | ||
| reason in the commit message and the PR, and the new value is still 1.5x | ||
| what was measured, never "whatever passes". | ||
| - The numbers are deterministic (scripted turns, no clock in the bodies), so | ||
| 1.5x leaves room for small prompt edits, not for noise. | ||
|
|
||
| ## The bench cost alarm (`bench/local/run.py`) | ||
|
|
||
| After each run, run.py reads the run's cost (`spend.costUsd`) and prompt | ||
| tokens (`spend.promptTokens`) from Maat's `job_end` event, and records the cost | ||
| in the row as `cost_usd`. The run trips the alarm when: | ||
|
|
||
| - its cost exceeds **max(5x the lane's running median cost, $0.10)**. The | ||
| median is over the lane's earlier runs that reported a cost (resumed rows | ||
| included; each paired arm is its own lane). With no earlier runs, the limit | ||
| is $0.10. A run with no cost (an unpriced model) is judged on tokens only. | ||
| - or its prompt tokens exceed **1,500,000**. | ||
|
|
||
| On an alarm the offending run's row is written as usual, then a `STOPPED` row: | ||
|
|
||
| {"stopped": true, "reason": "cost alarm", "run": "<tag>", "log": "<path>", | ||
| "detail": "...", "cost_usd": ..., "tokens_in": ..., "lane_median_usd": ..., | ||
| "limit_usd": ..., "limit_tokens": ...} | ||
|
|
||
| run.py prints `STOPPED: cost alarm on <tag>: <detail>` and the log path, and | ||
| exits with code 3. The marker has no `task`/`passed`: resume and the scoreboard | ||
| skip it. Resuming the lane after a look runs the remaining tasks. | ||
|
|
||
| | Env | Default | Meaning | | ||
| |---|---|---| | ||
| | `BENCH_COST_ALARM_X` | 5 | multiple of the lane's running median cost | | ||
| | `BENCH_COST_ALARM_USD` | 0.10 | dollar floor for the cost limit | | ||
| | `BENCH_TOKEN_ALARM` | 1500000 | prompt-token limit for one run | | ||
|
|
||
| Tests: `python3 -m pytest bench/local/tests` (`test_cost_alarm.py`). |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Non-blocking, medium-high confidence: these come only from
job_end. If the backstop kills the run or node crashes, both stayNoneand the alarm never fires. Fall back to the summedstep_summary.spend.promptTokensand the lastsessionCostUsd.