diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ba80e1..d21bb21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## Unreleased +### Added + +- **Lean-budget guards.** `test/lean-budget.test.ts` runs seven scripted + scenarios (small task, 400 KB read, repeated re-reads, a long session that + sheds, malformed calls with huge arguments on the tool and `act` paths, a + command printing 1 MB) with hard ceilings on the largest request, the total + request characters and the steps of the turn; the build before the malformed + excerpt fix fails it by 20-60x. `bench/local/run.py` stops the lane with a + `STOPPED` row (`reason: "cost alarm"`) and exit code 3 when one run costs + more than max(5x the lane's running median, $0.10) or sends more than 1.5M + prompt tokens (`BENCH_COST_ALARM_X`, `BENCH_COST_ALARM_USD`, + `BENCH_TOKEN_ALARM`). See `docs/lean.md`. + ### Changed - **OpenCode runs OpenCode Zen models only.** The OpenCode backend (worker and diff --git a/bench/local/run.py b/bench/local/run.py index a3e5e1b..94d3935 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -17,6 +17,7 @@ import json import os import shutil +import statistics import subprocess import sys import time @@ -148,7 +149,7 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: return { "provider_capped": capped, "secs": round(secs), "turns": steps, "calls": sum(per), "multi": sum(1 for x in per if x > 1), - "tokens_in": spend.get("promptTokens"), "claim": outcome, "timed_out": timed_out, + "tokens_in": spend.get("promptTokens"), "cost_usd": spend.get("costUsd"), "claim": outcome, "timed_out": timed_out, "said_done": (outcome or "").startswith("verified"), # The reviewer's label: a verified claim it did not confirm. "review": review, @@ -159,6 +160,42 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: } +# Per-run cost alarm (reports/loop-charter.md, non-regression rule 1): a run that +# costs more than max(5x the lane's running median cost, $0.10), or sends more +# than 1.5M prompt tokens, stops the lane before the bill grows. 2026-10-07: one +# task cost 2.68M tokens / $0.82 because malformed calls were resent in full. +COST_ALARM_EXIT = 3 + + +def alarm_limits() -> dict: + """The alarm thresholds, each overridable by env.""" + return { + "x": float(os.environ.get("BENCH_COST_ALARM_X") or 5), + "usd": float(os.environ.get("BENCH_COST_ALARM_USD") or 0.10), + "tokens": int(float(os.environ.get("BENCH_TOKEN_ALARM") or 1_500_000)), + } + + +def cost_alarm(row: dict, prior_costs: list[float]) -> dict | None: + """Why this run trips the alarm, or None. `prior_costs` are the lane's earlier runs + (this run excluded): the median is over runs that reported a cost. A run with no + cost (an unpriced model) is judged on its prompt tokens only.""" + lim = alarm_limits() + med = statistics.median(prior_costs) if prior_costs else None + limit_usd = max(lim["x"] * med, lim["usd"]) if med is not None else lim["usd"] + cost, toks = row.get("cost_usd"), row.get("tokens_in") + why = [] + if cost is not None and cost > limit_usd: + why.append(f"cost ${cost:.4f} > ${limit_usd:.4f} (max({lim['x']:g}x lane median " + f"{'n/a' if med is None else f'${med:.4f}'}, ${lim['usd']:g}))") + if toks is not None and toks > lim["tokens"]: + why.append(f"prompt tokens {toks:,} > {lim['tokens']:,}") + if not why: + return None + return {"detail": "; ".join(why), "cost_usd": cost, "tokens_in": toks, "lane_median_usd": med, + "limit_usd": round(limit_usd, 6), "limit_tokens": lim["tokens"]} + + def claims_done(text: str) -> bool: """ Whether a final message says the work is done, by one fixed rule: a hedge @@ -211,10 +248,15 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: arms = parse_arms(os.environ.get("ARMS")) out = Path(os.environ.get("RESULTS_DIR", HERE)) / os.environ.get("RESULTS", f"results-{which}-x{repeats}.jsonl") done = set() + lane_costs: dict = {} # arm -> costs of its runs so far, for the cost alarm's median if out.exists(): # resume: skip runs already recorded for line in out.read_text().splitlines(): r = json.loads(line) + if r.get("stopped"): # a STOPPED marker, not a run + continue done.add((r["task"], r["agent"], r["rep"], r.get("arm"))) + if r.get("cost_usd") is not None: + lane_costs.setdefault(r.get("arm"), []).append(r["cost_usd"]) for rep in range(repeats): for T in tasks: for arm, arm_env in arms: # paired: arm A then arm B on this task before the next task @@ -251,6 +293,23 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: with out.open("a") as f: f.write(json.dumps(r) + "\n") print(json.dumps({k: r[k] for k in ("task", "agent", "arm", "rep", "passed", "said_done", "turns", "secs") if k in r}), flush=True) + costs = lane_costs.setdefault(arm, []) + alarm = cost_alarm(r, costs) + if alarm: + stop_lane(out, tag, WORK / f"{tag}.log", alarm, arm) + if r.get("cost_usd") is not None: + costs.append(r["cost_usd"]) + + +def stop_lane(out: Path, tag: str, log: Path, alarm: dict, arm: str | None) -> None: + """Record a STOPPED row (no `passed`: not a run) and end the lane non-zero.""" + row = {"stopped": True, "reason": "cost alarm", "run": tag, "log": str(log), **alarm} + if arm: + row["arm"] = arm + with out.open("a") as f: + f.write(json.dumps(row) + "\n") + print(f"STOPPED: cost alarm on {tag}: {alarm['detail']}\n log: {log}", flush=True) + sys.exit(COST_ALARM_EXIT) diff --git a/bench/local/scoreboard.py b/bench/local/scoreboard.py index e477a4d..560149a 100644 --- a/bench/local/scoreboard.py +++ b/bench/local/scoreboard.py @@ -40,7 +40,9 @@ def load(path: str | Path) -> list[dict]: for line in Path(path).read_text().splitlines(): line = line.strip() if line.startswith("{"): - rows.append(json.loads(line)) + r = json.loads(line) + if not r.get("stopped"): # run.py's STOPPED marker (cost alarm) is not a run + rows.append(r) return rows diff --git a/bench/local/tests/test_cost_alarm.py b/bench/local/tests/test_cost_alarm.py new file mode 100644 index 0000000..98c108c --- /dev/null +++ b/bench/local/tests/test_cost_alarm.py @@ -0,0 +1,128 @@ +"""The per-run cost alarm: a run far over the lane's usual cost stops the lane. + +2026-10-07: one task cost 2.68M tokens / $0.82 because malformed tool calls were +resent in full every step. These tests pin the thresholds and that the lane +stops with a STOPPED row and a non-zero exit instead of running on. +""" +import json +import sys +import types +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import run # noqa: E402 +import scoreboard # noqa: E402 + + +@pytest.fixture(autouse=True) +def _no_env_overrides(monkeypatch): + for k in ("BENCH_COST_ALARM_X", "BENCH_COST_ALARM_USD", "BENCH_TOKEN_ALARM", "ARMS", "BENCH_TASKS", "MAAT_BUILD"): + monkeypatch.delenv(k, raising=False) + + +def test_floor_applies_with_no_history(): + assert run.cost_alarm({"cost_usd": 0.09, "tokens_in": 1000}, []) is None + a = run.cost_alarm({"cost_usd": 0.11, "tokens_in": 1000}, []) + assert a and a["limit_usd"] == 0.10 and "cost $0.1100" in a["detail"] + + +def test_five_times_the_median_when_that_is_above_the_floor(): + prior = [0.03, 0.04, 0.05] # median 0.04 -> limit 0.20 + assert run.cost_alarm({"cost_usd": 0.19}, prior) is None + a = run.cost_alarm({"cost_usd": 0.21}, prior) + assert a and a["limit_usd"] == pytest.approx(0.20) and a["lane_median_usd"] == 0.04 + + +def test_floor_wins_over_a_tiny_median(): + # A cheap lane (median $0.002) must not alarm on an ordinary $0.05 run. + assert run.cost_alarm({"cost_usd": 0.05}, [0.002, 0.002, 0.003]) is None + + +def test_the_regression_that_motivated_it_trips_both_alarms(): + a = run.cost_alarm({"cost_usd": 0.82, "tokens_in": 2_680_000}, [0.02, 0.03, 0.025]) + assert a and "cost $0.8200" in a["detail"] and "prompt tokens 2,680,000 > 1,500,000" in a["detail"] + + +def test_token_alarm_without_a_price(): + # Free models report no cost: prompt tokens are the only signal. + assert run.cost_alarm({"cost_usd": None, "tokens_in": 1_400_000}, []) is None + a = run.cost_alarm({"cost_usd": None, "tokens_in": 1_600_000}, []) + assert a and a["cost_usd"] is None and a["limit_tokens"] == 1_500_000 + + +def test_env_overrides(monkeypatch): + monkeypatch.setenv("BENCH_COST_ALARM_X", "2") + monkeypatch.setenv("BENCH_COST_ALARM_USD", "0.01") + monkeypatch.setenv("BENCH_TOKEN_ALARM", "1000") + assert run.cost_alarm({"cost_usd": 0.05}, [0.03]) is None # limit max(0.06, 0.01) + assert run.cost_alarm({"cost_usd": 0.07}, [0.03]) + assert run.cost_alarm({"tokens_in": 1001}, []) + + +class _Task: + def __init__(self, name): + self.name = name + self.PROMPT = "p" + + def setup(self, d): + pass + + def grade(self, d): + return True, "ok" + + +def _lane(monkeypatch, tmp_path, costs): + """Run main() over len(costs) fake tasks whose runs cost `costs`, in order.""" + monkeypatch.setattr(run, "WORK", tmp_path / "work") + monkeypatch.setattr(run, "TASKS", [_Task(f"t{i}") for i in range(len(costs))]) + monkeypatch.setitem(sys.modules, "tasks2", types.SimpleNamespace(TASKS2=[])) + monkeypatch.setitem(sys.modules, "tasks3", types.SimpleNamespace(TASKS3=[])) + monkeypatch.setenv("RESULTS_DIR", str(tmp_path)) + monkeypatch.setenv("RESULTS", "lane.jsonl") + it = iter(costs) + + def fake_molt(d, prompt, log): + cost, toks = next(it) + log.write_text("") + return {"provider_capped": False, "cost_usd": cost, "tokens_in": toks, "turns": 3, "secs": 1} + + monkeypatch.setattr(run, "run_molt", fake_molt) + return tmp_path / "lane.jsonl" + + +def test_lane_stops_with_a_stopped_row_and_nonzero_exit(monkeypatch, tmp_path, capsys): + out = _lane(monkeypatch, tmp_path, [(0.02, 50_000), (0.03, 60_000), (0.82, 2_680_000), (0.02, 50_000)]) + with pytest.raises(SystemExit) as e: + run.main("molt", 1, None) + assert e.value.code == run.COST_ALARM_EXIT != 0 + rows = [json.loads(x) for x in out.read_text().splitlines()] + assert [r.get("task") for r in rows[:3]] == ["t0", "t1", "t2"] # the offending run is recorded + stop = rows[3] + assert len(rows) == 4, "the lane ran on after the alarm" + assert stop["stopped"] is True and stop["reason"] == "cost alarm" and stop["run"] == "t2-molt-0" + assert stop["cost_usd"] == 0.82 and stop["tokens_in"] == 2_680_000 + assert stop["log"].endswith("t2-molt-0.log") + printed = capsys.readouterr().out + assert "STOPPED: cost alarm on t2-molt-0" in printed and "t2-molt-0.log" in printed + # The marker is not a run: the scoreboard does not count it. + assert len(scoreboard.load(out)) == 3 + + +def test_an_ordinary_lane_runs_to_the_end(monkeypatch, tmp_path): + out = _lane(monkeypatch, tmp_path, [(0.02, 50_000), (0.06, 80_000), (0.03, 60_000)]) + run.main("molt", 1, None) + rows = [json.loads(x) for x in out.read_text().splitlines()] + assert len(rows) == 3 and not any(r.get("stopped") for r in rows) + + +def test_resume_skips_the_marker_and_keeps_the_median(monkeypatch, tmp_path): + out = _lane(monkeypatch, tmp_path, [(0.40, 1000)]) + out.write_text( + json.dumps({"task": "t9", "agent": "molt", "rep": 0, "cost_usd": 0.10}) + "\n" + + json.dumps({"stopped": True, "reason": "cost alarm", "run": "t9-molt-0"}) + "\n" + ) + # Earlier run's $0.10 is the median: limit max(0.50, 0.10), so $0.40 passes. + run.main("molt", 1, None) + assert json.loads(out.read_text().splitlines()[-1])["task"] == "t0" diff --git a/docs/lean.md b/docs/lean.md new file mode 100644 index 0000000..f7a8667 --- /dev/null +++ b/docs/lean.md @@ -0,0 +1,80 @@ +# Lean guards + +Maat's first product goal is lean token spend. Two guards keep a cost +regression from reaching the bill: a CI suite with hard request-size ceilings, +and a bench alarm that stops a lane on a run that costs far more than usual. +They exist because of 2026-10-07: malformed tool calls were resent in full on +every later step, and one task cost 2.68M tokens / $0.82 for a 181-byte file. +The rules they enforce are the non-regression rules in the loop charter. + +## The lean-budget suite (`test/lean-budget.test.ts`) + +Each scenario drives the engine with a scripted provider (no model, no +network) and measures three numbers for the turn: + +- **largest**: the biggest single request body, in characters +- **total**: all request bodies of the turn added up, in characters +- **steps**: how many requests the provider received + +| Scenario | Guards against | Measured (largest / total / steps) | Ceiling | +|---|---|---|---| +| a. small task: read, write, done | system prompt, tool schemas or per-step framing growing | 7,805 / 22,690 / 3 | 11,700 / 34,000 / 5 | +| b. one read of a 400 KB file | read_file paging lost; a big file sent whole or carried whole | 41,234 / 130,307 / 4 | 62,000 / 195,000 / 6 | +| c. the same 40 KB file read 8 times | the repeat pointer lost; every re-read a fresh copy | 40,732 / 115,399 / 9 | 61,000 / 173,000 / 14 | +| d. 60-step session (default auto-shed) | auto-shed not firing; requests growing every step | 259,597 / 7,262,564 / 61 | 390,000 / 10,900,000 / 92 | +| e1. 12 malformed calls, 60 KB unparseable arguments | malformed calls resent in full (PR #47); no stop after six | 12,450 / 69,197 / 7 | 18,700 / 104,000 / 11 | +| e2. 12 malformed `act` calls, 60 KB arguments (batch mode) | the same on the batch path | 9,384 / 48,455 / 7 | 14,100 / 73,000 / 11 | +| f. bash printing 1 MB | tool-result capping lost; huge output resent each step | 16,537 / 56,108 / 4 | 24,800 / 84,000 / 6 | + +Measured on 2026-10-07 on the PR #47 build. The build before #47 measures e1 +at 369,044 / 4,070,554 / 13 and e2 at 727,583 / 4,766,054 / 13, so those two +fail by 20-60x. Each scenario also checks that it really ran (the file, the +last step, the command's output reached the model), so a scenario that stops +early cannot pass on small numbers. + +### Updating ceilings + +Every run prints its measurements: + + lean-budget small largest=7805 total=22690 steps=3 + +To see them: `npx tsc -p tsconfig.test.json && node --test dist-test/test/lean-budget.test.js | grep lean-budget`. + +- A change made a scenario cheaper: set its ceiling to about 1.5x the new + value, and update the measured comment beside it and this table. +- A change made a scenario more expensive and fails the suite: that is the + guard working. Fix the change. Raising a ceiling is allowed only with the + reason in the commit message and the PR, and the new value is still 1.5x + what was measured, never "whatever passes". +- The numbers are deterministic (scripted turns, no clock in the bodies), so + 1.5x leaves room for small prompt edits, not for noise. + +## The bench cost alarm (`bench/local/run.py`) + +After each run, run.py reads the run's cost (`spend.costUsd`) and prompt +tokens (`spend.promptTokens`) from Maat's `job_end` event, and records the cost +in the row as `cost_usd`. The run trips the alarm when: + +- its cost exceeds **max(5x the lane's running median cost, $0.10)**. The + median is over the lane's earlier runs that reported a cost (resumed rows + included; each paired arm is its own lane). With no earlier runs, the limit + is $0.10. A run with no cost (an unpriced model) is judged on tokens only. +- or its prompt tokens exceed **1,500,000**. + +On an alarm the offending run's row is written as usual, then a `STOPPED` row: + + {"stopped": true, "reason": "cost alarm", "run": "", "log": "", + "detail": "...", "cost_usd": ..., "tokens_in": ..., "lane_median_usd": ..., + "limit_usd": ..., "limit_tokens": ...} + +run.py prints `STOPPED: cost alarm on : ` and the log path, and +exits with code 3. The marker has no `task`/`passed`: resume and the scoreboard +skip it. Resuming the lane after a look runs the remaining tasks. + +| Env | Default | Meaning | +|---|---|---| +| `BENCH_COST_ALARM_X` | 5 | multiple of the lane's running median cost | +| `BENCH_COST_ALARM_USD` | 0.10 | dollar floor for the cost limit | +| `BENCH_TOKEN_ALARM` | 1500000 | prompt-token limit for one run | + +Tests: `python3 -m pytest bench/local/tests` (`test_cost_alarm.py`). diff --git a/test/lean-budget.test.ts b/test/lean-budget.test.ts new file mode 100644 index 0000000..c607a74 --- /dev/null +++ b/test/lean-budget.test.ts @@ -0,0 +1,237 @@ +import assert from "node:assert/strict"; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { Engine, type EngineConfig } from "../src/engine.js"; +import type { EngineEvent } from "../src/types.js"; +import { allowAll, drain, scriptedProvider, workspace } from "./helpers.js"; + +/** + * Lean budget: hard ceilings on what one turn sends to the provider. + * + * 2026-10-07: malformed tool calls were resent in full on every later step, + * and one task cost 2.68M tokens / $0.82 for a 181-byte file. Every test in + * this file would have gone red on that build. Each scenario is scripted (no + * model, no network) and measures three things: + * + * largest the biggest single request body, in characters + * total every request body of the turn added up, in characters + * steps how many times the provider was asked + * + * Ceilings sit at about 1.5x the value measured when they were set, so noise + * never trips them and a resend-everything regression always does. Each run + * prints its numbers (`lean-budget ...`); when a change makes a + * scenario cheaper, tighten its ceiling to 1.5x the new value. Raising one + * needs a reason in the commit message. See docs/lean.md. + */ + +interface Ceiling { + largest: number; + total: number; + steps: number; +} + +interface Measured extends Ceiling { + events: EngineEvent[]; + /** Every request body, so a test can prove its scenario really ran. */ + sent: string; +} + +/** One scripted turn: `raw` sends tool-call arguments exactly as given, unparseable or not. */ +type Turn = + | { text: string } + | { calls: { name: string; args?: Record; raw?: string }[] }; + +/** + * scriptedProvider, plus raw argument strings: a real model's malformed call + * arrives as text that is not JSON, which JSON.stringify can never produce. + */ +function rawProvider(turns: Turn[]): { fetchFn: typeof fetch; bodies: string[] } { + const bodies: string[] = []; + let n = 0; + let id = 0; + const fetchFn = (async (_url: string, init?: RequestInit) => { + bodies.push(String(init?.body ?? "")); + const turn = turns[Math.min(n, turns.length - 1)]!; + n += 1; + const message = + "text" in turn + ? { role: "assistant", content: turn.text } + : { + role: "assistant", + content: null, + tool_calls: turn.calls.map((c) => ({ + id: `call_${++id}`, + type: "function", + function: { name: c.name, arguments: c.raw ?? JSON.stringify(c.args ?? {}) }, + })), + }; + return { + ok: true, + status: 200, + json: async () => ({ choices: [{ message }], usage: { prompt_tokens: 100, completion_tokens: 20 } }), + text: async () => "", + } as unknown as Response; + }) as unknown as typeof fetch; + return { fetchFn, bodies }; +} + +async function measure( + name: string, + turns: Turn[], + opts: { setup?: (dir: string) => void; cfg?: Partial; task?: string } = {}, +): Promise { + const ws = workspace(); + try { + opts.setup?.(ws.dir); + const raw = turns.some((t) => "calls" in t && t.calls.some((c) => c.raw !== undefined)); + const provider = raw ? rawProvider(turns) : scriptedProvider(turns as never); + const engine = new Engine({ + baseUrl: "http://p.test/v1", + model: "m", + cwd: ws.dir, + fetchFn: provider.fetchFn, + bar: null, + stream: false, + autonomy: "high", + ...opts.cfg, + }); + const events = await drain(engine.run(opts.task ?? "do the task", allowAll)); + const sizes = provider.bodies.map((b) => b.length); + const m = { + largest: Math.max(0, ...sizes), + total: sizes.reduce((a, b) => a + b, 0), + steps: sizes.length, + events, + sent: provider.bodies.join("\n"), + }; + console.log(`lean-budget ${name} largest=${m.largest} total=${m.total} steps=${m.steps}`); + return m; + } finally { + ws.cleanup(); + } +} + +function within(name: string, m: Measured, c: Ceiling): void { + assert.ok(m.largest <= c.largest, `${name}: largest request ${m.largest} chars > ceiling ${c.largest}`); + assert.ok(m.total <= c.total, `${name}: total request chars ${m.total} > ceiling ${c.total}`); + assert.ok(m.steps <= c.steps, `${name}: ${m.steps} provider requests > ceiling ${c.steps}`); +} + +/** + * About 1.5x the values measured on 2026-10-07 (the PR #47 build). Measured + * values in the comments; the build before #47 measured malformed-raw at + * 369,044 largest / 4,070,554 total / 13 steps and malformed-act at 727,583 / + * 4,766,054 / 13, which these ceilings fail by 20-60x. + */ +const CEILINGS = { + small: { largest: 11_700, total: 34_000, steps: 5 }, // 7,805 / 22,690 / 3 + bigRead: { largest: 62_000, total: 195_000, steps: 6 }, // 41,234 / 130,307 / 4 + reread: { largest: 61_000, total: 173_000, steps: 14 }, // 40,732 / 115,399 / 9 + longShed: { largest: 390_000, total: 10_900_000, steps: 92 }, // 259,597 / 7,262,564 / 61 + malformedRaw: { largest: 18_700, total: 104_000, steps: 11 }, // 12,450 / 69,197 / 7 + malformedAct: { largest: 14_100, total: 73_000, steps: 11 }, // 9,384 / 48,455 / 7 + hugeOutput: { largest: 24_800, total: 84_000, steps: 6 }, // 16,537 / 56,108 / 4 +} satisfies Record; + +const done = { text: "Done." }; + +describe("lean budget: request-size and step ceilings", () => { + // Guards: the baseline. System prompt, tool schemas or per-step framing + // growing until an ordinary three-step task costs noticeably more. + it("a: a small task (read, write, done)", async () => { + const m = await measure("small", [ + { calls: [{ name: "read_file", args: { path: "in.txt" } }] }, + { calls: [{ name: "write_file", args: { path: "out.txt", content: "HELLO\n" } }] }, + done, + ], { setup: (d) => writeFileSync(join(d, "in.txt"), "hello\n"), task: "uppercase in.txt into out.txt" }); + within("small", m, CEILINGS.small); + }); + + // Guards: read_file paging. A 400 KB file must arrive in bounded parts, + // never whole in one request, and must not be carried whole on every step. + it("b: one read of a 400 KB file, then work", async () => { + const m = await measure("big-read", [ + { calls: [{ name: "read_file", args: { path: "big.txt" } }] }, + { calls: [{ name: "write_file", args: { path: "out.txt", content: "summary\n" } }] }, + { calls: [{ name: "bash", args: { command: "wc -c out.txt" } }] }, + done, + ], { setup: (d) => writeFileSync(join(d, "big.txt"), bigText(400_000)), task: "summarise big.txt" }); + assert.ok(/line 100 abcdefghij/.test(m.sent), "the file never reached the model"); + within("big-read", m, CEILINGS.bigRead); + }); + + // Guards: the repeat pointer. Re-reading the same file must come back as a + // short "already shown" note, not a fresh copy on every step. + it("c: the same file read again and again", async () => { + const reread = { calls: [{ name: "read_file", args: { path: "mid.txt" } }] }; + const m = await measure("re-read", [ + ...Array.from({ length: 8 }, () => reread), + done, + ], { setup: (d) => writeFileSync(join(d, "mid.txt"), bigText(40_000)), task: "read mid.txt" }); + assert.ok(/line 100 abcdefghij/.test(m.sent), "the file never reached the model"); + within("re-read", m, CEILINGS.reread); + }); + + // Guards: auto-shed. A long session must shed its history at the default + // threshold, so requests plateau instead of growing with every step. + it("d: a long session sheds and stays bounded", async () => { + const m = await measure("long-shed", [ + ...Array.from({ length: 60 }, (_, i) => ({ + calls: [{ name: "bash", args: { command: `printf '%.0sstep ${i} output line\\n' $(seq 1 300)` } }], + })), + done, + ], { cfg: { maxSteps: 0 }, task: "a long job" }); + assert.ok(m.events.some((e) => e.kind === "shed"), "the long session never shed"); + assert.ok(/step 59 output line/.test(m.sent), "the session ended before its last step"); + within("long-shed", m, CEILINGS.longShed); + }); + + // Guards: the 2026-10-07 regression itself (PR #47). Malformed calls with + // huge arguments must go back as an excerpt and stop the turn at the limit. + it("e1: malformed calls with huge unparseable arguments", async () => { + const broken = `{"path": "out.txt", "content": "${"x".repeat(60_000)}`; + const m = await measure("malformed-raw", [ + ...Array.from({ length: 12 }, () => ({ calls: [{ name: "write_file", raw: broken }] })), + done, + ]); + within("malformed-raw", m, CEILINGS.malformedRaw); + }); + + // Guards: the same regression on the batch path, where it was first seen: + // an `act` whose actions cannot be read, carrying a huge payload. + it("e2: malformed act calls with huge arguments (batch mode)", async () => { + const m = await measure("malformed-act", [ + ...Array.from({ length: 12 }, () => ({ + calls: [{ name: "act", args: { analysis: "writing", actions: "y".repeat(60_000) } }], + })), + { calls: [{ name: "act", args: { analysis: "done", actions: [] } }] }, + ], { cfg: { batch: true } }); + within("malformed-act", m, CEILINGS.malformedAct); + }); + + // Guards: tool-result capping. A command that prints 1 MB must reach the + // model as a bounded result, and later steps must not resend all of it. + it("f: a command that prints 1 MB", async () => { + const m = await measure("huge-output", [ + { calls: [{ name: "bash", args: { command: "yes 'build output line 0123456789' | head -c 1048576" } }] }, + { calls: [{ name: "write_file", args: { path: "out.txt", content: "ok\n" } }] }, + { calls: [{ name: "bash", args: { command: "cat out.txt" } }] }, + done, + ]); + assert.ok(/(build output line 0123456789\\n){20}/.test(m.sent), "the command's output never reached the model"); + within("huge-output", m, CEILINGS.hugeOutput); + }); +}); + +/** Distinct numbered lines, so nothing upstream can compress or dedupe them. */ +function bigText(bytes: number): string { + const lines: string[] = []; + let size = 0; + for (let i = 0; size < bytes; i++) { + const line = `line ${i} ${"abcdefghij".repeat(6)}`; + lines.push(line); + size += line.length + 1; + } + return lines.join("\n") + "\n"; +}