diff --git a/bench/local/audit_replay.py b/bench/local/audit_replay.py new file mode 100644 index 0000000..59e9015 --- /dev/null +++ b/bench/local/audit_replay.py @@ -0,0 +1,245 @@ +"""What the post-work audit (--post-work-audit) would have done on 2026-10-07's runs. + + python3 audit_replay.py --image maat-bench:-u [--jobs 4] [--limit N] [LANE_PREFIX ...] + python3 audit_replay.py --report-only [LANE_PREFIX ...] + +The audit (src/post-audit.ts): when a claimed run is not verified by the checks +sealed before the work, an independent judge drafts checks from the task text +and an interface view of the finished work (names, signatures, usage; never +the transcript, the claim or the outputs). A check counts only if it quotes the +task for its expected value, passes on the work, fails on the pre-work copy, +fails on a mutant of the changed code that still runs, and clears the lints. + +Population: every saved -molt-- tree from today's lanes whose +grader row exists and whose claim was "unverified" (the outcome the audit +runs on: the sealed checks passed without earning "verified", only drafted +checks refused the claim, or nothing checked it), not timed out and not +stopped by the clock. "verified", "not proven", "error" and "stopped" runs are +out of scope: the audit never runs on them. + +Each run is replayed in its own throwaway container (the bench image, so the +tools are the bench's and port 8080 is free) with the PR's build installed: +the saved tree (minus .maat/) is the work, the task's own setup() rebuilds the +pre-work tree, and the judge is asked live. Results are cached per run under +~/.cache/maat-bench/audit-replay/, so a rerun only asks for what is missing. + +Judged arms (arm != self: a separate judge, qwen3-coder-30b-a3b, sealed the +checks) are the population the flag applies to as run. The self arm had no +judge, so the audit would have been skipped; it is replayed with the same judge +as a what-if and reported apart. + +Caveats, printed with the numbers: the judge is OpenRouter's +qwen/qwen3-coder-30b-a3b-instruct (the bench used the same model on the NUC, +a different quantisation, and is busy with other lanes); the saved trees are +the state after grading; setup() is re-run for the pre-work tree; one judge +sample per run. +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from collections import Counter, defaultdict +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +HERE = Path(__file__).resolve().parent +CACHE = Path.home() / ".cache/maat-bench" +OUT = Path(os.environ.get("REPLAY_OUT") or CACHE / "audit-replay") +JUDGE_URL = os.environ.get("REPLAY_JUDGE_URL") or "https://openrouter.ai/api/v1" +# The bench's judge (qwen3-coder-30b-a3b on the NUC) by default; REPLAY_JUDGE_MODEL for a what-if. +JUDGE_MODEL = os.environ.get("REPLAY_JUDGE_MODEL") or "qwen/qwen3-coder-30b-a3b-instruct" + + +# ---------------------------------------------------------------- inside the container + +def inside() -> int: + """Read one entry on stdin, rebuild its pre-work tree, run the audit, print the report.""" + sys.path.insert(0, str(HERE)) + from tasks import TASKS # noqa: PLC0415 + from tasks2 import TASKS2 # noqa: PLC0415 + from tasks3 import TASKS3 # noqa: PLC0415 + by = {T.name: T for T in TASKS + TASKS2 + TASKS3} + e = json.loads(sys.stdin.read()) + T = by[e["task"]] + root = Path(tempfile.mkdtemp(prefix="audit-")) + pre = root / "pre" / e["tree_name"] + work = root / "work" / e["tree_name"] + pre.mkdir(parents=True) + T.setup(pre) + shutil.copytree("/tree", work, symlinks=True, ignore=shutil.ignore_patterns(".maat", ".molt")) + inp = { + "task": T.PROMPT, + "workDir": str(work), + "preWorkDir": str(pre), + "judge": {"baseUrl": e["judge_url"], "apiKey": os.environ.get("JUDGE_KEY") or None, "model": e["judge_model"]}, + "reasoningEffort": "none", + } + p = subprocess.run(["node", str(HERE / "audit_replay_driver.mjs")], input=json.dumps(inp), capture_output=True, text=True, timeout=600) + if p.returncode != 0: + print(json.dumps({"error": f"driver exited {p.returncode}: {p.stderr[-800:]}"})) + return 0 + print(p.stdout.strip().splitlines()[-1]) + return 0 + + +# ---------------------------------------------------------------- on the host + +def population(prefixes: list[str]) -> tuple[list[dict], Counter]: + sys.path.insert(0, str(HERE)) + import discriminating_replay as dr # noqa: PLC0415 + lanes = sorted(p for p in dr.CW.iterdir() if p.is_dir() and any(re.fullmatch(rf"{re.escape(x)}-\d+", p.name) for x in prefixes)) + out: list[dict] = [] + other: Counter = Counter() + for lane in lanes: + rf = dr.results_file(lane) + if not rf: + continue + rows = dr.grader_rows(rf) + for tree in sorted(lane.iterdir()): + m = dr.RUN_DIR.match(tree.name) + if not m or not tree.is_dir(): + continue + row = rows.get((m["task"], m["arm"], int(m["rep"]))) + if not row: + continue + claim = row.get("claim") or "" + judged = m["arm"] != "self" + if claim.startswith("verified"): + other[("verified", judged, bool(row["passed"]))] += 1 + continue + if not claim.startswith("unverified") or row.get("timed_out") or row.get("endedBy") or row.get("deadline"): + other[("out of scope", judged, bool(row["passed"]))] += 1 + continue + out.append({"id": f"{lane.name}/{tree.name}", "lane": lane.name, "task": m["task"], "arm": m["arm"], "judged": judged, + "tree": str(tree), "tree_name": tree.name, "passed": bool(row["passed"]), "claim": claim, "tier": row.get("tier")}) + return out, other + + +def cached(e: dict) -> Path: + return OUT / (e["id"].replace("/", "__") + ".json") + + +def openrouter_key() -> str: + p = subprocess.run(["node", "-e", "import('%s').then(m=>process.stdout.write(m.readAuth().openrouter||''))" + % (Path.home() / "Documents/molt-desktop/dist/providers.js")], capture_output=True, text=True) + return p.stdout.strip() + + +def replay_one(e: dict, image: str, envf: str) -> dict: + entry = {"task": e["task"], "tree_name": e["tree_name"], "judge_url": JUDGE_URL, "judge_model": JUDGE_MODEL} + cmd = ["docker", "run", "--rm", "-i", "--env-file", envf, "-e", "PYTHONDONTWRITEBYTECODE=1", + "-v", f"{HERE}:/bench:ro", "-v", f"{e['tree']}:/tree:ro", + # A dist/ to test in place of the image's (development only; the reported numbers use the image). + *(["-v", f"{os.environ['REPLAY_DIST']}:/usr/local/lib/node_modules/@solvyx/molt/dist:ro"] if os.environ.get("REPLAY_DIST") else []), + image, + "sh", "-c", "export MOLT_DIST_ABS=$(npm root -g)/@solvyx/molt/dist; python3 /bench/audit_replay.py --inside"] + try: + p = subprocess.run(cmd, input=json.dumps(entry), capture_output=True, text=True, timeout=900) + line = (p.stdout.strip().splitlines() or [""])[-1] + r = json.loads(line) if line.startswith("{") else {"error": f"no report (exit {p.returncode}): {p.stderr[-500:]}"} + except subprocess.TimeoutExpired: + r = {"error": "the replay timed out"} + cached(e).write_text(json.dumps(r)) + return r + + +def report(pop: list[dict], other: Counter) -> None: + rows = [] + for e in pop: + f = cached(e) + if f.exists(): + rows.append((e, json.loads(f.read_text()))) + print(f"replayed {len(rows)} of {len(pop)} eligible runs (judge {JUDGE_MODEL})\n") + for judged, label in ((True, "JUDGED ARMS (the flag applies as run)"), (False, "SELF ARM (what-if: no judge was set, so the audit would have been skipped)")): + sel = [(e, r) for e, r in rows if e["judged"] == judged] + if not sel: + continue + ok = [(e, r) for e, r in sel if not r.get("error")] + err = [(e, r) for e, r in sel if r.get("error")] + p_all = [(e, r) for e, r in ok if e["passed"]] + f_all = [(e, r) for e, r in ok if not e["passed"]] + p_v = [e for e, r in p_all if r.get("accepted")] + f_v = [e for e, r in f_all if r.get("accepted")] + print(f"== {label}") + print(f" eligible (claim unverified): {len(sel)} replay errors: {len(err)}") + print(f" grader PASSED, Maat unverified: {len(p_all)} -> audit verifies {len(p_v)} (recall {pct(len(p_v), len(p_all))})") + print(f" grader FAILED, Maat unverified: {len(f_all)} -> audit verifies {len(f_v)} (false verifieds)") + print(f" precision of the audit tier: {len(p_v)}/{len(p_v) + len(f_v)} = {pct(len(p_v), len(p_v) + len(f_v))}") + vr = other[("verified", judged, True)] + vw = other[("verified", judged, False)] + print(f" pre-work 'verified' on the same lanes: {vr}/{vr + vw} right; with the audit: " + f"{vr + len(p_v)}/{vr + vw + len(p_v) + len(f_v)} right ({pct(vr + len(p_v), vr + vw + len(p_v) + len(f_v))})") + drafted = sum(r.get("drafted", 0) for _, r in ok) + dropped = sum(len(r.get("dropped", [])) for _, r in ok) + gates = Counter(c.get("rule") or "accepted" for _, r in ok for c in r.get("checks", [])) + print(f" checks drafted {drafted}, dropped ungrounded/malformed {dropped}, gated {sum(gates.values())}:") + for k, v in gates.most_common(): + print(f" {k:24} {v}") + no_mut = sum(1 for _, r in ok if r.get("drafted") and not r.get("mutants")) + print(f" runs where no mutant could be made: {no_mut}") + if f_v: + print(" WRONG audit verifieds:") + for e, r in f_all: + if r.get("accepted"): + acc = [c for c in r["checks"] if c.get("accepted")] + for c in acc: + print(f" {e['id']} {c['name']}: {c['run'][:150]} quote: {c['quote'][:80]!r}") + by_task = defaultdict(lambda: [0, 0]) + for e, r in p_all: + by_task[e["task"]][0] += 1 + by_task[e["task"]][1] += bool(r.get("accepted")) + print(" recall by task (verified/passing-unverified): " + ", ".join(f"{t} {v}/{n}" for t, (n, v) in sorted(by_task.items()))) + if err: + print(" errors: " + "; ".join(f"{e['id']}: {r['error'][:100]}" for e, r in err[:8])) + print() + + +def pct(a: int, b: int) -> str: + return f"{100 * a / b:.0f}%" if b else "n/a" + + +def main(argv: list[str]) -> int: + if argv[:1] == ["--inside"]: + return inside() + ap = argparse.ArgumentParser() + ap.add_argument("--image") + ap.add_argument("--jobs", type=int, default=4) + ap.add_argument("--limit", type=int) + ap.add_argument("--report-only", action="store_true") + ap.add_argument("--judged-only", action="store_true") + ap.add_argument("prefixes", nargs="*") + a = ap.parse_args(argv) + sys.path.insert(0, str(HERE)) + import discriminating_replay as dr # noqa: PLC0415 + pop, other = population(a.prefixes or dr.TODAY) + OUT.mkdir(parents=True, exist_ok=True) + if not a.report_only: + if not a.image: + print("--image maat-bench:-u is required (build it from the PR's tarball)", file=sys.stderr) + return 2 + todo = [e for e in pop if not cached(e).exists() and (e["judged"] or not a.judged_only)] + todo.sort(key=lambda e: (not e["judged"], e["id"])) + if a.limit: + todo = todo[: a.limit] + fd, envf = tempfile.mkstemp() + os.chmod(envf, 0o600) + with os.fdopen(fd, "w") as f: + f.write(f"JUDGE_KEY={openrouter_key()}\n") + try: + with ThreadPoolExecutor(a.jobs) as ex: + for i, r in enumerate(ex.map(lambda e: replay_one(e, a.image, envf), todo)): + print(f"[{i + 1}/{len(todo)}] {'error' if r.get('error') else len(r.get('accepted', []))}", file=sys.stderr, flush=True) + finally: + os.unlink(envf) + report(pop, other) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/bench/local/audit_replay_driver.mjs b/bench/local/audit_replay_driver.mjs new file mode 100644 index 0000000..b999f67 --- /dev/null +++ b/bench/local/audit_replay_driver.mjs @@ -0,0 +1,21 @@ +// One post-work audit (src/post-audit.ts) on a saved tree, for audit_replay.py. +// stdin: {"task", "workDir", "preWorkDir", "judge": {"baseUrl", "apiKey", "model"}, "reasoningEffort"?} +// stdout: the AuditReport as one JSON line. +const dist = process.env.MOLT_DIST_ABS; +if (!dist) { + console.error("MOLT_DIST_ABS is not set"); + process.exit(2); +} +const { postWorkAudit } = await import(`${dist}/post-audit.js`); +let raw = ""; +for await (const chunk of process.stdin) raw += chunk; +const input = JSON.parse(raw); +const report = await postWorkAudit({ + task: input.task, + workDir: input.workDir, + preWorkDir: input.preWorkDir, + judge: input.judge, + ...(input.reasoningEffort ? { reasoningEffort: input.reasoningEffort } : {}), + deadlineAt: Date.now() + 240_000, +}); +process.stdout.write(`${JSON.stringify(report)}\n`); diff --git a/bench/local/run.py b/bench/local/run.py index 8be41cd..3ec48d3 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -98,7 +98,9 @@ def claim_of(ev: dict) -> str | None: Builds from 2026-10-07 on carry job_end's own `claim`, which says who stood behind the word: "verified (independent checks: )", "verified (your - checks)", or "passed own checks (), not verified" (never counted as + checks)", "verified (post-work audit: )" (--post-work-audit, tier + verified-audit: checks drafted after the work, gated on the pre-work tree + and on mutants), or "passed own checks (), not verified" (never counted as verified: it does not start with "verified"). Older builds, and outcomes that are not a tier, carry the outcome as before, with " (self-checked)" when every check was drafted (whoever drafted it). @@ -161,7 +163,7 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: review = ev.get("review") disagree = ev.get("checksDisagree") or [] # Only what job_end carried; absent keys stay absent (older builds). - extra = {k: ev[k] for k in ("revealed", "deadline", "endedBy", "retired", "build", "tier", "tierReason", "providerStall", "checkAuthors") if k in ev} + extra = {k: ev[k] for k in ("revealed", "deadline", "endedBy", "retired", "build", "tier", "tierReason", "providerStall", "checkAuthors", "audit") if k in ev} # The provider's daily cap, not the work: every later task would fail the # same way (2026-10-05: eleven tasks per arm "failed" in 140 s, 0 turns). capped = provider_capped(out, steps) diff --git a/bench/local/stats.py b/bench/local/stats.py index f113020..38d5991 100644 --- a/bench/local/stats.py +++ b/bench/local/stats.py @@ -60,7 +60,7 @@ def no_verdict(row: dict) -> bool: return (not c) or c.startswith(("error", "stopped")) or bool(row.get("timed_out")) -TIERS = ("verified", "passed-checks", "passed-own-checks", "passed-untested") +TIERS = ("verified", "passed-checks", "passed-own-checks", "passed-untested", "verified-audit") def claim_basis(claim) -> str | None: @@ -71,6 +71,9 @@ def claim_basis(claim) -> str | None: "verified (your checks)" -> "person" "passed own checks (m), not verified" -> "own" "passed checks that did not test this work, not verified" -> "untested" + "verified (post-work audit: m)" -> "audit" (--post-work-audit: + checks drafted after the work, gated on the + pre-work tree and on mutants) "verified (self-checked)" (builds before 2026-10-07: the checks were drafted, by the worker or a judge; the text cannot say which) -> "self-checked" @@ -82,6 +85,8 @@ def claim_basis(claim) -> str | None: return None if claim.startswith("verified (independent checks"): return "independent" + if claim.startswith("verified (post-work audit"): + return "audit" if claim.startswith("verified (your checks)"): return "person" if claim.startswith("passed own checks"): @@ -96,10 +101,15 @@ def claim_basis(claim) -> str | None: def claim_judge(claim) -> list[str]: - """The judge model(s) named by "verified (independent checks: a, b)"; [] otherwise.""" - if not isinstance(claim, str) or not claim.startswith("verified (independent checks:"): + """The judge model(s) named by "verified (independent checks: a, b)" or "verified (post-work audit: j)"; [] otherwise.""" + if not isinstance(claim, str): + return [] + for head in ("verified (independent checks:", "verified (post-work audit:"): + if claim.startswith(head): + break + else: return [] - inner = claim[len("verified (independent checks:"):].rsplit(")", 1)[0] + inner = claim[len(head):].rsplit(")", 1)[0] return [m.strip() for m in inner.split(",") if m.strip()] diff --git a/bench/local/tests/test_claim_labels.py b/bench/local/tests/test_claim_labels.py index be55611..a07b9b1 100644 --- a/bench/local/tests/test_claim_labels.py +++ b/bench/local/tests/test_claim_labels.py @@ -63,3 +63,17 @@ def test_passed_untested_is_its_own_tier_and_never_verified(): assert not stats.is_verified(UNTESTED) assert stats.claim_basis(UNTESTED) == "untested" assert run.claim_of({"outcome": "unverified", "tier": "passed-untested", "claim": UNTESTED}) == UNTESTED + + +AUDIT = "verified (post-work audit: qwen3-coder-30b-a3b)" + + +def test_post_work_audit_label_parses(): + ev = {"kind": "job_end", "outcome": "verified", "tier": "verified-audit", "claim": AUDIT, + "audit": {"judge": "qwen3-coder-30b-a3b", "drafted": 3, "grounded": 2, "accepted": ["audit:sum"]}} + assert run.claim_of(ev) == AUDIT + assert stats.is_verified(AUDIT) + assert stats.claim_basis(AUDIT) == "audit" + assert stats.claim_judge(AUDIT) == ["qwen3-coder-30b-a3b"] + assert "verified-audit" in stats.TIERS + assert stats.tier_breakdown([{"tier": "verified-audit", "passed": True}]) == {"verified-audit": (1, 1)} diff --git a/src/cli.tsx b/src/cli.tsx index 8be2bca..16abd5d 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -168,6 +168,13 @@ options on the tree before the work and passes now; otherwise "passed checks that did not test this work", exit 3 (also MAAT_REQUIRE_DISCRIMINATING=1; off by default) + --post-work-audit when the checks sealed before the work did not verify a + claim, the --judge model drafts checks from the task and + the work's interface (never its transcript or outputs); + one that quotes the task, passes on the work, fails + before it and fails on a mutant of the changed code earns + "verified (post-work audit: )". Needs --judge + (also MAAT_POST_WORK_AUDIT=1; off by default) --signout before an unattended claim is judged, put each stated requirement to the model once beside the commands it ran (off by default: 60 rounds rescued no task) @@ -290,6 +297,8 @@ type Args = { reviewAdvisory?: boolean; /** `--require-discriminating`: see EngineConfig.requireDiscriminating. */ requireDiscriminating?: boolean; + /** `--post-work-audit`: see EngineConfig.postWorkAudit. */ + postWorkAudit?: boolean; /** `--signout`: see EngineConfig.signOut. */ signout?: boolean; /** `--arbiter-model` / `--dispute-votes`: see EngineConfig.dispute. */ @@ -529,6 +538,9 @@ export function parseArgs(argv: string[], stored: StoredEndpoint = {}): Args { case "--require-discriminating": out.requireDiscriminating = true; break; + case "--post-work-audit": + out.postWorkAudit = true; + break; case "--signout": out.signout = true; break; @@ -837,6 +849,7 @@ function engineFor(args: Args, session = false, extra: { files?: FileAccess } = ...(args.revealStuck === true ? { revealOnStuck: true } : args.revealStuck === false ? { revealOnStuck: false } : {}), ...(args.reviewAdvisory || env("REVIEW_ADVISORY") === "1" ? { reviewAdvisory: true } : {}), ...(args.requireDiscriminating || env("REQUIRE_DISCRIMINATING") === "1" ? { requireDiscriminating: true } : {}), + ...(args.postWorkAudit || env("POST_WORK_AUDIT") === "1" ? { postWorkAudit: true } : {}), ...(args.signout ? { signOut: true } : {}), ...(args.arbiterModel || args.disputeVotes ? { dispute: { model: args.arbiterModel, votes: args.disputeVotes } } : {}), ...(args.review ? { review: { votes: args.review, reasoningEffort: args.reasoningChecks ?? args.reasoning } } : {}), diff --git a/src/criteria.ts b/src/criteria.ts index 8c68ca3..7a47c9b 100644 --- a/src/criteria.ts +++ b/src/criteria.ts @@ -325,6 +325,8 @@ export async function preflightCriteria( * run. The task text is where a path the person stated is allowed from. */ stray?: { task: string }; + /** Kill what each check left running when it exits (run.ts killGroupOnExit). */ + killGroupOnExit?: boolean; }, ): Promise { const broken: BrokenCriterion[] = []; @@ -352,6 +354,7 @@ export async function preflightCriteria( timeoutMs: opts.timeoutMs ?? 5_000, maxBuffer: 1024 * 1024, signal: opts.signal, + ...(opts.killGroupOnExit ? { killGroupOnExit: true } : {}), }); // One decision point, shared with the bar. A command that outlived the // timeout plainly ran, and a timeout's exit code is never one of the @@ -983,6 +986,47 @@ export const PASSES_BEFORE_WORK = export const PRINTS_FAIL_EXITS_0 = "it printed a failure (FAIL or False) and still exited 0 before any work, so its exit code does not carry its verdict; make a failure exit non-zero (assert, sys.exit(1), jq -e, or compare in the shell)"; +/** + * The pre-work screen: each check is tried on a copy of the project taken + * before the work. One that already passes there cannot show the task was + * done (P1); one that exits 0 while printing FAIL cannot fail at all (L16). + * The rest are kept, including one that could not run before the work (its + * deliverable did not exist yet). Used at seal time and by the post-work + * audit (src/post-audit.ts). Null when the screen could not run, or the copy + * changed under it (`intact`): then it decided nothing. + */ +export async function preWorkScreen( + checks: readonly DraftedCheck[], + preWorkDir: string, + redraft = false, + opts: { killGroupOnExit?: boolean; timeoutMs?: number; intact?: () => Promise } = {}, +): Promise<{ ok: DraftedCheck[]; bad: LintDrop[] } | null> { + const passed: string[] = []; + const printedFail: string[] = []; + const { intact, ...run } = opts; + // A copy the worker changed (the draft can outlive the start of the work) + // is not the tree before the work: nothing tried on it decides anything. + if (intact && !(await intact())) return null; + try { + await preflightCriteria( + // As they will be sealed: hidden task checks, so under the bar's shell. + checks.map((c) => ({ name: c.name, kind: "command", run: c.run, hidden: true, tags: ["task"] })), + { cwd: preWorkDir, passed, printedFail, ...run }, + ); + } catch { + return null; + } + if (intact && !(await intact())) return null; + const ok: DraftedCheck[] = []; + const bad: LintDrop[] = []; + for (const c of checks) { + if (printedFail.includes(c.name)) bad.push({ name: c.name, run: c.run, rule: "L16-printed-fail", why: PRINTS_FAIL_EXITS_0, redraft }); + else if (passed.includes(c.name)) bad.push({ name: c.name, run: c.run, rule: "P1-passes-before-work", why: PASSES_BEFORE_WORK, redraft }); + else ok.push(c); + } + return { ok, bad }; +} + /** A drafted check the seal-time lint retired (src/checklint.ts), for the journal. */ export type LintDrop = { name: string; run: string; rule: string; why: string; redraft: boolean }; type Sink = { requirements: string[]; lint: LintDrop[] }; @@ -1049,29 +1093,8 @@ async function critiqued( const screen = async (checks: DraftedCheck[], redraft: boolean): Promise<{ ok: DraftedCheck[]; bad: LintDrop[] }> => { const l = lintSplit(checks, redraft); if (!opts.preWorkDir || !l.ok.length) return l; - const passed: string[] = []; - const printedFail: string[] = []; - // A copy the worker changed (the draft can outlive the start of the work) - // is not the tree before the work: nothing tried on it decides anything. - if (opts.preWorkIntact && !(await opts.preWorkIntact())) return l; - try { - await preflightCriteria( - // As they will be sealed: hidden task checks, so under the bar's shell. - l.ok.map((c) => ({ name: c.name, kind: "command", run: c.run, hidden: true, tags: ["task"] })), - { cwd: opts.preWorkDir, passed, printedFail }, - ); - } catch { - return l; - } - if (opts.preWorkIntact && !(await opts.preWorkIntact())) return l; - const ok: DraftedCheck[] = []; - const bad = [...l.bad]; - for (const c of l.ok) { - if (printedFail.includes(c.name)) bad.push({ name: c.name, run: c.run, rule: "L16-printed-fail", why: PRINTS_FAIL_EXITS_0, redraft }); - else if (passed.includes(c.name)) bad.push({ name: c.name, run: c.run, rule: "P1-passes-before-work", why: PASSES_BEFORE_WORK, redraft }); - else ok.push(c); - } - return { ok, bad }; + const s = await preWorkScreen(l.ok, opts.preWorkDir, redraft, { intact: opts.preWorkIntact }); + return s ? { ok: s.ok, bad: [...l.bad, ...s.bad] } : l; }; const drafted1 = await draftCriteria(opts); let first = drafted1.ok ? { ...drafted1, draft: fix(drafted1.draft) } : drafted1; diff --git a/src/engine.ts b/src/engine.ts index d4cb170..b46f361 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -22,8 +22,9 @@ import { runCommand } from "./run.js"; import { describeStart, listBackground, startBackground, stopBackground } from "./background.js"; import { reviewClaim, type Review } from "./review.js"; import { judgeEffort, judgeTarget } from "./judge.js"; -import { authorKey, authorWords, claimLabel, tierOf, withAuthor, type Tier } from "./tiers.js"; +import { auditClaim, authorKey, authorWords, claimLabel, contradictions, independentOf, tierOf, withAuthor, type Tier } from "./tiers.js"; import { recordGoldens, valueUnproven } from "./golden.js"; +import { postWorkAudit, type AuditReport } from "./post-audit.js"; import { arbitrate, parseDisputes, type Ruling } from "./dispute.js"; import { createHash } from "node:crypto"; import { execFileSync } from "node:child_process"; @@ -1423,6 +1424,16 @@ export type EngineConfig = { * redraft of checks that pass before the work). `reviewAdvisory` implies it. */ requireDiscriminating?: boolean; + /** + * Opt-in (`--post-work-audit`, MAAT_POST_WORK_AUDIT=1): when a claimed turn + * is not verified by the checks sealed before the work, an independent + * judge drafts checks from the task text and an interface view of the work + * (never its transcript, claim or outputs), and one that passes on the work, + * fails before it and fails on a mutant of the changed code earns + * "verified (post-work audit: )", tier `verified-audit` + * (src/post-audit.ts). Needs a judge that is not the worker model. + */ + postWorkAudit?: boolean; /** * Requirement sign-out (src/signout.ts): one round per unattended turn that * lists each stated requirement beside the commands run for it. Off unless @@ -3708,8 +3719,12 @@ export class Engine { try { yield* inner; } finally { - await this.preWorkTree?.cleanup(); - this.preWorkTree = null; + // The post-work audit tries its checks on this copy after the turn's + // events are over; it is removed when the job ends (runSealed). + if (this.cfg.postWorkAudit !== true) { + await this.preWorkTree?.cleanup(); + this.preWorkTree = null; + } } } @@ -3784,6 +3799,90 @@ export class Engine { }); } + /** + * Run the post-work audit (src/post-audit.ts) on the pre-work copy kept for + * it. Undefined, said once, when it cannot run: no copy, or no judge that is + * another model than the worker (#32: a model never both finds and judges). + */ + private async *runPostWorkAudit(task: string): AsyncGenerator { + const log = this.cfg.journal; + const judge = judgeTarget({ baseUrl: this.cfg.baseUrl, apiKey: this.cfg.apiKey, model: this.cfg.model }); + const skip = (why: string) => { + log?.append("note", { kind: "post-work-audit", ran: false, text: `post-work audit skipped: ${why}` }); + return { kind: "info" as const, text: `post-work audit skipped: ${why}.` }; + }; + if (!independentOf({ kind: "judge", model: judge.model }, this.workerNames())) { + yield skip(`the judge (${judge.model}) is the worker model; set --judge to another model`); + return undefined; + } + const pre = this.preWorkTree; + if (!pre) { + yield skip("no copy of the project was taken before the work (too big to copy)"); + return undefined; + } + if (!(await pre.intact())) { + yield skip("the copy of the project taken before the work has changed since, so nothing tried on it decides anything"); + return undefined; + } + yield { kind: "info", text: `post-work audit: ${judge.model} drafts checks from the task and the work's interface` }; + const now = Date.now(); + const turnEnd = this.deadlineAt(); + const deadlineAt = turnEnd === undefined ? now + 180_000 : Math.min(now + 180_000, Math.max(turnEnd, now + 45_000)); + this.running = new AbortController(); + let report: AuditReport; + try { + report = await postWorkAudit({ + task, + workDir: this.cwd, + preWorkDir: pre.dir, + preWorkIntact: pre.intact, + judge, + fetchFn: this.cfg.fetchFn, + acpSpawn: this.cfg.acpSpawn, + reasoningEffort: judgeEffort(this.cfg.reasoningEffort), + deadlineAt, + signal: this.running.signal, + // Masked like every hidden check while it runs, released below. + onCheck: (c) => this.withholdText(`audit:${c.name}`, c.run), + }); + } catch (e) { + log?.append("note", { kind: "post-work-audit", ran: false, text: `post-work audit failed: ${errorText(e)}` }); + yield { kind: "info", text: `post-work audit could not run: ${errorText(e)}` }; + return undefined; + } finally { + this.running = undefined; + this.releaseWithheld(); + } + log?.append("note", { + kind: "post-work-audit", + ran: true, + text: `post-work audit by ${report.judge}: ${report.drafted} drafted, ${report.dropped.length} dropped, ${report.accepted.length} cleared every gate`, + judge: report.judge, + viewSha: createHash("sha256").update(report.view).digest("hex").slice(0, 16), + changed: report.changed.map((c) => c.path), + dropped: report.dropped, + mutants: report.mutants, + checks: report.checks.map((c) => ({ + name: `audit:${c.name}`, + run: c.run, + sha256: commandSha(c.run), + quote: c.quote, + accepted: c.accepted, + ...(c.rule ? { rule: c.rule, why: c.why } : {}), + ...(c.work ? { work: c.work } : {}), + ...(c.preWork ? { preWork: c.preWork } : {}), + ...(c.mutants ? { mutantsKilled: c.mutants.killed, mutantsTotal: c.mutants.total, mutantsCrashedOnly: c.mutants.crashedOnly } : {}), + })), + ...(report.error ? { error: report.error } : {}), + }); + if (report.error) yield { kind: "info", text: `post-work audit: ${report.error}` }; + else if (!report.accepted.length) { + const why = report.checks.map((c) => `${c.name}: ${c.why}`).concat(report.dropped.map((d) => `${d.name}: ${d.why}`)); + yield { kind: "info", text: `post-work audit: no check cleared every gate${why.length ? ` (${why.join("; ")})` : " (the judge drafted none)"}` }; + } + return report; + } + /** tierOf's advisory-review argument: empty unless `reviewAdvisory`. */ private advisoryTier(): { reviewAdvisory?: true; requireDiscriminating?: true } { return { @@ -4435,6 +4534,8 @@ export class Engine { } finally { // A turn that threw or was abandoned still releases what it withheld. this.releaseWithheld(); + await this.preWorkTree?.cleanup(); + this.preWorkTree = null; this.turnActive = false; this.cancelRequested = false; } @@ -4705,6 +4806,33 @@ export class Engine { ...(t.reviewNote ? { review: t.reviewNote } : {}), }); } + // The post-work audit (opt-in): a claimed turn the sealed checks did not + // verify gets checks an independent judge drafts from the task and the + // work's interface, each gated on the work, the pre-work copy and mutants. + let audit: AuditReport | undefined; + const auditable = + this.cfg.postWorkAudit === true && + !opts.ask && + !cancelled && + !this.turnEndedBy && + // A claim the sealed checks did not verify: they passed without earning the + // word, only drafted checks refused it, or nothing checked it. Never one a + // person's or the project's check refused ("not proven"), or an error. + outcome === "unverified" && + // A reviewer that found the claim contradicts the task is not overruled here. + contradictions(review) === 0; + if (auditable) { + audit = yield* this.runPostWorkAudit(userText); + if (audit?.accepted.length) { + outcome = "verified"; + tier = "verified-audit"; + tierReason = undefined; + claim = auditClaim(audit.judge); + yield { kind: "info", text: `${claim}: ${audit.accepted.map((n) => `\`audit:${n}\``).join(", ")} passed on the work, failed before it, and failed on a mutant of the changed code.` }; + if (receiptPath) this.cfg.receipts?.amendTier(basename(receiptPath), { tier: "verified-audit", claim }); + this.cfg.journal?.append("note", { text: "tier: verified-audit", claim, by: [audit.judge], accepted: audit.accepted.map((n) => `audit:${n}`) }); + } + } yield { kind: "job_end", job, @@ -4714,6 +4842,7 @@ export class Engine { outcome, ...(tier ? { tier, ...(tierReason ? { tierReason } : {}) } : {}), ...(claim ? { claim } : {}), + ...(audit ? { audit: { judge: audit.judge, drafted: audit.drafted, grounded: audit.drafted - audit.dropped.length, accepted: audit.accepted.map((n) => `audit:${n}`), ...(audit.error ? { error: audit.error } : {}) } } : {}), ...(this.sealedChecks.length ? { checkAuthors: Object.fromEntries(this.sealedChecks.map((c) => [c.name.startsWith("task:") ? c.name : `task:${c.name}`, authorKey(withAuthor(c, this.cfg.model).author!)])) } : {}), ...(selfChecked ? { selfChecked: true } : {}), ...(this.reviewSkipped ? { unreviewed: true } : {}), @@ -5453,7 +5582,7 @@ export class Engine { // taken: mark the pending draft handled now (it is awaited, and its // failure reported, further down) so that is not an unhandled rejection. opts.pendingCriteria?.catch(() => {}); - if ((opts.pendingCriteria || opts.referenceCheck) && !opts.ask) { + if ((opts.pendingCriteria || opts.referenceCheck || this.cfg.postWorkAudit === true) && !opts.ask) { const t0 = Date.now(); this.preWorkTree = await preWorkCopy(this.cwd); // Journalled either way, so a replay can tell a late check that was diff --git a/src/interface-view.ts b/src/interface-view.ts new file mode 100644 index 0000000..6b96f1d --- /dev/null +++ b/src/interface-view.ts @@ -0,0 +1,248 @@ +/** + * What the post-work audit's judge may see of the finished work: its + * interface, never its answers. + * + * The audit (src/post-audit.ts) asks an independent judge for checks after the + * work exists, so the checks can fit the deliverable's real interface (a name, + * a flag, a file it reads). The risk is the obvious one: a judge shown what the + * work printed writes checks that confirm whatever it printed. So the view is + * built from the two trees alone, never from the worker's transcript or its + * claim (nothing here takes them), and it holds: + * + * - the files the work added or changed, by name; + * - for code, the signature lines only (def/class/function/export, argparse + * `add_argument`, a script's usage line), never a body or a constant; + * - for each runnable new or changed script, what `--help` prints, kept only + * when it reads as usage text, run on a throwaway copy; + * - for anything else (data the work wrote), the name and a format word, never + * the contents: that is the deliverable's output on the task's example; + * - the top-level project listing. + */ +import { createHash } from "node:crypto"; +import { lstatSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { extname, join, relative, sep } from "node:path"; +import { topLevel } from "./brief.js"; +import { runCommand } from "./run.js"; +import { copyTree, LINKED_DIRS } from "./scratch.js"; +import { STATE_DIRS } from "./statedir.js"; + +export type Changed = { path: string; status: "new" | "changed" }; + +const SKIP = new Set([".git", "__pycache__", ".DS_Store", ...LINKED_DIRS, ...STATE_DIRS]); +const WALK_MAX_FILES = 5_000; +const FILE_MAX_BYTES = 2 * 1024 * 1024; + +/** Every file under `root`, relative, with a content hash; skips state, VCS and dependency folders. */ +function hashes(root: string): Map { + const out = new Map(); + const walk = (dir: string) => { + if (out.size >= WALK_MAX_FILES) return; + let ents; + try { + ents = readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const e of ents) { + if (out.size >= WALK_MAX_FILES) return; + if (SKIP.has(e.name)) continue; + const p = join(dir, e.name); + if (e.isDirectory()) walk(p); + else if (e.isFile()) { + try { + const st = lstatSync(p); + const h = st.size > FILE_MAX_BYTES ? `size:${st.size}:${st.mtimeMs}` : createHash("sha256").update(readFileSync(p)).digest("hex"); + out.set(relative(root, p).split(sep).join("/"), h); + } catch { + /* gone */ + } + } + } + }; + walk(root); + return out; +} + +/** The files the work added or changed: the pre-work copy against the tree now. */ +export function changedFiles(preWorkDir: string, workDir: string): Changed[] { + const before = hashes(preWorkDir); + const after = hashes(workDir); + const out: Changed[] = []; + for (const [p, h] of [...after].sort(([a], [b]) => a.localeCompare(b))) { + const was = before.get(p); + if (was === undefined) out.push({ path: p, status: "new" }); + else if (was !== h) out.push({ path: p, status: "changed" }); + } + return out; +} + +const INTERPRETER: Record = { + ".py": "python3", + ".js": "node", + ".mjs": "node", + ".cjs": "node", + ".sh": "bash", + ".bash": "bash", + ".rb": "ruby", + ".pl": "perl", +}; +const CODE = new Set([...Object.keys(INTERPRETER), ".ts", ".tsx", ".jsx", ".go", ".rs", ".c", ".h", ".cpp", ".java", ".kt", ".swift", ".php", ".lua", ".r", ".R"]); + +/** True when the file is code: by extension, or a shebang on its first line. */ +export function isCode(path: string, text: string): boolean { + return CODE.has(extname(path)) || text.startsWith("#!"); +} + +const FORMAT: Record = { + ".json": "JSON", + ".jsonl": "JSON lines", + ".csv": "CSV", + ".tsv": "TSV", + ".txt": "text", + ".md": "Markdown", + ".yaml": "YAML", + ".yml": "YAML", + ".toml": "TOML", + ".ini": "INI", + ".xml": "XML", + ".html": "HTML", + ".log": "log text", + ".sql": "SQL", + ".db": "database file", + ".sqlite": "database file", +}; + +/** Lines that say how code is called, and nothing it computes. Each cut at 160 characters. */ +export function signatures(path: string, text: string, max = 25): string[] { + const ext = extname(path); + const lines = text.split("\n"); + const keep: string[] = []; + const add = (l: string) => { + const t = l.trimEnd().slice(0, 160); + if (t.trim() && keep.length < max) keep.push(t); + }; + for (const l of lines) { + const t = l.trim(); + if (ext === ".py" || (!CODE.has(ext) && /^#!.*python/.test(lines[0] ?? ""))) { + if (/^(async\s+)?def\s+\w+\s*\(/.test(t) || /^class\s+\w+/.test(t)) add(l.replace(/:\s*(#.*)?$/, "")); + else if (/\badd_argument\s*\(/.test(t) || /\badd_parser\s*\(/.test(t)) add(l); + } else if ([".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx"].includes(ext)) { + if (/^(export\s+)?(default\s+)?(async\s+)?function\b/.test(t) || /^(export\s+)?(default\s+)?class\s+\w+/.test(t)) add(l.replace(/\{\s*$/, "")); + else if (/^export\s+(const|let|var)\s+\w+\s*=\s*(async\s*)?(\([^)]*\)|\w+)\s*=>/.test(t)) add(l.replace(/=>.*$/, "=>")); + else if (/^(module\.)?exports(\.\w+)?\s*=/.test(t) && t.length < 120) add(l); + else if (/^(app|router|server)\.(get|post|put|delete|patch|use)\s*\(\s*["'`]/.test(t)) add(l.replace(/,.*$/, ", …)")); + } else if (ext === ".go") { + if (/^func\s/.test(t) || /^type\s+\w+\s+(struct|interface)/.test(t)) add(l.replace(/\{\s*$/, "")); + } else if (ext === ".rs") { + if (/^(pub\s+)?(async\s+)?fn\s/.test(t) || /^(pub\s+)?struct\s/.test(t)) add(l.replace(/\{\s*$/, "")); + } else if ([".c", ".h", ".cpp", ".java", ".kt", ".swift", ".php"].includes(ext)) { + if (/^[\w:<>*&\s]+\s+\**\w+\s*\([^;]*\)\s*\{?\s*$/.test(t) && !/^(if|for|while|switch|return)\b/.test(t)) add(l.replace(/\{\s*$/, "")); + } + // Usage text is the interface, whatever the language (a shell script's `echo "usage: ..."`). + if (/\busage\s*:/i.test(t) && !keep.includes(l.trimEnd().slice(0, 160))) add(l); + } + return keep; +} + +/** Text that reads as a program's usage message rather than its answer. */ +export function looksLikeUsage(text: string): boolean { + return /\busage\b/i.test(text) || /^\s*(options|arguments|positional arguments)\s*:/im.test(text); +} + +export const HELP_TIMEOUT_MS = 3_000; +export const HELP_MAX_CHARS = 1_200; + +/** How the file is run, or null when it is not a script. */ +function invocation(path: string, text: string, workDir: string): string | null { + const interp = INTERPRETER[extname(path)]; + if (interp) return `${interp} ${shellWord(path)}`; + if (text.startsWith("#!")) { + try { + if (statSync(join(workDir, path)).mode & 0o111) return `./${shellWord(path)}`; + } catch { + return null; + } + } + return null; +} + +function shellWord(s: string): string { + return /^[\w./-]+$/.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`; +} + +export type InterfaceViewInput = { + /** The project as it was before the work: a copy taken at turn start. */ + preWorkDir: string; + /** The project now, after the work. Only read; `--help` runs on a copy of it. */ + workDir: string; + /** Bound on the scripts whose `--help` is asked. */ + maxHelp?: number; + signal?: AbortSignal; +}; + +export type InterfaceView = { text: string; changed: Changed[] }; + +/** + * The view. Takes the two trees and nothing else: no transcript, no claim, + * no check output, so none of them can reach the judge through it. + */ +export async function interfaceView(input: InterfaceViewInput): Promise { + const changed = changedFiles(input.preWorkDir, input.workDir); + const lines: string[] = []; + lines.push( + changed.length + ? `Files the work added or changed: ${changed.map((c) => `${c.path} (${c.status})`).join(", ")}` + : "Files the work added or changed: (none found)", + ); + const helpable: { path: string; cmd: string }[] = []; + const code: string[] = []; + const data: string[] = []; + for (const c of changed) { + let text = ""; + try { + text = readFileSync(join(input.workDir, c.path)).subarray(0, 512 * 1024).toString("utf8"); + } catch { + continue; + } + if (text.includes("\u0000")) { + data.push(` ${c.path}: binary file (contents not shown)`); + continue; + } + if (isCode(c.path, text)) { + const sig = signatures(c.path, text); + code.push(` ${c.path}:${sig.length ? `\n${sig.map((s) => ` ${s.trim()}`).join("\n")}` : " (no signature lines found)"}`); + const cmd = invocation(c.path, text, input.workDir); + if (cmd) helpable.push({ path: c.path, cmd }); + } else { + data.push(` ${c.path}: ${FORMAT[extname(c.path).toLowerCase()] ?? "file"} (contents not shown)`); + } + } + if (code.length) lines.push("Signatures (declarations only, no bodies):", ...code); + if (helpable.length) { + const usage: string[] = []; + // On a copy: a script that ignores --help and does its job must not do it here. + const copy = await copyTree(input.workDir); + try { + for (const h of helpable.slice(0, input.maxHelp ?? 4)) { + let said = ""; + if (copy) { + try { + const r = await runCommand(`${h.cmd} --help`, { cwd: copy.dir, timeoutMs: HELP_TIMEOUT_MS, maxBuffer: 64 * 1024, signal: input.signal, killGroupOnExit: true }); + const out = copy.unmap(`${r.stdout}\n${r.stderr}`).trim(); + if (!r.timedOut && looksLikeUsage(out)) said = out.slice(0, HELP_MAX_CHARS); + } catch { + /* could not spawn: no usage text */ + } + } + usage.push(` run as: ${h.cmd}${said ? `\n \`--help\` prints:\n${said.split("\n").map((l) => ` ${l}`).join("\n")}` : " (no usage text)"}`); + } + } finally { + await copy?.cleanup(); + } + lines.push("How to run the new or changed scripts:", ...usage); + } + if (data.length) lines.push("Other files the work wrote (formats only):", ...data); + const listing = topLevel(input.workDir, 41).filter((n) => !STATE_DIRS.some((s) => n === s || n === `${s}/`)).slice(0, 40); + lines.push(`Project files (top level): ${listing.join(" ") || "(empty)"}`); + return { text: lines.join("\n"), changed }; +} diff --git a/src/post-audit.ts b/src/post-audit.ts new file mode 100644 index 0000000..65fecef --- /dev/null +++ b/src/post-audit.ts @@ -0,0 +1,647 @@ +/** + * The post-work audit (`--post-work-audit`, MAAT_POST_WORK_AUDIT=1). + * + * "verified" normally rests on hidden checks an independent judge drafted + * before the work, from a frozen snapshot. That is safe and misses a lot: a + * check written before the deliverable exists cannot know its interface, and + * a weak judge's checks are often screened out because they could not fail. + * This adds checks drafted AFTER the work, without letting the judge confirm + * whatever the worker produced: + * + * 1. The judge (MAAT_JUDGE_*, never the worker's model) sees the task text + * and an interface view of the work (src/interface-view.ts): names, + * signatures, usage. Never the transcript, the claim, or the outputs. + * 2. Every check quotes the task words its expected value comes from; a + * quote that is not in the task drops the check (dispute.ts quotedIn). + * 3. A check counts only when it clears every gate: + * (a) it passes on the work; + * (b) it fails on the copy taken before the work (criteria.ts + * preWorkScreen, the seal-time screen); + * (c) mutation: Maat breaks the changed files 3-5 ways (flip a + * comparison, nudge a number, delete a statement, swap two + * branches) and the check must fail on at least one mutant that + * still runs (compiles, and did not just crash); + * (d) the standing lints: it must not change the work (L15), must be + * able to fail (L16), and must assert a value (tiers.ts). + * 4. One check through every gate earns "verified (post-work audit: J)". + * + * Every run is on a throwaway copy (src/scratch.ts); the work is never touched. + */ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, extname, join } from "node:path"; +import { askModel, type AskOptions } from "./ask.js"; +import { diagnoseFailure } from "./bar.js"; +import { reportsFailure } from "./evidence.js"; +import { cannotFail } from "./checklint.js"; +import { checkMutates } from "./checkwrites.js"; +import { CRITERIA_MAX_NAME, CRITERIA_MAX_RUN, preWorkScreen, repairEscapes, SHELL_PARSE_ERROR } from "./criteria.js"; +import { quotedIn } from "./dispute.js"; +import { parseLenient } from "./lenient-json.js"; +import { interfaceView, isCode, type Changed } from "./interface-view.js"; +import { bashPath, runCommand } from "./run.js"; +import { copyTree } from "./scratch.js"; +import { assertsValue } from "./tiers.js"; + +export const AUDIT_MAX_CHECKS = 4; +export const AUDIT_MIN_QUOTE = 8; +export const AUDIT_MAX_MUTANTS = 5; +export const AUDIT_CHECK_TIMEOUT_MS = 10_000; + +export const AUDIT_SYSTEM = [ + "You audit finished work on one coding task. You did not do the work and you will not see what", + "it printed or wrote. You get the task text and the INTERFACE of the finished work: which files", + "it added or changed, their signatures, how to run them, and the project listing.", + "", + "Write 2 to 4 shell checks. Each runs the deliverable (a script, a function, a query) on an input", + "and compares what it produces with an expected value, exiting non-zero on a mismatch. Almost", + "every task states something checkable: an output for a given input, a format, a count, a rule", + "(\"blank lines are ignored\", \"sorted by total, highest first\"), a file it must write.", + "Return an empty list only when the task states nothing a command could test.", + "", + "The expected value must come from the task text: either stated outright, or the result of", + "applying a rule the task states to a small input your check builds itself (write it under", + "$(mktemp -d)). For each check give `quote`: the task words that state that value or rule,", + "copied character for character from the task text (a short span is best, 5 to 20 words).", + "A quote that is not word for word in the task, even a close paraphrase, drops the check. Never", + "use a value you would only know by running the work or by guessing.", + "", + "Rules: one line each, under 500 characters, run under bash from the project root on a", + "throwaway copy. A check that only prints is useless: end every check in a comparison, e.g.", + " [ \"$(python3 tool.py 3 4)\" = \"7\" ]", + " python3 -c \"import json; d=json.load(open('out.json')); assert d['total'] == 12, d\"", + " [ \"$(grep -c '^ERROR' report.txt)\" -eq 3 ]", + "A wrong result must make the check exit non-zero: no `|| true`, no `|| echo`, no", + "printing PASS/FAIL with exit 0. Do not write into the project; scratch files go under", + "$(mktemp -d). Use the file names, flags and signatures shown; do not invent others.", + "", + "Reply with JSON only:", + '{"checks":[{"name":"kebab-name","run":"shell command","quote":"verbatim task words"}]}', +].join("\n"); + +export type AuditCheck = { name: string; run: string; quote: string }; +export type AuditDrop = { name: string; run: string; why: string }; + +/** The judge's reply as checks, with every ungrounded or malformed one dropped and said. */ +export function parseAuditChecks(reply: string, task: string): { kept: AuditCheck[]; dropped: AuditDrop[] } | null { + const body = reply.replace(/^\s*```(?:json)?/i, "").replace(/```\s*$/, "").trim(); + // Some judges answer with the bare list of checks. + const array = body.startsWith("["); + const start = body.indexOf(array ? "[" : "{"); + const end = body.lastIndexOf(array ? "]" : "}"); + if (start === -1 || end === -1) return null; + const json = body.slice(start, end + 1); + let raw: unknown; + try { + raw = JSON.parse(json); + } catch { + try { + raw = parseLenient(json); + } catch { + try { + raw = JSON.parse(repairEscapes(json)); + } catch { + return null; + } + } + } + const list = Array.isArray(raw) ? raw : (raw as { checks?: unknown }).checks; + if (!Array.isArray(list)) return null; + const kept: AuditCheck[] = []; + const dropped: AuditDrop[] = []; + const seen = new Set(); + for (const v of list) { + if (!v || typeof v !== "object") continue; + const o = v as Record; + let name = String(o.name ?? "").trim().replace(/\s+/g, "-").slice(0, CRITERIA_MAX_NAME) || `audit-${kept.length + dropped.length + 1}`; + const run = String(o.run ?? "").trim(); + const quote = String(o.quote ?? "").trim(); + while (seen.has(name)) name = `${name}-2`; + seen.add(name); + if (!run) dropped.push({ name, run, why: "no command" }); + else if (run.length > CRITERIA_MAX_RUN) dropped.push({ name, run, why: `the command is over ${CRITERIA_MAX_RUN} characters` }); + else if (!quote) dropped.push({ name, run, why: "it quotes no task words for its expected value" }); + else if (quote.replace(/\s+/g, " ").length < AUDIT_MIN_QUOTE || !quotedIn(quote, task)) + dropped.push({ name, run, why: `its quote is not in the task text: "${quote.slice(0, 80)}"` }); + else if (kept.length >= AUDIT_MAX_CHECKS) dropped.push({ name, run, why: `over ${AUDIT_MAX_CHECKS} checks` }); + else kept.push({ name, run, quote }); + } + return { kept, dropped }; +} + +// ---------------------------------------------------------------- mutants + +export type Mutant = { path: string; line: number; operator: string; before: string; after: string; text: string }; + +/** Same length as the line, string literals and comments blanked, so operators inside them are left alone. */ +function maskLine(line: string, ext: string): string { + let out = ""; + let q = ""; + for (let i = 0; i < line.length; i++) { + const c = line[i]!; + if (q) { + if (c === "\\") { + out += "__"; + i++; + continue; + } + if (c === q) q = ""; + out += "_"; + continue; + } + if (c === '"' || c === "'" || c === "`") { + q = c; + out += "_"; + continue; + } + if ((c === "#" && [".py", ".sh", ".bash", ".rb", ".pl", ""].includes(ext)) || (c === "/" && line[i + 1] === "/" && ![".py", ".sh", ".bash"].includes(ext))) { + out += "_".repeat(line.length - i); + break; + } + out += c; + } + return out; +} + +/** Replace `len` characters at `at` in `s`. */ +function splice(s: string, at: number, len: number, by: string): string { + return s.slice(0, at) + by + s.slice(at + len); +} + +const FLIPS: { find: RegExp; to: string; shell?: true }[] = [ + { find: /(?=!])<=(?!=)/, to: ">" }, + { find: /(?=!])>=(?!=)/, to: "<" }, + { find: /===/, to: "!==" }, + { find: /!==/, to: "===" }, + { find: /(?])==(?!=)/, to: "!=" }, + { find: /!=(?!=)/, to: "==" }, + { find: /(?<=\s)<(?=\s)/, to: ">=" }, + { find: /(?<=\s)>(?=\s)/, to: "<=" }, + { find: /(?<=\s)-lt(?=\s)/, to: "-ge", shell: true }, + { find: /(?<=\s)-gt(?=\s)/, to: "-le", shell: true }, + { find: /(?<=\s)-le(?=\s)/, to: "-gt", shell: true }, + { find: /(?<=\s)-ge(?=\s)/, to: "-lt", shell: true }, + { find: /(?<=\s)-eq(?=\s)/, to: "-ne", shell: true }, + { find: /(?<=\s)-ne(?=\s)/, to: "-eq", shell: true }, +]; + +const SHELLISH = new Set([".sh", ".bash", ""]); + +function isImport(t: string): boolean { + return /^(import\s|from\s+\S+\s+import\s|#include|use\s|require\(|const\s+\w+\s*=\s*require\(|package\s|using\s)/.test(t) || /^(export\s+)?\{?.*\}?\s*from\s+["']/.test(t) || t.startsWith("#!"); +} + +function balanced(t: string): boolean { + let n = 0; + for (const c of t) { + if ("([{".includes(c)) n++; + else if (")]}".includes(c)) n--; + if (n < 0) return false; + } + return n === 0; +} + +function flipComparison(line: string, ext: string): { after: string; operator: string } | null { + const masked = maskLine(line, ext); + if (/^\s*(import|from|#include)\b/.test(line) || line.trim().startsWith("#!")) return null; + for (const f of FLIPS) { + if (SHELLISH.has(ext)) { + // In a shell script `<` and `>` are redirects: only the test operators, and `==`/`!=` inside a test. + if (!f.shell && (!/\[\[?\s/.test(masked) || !/^[=!]=$/.test(f.to))) continue; + } else if (f.shell) continue; + const m = f.find.exec(masked); + if (!m) continue; + return { after: splice(line, m.index, m[0].length, f.to), operator: `flip ${m[0].trim()} to ${f.to}` }; + } + return null; +} + +function offByOne(line: string, ext: string): { after: string; operator: string } | null { + const t = line.trim(); + if (!t || isImport(t) || /^(#|\/\/|\*|\/\*)/.test(t)) return null; + const masked = maskLine(line, ext); + const m = /(? l.length - l.trimStart().length; + if (ext === ".py") { + if (!/^\s*(if|elif)\b.*:\s*(#.*)?$/.test(line)) return null; + const base = ind(line); + let j = i + 1; + while (j < lines.length && (lines[j]!.trim() === "" || ind(lines[j]!) > base)) j++; + if (j >= lines.length || ind(lines[j]!) !== base || !/^\s*else\s*:\s*(#.*)?$/.test(lines[j]!)) return null; + let k = j + 1; + while (k < lines.length && (lines[k]!.trim() === "" || ind(lines[k]!) > base)) k++; + while (k > j + 1 && lines[k - 1]!.trim() === "") k--; + const a = lines.slice(i + 1, j); + const b = lines.slice(j + 1, k); + if (!a.some((l) => l.trim()) || !b.some((l) => l.trim())) return null; + if (a.join("\n").trim() === b.join("\n").trim()) return null; + return { text: [...lines.slice(0, i + 1), ...b, lines[j]!, ...a, ...lines.slice(k)], operator: "swap the if/else branches", end: k }; + } + if ([".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".c", ".cpp", ".java", ".go", ".rs", ".kt", ".swift", ".php"].includes(ext)) { + if (!/^\s*(\}\s*else\s+)?if\b.*\{\s*$/.test(line)) return null; + const base = ind(line); + let j = i + 1; + while (j < lines.length && !(ind(lines[j]!) === base && /^\s*\}\s*else\s*\{\s*$/.test(lines[j]!))) { + if (ind(lines[j]!) <= base && lines[j]!.trim() !== "" && !/^\s*\}/.test(lines[j]!)) return null; + if (ind(lines[j]!) === base && /^\s*\}/.test(lines[j]!)) return null; + j++; + } + if (j >= lines.length) return null; + let k = j + 1; + while (k < lines.length && !(ind(lines[k]!) === base && /^\s*\}\s*$/.test(lines[k]!))) k++; + if (k >= lines.length) return null; + const a = lines.slice(i + 1, j); + const b = lines.slice(j + 1, k); + if (!a.some((l) => l.trim()) || !b.some((l) => l.trim()) || a.join("\n").trim() === b.join("\n").trim()) return null; + return { text: [...lines.slice(0, i + 1), ...b, lines[j]!, ...a, ...lines.slice(k)], operator: "swap the if/else branches", end: k }; + } + return null; +} + +/** Line numbers (1-based) of `after` whose text is not in `before`: what the work wrote. */ +export function changedLines(before: string | undefined, after: string): Set { + const lines = after.split("\n"); + if (before === undefined) return new Set(lines.map((_, i) => i + 1)); + const pool = new Map(); + for (const l of before.split("\n")) pool.set(l, (pool.get(l) ?? 0) + 1); + const out = new Set(); + lines.forEach((l, i) => { + const n = pool.get(l) ?? 0; + if (n > 0) pool.set(l, n - 1); + else out.add(i + 1); + }); + return out; +} + +/** Every mechanical mutant of the file's changed lines, by operator. */ +export function mutantCandidates(path: string, text: string, before?: string): Record<"flip" | "nudge" | "delete" | "swap", Mutant[]> { + const ext = extname(path) === "" && text.startsWith("#!") ? (/python/.test(text.split("\n")[0]!) ? ".py" : /node/.test(text.split("\n")[0]!) ? ".js" : "") : extname(path); + const lines = text.split("\n"); + const touched = changedLines(before, text); + const out: Record<"flip" | "nudge" | "delete" | "swap", Mutant[]> = { flip: [], nudge: [], delete: [], swap: [] }; + const one = (i: number, r: { after: string; operator: string } | null, into: Mutant[]) => { + if (!r || r.after === lines[i]) return; + const copy = [...lines]; + copy[i] = r.after; + into.push({ path, line: i + 1, operator: r.operator, before: lines[i]!, after: r.after, text: copy.join("\n") }); + }; + for (let i = 0; i < lines.length; i++) { + if (!touched.has(i + 1)) continue; + one(i, flipComparison(lines[i]!, ext), out.flip); + one(i, offByOne(lines[i]!, ext), out.nudge); + one(i, deleteStatement(lines[i]!, ext, lines[i + 1], lines[i - 1]), out.delete); + const s = swapBranches(lines, i, ext); + if (s) out.swap.push({ path, line: i + 1, operator: s.operator, before: lines[i]!, after: lines[i]!, text: s.text.join("\n") }); + } + return out; +} + +/** Mutants of a data file's changed lines: the first number nudged, or the line removed. */ +export function dataMutants(path: string, text: string, before?: string): { nudge: Mutant[]; delete: Mutant[] } { + const lines = text.split("\n"); + const touched = changedLines(before, text); + const out = { nudge: [] as Mutant[], delete: [] as Mutant[] }; + for (let i = 0; i < lines.length; i++) { + const l = lines[i]!; + if (!touched.has(i + 1) || !l.trim()) continue; + const m = /(? string> = { + ".py": (f) => `python3 -m py_compile ${q(f)}`, + ".js": (f) => `node --check ${q(f)}`, + ".mjs": (f) => `node --check ${q(f)}`, + ".cjs": (f) => `node --check ${q(f)}`, + ".sh": (f) => `bash -n ${q(f)}`, + ".bash": (f) => `bash -n ${q(f)}`, +}; + +function q(s: string): string { + return `'${s.replace(/'/g, `'\\''`)}'`; +} + +/** Does the mutant still parse? Languages with no cheap parser here are taken as parsing. */ +export async function mutantParses(m: Mutant): Promise { + if (extname(m.path).toLowerCase() === ".json") { + try { + JSON.parse(m.text); + return true; + } catch { + return false; + } + } + const compile = COMPILE[extname(m.path)]; + if (!compile) return true; + const dir = mkdtempSync(join(tmpdir(), "maat-mutant-")); + try { + const file = join(dir, basename(m.path)); + writeFileSync(file, m.text); + const r = await runCommand(compile(file), { cwd: dir, timeoutMs: 10_000, maxBuffer: 64 * 1024, env: { ...process.env, PYTHONDONTWRITEBYTECODE: "1" } }); + // A missing interpreter says nothing about the mutant. + if (r.code === 127) return true; + return r.code === 0 && !r.timedOut; + } catch { + return true; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** + * Up to `max` mutants of the changed code files that still parse, spread + * over the four operators and over the files, in a fixed order so a replay + * makes the same ones. + */ +export async function planAuditMutants( + files: readonly { path: string; text: string; before?: string }[], + max = AUDIT_MAX_MUTANTS, +): Promise { + const pools: Mutant[][] = []; + // When the work changed code, only the code is broken: a check that reads an output file the + // code wrote once is killed by editing that file and says nothing about the code. Data is + // broken only when the deliverable is data (a migrated config, a regex file, a hand-made report). + const anyCode = files.some((f) => isCode(f.path, f.text)); + for (const f of files) { + if (anyCode && !isCode(f.path, f.text)) continue; + if (!isCode(f.path, f.text)) { + // A deliverable that is data (a migrated config, a regex, a report): nudge a number, drop a line. + const d = dataMutants(f.path, f.text, f.before); + pools.push(spread(d.nudge), spread(d.delete)); + continue; + } + const c = mutantCandidates(f.path, f.text, f.before); + // Spread within an operator too: every other candidate, from both ends. + for (const k of ["flip", "swap", "nudge", "delete"] as const) pools.push(spread(c[k])); + } + const chosen: Mutant[] = []; + const texts = new Set(); + let tried = 0; + for (let round = 0; chosen.length < max && tried < max * 6; round++) { + let any = false; + for (const p of pools) { + const m = p[round]; + if (!m) continue; + any = true; + const key = `${m.path}\0${m.text}`; + if (texts.has(key)) continue; + texts.add(key); + tried++; + if (await mutantParses(m)) chosen.push(m); + if (chosen.length >= max || tried >= max * 6) break; + } + if (!any) break; + } + return chosen; +} + +function spread(xs: T[]): T[] { + const out: T[] = []; + let lo = 0; + let hi = xs.length - 1; + while (lo <= hi) { + out.push(xs[lo++]!); + if (lo <= hi) out.push(xs[hi--]!); + } + return out; +} + +// ---------------------------------------------------------------- gates + +export type RunVerdict = { status: "pass" | "fail" | "broken"; printedFail: boolean; crashed: boolean; output: string }; + +const CRASH = /Traceback \(most recent call last\)|\b(?:SyntaxError|ReferenceError|NameError|UnboundLocalError|IndentationError|ImportError|ModuleNotFoundError)\b/; + +/** Run a check on a throwaway copy of `dir`, optionally with one file replaced. */ +export async function runOnCopy( + dir: string, + run: string, + opts: { timeoutMs?: number; replace?: { path: string; text: string }; signal?: AbortSignal } = {}, +): Promise { + const copy = await copyTree(dir); + if (!copy) return { status: "broken", printedFail: false, crashed: false, output: "the project could not be copied, so the check was not run" }; + try { + if (opts.replace) writeFileSync(join(copy.dir, opts.replace.path), opts.replace.text); + const r = await runCommand(run, { + cwd: copy.dir, + shell: bashPath() ?? true, + timeoutMs: opts.timeoutMs ?? AUDIT_CHECK_TIMEOUT_MS, + maxBuffer: 1024 * 1024, + signal: opts.signal, + killGroupOnExit: true, + }); + const output = copy.unmap(`${r.stdout}\n${r.stderr}`).trim().slice(-2_000); + const d = diagnoseFailure(r.code ?? 0, r.stdout, r.stderr); + if (d.didNotRun || (SHELL_PARSE_ERROR.test(r.stderr) && r.code !== 0 && !CRASH.test(r.stderr))) return { status: "broken", printedFail: false, crashed: false, output }; + const printedFail = /\bFAIL(?:ED)?\b/.test(`${r.stdout}\n${r.stderr}`); + // As at the bar: a check whose deciding step printed False/FAIL (or a non-zero `echo $?`) failed. + const ok = !r.timedOut && r.code === 0 && !reportsFailure(run, r.stdout); + return { status: ok && !printedFail ? "pass" : "fail", printedFail: ok && printedFail, crashed: !ok && CRASH.test(output), output }; + } catch (e) { + return { status: "broken", printedFail: false, crashed: false, output: (e as Error).message }; + } finally { + await copy.cleanup(); + } +} + +export type GateResult = { + name: string; + run: string; + quote: string; + accepted: boolean; + /** Why it was not accepted: the first gate it did not clear. */ + why?: string; + rule?: string; + work?: RunVerdict["status"]; + preWork?: "failed" | "passed" | "printed-fail"; + mutants?: { total: number; killed: number; crashedOnly: number; survived: number; killedBy?: string[] }; +}; + +/** Gates (d), (a), (b), (c), in that order: cheapest first, and each run only if the last was cleared. */ +export async function auditGates( + c: AuditCheck, + ctx: { workDir: string; preWorkDir: string; mutants: readonly Mutant[]; timeoutMs?: number; signal?: AbortSignal; preWorkIntact?: () => Promise }, +): Promise { + const base = { name: c.name, run: c.run, quote: c.quote }; + const no = (rule: string, why: string, extra: Partial = {}): GateResult => ({ ...base, accepted: false, rule, why, ...extra }); + const writes = checkMutates(c.run); + if (writes) return no("L15-mutates", writes); + const never = cannotFail(c.run); + if (never) return no("L16-cannot-fail", never); + if (!assertsValue(c.run)) return no("V-no-value", "it does not assert an expected value"); + const work = await runOnCopy(ctx.workDir, c.run, { timeoutMs: ctx.timeoutMs, signal: ctx.signal }); + if (work.printedFail) return no("L16-printed-fail", "it printed FAIL and still exited 0 on the work", { work: work.status }); + if (work.status !== "pass") return no("A-fails-on-work", `it does not pass on the work (${work.status}): ${work.output.split("\n").slice(-1)[0]?.slice(0, 160) ?? ""}`, { work: work.status }); + const screened = await preWorkScreen([{ name: c.name, run: c.run }], ctx.preWorkDir, false, { + killGroupOnExit: true, + timeoutMs: ctx.timeoutMs ?? AUDIT_CHECK_TIMEOUT_MS, + ...(ctx.preWorkIntact ? { intact: ctx.preWorkIntact } : {}), + }); + if (!screened) return no("B-untried", "it could not be tried on the copy taken before the work (the copy changed, or the try could not run)", { work: "pass" }); + const hit = screened.bad[0]; + if (hit) return no(hit.rule, hit.why, { work: "pass", preWork: hit.rule === "L16-printed-fail" ? "printed-fail" : "passed" }); + if (!ctx.mutants.length) return no("C-no-mutants", "no mutant of the changed files could be made, so nothing shows the check reads the work", { work: "pass", preWork: "failed" }); + let killed = 0; + const killedBy: string[] = []; + let crashedOnly = 0; + let survived = 0; + for (const m of ctx.mutants) { + const r = await runOnCopy(ctx.workDir, c.run, { timeoutMs: ctx.timeoutMs, replace: { path: m.path, text: m.text }, signal: ctx.signal }); + if (r.status === "pass") survived++; + else if (r.status === "fail" && !r.crashed) { + killed++; + killedBy.push(`${m.operator} at ${m.path}:${m.line}`); + } else crashedOnly++; + } + const mutants = { total: ctx.mutants.length, killed, crashedOnly, survived, killedBy }; + if (survived === ctx.mutants.length) return no("C-passes-every-mutant", `it passed on all ${ctx.mutants.length} mutants of the changed files`, { work: "pass", preWork: "failed", mutants }); + if (killed === 0) return no("C-only-crashes", "it failed only on mutants that crashed or could not run, never on one that still ran", { work: "pass", preWork: "failed", mutants }); + return { ...base, accepted: true, work: "pass", preWork: "failed", mutants }; +} + +// ---------------------------------------------------------------- the audit + +export type AuditReport = { + judge: string; + /** What the judge was shown, besides the task text. */ + view: string; + changed: Changed[]; + drafted: number; + dropped: AuditDrop[]; + mutants: { path: string; line: number; operator: string }[]; + checks: GateResult[]; + /** Names of the checks that cleared every gate. */ + accepted: string[]; + /** Set when the audit could not ask the judge or read its reply. */ + error?: string; +}; + +export async function postWorkAudit(opts: { + task: string; + workDir: string; + preWorkDir: string; + /** Whether the pre-work copy is still as taken (scratch.ts preWorkCopy). */ + preWorkIntact?: () => Promise; + judge: { baseUrl: string; apiKey?: string; model: string }; + fetchFn?: typeof fetch; + acpSpawn?: AskOptions["acpSpawn"]; + cliRun?: AskOptions["cliRun"]; + reasoningEffort?: string; + askTimeoutMs?: number; + deadlineAt?: number; + checkTimeoutMs?: number; + maxMutants?: number; + signal?: AbortSignal; + /** Called with each check before it is run, so a caller can withhold its command. */ + onCheck?: (c: AuditCheck) => void; +}): Promise { + const view = await interfaceView({ preWorkDir: opts.preWorkDir, workDir: opts.workDir, signal: opts.signal }); + const report: AuditReport = { judge: opts.judge.model, view: view.text, changed: view.changed, drafted: 0, dropped: [], mutants: [], checks: [], accepted: [] }; + const prompt = [ + `TASK TEXT:\n${opts.task}`, + "", + "INTERFACE OF THE FINISHED WORK (names and signatures only; what it printed or wrote is not shown):", + view.text, + ...(process.platform === "darwin" ? ["", "This is macOS: BSD tools, not GNU. Prefer python3 for anything beyond plain shell."] : []), + ].join("\n"); + // A subprocess judge starts in an empty folder of its own, never the work: it must not read the outputs. + const askDir = mkdtempSync(join(tmpdir(), "maat-audit-")); + let parsed: ReturnType = null; + let lastText = ""; + try { + for (let i = 0; i < 2 && !parsed; i++) { + const asked = await askModel({ + ...opts.judge, + system: AUDIT_SYSTEM, + prompt: i === 0 ? prompt : `${prompt}\n\nYour last reply could not be parsed as JSON. Reply with the JSON object only.`, + cwd: askDir, + what: "drafting post-work audit checks", + fetchFn: opts.fetchFn, + acpSpawn: opts.acpSpawn, + cliRun: opts.cliRun, + reasoningEffort: opts.reasoningEffort, + timeoutMs: opts.askTimeoutMs, + deadlineAt: opts.deadlineAt, + }); + if (!asked.ok) { + report.error = asked.error; + return report; + } + lastText = asked.text; + parsed = parseAuditChecks(asked.text, opts.task); + } + } finally { + rmSync(askDir, { recursive: true, force: true }); + } + if (!parsed) { + const said = lastText.trim().replace(/\s+/g, " "); + report.error = `the judge's reply was not JSON: ${said ? `"${said.slice(0, 160)}${said.length > 160 ? "…" : ""}"` : "the reply was empty"}`; + return report; + } + report.drafted = parsed.kept.length + parsed.dropped.length; + report.dropped = parsed.dropped; + if (!parsed.kept.length) return report; + const files = view.changed.map((c) => { + let text = ""; + let before: string | undefined; + try { + text = readFileSync(join(opts.workDir, c.path), "utf8"); + if (c.status === "changed") before = readFileSync(join(opts.preWorkDir, c.path), "utf8"); + } catch { + /* unreadable: no mutants from it */ + } + return { path: c.path, text, ...(before !== undefined ? { before } : {}) }; + }); + const mutants = await planAuditMutants(files.filter((f) => f.text && !f.text.includes("\u0000")), opts.maxMutants ?? AUDIT_MAX_MUTANTS); + report.mutants = mutants.map((m) => ({ path: m.path, line: m.line, operator: m.operator })); + for (const c of parsed.kept) { + if (opts.signal?.aborted || (opts.deadlineAt !== undefined && Date.now() >= opts.deadlineAt)) { + report.checks.push({ ...c, accepted: false, rule: "T-out-of-time", why: "the run's time ran out before this check was tried" }); + continue; + } + opts.onCheck?.(c); + const g = await auditGates(c, { workDir: opts.workDir, preWorkDir: opts.preWorkDir, mutants, timeoutMs: opts.checkTimeoutMs, signal: opts.signal, ...(opts.preWorkIntact ? { preWorkIntact: opts.preWorkIntact } : {}) }); + report.checks.push(g); + if (g.accepted) report.accepted.push(g.name); + } + return report; +} diff --git a/src/run.ts b/src/run.ts index cba5922..121426c 100644 --- a/src/run.ts +++ b/src/run.ts @@ -30,6 +30,13 @@ export type RunOptions = { shell?: string | true; /** Kills the command when it aborts, so a turn can be cancelled mid-command. */ signal?: AbortSignal; + /** + * When the shell exits, kill whatever it left running in its process group + * (a server started with `&`). Used where runs follow one another on the + * same ports (src/post-audit.ts): a server left by the run on the work would + * otherwise answer for the pre-work copy and for every mutant. + */ + killGroupOnExit?: boolean; }; export type RunResult = { @@ -195,6 +202,7 @@ export function runCommand(command: string, opts: RunOptions): Promise finish(code, signal)); child.on("exit", (code, signal) => { + if (opts.killGroupOnExit) signalAll("SIGKILL"); drainTimer ??= setTimeout(() => finish(code, signal, true), DRAIN_GRACE_MS); }); child.on("error", (e) => { diff --git a/src/tiers.ts b/src/tiers.ts index 21ced45..43bf441 100644 --- a/src/tiers.ts +++ b/src/tiers.ts @@ -146,7 +146,7 @@ export function evidenceTags(run: string, surface: boolean | undefined): string[ * ... || echo fail` passed before server.py existed and earned a wrong * "verified" (the grader got a non-JSON 404). */ -export type Tier = "verified" | "passed-checks" | "passed-own-checks" | "passed-untested"; +export type Tier = "verified" | "passed-checks" | "passed-own-checks" | "passed-untested" | "verified-audit"; export type TierVerdict = { tier: Tier; reason?: string; @@ -280,6 +280,19 @@ export function claimLabel(outcome: string, tier: Pick", "judge ", "person", "reference ". */ checkAuthors?: Record; + /** + * The post-work audit (`--post-work-audit`, src/post-audit.ts), when it + * ran: the judge, how many checks it drafted, how many quoted the task, + * and which cleared every gate. Absent when the flag is off or the run + * was already verified. + */ + audit?: { judge: string; drafted: number; grounded: number; accepted: string[]; error?: string }; } | { kind: "step_summary"; diff --git a/test/post-work-audit.test.ts b/test/post-work-audit.test.ts new file mode 100644 index 0000000..03454e7 --- /dev/null +++ b/test/post-work-audit.test.ts @@ -0,0 +1,258 @@ +/** + * The post-work audit (`--post-work-audit`, src/post-audit.ts): checks an + * independent judge drafts after the work, from the task text and the work's + * interface only, each gated on the work, the pre-work copy and mutants of + * the changed code. + */ +import assert from "node:assert/strict"; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, before, describe, it } from "node:test"; +import { parseArgs } from "../src/cli.js"; +import { Engine } from "../src/engine.js"; +import { interfaceView, changedFiles, signatures } from "../src/interface-view.js"; +import { Journal } from "../src/journal.js"; +import { auditGates, changedLines, dataMutants, mutantCandidates, parseAuditChecks, planAuditMutants } from "../src/post-audit.js"; +import { Receipts } from "../src/receipts.js"; +import { auditClaim } from "../src/tiers.js"; +import type { Check } from "../src/types.js"; +import { allowAll, drain, scriptedProvider, workspace, type ScriptedTurn } from "./helpers.js"; + +const TASK = + "Write sum.js so that `node sum.js FILE` prints the total of the numbers in FILE, one number per line, blank lines ignored.\n" + + "For input.txt the total is 7."; +const SECRET = "ANSWER-IS-7"; + +const SUM_JS = [ + "const fs = require(\"fs\");", + "function total(lines) {", + " let t = 0;", + " for (const l of lines) {", + " if (l.trim() !== \"\") t += Number(l);", + " }", + " return t;", + "}", + "if (process.argv[2] === \"--help\") {", + " console.log(\"usage: node sum.js FILE\");", + " process.exit(0);", + "}", + "const note = \"" + SECRET + "\";", + "console.log(total(fs.readFileSync(process.argv[2], \"utf8\").split(\"\\n\")));", + "", +].join("\n"); + +/** A pre-work tree and a work tree: input.txt before; sum.js, an answer-printing script and out.txt after. */ +function trees(): { pre: string; work: string; cleanup: () => void } { + const root = mkdtempSync(join(tmpdir(), "maat-audit-test-")); + const pre = join(root, "pre"); + const work = join(root, "work"); + mkdirSync(pre); + writeFileSync(join(pre, "input.txt"), "3\n\n4\n"); + cpSync(pre, work, { recursive: true }); + writeFileSync(join(work, "sum.js"), SUM_JS); + // Ignores --help and prints the answer: its output must never reach the view. + writeFileSync(join(work, "show.js"), `console.log(${JSON.stringify(SECRET)});\n`); + writeFileSync(join(work, "out.txt"), `7 ${SECRET}\n`); + return { pre, work, cleanup: () => rmSync(root, { recursive: true, force: true }) }; +} + +describe("the interface view", () => { + let t: ReturnType; + before(() => { + t = trees(); + }); + after(() => t.cleanup()); + + it("names the new files, shows signatures and usage, and none of the outputs", async () => { + const v = await interfaceView({ preWorkDir: t.pre, workDir: t.work }); + assert.deepEqual(changedFiles(t.pre, t.work).map((c) => `${c.path}:${c.status}`), ["out.txt:new", "show.js:new", "sum.js:new"]); + assert.match(v.text, /Files the work added or changed: out\.txt \(new\), show\.js \(new\), sum\.js \(new\)/); + assert.match(v.text, /function total\(lines\)/); + assert.match(v.text, /run as: node sum\.js\n `--help` prints:\n usage: node sum\.js FILE/); + assert.match(v.text, /run as: node show\.js \(no usage text\)/, "an answer printed in reply to --help is not usage"); + assert.match(v.text, /out\.txt: text \(contents not shown\)/); + assert.match(v.text, /Project files \(top level\): input\.txt out\.txt show\.js sum\.js/); + assert.ok(!v.text.includes(SECRET), "no output, file content or constant of the work reaches the judge"); + assert.ok(!v.text.includes("t += Number"), "no function body"); + }); + + it("signatures are declarations only", () => { + assert.deepEqual(signatures("a.py", "import sys\ndef parse(line, strict=False):\n return 42\nclass Row:\n pass\np.add_argument('--port', type=int)\n"), [ + "def parse(line, strict=False)", + "class Row", + "p.add_argument('--port', type=int)", + ]); + }); +}); + +describe("the judge's checks", () => { + it("a check whose quote is not in the task is dropped; one with a verbatim quote is kept", () => { + const reply = JSON.stringify({ + checks: [ + { name: "total", run: `[ "$(node sum.js input.txt)" = "7" ]`, quote: "For input.txt the total is 7." }, + { name: "guess", run: `[ "$(node sum.js input.txt)" = "8" ]`, quote: "the total for input.txt is 8" }, + { name: "unquoted", run: `[ "$(node sum.js input.txt)" = "7" ]` }, + { name: "tiny", run: `[ "$(node sum.js input.txt)" = "7" ]`, quote: "7." }, + ], + }); + const p = parseAuditChecks(`\`\`\`json\n${reply}\n\`\`\``, TASK)!; + assert.deepEqual(p.kept.map((c) => c.name), ["total"]); + assert.deepEqual(p.dropped.map((d) => d.name), ["guess", "unquoted", "tiny"]); + assert.match(p.dropped[0]!.why, /not in the task text/); + assert.equal(parseAuditChecks("no json here", TASK), null); + const bare = parseAuditChecks(`[{"name":"t","run":"[ \\"$(node sum.js input.txt)\\" = \\"7\\" ]","quote":"For input.txt the total is 7."}]`, TASK)!; + assert.deepEqual(bare.kept.map((c) => c.name), ["t"], "a bare list of checks is read too"); + }); +}); + +describe("mutants", () => { + it("only the lines the work wrote are broken, and every mutant still parses", async () => { + assert.deepEqual([...changedLines("a\nb\nc", "a\nB\nc\nd")], [2, 4]); + const c = mutantCandidates("sum.js", SUM_JS); + assert.ok(c.flip.some((m) => m.after.includes('l.trim() === ""')), "a comparison is flipped"); + assert.ok(c.nudge.some((m) => m.after.includes("let t = 1")), "a number is nudged"); + assert.ok(c.delete.length > 0, "a statement can be deleted"); + const plan = await planAuditMutants([{ path: "sum.js", text: SUM_JS }]); + assert.ok(plan.length >= 3 && plan.length <= 5, `3-5 mutants, got ${plan.length}`); + assert.equal(new Set(plan.map((m) => m.text)).size, plan.length); + const py = mutantCandidates("f.py", "def f(x):\n if x > 3:\n return 1\n else:\n return 2\n"); + assert.ok(py.swap.some((m) => m.text.includes("if x > 3:\n return 2\n else:\n return 1")), "if/else bodies swapped"); + assert.equal(mutantCandidates("f.py", "x = 1\n", "x = 1\n").nudge.length, 0, "an unchanged line is left alone"); + const cfg = dataMutants("c.json", '{\n "version": 2,\n "name": "api"\n}\n', '{\n "name": "api"\n}\n'); + assert.deepEqual(cfg.nudge.map((m) => m.after), [' "version": 3,'], "a data deliverable's changed number is nudged"); + const planned = await planAuditMutants([{ path: "c.json", text: '{\n "version": 2,\n "name": "api"\n}\n', before: '{\n "name": "api"\n}\n' }]); + assert.ok(planned.every((m) => { try { JSON.parse(m.text); return true; } catch { return false; } }), "a JSON mutant that no longer parses is not used"); + assert.ok(planned.length >= 1); + const mixed = await planAuditMutants([{ path: "out.csv", text: "a,1\nb,2\n" }, { path: "sum.js", text: SUM_JS }]); + assert.ok(mixed.length > 0 && mixed.every((m) => m.path === "sum.js"), "when code changed, an output file it wrote is not what gets broken"); + }); +}); + +describe("the gates", () => { + let t: ReturnType; + before(() => { + t = trees(); + }); + after(() => t.cleanup()); + + it("a check that passes on every mutant is rejected", async () => { + const mutants = await planAuditMutants([{ path: "sum.js", text: SUM_JS }]); + const g = await auditGates( + { name: "exists", run: `test -f sum.js && [ "$(echo 7)" = "7" ]`, quote: "For input.txt the total is 7." }, + { workDir: t.work, preWorkDir: t.pre, mutants }, + ); + assert.equal(g.accepted, false); + assert.equal(g.rule, "C-passes-every-mutant"); + assert.deepEqual([g.work, g.preWork], ["pass", "failed"], "it cleared (a) and (b) and still does not show the code works"); + }); + + it("a check that passes on the work, fails before it and fails on a mutant is accepted", async () => { + const mutants = await planAuditMutants([{ path: "sum.js", text: SUM_JS }]); + const g = await auditGates( + { name: "total", run: `[ "$(node sum.js input.txt)" = "7" ]`, quote: "For input.txt the total is 7." }, + { workDir: t.work, preWorkDir: t.pre, mutants }, + ); + assert.equal(g.accepted, true, g.why); + assert.ok(g.mutants!.killed >= 1); + assert.ok(g.mutants!.survived < g.mutants!.total); + }); + + it("a check that passes before the work, cannot fail, writes, or asserts nothing is rejected before the mutants", async () => { + const ctx = { workDir: t.work, preWorkDir: t.pre, mutants: [] }; + const q = "For input.txt the total is 7."; + assert.equal((await auditGates({ name: "a", run: `[ "$(cat input.txt | wc -l | tr -d ' ')" = "3" ]`, quote: q }, ctx)).rule, "P1-passes-before-work"); + assert.equal((await auditGates({ name: "b", run: `[ "$(node sum.js input.txt)" = "7" ] || echo fail`, quote: q }, ctx)).rule, "L16-cannot-fail"); + assert.equal((await auditGates({ name: "c", run: `node sum.js input.txt > out.txt && grep -qx 7 out.txt`, quote: q }, ctx)).rule, "L15-mutates"); + assert.equal((await auditGates({ name: "d", run: `node sum.js input.txt`, quote: q }, ctx)).rule, "V-no-value"); + assert.equal((await auditGates({ name: "e", run: `[ "$(node sum.js input.txt)" = "8" ]`, quote: q }, ctx)).rule, "A-fails-on-work"); + }); +}); + +describe("the audit in a turn", () => { + const work: ScriptedTurn[] = [ + { calls: [{ name: "write_file", args: { path: "sum.js", content: SUM_JS } }] }, + { text: `Done. The total is 7 and the note says ${SECRET}.` }, + ]; + // The worker's own check: it passes, and earns "passed own checks", not "verified". + const own: Check = { name: "runs", kind: "command", run: `[ "$(node sum.js input.txt)" = "7" ]`, timeoutMs: 5_000, expectExit: 0, tags: ["task", "value"], hidden: true, author: { kind: "worker", model: "m" } } as Check; + const JUDGE_REPLY = JSON.stringify({ + checks: [ + { name: "total", run: `[ "$(node sum.js input.txt)" = "7" ]`, quote: "For input.txt the total is 7." }, + { name: "made-up", run: `[ "$(node sum.js input.txt)" = "9" ]`, quote: "the total is 9" }, + ], + }); + + async function turn(postWorkAudit: boolean, judgeModel: string | null = "judge-x") { + const ws = workspace(); + const was = process.env.MAAT_JUDGE_MODEL; + if (judgeModel) process.env.MAAT_JUDGE_MODEL = judgeModel; + else delete process.env.MAAT_JUDGE_MODEL; + try { + writeFileSync(join(ws.dir, "input.txt"), "3\n\n4\n"); + const provider = scriptedProvider(work); + const judgePrompts: string[] = []; + const fetchFn = (async (url: string, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string; messages?: { content?: string }[] }; + if (judgeModel && body.model === judgeModel) { + judgePrompts.push((body.messages ?? []).map((m) => m.content ?? "").join("\n")); + return { ok: true, status: 200, json: async () => ({ choices: [{ message: { content: JUDGE_REPLY }, finish_reason: "stop" }] }), text: async () => "" } as unknown as Response; + } + return provider.fetchFn(url, init); + }) as typeof fetch; + const journal = new Journal(ws.dir, "audit"); + const engine = new Engine({ + baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn, bar: null, + receipts: new Receipts(ws.dir), stream: false, autonomy: "high", journal, + ...(postWorkAudit ? { postWorkAudit: true } : {}), + }); + const events = await drain(engine.run(TASK, allowAll, { taskChecks: [own], taskNotes: [] })); + const end = events.find((e) => e.kind === "job_end"); + assert.ok(end && end.kind === "job_end"); + const rows = readFileSync(join(ws.dir, ".maat", "receipts", "index.jsonl"), "utf8").trim().split("\n").map((l) => JSON.parse(l)); + const notes = Journal.read(journal.path).filter((e) => e.kind === "note" && e.data.kind === "post-work-audit"); + return { end, events, rows, notes, judgePrompts }; + } finally { + if (was === undefined) delete process.env.MAAT_JUDGE_MODEL; + else process.env.MAAT_JUDGE_MODEL = was; + ws.cleanup(); + } + } + + it("an unverified claim earns verified (post-work audit: judge) from a check that clears every gate", async () => { + const { end, rows, notes, judgePrompts } = await turn(true); + assert.deepEqual([end.outcome, end.tier, end.claim], ["verified", "verified-audit", auditClaim("judge-x")]); + assert.equal(end.claim, "verified (post-work audit: judge-x)"); + assert.deepEqual(end.audit, { judge: "judge-x", drafted: 2, grounded: 1, accepted: ["audit:total"] }); + assert.deepEqual([rows.at(-1).tier, rows.at(-1).claim], ["verified-audit", "verified (post-work audit: judge-x)"]); + const checks = notes[0]!.data.checks as { name: string; accepted: boolean; preWork?: string; mutantsKilled?: number }[]; + assert.deepEqual(checks.map((c) => [c.name, c.accepted, c.preWork]), [["audit:total", true, "failed"]]); + assert.ok(checks[0]!.mutantsKilled! >= 1); + assert.equal(judgePrompts.length, 1); + assert.ok(judgePrompts[0]!.includes("For input.txt the total is 7."), "the judge reads the task"); + assert.ok(judgePrompts[0]!.includes("sum.js (new)"), "and the interface"); + assert.ok(!judgePrompts[0]!.includes(SECRET), "never the worker's claim or the work's contents"); + assert.ok(!judgePrompts[0]!.includes("Done."), "never the transcript"); + }); + + it("with the flag off nothing changes: no audit, no judge request, the tier is the sealed checks'", async () => { + const { end, notes, judgePrompts } = await turn(false); + assert.deepEqual([end.outcome, end.tier], ["unverified", "passed-own-checks"]); + assert.equal(end.audit, undefined); + assert.equal(notes.length, 0); + assert.equal(judgePrompts.length, 0); + }); + + it("with no judge apart from the worker, the audit is skipped and the tier stands", async () => { + const { end, notes } = await turn(true, null); + assert.deepEqual([end.outcome, end.tier, end.audit], ["unverified", "passed-own-checks", undefined]); + assert.match(String(notes[0]!.data.text), /skipped: the judge \(m\) is the worker model/); + }); +}); + +describe("parseArgs --post-work-audit", () => { + it("is off unless asked for", () => { + assert.equal(parseArgs([]).postWorkAudit, undefined); + assert.equal(parseArgs(["--post-work-audit"]).postWorkAudit, true); + }); +});