From 40140184a7f4f8e234aa8f31ce9b8f511a9f23eb Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 14:38:03 -0700 Subject: [PATCH 1/4] Verified needs a check that failed before the work On the 2026-10-07 container bench a separate judge drafted checks like `python3 server.py 8080 & sleep 1; curl -f .../items || echo 'fail'`. They exit 0 whatever happens, so they passed on the tree before server.py existed, and the run was labelled verified; the grader got a non-JSON 404 for GET /items/99. The receipt already said "pass (nothing to establish)" for every one of them, and the tier ignored it. A passing independent runs+value check now earns "verified" only when it FAILED on the tree as it was before the work (the pre-work try in sealCriteria). One that passed then too, could not run then, or joined later with no pre-work try guards against a regression and does not count. Otherwise the tier is passed-untested: "passed checks that did not test this work, not verified", outcome unverified, exit code 3. A person's check still verifies as before. Checks that join late (drafts past the time cut, the reference check) are tried against a copy of the project taken at turn start, so they get the same pre-work try. The try's outcome is journalled (pre-work-try), each receipt row says how the check fared before the work, and each "Recorded, not verified" note says whether a discriminating check plausibly covers it (word match, display only). bench/local/discriminating_replay.py recomputes the tier offline from the saved journals and receipts. Over 2026-10-07's lanes it removes 2 of the 4 wrong verifieds left after independence (q235 http-json-server, ml4b config-migrate) and denies 10 right ones; 8 of those rested only on checks that passed on the untouched tree (`|| echo fail`, `find -exec` exit codes) and 2 on late checks this build could not try. --- bench/harbor/summarize.py | 1 + bench/local/discriminating_replay.py | 232 +++++++++++++++++++++++++ bench/local/stats.py | 5 +- bench/local/tests/test_claim_labels.py | 10 ++ src/criteria.ts | 10 ++ src/engine.ts | 138 +++++++++++++-- src/receipts.ts | 56 +++++- src/tiers.ts | 99 +++++++++-- src/types.ts | 9 +- test/discriminating-checks.test.ts | 142 +++++++++++++++ test/exit-codes.test.ts | 11 +- test/harness.test.ts | 2 +- test/independent-checks.test.ts | 16 +- test/judgment.test.ts | 2 +- test/review-advisory.test.ts | 27 +-- test/tiers.test.ts | 3 +- 16 files changed, 699 insertions(+), 64 deletions(-) create mode 100644 bench/local/discriminating_replay.py create mode 100644 test/discriminating-checks.test.ts diff --git a/bench/harbor/summarize.py b/bench/harbor/summarize.py index 3999c1c..2bc02e7 100755 --- a/bench/harbor/summarize.py +++ b/bench/harbor/summarize.py @@ -84,6 +84,7 @@ def trial_row(d: Path) -> dict | None: "unconfirmed" if (je or {}).get("outcome") == "verified" and ((je or {}).get("review") or {}).get("confirmed") is False # Builds from 2026-10-07 say who wrote the checks in job_end's `claim`. else "passed-own-checks" if (je or {}).get("tier") == "passed-own-checks" + else "passed-untested" if (je or {}).get("tier") == "passed-untested" else "self-checked" if (je or {}).get("outcome") == "verified" and (je or {}).get("selfChecked") and not (je or {}).get("claim") else (je or {}).get("outcome") ), diff --git a/bench/local/discriminating_replay.py b/bench/local/discriminating_replay.py new file mode 100644 index 0000000..5c716df --- /dev/null +++ b/bench/local/discriminating_replay.py @@ -0,0 +1,232 @@ +"""Replay the "verified" tier with and without the discrimination rule. + + python3 discriminating_replay.py [LANE_PREFIX ...] (default: 2026-10-07's lanes) + +The rule (src/tiers.ts tierOf, `failedBefore`): "verified" needs a passing +independent runs+value check that FAILED on the tree before the work. One that +passed then too (receipt: `pass-vacuous`), was broken then ("sealed criteria +that did not run when tried before the work"), or joined after the work began +(late drafts, the reference check) with no pre-work try does not count. + +Reads, offline, for every -molt-- tree under +~/.cache/maat-bench/container-work//: the session journal (.maat/log), the +last accepted full receipt, and the lane's grader row in container-results. No +model, no network. "Value" is src/tiers.ts assertsValue on the released +command, through the built dist/tiers.js; the critic's `surface` tag is not +journalled, so a value check the critic marked surface still counts here +(this can only overstate what the rule keeps). + +Baselines: + recorded the run's own claim started with "verified" (what the build said) + pr32 recorded, and an independent check stood behind it: the claim names + independent checks, or (builds before 2026-10-07) the arm is not + `self`, i.e. a separate judge drafted the checks +The rule is applied on top of pr32. +""" +from __future__ import annotations + +import json +import re +import subprocess +import sys +from collections import defaultdict +from pathlib import Path + +CACHE = Path.home() / ".cache/maat-bench" +CW = CACHE / "container-work" +RESULTS = CACHE / "container-results" +REPO = Path(__file__).resolve().parents[2] +TODAY = ["q235", "mm3", "ml4b", "hk55", "dsv4", "gh", "ga3b", "gx"] +RUN_DIR = re.compile(r"^(?P.+)-molt-(?P\d+)-(?P[a-z0-9]+)$") +RESULTS_LINE = re.compile(r"container-results/(results-[\w.-]+\.jsonl)") + + +def results_file(lane: Path) -> Path | None: + log = CACHE / f"{lane.name}.log" + if log.exists(): + m = RESULTS_LINE.search(log.read_text(errors="replace")[:4000]) + if m and (RESULTS / m.group(1)).exists(): + return RESULTS / m.group(1) + return None + + +def grader_rows(path: Path) -> dict[tuple[str, str, int], dict]: + rows: dict[tuple[str, str, int], dict] = {} + for line in path.read_text().splitlines(): + try: + r = json.loads(line) + except ValueError: + continue + if r.get("agent") == "molt": + rows[(r["task"], r["arm"], int(r["rep"]))] = r # the last row for a trial wins + return rows + + +def journal(tree: Path) -> list[dict]: + out: list[dict] = [] + for f in sorted((tree / ".maat/log").glob("*.jsonl")): + for line in f.read_text(errors="replace").splitlines(): + try: + out.append(json.loads(line)) + except ValueError: + pass + return out + + +def task(n: str) -> str: + return n if n.startswith("task:") else f"task:{n}" + + +def receipt_results(tree: Path) -> dict[str, str]: + """{check: result} from the last accepted full receipt: pass, pass-vacuous, fail, ...""" + # Builds that withhold hidden checks keep the full text under full/; older ones have only the one file. + full = sorted((tree / ".maat/receipts/full").glob("*-accepted.md")) or sorted((tree / ".maat/receipts").glob("*-accepted.md")) + if not full: + return {} + out: dict[str, str] = {} + name = None + for line in full[-1].read_text(errors="replace").splitlines(): + if line.startswith("check: "): + name = line[len("check: "):].strip() + elif line.startswith("result: ") and name: + out[name] = line[len("result: "):].strip() + name = None + return out + + +def asserts_value(runs: list[str]) -> list[bool]: + js = ( + f"import {{ assertsValue }} from {json.dumps(str(REPO / 'dist/tiers.js'))};" + "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>console.log(JSON.stringify(JSON.parse(s).map(assertsValue))));" + ) + p = subprocess.run(["node", "--input-type=module", "-e", js], input=json.dumps(runs), capture_output=True, text=True, check=True) + return json.loads(p.stdout) + + +def analyse(tree: Path, row: dict) -> dict: + ev = journal(tree) + notes = [e["data"] for e in ev if e.get("kind") == "note"] + upfront: set[str] = set() + late: set[str] = set() + broken: set[str] = set() + runs: dict[str, str] = {} + pre_failed: set[str] | None = None + for e in ev: # builds before hidden checks were withheld journalled the command with each run + if e.get("kind") == "bar_run": + for c in e["data"].get("checks", []): + if c.get("name", "").startswith("task:") and not str(c.get("detail", "")).startswith("[withheld"): + runs.setdefault(c["name"], c.get("detail", "")) + for d in notes: + text = d.get("text", "") + if text.startswith("task criteria sealed"): + upfront |= {task(c) for c in d.get("checks", [])} + elif d.get("kind") == "late-checks" and d.get("arrived") and "joined" in text and "NOT joined" not in text: + late |= {task(c) for c in d.get("checks", [])} + elif text.startswith("reference check joined"): + late.add("task:reference") + elif text.startswith("sealed criteria that did not run"): + broken |= {task(c) for c in d.get("checks", [])} + elif d.get("kind") == "checks-released": + for c in d.get("checks", []): + runs[task(c["name"])] = c.get("run", "") # the released command outranks a journalled detail + elif d.get("kind") == "pre-work-try": # builds with the rule journal it directly + pre_failed = (pre_failed or set()) | set(d.get("failed", [])) + late -= upfront + res = receipt_results(tree) + passing = [n for n, r in res.items() if n.startswith("task:") and r in ("pass", "pass-vacuous")] + values = dict(zip(passing, asserts_value([runs.get(n, "") for n in passing]))) if passing else {} + strong = [n for n in passing if values.get(n)] + status = {} + for n in strong: + if pre_failed is not None: + status[n] = "failed" if n in pre_failed else ("passed" if res[n] == "pass-vacuous" else "untried") + elif res[n] == "pass-vacuous": + status[n] = "passed" + elif n in broken: + status[n] = "untried" + elif n in late: + status[n] = "late" + elif n in upfront: + status[n] = "failed" + else: + status[n] = "untried" + return {"strong": strong, "status": status} + + +def main(argv: list[str]) -> int: + prefixes = argv or TODAY + lanes = sorted(p for p in CW.iterdir() if p.is_dir() and any(re.fullmatch(rf"{re.escape(x)}-\d+", p.name) for x in prefixes)) + tally: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) + lost_right: list[str] = [] + removed_wrong: list[str] = [] + kept_wrong: list[str] = [] + late_only: list[str] = [] + for lane in lanes: + rf = results_file(lane) + if not rf: + print(f"# {lane.name}: no results file found, skipped", file=sys.stderr) + continue + rows = grader_rows(rf) + for tree in sorted(lane.iterdir()): + m = RUN_DIR.match(tree.name) + if not m or not tree.is_dir() or not (tree / ".maat/log").exists(): + continue + row = rows.get((m["task"], m["arm"], int(m["rep"]))) + if not row: + continue + claim = row.get("claim") or "" + if not claim.startswith("verified"): + continue + if claim.startswith(("verified (independent checks", "verified (your checks)")): + independent = True + else: # builds before 2026-10-07: a separate judge drafted unless the arm is self + independent = m["arm"] != "self" + right = bool(row.get("passed")) + key = f"{lane.name}/{tree.name}" + lane_key = lane.name.rsplit("-", 1)[0] + t = tally[lane_key] + t["recorded"] += 1 + t["recorded_right"] += right + if not independent: + continue + t["pr32"] += 1 + t["pr32_right"] += right + a = analyse(tree, row) + disc = [n for n in a["strong"] if a["status"][n] == "failed"] + if disc: + t["rule"] += 1 + t["rule_right"] += right + if not right: + kept_wrong.append(f"{key} discriminating: {', '.join(disc)}") + else: + why = ", ".join(f"{n}={a['status'][n]}" for n in a["strong"]) or "no value check in the receipt" + if any(s == "late" for s in a["status"].values()): + late_only.append(f"{key} ({'right' if right else 'WRONG'}) {why}") + (lost_right if right else removed_wrong).append(f"{key} {why}") + tot: dict[str, int] = defaultdict(int) + print(f"{'lane':8} {'recorded':>12} {'PR #32':>12} {'+ rule':>12}") + for lane_key in sorted(tally): + t = tally[lane_key] + for k, v in t.items(): + tot[k] += v + print(f"{lane_key:8} {t['recorded_right']:>5}/{t['recorded']:<6} {t['pr32_right']:>5}/{t['pr32']:<6} {t['rule_right']:>5}/{t['rule']:<6}") + print(f"{'all':8} {tot['recorded_right']:>5}/{tot['recorded']:<6} {tot['pr32_right']:>5}/{tot['pr32']:<6} {tot['rule_right']:>5}/{tot['rule']:<6} (right/verified)") + print(f"\nwrong verifieds the rule removes ({len(removed_wrong)}):") + for x in removed_wrong: + print(" " + x) + print(f"\nright verifieds the rule loses ({len(lost_right)}):") + for x in lost_right: + print(" " + x) + print(f"\nwrong verifieds the rule keeps ({len(kept_wrong)}):") + for x in kept_wrong: + print(" " + x) + if late_only: + print(f"\nof those denied, rested on a late-joined check with no pre-work try in this build ({len(late_only)}); " + "with the rule, Maat tries late checks on a copy taken before the work, so these may differ live:") + for x in late_only: + print(" " + x) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/bench/local/stats.py b/bench/local/stats.py index 2969f9f..f113020 100644 --- a/bench/local/stats.py +++ b/bench/local/stats.py @@ -60,7 +60,7 @@ def no_verdict(row: dict) -> bool: return (not c) or c.startswith(("error", "stopped")) or bool(row.get("timed_out")) -TIERS = ("verified", "passed-checks", "passed-own-checks") +TIERS = ("verified", "passed-checks", "passed-own-checks", "passed-untested") def claim_basis(claim) -> str | None: @@ -70,6 +70,7 @@ def claim_basis(claim) -> str | None: "verified (independent checks: m)" -> "independent" "verified (your checks)" -> "person" "passed own checks (m), not verified" -> "own" + "passed checks that did not test this work, not verified" -> "untested" "verified (self-checked)" (builds before 2026-10-07: the checks were drafted, by the worker or a judge; the text cannot say which) -> "self-checked" @@ -85,6 +86,8 @@ def claim_basis(claim) -> str | None: return "person" if claim.startswith("passed own checks"): return "own" + if claim.startswith("passed checks that did not test this work"): + return "untested" if claim.startswith("verified (self-checked)"): return "self-checked" if claim.startswith("verified"): diff --git a/bench/local/tests/test_claim_labels.py b/bench/local/tests/test_claim_labels.py index 9c0624e..be55611 100644 --- a/bench/local/tests/test_claim_labels.py +++ b/bench/local/tests/test_claim_labels.py @@ -53,3 +53,13 @@ def test_passed_own_checks_is_a_tier_of_its_own(): assert "passed-own-checks" in stats.TIERS rows = [{"tier": "passed-own-checks", "passed": True}, {"tier": "verified", "passed": False}] assert stats.tier_breakdown(rows) == {"passed-own-checks": (1, 1), "verified": (1, 0)} + + +UNTESTED = "passed checks that did not test this work, not verified" + + +def test_passed_untested_is_its_own_tier_and_never_verified(): + assert "passed-untested" in stats.TIERS + assert not stats.is_verified(UNTESTED) + assert stats.claim_basis(UNTESTED) == "untested" + assert run.claim_of({"outcome": "unverified", "tier": "passed-untested", "claim": UNTESTED}) == UNTESTED diff --git a/src/criteria.ts b/src/criteria.ts index d352f66..e3c6488 100644 --- a/src/criteria.ts +++ b/src/criteria.ts @@ -303,6 +303,15 @@ export async function preflightCriteria( * evidence the task was done. The engine uses this to say so. */ passed?: string[]; + /** + * Filled with the names of criteria that RAN before the work and did not + * pass: they parsed, executed, did not err in their own code, and exited + * otherwise than expected (or ran out of time). Only such a criterion can + * tell the finished task from the untouched tree (src/tiers.ts + * `failedBefore`). One that was broken, skipped or could not be spawned is + * in neither list: it was not tried. + */ + failed?: string[]; /** * Set for DRAFTED checks: a command that reaches for an absolute path * outside the project (see strayPath) is reported broken without being @@ -347,6 +356,7 @@ export async function preflightCriteria( else if (unparsed) { broken.push({ name: c.name, run: c.run, why: `the shell could not parse it: ${firstLine(copy ? copy.unmap(r.stderr) : r.stderr)}` }); } else if (selfError) broken.push({ name: c.name, run: c.run, why: selfError }); else if (!r.timedOut && r.code === (c.expectExit ?? 0)) opts.passed?.push(c.name); + else opts.failed?.push(c.name); } catch { // Failing to spawn it here is molt's problem, not the criterion's. // Reporting it as broken would block work for the wrong reason. diff --git a/src/engine.ts b/src/engine.ts index 92a1212..2a9195d 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -22,7 +22,7 @@ import { runCommand } from "./run.js"; import { describeStart, listBackground, startBackground, stopBackground } from "./background.js"; import { reviewClaim, type Review } from "./review.js"; import { judgeEffort, judgeTarget } from "./judge.js"; -import { authorKey, authorWords, claimLabel, tierOf, withAuthor } from "./tiers.js"; +import { authorKey, authorWords, claimLabel, tierOf, withAuthor, type Tier } from "./tiers.js"; import { recordGoldens, valueUnproven } from "./golden.js"; import { arbitrate, parseDisputes, type Ruling } from "./dispute.js"; import { createHash } from "node:crypto"; @@ -43,6 +43,7 @@ import { barPath, } from "./bar.js"; import { checkSelfError, preflightCriteria } from "./criteria.js"; +import { copyTree, type TreeCopy } from "./scratch.js"; import { AUTONOMY_SUMMARY, DEFAULT_AUTONOMY, @@ -2195,6 +2196,24 @@ export class Engine { * proves no value. */ private goldensBefore: Map = new Map(); + /** + * Task criteria that ran before the work began and FAILED there. + * + * The only checks whose pass at the end shows this work did something: one + * that passed on the untouched tree too, or was never tried there (broken + * then, or joined after the work began), cannot tell the work from none. + * "verified" needs a passing independent value check from this set + * (src/tiers.ts `failedBefore`). + */ + private failedBeforeWork: ReadonlySet = new Set(); + /** + * A copy of the project taken at turn start, before any request, kept only + * while checks are still to join late (drafts past the time cut, the + * reference check). A late check is tried against it as it joins, so it + * gets the same pre-work try as one sealed up front. Null when nothing is + * pending or the tree is too big to copy: a late check is then untried. + */ + private preWorkTree: TreeCopy | null = null; /** True only while `proveNow` runs: a bar with no turn behind it. */ private standalone = false; private barHash: string | null; @@ -3674,9 +3693,49 @@ export class Engine { return this.markGuards(await runBar(bar, this.barContext(claim))); } - /** tierOf's advisory-review arguments: empty unless `reviewAdvisory`. */ - private advisoryTier(): { reviewAdvisory?: true; guards?: ReadonlySet } { - return this.cfg.reviewAdvisory === true ? { reviewAdvisory: true, guards: this.passedBeforeWork } : {}; + /** Pass a turn's events through, and remove the pre-work copy however it ends. */ + private async *dropPreWorkTreeAfter(inner: AsyncGenerator): AsyncGenerator { + try { + yield* inner; + } finally { + await this.preWorkTree?.cleanup(); + this.preWorkTree = null; + } + } + + /** + * Try checks that joined after the work began against the pre-work copy, + * as sealCriteria tries the ones sealed up front. Without a copy they stay + * untried, and an untried check cannot earn "verified". + */ + private async tryLateBeforeWork(checks: readonly Check[], log?: Journal): Promise { + const tree = this.preWorkTree; + if (!tree || !checks.length) return; + const passed: string[] = []; + const failed: string[] = []; + this.running = new AbortController(); + try { + await preflightCriteria(checks, { cwd: tree.dir, signal: this.running.signal, passed, failed }); + } catch { + return; + } finally { + this.running = undefined; + } + const asTask = (n: string) => (n.startsWith("task:") ? n : `task:${n}`); + this.passedBeforeWork = new Set([...this.passedBeforeWork, ...passed.map(asTask)]); + this.failedBeforeWork = new Set([...this.failedBeforeWork, ...failed.map(asTask)]); + log?.append("note", { + kind: "pre-work-try", + text: `late checks tried on the copy taken before the work: ${failed.length} failed, ${passed.length} passed`, + late: true, + failed: failed.map(asTask), + passed: passed.map(asTask), + }); + } + + /** tierOf's advisory-review argument: empty unless `reviewAdvisory`. */ + private advisoryTier(): { reviewAdvisory?: true } { + return this.cfg.reviewAdvisory === true ? { reviewAdvisory: true } : {}; } /** Who wrote each sealed check, keyed by the name it runs under in the bar (and its bare name). */ @@ -3695,16 +3754,19 @@ export class Engine { return [...new Set([this.cfg.model, this.modelOfRecord()].filter((m): m is string => typeof m === "string" && m.trim().length > 0))]; } - /** Everything tierOf weighs beside the results: advisory mode, the worker, who wrote each check, and which golden files predate the work. */ + /** Everything tierOf weighs beside the results: advisory mode, the worker, who wrote each check, what the pre-work try found, and which golden files predate the work. */ private tierContext(): { reviewAdvisory?: true; - guards?: ReadonlySet; + guards: ReadonlySet; + failedBefore: ReadonlySet; worker: string[]; authors: Map; valueUnproven: Set; } { return { ...this.advisoryTier(), + guards: this.passedBeforeWork, + failedBefore: this.failedBeforeWork, worker: this.workerNames(), authors: this.sealedAuthors(), valueUnproven: valueUnproven(this.sealedChecks.map((c) => ({ name: c.name, run: c.kind === "command" ? c.run : undefined, tags: c.tags })), this.cwd, this.goldensBefore), @@ -3716,23 +3778,36 @@ export class Engine { return Object.fromEntries(this.sealedAuthors()); } - /** See `passedBeforeWork`. Only a passing command criterion is relabelled. */ + /** + * See `passedBeforeWork` and `failedBeforeWork`. Only a passing command + * criterion is relabelled; every sealed task check is stamped with how it + * fared on the tree before the work, for the receipt. + */ private markGuards(result: BarResult): BarResult { - if (this.passedBeforeWork.size === 0) return result; + if (this.sealedChecks.length === 0 && this.passedBeforeWork.size === 0) return result; + const sealed = new Set(this.sealedChecks.map((c) => (c.name.startsWith("task:") ? c.name : `task:${c.name}`))); return { ...result, - results: result.results.map((r) => - r.ok && r.kind === "command" && this.passedBeforeWork.has(r.name) + results: result.results.map((r) => { + const beforeWork: CheckResult["beforeWork"] = this.passedBeforeWork.has(r.name) + ? "passed" + : this.failedBeforeWork.has(r.name) + ? "failed" + : sealed.has(r.name) + ? "untried" + : undefined; + const stamped = beforeWork ? { ...r, beforeWork } : r; + return r.ok && r.kind === "command" && this.passedBeforeWork.has(r.name) ? { - ...r, + ...stamped, established: false, output: "passed before the work began too, so it guards against a regression and " + "does not show this task was done · " + r.output, } - : r, - ), + : stamped; + }), }; } @@ -4352,7 +4427,7 @@ export class Engine { let receiptPath: string | undefined; let exhaustedResult: BarResult | undefined; let lastProof: BarResult | undefined; - for await (const ev of this.runTurn(userText, confirm, job, opts)) { + for await (const ev of this.dropPreWorkTreeAfter(this.runTurn(userText, confirm, job, opts))) { switch (ev.kind) { case "receipt": receiptPath = ev.path; @@ -4531,7 +4606,7 @@ export class Engine { // right 7/7 where everything else carrying the word was right 28/50. What // passed without earning it is reported as what it is. After the judgment // case above, which is opened on what the reviewers said. - let tier: "verified" | "passed-checks" | "passed-own-checks" | undefined; + let tier: Tier | undefined; let tierReason: string | undefined; let claim: string | undefined; if (outcome === "verified" && lastProof) { @@ -4553,6 +4628,14 @@ export class Engine { tierReason = t.reason; claim = claimLabel(outcome, t); yield { kind: "info", text: `${claim}: ${t.reason}. Use --judge , or approve the checks yourself, for "verified".` }; + } else if (t.tier === "passed-untested") { + // Independent checks passed, but every one of them passed on the + // untouched tree too or was never tried there: they guard against a + // regression and cannot tell this work from none. + outcome = "unverified"; + tierReason = t.reason; + claim = claimLabel(outcome, t); + yield { kind: "info", text: `${claim}: ${t.reason}.` }; } else { claim = claimLabel(outcome, t); } @@ -5308,7 +5391,14 @@ export class Engine { let signedOut = false; this.passedBeforeWork = new Set(); this.goldensBefore = new Map(); + this.failedBeforeWork = new Set(); const log = this.cfg.journal; + await this.preWorkTree?.cleanup(); + // The copy is taken asynchronously, and a draft can fail while it is + // taken: mark the pending draft handled now (it is awaited, and its + // failure reported, further down) so that is not an unhandled rejection. + opts.pendingCriteria?.catch(() => {}); + this.preWorkTree = (opts.pendingCriteria || opts.referenceCheck) && !opts.ask ? await copyTree(this.cwd) : null; /** * This turn's criteria, copied and sealed before anything runs. * @@ -5382,16 +5472,26 @@ export class Engine { self.running = new AbortController(); let broken: Awaited> = []; const passed: string[] = []; + const failed: string[] = []; const beforeTry = self.cfg.unattended ? listProject(self.cwd) : null; try { broken = await preflightCriteria(taskChecks, { cwd: self.cwd, signal: self.running.signal, passed, + failed, + }); + const asTask = (n: string) => (n.startsWith("task:") ? n : `task:${n}`); + self.passedBeforeWork = new Set(passed.map(asTask)); + self.failedBeforeWork = new Set(failed.map(asTask)); + // Journalled so a replay can tell a check that discriminates from one + // that only guards (src/tiers.ts failedBefore). + log?.append("note", { + kind: "pre-work-try", + text: `tried before the work: ${failed.length} failed, ${passed.length} passed`, + failed: [...self.failedBeforeWork], + passed: [...self.passedBeforeWork], }); - self.passedBeforeWork = new Set( - passed.map((n) => (n.startsWith("task:") ? n : `task:${n}`)), - ); } finally { self.running = undefined; if (self.cfg.unattended) { @@ -5614,6 +5714,7 @@ export class Engine { withAuthor({ ...got.check, name: got.check.name.startsWith("task:") ? got.check.name : `task:${got.check.name}` }, self.cfg.model), ); if (taskChecks.some((c) => c.name === check.name)) return; + await self.tryLateBeforeWork([check], log); taskChecks = Object.freeze([...taskChecks, check]) as Check[]; self.sealedChecks = taskChecks; self.withholdChecks([check]); @@ -5693,6 +5794,7 @@ export class Engine { log?.append("note", { kind: "late-checks", text: "the drafter finished with no checks to join", inputsSha: draftInputs }); return; } + await self.tryLateBeforeWork(added, log); taskChecks = Object.freeze([...taskChecks, ...added]) as Check[]; taskNotes = Object.freeze([...taskNotes, ...(got?.taskNotes ?? [])]) as string[]; taskSeal = sealOf(taskChecks, taskNotes); diff --git a/src/receipts.ts b/src/receipts.ts index f3b63c2..681bdd1 100644 --- a/src/receipts.ts +++ b/src/receipts.ts @@ -12,7 +12,7 @@ import { createHash } from "node:crypto"; import { join } from "node:path"; import { MIN_SECRET_CHARS, redact } from "./redact.js"; import type { BarResult, CheckAuthor } from "./types.js"; -import { authorWords, claimLabel, resultAuthorWords } from "./tiers.js"; +import { authorWords, claimLabel, noteCoverage, resultAuthorWords, UNTESTED_CLAIM, type Tier } from "./tiers.js"; import { stateDir } from "./statedir.js"; import { WITHHELD, maskDeep, maskText } from "./withhold.js"; @@ -86,7 +86,7 @@ export type ReceiptRecord = { * (`amendTier`), so the index row is the last word and the receipt file, * hash-bound when it was written, is never rewritten. */ - tier?: "verified" | "passed-checks" | "passed-own-checks"; + tier?: Tier; tierReason?: string; /** The claim in words (src/tiers.ts claimLabel), e.g. "verified (independent checks: m)". */ claim?: string; @@ -372,7 +372,7 @@ export class Receipts { /** True when that figure rests on molt's own token estimate. */ costEstimated?: boolean; /** The tier the passing checks earned, before any independent review (src/tiers.ts). */ - tier?: { tier: "verified" | "passed-checks" | "passed-own-checks"; reason?: string; evidence: string; basis?: "person" | "independent" | "own"; by?: string[]; worker?: string }; + tier?: { tier: Tier; reason?: string; evidence: string; basis?: "person" | "independent" | "own"; by?: string[]; worker?: string }; /** * Who wrote each check, by check name, recorded at seal time: the worker * model, a separate judge, a person, or the reference writer. A check not @@ -450,8 +450,16 @@ export class Receipts { ? `\n\nEvidence: ${args.tier.evidence}. A passing check of this class earns the word "verified".` : args.tier.tier === "passed-own-checks" ? `\n\nPassed own checks, not verified: ${args.tier.reason}. A model never judges its own work; a check from a person or another model is needed for "verified".` - : `\n\nPassed its checks, not verified: ${args.tier.reason}. The strongest passing check is ${args.tier.evidence}.`; - verdictLine += `\n\nClaim: ${args.tier.tier === "passed-checks" ? "passed its checks, not verified" : claimLabel("verified", args.tier)}.`; + : args.tier.tier === "passed-untested" + ? `\n\nPassed checks that did not test this work, not verified: ${args.tier.reason}. A check that passes on the tree as it was before the work cannot tell this work from none; "verified" needs one that failed before the work and passes now.` + : `\n\nPassed its checks, not verified: ${args.tier.reason}. The strongest passing check is ${args.tier.evidence}.`; + verdictLine += `\n\nClaim: ${ + args.tier.tier === "passed-checks" + ? "passed its checks, not verified" + : args.tier.tier === "passed-untested" + ? UNTESTED_CLAIM + : claimLabel("verified", args.tier) + }.`; } if (args.revealed?.length) { verdictLine += `\n\nThe command of ${args.revealed.map((n) => `\`${n}\``).join(", ")} was shown to the model after it failed the same way twice; the work was judged against a check the model had read.`; @@ -486,8 +494,29 @@ export class Receipts { "because they were asked for, and Maat will not report them as met:", "", ); - for (const n of task.notes) asked.push(`- ${n}`); - asked.push(""); + // Coverage, display only: which of these a check that failed before + // the work and passes now plausibly speaks to, matched by shared words. + const disc = args.result.results.filter((r) => r.ok && r.beforeWork === "failed"); + const commandOf = (name: string) => { + const bare = name.replace(/^task:/, ""); + const line = task.checks.find((c) => c === bare || c.startsWith(`${bare}: `) || c === name || c.startsWith(`${name}: `)); + return line ? line.slice(line.indexOf(": ") + 2) : ""; + }; + const cover = noteCoverage(task.notes, disc.map((r) => ({ name: r.name, text: `${commandOf(r.name)} ${r.detail ?? ""}` }))); + for (const c of cover) { + asked.push( + `- ${c.note} — ${ + c.by.length + ? `plausibly covered by ${c.by.map((b) => `\`${b}\``).join(", ")}, which failed before the work and passes now` + : "not matched to any check that failed before the work and passes now" + }`, + ); + } + asked.push( + "", + "Coverage is matched by shared words, for the reader only; it gates nothing.", + "", + ); } } @@ -613,6 +642,17 @@ export class Receipts { `check: ${r.name}`, `kind: ${r.kind}`, `written by: ${resultAuthorWords(r, args.authors?.[r.name])}`, + ...(r.beforeWork + ? [ + `before the work: ${ + r.beforeWork === "failed" + ? "failed (this check can tell the work from none)" + : r.beforeWork === "passed" + ? "passed (a guard: it cannot tell the work from none)" + : "not tried (it could not run then, or joined later), so it cannot tell the work from none" + }`, + ] + : []), `command: ${r.detail}`, ...(r.ranInPlace ? [`ran in place: no throwaway copy of the tree — ${r.ranInPlace}`] : []), `exit: ${r.exitCode ?? "n/a"}`, @@ -740,7 +780,7 @@ export class Receipts { * is hash-bound and stays byte for byte as written. A row that is not * there, or an index that cannot be read, is left alone. */ - amendTier(file: string, tier: { tier: "verified" | "passed-checks" | "passed-own-checks"; reason?: string; claim?: string }): boolean { + amendTier(file: string, tier: { tier: Tier; reason?: string; claim?: string }): boolean { try { if (!existsSync(this.indexPath)) return false; let hit = false; diff --git a/src/tiers.ts b/src/tiers.ts index 639954b..6e6896d 100644 --- a/src/tiers.ts +++ b/src/tiers.ts @@ -137,7 +137,16 @@ export function evidenceTags(run: string, surface: boolean | undefined): string[ * bench, self-judged arms were right in 125 of 174 "verified" claims (72%), * separate-judge arms in 48 of 55 (87%). */ -export type Tier = "verified" | "passed-checks" | "passed-own-checks"; +/** + * "passed-untested": an independent check that ran the work and asserted a + * value passed, but none of them FAILED on the tree as it was before the work. + * A check that passes on the untouched tree too cannot tell this work from no + * work; it guards against a regression and proves nothing here. On the + * 2026-10-07 container bench, a judge's `python3 server.py & sleep 1; curl + * ... || echo fail` passed before server.py existed and earned a wrong + * "verified" (the grader got a non-JSON 404). + */ +export type Tier = "verified" | "passed-checks" | "passed-own-checks" | "passed-untested"; export type TierVerdict = { tier: Tier; reason?: string; @@ -263,6 +272,7 @@ export function withAuthor | undefined): string { const gap = tier?.reviewGap ? `, ${tier.reviewGap}` : ""; if (tier?.tier === "passed-own-checks") return `passed own checks (${tier.worker || "the worker model"}), not verified${gap}`; + if (tier?.tier === "passed-untested") return `${UNTESTED_CLAIM}${gap}`; if (outcome === "verified" && tier?.tier === "verified") { if (tier.basis === "person") return `verified (your checks)${gap}`; if (tier.basis === "independent") return `verified (independent checks: ${(tier.by ?? []).join(", ") || "another model"})${gap}`; @@ -270,6 +280,20 @@ export function claimLabel(outcome: string, tier: Pick | undefined): string { + const why = (n: string) => `\`${n}\` ${passedBefore?.has(n) ? "passed before the work began too" : "was not tried before the work began"}`; + return names.length === 1 + ? `the only independent check that ran the work and asserted a value did not fail before the work: ${why(names[0]!)}` + : `no independent check that ran the work and asserted a value failed before the work: ${names.map(why).join("; ")}`; +} + /** The parts of a job_end event the verdict words are made from. */ export type JobEndWords = { outcome?: unknown; @@ -299,7 +323,7 @@ export function jobEndWords(ev: JobEndWords): string { const contradicted = verified && !!ev.review && ev.review.confirmed === false; const unreviewed = verified && !ev.review && !!ev.unreviewed; if (ev.tier === "passed-checks") return passedChecksWords(typeof ev.tierReason === "string" ? ev.tierReason : undefined); - if (ev.tier === "passed-own-checks" && claim) return claim; + if ((ev.tier === "passed-own-checks" || ev.tier === "passed-untested") && claim) return claim; if (verified && Array.isArray(ev.revealed) && ev.revealed.length) return "verified (checks shown after a repeat failure)"; if (verified && claim) { if (contradicted) return /, unconfirmed$/.test(claim) ? claim : `${claim}, unconfirmed`; @@ -334,7 +358,8 @@ export function contradictions(review: { votes: string; violations: unknown[] } * The tier a passing turn has earned. "verified" needs (a) a passing check a * person wrote, or a passing drafted check that runs the deliverable (not * surface) AND asserts a value AND was written by someone other than the - * worker model, and (b) an independent review, when one ran, that found no + * worker model AND failed on the tree before the work began (it discriminates: + * see `failedBefore`), and (b) an independent review, when one ran, that found no * contradiction. (c) — the turn not ended by the clock or the provider — is * decided before this, by `passedAtEnd`. * @@ -348,6 +373,11 @@ export function contradictions(review: { votes: string; violations: unknown[] } * was written by the worker model (no judge, or a judge that is the same * model reached another way), the best the turn earns is "passed-own-checks". * A check whose author was never recorded counts as the worker's. + * + * Never let a check that cannot tell the work from no work vouch for it: + * when independent runs+value checks passed but none had failed before the + * work, the turn earns "passed-untested" — they guard against a regression + * and did not test this work. */ export function tierOf(args: { results: readonly (Pick & { name?: string; kind?: CheckResult["kind"] })[]; @@ -356,14 +386,22 @@ export function tierOf(args: { unreviewed?: boolean; /** * Experimental (MAAT_REVIEW_ADVISORY=1, `--review-advisory`): the review is - * recorded but gates nothing, and the evidence rule is stricter in return — - * the passing runs+value drafted check must also have FAILED on the untouched - * project, i.e. not be in `guards` (the checks that passed before the work). - * reports/checkquality-2026-10-06.md §2.5 measured the 3-vote review as a + * recorded but gates nothing. (The evidence rule it once added here — the + * value check must have FAILED on the untouched project — now holds in every + * mode: see `failedBefore`.) reports/checkquality-2026-10-06.md §2.5 measured the 3-vote review as a * likelihood ratio of about 1. */ reviewAdvisory?: boolean; - /** Names (as in `results`) of checks that already passed before the work. */ + /** + * Names (as in `results`) of checks that were tried on the tree before the + * work began and FAILED there. Only such a check discriminates: "verified" + * needs a passing independent runs+value check named here. A check that + * passed before the work, could not run then, or joined after the work + * began (no try) is not named here and cannot earn the word. Absent means + * no check was tried, so none discriminates. + */ + failedBefore?: ReadonlySet; + /** Names of checks that passed before the work too: only words the reason, never the tier. */ guards?: ReadonlySet; /** The worker model, under every name it ran as (configured id, the id the backend reported). */ worker?: string | readonly string[]; @@ -394,11 +432,12 @@ export function tierOf(args: { // worker that changes any file passes them. const person = passing.some((r) => personCheck(r) && authorOf(r).kind === "person"); const drafted = passing.filter((r) => r.hidden === true && r.tags?.includes("task") && authorOf(r).kind !== "person"); - const strongAll = drafted.filter( - (r) => !r.tags?.includes("surface") && r.tags?.includes("value") && !(args.reviewAdvisory && args.guards?.has(r.name ?? "")), - ); + const strongAll = drafted.filter((r) => !r.tags?.includes("surface") && r.tags?.includes("value")); const strongIndependent = strongAll.filter((r) => independentOf(authorOf(r), workerNames)); const strong = strongAll.length > 0; + // Of those, the ones that failed on the tree before the work and pass now: + // the only passes that show THIS work did something. + const discriminating = strongIndependent.filter((r) => args.failedBefore?.has(r.name ?? "") === true); const by = [...new Set(strongIndependent.map((r) => authorOf(r).model ?? "another model"))]; const basis: TierVerdict["basis"] = person ? "person" : strongIndependent.length ? "independent" : strong ? "own" : undefined; const who = { ...(basis ? { basis } : {}), ...(by.length && !person ? { by } : {}), ...(worker ? { worker } : {}) }; @@ -412,20 +451,21 @@ export function tierOf(args: { ? "surface" : "none"; const ownReason = `every passing check that ran the work and asserted a value was written by the worker model${worker ? ` (${worker})` : ""}`; - const earned = person || strongIndependent.length > 0; + const earned = person || discriminating.length > 0; + const untestedReason = untestedWords(strongIndependent.map((r) => r.name ?? ""), args.guards); if (args.reviewAdvisory) { const n = contradictions(args.review); const reviewNote = n > 0 ? `advisory: the independent review found ${args.review!.votes} contradicting the task` : args.unreviewed ? "advisory: the independent review did not run" : undefined; const gap: Pick = n > 0 ? { reviewGap: "unconfirmed" } : args.unreviewed ? { reviewGap: "unreviewed" } : {}; if (earned) return { tier: "verified", evidence, ...who, ...(reviewNote ? { reviewNote } : {}), ...gap }; + if (strongIndependent.length) return { tier: "passed-untested", evidence, ...who, reason: untestedReason, ...(reviewNote ? { reviewNote } : {}), ...gap }; if (strong) return { tier: "passed-own-checks", evidence, ...who, reason: ownReason, ...(reviewNote ? { reviewNote } : {}), ...gap }; return { tier: "passed-checks", evidence, ...who, - reason: drafted.some((r) => !r.tags?.includes("surface") && r.tags?.includes("value")) - ? "the only passing value check also passed before the work began" - : evidence === "runs" + reason: + evidence === "runs" ? "no check that ran the work asserted an expected value" : "no passing check ran the work and asserted an expected value", ...(reviewNote ? { reviewNote } : {}), @@ -436,6 +476,7 @@ export function tierOf(args: { } if (args.unreviewed && !person) return { tier: "passed-checks", evidence, ...who, reason: "the independent review did not run" }; if (earned) return { tier: "verified", evidence, ...who }; + if (strongIndependent.length) return { tier: "passed-untested", evidence, ...who, reason: untestedReason }; if (strong) return { tier: "passed-own-checks", evidence, ...who, reason: ownReason }; return { tier: "passed-checks", @@ -447,3 +488,31 @@ export function tierOf(args: { : "no passing check ran the work and asserted an expected value", }; } + +const COVER_STOP = new Set( + "the and for with that this from into must should shall will each every only all any are has have not but its was were been being file files output input when then than also exactly whole same handle handles".split(" "), +); + +/** The words of a note or check that carry meaning: 3+ letters, not glue. */ +function contentWords(text: string): Set { + return new Set((text.toLowerCase().match(/[a-z][a-z0-9]{2,}/g) ?? []).filter((w) => !COVER_STOP.has(w))); +} + +/** + * Which discriminating checks plausibly cover each "Recorded, not verified" + * note, matched by the words they share: at least two, and at least 60% of + * the note's meaningful words appearing in the check's name or command. + * Display only — it gates nothing, and a word match is a hint, not proof. + */ +export function noteCoverage( + notes: readonly string[], + checks: readonly { name: string; text: string }[], +): { note: string; by: string[] }[] { + const cw = checks.map((c) => ({ name: c.name, words: contentWords(`${c.name.replace(/^task:/, "")} ${c.text}`) })); + return notes.map((note) => { + const nw = [...contentWords(note)]; + const need = Math.max(2, Math.ceil(nw.length * 0.6)); + const by = cw.filter((c) => nw.filter((w) => c.words.has(w)).length >= need).map((c) => c.name); + return { note, by }; + }); +} diff --git a/src/types.ts b/src/types.ts index 674acc5..09ff34d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -364,6 +364,13 @@ export type CheckResult = { * displayed as though it had proven something. */ established?: boolean; + /** + * How this sealed task check fared on the tree before the work began: + * "failed" (it discriminates: it cannot pass on the untouched tree), + * "passed" (a guard: it passes with or without the work), or "untried" + * (broken then, or joined after the work began). Absent for every other check. + */ + beforeWork?: "failed" | "passed" | "untried"; /** * Why this check was not run at all, when it was not. * @@ -680,7 +687,7 @@ export type EngineEvent = * check) and no reviewer contradicted it. "passed-checks" is the outcome * "unverified" with `tierReason` saying why the word was not earned. */ - tier?: "verified" | "passed-checks" | "passed-own-checks"; + tier?: "verified" | "passed-checks" | "passed-own-checks" | "passed-untested"; tierReason?: string; /** * The claim in words, the same on every surface and in the bench's diff --git a/test/discriminating-checks.test.ts b/test/discriminating-checks.test.ts new file mode 100644 index 0000000..1db5232 --- /dev/null +++ b/test/discriminating-checks.test.ts @@ -0,0 +1,142 @@ +/** + * "verified" needs a check that tells this work from no work. + * + * On the 2026-10-07 container bench a separate judge drafted + * `python3 server.py 8080 & sleep 1; curl -f .../items || echo 'fail'` and two + * more like it. Each exits 0 whatever happens, so each passed on the tree + * before server.py existed, and the run was labelled verified; the grader got + * a non-JSON 404. A passing independent runs+value check now earns the word + * only when it FAILED on the tree before the work (the pre-work try in + * sealCriteria). One that passed then too, or was never tried then, guards + * against a regression and proves nothing about this work: the tier is + * "passed-untested", outcome unverified, exit code 3. + */ +import assert from "node:assert/strict"; +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { Engine } from "../src/engine.js"; +import { Journal } from "../src/journal.js"; +import { Receipts } from "../src/receipts.js"; +import { claimLabel, noteCoverage, tierOf, UNTESTED_CLAIM } from "../src/tiers.js"; +import type { Check, CheckAuthor } from "../src/types.js"; +import { allowAll, drain, scriptedProvider, workspace, type ScriptedTurn } from "./helpers.js"; + +const JUDGE: CheckAuthor = { kind: "judge", model: "qwen3-coder-30b-a3b" }; +const value = (name: string) => ({ name, ok: true, hidden: true as const, tags: ["task", "value"] }); + +describe("tierOf: a check that did not fail before the work cannot earn verified", () => { + const names = ["task:server-starts-and-listens", "task:get-all-items", "task:post-new-item"]; + const authors = new Map(names.map((n) => [n, JUDGE])); + const worker = "qwen/qwen3-235b-a22b-2507"; + + it("the http-json-server run: every independent value check passed before the work too, so not verified", () => { + const t = tierOf({ results: names.map(value), worker, authors, guards: new Set(names), failedBefore: new Set() }); + assert.equal(t.tier, "passed-untested"); + assert.equal(t.basis, "independent", "the checks were independent; independence was not the problem"); + assert.match(t.reason!, /`task:get-all-items` passed before the work began too/); + assert.equal(claimLabel("unverified", t), UNTESTED_CLAIM); + assert.equal(UNTESTED_CLAIM, "passed checks that did not test this work, not verified"); + }); + + it("a check that failed before the work and passes after: verified", () => { + const t = tierOf({ results: names.map(value), worker, authors, guards: new Set(names.slice(0, 2)), failedBefore: new Set([names[2]!]) }); + assert.deepEqual([t.tier, t.basis, t.by], ["verified", "independent", ["qwen3-coder-30b-a3b"]]); + }); + + it("a check with no pre-work try (broken then, or joined late with no copy) does not discriminate", () => { + const t = tierOf({ results: [value(names[0]!)], worker, authors, failedBefore: new Set() }); + assert.equal(t.tier, "passed-untested"); + assert.match(t.reason!, /was not tried before the work began/); + assert.equal(tierOf({ results: [value(names[0]!)], worker, authors }).tier, "passed-untested", "no pre-work record at all: nothing discriminates"); + }); + + it("only independent value checks count: a discriminating check the worker wrote is still its own", () => { + const mixed = new Map([[names[0]!, JUDGE], [names[1]!, { kind: "worker", model: worker }]]); + const t = tierOf({ results: [value(names[0]!), value(names[1]!)], worker, authors: mixed, guards: new Set([names[0]!]), failedBefore: new Set([names[1]!]) }); + assert.equal(t.tier, "passed-untested"); + const own = tierOf({ results: [value(names[1]!)], worker, authors: mixed, failedBefore: new Set([names[1]!]) }); + assert.equal(own.tier, "passed-own-checks", "the authorship rule is unchanged"); + }); + + it("a failing or surface-only discriminating check carries nothing; a person's check still verifies", () => { + const fb = new Set([names[0]!]); + assert.equal(tierOf({ results: [{ ...value(names[0]!), ok: false }], worker, authors, failedBefore: fb }).tier, "passed-checks"); + assert.equal(tierOf({ results: [{ ...value(names[0]!), tags: ["task", "surface", "value"] }], worker, authors, failedBefore: fb }).tier, "passed-checks"); + assert.equal(tierOf({ results: [{ name: "project", ok: true, kind: "command" as const, tags: [] }], worker }).tier, "verified"); + }); +}); + +describe("noteCoverage", () => { + it("matches a note to a check only on most of its meaningful words", () => { + const cover = noteCoverage( + ["Server must handle GET /items returning sorted list by id", "Rows keep the input order"], + [ + { name: "task:get-all-items", text: "python3 server.py 8081 & sleep 1; curl -s http://127.0.0.1:8081/items | jq -e '.[0].id == 1'" }, + { name: "task:order-kept", text: "python3 -c 'rows keep input order' && diff clean.csv expected.csv" }, + ], + ); + assert.deepEqual(cover[0]!.by, [], "sharing get/items/server is not covering a sort"); + assert.deepEqual(cover[1]!.by, ["task:order-kept"]); + }); +}); + +describe("the rule in a turn", () => { + const TASK = "Write out.txt containing exactly the word hello."; + const work: ScriptedTurn[] = [ + { calls: [{ name: "write_file", args: { path: "out.txt", content: "hello\n" } }, { name: "write_file", args: { path: "tool.sh", content: "exit 0\n" } }] }, + { text: "Done." }, + ]; + const check = (name: string, run: string): Check => + ({ name, kind: "command", run, timeoutMs: 5_000, expectExit: 0, tags: ["task", "value"], hidden: true, author: JUDGE }) as Check; + + async function turn(checks: Check[], notes: string[] = []) { + const ws = workspace(); + try { + const provider = scriptedProvider(work); + const journal = new Journal(ws.dir, "disc"); + const engine = new Engine({ + baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: provider.fetchFn, bar: null, + receipts: new Receipts(ws.dir), stream: false, autonomy: "high", journal, + }); + const events = await drain(engine.run(TASK, allowAll, { taskChecks: checks, taskNotes: notes })); + const end = events.find((e) => e.kind === "job_end"); + assert.ok(end && end.kind === "job_end"); + const dir = join(ws.dir, ".maat", "receipts"); + const rows = readFileSync(join(dir, "index.jsonl"), "utf8").trim().split("\n").map((l) => JSON.parse(l)); + const receipt = readdirSync(dir).filter((f) => f.endsWith(".md")).sort().map((f) => readFileSync(join(dir, f), "utf8")).join("\n"); + const tried = Journal.read(journal.path).filter((e) => e.kind === "note" && e.data.kind === "pre-work-try"); + return { end, events, rows, receipt, tried }; + } finally { + ws.cleanup(); + } + } + + it("only checks that passed before the work: passed checks that did not test this work, unverified", async () => { + // Exits 0 with or without the work, like `curl ... || echo 'fail'`. + const { end, events, rows, receipt, tried } = await turn([check("always", `[ "$(echo ok)" = "ok" ] || echo fail`)], ["out.txt holds hello"]); + assert.deepEqual([end.outcome, end.tier, end.claim], ["unverified", "passed-untested", UNTESTED_CLAIM]); + assert.match(end.tierReason!, /`task:always` passed before the work began too/); + assert.ok(events.some((e) => e.kind === "info" && e.text.startsWith(UNTESTED_CLAIM))); + assert.deepEqual([rows.at(-1).tier, rows.at(-1).claim], ["passed-untested", UNTESTED_CLAIM]); + assert.match(receipt, /before the work: passed \(a guard/); + assert.match(receipt, /out\.txt holds hello — not matched to any check that failed before the work/); + assert.deepEqual(tried[0]!.data.passed, ["task:always"]); + }); + + it("a check that failed before the work and passes after: verified, and the receipt says it discriminates", async () => { + const { end, receipt, tried } = await turn([check("always", `[ "$(echo ok)" = "ok" ]`), check("greeting", "grep -qx hello out.txt")], ["out.txt holds exactly hello greeting"]); + assert.deepEqual([end.outcome, end.tier, end.claim], ["verified", "verified", "verified (independent checks: qwen3-coder-30b-a3b)"]); + assert.match(receipt, /before the work: failed \(this check can tell the work from none\)/); + assert.match(receipt, /out\.txt holds exactly hello greeting — plausibly covered by `task:greeting`/); + assert.deepEqual(tried[0]!.data.failed, ["task:greeting"]); + }); + + it("a check that could not run before the work has no pre-work try: not verified", async () => { + // `sh tool.sh` before tool.sh exists exits 127: the try found it broken, not failing. + const { end, receipt } = await turn([check("tool", "sh tool.sh")]); + assert.deepEqual([end.outcome, end.tier], ["unverified", "passed-untested"]); + assert.match(end.tierReason!, /`task:tool` was not tried before the work began/); + assert.match(receipt, /before the work: not tried/); + }); +}); diff --git a/test/exit-codes.test.ts b/test/exit-codes.test.ts index e8842c4..d4512da 100644 --- a/test/exit-codes.test.ts +++ b/test/exit-codes.test.ts @@ -56,7 +56,7 @@ async function provider(): Promise { * criteria drafter gets one check that the work satisfies, and the critic * judges it a real run of the deliverable. */ -async function draftingProvider(): Promise { +async function draftingProvider(check = "[ \"$(cat a.txt)\" = \"a\" ]"): Promise { let n = 0; const server: Server = createServer((req, res) => { let body = ""; @@ -67,7 +67,7 @@ async function draftingProvider(): Promise { res.end(JSON.stringify({ choices: [{ message: content, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })); }; if (body.includes("You draft acceptance criteria")) { - return reply({ role: "assistant", content: JSON.stringify({ checks: [{ name: "made", run: "[ \"$(cat a.txt)\" = \"a\" ]" }], notes: [] }) }); + return reply({ role: "assistant", content: JSON.stringify({ checks: [{ name: "made", run: check }], notes: [] }) }); } if (body.includes("You review acceptance checks")) { return reply({ role: "assistant", content: JSON.stringify({ checks: [{ name: "made", verdict: "runs", quote: "" }] }) }); @@ -168,6 +168,13 @@ describe("molt run's exit code", () => { assert.equal(same, 3); }); + it("is 3 when the only independent check met also passed before the work (it did not test this work)", async () => { + const url = await draftingProvider("[ \"$(echo a)\" = \"a\" ]"); + const code = await molt(["run", "write a.txt", "--url", url, "--model", "m", "--key", "k", + "--cwd", project(), "--no-stream", "--yes", "--criteria", "auto", "--judge", "j"]); + assert.equal(code, 3); + }); + it("is 3 when required checks were left out, not 1 as though something failed", async () => { assert.equal(await run(project(BAR), "--skip", "slow"), 3); }); diff --git a/test/harness.test.ts b/test/harness.test.ts index 8116bb8..be9a3d5 100644 --- a/test/harness.test.ts +++ b/test/harness.test.ts @@ -488,7 +488,7 @@ describe("self-checked work", () => { const ws = workspace(); try { const hidden = { name: "own", kind: "command" as const, run: "grep -qx x x.txt", timeoutMs: 5_000, expectExit: 0, tags: ["task", "value"], hidden: true, author: { kind: "judge" as const, model: "judge-j" } }; - const shown = { ...hidden, name: "chosen", hidden: undefined }; + const shown = { ...hidden, name: "chosen", hidden: undefined, author: { kind: "person" as const } }; const a = engineFor(ws.dir, [{ calls: [{ name: "write_file", args: { path: "x.txt", content: "x" } }] }, { text: "done" }]); const ea = await drain(a.engine.run("make x", allowAll, { taskChecks: [hidden] })); const ja = ea.find((e) => e.kind === "job_end"); diff --git a/test/independent-checks.test.ts b/test/independent-checks.test.ts index 6a71efa..070c702 100644 --- a/test/independent-checks.test.ts +++ b/test/independent-checks.test.ts @@ -61,6 +61,8 @@ describe("independentOf", () => { describe("tierOf: who wrote the checks", () => { const value = (name: string) => ({ name, ok: true, hidden: true as const, kind: "command" as const, tags: ["task", "value"] }); const by = (a: CheckAuthor, name = "task:v") => new Map([[name, a]]); + // Every check here failed before the work: the discrimination rule has its own tests. + const failedBefore = new Set(["task:v", "task:own", "task:judged"]); it("worker-authored checks only: not verified", () => { const t = tierOf({ results: [value("task:v")], worker: "m", authors: by({ kind: "worker", model: "m" }) }); @@ -76,7 +78,7 @@ describe("tierOf: who wrote the checks", () => { }); it("judge-authored: verified, and the label names the judge", () => { - const t = tierOf({ results: [value("task:v")], worker: "m", authors: by({ kind: "judge", model: "judge-j" }) }); + const t = tierOf({ results: [value("task:v")], worker: "m", authors: by({ kind: "judge", model: "judge-j" }), failedBefore }); assert.deepEqual([t.tier, t.basis, t.by], ["verified", "independent", ["judge-j"]]); assert.equal(claimLabel("verified", t), "verified (independent checks: judge-j)"); }); @@ -104,7 +106,7 @@ describe("tierOf: who wrote the checks", () => { ["task:own", { kind: "worker", model: "m" }], ["task:judged", { kind: "judge", model: "j" }], ]); - const t = tierOf({ results: [value("task:own"), value("task:judged")], worker: "m", authors }); + const t = tierOf({ results: [value("task:own"), value("task:judged")], worker: "m", authors, failedBefore }); assert.equal(claimLabel("verified", t), "verified (independent checks: j)"); // ...but an independent check that asserted no value does not carry it. const weak = tierOf({ @@ -141,12 +143,14 @@ describe("tierOf: who wrote the checks", () => { it("review-advisory: a contradicted or missing review qualifies every label", () => { const judged = by({ kind: "judge", model: "j" }); - const contradicted = tierOf({ results: [value("task:v")], worker: "m", authors: judged, review: { votes: "2/3", violations: [] }, reviewAdvisory: true }); + // Advisory mode also needs the check to have failed before the work (#34). + const fb = { failedBefore: new Set(["task:v"]) }; + const contradicted = tierOf({ results: [value("task:v")], worker: "m", authors: judged, review: { votes: "2/3", violations: [] }, reviewAdvisory: true, ...fb }); assert.deepEqual([contradicted.tier, contradicted.reviewGap], ["verified", "unconfirmed"]); assert.equal(claimLabel("verified", contradicted), "verified (independent checks: j), unconfirmed"); - const missing = tierOf({ results: [value("task:v")], worker: "m", authors: judged, unreviewed: true, reviewAdvisory: true }); + const missing = tierOf({ results: [value("task:v")], worker: "m", authors: judged, unreviewed: true, reviewAdvisory: true, ...fb }); assert.equal(claimLabel("verified", missing), "verified (independent checks: j), unreviewed"); - const fine = tierOf({ results: [value("task:v")], worker: "m", authors: judged, review: { votes: "0/3", violations: [] }, reviewAdvisory: true }); + const fine = tierOf({ results: [value("task:v")], worker: "m", authors: judged, review: { votes: "0/3", violations: [] }, reviewAdvisory: true, ...fb }); assert.equal(claimLabel("verified", fine), "verified (independent checks: j)"); const own = tierOf({ results: [value("task:v")], worker: "m", review: { votes: "2/3", violations: [] }, reviewAdvisory: true }); assert.equal(claimLabel("unverified", own), "passed own checks (m), not verified, unconfirmed"); @@ -166,7 +170,7 @@ describe("tierOf: who wrote the checks", () => { }); it("a reviewer contradiction still outranks authorship", () => { - const t = tierOf({ results: [value("task:v")], worker: "m", authors: by({ kind: "judge", model: "j" }), review: { votes: "2/3", violations: [] } }); + const t = tierOf({ results: [value("task:v")], worker: "m", authors: by({ kind: "judge", model: "j" }), review: { votes: "2/3", violations: [] }, failedBefore }); assert.equal(t.tier, "passed-checks"); assert.equal(claimLabel("unverified", t), "unverified"); }); diff --git a/test/judgment.test.ts b/test/judgment.test.ts index 4c4d898..3bc45b0 100644 --- a/test/judgment.test.ts +++ b/test/judgment.test.ts @@ -81,7 +81,7 @@ describe("which jobs open a case", () => { it("verified: the scale settled it, no case", async () => { const ws = workspace(); try { - const { end } = await jobEnd(engineIn(ws.dir, writesThenClaims()), "make a", [check("fine", "true", true)]); + const { end } = await jobEnd(engineIn(ws.dir, writesThenClaims()), "make a", [check("fine", "grep -qx 1 a.txt", true)]); assert.equal(end.outcome, "verified"); assert.equal(end.case, undefined); assert.equal(new Judgments(ws.dir).all().length, 0); diff --git a/test/review-advisory.test.ts b/test/review-advisory.test.ts index dfa0fa1..0e707b1 100644 --- a/test/review-advisory.test.ts +++ b/test/review-advisory.test.ts @@ -1,7 +1,8 @@ /** * Experimental review-advisory mode (`--review-advisory`, MAAT_REVIEW_ADVISORY=1): - * the review is recorded and does not gate; "verified" additionally needs a - * drafted runs+value check that did not already pass before the work. + * the review is recorded and does not gate. The rule that "verified" needs a + * drafted runs+value check that failed before the work holds here as in + * every mode. * Default behaviour is pinned beside it. */ import assert from "node:assert/strict"; @@ -13,7 +14,11 @@ describe("tierOf, review advisory", () => { const ck = (name: string, tags: string[]) => ({ name, ok: true, hidden: true as const, tags }); const strong = ck("task:counts", ["task", "value"]); // Drafted by a separate judge; tierOf's authorship rule has its own tests (independent-checks.test.ts). - const J = { worker: "worker-m", authors: new Map(["task:counts", "task:other", "task:a"].map((n) => [n, { kind: "judge" as const, model: "judge-j" }])) }; + const J = { + worker: "worker-m", + authors: new Map(["task:counts", "task:other", "task:a"].map((n) => [n, { kind: "judge" as const, model: "judge-j" }])), + failedBefore: new Set(["task:counts", "task:other", "task:a"]), + }; it("the review is recorded and does not gate", () => { const review = { votes: "2/3", violations: [] }; @@ -25,14 +30,16 @@ describe("tierOf, review advisory", () => { assert.equal(tierOf({ ...J, results: [strong], unreviewed: true }).tier, "passed-checks", "default unchanged"); }); - it("needs a value check that did not already pass before the work", () => { + it("needs a value check that failed before the work, in this mode as in every other", () => { const guards = new Set(["task:counts"]); - assert.equal(tierOf({ ...J, results: [strong], guards }).tier, "verified", "off by default: guards are ignored"); - const r = tierOf({ ...J, results: [strong], guards, reviewAdvisory: true }); - assert.equal(r.tier, "passed-checks"); - assert.match(r.reason!, /also passed before the work/); - const two = tierOf({ ...J, results: [strong, ck("task:other", ["task", "value"])], guards, reviewAdvisory: true }); - assert.equal(two.tier, "verified", "another value check that failed on the pristine tree carries it"); + const failedBefore = new Set(["task:other"]); + for (const reviewAdvisory of [false, true]) { + const r = tierOf({ ...J, results: [strong], guards, failedBefore, reviewAdvisory }); + assert.equal(r.tier, "passed-untested"); + assert.match(r.reason!, /`task:counts` passed before the work began too/); + const two = tierOf({ ...J, results: [strong, ck("task:other", ["task", "value"])], guards, failedBefore, reviewAdvisory }); + assert.equal(two.tier, "verified", "another value check that failed on the pristine tree carries it"); + } }); it("a surface-only or value-less pass is still not verified", () => { diff --git a/test/tiers.test.ts b/test/tiers.test.ts index 72070ed..9da9dda 100644 --- a/test/tiers.test.ts +++ b/test/tiers.test.ts @@ -78,7 +78,8 @@ describe("the value tag", () => { describe("tierOf", () => { const ok = (tags: string[], hidden = true) => ({ name: "c", ok: true, hidden: hidden || undefined, kind: "command" as const, tags }); // Drafted by a separate judge model: the authorship rule is pinned in independent-checks.test.ts. - const J = { worker: "worker-m", authors: new Map([["c", { kind: "judge" as const, model: "judge-j" }]]) }; + // ...and it failed on the tree before the work (the discrimination rule is pinned in discriminating-checks.test.ts). + const J = { worker: "worker-m", authors: new Map([["c", { kind: "judge" as const, model: "judge-j" }]]), failedBefore: new Set(["c"]) }; it("needs a passing check that runs the work and asserts a value", () => { assert.equal(tierOf({ ...J, results: [ok(["task", "value"])] }).tier, "verified"); assert.equal(tierOf({ ...J, results: [ok(["task"]), ok(["task", "value"])] }).tier, "verified"); From 13c050c8909027ea2ab3997ed261fe7517d36d8b Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 14:50:58 -0700 Subject: [PATCH 2/4] Redraft checks that pass before the work; discrimination gate opt-in The gate from the previous commit is now opt-in (--require-discriminating, MAAT_REQUIRE_DISCRIMINATING=1; --review-advisory implies it). Replayed over 2026-10-07's lanes it cost more right verifieds than it removed wrong ones, so default tiering is #32's again. The pre-work try is still journalled and shown on each receipt row, late checks are still tried on the turn-start copy, and the coverage notes stay. The cause is fixed where the checks are drafted instead. --criteria auto copies the project before the first step, and each drafted check is tried on that copy before it is sealed. One that already passes there cannot show the task was done: it goes back to the drafter once with "this check passes before any work, so it cannot show the task was done; make it fail on the current tree", and is dropped if the redraft still passes. Checks that cannot fail by construction get the same treatment through an always-on lint (L16): a trailing `|| echo ...`, `|| true`, `; exit 0`, `find ... -exec ... \;` (find ignores what -exec returns), PASS/FAIL printed with no failing exit, and, at the try, FAIL printed with exit 0. Drops are journalled as check-lint-dropped with rules P1-passes-before-work and L16-*. bench/local/prework_screen_replay.py runs today's 514 sealed drafted checks against their tasks' pristine trees: 112 would have been sent back (63 by the lint, 49 passing before the work). Of the 29 independent verifieds, 15 rested only on such checks: 3 of the 4 wrong ones (q235 http-json-server, q235 csv-clean, ml4b config-migrate) and 12 right ones, which would be judged by the redraft instead. --- bench/local/prework_screen_replay.py | 167 +++++++++++++++++++++++++++ src/checklint.ts | 46 ++++++++ src/cli.tsx | 35 ++++++ src/criteria.ts | 73 ++++++++++-- src/engine.ts | 18 ++- src/tiers.ts | 34 ++++-- test/discriminating-checks.test.ts | 49 ++++++-- test/exit-codes.test.ts | 2 +- test/prework-screen.test.ts | 136 ++++++++++++++++++++++ test/review-advisory.test.ts | 21 ++-- 10 files changed, 535 insertions(+), 46 deletions(-) create mode 100644 bench/local/prework_screen_replay.py create mode 100644 test/prework-screen.test.ts diff --git a/bench/local/prework_screen_replay.py b/bench/local/prework_screen_replay.py new file mode 100644 index 0000000..b7ecf8e --- /dev/null +++ b/bench/local/prework_screen_replay.py @@ -0,0 +1,167 @@ +"""What the seal-time pre-work screen would have done to 2026-10-07's checks. + + python3 prework_screen_replay.py [LANE_PREFIX ...] (default: 2026-10-07's lanes) + +The screen (src/criteria.ts `screen`, src/checklint.ts `cannotFail`): before a +drafted check is sealed it is linted for constructs that cannot fail (L16) and +tried on a copy of the project taken before the work. One that is flagged, or +already passes there (P1), or prints FAIL and exits 0 (L16), is sent back to +the drafter once; a redraft that is flagged again is dropped. + +Offline proxy: every drafted check sealed in today's runs (released commands, +or the commands older builds journalled with each bar run) is linted through +the built dist/checklint.js and run, under bash with a 10 s limit, on a fresh +copy of its task's pristine tree (bench tasks' own setup). Ports 8080-8099 are +moved to 28080-28099 so a listener on this machine cannot answer for a server +that does not exist yet. The bench container's tool set differs from this +Mac's, so a P1 verdict here is an estimate. + +Then, for each run labelled verified with an independent check behind it +(#32), it asks whether the verdict rested only on checks the screen would +have sent back: such a run would have been judged by whatever the redraft +produced, or by nothing. +""" +from __future__ import annotations + +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +from collections import Counter, defaultdict +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import discriminating_replay as dr # noqa: E402 +from replay_common import pristine, cleanup as cleanup_pristine # noqa: E402 + +REPO = HERE.parents[1] + + +def cannot_fail(runs: list[str]) -> list[str | None]: + js = ( + f"import {{ cannotFail }} from {json.dumps(str(REPO / 'dist/checklint.js'))};" + "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>console.log(JSON.stringify(JSON.parse(s).map(cannotFail))));" + ) + p = subprocess.run(["node", "--input-type=module", "-e", js], input=json.dumps(runs), capture_output=True, text=True, check=True) + return json.loads(p.stdout) + + +def try_before(task: str, run: str) -> tuple[str, bool]: + """('passed'|'failed'|'broken', printed FAIL) for `run` on a fresh copy of the task's pristine tree.""" + tmp = Path(tempfile.mkdtemp(prefix="prework-")) + work = tmp / "w" + shutil.copytree(pristine(task), work, symlinks=True) + cmd = re.sub(r"\b80([89]\d)\b", r"280\1", run) + try: + p = subprocess.Popen(["bash", "-c", cmd], cwd=work, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + start_new_session=True, stdin=subprocess.DEVNULL) + try: + out, err = p.communicate(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(p.pid, signal.SIGKILL) + p.communicate() + return "failed", False + finally: + try: + os.killpg(p.pid, signal.SIGKILL) # whatever it left running in the background + except ProcessLookupError: + pass + if p.returncode in (126, 127): + return "broken", False + if p.returncode == 0: + return "passed", bool(re.search(r"\bFAIL(?:ED)?\b", out + err)) + return "failed", False + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def sealed_checks(tree: Path) -> dict[str, str]: + runs: dict[str, str] = {} + for e in dr.journal(tree): + d = e.get("data", {}) + if e.get("kind") == "bar_run": + for c in d.get("checks", []): + if c.get("name", "").startswith("task:") and not str(c.get("detail", "")).startswith("[withheld"): + runs.setdefault(c["name"], c.get("detail", "")) + elif e.get("kind") == "note" and d.get("kind") == "checks-released": + for c in d.get("checks", []): + runs[dr.task(c["name"])] = c.get("run", "") + return {n: r for n, r in runs.items() if r and n != "task:reference"} + + +def main(argv: list[str]) -> int: + prefixes = argv or dr.TODAY + lanes = sorted(p for p in dr.CW.iterdir() if p.is_dir() and any(re.fullmatch(rf"{re.escape(x)}-\d+", p.name) for x in prefixes)) + cache: dict[tuple[str, str], tuple[str, bool]] = {} + checks: list[dict] = [] + verified: list[dict] = [] + for lane in lanes: + rf = dr.results_file(lane) + rows = dr.grader_rows(rf) if rf else {} + for tree in sorted(lane.iterdir()): + m = dr.RUN_DIR.match(tree.name) + if not m or not tree.is_dir() or not (tree / ".maat/log").exists(): + continue + sealed = sealed_checks(tree) + for n, run in sealed.items(): + checks.append({"run_id": f"{lane.name}/{tree.name}", "task": m["task"], "name": n, "run": run}) + row = rows.get((m["task"], m["arm"], int(m["rep"]))) + claim = (row or {}).get("claim") or "" + if not claim.startswith("verified"): + continue + independent = claim.startswith(("verified (independent checks", "verified (your checks)")) or m["arm"] != "self" + if not independent: + continue + a = dr.analyse(tree, row) + verified.append({"run_id": f"{lane.name}/{tree.name}", "right": bool(row.get("passed")), "strong": a["strong"]}) + flags = cannot_fail([c["run"] for c in checks]) + for c, f in zip(checks, flags): + key = (c["task"], c["run"]) + if key not in cache: + cache[key] = try_before(c["task"], c["run"]) + c["static"] = f + c["before"], c["printed_fail"] = cache[key] + c["redraft"] = bool(f) or c["before"] == "passed" + cleanup_pristine() + by = Counter() + for c in checks: + by["checks"] += 1 + by["static L16"] += bool(c["static"]) + by["P1 passes before (not L16)"] += (not c["static"]) and c["before"] == "passed" + by["L16 printed FAIL, exit 0"] += c["printed_fail"] + by["redrafted (any)"] += c["redraft"] + by["broken before (127/126)"] += c["before"] == "broken" + print("## Sealed drafted checks in today's runs") + for k, v in by.items(): + print(f" {k:32} {v}") + print("\n## Unique commands flagged, by reason") + seen = set() + for c in checks: + if c["redraft"] and (c["task"], c["run"]) not in seen: + seen.add((c["task"], c["run"])) + print(f" {len(seen)} unique (task, command) pairs") + flagged = {(c["run_id"], c["name"]) for c in checks if c["redraft"]} + out = defaultdict(list) + for v in verified: + if not v["strong"]: + kind = "no value check" + elif all((v["run_id"], n) in flagged for n in v["strong"]): + kind = "rested only on checks the screen sends back" + else: + kind = "kept a value check the screen passes" + out[(kind, v["right"])].append(v["run_id"]) + print("\n## Verified runs after #32 (independent), by what the screen does to their value checks") + for (kind, right), ids in sorted(out.items()): + print(f" {'right' if right else 'WRONG'} {kind}: {len(ids)}") + for i in ids: + print(f" {i}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/src/checklint.ts b/src/checklint.ts index 9e40ac0..2753a51 100644 --- a/src/checklint.ts +++ b/src/checklint.ts @@ -30,6 +30,12 @@ * L15 the check changes the work: git checkout/merge/commit/…, rm, * mv, sed -i, tee or a redirect into the project, a package * install (src/checkwrites.ts). Always on, MAAT_CHECK_LINT or not. + * L16 the check cannot fail (cannotFail): a trailing `|| echo …`, + * `|| true`, `; exit 0`; `find … -exec … \;`, whose status ignores + * what -exec ran; a script that prints PASS/FAIL and never exits + * non-zero. Always on. On 2026-10-07 a judge's `curl … || echo + * 'fail'` checks passed before server.py existed and earned a wrong + * "verified". * Left out as noisy in the replay: hand-computed numbers (L14) and paths not * in the tree (L10) — each fired as often on good checks as on bad ones. */ @@ -360,9 +366,49 @@ export function lintAll(run: string, ctx: LintCtx): LintHit[] { // a check that does the merge passes on its own effort. const writes = checkMutates(cmd); if (writes) add("L15-mutates", writes); + const never = cannotFail(cmd); + if (never && !sw) add("L16-cannot-fail", never); return hits; } +/** Mask quoted text so operators inside strings are not read as the shell's. */ +function maskQuotes(s: string): string { + return s.replace(/'[^']*'|"(?:\\.|[^"\\])*"/g, (m) => "_".repeat(m.length)); +} + +/** + * Why this command exits 0 whatever happens, or null. Read off the command + * alone, before anything runs: + * - its last top-level step is `|| echo …`/`|| printf …`, or it ends in + * `|| true`, `|| :`, `|| exit 0`, `; true`, `; exit 0` (evidence.ts); + * - it ends in `find … -exec … \;` (or `';'`): find's own status ignores + * what -exec ran, so `-exec sh -c '… exit 1' \;` reports nothing; + * - it prints a literal PASS/FAIL verdict (`print('PASS' if … else 'FAIL')`, + * `echo FAIL`) and nothing in it exits non-zero, asserts or raises. + */ +export function cannotFail(run: string): string | null { + const cmd = run.trim(); + const masked = maskQuotes(cmd); + const sw = swallowsExit(cmd); + if (sw) return `it ends in \`${sw}\`, so it exits 0 whatever happened`; + const echoTail = /\|\|\s*(?:echo|printf)\b[^|;&]*$/.exec(masked); + if (echoTail) return `it ends in \`${cmd.slice(echoTail.index).trim()}\`, so a failure prints a word and still exits 0`; + const find = /(?:^|[;&|(]\s*)find\s[^|;]*-exec(?:dir)?\s/.exec(masked); + if (find) { + const rest = cmd.slice(find.index); + const end = /(?:\\;|';'|";")/.exec(rest); + const after = end ? maskQuotes(rest.slice(end.index + end[0].length)).trim() : ""; + // Only when nothing after the find reads its output: `find … | grep -q .` is fine. + if (end && !/^\|(?!\|)/.test(after) && !/^[;&|]*\s*\S/.test(after.replace(/^&&\s*(?:echo|printf|true|:)\b.*$/, ""))) { + return "it ends in `find … -exec … \\;`, whose exit status ignores what -exec ran, so it passes whatever the files hold"; + } + } + if (/\bFAIL(?:ED)?\b/.test(cmd) && /\bPASS(?:ED)?\b/.test(cmd) && !/\bexit\s*\(?\s*[1-9]|sys\.exit|\bassert\b|\braise\b|process\.exit|\bfalse\b/.test(cmd)) { + return "it prints PASS or FAIL but always exits 0, and only the exit status is read"; + } + return null; +} + /** The first rule broken, or null. */ export function lintCheck(run: string, ctx: LintCtx): LintHit | null { return lintAll(run, ctx)[0] ?? null; diff --git a/src/cli.tsx b/src/cli.tsx index 0ebb72b..2dad220 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -53,6 +53,7 @@ import type { BarResult, Check, CheckAuthor, EngineEvent } from "./types.js"; import { stateDir } from "./statedir.js"; import { draftReference, snapshotProject } from "./reference.js"; import { env } from "./env.js"; +import { copyTree } from "./scratch.js"; /** * The version, from the manifest that npm actually publishes. @@ -162,6 +163,11 @@ options gate; "verified" then needs a drafted check that asserts a value, ran the work, and failed before the work began (also MAAT_REVIEW_ADVISORY=1) + --require-discriminating + "verified" needs an independent value check that failed + on the tree before the work and passes now; otherwise + "passed checks that did not test this work", exit 3 + (also MAAT_REQUIRE_DISCRIMINATING=1; off by default) --signout before an unattended claim is judged, put each stated requirement to the model once beside the commands it ran (off by default: 60 rounds rescued no task) @@ -282,6 +288,8 @@ type Args = { revealStuck?: boolean; /** `--review-advisory`: see EngineConfig.reviewAdvisory. */ reviewAdvisory?: boolean; + /** `--require-discriminating`: see EngineConfig.requireDiscriminating. */ + requireDiscriminating?: boolean; /** `--signout`: see EngineConfig.signOut. */ signout?: boolean; /** `--arbiter-model` / `--dispute-votes`: see EngineConfig.dispute. */ @@ -518,6 +526,9 @@ export function parseArgs(argv: string[], stored: StoredEndpoint = {}): Args { case "--review-advisory": out.reviewAdvisory = true; break; + case "--require-discriminating": + out.requireDiscriminating = true; + break; case "--signout": out.signout = true; break; @@ -825,6 +836,7 @@ function engineFor(args: Args, session = false, extra: { files?: FileAccess } = batch: args.batch === true, ...(args.revealStuck === true ? { revealOnStuck: true } : args.revealStuck === false ? { revealOnStuck: false } : {}), ...(args.reviewAdvisory || env("REVIEW_ADVISORY") === "1" ? { reviewAdvisory: true } : {}), + ...(args.requireDiscriminating || env("REQUIRE_DISCRIMINATING") === "1" ? { requireDiscriminating: true } : {}), ...(args.signout ? { signOut: true } : {}), ...(args.arbiterModel || args.disputeVotes ? { dispute: { model: args.arbiterModel, votes: args.disputeVotes } } : {}), ...(args.review ? { review: { votes: args.review, reasoningEffort: args.reasoningChecks ?? args.reasoning } } : {}), @@ -1223,6 +1235,28 @@ async function autoDraft( // Taken once, now, before the first step: the second try below and every // stage of each draft read this and never the folder the work is changing. const snapshot = inputs?.snapshot ?? drafterSnapshotFor(args); + // And a copy of the project as it is now, before the first step: each + // drafted check is tried on it, and one that already passes there cannot + // show this task was done, so it is sent back to the drafter once and + // dropped if it still passes (criteria.ts screen). Null when the project is + // too big to copy: the checks are then not tried here. + const preWork = args.cwd ? await copyTree(args.cwd) : null; + try { + return await autoDraftFrom(engine, args, snapshot, preWork?.dir, soFar, inputs, deadlineAt); + } finally { + await preWork?.cleanup(); + } +} + +async function autoDraftFrom( + engine: Engine, + args: Args, + snapshot: DrafterInputs, + preWorkDir: string | undefined, + soFar?: { draft?: Draft; sealed?: boolean; late?: boolean }, + inputs?: { snapshot: DrafterInputs; used: string[] }, + deadlineAt?: number, +): Promise> { const none: ReturnType = { taskChecks: [], taskNotes: [] }; // Under --for, no draft, critique or retry waits past the run's budget // (the job's own deadline, taken once by the caller: runDeadlineAt). @@ -1240,6 +1274,7 @@ async function autoDraft( barChecks: (engine.cfg.bar?.checks ?? []).map((c) => c.name), ...judgeTarget({ baseUrl: args.url, apiKey: args.key, model: args.model }), cwd: args.cwd, + ...(preWorkDir ? { preWorkDir } : {}), reasoningEffort: judgeEffort(args.reasoningChecks ?? args.reasoning), latency: engine.askLatency, deadlineAt, diff --git a/src/criteria.ts b/src/criteria.ts index e3c6488..be19da9 100644 --- a/src/criteria.ts +++ b/src/criteria.ts @@ -30,7 +30,7 @@ import { topLevel } from "./brief.js"; import { namedInputs, profileLine } from "./inspect.js"; import { askModel, type AskOptions } from "./ask.js"; import { runCommand, draftedShell, bashPath } from "./run.js"; -import { lintAll, readTree, type LintCtx, type Tree } from "./checklint.js"; +import { cannotFail, lintAll, readTree, type LintCtx, type Tree } from "./checklint.js"; import { checkMutates } from "./checkwrites.js"; import { copyTree } from "./scratch.js"; import { diagnoseFailure } from "./bar.js"; @@ -312,6 +312,12 @@ export async function preflightCriteria( * in neither list: it was not tried. */ failed?: string[]; + /** + * Filled with the names of criteria that exited as expected while printing + * a FAIL verdict (`FAIL`, `FAILED`): a script that reports failure in words + * and still exits 0 cannot fail, whatever it checks (checklint L16). + */ + printedFail?: string[]; /** * Set for DRAFTED checks: a command that reaches for an absolute path * outside the project (see strayPath) is reported broken without being @@ -355,7 +361,10 @@ export async function preflightCriteria( if (d.didNotRun) broken.push({ name: c.name, run: c.run, why: d.hint ?? "did not run" }); else if (unparsed) { broken.push({ name: c.name, run: c.run, why: `the shell could not parse it: ${firstLine(copy ? copy.unmap(r.stderr) : r.stderr)}` }); - } else if (selfError) broken.push({ name: c.name, run: c.run, why: selfError }); else if (!r.timedOut && r.code === (c.expectExit ?? 0)) opts.passed?.push(c.name); + } else if (selfError) broken.push({ name: c.name, run: c.run, why: selfError }); else if (!r.timedOut && r.code === (c.expectExit ?? 0)) { + opts.passed?.push(c.name); + if (/\bFAIL(?:ED)?\b/.test(`${r.stdout}\n${r.stderr}`)) opts.printedFail?.push(c.name); + } else opts.failed?.push(c.name); } catch { // Failing to spawn it here is molt's problem, not the criterion's. @@ -927,6 +936,8 @@ export async function draftCriteriaCritiqued( onProgress?: (d: Draft) => void; /** Bound on the step that adds checks for uncovered requirements (COVER_MAX_MS). */ coverMaxMs?: number; + /** A copy of the project taken before the work (see PreWork). Without it, no check is tried here. */ + preWorkDir?: string; }, ): Promise<{ ok: true; draft: Draft; critique: string[]; lint?: LintDrop[] } | { ok: false; error: string }> { // The critic's list of stated requirements rides on whatever draft comes @@ -950,6 +961,13 @@ export async function draftCriteriaCritiqued( } } +/** Why a drafted check that passes on the pre-work copy is sent back: the drafter reads this. */ +export const PASSES_BEFORE_WORK = + "this check passes before any work, so it cannot show the task was done; make it fail on the current tree"; +/** Why a drafted check that printed FAIL and exited 0 on the pre-work copy is sent back. */ +export const PRINTS_FAIL_EXITS_0 = + "it printed FAIL and still exited 0 before any work, so it can never fail; make a failure exit non-zero"; + /** A drafted check the seal-time lint retired (src/checklint.ts), for the journal. */ export type LintDrop = { name: string; run: string; rule: string; why: string; redraft: boolean }; type Sink = { requirements: string[]; lint: LintDrop[] }; @@ -960,6 +978,7 @@ async function critiqued( onProgress?: (d: Draft) => void; /** Bound on the step that adds checks for uncovered requirements (COVER_MAX_MS). */ coverMaxMs?: number; + preWorkDir?: string; }, sink: Sink, ): Promise<{ ok: true; draft: Draft; critique: string[] } | { ok: false; error: string }> { @@ -984,8 +1003,17 @@ async function critiqued( // Off unless MAAT_CHECK_LINT=1: in the v13 paired run the lint left fewer, shallower // checks sealed (the redraft lands after the seal), which cost passes and let wrong work through. // Except L15: a check that changes the work it judges is never sealed (checkwrites.ts). + // And L16: a check that cannot fail cannot show the task was done (checklint.ts cannotFail). const writes = checkMutates(c.run); - const hit = process.env.MAAT_CHECK_LINT === "1" ? lintAll(c.run, lintCtx)[0] : writes ? { rule: "L15-mutates", why: writes } : undefined; + const never = cannotFail(c.run); + const hit = + process.env.MAAT_CHECK_LINT === "1" + ? lintAll(c.run, lintCtx)[0] + : writes + ? { rule: "L15-mutates", why: writes } + : never + ? { rule: "L16-cannot-fail", why: never } + : undefined; if (!hit) ok.push(c); else bad.push({ name: c.name, run: c.run, rule: hit.rule, why: hit.why, redraft }); } @@ -997,10 +1025,37 @@ async function critiqued( said.push(`dropped ${b.name} (${b.run}) ${where}: ${b.why} [${b.rule}]`); } }; + /** + * The lint, then a try on the copy of the project taken before the work: a + * check that already passes there cannot show the task was done (P1), and + * one that exits 0 while printing FAIL cannot fail at all (L16). + */ + const screen = async (checks: DraftedCheck[], redraft: boolean): Promise<{ ok: DraftedCheck[]; bad: LintDrop[] }> => { + const l = lintSplit(checks, redraft); + if (!opts.preWorkDir || !l.ok.length) return l; + const passed: string[] = []; + const printedFail: string[] = []; + try { + await preflightCriteria( + l.ok.map((c) => ({ name: c.name, kind: "command", run: c.run })), + { cwd: opts.preWorkDir, passed, printedFail }, + ); + } catch { + return l; + } + const ok: DraftedCheck[] = []; + const bad = [...l.bad]; + for (const c of l.ok) { + if (printedFail.includes(c.name)) bad.push({ name: c.name, run: c.run, rule: "L16-printed-fail", why: PRINTS_FAIL_EXITS_0, redraft }); + else if (passed.includes(c.name)) bad.push({ name: c.name, run: c.run, rule: "P1-passes-before-work", why: PASSES_BEFORE_WORK, redraft }); + else ok.push(c); + } + return { ok, bad }; + }; const drafted1 = await draftCriteria(opts); let first = drafted1.ok ? { ...drafted1, draft: fix(drafted1.draft) } : drafted1; if (first.ok && first.draft.checks.length) { - const l1 = lintSplit(first.draft.checks, false); + const l1 = await screen(first.draft.checks, false); if (l1.bad.length) { // Sent back once with the reasons. What passes the lint is already ready // to seal while that second ask runs. @@ -1013,7 +1068,7 @@ async function critiqued( }); const have = new Set(l1.ok.map((c) => c.name)); const ran = new Set(l1.ok.map((c) => c.run)); - const l2 = again.ok ? lintSplit(fix(again.draft).checks, true) : { ok: [] as DraftedCheck[], bad: [] as LintDrop[] }; + const l2 = again.ok ? await screen(fix(again.draft).checks, true) : { ok: [] as DraftedCheck[], bad: [] as LintDrop[] }; drop(l1.bad, "before sealing"); drop(l2.bad, "from the redraft"); const added = l2.ok.filter((c) => !have.has(c.name) && !ran.has(c.run)).slice(0, Math.max(0, CRITERIA_MAX_CHECKS - l1.ok.length)); @@ -1022,8 +1077,8 @@ async function critiqued( } } if (!first.ok || !first.draft.checks.length) return first.ok ? { ...first, critique: said } : first; - const lintCover = (checks: DraftedCheck[]): DraftedCheck[] => { - const l = lintSplit(checks, false); + const lintCover = async (checks: DraftedCheck[]): Promise => { + const l = await screen(checks, false); drop(l.bad, "from the cover draft"); return l.ok; }; @@ -1070,7 +1125,7 @@ async function critiqued( `Nothing tests these stated requirements yet: ${c1.uncovered.map((q) => `"${q}"`).join("; ")}. ` + `Draft checks for these requirements only.)`, }); - const extra = cover.ok ? lintCover(fix(cover.draft).checks) : []; + const extra = cover.ok ? await lintCover(fix(cover.draft).checks) : []; return extra.length ? await critic({ checks: extra, notes: [] }) : null; }; let coverTimer: NodeJS.Timeout | undefined; @@ -1099,7 +1154,7 @@ async function critiqued( const drafted2 = await draftCriteria({ ...opts, task: `${opts.task}\n\n(Review of an earlier draft of the checks: ${findings})` }); let second = drafted2.ok ? { ...drafted2, draft: fix(drafted2.draft) } : drafted2; if (second.ok) { - const l = lintSplit(second.draft.checks, false); + const l = await screen(second.draft.checks, false); drop(l.bad, "from the redraft"); second = { ...second, draft: { ...second.draft, checks: l.ok } }; } diff --git a/src/engine.ts b/src/engine.ts index 2a9195d..6d05da7 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1413,6 +1413,16 @@ export type EngineConfig = { * failed before the work (tiers.ts `reviewAdvisory`). Off by default. */ reviewAdvisory?: boolean; + /** + * Opt-in (`--require-discriminating`, MAAT_REQUIRE_DISCRIMINATING=1): + * "verified" needs a passing independent runs+value check that FAILED on + * the tree before the work (tiers.ts `requireDiscriminating`); otherwise the + * tier is passed-untested. Off by default: replayed over the 2026-10-07 + * lanes it removed 2 wrong verifieds and denied 10 right ones (precision + * 18/22 -> 8/10). The cause is fixed at seal time instead (seal-time + * redraft of checks that pass before the work). `reviewAdvisory` implies it. + */ + requireDiscriminating?: boolean; /** * Requirement sign-out (src/signout.ts): one round per unattended turn that * lists each stated requirement beside the commands run for it. Off unless @@ -3734,8 +3744,11 @@ export class Engine { } /** tierOf's advisory-review argument: empty unless `reviewAdvisory`. */ - private advisoryTier(): { reviewAdvisory?: true } { - return this.cfg.reviewAdvisory === true ? { reviewAdvisory: true } : {}; + private advisoryTier(): { reviewAdvisory?: true; requireDiscriminating?: true } { + return { + ...(this.cfg.reviewAdvisory === true ? { reviewAdvisory: true as const } : {}), + ...(this.cfg.requireDiscriminating === true ? { requireDiscriminating: true as const } : {}), + }; } /** Who wrote each sealed check, keyed by the name it runs under in the bar (and its bare name). */ @@ -3757,6 +3770,7 @@ export class Engine { /** Everything tierOf weighs beside the results: advisory mode, the worker, who wrote each check, what the pre-work try found, and which golden files predate the work. */ private tierContext(): { reviewAdvisory?: true; + requireDiscriminating?: true; guards: ReadonlySet; failedBefore: ReadonlySet; worker: string[]; diff --git a/src/tiers.ts b/src/tiers.ts index 6e6896d..21ced45 100644 --- a/src/tiers.ts +++ b/src/tiers.ts @@ -358,8 +358,8 @@ export function contradictions(review: { votes: string; violations: unknown[] } * The tier a passing turn has earned. "verified" needs (a) a passing check a * person wrote, or a passing drafted check that runs the deliverable (not * surface) AND asserts a value AND was written by someone other than the - * worker model AND failed on the tree before the work began (it discriminates: - * see `failedBefore`), and (b) an independent review, when one ran, that found no + * worker model (and, with `requireDiscriminating`, failed on the tree before + * the work began: see `failedBefore`), and (b) an independent review, when one ran, that found no * contradiction. (c) — the turn not ended by the clock or the provider — is * decided before this, by `passedAtEnd`. * @@ -374,10 +374,10 @@ export function contradictions(review: { votes: string; violations: unknown[] } * model reached another way), the best the turn earns is "passed-own-checks". * A check whose author was never recorded counts as the worker's. * - * Never let a check that cannot tell the work from no work vouch for it: - * when independent runs+value checks passed but none had failed before the - * work, the turn earns "passed-untested" — they guard against a regression - * and did not test this work. + * With `requireDiscriminating`, never let a check that cannot tell the work + * from no work vouch for it: when independent runs+value checks passed but + * none had failed before the work, the turn earns "passed-untested" — they + * guard against a regression and did not test this work. */ export function tierOf(args: { results: readonly (Pick & { name?: string; kind?: CheckResult["kind"] })[]; @@ -386,16 +386,25 @@ export function tierOf(args: { unreviewed?: boolean; /** * Experimental (MAAT_REVIEW_ADVISORY=1, `--review-advisory`): the review is - * recorded but gates nothing. (The evidence rule it once added here — the - * value check must have FAILED on the untouched project — now holds in every - * mode: see `failedBefore`.) reports/checkquality-2026-10-06.md §2.5 measured the 3-vote review as a + * recorded but gates nothing, and the evidence rule is stricter in return: + * it implies `requireDiscriminating`. reports/checkquality-2026-10-06.md §2.5 measured the 3-vote review as a * likelihood ratio of about 1. */ reviewAdvisory?: boolean; + /** + * Opt-in (MAAT_REQUIRE_DISCRIMINATING=1, `--require-discriminating`; implied + * by `reviewAdvisory`): only a check in `failedBefore` can earn "verified". + * Off, `failedBefore` is ignored and the tier is #32's. Replayed over the + * 2026-10-07 lanes the gate removed 2 wrong verifieds and denied 10 right + * ones, so the default fixes the cause at seal time instead: a drafted + * check that already passes before the work is redrafted or dropped. + */ + requireDiscriminating?: boolean; /** * Names (as in `results`) of checks that were tried on the tree before the - * work began and FAILED there. Only such a check discriminates: "verified" - * needs a passing independent runs+value check named here. A check that + * work began and FAILED there. Only such a check discriminates: with + * `requireDiscriminating`, "verified" needs a passing independent runs+value + * check named here. A check that * passed before the work, could not run then, or joined after the work * began (no try) is not named here and cannot earn the word. Absent means * no check was tried, so none discriminates. @@ -437,7 +446,8 @@ export function tierOf(args: { const strong = strongAll.length > 0; // Of those, the ones that failed on the tree before the work and pass now: // the only passes that show THIS work did something. - const discriminating = strongIndependent.filter((r) => args.failedBefore?.has(r.name ?? "") === true); + const gate = args.requireDiscriminating === true || args.reviewAdvisory === true; + const discriminating = gate ? strongIndependent.filter((r) => args.failedBefore?.has(r.name ?? "") === true) : strongIndependent; const by = [...new Set(strongIndependent.map((r) => authorOf(r).model ?? "another model"))]; const basis: TierVerdict["basis"] = person ? "person" : strongIndependent.length ? "independent" : strong ? "own" : undefined; const who = { ...(basis ? { basis } : {}), ...(by.length && !person ? { by } : {}), ...(worker ? { worker } : {}) }; diff --git a/test/discriminating-checks.test.ts b/test/discriminating-checks.test.ts index 1db5232..16359f5 100644 --- a/test/discriminating-checks.test.ts +++ b/test/discriminating-checks.test.ts @@ -1,5 +1,8 @@ /** - * "verified" needs a check that tells this work from no work. + * With --require-discriminating (MAAT_REQUIRE_DISCRIMINATING=1), "verified" + * needs a check that tells this work from no work. Off by default: replayed + * over the 2026-10-07 lanes it removed 2 wrong verifieds and denied 10 right + * ones; the cause is fixed at seal time instead (criteria.ts screen). * * On the 2026-10-07 container bench a separate judge drafted * `python3 server.py 8080 & sleep 1; curl -f .../items || echo 'fail'` and two @@ -15,6 +18,7 @@ import assert from "node:assert/strict"; import { readFileSync, readdirSync } from "node:fs"; import { join } from "node:path"; import { describe, it } from "node:test"; +import { parseArgs } from "../src/cli.js"; import { Engine } from "../src/engine.js"; import { Journal } from "../src/journal.js"; import { Receipts } from "../src/receipts.js"; @@ -31,7 +35,7 @@ describe("tierOf: a check that did not fail before the work cannot earn verified const worker = "qwen/qwen3-235b-a22b-2507"; it("the http-json-server run: every independent value check passed before the work too, so not verified", () => { - const t = tierOf({ results: names.map(value), worker, authors, guards: new Set(names), failedBefore: new Set() }); + const t = tierOf({ requireDiscriminating: true, results: names.map(value), worker, authors, guards: new Set(names), failedBefore: new Set() }); assert.equal(t.tier, "passed-untested"); assert.equal(t.basis, "independent", "the checks were independent; independence was not the problem"); assert.match(t.reason!, /`task:get-all-items` passed before the work began too/); @@ -40,30 +44,38 @@ describe("tierOf: a check that did not fail before the work cannot earn verified }); it("a check that failed before the work and passes after: verified", () => { - const t = tierOf({ results: names.map(value), worker, authors, guards: new Set(names.slice(0, 2)), failedBefore: new Set([names[2]!]) }); + const t = tierOf({ requireDiscriminating: true, results: names.map(value), worker, authors, guards: new Set(names.slice(0, 2)), failedBefore: new Set([names[2]!]) }); assert.deepEqual([t.tier, t.basis, t.by], ["verified", "independent", ["qwen3-coder-30b-a3b"]]); }); it("a check with no pre-work try (broken then, or joined late with no copy) does not discriminate", () => { - const t = tierOf({ results: [value(names[0]!)], worker, authors, failedBefore: new Set() }); + const t = tierOf({ requireDiscriminating: true, results: [value(names[0]!)], worker, authors, failedBefore: new Set() }); assert.equal(t.tier, "passed-untested"); assert.match(t.reason!, /was not tried before the work began/); - assert.equal(tierOf({ results: [value(names[0]!)], worker, authors }).tier, "passed-untested", "no pre-work record at all: nothing discriminates"); + assert.equal(tierOf({ requireDiscriminating: true, results: [value(names[0]!)], worker, authors }).tier, "passed-untested", "no pre-work record at all: nothing discriminates"); }); it("only independent value checks count: a discriminating check the worker wrote is still its own", () => { const mixed = new Map([[names[0]!, JUDGE], [names[1]!, { kind: "worker", model: worker }]]); - const t = tierOf({ results: [value(names[0]!), value(names[1]!)], worker, authors: mixed, guards: new Set([names[0]!]), failedBefore: new Set([names[1]!]) }); + const t = tierOf({ requireDiscriminating: true, results: [value(names[0]!), value(names[1]!)], worker, authors: mixed, guards: new Set([names[0]!]), failedBefore: new Set([names[1]!]) }); assert.equal(t.tier, "passed-untested"); - const own = tierOf({ results: [value(names[1]!)], worker, authors: mixed, failedBefore: new Set([names[1]!]) }); + const own = tierOf({ requireDiscriminating: true, results: [value(names[1]!)], worker, authors: mixed, failedBefore: new Set([names[1]!]) }); assert.equal(own.tier, "passed-own-checks", "the authorship rule is unchanged"); }); it("a failing or surface-only discriminating check carries nothing; a person's check still verifies", () => { const fb = new Set([names[0]!]); - assert.equal(tierOf({ results: [{ ...value(names[0]!), ok: false }], worker, authors, failedBefore: fb }).tier, "passed-checks"); - assert.equal(tierOf({ results: [{ ...value(names[0]!), tags: ["task", "surface", "value"] }], worker, authors, failedBefore: fb }).tier, "passed-checks"); - assert.equal(tierOf({ results: [{ name: "project", ok: true, kind: "command" as const, tags: [] }], worker }).tier, "verified"); + assert.equal(tierOf({ requireDiscriminating: true, results: [{ ...value(names[0]!), ok: false }], worker, authors, failedBefore: fb }).tier, "passed-checks"); + assert.equal(tierOf({ requireDiscriminating: true, results: [{ ...value(names[0]!), tags: ["task", "surface", "value"] }], worker, authors, failedBefore: fb }).tier, "passed-checks"); + assert.equal(tierOf({ requireDiscriminating: true, results: [{ name: "project", ok: true, kind: "command" as const, tags: [] }], worker }).tier, "verified"); + }); +}); + +describe("tierOf without --require-discriminating: #32's tiering, unchanged", () => { + it("ignores the pre-work record", () => { + const authors = new Map([["task:v", JUDGE]]); + const t = tierOf({ results: [value("task:v")], worker: "w", authors, guards: new Set(["task:v"]), failedBefore: new Set() }); + assert.deepEqual([t.tier, t.basis], ["verified", "independent"]); }); }); @@ -90,7 +102,7 @@ describe("the rule in a turn", () => { const check = (name: string, run: string): Check => ({ name, kind: "command", run, timeoutMs: 5_000, expectExit: 0, tags: ["task", "value"], hidden: true, author: JUDGE }) as Check; - async function turn(checks: Check[], notes: string[] = []) { + async function turn(checks: Check[], notes: string[] = [], requireDiscriminating = true) { const ws = workspace(); try { const provider = scriptedProvider(work); @@ -98,6 +110,7 @@ describe("the rule in a turn", () => { const engine = new Engine({ baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: provider.fetchFn, bar: null, receipts: new Receipts(ws.dir), stream: false, autonomy: "high", journal, + ...(requireDiscriminating ? { requireDiscriminating: true } : {}), }); const events = await drain(engine.run(TASK, allowAll, { taskChecks: checks, taskNotes: notes })); const end = events.find((e) => e.kind === "job_end"); @@ -124,6 +137,13 @@ describe("the rule in a turn", () => { assert.deepEqual(tried[0]!.data.passed, ["task:always"]); }); + it("off by default: the same turn is verified as under #32, and the receipt still records the pre-work try", async () => { + const { end, receipt, tried } = await turn([check("always", `[ "$(echo ok)" = "ok" ] || echo fail`)], [], false); + assert.deepEqual([end.outcome, end.tier], ["verified", "verified"]); + assert.match(receipt, /before the work: passed \(a guard/); + assert.deepEqual(tried[0]!.data.passed, ["task:always"]); + }); + it("a check that failed before the work and passes after: verified, and the receipt says it discriminates", async () => { const { end, receipt, tried } = await turn([check("always", `[ "$(echo ok)" = "ok" ]`), check("greeting", "grep -qx hello out.txt")], ["out.txt holds exactly hello greeting"]); assert.deepEqual([end.outcome, end.tier, end.claim], ["verified", "verified", "verified (independent checks: qwen3-coder-30b-a3b)"]); @@ -140,3 +160,10 @@ describe("the rule in a turn", () => { assert.match(receipt, /before the work: not tried/); }); }); + +describe("parseArgs --require-discriminating", () => { + it("is off unless asked for", () => { + assert.equal(parseArgs([]).requireDiscriminating, undefined); + assert.equal(parseArgs(["--require-discriminating"]).requireDiscriminating, true); + }); +}); diff --git a/test/exit-codes.test.ts b/test/exit-codes.test.ts index d4512da..f0fa9ac 100644 --- a/test/exit-codes.test.ts +++ b/test/exit-codes.test.ts @@ -168,7 +168,7 @@ describe("molt run's exit code", () => { assert.equal(same, 3); }); - it("is 3 when the only independent check met also passed before the work (it did not test this work)", async () => { + it("is 3 when the only drafted check already passes before the work: it is redrafted, then dropped", async () => { const url = await draftingProvider("[ \"$(echo a)\" = \"a\" ]"); const code = await molt(["run", "write a.txt", "--url", url, "--model", "m", "--key", "k", "--cwd", project(), "--no-stream", "--yes", "--criteria", "auto", "--judge", "j"]); diff --git a/test/prework-screen.test.ts b/test/prework-screen.test.ts new file mode 100644 index 0000000..c50ea5f --- /dev/null +++ b/test/prework-screen.test.ts @@ -0,0 +1,136 @@ +/** + * Drafted checks that cannot show the task was done are fixed at seal time. + * + * On 2026-10-07 a separate judge drafted `python3 server.py 8080 & sleep 1; + * curl -f .../items || echo 'fail'`: it exits 0 whatever happens, passed + * before server.py existed, and earned a wrong "verified". Two screens now + * run before a drafted check is sealed: + * - L16 (always on): constructs that cannot fail — `|| echo …`, `|| true`, + * `; exit 0`, `find … -exec … \;`, PASS/FAIL printed with exit 0; + * - P1: the check is tried on a copy of the project taken before the work; + * one that already passes there is sent back to the drafter once with the + * reason, and dropped if the redraft still passes. + */ +import assert from "node:assert/strict"; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { cannotFail } from "../src/checklint.js"; +import { draftCriteriaCritiqued, PASSES_BEFORE_WORK } from "../src/criteria.js"; +import { workspace } from "./helpers.js"; + +describe("cannotFail", () => { + it("flags the constructs that exit 0 whatever happens", () => { + const flagged = [ + "python3 server.py 8080 & sleep 1; curl -f http://127.0.0.1:8080/items || echo 'fail'", + "python3 wc.py words.txt || true", + "grep -q x out.txt; exit 0", + String.raw`find configs/ -name '*.json' -exec sh -c 'jq -e ".version == 2" {} 2>/dev/null || exit 1' \;`, + `python3 -c "import csv; h = next(csv.reader(open('clean.csv'))); print('PASS' if len(h) == 5 else 'FAIL')"`, + ]; + for (const run of flagged) assert.ok(cannotFail(run), run); + }); + + it("leaves checks whose status means something alone", () => { + const fine = [ + "grep -qx hello out.txt", + String.raw`find . -name '*.py' -exec grep -l foo {} \; | grep -q .`, + "find . -name '*.py' -exec grep -q foo {} +", + `[ "$(cat a.txt)" = "a" ] && echo PASS || { echo FAIL; exit 1; }`, + "rm -f x || true; python3 wc.py words.txt | grep -qx 3", + "test -f a || echo missing; exit 1", + ]; + for (const run of fine) assert.equal(cannotFail(run), null, run); + }); +}); + +describe("drafted checks tried on the pre-work copy", () => { + type Msgs = { messages: { role: string; content: string }[] }; + function drafter(script: { first: string; redraft: string }) { + const asked: string[] = []; + const prompts: string[] = []; + const fetchFn = (async (_u: string, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as Msgs; + const system = body.messages[0]!.content; + const prompt = body.messages.filter((m) => m.role !== "system").map((m) => m.content).join("\n"); + const kind = system.startsWith("You review acceptance checks") ? "critic" : prompt.includes("was dropped:") ? "redraft" : "first"; + asked.push(kind); + prompts.push(prompt); + const content = + kind === "critic" ? JSON.stringify({ checks: [], uncovered: [], requirements: [] }) : kind === "redraft" ? script.redraft : script.first; + return { ok: true, status: 200, json: async () => ({ choices: [{ message: { content }, finish_reason: "stop" }] }), text: async () => "" } as unknown as Response; + }) as unknown as typeof fetch; + return { fetchFn, asked, prompts }; + } + const base = { scripts: [], barChecks: [], baseUrl: "http://p.test/v1", model: "m", task: "Write out.txt holding the number of words in words.txt" }; + const draft = (...checks: [string, string][]) => JSON.stringify({ checks: checks.map(([name, run]) => ({ name, run })), notes: [] }); + const GOOD: [string, string] = ["counts", "grep -qx 3 out.txt"]; + const ALWAYS: [string, string] = ["input-there", `[ "$(wc -w < words.txt | tr -d ' ')" = "3" ]`]; + + /** The project before the work, and a copy of it standing in for the turn-start copy. */ + function before() { + const ws = workspace(); + const pre = workspace(); + for (const dir of [ws.dir, pre.dir]) writeFileSync(join(dir, "words.txt"), "a b c\n"); + return { cwd: ws.dir, preWorkDir: pre.dir, cleanup: () => (ws.cleanup(), pre.cleanup()) }; + } + + it("a check that already passes is sent back once with the reason, and the replacement is sealed", async () => { + const p = before(); + try { + const { fetchFn, asked, prompts } = drafter({ first: draft(GOOD, ALWAYS), redraft: draft(["has-count", "grep -q '^[0-9]' out.txt"]) }); + const r = await draftCriteriaCritiqued({ ...base, cwd: p.cwd, preWorkDir: p.preWorkDir, fetchFn }); + assert.ok(r.ok); + assert.deepEqual(asked.filter((k) => k !== "critic"), ["first", "redraft"]); + assert.ok(prompts.find((x) => x.includes("was dropped:"))!.includes(PASSES_BEFORE_WORK)); + assert.deepEqual(r.draft.checks.map((c) => c.name), ["counts", "has-count"]); + assert.deepEqual(r.lint!.map((l) => [l.name, l.rule, l.redraft]), [["input-there", "P1-passes-before-work", false]]); + } finally { + p.cleanup(); + } + }); + + it("a redraft that still passes before the work is dropped", async () => { + const p = before(); + try { + const { fetchFn, asked } = drafter({ first: draft(GOOD, ALWAYS), redraft: draft(["still", "test -f words.txt"]) }); + const r = await draftCriteriaCritiqued({ ...base, cwd: p.cwd, preWorkDir: p.preWorkDir, fetchFn }); + assert.ok(r.ok); + assert.equal(asked.filter((k) => k === "redraft").length, 1, "sent back once, no more"); + assert.deepEqual(r.draft.checks.map((c) => c.name), ["counts"]); + assert.deepEqual(r.lint!.map((l) => [l.name, l.rule, l.redraft]), [["input-there", "P1-passes-before-work", false], ["still", "P1-passes-before-work", true]]); + } finally { + p.cleanup(); + } + }); + + it("a check that prints FAIL and exits 0 is caught when it runs, and a can't-fail tail by the lint", async () => { + const p = before(); + try { + const printsFail = ["verdict", "test -f out.txt && echo ok || echo FAIL"] as [string, string]; + const tail = ["server", "python3 server.py & sleep 1; curl -sf localhost:9/items || echo 'fail'"] as [string, string]; + const { fetchFn } = drafter({ first: draft(GOOD, printsFail, tail), redraft: draft() }); + const r = await draftCriteriaCritiqued({ ...base, cwd: p.cwd, preWorkDir: p.preWorkDir, fetchFn }); + assert.ok(r.ok); + assert.deepEqual(r.draft.checks.map((c) => c.name), ["counts"]); + const rules = Object.fromEntries(r.lint!.map((l) => [l.name, l.rule])); + assert.equal(rules.server, "L16-cannot-fail"); + assert.ok(rules.verdict === "L16-cannot-fail" || rules.verdict === "L16-printed-fail", rules.verdict); + } finally { + p.cleanup(); + } + }); + + it("without a pre-work copy nothing is tried; the lint still applies", async () => { + const p = before(); + try { + const { fetchFn, asked } = drafter({ first: draft(GOOD, ALWAYS), redraft: draft() }); + const r = await draftCriteriaCritiqued({ ...base, cwd: p.cwd, fetchFn }); + assert.ok(r.ok); + assert.deepEqual(asked.filter((k) => k !== "critic"), ["first"]); + assert.deepEqual(r.draft.checks.map((c) => c.name), ["counts", "input-there"]); + } finally { + p.cleanup(); + } + }); +}); diff --git a/test/review-advisory.test.ts b/test/review-advisory.test.ts index 0e707b1..b22b88f 100644 --- a/test/review-advisory.test.ts +++ b/test/review-advisory.test.ts @@ -1,8 +1,8 @@ /** * Experimental review-advisory mode (`--review-advisory`, MAAT_REVIEW_ADVISORY=1): - * the review is recorded and does not gate. The rule that "verified" needs a - * drafted runs+value check that failed before the work holds here as in - * every mode. + * the review is recorded and does not gate; "verified" additionally needs a + * drafted runs+value check that failed before the work (it implies + * --require-discriminating). * Default behaviour is pinned beside it. */ import assert from "node:assert/strict"; @@ -30,16 +30,15 @@ describe("tierOf, review advisory", () => { assert.equal(tierOf({ ...J, results: [strong], unreviewed: true }).tier, "passed-checks", "default unchanged"); }); - it("needs a value check that failed before the work, in this mode as in every other", () => { + it("needs a value check that failed before the work", () => { const guards = new Set(["task:counts"]); const failedBefore = new Set(["task:other"]); - for (const reviewAdvisory of [false, true]) { - const r = tierOf({ ...J, results: [strong], guards, failedBefore, reviewAdvisory }); - assert.equal(r.tier, "passed-untested"); - assert.match(r.reason!, /`task:counts` passed before the work began too/); - const two = tierOf({ ...J, results: [strong, ck("task:other", ["task", "value"])], guards, failedBefore, reviewAdvisory }); - assert.equal(two.tier, "verified", "another value check that failed on the pristine tree carries it"); - } + assert.equal(tierOf({ ...J, results: [strong], guards, failedBefore }).tier, "verified", "off by default: the pre-work record is ignored"); + const r = tierOf({ ...J, results: [strong], guards, failedBefore, reviewAdvisory: true }); + assert.equal(r.tier, "passed-untested"); + assert.match(r.reason!, /`task:counts` passed before the work began too/); + const two = tierOf({ ...J, results: [strong, ck("task:other", ["task", "value"])], guards, failedBefore, reviewAdvisory: true }); + assert.equal(two.tier, "verified", "another value check that failed on the pristine tree carries it"); }); it("a surface-only or value-less pass is still not verified", () => { From be6883fec18b007797a1b1f521fb5a2fa5d1c070 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 17:52:10 -0700 Subject: [PATCH 3/4] A check whose exit code ignores its assertion cannot verify; try checks under the bar's shell A check like `python3 -c "print(rows == expected)"` exits 0 whether it prints True or False. It passed the L16 lint and the pre-work screen (it raised before the work, so it "failed" there), and after wrong work it printed False, exited 0 and earned "verified". Two layers now: - L16 reads the step that decides the exit (top-level parse: quotes, heredocs, subshells and [[ ]] kept whole) and flags inline programs that print a comparison or boolean with nothing that exits non-zero, jq without -e, awk with no exit, a `; echo` / `; echo $?` tail without set -e or exit 1, and an `if ... fi` with no failing branch. Output read by a later pipe stage (`... | grep -qx PASS`) is no longer flagged. - At the bar and in the pre-work try, a task check whose deciding step printed its verdict (verdictPrinter) and whose last line is False, FAIL, FAILED or NO, or whose `echo $?` printed non-zero, has failed. jq's own output inside `find -exec` or `jq -e` is not read as a verdict. The pre-work try now runs each check under the shell the bar runs it with (draftedShell per check). It used /bin/sh unless `stray` was set, so on dash a bashism failed before the work and passed at the bar, and counted as discriminating. dash's exit 2 for a missing `sh script` reads like bash's 127, and a try that timed out counts as not tried. Replay (prework_screen_replay.py, today's lanes, 549 sealed checks): L16 static 65 -> 110, P1 53 -> 46, redrafted 118 -> 156. The new flags are all print-only checks. On the 31 independent verifieds, the bar-time read and the golden-file rule change no verdict; the wrong csv-clean verified rested on a PASS/FAIL print that L16 already sent back. --- bench/local/prework_screen_replay.py | 88 ++++++++++++++++++++++- src/bar.ts | 21 +++++- src/checklint.ts | 73 +++++++++++++++---- src/criteria.ts | 41 ++++++++--- src/evidence.ts | 102 +++++++++++++++++++++++++++ test/prework-screen.test.ts | 94 +++++++++++++++++++++++- 6 files changed, 391 insertions(+), 28 deletions(-) diff --git a/bench/local/prework_screen_replay.py b/bench/local/prework_screen_replay.py index b7ecf8e..ee013a7 100644 --- a/bench/local/prework_screen_replay.py +++ b/bench/local/prework_screen_replay.py @@ -51,6 +51,56 @@ def cannot_fail(runs: list[str]) -> list[str | None]: return json.loads(p.stdout) +def reports_failure(run: str, stdout: str) -> bool: + """src/evidence.ts reportsFailure, through the built dist/evidence.js.""" + if not stdout.strip(): + return False + js = ( + f"import {{ reportsFailure }} from {json.dumps(str(REPO / 'dist/evidence.js'))};" + "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{const a=JSON.parse(s);console.log(JSON.stringify(!!reportsFailure(a.run,a.out)))});" + ) + p = subprocess.run(["node", "--input-type=module", "-e", js], input=json.dumps({"run": run, "out": stdout[-65536:]}), capture_output=True, text=True, check=True) + return json.loads(p.stdout) + + +def run_in(tree: Path, run: str) -> tuple[int | None, str]: + """(exit code or None on timeout, stdout) for `run` under bash on a fresh copy of `tree` without .maat.""" + tmp = Path(tempfile.mkdtemp(prefix="atbar-")) + work = tmp / "w" + shutil.copytree(tree, work, symlinks=True, ignore=shutil.ignore_patterns(".maat", ".molt")) + cmd = re.sub(r"\b80([89]\d)\b", r"280\1", run) + try: + p = subprocess.Popen(["bash", "-c", cmd], cwd=work, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + start_new_session=True, stdin=subprocess.DEVNULL) + try: + out, _ = p.communicate(timeout=20) + except subprocess.TimeoutExpired: + os.killpg(p.pid, signal.SIGKILL) + p.communicate() + return None, "" + finally: + try: + os.killpg(p.pid, signal.SIGKILL) + except ProcessLookupError: + pass + return p.returncode, out + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def golden_unproven(task: str, runs: list[str]) -> list[bool]: + """Per command: its value rests only on an expected file that is not in the task's pristine tree (src/golden.ts).""" + base = pristine(task) + js = ( + f"import {{ assertsValue }} from {json.dumps(str(REPO / 'dist/tiers.js'))};" + "import { existsSync } from 'node:fs'; import { join, isAbsolute } from 'node:path';" + "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{const {base, runs}=JSON.parse(s);" + "console.log(JSON.stringify(runs.map(r=>assertsValue(r) && !assertsValue(r, g=>existsSync(isAbsolute(g)?g:join(base,g))))))});" + ) + p = subprocess.run(["node", "--input-type=module", "-e", js], input=json.dumps({"base": str(base), "runs": runs}), capture_output=True, text=True, check=True) + return json.loads(p.stdout) + + def try_before(task: str, run: str) -> tuple[str, bool]: """('passed'|'failed'|'broken', printed FAIL) for `run` on a fresh copy of the task's pristine tree.""" tmp = Path(tempfile.mkdtemp(prefix="prework-")) @@ -74,6 +124,11 @@ def try_before(task: str, run: str) -> tuple[str, bool]: if p.returncode in (126, 127): return "broken", False if p.returncode == 0: + # As Maat reads it (src/evidence.ts reportsFailure): a check that + # printed False/FAIL as its last line, or whose `echo $?` tail was + # not 0, failed whatever it exited. + if reports_failure(run, out): + return "failed", True return "passed", bool(re.search(r"\bFAIL(?:ED)?\b", out + err)) return "failed", False finally: @@ -118,7 +173,8 @@ def main(argv: list[str]) -> int: if not independent: continue a = dr.analyse(tree, row) - verified.append({"run_id": f"{lane.name}/{tree.name}", "right": bool(row.get("passed")), "strong": a["strong"]}) + verified.append({"run_id": f"{lane.name}/{tree.name}", "right": bool(row.get("passed")), "strong": a["strong"], + "task": m["task"], "tree": tree, "runs": {n: sealed.get(n, "") for n in a["strong"]}}) flags = cannot_fail([c["run"] for c in checks]) for c, f in zip(checks, flags): key = (c["task"], c["run"]) @@ -127,7 +183,6 @@ def main(argv: list[str]) -> int: c["static"] = f c["before"], c["printed_fail"] = cache[key] c["redraft"] = bool(f) or c["before"] == "passed" - cleanup_pristine() by = Counter() for c in checks: by["checks"] += 1 @@ -155,6 +210,35 @@ def main(argv: list[str]) -> int: else: kind = "kept a value check the screen passes" out[(kind, v["right"])].append(v["run_id"]) + # At the bar, on the finished tree: does a strong check now fail on its own + # words (printed False/FAIL with exit 0), and does its value rest only on a + # golden file the worker could have written? + print("\n## Verified runs after #32 (independent): what the bar-time reads do to their value checks") + bar = defaultdict(list) + for v in verified: + runs = {n: r for n, r in v["runs"].items() if r} + if not runs: + continue + unproven = dict(zip(runs, golden_unproven(v["task"], list(runs.values())))) + words = {} + for n, r in runs.items(): + code, said = run_in(v["tree"], r) + words[n] = code == 0 and reports_failure(r, said) + left = [n for n in runs if not words[n] and not unproven[n]] + if any(words.values()): + bar[("a value check now fails on its own words", v["right"])].append(f"{v['run_id']} {[n for n in runs if words[n]]}") + if any(unproven.values()): + bar[("a value check rests on a golden file not in the pristine tree", v["right"])].append(f"{v['run_id']} {[n for n in runs if unproven[n]]}") + if not left: + bar[("no value check left: would not be verified", v["right"])].append(v["run_id"]) + cleanup_pristine() + if not bar: + print(" none affected") + for (kind, right), ids in sorted(bar.items()): + print(f" {'right' if right else 'WRONG'} {kind}: {len(ids)}") + for i in ids: + print(f" {i}") + print("\n## Verified runs after #32 (independent), by what the screen does to their value checks") for (kind, right), ids in sorted(out.items()): print(f" {'right' if right else 'WRONG'} {kind}: {len(ids)}") diff --git a/src/bar.ts b/src/bar.ts index 8291e4a..774e334 100644 --- a/src/bar.ts +++ b/src/bar.ts @@ -9,7 +9,7 @@ * Nothing here asks a model anything. A bar result is an exit code. */ import { DISPUTE_HINT } from "./dispute.js"; -import { judgePass } from "./evidence.js"; +import { judgePass, reportsFailure } from "./evidence.js"; import { REFERENCE_SENTINEL, referenceComparedAll } from "./reference.js"; import { testsRealFor } from "./tests-real.js"; import { runCommand, draftedShell } from "./run.js"; @@ -627,6 +627,15 @@ export function diagnoseFailure( hint: "the command was not found, so nothing ran and nothing was established", }; } + // dash (Debian's and Ubuntu's /bin/sh) exits 2, not 127, for `sh tool.sh` + // when tool.sh is missing. Read the same way on every platform: on macOS, + // where sh is bash, the same check exits 127. + if (exitCode === 2 && /^\S*sh: \d+: (?:cannot open|Can't open) [^\n]*(?:No such file|nonexistent)/im.test(stderr)) { + return { + didNotRun: true, + hint: "the script it runs was not found, so nothing ran and nothing was established", + }; + } if (exitCode === 126) { return { didNotRun: true, @@ -2178,6 +2187,7 @@ export async function runCheck(check: Check, ctx: BarContext): Promise "_".repeat(m.length)); -} +/** Something in the command that can end it non-zero whatever the last step does: `exit 1`, `set -e`. */ +const EXITS_NONZERO = /\bexit\s+[1-9]|\bset\s+-[a-z]*e|\berrexit\b/; +/** In an inline program: something that fails the process on a false condition. */ +const PROGRAM_FAILS = /\bassert\b|sys\.exit|\bexit\s*\(|\bquit\s*\(|\braise\b|os\._exit|process\.exit|exitCode|\bthrow\b|\bdie\b|\babort\b|\bexit\s+[1-9]/; +/** In an inline program: a comparison or a boolean-valued test. */ +const COMPARES = /==|!=|<=|>=|(?(?![>=])|\bin\b|\bis\b|\bnot\b|startswith\(|endswith\(|isinstance\(|\ball\(|\bany\(|issubset|issuperset|\.includes\(|\.equals?\(|\.every\(|\.some\(/; +/** An inline program that prints. */ +const PRINTS = /\bprint\s*\(|\bprint\s|console\.log|process\.stdout\.write|\bputs\b|\bsay\b|\bp\s/; +const INTERPRETER = /^(?:\w+=\S*\s+)*(?:timeout\s+\S+\s+)?(?:python3?|node|ruby|perl|php)\b/; /** - * Why this command exits 0 whatever happens, or null. Read off the command - * alone, before anything runs: + * Why this command exits 0 whatever its assertion found, or null. Read off + * the command alone, before anything runs: * - its last top-level step is `|| echo …`/`|| printf …`, or it ends in * `|| true`, `|| :`, `|| exit 0`, `; true`, `; exit 0` (evidence.ts); + * - its last step is `echo`/`printf` after `;` or a newline (`…; echo $?`, + * `…; echo done`), with no `exit 1` or `set -e` anywhere: the check exits + * with echo's status; * - it ends in `find … -exec … \;` (or `';'`): find's own status ignores * what -exec ran, so `-exec sh -c '… exit 1' \;` reports nothing; - * - it prints a literal PASS/FAIL verdict (`print('PASS' if … else 'FAIL')`, - * `echo FAIL`) and nothing in it exits non-zero, asserts or raises. + * - its deciding step is an inline program (`python3 -c`, `node -e`, a + * heredoc) that prints a comparison or a boolean and has nothing that + * exits non-zero: `print(x == y)` exits 0 whether it printed True or + * False. The same for `jq` without `-e`, and `awk` with no `exit`; + * - it prints a literal PASS/FAIL verdict and nothing fails the process; + * - it ends in an `if … fi` with no branch that fails. + * A step whose output a later pipe stage reads (`… | grep -qx PASS`) is not + * the deciding step: the reader's status is the check's. */ export function cannotFail(run: string): string | null { const cmd = run.trim(); - const masked = maskQuotes(cmd); + const masked = maskShell(cmd); const sw = swallowsExit(cmd); if (sw) return `it ends in \`${sw}\`, so it exits 0 whatever happened`; const echoTail = /\|\|\s*(?:echo|printf)\b[^|;&]*$/.exec(masked); @@ -396,15 +410,46 @@ export function cannotFail(run: string): string | null { const find = /(?:^|[;&|(]\s*)find\s[^|;]*-exec(?:dir)?\s/.exec(masked); if (find) { const rest = cmd.slice(find.index); - const end = /(?:\\;|';'|";")/.exec(rest); + const end = /(?:\;|';'|";")/.exec(rest); const after = end ? maskQuotes(rest.slice(end.index + end[0].length)).trim() : ""; // Only when nothing after the find reads its output: `find … | grep -q .` is fine. if (end && !/^\|(?!\|)/.test(after) && !/^[;&|]*\s*\S/.test(after.replace(/^&&\s*(?:echo|printf|true|:)\b.*$/, ""))) { - return "it ends in `find … -exec … \\;`, whose exit status ignores what -exec ran, so it passes whatever the files hold"; + return "it ends in `find … -exec … \;`, whose exit status ignores what -exec ran, so it passes whatever the files hold"; } } - if (/\bFAIL(?:ED)?\b/.test(cmd) && /\bPASS(?:ED)?\b/.test(cmd) && !/\bexit\s*\(?\s*[1-9]|sys\.exit|\bassert\b|\braise\b|process\.exit|\bfalse\b/.test(cmd)) { - return "it prints PASS or FAIL but always exits 0, and only the exit status is read"; + const steps = topLevelCommands(cmd); + const last = steps.at(-1); + if (!last) return null; + const exitsNonzero = EXITS_NONZERO.test(masked); + if (steps.length > 1 && (last.op === ";" || last.op === "\n") && /^(?:echo|printf)\b/.test(last.text) && !exitsNonzero) { + return `it ends in \`${last.op === ";" ? "; " : ""}${last.text.slice(0, 40)}\`, so it exits with echo's status (0) whatever ran before`; + } + const ifAt = steps.findLastIndex((st) => /^if\b/.test(st.text)); + const ifBlock = ifAt >= 0 ? steps.slice(ifAt).map((st) => st.text).join("; ") : ""; + if (/\bfi\s*$/.test(masked) && ifAt >= 0 && !exitsNonzero && !/\bfalse\b|\breturn\s+[1-9]/.test(maskQuotes(ifBlock))) { + return "it ends in an `if … fi` with no branch that fails, so it exits 0 when the condition is false"; + } + // The deciding step: the last one, or the one before an `|| exit 1` / `|| false`. + let decider = last; + if (steps.length > 1 && last.op === "||" && /^(?:exit\s+[1-9]\d*|false)\s*$/.test(last.text)) decider = steps.at(-2)!; + // When a later pipe stage reads the output (`… | grep -qx PASS`), that + // stage is the last step, and it is what is read here. + { + const d = decider.text; + if (INTERPRETER.test(d) && /\s-(?:c|e|E)\b|\s-\s*<<|<<|\s-\s*$/.test(d) && PRINTS.test(d) && !PROGRAM_FAILS.test(d)) { + if (/\b(?:True|False|true|false)\b/.test(d) || COMPARES.test(d.replace(/^\S+\s+-[a-zA-Z]\s*/, ""))) { + return "it prints the result of a comparison and exits 0 whether that is true or false; make it fail (assert, sys.exit(1), or compare in the shell)"; + } + } + if (/^jq\b/.test(d) && !/\s(?:-e|--exit-status)\b|\s-[a-zA-Z]*e[a-zA-Z]*\s/.test(` ${d} `) && /==|!=|[<>]|\btest\(|\bcontains\(|\bany\b|\ball\b|\bselect\(/.test(d)) { + return "it runs `jq` without `-e`, which prints true or false and exits 0 either way"; + } + if (/^awk\b/.test(d) && /==|!=|[<>]|~/.test(d) && !/\bexit\b/.test(d)) { + return "it runs `awk` with no `exit`, which exits 0 whatever its comparison found"; + } + if (/\bFAIL(?:ED)?\b/.test(d) && /\bPASS(?:ED)?\b/.test(d) && !/^(?:\[|test\b)/.test(d) && !PROGRAM_FAILS.test(d) && !/\bfalse\b/.test(d) && !exitsNonzero) { + return "it prints PASS or FAIL but always exits 0, and only the exit status is read"; + } } return null; } diff --git a/src/criteria.ts b/src/criteria.ts index be19da9..8c68ca3 100644 --- a/src/criteria.ts +++ b/src/criteria.ts @@ -34,6 +34,7 @@ import { cannotFail, lintAll, readTree, type LintCtx, type Tree } from "./checkl import { checkMutates } from "./checkwrites.js"; import { copyTree } from "./scratch.js"; import { diagnoseFailure } from "./bar.js"; +import { reportsFailure } from "./evidence.js"; import { normalizeRequirements } from "./signout.js"; import { evidenceTags } from "./tiers.js"; import type { CheckAuthor } from "./types.js"; @@ -290,7 +291,7 @@ export function strayPath(run: string, opts: { cwd: string; task?: string }): st } export async function preflightCriteria( - checks: readonly { name: string; kind?: string; run?: string; expectExit?: number }[], + checks: readonly { name: string; kind?: string; run?: string; expectExit?: number; hidden?: boolean; tags?: readonly string[] }[], opts: { cwd: string; timeoutMs?: number; @@ -342,8 +343,12 @@ export async function preflightCriteria( try { const r = await runCommand(c.run, { cwd: copy?.dir ?? opts.cwd, - // Drafted checks (stray set) run under the shell the bar will use. - shell: draftedShell({ hidden: !!opts.stray, tags: ["task"] }), + // Under the shell the bar will run this check with (src/run.ts + // draftedShell): bash for a drafted check, sh otherwise. Tried under + // one shell and judged under another, a bashism (`[[ ]]`, `==` in + // `[ ]`, `echo -e`) failed here under dash and passed at the bar, and + // counted as a check that tells the work from none. + shell: draftedShell(c), timeoutMs: opts.timeoutMs ?? 5_000, maxBuffer: 1024 * 1024, signal: opts.signal, @@ -361,11 +366,19 @@ export async function preflightCriteria( if (d.didNotRun) broken.push({ name: c.name, run: c.run, why: d.hint ?? "did not run" }); else if (unparsed) { broken.push({ name: c.name, run: c.run, why: `the shell could not parse it: ${firstLine(copy ? copy.unmap(r.stderr) : r.stderr)}` }); - } else if (selfError) broken.push({ name: c.name, run: c.run, why: selfError }); else if (!r.timedOut && r.code === (c.expectExit ?? 0)) { - opts.passed?.push(c.name); - if (/\bFAIL(?:ED)?\b/.test(`${r.stdout}\n${r.stderr}`)) opts.printedFail?.push(c.name); - } - else opts.failed?.push(c.name); + } else if (selfError) broken.push({ name: c.name, run: c.run, why: selfError }); + else if (r.timedOut) { + // Out of time is not a verdict: a check slower than the try (an + // existing suite) that would pass on the untouched tree is not one + // that fails there. Neither list: not tried. + } else if (r.code === (c.expectExit ?? 0)) { + // Read as the bar reads it: a task check that printed False or FAIL + // and exited 0 failed (evidence.ts reportsFailure). + const said = (c.expectExit ?? 0) === 0 && (c.tags?.includes("task") ?? true) ? reportsFailure(c.run, r.stdout) : null; + if (said || /\bFAIL(?:ED)?\b/.test(`${r.stdout}\n${r.stderr}`)) opts.printedFail?.push(c.name); + if (said) opts.failed?.push(c.name); + else opts.passed?.push(c.name); + } else opts.failed?.push(c.name); } catch { // Failing to spawn it here is molt's problem, not the criterion's. // Reporting it as broken would block work for the wrong reason. @@ -938,6 +951,8 @@ export async function draftCriteriaCritiqued( coverMaxMs?: number; /** A copy of the project taken before the work (see PreWork). Without it, no check is tried here. */ preWorkDir?: string; + /** Whether that copy is still as it was taken (src/scratch.ts preWorkCopy). A copy that changed is not tried on. */ + preWorkIntact?: () => Promise; }, ): Promise<{ ok: true; draft: Draft; critique: string[]; lint?: LintDrop[] } | { ok: false; error: string }> { // The critic's list of stated requirements rides on whatever draft comes @@ -966,7 +981,7 @@ export const PASSES_BEFORE_WORK = "this check passes before any work, so it cannot show the task was done; make it fail on the current tree"; /** Why a drafted check that printed FAIL and exited 0 on the pre-work copy is sent back. */ export const PRINTS_FAIL_EXITS_0 = - "it printed FAIL and still exited 0 before any work, so it can never fail; make a failure exit non-zero"; + "it printed a failure (FAIL or False) and still exited 0 before any work, so its exit code does not carry its verdict; make a failure exit non-zero (assert, sys.exit(1), jq -e, or compare in the shell)"; /** A drafted check the seal-time lint retired (src/checklint.ts), for the journal. */ export type LintDrop = { name: string; run: string; rule: string; why: string; redraft: boolean }; @@ -979,6 +994,7 @@ async function critiqued( /** Bound on the step that adds checks for uncovered requirements (COVER_MAX_MS). */ coverMaxMs?: number; preWorkDir?: string; + preWorkIntact?: () => Promise; }, sink: Sink, ): Promise<{ ok: true; draft: Draft; critique: string[] } | { ok: false; error: string }> { @@ -1035,14 +1051,19 @@ async function critiqued( if (!opts.preWorkDir || !l.ok.length) return l; const passed: string[] = []; const printedFail: string[] = []; + // A copy the worker changed (the draft can outlive the start of the work) + // is not the tree before the work: nothing tried on it decides anything. + if (opts.preWorkIntact && !(await opts.preWorkIntact())) return l; try { await preflightCriteria( - l.ok.map((c) => ({ name: c.name, kind: "command", run: c.run })), + // As they will be sealed: hidden task checks, so under the bar's shell. + l.ok.map((c) => ({ name: c.name, kind: "command", run: c.run, hidden: true, tags: ["task"] })), { cwd: opts.preWorkDir, passed, printedFail }, ); } catch { return l; } + if (opts.preWorkIntact && !(await opts.preWorkIntact())) return l; const ok: DraftedCheck[] = []; const bad = [...l.bad]; for (const c of l.ok) { diff --git a/src/evidence.ts b/src/evidence.ts index 85459ea..bfb17e0 100644 --- a/src/evidence.ts +++ b/src/evidence.ts @@ -242,3 +242,105 @@ export function judgePass(run: string, output: string, emptyAllowed = false): Pa } return { ok: true, summary }; } + +/** Mask quoted text so operators inside strings are not read as the shell's. */ +export function maskQuotes(s: string): string { + return s.replace(/'[^']*'|"(?:\\.|[^"\\])*"/g, (m) => "_".repeat(m.length)); +} + +/** Mask heredoc bodies (and quotes) so a script's own `;` and newlines are not the shell's. */ +export function maskShell(s: string): string { + let m = maskQuotes(s); + for (const h of s.matchAll(/<<-?\s*(['"]?)(\w+)\1/g)) { + const bodyStart = s.indexOf("\n", (h.index ?? 0) + h[0].length); + if (bodyStart < 0) continue; + const end = new RegExp(String.raw`\n[ \t]*${h[2]}[ \t]*(?:\n|$)`).exec(s.slice(bodyStart)); + const stop = end ? bodyStart + end.index + end[0].trimEnd().length : s.length; + m = m.slice(0, bodyStart) + "_".repeat(stop - bodyStart) + m.slice(stop); + } + return m; +} + +type Segment = { text: string; op: string }; + +/** + * The top-level commands of a shell command, each with the operator before + * it (`;`, newline, `&&`, `||`, `|`, `&`, or "" for the first). Quotes, + * heredoc bodies, `( )`, `{ }`, `$( )` and `[[ ]]` are not split. + */ +export function topLevelCommands(run: string): Segment[] { + const m = maskShell(run); + const out: Segment[] = []; + let depth = 0; + let from = 0; + let op = ""; + for (let i = 0; i < m.length; i++) { + const c = m[i]!; + const two = m.slice(i, i + 2); + if (two === "[[") { depth++; i++; continue; } + if (two === "]]") { depth = Math.max(0, depth - 1); i++; continue; } + if (c === "(" || c === "{") { depth++; continue; } + if (c === ")" || c === "}") { depth = Math.max(0, depth - 1); continue; } + if (depth > 0) continue; + let sep = ""; + if (two === "&&" || two === "||") sep = two; + else if (c === ";" || c === "\n" || c === "|" || (c === "&" && m[i - 1] !== ">" && m[i + 1] !== ">")) sep = c; + if (!sep) continue; + const text = run.slice(from, i).trim(); + if (text) out.push({ text, op }); + op = sep; + from = i + sep.length; + i += sep.length - 1; + } + const text = run.slice(from).trim(); + if (text) out.push({ text, op }); + return out; +} + +/** + * The step whose exit status is the check's, when that step prints its + * verdict instead of exiting with it: an inline program (`python3 -c`, + * `node -e`, a heredoc), `jq` without `-e`, `awk`, or `echo`/`printf` + * (`… || echo fail`, `…; echo $?`). Null when the deciding step is anything + * else, such as `grep`, `test`, `jq -e` or `find`: their own output (jq + * printing false inside `find -exec`) is not the check's verdict. + */ +export function verdictPrinter(run: string): string | null { + const steps = topLevelCommands(run.trim()); + let d = steps.at(-1); + if (!d) return null; + if (steps.length > 1 && d.op === "||" && /^(?:exit\s+[1-9]\d*|false)\s*$/.test(d.text)) d = steps.at(-2)!; + const t = d.text; + if (/^(?:\w+=\S*\s+)*(?:timeout\s+\S+\s+)?(?:python3?|node|ruby|perl|php)\b/.test(t) && /\s-(?:c|e|E)\b|<<|\s-\s*$/.test(t)) return t; + if (/^jq\b/.test(t) && !/\s(?:-e|--exit-status)\b|\s-[a-zA-Z]*e[a-zA-Z]*\s/.test(` ${t} `)) return t; + if (/^awk\b/.test(t)) return t; + if (/^(?:echo|printf)\b/.test(t)) return t; + return null; +} + +/** + * Did a check that exited 0 say, in words, that it failed? + * + * `python3 -c "print(rows == [['a'], ['b']])"` exits 0 whether it printed + * True or False, and so does `jq '.n == 3'`, `…; echo $?` and a script that + * prints FAIL. The exit code of such a check does not depend on what it + * asserted. Its own last line does: a check whose last line on stdout is + * False, FAIL(ED) or NO, or whose `echo $?` tail printed something other + * than 0, reported a failure and is read as one, when the step that decides + * the exit is the one that printed (verdictPrinter): `find … -exec sh -c + * 'jq -e …'` prints jq's `false` for files that rightly do not match, and + * that is not the check's verdict. Only for task checks (the drafted and + * approved criteria), never a project's own commands. + */ +export function reportsFailure(run: string, stdout: string): string | null { + // Only when the step that decides the exit prints the verdict instead. + if (!verdictPrinter(run)) return null; + const lines = stdout.split(/\r?\n/).map((l) => l.trim()).filter(Boolean); + const lastLine = lines.at(-1); + if (lastLine === undefined) return null; + if (/^(?:false|fail(?:ed|ure)?|no)$/i.test(lastLine)) return `it printed \`${lastLine}\` and exited 0: a check that reports a failure in words has failed`; + if (/(?:^|[;\n&]\s*)echo\s+["']?\$\?["']?\s*$/.test(run.trim()) && lastLine !== "0") { + return `it ends in \`echo $?\`, which printed ${lastLine}: the step before it failed`; + } + return null; +} diff --git a/test/prework-screen.test.ts b/test/prework-screen.test.ts index c50ea5f..920d515 100644 --- a/test/prework-screen.test.ts +++ b/test/prework-screen.test.ts @@ -16,8 +16,11 @@ import { writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, it } from "node:test"; import { cannotFail } from "../src/checklint.js"; +import { reportsFailure } from "../src/evidence.js"; import { draftCriteriaCritiqued, PASSES_BEFORE_WORK } from "../src/criteria.js"; -import { workspace } from "./helpers.js"; +import { allowAll, drain, scriptedProvider, workspace } from "./helpers.js"; +import { Engine } from "../src/engine.js"; +import type { Check } from "../src/types.js"; describe("cannotFail", () => { it("flags the constructs that exit 0 whatever happens", () => { @@ -31,6 +34,60 @@ describe("cannotFail", () => { for (const run of flagged) assert.ok(cannotFail(run), run); }); + it("flags checks whose exit code does not depend on what they assert", () => { + const flagged = [ + // The csv-clean class: prints True or False, exits 0 either way. + `python3 -c "import csv; print(list(csv.reader(open('out.csv'))) == [['a'],['b']])"`, + `python3 -c "print(open('o').read().strip() == 'x')" || exit 1`, + `cd sub && python3 -c "print(1 in [1])"`, + "python3 - <<'EOF'\nimport json\nd = json.load(open('o.json'))\nprint(d['n'] == 3)\nEOF", + `node -e "console.log(require('fs').readFileSync('o','utf8').trim() === 'x')"`, + "jq '.count == 3' out.json", + "awk '$1 > 3' data.txt", + // An echo of a result. + "grep -q 42 out.txt; echo $?", + "test -f out.txt; echo done", + `if [ "$(cat out)" = 3 ]; then echo yes; fi`, + `if [ "$(cat out)" = 3 ]; then echo yes; else echo no; fi`, + ]; + for (const run of flagged) assert.ok(cannotFail(run), run); + }); + + it("leaves a comparison alone when something fails on it, or a later stage reads the output", () => { + const fine = [ + `python3 -c "print('PASS' if 1 == 1 else 'FAIL')" | grep -qx PASS`, + `[ "$(python3 -c "print(1 == 1)")" = True ]`, + `python3 -c "import json; assert json.load(open('o.json'))['n'] == 3"`, + `python3 -c "import sys; sys.exit(0 if open('o').read() == 'x' else 1)"`, + "jq -e '.count == 3' out.json", + "awk 'END { exit !(NR == 3) }' data.txt", + "if grep -q x f; then exit 0; else exit 1; fi", + "set -e; grep -q x f; echo ok", + "test -f out.txt && echo done", + "python3 server.py & sleep 1; curl -sf localhost:8080/items | jq -e 'length == 2'", + `python3 -c "import out; print(out.f(2))" | grep -qx 4`, + `node -e "const a=require('assert'); a.strictEqual(1,1)"`, + "for f in a b; do grep -q x $f || exit 1; done; echo ok", + ]; + for (const run of fine) assert.equal(cannotFail(run), null, run); + }); + + it("reportsFailure: a task check's own words decide when its exit code cannot", () => { + assert.match(reportsFailure("python3 -c 'print(x == y)'", "False\n")!, /printed `False`/); + assert.ok(reportsFailure(`python3 -c "print('rows: 2'); print('FAIL')"`, "rows: 2\nFAIL\n")); + assert.ok(reportsFailure("grep -qx 3 out.txt || echo fail", "fail\n")); + assert.ok(reportsFailure("jq '.n == 3' o.json", "false\n")); + assert.ok(reportsFailure("grep -q 42 out.txt; echo $?", "1\n")); + assert.equal(reportsFailure("grep -q 42 out.txt; echo $?", "0\n"), null); + assert.equal(reportsFailure("python3 -c 'print(x == y)'", "True\n"), null); + assert.equal(reportsFailure(`python3 -c "print('3 failed, then fixed'); print('ok')"`, "3 failed, then fixed\nok\n"), null, "only the last line is the verdict"); + assert.equal(reportsFailure("python3 -c 'print(1)'", ""), null); + // The step that decides the exit did not print the verdict: jq inside find, jq -e, grep. + assert.equal(reportsFailure(String.raw`find configs/ -name '*.json' -exec sh -c 'jq -e ".version == 1" {} && exit 1 || exit 0' \;`, "false\nfalse\n"), null); + assert.equal(reportsFailure("jq -e '.ok' o.json", "false\n"), null); + assert.equal(reportsFailure("cat out.txt | grep -c x", "no\n"), null); + }); + it("leaves checks whose status means something alone", () => { const fine = [ "grep -qx hello out.txt", @@ -134,3 +191,38 @@ describe("drafted checks tried on the pre-work copy", () => { } }); }); + +describe("a check that prints a boolean, at the bar", () => { + // The reviewer's reproduction: sealed as a judge's check, it raised before the + // work (no out.csv), so the pre-work try counted it as failing there; after + // WRONG work it printed False and exited 0, and the run read "verified". + const run = `python3 -c "import csv; print(list(csv.reader(open('out.csv'))) == [['a'],['b']])"`; + const check = { name: "rows", kind: "command", run, timeoutMs: 10_000, expectExit: 0, tags: ["task", "value"], hidden: true, author: { kind: "judge", model: "judge-j" } } as Check; + async function turn(csv: string) { + const ws = workspace(); + try { + const provider = scriptedProvider([ + { calls: [{ name: "write_file", args: { path: "out.csv", content: csv } }] }, + { text: "Done." }, + ]); + const engine = new Engine({ baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: provider.fetchFn, bar: null, stream: false, autonomy: "high", maxProofAttempts: 1 }); + const events = await drain(engine.run("Write out.csv with rows a then b.", allowAll, { taskChecks: [check] })); + const end = events.find((e) => e.kind === "job_end"); + assert.ok(end && end.kind === "job_end"); + return { end, events }; + } finally { + ws.cleanup(); + } + } + it("wrong work: it printed False, so it failed, and nothing is verified", async () => { + const { end, events } = await turn("b\na\n"); + assert.notEqual(end.outcome, "verified"); + const refused = events.find((e) => e.kind === "proof_refused" || e.kind === "proof_exhausted"); + assert.match(JSON.stringify(refused), /printed `False` and exited 0/); + }); + it("right work: it printed True and passes as before", async () => { + const { end } = await turn("a\nb\n"); + assert.deepEqual([end.outcome, end.claim], ["verified", "verified (independent checks: judge-j)"]); + }); +}); + From 8d0303c2e7b3f3ff9a0b2cb599105bfbdc6a4d8d Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 17:52:10 -0700 Subject: [PATCH 4/4] Late checks: trust the pre-work copy only while it is unchanged The copy taken at turn start for late checks lived in $TMPDIR, owned by the uid the worker's bash runs as. A worker could edit it so a check that passes on the pristine tree "failed before the work" and counted as discriminating. preWorkCopy takes a content digest when the copy is made (0700 parent; under --worker-user that is Maat's private state dir) and checks it before and after every late try and every seal-time screen. A copy that changed counts as not tried, journalled with tampered: true. A late check that names the project's absolute path reads the finished tree, so it is not tried either. Golden files a late check diffs against are recorded from the copy, so the #32 golden rule covers late checks too. The late try now honours a pending cancel, the copy's size and timing are journalled (pre-work-copy), and a missing copy is journalled instead of skipped silently. CHANGELOG, README and bench/harbor/README.md describe --require-discriminating, passed-untested, the P1 redraft and drop, the verdict read and the shell change. --- CHANGELOG.md | 33 ++++++++ README.md | 9 ++- bench/harbor/README.md | 7 +- src/cli.tsx | 13 +-- src/engine.ts | 76 +++++++++++++++--- src/scratch.ts | 66 ++++++++++++++- test/discriminating-checks.test.ts | 124 +++++++++++++++++++++++++++-- 7 files changed, 301 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 431384e..15eb5de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,39 @@ module the work's code can reach, and expected values never leave the driver's memory. A pass also needs the driver's final `REFERENCE COMPARED n/n` line: an exit 0 without it is a failure. +- **Drafted checks that pass before the work are redrafted, then dropped.** + With `--criteria auto` each drafted check is tried on a copy of the project + taken before the first step. One that already passes there cannot show the + task was done: it goes back to the drafter once, and is dropped if the + redraft still passes (journal: `check-lint-dropped`, rule + `P1-passes-before-work`). An always-on lint (L16) does the same for checks + that cannot fail by construction: `|| echo …`, `|| true`, `; exit 0`, + `…; echo $?`, `find … -exec … \;`, an `if … fi` with no failing branch, + and inline programs whose exit code does not depend on what they assert + (`python3 -c "print(x == y)"`, `node -e "console.log(a === b)"`, `jq` + without `-e`, `awk` with no `exit`, a printed PASS/FAIL). +- **A task check that prints its failure has failed.** A drafted or approved + check that exits 0 but whose last line of output is `False`, `FAIL`, + `FAILED` or `NO` (or whose `echo $?` tail printed non-zero) is read as a + failure at the bar and before the work. `print(rows == expected)` printing + False on wrong work used to pass and earn "verified". +- **`--require-discriminating`** (`MAAT_REQUIRE_DISCRIMINATING=1`, off by + default; `--review-advisory` implies it): "verified" needs an independent + value check that failed on the tree before the work and passes now. + Otherwise the new tier **`passed-untested`** ("passed checks that did not + test this work, not verified"): outcome `unverified`, exit code 3. Each + receipt row says how its check fared before the work. +- **Checks are tried before the work under the shell the bar runs them + with** (bash for drafted checks), so a bashism no longer "fails before the + work" under dash and passes at the bar. dash's exit 2 for a missing + `sh script` is read like bash's 127. A try that timed out counts as not + tried, not as failing. +- **The pre-work copy is checked before it is trusted.** Late checks (drafts + past the time cut, the reference check) are tried on a copy taken at turn + start. Its digest is taken when it is made and compared around every try; + a copy that changed during the work, or a check that names the project's + absolute path, counts as not tried. The copy's size and timing are + journalled (`pre-work-copy`). - **Claims say who stood behind them** on the terminal, the window, job_end's `claim`, the receipt index and the receipt: `verified (independent checks: )`, `verified (your checks)`, `passed own checks (), not diff --git a/README.md b/README.md index 46f1d4f..e209408 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,8 @@ Exit codes: `0` the bar was met · `1` it was not · `2` usage · `3` finished without a verdict (no bar, or `--only`/`--skip` left required checks unrun, which is never reported as a pass), or passed without earning "verified": every check that could prove the work was written by the worker model itself -(`passed-own-checks`), or nothing but builtins passed. "Verified" needs a +(`passed-own-checks`), with `--require-discriminating` none of them failed +before the work (`passed-untested`), or nothing but builtins passed. "Verified" needs a passing check from another model (`--judge`) or a command check you wrote or approved; builtins never carry it. @@ -410,6 +411,12 @@ the task are drafted before the work, they are sealed, and the worker is held to them. Passing checks the worker model drafted for itself are never "verified": without `--judge ` (or checks a person approved) the best a run earns is `passed own checks (), not verified`, exit 3. +Drafted checks are tried on a copy of the project before the work: one that +already passes there, or cannot fail by construction, is sent back to the +drafter once and dropped if the redraft is no better. `--require-discriminating` +goes further: "verified" then needs an independent value check that failed +before the work, and anything less is `passed checks that did not test this +work, not verified` (exit 3). ## Docs diff --git a/bench/harbor/README.md b/bench/harbor/README.md index 915a5e7..36eaa36 100644 --- a/bench/harbor/README.md +++ b/bench/harbor/README.md @@ -13,7 +13,12 @@ model, and since 2026-10-07 a run that passes only its own checks ends `passed own checks (), not verified` (job_end `tier: passed-own-checks`, exit 3), never `verified`. Set `MAAT_JUDGE_MODEL` (and `MAAT_JUDGE_URL` / `MAAT_JUDGE_KEY`) in the container to another model to make "verified" -reachable. The exit code never fails a trial (see below). +reachable. Drafted checks that pass on the untouched task, or cannot fail +by construction, are redrafted once and then dropped before they are +sealed. With `--require-discriminating` (or `MAAT_REQUIRE_DISCRIMINATING=1`) +a run whose independent checks never failed before the work ends +`passed-untested`, also exit 3. The exit code never fails a trial (see +below). ## What it measures diff --git a/src/cli.tsx b/src/cli.tsx index 2dad220..8be2bca 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -53,7 +53,7 @@ import type { BarResult, Check, CheckAuthor, EngineEvent } from "./types.js"; import { stateDir } from "./statedir.js"; import { draftReference, snapshotProject } from "./reference.js"; import { env } from "./env.js"; -import { copyTree } from "./scratch.js"; +import { preWorkCopy } from "./scratch.js"; /** * The version, from the manifest that npm actually publishes. @@ -1240,9 +1240,12 @@ async function autoDraft( // show this task was done, so it is sent back to the drafter once and // dropped if it still passes (criteria.ts screen). Null when the project is // too big to copy: the checks are then not tried here. - const preWork = args.cwd ? await copyTree(args.cwd) : null; + // Fingerprinted: the drafting can outlive the start of the work (a late + // draft), and the worker runs as this uid, so every try checks the copy is + // still as taken (src/scratch.ts preWorkCopy). + const preWork = args.cwd ? await preWorkCopy(args.cwd) : null; try { - return await autoDraftFrom(engine, args, snapshot, preWork?.dir, soFar, inputs, deadlineAt); + return await autoDraftFrom(engine, args, snapshot, preWork ? { dir: preWork.dir, intact: preWork.intact } : undefined, soFar, inputs, deadlineAt); } finally { await preWork?.cleanup(); } @@ -1252,7 +1255,7 @@ async function autoDraftFrom( engine: Engine, args: Args, snapshot: DrafterInputs, - preWorkDir: string | undefined, + preWork: { dir: string; intact: () => Promise } | undefined, soFar?: { draft?: Draft; sealed?: boolean; late?: boolean }, inputs?: { snapshot: DrafterInputs; used: string[] }, deadlineAt?: number, @@ -1274,7 +1277,7 @@ async function autoDraftFrom( barChecks: (engine.cfg.bar?.checks ?? []).map((c) => c.name), ...judgeTarget({ baseUrl: args.url, apiKey: args.key, model: args.model }), cwd: args.cwd, - ...(preWorkDir ? { preWorkDir } : {}), + ...(preWork ? { preWorkDir: preWork.dir, preWorkIntact: preWork.intact } : {}), reasoningEffort: judgeEffort(args.reasoningChecks ?? args.reasoning), latency: engine.askLatency, deadlineAt, diff --git a/src/engine.ts b/src/engine.ts index 6d05da7..d4cb170 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -27,7 +27,7 @@ import { recordGoldens, valueUnproven } from "./golden.js"; import { arbitrate, parseDisputes, type Ruling } from "./dispute.js"; import { createHash } from "node:crypto"; import { execFileSync } from "node:child_process"; -import { mkdirSync, readFileSync, writeFileSync, existsSync, statSync } from "node:fs"; +import { mkdirSync, readFileSync, realpathSync, writeFileSync, existsSync, statSync } from "node:fs"; import { excludeMoltFromGit, listProject, removeNew, unnamedNewFiles, type ProjectListing } from "./leftovers.js"; import { inspectDir, inspectFile, namedInputs, profileLine } from "./inspect.js"; import { basename, dirname, resolve, relative, isAbsolute, join } from "node:path"; @@ -43,7 +43,7 @@ import { barPath, } from "./bar.js"; import { checkSelfError, preflightCriteria } from "./criteria.js"; -import { copyTree, type TreeCopy } from "./scratch.js"; +import { preWorkCopy, type PreWorkCopy } from "./scratch.js"; import { AUTONOMY_SUMMARY, DEFAULT_AUTONOMY, @@ -2223,7 +2223,7 @@ export class Engine { * gets the same pre-work try as one sealed up front. Null when nothing is * pending or the tree is too big to copy: a late check is then untried. */ - private preWorkTree: TreeCopy | null = null; + private preWorkTree: PreWorkCopy | null = null; /** True only while `proveNow` runs: a bar with no turn behind it. */ private standalone = false; private barHash: string | null; @@ -3721,17 +3721,58 @@ export class Engine { private async tryLateBeforeWork(checks: readonly Check[], log?: Journal): Promise { const tree = this.preWorkTree; if (!tree || !checks.length) return; + const asTask = (n: string) => (n.startsWith("task:") ? n : `task:${n}`); + // A check that names the project by its absolute path (Terminal-Bench's + // /app/...) would read the finished tree, not the copy: its try says + // nothing about the tree before the work. Not tried. + const roots = [this.cwd, (() => { try { return realpathSync(this.cwd); } catch { return this.cwd; } })()]; + const named = (c: Check) => c.kind === "command" && roots.some((r) => r.length > 1 && c.run.includes(r)); + const absolute = checks.filter(named); + const tryable = checks.filter((c) => !named(c)); const passed: string[] = []; const failed: string[] = []; - this.running = new AbortController(); - try { - await preflightCriteria(checks, { cwd: tree.dir, signal: this.running.signal, passed, failed }); - } catch { - return; - } finally { - this.running = undefined; + // The worker may have run by now, as this same uid, so the copy is + // checked against the digest taken when it was made, before and after the + // try. A copy that changed is not the tree before the work: nothing tried + // on it counts (src/scratch.ts preWorkCopy). + const intactBefore = tryable.length ? await tree.intact() : false; + if (intactBefore) { + this.running = new AbortController(); + if (this.cancelRequested) this.running.abort(); + try { + await preflightCriteria(tryable, { cwd: tree.dir, signal: this.running.signal, passed, failed }); + } catch { + return; + } finally { + this.running = undefined; + } + } + const intact = intactBefore && (await tree.intact()); + if (!intact || absolute.length) { + log?.append("note", { + kind: "pre-work-try", + text: !intact && tryable.length + ? "the copy of the project taken before the work changed during the work: late checks count as not tried" + : "late checks that name the project's absolute path read the finished tree: not tried before the work", + late: true, + untried: [...(!intact ? tryable : []), ...absolute].map((c) => asTask(c.name)), + ...(!intact && tryable.length ? { tampered: true } : {}), + }); + } + if (!intact) return; + // Golden files a late check diffs against: what they held before the work, read from the copy. + for (const c of tryable) { + if (c.kind !== "command") continue; + const g = recordGoldens([{ run: c.run }], tree.dir); + for (const [abs, digest] of g) { + // Only files inside the copy: an absolute path outside it is read as + // it is now, after the work, and proves nothing about before. + const rel = relative(tree.dir, abs); + if (rel.startsWith("..") || isAbsolute(rel)) continue; + const key = join(this.cwd, rel); + if (!this.goldensBefore.has(key)) this.goldensBefore.set(key, digest); + } } - const asTask = (n: string) => (n.startsWith("task:") ? n : `task:${n}`); this.passedBeforeWork = new Set([...this.passedBeforeWork, ...passed.map(asTask)]); this.failedBeforeWork = new Set([...this.failedBeforeWork, ...failed.map(asTask)]); log?.append("note", { @@ -5412,7 +5453,18 @@ export class Engine { // taken: mark the pending draft handled now (it is awaited, and its // failure reported, further down) so that is not an unhandled rejection. opts.pendingCriteria?.catch(() => {}); - this.preWorkTree = (opts.pendingCriteria || opts.referenceCheck) && !opts.ask ? await copyTree(this.cwd) : null; + if ((opts.pendingCriteria || opts.referenceCheck) && !opts.ask) { + const t0 = Date.now(); + this.preWorkTree = await preWorkCopy(this.cwd); + // Journalled either way, so a replay can tell a late check that was + // never tried because there was no copy from one that failed its try. + log?.append( + "note", + this.preWorkTree + ? { kind: "pre-work-copy", text: `copied the project before the work for late checks: ${this.preWorkTree.files} file(s) in ${Date.now() - t0} ms`, files: this.preWorkTree.files, ms: Date.now() - t0 } + : { kind: "pre-work-copy", text: "no copy of the project before the work (too large, or it could not be copied): late checks are not tried before the work", ms: Date.now() - t0 }, + ); + } else this.preWorkTree = null; /** * This turn's criteria, copied and sealed before anything runs. * diff --git a/src/scratch.ts b/src/scratch.ts index 32e4c88..834ec2d 100644 --- a/src/scratch.ts +++ b/src/scratch.ts @@ -36,7 +36,8 @@ * then runs in place and says so; a reviewer's objection is not run). */ import { constants } from "node:fs"; -import { copyFile, lstat, mkdir, mkdtemp, readdir, readlink, realpath, rm, symlink, utimes, writeFile } from "node:fs/promises"; +import { chmod, copyFile, lstat, mkdir, mkdtemp, readdir, readFile, readlink, realpath, rm, symlink, utimes, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import { STATE_DIRS } from "./statedir.js"; @@ -333,3 +334,66 @@ export function runsInCopy(check: { kind: string; hidden?: boolean; tags?: reado if (check.kind !== "command") return false; return check.hidden === true || (check.tags ?? []).some((t) => t === "task" || t === "mission"); } + +/** + * A digest of a tree as it stands: every path, its type and mode, each file's + * size and content hash, each link's target. Links are not followed. Two + * calls agree only if nothing under `dir` was added, removed or changed. + */ +export async function digestTree(dir: string): Promise { + const h = createHash("sha256"); + const walk = async (abs: string, rel: string): Promise => { + const names = (await readdir(abs)).sort(); + for (const n of names) { + const p = join(abs, n); + const r = rel ? `${rel}/${n}` : n; + const st = await lstat(p); + if (st.isSymbolicLink()) h.update(`l ${r} ${await readlink(p)}\0`); + else if (st.isDirectory()) { + h.update(`d ${r} ${st.mode}\0`); + await walk(p, r); + } else if (st.isFile()) { + h.update(`f ${r} ${st.mode} ${st.size} `).update(createHash("sha256").update(await readFile(p)).digest("hex")).update("\0"); + } else h.update(`o ${r} ${st.mode}\0`); + } + }; + await walk(dir, ""); + return h.digest("hex"); +} + +/** + * A copy of the project taken before the work, that can say whether it is + * still as it was taken. + * + * The copy lives in a fresh 0700 directory under Maat's temporary folder + * (with --worker-user that folder is Maat's private state dir, out of the + * worker uid's reach). A worker running as Maat's own uid can still reach it, + * and editing it would turn a check that passes on the pristine tree into one + * that "failed before the work". So its digest is taken when it is made and + * checked around every later try: a copy that changed is not the tree before + * the work, and checks tried on it count as not tried. + */ +export type PreWorkCopy = TreeCopy & { digest: string; intact: () => Promise }; + +export async function preWorkCopy(root: string): Promise { + const c = await copyTree(root); + if (!c) return null; + try { + await chmod(join(c.dir, ".."), 0o700); + const digest = await digestTree(c.dir); + return { + ...c, + digest, + intact: async () => { + try { + return (await digestTree(c.dir)) === digest; + } catch { + return false; + } + }, + }; + } catch { + await c.cleanup(); + return null; + } +} diff --git a/test/discriminating-checks.test.ts b/test/discriminating-checks.test.ts index 16359f5..e9fd47b 100644 --- a/test/discriminating-checks.test.ts +++ b/test/discriminating-checks.test.ts @@ -15,13 +15,15 @@ * "passed-untested", outcome unverified, exit code 3. */ import assert from "node:assert/strict"; -import { readFileSync, readdirSync } from "node:fs"; +import { readFileSync, readdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, it } from "node:test"; import { parseArgs } from "../src/cli.js"; +import { diagnoseFailure } from "../src/bar.js"; import { Engine } from "../src/engine.js"; import { Journal } from "../src/journal.js"; import { Receipts } from "../src/receipts.js"; +import { preWorkCopy } from "../src/scratch.js"; import { claimLabel, noteCoverage, tierOf, UNTESTED_CLAIM } from "../src/tiers.js"; import type { Check, CheckAuthor } from "../src/types.js"; import { allowAll, drain, scriptedProvider, workspace, type ScriptedTurn } from "./helpers.js"; @@ -152,12 +154,30 @@ describe("the rule in a turn", () => { assert.deepEqual(tried[0]!.data.failed, ["task:greeting"]); }); - it("a check that could not run before the work has no pre-work try: not verified", async () => { - // `sh tool.sh` before tool.sh exists exits 127: the try found it broken, not failing. - const { end, receipt } = await turn([check("tool", "sh tool.sh")]); - assert.deepEqual([end.outcome, end.tier], ["unverified", "passed-untested"]); - assert.match(end.tierReason!, /`task:tool` was not tried before the work began/); - assert.match(receipt, /before the work: not tried/); + it("a check that could not run before the work has no pre-work try: not verified, on every platform", async () => { + // Before tool.sh exists, `bash tool.sh` exits 127, and `sh tool.sh` exits 127 + // where sh is bash (macOS) and 2 where it is dash (Debian, Ubuntu): both are + // read as broken, not failing, so neither counts as tried. + for (const run of ["bash tool.sh", "sh tool.sh"]) { + const { end, receipt } = await turn([check("tool", run)]); + assert.deepEqual([end.outcome, end.tier], ["unverified", "passed-untested"], run); + assert.match(end.tierReason!, /`task:tool` was not tried before the work began/); + assert.match(receipt, /before the work: not tried/); + } + }); + + it("dash's missing-script exit reads as bash's", () => { + assert.equal(diagnoseFailure(2, "", "sh: 0: cannot open tool.sh: No such file\n").didNotRun, true); + assert.equal(diagnoseFailure(127, "", "bash: tool.sh: No such file or directory\n").didNotRun, true); + assert.equal(diagnoseFailure(2, "", "python3: can't open file 'x.py': [Errno 2] No such file or directory\n").didNotRun, false, "an interpreter's own missing file is the work missing"); + }); + + it("a drafted check is tried before the work under the shell the bar runs it with", async () => { + // `==` inside `[ ]` and `echo -e` are bash; under dash the first errs and the + // second prints "-e x". Tried under sh it "failed before the work" on Linux + // and passed at the bar under bash: a guard counted as discriminating. + const { tried } = await turn([check("bashism", `[ "$(echo -e x)" == "x" ]`)], [], false); + assert.deepEqual(tried[0]!.data.passed, ["task:bashism"]); }); }); @@ -167,3 +187,93 @@ describe("parseArgs --require-discriminating", () => { assert.equal(parseArgs(["--require-discriminating"]).requireDiscriminating, true); }); }); + +describe("late checks and the copy taken before the work", () => { + const later = (v: T, ms: number) => new Promise((r) => setTimeout(() => r(v), ms)); + const nothing = async () => ({ taskChecks: [] as Check[], taskNotes: [] as string[] }); + const mk = (name: string, run: string): Check => ({ name, kind: "command", run, timeoutMs: 5_000, expectExit: 0, tags: ["task", "value"], hidden: true, author: JUDGE }) as Check; + + const TASK = "Write out.txt containing exactly the word hello."; + async function lateTurn(dir: string, calls: { name: string; args: Record }[], checks: Check[]) { + const provider = scriptedProvider([{ calls }, { text: "Done." }]); + const journal = new Journal(dir, "late-copy"); + const engine = new Engine({ + baseUrl: "http://provider.test/v1", model: "m", cwd: dir, fetchFn: provider.fetchFn, bar: null, stream: false, autonomy: "high", journal, + requireDiscriminating: true, + }); + engine.setTurnDeadline(600_000); + const events = await drain(engine.run(TASK, allowAll, { pendingCriteria: later({ taskChecks: checks, taskNotes: [] }, 300), criteriaSoFar: nothing, criteriaWaitMs: 100 })); + const end = events.find((e) => e.kind === "job_end"); + assert.ok(end && end.kind === "job_end"); + return { end, tried: Journal.read(journal.path).filter((e) => e.kind === "note" && e.data.kind === "pre-work-try"), copies: Journal.read(journal.path).filter((e) => e.kind === "note" && e.data.kind === "pre-work-copy") }; + } + + it("is tried on the copy when the copy is untouched: the check failed before the work and discriminates", async () => { + const ws = workspace(); + try { + const { end, tried, copies } = await lateTurn(ws.dir, [{ name: "write_file", args: { path: "out.txt", content: "hello\n" } }], [mk("greeting", "grep -qx hello out.txt")]); + assert.equal(copies.length, 1, "the copy is journalled with its timing"); + assert.equal(typeof copies[0]!.data.ms, "number"); + assert.deepEqual(tried.at(-1)!.data.failed, ["task:greeting"]); + assert.equal(end.outcome, "verified"); + } finally { + ws.cleanup(); + } + }); + + it("a copy the worker changed during the work is not the tree before it: late checks count as not tried", async () => { + const ws = workspace(); + const seed = `seed-${process.pid}-${Date.now()}.txt`; + writeFileSync(join(ws.dir, seed), "pristine\n"); + try { + // The worker, as Maat's own uid, finds the copy under the temp folder and edits it. + const tamper = `for f in "\${TMPDIR:-/tmp}"/maat-check-*/*/${seed}; do echo hello > "$f"; done; cp ${seed} /dev/null`; + const { end, tried } = await lateTurn( + ws.dir, + [{ name: "bash", args: { command: tamper } }, { name: "write_file", args: { path: "out.txt", content: "hello\n" } }], + [mk("seeded", `grep -qx hello ${seed} || grep -qx hello out.txt`)], + ); + const note = tried.find((e) => e.data.tampered === true); + assert.ok(note, "the change to the copy is journalled"); + assert.deepEqual(note!.data.untried, ["task:seeded"]); + assert.equal(end.tier, "passed-untested"); + } finally { + ws.cleanup(); + } + }); + + it("a late check that names the project's absolute path is not tried on the copy", async () => { + const ws = workspace(); + try { + const { tried } = await lateTurn(ws.dir, [{ name: "write_file", args: { path: "out.txt", content: "hello\n" } }], [mk("abs", `grep -qx hello ${join(ws.dir, "out.txt")}`)]); + assert.ok(tried.some((e) => Array.isArray(e.data.untried) && (e.data.untried as string[]).includes("task:abs"))); + } finally { + ws.cleanup(); + } + }); +}); + +describe("preWorkCopy", () => { + it("knows when the copy changed", async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "a.txt"), "a\n"); + const c = await preWorkCopy(ws.dir); + assert.ok(c); + try { + assert.equal(await c.intact(), true); + writeFileSync(join(c.dir, "a.txt"), "b\n"); + assert.equal(await c.intact(), false, "same size, different content"); + writeFileSync(join(c.dir, "a.txt"), "a\n"); + assert.equal(await c.intact(), true); + writeFileSync(join(c.dir, "new.txt"), ""); + assert.equal(await c.intact(), false, "a file added"); + } finally { + await c.cleanup(); + } + } finally { + ws.cleanup(); + } + }); +}); +