From f66479c86fcde380663bff0088815ac222c54794 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 18:43:45 -0700 Subject: [PATCH 1/2] bench: audit the drafted checks Maat drops before sealing drop_audit.py collects every drop line the lanes wrote to stderr (trial, review before sealing, redraft, cover draft, critic), groups them by reason and samples commands. --run executes each dropped command in the bench image on the task's pristine tree and on the lane's finished tree, and grades the finished tree, so a check that failed before the work and passed on accepted work is counted as a good check thrown away. --lint re-reads the drops with a build's dist/ to show which it still makes, and which P1 drops it keeps as guards. --- bench/local/drop_audit.py | 358 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 358 insertions(+) create mode 100644 bench/local/drop_audit.py diff --git a/bench/local/drop_audit.py b/bench/local/drop_audit.py new file mode 100644 index 0000000..a0e8db1 --- /dev/null +++ b/bench/local/drop_audit.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +"""Audit of drafted checks Maat dropped before sealing. + +Collects every drop line the bench lanes wrote to stderr: + + maat: dropped drafted criterion NAME (RUN) — WHY (seal-time trial, cli.tsx) + maat: criteria review — dropped NAME (RUN) before sealing: WHY [RULE] + maat: criteria review — dropped NAME (RUN) from the redraft: WHY [RULE] + maat: criteria review — dropped NAME (RUN) from the cover draft: WHY [RULE] + maat: criteria review — dropped NAME from the redraft — "QUOTE" (the critic's own drop, no run) + +groups them by reason, and prints a table with samples per reason. + +With --run, every dropped command whose task is known is also executed in the +bench image (Linux, the tools the lanes had): once on the task's pristine +tree (tasks*.py setup) and once on the lane's saved finished tree, which is +then graded. A check that FAILS on the pristine tree, PASSES on a finished +tree the grader accepted, and is not one of the rules that is right whatever +the outcome (it changes the work, it cannot fail), told the work from none: +dropping it threw away a good check. That count is the automatic part of +the false-drop estimate; the rest is reading the samples. + +With --lint DIST, every dropped command is also re-linted by the built +dist/checklint.js (and dist/criteria.js strayPath) of a checkout, to show +which drops a given build would still make ("after" counts). + +Usage: + python3 bench/local/drop_audit.py [--since 2026-10-07] [--samples 3] + [--run [--image maat-bench:TAG]] [--lint ~/Documents/molt-wt-drops/dist] + [--json out.json] +""" +from __future__ import annotations + +import argparse +import collections +import datetime as dt +import json +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +CW = Path.home() / ".cache/maat-bench/container-work" + +DRAFTED = re.compile(r"^maat: dropped drafted criterion (?P\S+) \((?P.*)\) — (?P.*)$", re.S) +REVIEW = re.compile( + r"^maat: criteria review — dropped (?P\S+) \((?P.*)\) (?Pbefore sealing|from the redraft|from the cover draft): " + r"(?P.*?)(?: \[(?P[^\]]+)\])?$", + re.S, +) +CRITIC_RUN = re.compile( + r"^maat: criteria review — dropped (?P\S+) \((?P.*)\): it (?P(?:guesses an answer the task does not give|demands what the task does not state) — .*)$", re.S +) +CRITIC = re.compile(r"^maat: criteria review — dropped (?P\S+) from the (?Predraft|draft) — (?P.*)$", re.S) +START = re.compile(r"^maat: (?:dropped drafted criterion |criteria review — dropped )") +RUN_FILE = re.compile(r"^(?P.+?)-(?Pmolt)-(?P\d+)(?:-(?P[\w.-]+))?\.err$") + +# What a trial (preflightCriteria) said, as a reason key. +TRIAL_KEYS = [ + (re.compile(r"an absolute path outside the project"), "trial:L11-abs"), + (re.compile(r"the command was not found"), "trial:not-found"), + (re.compile(r"could not parse it"), "trial:unparsed"), + (re.compile(r"git outside a git repository"), "trial:git-norepo"), + (re.compile(r"reads a file from HEAD"), "trial:git-head-missing"), + (re.compile(r"git rejected the check"), "trial:git-usage"), + (re.compile(r"own program has a bug"), "trial:own-bug"), + (re.compile(r"rejected the check's options"), "trial:bad-options"), + (re.compile(r"rejected the check's date"), "trial:bad-date"), + (re.compile(r"misuses a library call"), "trial:lib-misuse"), +] + +# Rules whose drop is right whatever the check would have reported: a check +# that changes the work, or that exits 0 whatever happened, is unsafe even +# when it happens to tell the work from none on one tree. Their false drops +# are mis-fires of the rule itself, which only reading the command shows. +OUTCOME_INDEPENDENT = ("L15-mutates", "L16-cannot-fail", "L9-swallows-exit", "L9-pipe-no-pipefail") + + +def reason_of(rec: dict) -> str: + if rec["src"] == "trial": + for rx, key in TRIAL_KEYS: + if rx.search(rec["why"]): + return key + return "trial:" + re.sub(r"\W+", "-", rec["why"].lower())[:40].strip("-") + if rec["src"] == "critic": + if rec["why"].startswith("guesses"): + return "critic:guesses" + if rec["why"].startswith("demands"): + return "critic:invents" + return "critic:off-task" + rule = rec.get("rule") or "unlabelled" + return re.sub(r":.*$", "", rule) + + +def collect(since: dt.date, root: Path = CW, until: dt.datetime | None = None) -> list[dict]: + recs: list[dict] = [] + cutoff = dt.datetime.combine(since, dt.time()).timestamp() + stop = until.timestamp() if until else float("inf") + for err in sorted(root.glob("*/*.err")): + if not cutoff <= err.stat().st_mtime <= stop: + continue + m = RUN_FILE.match(err.name) + task = m["task"] if m else err.stem + cur: list[str] | None = None + blocks: list[str] = [] + for line in err.read_text(errors="replace").splitlines(): + if line.startswith("maat:"): + if cur is not None: + blocks.append("\n".join(cur)) + cur = [line] if START.match(line) else None + elif cur is not None: + cur.append(line) + if cur is not None: + blocks.append("\n".join(cur)) + for b in blocks: + base = {"lane": err.parent.name, "task": task, "err": str(err), "tree": str(err.with_suffix(""))} + if mm := DRAFTED.match(b): + rec = {**base, "src": "trial", "where": "trial", "name": mm["name"], "run": mm["run"], "why": mm["why"].strip(), "rule": None} + elif mm := REVIEW.match(b): + rec = {**base, "src": "lint", "where": mm["where"], "name": mm["name"], "run": mm["run"], "why": mm["why"].strip(), "rule": mm["rule"]} + elif mm := CRITIC_RUN.match(b): + rec = {**base, "src": "critic", "where": "critic", "name": mm["name"], "run": mm["run"], "why": mm["why"].strip(), "rule": None} + elif mm := CRITIC.match(b): + rec = {**base, "src": "critic", "where": "critic", "name": mm["name"], "run": None, "why": mm["why"].strip(), "rule": None} + else: + rec = {**base, "src": "unparsed", "where": "?", "name": "?", "run": None, "why": b[:200], "rule": None} + rec["reason"] = reason_of(rec) + recs.append(rec) + return recs + + +# ---------------------------------------------------------------- running the commands + +RUNNER = r''' +import json, os, re, shutil, signal, subprocess, sys, tempfile +from pathlib import Path +sys.path.insert(0, "/bench") +from tasks import TASKS +from tasks2 import TASKS2 +try: + from tasks3 import TASKS3 +except Exception: + TASKS3 = [] +BY = {T.name: T for T in TASKS + TASKS2 + TASKS3} +items = json.load(open("/io/in.json")) + +def run(tree, cmd): + tmp = Path(tempfile.mkdtemp()) + w = tmp / "w" + shutil.copytree(tree, w, symlinks=True, ignore=shutil.ignore_patterns(".maat", ".molt")) + try: + p = subprocess.Popen(["bash", "-c", cmd], cwd=w, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + start_new_session=True, stdin=subprocess.DEVNULL) + try: + out, err = p.communicate(timeout=20) + code = p.returncode + except subprocess.TimeoutExpired: + os.killpg(p.pid, signal.SIGKILL); p.communicate(); code, out, err = None, "", "timeout" + finally: + try: os.killpg(p.pid, signal.SIGKILL) + except ProcessLookupError: pass + return {"code": code, "tail": (out + err)[-300:]} + finally: + shutil.rmtree(tmp, ignore_errors=True) + +pristine = {} +graded = {} +out = [] +for it in items: + T = BY.get(it["task"]) + r = {"i": it["i"]} + if T is None: + r["skip"] = "task unknown"; out.append(r); continue + if it["task"] not in pristine: + d = Path(tempfile.mkdtemp()) / "w"; d.mkdir() + T.setup(d); pristine[it["task"]] = d + r["pre"] = run(pristine[it["task"]], it["run"]) + post = Path("/cw") / it["tree"] + if post.is_dir(): + if it["tree"] not in graded: + tmp = Path(tempfile.mkdtemp()) / "w" + shutil.copytree(post, tmp, symlinks=True, ignore=shutil.ignore_patterns(".maat", ".molt")) + try: + ok, why = T.grade(tmp) + except Exception as e: + ok, why = False, f"grader raised {e!r}" + graded[it["tree"]] = (bool(ok), str(why)[:120]) + shutil.rmtree(tmp.parent, ignore_errors=True) + r["post"] = run(post, it["run"]) + r["graded"] = graded[it["tree"]] + out.append(r) +json.dump(out, open("/io/out.json", "w")) +''' + + +def run_all(recs: list[dict], image: str) -> None: + todo = [{"i": i, "task": r["task"], "run": r["run"], "tree": str(Path(r["tree"]).relative_to(CW))} for i, r in enumerate(recs) if r["run"]] + with tempfile.TemporaryDirectory() as io: + Path(io, "in.json").write_text(json.dumps(todo)) + Path(io, "runner.py").write_text(RUNNER) + subprocess.run( + ["docker", "run", "--rm", "--network=none", "-v", f"{HERE}:/bench:ro", "-v", f"{CW}:/cw:ro", "-v", f"{io}:/io", + "--entrypoint", "python3", image, "/io/runner.py"], + check=True, timeout=3600, + ) + for r in json.loads(Path(io, "out.json").read_text()): + recs[r["i"]]["exec"] = r + + +def verdict(rec: dict) -> str: + """good: failed on the pristine tree, passed on a finished tree the grader accepted.""" + ex = rec.get("exec") + if not ex or "skip" in ex or "post" not in ex: + return "n/a" + pre, post, (ok, _) = ex["pre"]["code"], ex["post"]["code"], ex["graded"] + if pre == 0: + return "passes-before" + if post == 0 and ok: + return "good" + if post == 0 and not ok: + return "passes-wrong-work" + if ok: + return "fails-correct-work" + return "fails-both" + + +# ---------------------------------------------------------------- re-linting with a build + +LINT_JS = r''' +import { cannotFail } from "%(dist)s/checklint.js"; +import { checkMutates } from "%(dist)s/checkwrites.js"; +import * as criteria from "%(dist)s/criteria.js"; +let s = ""; process.stdin.on("data", (d) => (s += d)).on("end", () => { + const items = JSON.parse(s); + const out = items.map((it) => ({ + // What the seal-time review drops by default (criteria.ts lintSplit): L15, then L16. + L15: checkMutates(it.run), + L16: cannotFail(it.run), + // What the seal-time trial drops without running (cli.tsx sealDraft → preflightCriteria stray). + stray: criteria.strayPath(it.run, { cwd: "/nonexistent-project-root", task: it.task }), + // A "not found" that is the deliverable missing (absent in older builds). + missing: criteria.missingDeliverable ? criteria.missingDeliverable(it.stderr ?? "", "/nonexistent-project-root") : null, + // P1 drops this build seals as refuse-only guards (criteria.ts guardsFrom), before the per-draft cap. + guard: criteria.pinsCurrentValue ? !criteria.pinsCurrentValue(it.run) : false, + })); + console.log(JSON.stringify(out)); +}); +''' + + +def relint(recs: list[dict], dist: Path) -> None: + todo = [r for r in recs if r["run"]] + task_text = {} + try: + sys.path.insert(0, str(HERE)) + from tasks import TASKS # noqa: E402 + from tasks2 import TASKS2 # noqa: E402 + from tasks3 import TASKS3 # noqa: E402 + task_text = {T.name: getattr(T, "PROMPT", "") for T in TASKS + TASKS2 + TASKS3} + except Exception: + pass + items = [{"run": r["run"], "task": task_text.get(r["task"], ""), "stderr": ((r.get("exec") or {}).get("pre") or {}).get("tail", "")} for r in todo] + p = subprocess.run(["node", "--input-type=module", "-e", LINT_JS % {"dist": str(dist.resolve())}], input=json.dumps(items), + capture_output=True, text=True, check=True) + for r, o in zip(todo, json.loads(p.stdout)): + r["relint"] = o + + +def still_dropped(rec: dict) -> bool | None: + """Would this build still drop the command for the same reason? None when a re-lint cannot say (a run's outcome decides).""" + o = rec.get("relint") + if o is None: + return None + reason = rec["reason"] + if reason in ("trial:L11-abs", "L11-abs"): + return bool(o["stray"]) + if reason == "L15-mutates": + return bool(o["L15"]) + if reason == "L16-cannot-fail": + return bool(o["L16"]) + if reason == "trial:not-found": + # Needs the run's stderr (--run); a missing deliverable is now a failure before the work. + return None if not rec.get("exec") else not o["missing"] + return None + + +# ---------------------------------------------------------------- report + +def short(s: str, n: int = 150) -> str: + s = " ".join(s.split()) + return s if len(s) <= n else s[: n - 1] + "…" + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--since", default=dt.date.today().isoformat()) + ap.add_argument("--until", default=None, help="only .err files last written before this local time (2026-10-07T18:20), so a re-run reads the same set while lanes are still writing") + ap.add_argument("--samples", type=int, default=3) + ap.add_argument("--run", action="store_true") + ap.add_argument("--image", default=None) + ap.add_argument("--lint", type=Path, default=None, help="a dist/ directory to re-lint the drops with") + ap.add_argument("--json", type=Path, default=None) + ap.add_argument("--reason", default=None, help="print every record of this reason") + ap.add_argument("--reuse", type=Path, default=None, help="take --run results from an earlier --json instead of running again") + a = ap.parse_args(argv) + recs = collect(dt.date.fromisoformat(a.since), until=dt.datetime.fromisoformat(a.until) if a.until else None) + if a.reuse: + saved = {(r["err"], r["name"], r["run"], r["where"]): r.get("exec") for r in json.loads(a.reuse.read_text())} + for r in recs: + if saved.get((r["err"], r["name"], r["run"], r["where"])): + r["exec"] = saved[(r["err"], r["name"], r["run"], r["where"])] + if a.run: + image = a.image or subprocess.run(["docker", "images", "--format", "{{.Repository}}:{{.Tag}}", "maat-bench"], + capture_output=True, text=True, check=True).stdout.split()[0] + print(f"running {sum(1 for r in recs if r['run'])} commands in {image}", file=sys.stderr) + run_all(recs, image) + if a.lint: + relint(recs, a.lint) + by = collections.defaultdict(list) + for r in recs: + by[r["reason"]].append(r) + lanes = len({r["lane"] for r in recs}) + print(f"{len(recs)} drop lines in {lanes} lanes since {a.since}{f' until {a.until}' if a.until else ''} ({len({(r['lane'], r['task'], r['name'], r['run']) for r in recs})} distinct)\n") + hdr = f"{'reason':28} {'lines':>5} {'uniq':>5}" + if a.run or a.reuse: + hdr += f" {'good':>5} {'pass-bef':>8} {'n/a':>4}" + if a.lint: + hdr += f" {'still':>5} {'guard':>5}" + print(hdr) + for reason, rs in sorted(by.items(), key=lambda kv: -len(kv[1])): + uniq = {(r["lane"], r["task"], r["name"], r["run"]) for r in rs} + line = f"{reason:28} {len(rs):5} {len(uniq):5}" + if a.run or a.reuse: + vs = collections.Counter(verdict(r) for r in rs) + line += f" {vs['good']:5} {vs['passes-before']:8} {vs['n/a']:4}" + if a.lint: + st = [still_dropped(r) for r in rs] + line += f" {sum(1 for s in st if s):5}" if any(s is not None for s in st) else f" {'-':>5}" + g = sum(1 for r in rs if r["reason"] == "P1-passes-before-work" and (r.get("relint") or {}).get("guard")) + line += f" {g:5}" if reason == "P1-passes-before-work" else f" {'-':>5}" + print(line) + print() + for reason, rs in sorted(by.items(), key=lambda kv: -len(kv[1])): + pick = rs if a.reason == reason else rs[: a.samples] + print(f"== {reason} ({len(rs)})") + for r in pick: + v = f" [{verdict(r)}]" if (a.run or a.reuse) else "" + print(f" {r['lane']}/{r['task']} {r['name']}{v}: {short(r['run'] or '')}") + print(f" → {short(r['why'], 160)}") + if a.json: + a.json.write_text(json.dumps(recs, indent=1)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From e1c4668d717c89e6d63ce3aaec5e153f6145a8ae Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 18:43:45 -0700 Subject: [PATCH 2/2] Stop dropping good drafted checks; keep pre-work passes as guards Measured over the 2026-10-07 lanes with bench/local/drop_audit.py: - The absolute-path rule read the raw command, so a sed program's /g and a value glued onto sys.argv[1] + "/old.py" were paths (all 4 of its drops that day). src/shellwords.ts parses words, quotes, substitutions, redirects and heredocs, and absolutePathArgs returns only what programs open: operands, redirect targets, assignments, --opt= values, and string literals of inline programs that are a path on their own. sed and awk programs, grep patterns, jq filters, printf/echo data, find patterns and test's compared strings are skipped. - A trial of ./rotate.sh before rotate.sh exists exits 127, and was dropped as 'the command was not found' (6 checks). A project path the shell could not open that is absent from the tree is the deliverable missing: it counts as failing before the work. A tool missing from PATH is still broken. - L15 read touch -d's DATE as a file the check creates, lost track of f="$d/x", and of > "$(mktemp -d)/out" (13 drops). - L16 read '...|| exit 1; done; exit 0' and 'cond && exit 1 || exit 0' as checks that cannot fail (4 drops). - A check naming the project by its absolute path is pointed at the copy it is tried on before the work, and at the bar's throwaway copy. Checks the pre-work screen drops for passing before the work (P1) are sealed as refuse-only guards (tag guard): they run at the bar and can refuse, tierOf never counts them, receipts and the CLI label them. A check that pins today's output of the project's program on its own data is not kept: 13 of the 19 P1 drops that failed an accepted finished tree were of that shape. --- CHANGELOG.md | 27 ++ README.md | 4 +- bench/harbor/README.md | 3 +- src/bar.ts | 7 +- src/checkwrites.ts | 31 +- src/cli.tsx | 2 +- src/criteria.ts | 160 ++++++++-- src/evidence.ts | 29 +- src/receipts.ts | 6 +- src/scratch.ts | 28 ++ src/shellwords.ts | 463 +++++++++++++++++++++++++++++ src/tiers.ts | 4 +- test/discriminating-checks.test.ts | 37 ++- test/false-drops.test.ts | 352 ++++++++++++++++++++++ 14 files changed, 1117 insertions(+), 36 deletions(-) create mode 100644 src/shellwords.ts create mode 100644 test/false-drops.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 15eb5de..1089a35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,33 @@ a copy that changed during the work, or a check that names the project's absolute path, counts as not tried. The copy's size and timing are journalled (`pre-work-copy`). +- **Checks that pass before the work are kept as refuse-only guards.** A + drafted check the pre-work screen drops (P1) is still sealed, tagged + `guard`: it runs at the bar and a failure refuses the claim, but it never + counts toward "verified", and receipts label it "refuse-only guard". Up to + four, beside (not inside) the four-check cap. Not kept: a duplicate of a + sealed check, and one that pins today's output of the project's program on + the project's own data (`node summarize.js transactions.json | jq -e + '.zoe == "-3.50"'`, `sqlite3 app.db 'PRAGMA user_version' | grep -q '^1$'`), + which on the 2026-10-07 lanes failed 13 finished trees the grader accepted. +- **Fewer good drafted checks dropped before sealing** (measured with + `bench/local/drop_audit.py` over the 2026-10-07 lanes): + - the absolute-path rule reads what each program opens (a shell parser, + `src/shellwords.ts`), not the raw text: a sed program's `/g`, an awk + program, a grep pattern, a jq filter, printf's data and `sys.argv[1] + + "/old.py"` are not paths. Its four drops that day were all wrong. + - `./rotate.sh …` before rotate.sh exists is the deliverable missing: + tried before the work it now counts as failing there (it discriminates), + not as a command that could not run. A tool missing from PATH is still + broken. + - the mutation rule (L15) skips the value of `touch -d/-t/-r` and + `mkdir -m`, follows variables built on a mktemp directory + (`f="$d/x"; echo > "$f"`), and lets `> "$(mktemp -d)/out"` through. + - the cannot-fail rule (L16) no longer flags `…; exit 0` after an earlier + `exit 1` in the same shell, or `cond && exit 1 || exit 0`. + - a check that names the project by its absolute path is tried on the + pre-work copy, and run in the bar's throwaway copy, instead of reaching + the live folder. - **Claims say who stood behind them** on the terminal, the window, job_end's `claim`, the receipt index and the receipt: `verified (independent checks: )`, `verified (your checks)`, `passed own checks (), not diff --git a/README.md b/README.md index e209408..bd89378 100644 --- a/README.md +++ b/README.md @@ -413,7 +413,9 @@ to them. Passing checks the worker model drafted for itself are never the best a run earns is `passed own checks (), not verified`, exit 3. Drafted checks are tried on a copy of the project before the work: one that already passes there, or cannot fail by construction, is sent back to the -drafter once and dropped if the redraft is no better. `--require-discriminating` +drafter once and dropped if the redraft is no better; one that already +passed is still sealed as a refuse-only guard, which can fail the claim but +never makes it verified. `--require-discriminating` goes further: "verified" then needs an independent value check that failed before the work, and anything less is `passed checks that did not test this work, not verified` (exit 3). diff --git a/bench/harbor/README.md b/bench/harbor/README.md index 36eaa36..92b405e 100644 --- a/bench/harbor/README.md +++ b/bench/harbor/README.md @@ -15,7 +15,8 @@ passed-own-checks`, exit 3), never `verified`. Set `MAAT_JUDGE_MODEL` (and `MAAT_JUDGE_URL` / `MAAT_JUDGE_KEY`) in the container to another model to make "verified" reachable. Drafted checks that pass on the untouched task, or cannot fail by construction, are redrafted once and then dropped before they are -sealed. With `--require-discriminating` (or `MAAT_REQUIRE_DISCRIMINATING=1`) +sealed; the ones that only passed before the work are kept as refuse-only +guards (they can fail a run, never verify it). With `--require-discriminating` (or `MAAT_REQUIRE_DISCRIMINATING=1`) a run whose independent checks never failed before the work ends `passed-untested`, also exit 3. The exit code never fails a trial (see below). diff --git a/src/bar.ts b/src/bar.ts index 774e334..9c55805 100644 --- a/src/bar.ts +++ b/src/bar.ts @@ -2201,7 +2201,11 @@ export async function runCheck(check: Check, ctx: BarContext): Promise r.hidden)) lines.push(DISPUTE_HINT, ""); for (const r of failed) { lines.push(`--- FAILED: ${r.name} (${r.hidden ? "command withheld" : r.detail})`); + if (r.tags?.includes("guard")) lines.push("a guard: this held on the project before the work, and it does not now."); if (r.exitCode !== undefined) lines.push(`exit code: ${r.exitCode}`); lines.push(r.output.trim() || "(no output)"); lines.push(""); diff --git a/src/checkwrites.ts b/src/checkwrites.ts index 7dc8bb2..04c388c 100644 --- a/src/checkwrites.ts +++ b/src/checkwrites.ts @@ -91,6 +91,20 @@ export function maskQuoted(cmd: string): string { function tempVars(cmd: string): Set { const out = new Set(["TMPDIR", "TMP", "TEMP", "TEMPDIR"]); for (const m of cmd.matchAll(/\b([A-Za-z_]\w*)=["']?(?:\$\(|`)\s*mktemp\b/g)) out.add(m[1]!); + // And what is built on one: `f="$d/f $i.log"` names a file in the mktemp + // directory, so `echo x > "$f"` writes there. Followed to a fixed point: + // `g=$f.bak` is scratch once f is. + const assigns = [...cmd.matchAll(/(?; depth: n if (!tm) continue; const target = seg.raw.slice(start, start + tm[0].length); for (let k = r.index ?? 0; k < start + tm[0].length; k++) consumed.add(k); - if (!isScratchPath(target, temps) && !inScratch[seg.at + start]) { + // `> "$(mktemp -d)/out"`: the segment stops at the `$(`, so read the target from the whole command. + const intoTemp = /^["']?(?:\$\(|`)\s*mktemp\b/.test(cmd.slice(seg.at + start)); + if (!intoTemp && !isScratchPath(target, temps) && !inScratch[seg.at + start]) { return `it writes to ${unquote(target)} with a redirect; a check only reads (write to a mktemp directory instead)`; } } @@ -344,9 +360,18 @@ export function checkMutates(cmd: string, outer?: { temps: Set; depth: n } case "touch": case "mkdir": - case "mkfifo": - if (!allScratch(operands)) return `it creates ${quoteOne(operands.find((o) => !scratch(o)))} (\`${name}\`) in the project; a check only reads`; + case "mkfifo": { + // `touch -d "2020-01-01" f`, `touch -t 2001010000 f`, `touch -r ref f`, `mkdir -m 700 d`: + // the option's value is not a file it creates. + const valued = name === "touch" ? /^(-[dtr]|--date|--reference)$/ : /^(-m|--mode|-Z|--context)$/; + const files: Word[] = []; + for (let k = 0; k < args.length; k++) { + if (valued.test(args[k]!.text)) k++; + else if (!isFlag(args[k]!.text)) files.push(args[k]!); + } + if (!allScratch(files)) return `it creates ${quoteOne(files.find((o) => !scratch(o)))} (\`${name}\`) in the project; a check only reads`; break; + } case "chmod": case "chown": case "chgrp": diff --git a/src/cli.tsx b/src/cli.tsx index 8be2bca..fc96fc8 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -1388,7 +1388,7 @@ async function sealDraft(draft: Draft, args: Args, late = false): Promise !drop.has(c.name)); if (!args.json) { // Names only: the commands are printed when the job releases them (checks_released). - for (const c of taskChecks) process.stdout.write(`· criterion ${c.name} (command withheld until the job ends)\n`); + for (const c of taskChecks) process.stdout.write(`· ${c.tags.includes("guard") ? "guard" : "criterion"} ${c.name} (command withheld until the job ends)\n`); for (const n of sealed.taskNotes) process.stdout.write(`· note ${n}\n`); } return { taskChecks, taskNotes: sealed.taskNotes, ...(sealed.requirements ? { requirements: sealed.requirements } : {}) }; diff --git a/src/criteria.ts b/src/criteria.ts index 8c68ca3..e6e4540 100644 --- a/src/criteria.ts +++ b/src/criteria.ts @@ -32,7 +32,8 @@ import { askModel, type AskOptions } from "./ask.js"; import { runCommand, draftedShell, bashPath } from "./run.js"; import { cannotFail, lintAll, readTree, type LintCtx, type Tree } from "./checklint.js"; import { checkMutates } from "./checkwrites.js"; -import { copyTree } from "./scratch.js"; +import { copyTree, replacePathPrefix } from "./scratch.js"; +import { absolutePathArgs } from "./shellwords.js"; import { diagnoseFailure } from "./bar.js"; import { reportsFailure } from "./evidence.js"; import { normalizeRequirements } from "./signout.js"; @@ -43,8 +44,13 @@ import type { CheckAuthor } from "./types.js"; * `surface`: the critic read it as only looking (a file exists, a word * appears, it compiles) rather than running the deliverable. Kept, but such a * check alone cannot carry a "verified" once the ones that ran it are retired. + * + * `guard`: a check that passed on the tree before the work (P1). It cannot + * show the task was done, so it never counts toward "verified"; it is sealed + * anyway, as a refuse-only guard, because what it asserts (no conflict + * markers, a clean tree, the suite still passing) must still hold after. */ -export type DraftedCheck = { name: string; run: string; surface?: true }; +export type DraftedCheck = { name: string; run: string; surface?: true; guard?: true }; export type Draft = { checks: DraftedCheck[]; notes: string[]; @@ -58,6 +64,8 @@ export type Draft = { /** Same bounds the drafter uses — applied again at `session:run`. */ export const CRITERIA_MAX_CHECKS = 4; +/** Refuse-only guards sealed beside the checks, at most (they are not counted in CRITERIA_MAX_CHECKS). */ +export const CRITERIA_MAX_GUARDS = 4; export const CRITERIA_MAX_NOTES = 3; export const CRITERIA_MAX_NAME = 40; /** @@ -95,13 +103,18 @@ export function sanitizeCriteria(raw: unknown): Draft { // check does not use up a slot: see CRITERIA_MAX_RUN. (c as DraftedCheck).run.trim().length <= CRITERIA_MAX_RUN, ) - .slice(0, CRITERIA_MAX_CHECKS) .map((c) => ({ name: c.name.trim().slice(0, CRITERIA_MAX_NAME), run: c.run.trim(), ...(c.surface === true ? { surface: true as const } : {}), + ...(c.guard === true ? { guard: true as const } : {}), })) : []; + // Guards have their own cap: they can only refuse, so they never crowd out a check that can verify. + const capped = [ + ...checks.filter((c) => !c.guard).slice(0, CRITERIA_MAX_CHECKS), + ...checks.filter((c) => c.guard).slice(0, CRITERIA_MAX_GUARDS), + ]; const notes: string[] = Array.isArray(o.notes) ? o.notes .filter((n): n is string => typeof n === "string" && n.trim().length > 0) @@ -109,7 +122,7 @@ export function sanitizeCriteria(raw: unknown): Draft { .map((n) => n.trim().slice(0, CRITERIA_MAX_NOTE)) : []; const requirements = normalizeRequirements(Array.isArray(o.requirements) ? o.requirements : undefined); - return { checks, notes, ...(requirements.length ? { requirements } : {}) }; + return { checks: capped, notes, ...(requirements.length ? { requirements } : {}) }; } /** @@ -149,8 +162,9 @@ export function taskChecksFrom( run: c.run, timeoutMs: 120_000, expectExit: 0, - // surface from the critic, value from the command itself (evidence.ts). - tags: evidenceTags(c.run, c.surface), + // surface from the critic, value from the command itself (evidence.ts); + // a guard is marked so tierOf never counts it toward "verified". + tags: [...evidenceTags(c.run, c.surface), ...(c.guard ? ["guard"] : [])], ...(opts.hidden ? { hidden: true } : {}), ...(opts.author ? { author: { ...opts.author } } : opts.hidden ? {} : { author: { kind: "person" as const } }), })), @@ -264,10 +278,15 @@ function under(p: string, root: string): boolean { * * Seen in real runs: `python3 /wc.py` and `/home/user/data.csv` in checks * drafted without any view of the project. They fail for ever, whatever the - * work does, and refuse correct work. Paths with a trailing slash are skipped - * (`awk '/x/'`, `sed 's/a/b/'` delimiters), as is anything glued to a word, - * `$VAR`, `./`, or `://` — those are relative paths, expansions and URLs — - * and so are closing tags (``) and cron steps. A deliberately missing path + * work does, and refuse correct work. + * + * Only what a program will open is read (src/shellwords.ts absolutePathArgs): + * operands, redirect targets, assignments, and the string literals of an + * inline program that are a path on their own. A sed or awk program, a grep + * pattern, a jq filter, printf's data and a value glued onto `$d` or + * `sys.argv[1] +` are not paths: on 2026-10-07 a regex over the raw command + * dropped three good differential checks for the `/g` of `sed -E 's/ +/ /g'`. + * A path with a trailing slash is skipped, and a deliberately missing path * (`/nonexistent`) tests error handling and is allowed. */ export function strayPath(run: string, opts: { cwd: string; task?: string }): string | null { @@ -278,8 +297,10 @@ export function strayPath(run: string, opts: { cwd: string; task?: string }): st /* a cwd that is not there: compare as given */ } const roots = [opts.cwd, real, ...TEMP_PATHS, tmpdir(), ...SYSTEM_PATHS]; - for (const m of run.matchAll(/(?()*?\[\]{}$,=\\]*)/g)) { - const p = m[1]!; + for (const arg of absolutePathArgs(run)) { + // A glob or a variable tail is cut off: `/srv/out/*.txt` is about /srv/out. + const p = arg.replace(/[*?[{$,].*$/s, ""); + if (!/^\/[\w.@+-]/.test(p)) continue; if (p.endsWith("/")) continue; // A path built to not exist is an error-handling test, not an invention. if (/nonexist|no[-_]such|does[-_]?not[-_]?exist|missing/i.test(p)) continue; @@ -290,6 +311,30 @@ export function strayPath(run: string, opts: { cwd: string; task?: string }): st return null; } +/** + * The project file a shell could not run because it is not there yet, or + * null. Only a path the shell was asked to open (`./rotate.sh`, `bin/tool`, + * `bash backup.sh`) and that is absent from the tree counts: a bare command + * name the shell looked up on PATH (`pytest: command not found`) is a tool + * the machine lacks, and stays broken. + */ +export function missingDeliverable(stderr: string, cwd: string): string | null { + const said = + /^(?:\S*sh|bash|dash|zsh)(?::\s*line \d+)?:\s*(?:\d+:\s*)?([^\s:]+): (?:No such file or directory|not found)\s*$/m.exec(stderr) ?? + /^\S*sh: \d+: (?:cannot open|Can't open) ([^\s:]+)/m.exec(stderr); + if (!said) return null; + const p = said[1]!; + if (isAbsolute(p) || p.startsWith("~")) return null; + // PATH lookups name no directory; bash's "No such file" for `bash x.sh` names a file. + if (!p.includes("/") && !/No such file or directory|cannot open|Can't open/.test(said[0])) return null; + try { + statSync(join(cwd, p)); + return null; + } catch { + return p; + } +} + export async function preflightCriteria( checks: readonly { name: string; kind?: string; run?: string; expectExit?: number; hidden?: boolean; tags?: readonly string[] }[], opts: { @@ -325,9 +370,24 @@ export async function preflightCriteria( * run. The task text is where a path the person stated is allowed from. */ stray?: { task: string }; + /** + * The project's own absolute path, when `cwd` is a copy of it (the + * pre-work copy the screen tries on). A check that names the project + * (`cd /app && …`) is tried on the copy it runs in, not on the live + * folder the worker may already be changing. Defaults to `cwd`. + */ + root?: string; }, ): Promise { const broken: BrokenCriterion[] = []; + const roots = (() => { + const r = opts.root ?? opts.cwd; + try { + return [...new Set([r, realpathSync(r), opts.cwd, realpathSync(opts.cwd)])]; + } catch { + return [...new Set([r, opts.cwd])]; + } + })(); for (const c of checks) { if (c.kind && c.kind !== "command") continue; if (!c.run) continue; @@ -340,9 +400,14 @@ export async function preflightCriteria( // that checks out a branch or deletes a file would change the folder the // work starts from. const copy = process.env.MAAT_CHECK_COPY === "0" ? null : await copyTree(opts.cwd); + // Where the check will really run: the copy, under the project's layout, + // `.git` included. A check that names the project by its absolute path is + // pointed at the copy too, as the bar points it (src/bar.ts). + const where = copy?.dir ?? opts.cwd; + const run = roots.reduce((t, r) => replacePathPrefix(t, r, where), c.run); try { - const r = await runCommand(c.run, { - cwd: copy?.dir ?? opts.cwd, + const r = await runCommand(run, { + cwd: where, // Under the shell the bar will run this check with (src/run.ts // draftedShell): bash for a drafted check, sh otherwise. Tried under // one shell and judged under another, a bashism (`[[ ]]`, `==` in @@ -363,7 +428,14 @@ export async function preflightCriteria( // satisfy a command that cannot be satisfied. const unparsed = SHELL_PARSE_ERROR.test(r.stderr); const selfError = r.code !== (c.expectExit ?? 0) ? checkSelfError(`${r.stdout}\n${r.stderr}`) : null; - if (d.didNotRun) broken.push({ name: c.name, run: c.run, why: d.hint ?? "did not run" }); + const absent = d.didNotRun ? missingDeliverable(r.stderr, where) : null; + if (absent) { + // `./rotate.sh` before rotate.sh exists: the shell's "not found" is + // the deliverable missing, which is what a check is meant to find + // before the work. On 2026-10-07 six checks of a script the task + // asked for were dropped as "the command was not found". + opts.failed?.push(c.name); + } else if (d.didNotRun) broken.push({ name: c.name, run: c.run, why: d.hint ?? "did not run" }); else if (unparsed) { broken.push({ name: c.name, run: c.run, why: `the shell could not parse it: ${firstLine(copy ? copy.unmap(r.stderr) : r.stderr)}` }); } else if (selfError) broken.push({ name: c.name, run: c.run, why: selfError }); @@ -970,12 +1042,66 @@ export async function draftCriteriaCritiqued( { ...opts, snapshot, askCwd: opts.askCwd ?? ownDir, onProgress: opts.onProgress && ((d) => opts.onProgress!(withReqs(d))) }, sink, ); - return r.ok ? { ...r, draft: withReqs(r.draft), ...(sink.lint.length ? { lint: sink.lint } : {}) } : r; + if (!r.ok) return r; + const kept = guardsFrom(sink.lint, r.draft.checks); + const draft = kept.length ? { ...r.draft, checks: [...r.draft.checks, ...kept] } : r.draft; + const critique = kept.length ? [...r.critique, `kept as refuse-only guards (they passed before the work, so they can refuse the claim but never verify it): ${kept.map((g) => g.name).join(", ")}`] : r.critique; + return { ...r, draft: withReqs(draft), critique, ...(sink.lint.length ? { lint: sink.lint } : {}) }; } finally { if (ownDir) rmSync(ownDir, { recursive: true, force: true }); } } +/** + * Does this check pin a value the project's own program prints for the + * project's own data, today? Such a check passing before the work says the + * literal is the CURRENT output, and the task is usually to change it: as a + * guard it would refuse the correct fix. Replayed over the P1 drops of the + * 2026-10-07 lanes (bench/local/drop_audit.py), 13 of the 19 that failed on a + * finished tree the grader accepted were of this shape + * (`node summarize.js transactions.json | jq -e '.zoe == "-3.50"'`, + * `sqlite3 app.db 'PRAGMA user_version' | grep -q '^1$'`, `test -f cache.py` + * in a revert), against 7 of 147 that held. A literal 0 or an empty answer + * ("no duplicates", "nothing changed") is an invariant, not a pin, and an + * input the check makes itself (echo, printf, a heredoc, mktemp) is not the + * project's data. + */ +export function pinsCurrentValue(run: string): boolean { + const literal = + /==\s*(?:["'][^"']+["']|-?[1-9][\d.]*|-?0\.\d|\[)/.test(run) || + /\bgrep\s+(?:-\w+\s+)*-\w*[qx]\w*\s+['"]\^(?!0\$)[^'"|]+\$['"]/.test(run) || + /(?:\[|\btest)\s+"?\$\(.*\)"?\s*=\s*["'][^$]/.test(run) || + /\btest\s+-[fe]\s+[\w./-]+\s*(?:&&\s*echo\b.*)?$/.test(run.trim()); + const ownInput = /\b(?:echo|printf)\b[^|]*\|\s*\S|<<|mktemp|\/dev\/null/.test(run); + return literal && !ownInput; +} + +/** + * The checks the pre-work screen dropped for passing before the work (P1), + * kept as refuse-only guards: sealed, run at the bar, able to refuse, never + * counted toward "verified" (tags `guard`, src/tiers.ts). On 2026-10-07 the + * fix-git judge's `! grep -qE '^(<<<<<<<|>>>>>>>)' about.md index.md` and + * "on master with a clean tree" were dropped this way: right for the word + * "verified", but a merge that leaves conflict markers should still be + * refused. Not kept: a check that pins today's output (pinsCurrentValue), one + * that duplicates a sealed check, and any beyond CRITERIA_MAX_GUARDS. + */ +export function guardsFrom(dropped: readonly LintDrop[], sealed: readonly DraftedCheck[]): DraftedCheck[] { + const runs = new Set(sealed.map((c) => c.run)); + const names = new Set(sealed.map((c) => c.name)); + const out: DraftedCheck[] = []; + for (const d of dropped) { + if (d.rule !== "P1-passes-before-work" || runs.has(d.run) || pinsCurrentValue(d.run)) continue; + runs.add(d.run); + let name = d.name; + for (let k = 2; names.has(name); k++) name = `${d.name}-${k}`; + names.add(name); + out.push({ name: name.slice(0, CRITERIA_MAX_NAME), run: d.run, guard: true }); + if (out.length >= CRITERIA_MAX_GUARDS) break; + } + return out; +} + /** Why a drafted check that passes on the pre-work copy is sent back: the drafter reads this. */ export const PASSES_BEFORE_WORK = "this check passes before any work, so it cannot show the task was done; make it fail on the current tree"; @@ -1058,7 +1184,7 @@ async function critiqued( await preflightCriteria( // As they will be sealed: hidden task checks, so under the bar's shell. l.ok.map((c) => ({ name: c.name, kind: "command", run: c.run, hidden: true, tags: ["task"] })), - { cwd: opts.preWorkDir, passed, printedFail }, + { cwd: opts.preWorkDir, passed, printedFail, ...(opts.cwd ? { root: opts.cwd } : {}) }, ); } catch { return l; diff --git a/src/evidence.ts b/src/evidence.ts index bfb17e0..6fc974f 100644 --- a/src/evidence.ts +++ b/src/evidence.ts @@ -133,8 +133,33 @@ export function readSummary(output: string): RunnerSummary | null { const SWALLOW_TAIL = /(?:\|\|\s*(?:true|:|exit\s+0)|;\s*(?:true|:|exit\s+0))\s*$/; export function swallowsExit(run: string): string | null { - const m = SWALLOW_TAIL.exec(run.trim()); - return m ? m[0].trim() : null; + const cmd = run.trim(); + const m = SWALLOW_TAIL.exec(cmd); + if (!m) return null; + // `cond && exit 1 || exit 0` is `! cond`: the `exit 1` ends the shell + // before the `||` is reached. A redact-secrets check of exactly this shape + // was dropped on 2026-10-07 as one that could not fail. + const steps = topLevelCommands(cmd); + const before = steps.at(-2); + if (steps.at(-1)?.op === "||" && before?.op === "&&" && /^exit\s+[1-9]\d*$/.test(before.text)) return null; + // `for f in …; do … || exit 1; done; exit 0`: an earlier `exit N` in this + // same shell can still end the check non-zero; the tail only names the + // success path. Not one inside `( … )`, `$( … )` or a quoted `sh -c`. + if (/^;/.test(m[0].trim()) && exitsEarlier(cmd.slice(0, m.index))) return null; + return m[0].trim(); +} + +/** An `exit N` (N > 0) at the top level of the shell, outside quotes, heredocs and subshells. */ +function exitsEarlier(head: string): boolean { + const masked = maskShell(head); + let depth = 0; + for (let i = 0; i < masked.length; i++) { + const c = masked[i]!; + if (c === "(") depth++; + else if (c === ")") depth = Math.max(0, depth - 1); + else if (depth === 0 && c === "e" && /^exit\s+[1-9]/.test(masked.slice(i, i + 8)) && !/[\w$-]/.test(masked[i - 1] ?? "")) return true; + } + return false; } /** A top-level pipe (`a | b`, not `a || b`), without pipefail in force. */ diff --git a/src/receipts.ts b/src/receipts.ts index 681bdd1..834b83b 100644 --- a/src/receipts.ts +++ b/src/receipts.ts @@ -484,7 +484,8 @@ export class Receipts { for (const c of task.checks) { const name = c.slice(0, c.indexOf(": ") >= 0 ? c.indexOf(": ") : c.length); const a = args.authors?.[name] ?? args.authors?.[`task:${name}`]; - asked.push(`- \`${c}\` — written by ${authorWords(a)}`); + const guard = args.result.results.some((r) => (r.name === name || r.name === `task:${name}`) && r.tags?.includes("guard")); + asked.push(`- \`${c}\` — written by ${authorWords(a)}${guard ? " · refuse-only guard: it can refuse this claim, never verify it" : ""}`); } asked.push(""); } @@ -642,6 +643,9 @@ export class Receipts { `check: ${r.name}`, `kind: ${r.kind}`, `written by: ${resultAuthorWords(r, args.authors?.[r.name])}`, + ...(r.tags?.includes("guard") + ? ["role: refuse-only guard (it passed before the work: it can refuse this claim, never count toward verified)"] + : []), ...(r.beforeWork ? [ `before the work: ${ diff --git a/src/scratch.ts b/src/scratch.ts index 834ec2d..db7e328 100644 --- a/src/scratch.ts +++ b/src/scratch.ts @@ -84,8 +84,31 @@ export type TreeCopy = { cleanup: () => Promise; /** Rewrite the copy's path back to the project's in text a check printed. */ unmap: (text: string) => string; + /** + * Rewrite the project's absolute path to the copy's in a command, so a + * check that names the project (`cd /app && …`, `/work/x/out.csv`) reads + * and writes the copy like every relative path in it does. + */ + map: (text: string) => string; }; +/** Replace `from` with `to` wherever `from` is a whole path or a path's leading part. */ +export function replacePathPrefix(text: string, from: string, to: string): string { + if (!from || from === to || from === "/") return text; + let out = ""; + let i = 0; + for (;;) { + const at = text.indexOf(from, i); + if (at < 0) return out + text.slice(i); + const next = text[at + from.length] ?? ""; + const prev = text[at - 1] ?? ""; + // `/work/p` inside `/work/p2` or `/x/work/p` is another path. + const whole = !/[\w.-]/.test(next) && !/[\w./-]/.test(prev); + out += text.slice(i, at) + (whole ? to : from); + i = at + from.length; + } +} + class TooBig extends Error {} /** @@ -319,6 +342,11 @@ export async function copyTreeOrWhy( for (const [from, to] of pairs) if (from !== to) out = out.split(from).join(to); return out; }, + map: (text) => { + let out = text; + for (const [to, from] of pairs) out = replacePathPrefix(out, from, to); + return out; + }, }; } diff --git a/src/shellwords.ts b/src/shellwords.ts new file mode 100644 index 0000000..5c3ecbb --- /dev/null +++ b/src/shellwords.ts @@ -0,0 +1,463 @@ +/** + * Enough of the shell's grammar to read a drafted check's arguments as the + * shell will hand them to each program. + * + * The seal-time path lint used to scan the raw command with a regex, so it + * read the inside of a sed program as a path: on 2026-10-07 three good + * differential checks (`… | sed -E 's/^ +//; s/ +/ /g' | diff - $d/out`) were + * dropped for using "/g, an absolute path outside the project", and a check + * that built a Python file with `printf` lost to the `"/old.py"` it + * concatenated onto `sys.argv[1]`. A path is an argument a program opens, and + * which arguments those are depends on the program: a sed script, an awk + * program, a grep pattern, a jq filter, printf's data are not paths. + * + * Not a full shell. Words, quotes, `$(…)`, backticks, `$((…))`, redirects, + * heredocs and the command separators. Command substitutions and heredoc + * bodies are kept as text for the caller to read again. + */ + +/** One shell word: its value after quote removal, with every `$(…)` replaced by SUBST. */ +export type ShellWord = { + raw: string; + value: string; + /** Inner text of each command substitution in the word, in order. */ + substs: string[]; + /** True when any part of the word was quoted. */ + quoted: boolean; +}; + +export type ShellRedirect = { op: string; target: ShellWord }; + +export type ShellCommand = { + words: ShellWord[]; + redirects: ShellRedirect[]; + /** Bodies of the command's `<<` heredocs. */ + heredocs: string[]; +}; + +/** Stands in for a command substitution inside a word's value. Never the start of a path. */ +export const SUBST = "\u0001"; + +const SEPARATORS = new Set([";", "&", "|", "\n", "(", ")"]); + +/** The text of a `$(…)` or `` `…` `` starting at `open` (just past the opener), and where it ends. */ +function readSubst(s: string, open: number, closer: ")" | "`"): { text: string; end: number } { + if (closer === "`") { + let j = open; + while (j < s.length && s[j] !== "`") j += s[j] === "\\" ? 2 : 1; + return { text: s.slice(open, j), end: Math.min(j + 1, s.length) }; + } + let depth = 1; + let j = open; + for (; j < s.length; j++) { + const ch = s[j]!; + if (ch === "\\") { + j++; + continue; + } + if (ch === "'") { + const e = s.indexOf("'", j + 1); + j = e < 0 ? s.length : e; + continue; + } + if (ch === '"') { + for (j++; j < s.length && s[j] !== '"'; j++) if (s[j] === "\\") j++; + continue; + } + if (ch === "(") depth++; + else if (ch === ")" && --depth === 0) break; + } + return { text: s.slice(open, j), end: Math.min(j + 1, s.length) }; +} + +/** + * The simple commands of a shell command line, in order. Compound syntax + * (`if`, `for`, `{ }`, `( )`) is flattened: its keywords come out as words + * of the commands they start, which is what a reader of arguments needs. + */ +export function parseShell(src: string): ShellCommand[] { + const out: ShellCommand[] = []; + let cmd: ShellCommand = { words: [], redirects: [], heredocs: [] }; + let word: ShellWord | null = null; + let pendingRedirect: string | null = null; + const pendingHeredocs: { delim: string; strip: boolean }[] = []; + + const endWord = () => { + if (!word) return; + if (pendingRedirect !== null) { + cmd.redirects.push({ op: pendingRedirect, target: word }); + if (pendingRedirect.startsWith("<<") && pendingRedirect !== "<<<") pendingHeredocs.push({ delim: word.value, strip: pendingRedirect === "<<-" }); + pendingRedirect = null; + } else cmd.words.push(word); + word = null; + }; + const endCommand = () => { + endWord(); + pendingRedirect = null; + if (cmd.words.length || cmd.redirects.length) out.push(cmd); + cmd = { words: [], redirects: [], heredocs: [] }; + }; + const w = (): ShellWord => (word ??= { raw: "", value: "", substs: [], quoted: false }); + + let i = 0; + let wordStart = 0; + const readHeredocs = () => { + // Bodies start on the line after the operator; `i` is at that newline. + let pos = i + 1; + const owner = cmd.words.length || cmd.redirects.length ? cmd : out.at(-1); + for (const h of pendingHeredocs.splice(0)) { + const lines: string[] = []; + for (;;) { + if (pos >= src.length) break; + const nl = src.indexOf("\n", pos); + const line = src.slice(pos, nl < 0 ? src.length : nl); + pos = nl < 0 ? src.length : nl + 1; + if ((h.strip ? line.replace(/^\t+/, "") : line).trim() === h.delim) break; + lines.push(line); + } + owner?.heredocs.push(lines.join("\n")); + } + i = pos - 1; + }; + + for (i = 0; i < src.length; i++) { + const ch = src[i]!; + if (ch === " " || ch === "\t") { + endWord(); + continue; + } + if (ch === "#" && !word) { + while (i < src.length && src[i] !== "\n") i++; + i--; + continue; + } + if (ch === "\\") { + if (src[i + 1] === "\n") { + i++; + continue; + } + if (!word) wordStart = i; + w().value += src[i + 1] ?? ""; + w().quoted = true; + w().raw = src.slice(wordStart, i + 2); + i++; + continue; + } + if (ch === "'") { + if (!word) wordStart = i; + const e = src.indexOf("'", i + 1); + const end = e < 0 ? src.length : e; + w().value += src.slice(i + 1, end); + w().quoted = true; + i = end; + w().raw = src.slice(wordStart, i + 1); + continue; + } + if (ch === '"') { + if (!word) wordStart = i; + const cur = w(); + cur.quoted = true; + let j = i + 1; + for (; j < src.length && src[j] !== '"'; j++) { + const c = src[j]!; + if (c === "\\" && /["\\$`\n]/.test(src[j + 1] ?? "")) { + cur.value += src[++j]; + } else if (c === "$" && src[j + 1] === "(") { + const r = readSubst(src, j + 2, ")"); + cur.substs.push(r.text); + cur.value += SUBST; + j = r.end - 1; + } else if (c === "`") { + const r = readSubst(src, j + 1, "`"); + cur.substs.push(r.text); + cur.value += SUBST; + j = r.end - 1; + } else cur.value += c; + } + i = j; + cur.raw = src.slice(wordStart, i + 1); + continue; + } + if (ch === "$" && src[i + 1] === "(") { + if (!word) wordStart = i; + // `$((…))` is arithmetic, `$(…)` a command: both are a value here, not a path. + const r = readSubst(src, i + 2, ")"); + if (src[i + 2] !== "(") w().substs.push(r.text); + w().value += SUBST; + i = r.end - 1; + w().raw = src.slice(wordStart, i + 1); + continue; + } + if (ch === "`") { + if (!word) wordStart = i; + const r = readSubst(src, i + 1, "`"); + w().substs.push(r.text); + w().value += SUBST; + i = r.end - 1; + w().raw = src.slice(wordStart, i + 1); + continue; + } + if (ch === ">" || ch === "<") { + // `2>`, `&>`: the digits or & just read belong to the operator. + const cur = word as ShellWord | null; + if (cur && /^\d+$/.test(cur.raw) && !cur.quoted) word = null; + else endWord(); + if (ch === "<" && src[i + 1] === "(") { + // Process substitution: a command, not a file. + const r = readSubst(src, i + 2, ")"); + wordStart = i; + w().substs.push(r.text); + w().value += SUBST; + i = r.end - 1; + w().raw = src.slice(wordStart, i + 1); + continue; + } + let op = ch; + while (/[<>|&-]/.test(src[i + 1] ?? "") && op.length < 3) { + const nx = src[i + 1]!; + if (nx === "&") { + // `>&2`, `<&0`: a descriptor, not a file. + op += nx; + i++; + break; + } + if (nx === "-" && op !== "<<") break; + if (nx === "|" && op !== ">") break; + op += nx; + i++; + } + if (op.endsWith("&")) { + // Skip the descriptor number. + while (/[\d-]/.test(src[i + 1] ?? "")) i++; + continue; + } + pendingRedirect = op; + continue; + } + if (ch === "\n") { + endCommand(); + if (pendingHeredocs.length) readHeredocs(); + continue; + } + if (ch === "&" && src[i + 1] === ">") { + // `&> file`: both streams to the file. The `>` is read next. + endWord(); + continue; + } + if (SEPARATORS.has(ch)) { + endCommand(); + continue; + } + if (!word) wordStart = i; + w().value += ch; + w().raw = src.slice(wordStart, i + 1); + } + endCommand(); + return out; +} + +/** Leading words that run the next word as the command (`timeout 5 cmd`, `env A=1 cmd`, `if cmd`). */ +const PREFIX = new Set(["{", "}", "then", "do", "else", "elif", "if", "while", "until", "!", "time", "sudo", "exec", "command", "nohup", "builtin", "nice", "stdbuf", "done", "fi"]); + +/** The command's own words: assignments, keyword and wrapper prefixes peeled off. */ +export function commandWords(c: ShellCommand): { assignments: ShellWord[]; argv: ShellWord[] } { + const assignments: ShellWord[] = []; + let ws = c.words; + for (;;) { + const h = ws[0]; + if (!h) break; + if (!h.quoted && PREFIX.has(h.value)) { + ws = ws.slice(1); + continue; + } + if (!h.quoted && h.value === "timeout") { + ws = ws.slice(1); + while (ws[0] && /^(-|\d)/.test(ws[0].value)) ws = ws.slice(/^-[sk]$/.test(ws[0].value) ? 2 : 1); + continue; + } + if (!h.quoted && h.value === "env") { + ws = ws.slice(1); + while (ws[0] && (/^-/.test(ws[0].value) || /^[A-Za-z_]\w*=/.test(ws[0].raw))) { + if (/^[A-Za-z_]\w*=/.test(ws[0].raw)) assignments.push(ws[0]); + ws = ws.slice(1); + } + continue; + } + if (/^[A-Za-z_]\w*(\[[^\]]*\])?\+?=/.test(h.raw)) { + assignments.push(h); + ws = ws.slice(1); + continue; + } + break; + } + return { assignments, argv: ws }; +} + +// ---------------------------------------------------------------- arguments that are paths + +/** Programs whose first operand is a program or pattern, not a file (unless given by an option). */ +const SCRIPT_FIRST: Record = { + sed: { script: /^(-e|--expression)$/, file: /^(-f|--file)$/, valued: /^(-l|--line-length)$/ }, + awk: { script: /^$/, file: /^(-f|--file)$/, valued: /^(-F|-v|--assign|--field-separator)$/ }, + grep: { script: /^(-e|--regexp)$/, file: /^(-f|--file)$/, valued: /^(-m|-A|-B|-C|-d|-D|--max-count|--context|--after-context|--before-context|--label|--color|--colour)$/ }, + jq: { script: /^$/, file: /^(-f|--from-file)$/, valued: /^(--indent|--tab)$/ }, +}; +for (const a of ["gawk", "mawk", "nawk", "busybox-awk"]) SCRIPT_FIRST[a] = SCRIPT_FIRST.awk!; +for (const g of ["egrep", "fgrep", "zgrep", "rg", "ag", "ack"]) SCRIPT_FIRST[g] = SCRIPT_FIRST.grep!; +SCRIPT_FIRST.gsed = SCRIPT_FIRST.sed!; + +/** Programs whose every argument is data or an operator, never a file to open. */ +const DATA_ONLY = new Set(["echo", "printf", "tr", "expr", "true", "false", "exit", "return", "basename", "dirname", "seq", "sleep", "kill", "pkill", "date", "export", "local", "declare", "set", "shift", "read", "trap", "let", "case", "esac", "in", "for", "unset"]); + +/** Interpreters and the option that gives them a program as text. */ +const INLINE: Record = { + python: /^-c$/, + node: /^(-e|--eval|-p|--print)$/, + deno: /^eval$/, + perl: /^-[a-zA-Z]*[eE]$/, + ruby: /^-[a-zA-Z]*e$/, + php: /^-r$/, + osascript: /^-e$/, +}; +const SHELLS = /^(ba|da|z|k|a)?sh$/; + +/** `test`/`[` operators whose operand is a file. */ +const FILE_TEST = /^-[efdsrwxLhpSbcgukOGN]$/; + +/** Every string literal in an inline program that is a whole absolute path, not one glued onto something with `+`. */ +export function programPathLiterals(prog: string): string[] { + const out: string[] = []; + for (const m of prog.matchAll(/(['"])(\/[\w.@+-][^'"\s{}$]*)\1/g)) { + const at = m.index ?? 0; + const before = prog.slice(0, at).replace(/[rbRB]$/, "").trimEnd(); + const after = prog.slice(at + m[0].length).trimStart(); + // `base + "/old.py"`, `"/x" + y`, `os.path.join(d, "/x")`: built from a value, not a path of its own. + if (/[+\w]$/.test(before) || after.startsWith("+")) continue; + out.push(m[2]!); + } + return out; +} + +const looksAbsolute = (v: string) => /^\/[\w.@+-]/.test(v); + +/** + * The absolute paths a command gives programs to open, in order: operands, + * redirect targets, assignment values, `--opt=/path` values, and the + * string literals of inline programs (`python3 -c`, heredocs to an + * interpreter) that are a path on their own. Never a sed/awk/jq program, a + * grep pattern, printf's or echo's data, a `find -name` pattern, a string + * `test` compares, or a value built onto a variable or `$(…)`. + */ +export function absolutePathArgs(run: string, depth = 0): string[] { + const out: string[] = []; + if (depth > 4) return out; + const word = (w: ShellWord | undefined) => { + if (w && looksAbsolute(w.value)) out.push(w.value.split(SUBST)[0]!); + }; + const program = (text: string) => out.push(...programPathLiterals(text)); + for (const c of parseShell(run)) { + const { assignments, argv } = commandWords(c); + for (const ws of c.words) for (const s of ws.substs) out.push(...absolutePathArgs(s, depth + 1)); + for (const r of c.redirects) { + for (const s of r.target.substs) out.push(...absolutePathArgs(s, depth + 1)); + if (!r.op.startsWith("<<")) word(r.target); + } + for (const a of assignments) { + const v = { ...a, value: a.value.replace(/^[A-Za-z_]\w*(\[[^\]]*\])?\+?=/, "") }; + // PATH-like lists: each element. + for (const part of v.value.split(":")) word({ ...v, value: part }); + } + const head = argv[0]; + if (!head) continue; + word(head); + const name = head.value.replace(/^.*\//, "").replace(/^(python|pypy)[\d.]*$/, "python"); + const args = argv.slice(1); + const generic = (list: ShellWord[]) => { + for (const a of list) { + const eq = /^--?[\w-]+=(.*)$/s.exec(a.value); + if (eq) word({ ...a, value: eq[1]! }); + else if (!a.value.startsWith("-")) word(a); + } + }; + if (DATA_ONLY.has(name)) continue; + if (name === "xargs") { + let k = 0; + while (args[k] && args[k]!.value.startsWith("-")) k += /^-[IdnLPsEa]$/.test(args[k]!.value) ? 2 : 1; + const rest = args.slice(k).map((a) => a.raw).join(" "); + if (rest.trim()) out.push(...absolutePathArgs(rest, depth + 1)); + continue; + } + if (name === "test" || name === "[" || name === "[[") { + args.forEach((a, k) => { + if (FILE_TEST.test(a.value)) word(args[k + 1]); + }); + continue; + } + if (name === "find") { + // Start points, then an expression of patterns and actions. + let k = 0; + while (args[k] && !/^[-!(]/.test(args[k]!.value)) word(args[k++]); + continue; + } + if (SCRIPT_FIRST[name]) { + const spec = SCRIPT_FIRST[name]!; + let scriptGiven = false; + const operands: ShellWord[] = []; + for (let k = 0; k < args.length; k++) { + const a = args[k]!.value; + if (a === "--") { + operands.push(...args.slice(k + 1)); + break; + } + if (spec.script.source !== "^$" && spec.script.test(a)) { + scriptGiven = true; + k++; + } else if (spec.file?.test(a)) { + scriptGiven = true; + word(args[++k]); + } else if (spec.valued.test(a)) k++; + else if (name === "jq" && /^--(arg|argjson)$/.test(a)) k += 2; + else if (name === "jq" && /^--(slurpfile|rawfile)$/.test(a)) { + word(args[k + 2]); + k += 2; + } else if (a.startsWith("-") && a.length > 1) { + // `-e'expr'` glued, `--regexp=x`: the script, given inline. + if (/^-e./.test(a) && name !== "jq") scriptGiven = true; + } else operands.push(args[k]!); + } + // awk `var=value` operands are assignments, not files. + const files = (scriptGiven ? operands : operands.slice(1)).filter((o) => !/^[A-Za-z_]\w*=/.test(o.value)); + for (const f of files) word(f); + continue; + } + if (SHELLS.test(name) || name === "eval") { + const k = args.findIndex((a) => /^-[a-zA-Z]*c[a-zA-Z]*$/.test(a.value)); + if (name === "eval") out.push(...absolutePathArgs(args.map((a) => a.value).join(" "), depth + 1)); + else if (k >= 0) { + if (args[k + 1]) out.push(...absolutePathArgs(args[k + 1]!.value, depth + 1)); + } else { + generic(args.slice(0, 1)); + for (const h of c.heredocs) out.push(...absolutePathArgs(h, depth + 1)); + } + continue; + } + if (INLINE[name]) { + const opt = INLINE[name]!; + const k = args.findIndex((a) => opt.test(a.value)); + if (k >= 0) { + if (args[k + 1]) program(args[k + 1]!.value); + // python3 -c '…' arg1 arg2: what follows is the program's argv. + generic(args.slice(k + 2)); + generic(args.slice(0, k).filter((a) => !/^-[mW]$/.test(a.value))); + } else { + // The script is the first operand; a `-m module` is not a file. + const m = args.findIndex((a) => a.value === "-m"); + generic(m >= 0 ? args.slice(0, m) : args); + for (const h of c.heredocs) program(h); + } + continue; + } + generic(args); + } + return out; +} diff --git a/src/tiers.ts b/src/tiers.ts index 21ced45..4441963 100644 --- a/src/tiers.ts +++ b/src/tiers.ts @@ -423,8 +423,10 @@ export function tierOf(args: { */ valueUnproven?: ReadonlySet; }): TierVerdict { + // A guard (tag `guard`, criteria.ts guardsFrom) passed before the work by + // construction: it can refuse a claim at the bar, never carry the word. const passing = args.results - .filter((r) => r.ok && !r.advisory && !r.skipped) + .filter((r) => r.ok && !r.advisory && !r.skipped && !r.tags?.includes("guard")) .map((r) => (args.valueUnproven?.has(r.name ?? "") && r.tags?.includes("value") ? { ...r, tags: r.tags.filter((t) => t !== "value") } : r)); const workerNames = (typeof args.worker === "string" ? [args.worker] : [...(args.worker ?? [])]).filter((w) => w.trim()); const worker = workerNames[0]; diff --git a/test/discriminating-checks.test.ts b/test/discriminating-checks.test.ts index e9fd47b..8f3b5d9 100644 --- a/test/discriminating-checks.test.ts +++ b/test/discriminating-checks.test.ts @@ -20,6 +20,7 @@ import { join } from "node:path"; import { describe, it } from "node:test"; import { parseArgs } from "../src/cli.js"; import { diagnoseFailure } from "../src/bar.js"; +import { missingDeliverable } from "../src/criteria.js"; import { Engine } from "../src/engine.js"; import { Journal } from "../src/journal.js"; import { Receipts } from "../src/receipts.js"; @@ -154,18 +155,38 @@ describe("the rule in a turn", () => { assert.deepEqual(tried[0]!.data.failed, ["task:greeting"]); }); - it("a check that could not run before the work has no pre-work try: not verified, on every platform", async () => { + it("a check whose script the work creates failed before the work: it discriminates, on every platform", async () => { // Before tool.sh exists, `bash tool.sh` exits 127, and `sh tool.sh` exits 127 - // where sh is bash (macOS) and 2 where it is dash (Debian, Ubuntu): both are - // read as broken, not failing, so neither counts as tried. - for (const run of ["bash tool.sh", "sh tool.sh"]) { - const { end, receipt } = await turn([check("tool", run)]); - assert.deepEqual([end.outcome, end.tier], ["unverified", "passed-untested"], run); - assert.match(end.tierReason!, /`task:tool` was not tried before the work began/); - assert.match(receipt, /before the work: not tried/); + // where sh is bash (macOS) and 2 where it is dash (Debian, Ubuntu). The + // shell could not open a project file that is not there yet: that is the + // deliverable missing, which is what a check is for, not a check that + // cannot run. On 2026-10-07 six `./rotate.sh …` checks were dropped as + // "the command was not found" this way. + for (const run of ["bash tool.sh", "sh tool.sh", "./tool.sh"]) { + const { end, receipt, tried } = await turn([check("tool", run)]); + if (run === "./tool.sh") { + // The work wrote tool.sh without the execute bit: 126 at the bar. + assert.notEqual(end.outcome, "verified", run); + continue; + } + assert.deepEqual([end.outcome, end.tier], ["verified", "verified"], run); + assert.match(receipt, /before the work: failed/); + assert.deepEqual(tried[0]!.data.failed, ["task:tool"], run); } }); + it("a tool the machine lacks is still not tried: only a missing project file counts as failing", async () => { + const { receipt, tried } = await turn([check("tool", "no-such-tool-xyz --version")]); + assert.deepEqual([tried[0]!.data.failed, tried[0]!.data.passed], [[], []]); + assert.match(receipt, /before the work: not tried/); + assert.equal(missingDeliverable("bash: line 1: pytest: command not found\n", "/nonexistent"), null); + assert.equal(missingDeliverable("sh: 1: rg: not found\n", "/nonexistent"), null); + assert.equal(missingDeliverable("bash: line 1: ./rotate.sh: No such file or directory\n", "/nonexistent"), "./rotate.sh"); + assert.equal(missingDeliverable("sh: 1: ./rotate.sh: not found\n", "/nonexistent"), "./rotate.sh"); + assert.equal(missingDeliverable("sh: 0: cannot open backup.sh: No such file\n", "/nonexistent"), "backup.sh"); + assert.equal(missingDeliverable("bash: /opt/x/run.sh: No such file or directory\n", "/nonexistent"), null, "an absolute path is not the project's"); + }); + it("dash's missing-script exit reads as bash's", () => { assert.equal(diagnoseFailure(2, "", "sh: 0: cannot open tool.sh: No such file\n").didNotRun, true); assert.equal(diagnoseFailure(127, "", "bash: tool.sh: No such file or directory\n").didNotRun, true); diff --git a/test/false-drops.test.ts b/test/false-drops.test.ts new file mode 100644 index 0000000..b7ccc01 --- /dev/null +++ b/test/false-drops.test.ts @@ -0,0 +1,352 @@ +/** + * Drafted checks Maat dropped before sealing that it should have kept. + * + * bench/local/drop_audit.py read every drop line of the 2026-10-07 lanes and + * ran each dropped command on the task's pristine tree and on the finished + * tree. The classes it found wrong, each tested here with the commands the + * judges really drafted, next to the true drops of the same rule: + * - the absolute-path rule read a sed program's `/g` and a value glued onto + * `sys.argv[1] +` as paths (all 4 of its drops); + * - the trial called `./rotate.sh` before rotate.sh existed "not found", + * though a missing deliverable is what a check is meant to find (6); + * - the mutation rule read `touch -d DATE file`'s DATE as a file it creates, + * and lost track of `f="$d/…"` and `> "$(mktemp -d)/x"` (14); + * - the cannot-fail rule read `… || exit 1; done; exit 0` and + * `… && exit 1 || exit 0` as checks that exit 0 whatever happened (3); + * - a check tried before the work that named the project by its absolute + * path read the live folder, not the copy it was supposed to be tried on; + * - and checks that pass before the work (P1) are kept as refuse-only + * guards instead of thrown away. + */ +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { cannotFail } from "../src/checklint.js"; +import { checkMutates } from "../src/checkwrites.js"; +import { + CRITERIA_MAX_CHECKS, + draftCriteriaCritiqued, + guardsFrom, + missingDeliverable, + pinsCurrentValue, + preflightCriteria, + sanitizeCriteria, + strayPath, + taskChecksFrom, + type LintDrop, +} from "../src/criteria.js"; +import { swallowsExit } from "../src/evidence.js"; +import { absolutePathArgs, parseShell } from "../src/shellwords.js"; +import { runCheck, type BarContext } from "../src/bar.js"; +import { replacePathPrefix } from "../src/scratch.js"; +import { Engine } from "../src/engine.js"; +import { Journal } from "../src/journal.js"; +import { Receipts } from "../src/receipts.js"; +import { tierOf } from "../src/tiers.js"; +import type { Check, CheckAuthor } from "../src/types.js"; +import { allowAll, drain, scriptedProvider, workspace, type ScriptedTurn } from "./helpers.js"; + +const JUDGE: CheckAuthor = { kind: "judge", model: "judge-j" }; + +describe("the absolute-path rule reads what programs open, not the raw text", () => { + const stray = (run: string, task = "") => strayPath(run, { cwd: "/work/proj", task }); + + it("keeps the 2026-10-07 differential checks it dropped", () => { + // wc-tool, three times: `/g` is sed's flag inside its program. + const wc = [ + String.raw`d=$(mktemp -d); printf 'a b\nc d e\n' > $d/a; printf 'h\xc3\xa9llo world\nx' > $d/b; python3 wc.py $d/a $d/b > $d/out && wc $d/a $d/b | sed 's/^ *//; s/ */ /g' | diff - $d/out`, + String.raw`d=$(mktemp -d); printf 'one two\nthree\n' > $d/a; python3 wc.py $d/a > $d/out && wc $d/a | sed 's/^ *//; s/ */ /g' | diff - $d/out`, + String.raw`d=$(mktemp -d); printf 'a b\nc\n\n' | python3 wc.py > $d/out && printf 'a b\nc\n\n' | wc | sed 's/^ *//; s/ */ /g' | diff - $d/out`, + ]; + for (const run of wc) assert.equal(stray(run), null, run); + // refactor-pricing: a Python file built with printf; "/old.py" is glued onto sys.argv[1]. + const pricing = + `d=$(mktemp -d) && git show HEAD:pricing.py > $d/old.py && printf '%s\\n' 'import sys,importlib.util as u' 'sys.path.insert(0,".")' 'import pricing as n' ` + + `'s=u.spec_from_file_location("o",sys.argv[1]+"/old.py");o=u.module_from_spec(s);s.loader.exec_module(o)' > $d/t.py && python3 $d/t.py $d`; + assert.equal(stray(pricing), null); + }); + + it("still finds the invented paths it was written for", () => { + assert.equal(stray("python3 /wc.py words.txt"), "/wc.py"); + assert.equal(stray("cat /home/user/data.csv | wc -l"), "/home/user/data.csv"); + assert.equal(stray("python3 /app/server.py & sleep 1; curl -s localhost:8000/"), "/app/server.py"); + assert.equal(stray(`python3 -c "import csv; rows = list(csv.reader(open('/home/user/data.csv')))"`), "/home/user/data.csv"); + assert.equal(stray("python3 - <<'EOF'\nprint(open('/data/in.txt').read())\nEOF"), "/data/in.txt"); + assert.equal(stray("python3 wc.py words.txt > /home/user/out.txt"), "/home/user/out.txt"); + assert.equal(stray("OUT=/srv/report.txt python3 report.py"), "/srv/report.txt"); + assert.equal(stray("python3 report.py --out=/srv/report.txt"), "/srv/report.txt"); + assert.equal(stray("diff <(python3 wc.py /home/u/a) expected.txt"), "/home/u/a"); + assert.equal(stray("bash -c 'cat /home/u/a'"), "/home/u/a"); + assert.equal(stray("sed -n 1p /home/u/a.txt"), "/home/u/a.txt", "sed's file operand is a path; its program is not"); + assert.equal(stray("awk -f /home/u/prog.awk data.txt"), "/home/u/prog.awk"); + assert.equal(stray("grep -q x /home/u/log"), "/home/u/log"); + assert.equal(stray("test -s /home/u/out.txt"), "/home/u/out.txt"); + }); + + it("does not read programs, patterns and data as paths", () => { + const fine = [ + "awk -F/ '/^\\/usr/ {print $2}' paths.txt", + "grep -E '^/api/v1' routes.txt", + "grep -e /etc/passwd -q notes.txt", + "jq -e '.path == \"/srv/x\"' out.json", + "echo /home/user/x | python3 norm.py", + "printf '/a/b\\n/c/d\\n' | python3 dedupe.py", + "find . -path '/proc/*' -prune -o -name '*.py' -print", + `[ "$(python3 norm.py a//b)" = "/a/b" ]`, + "tr '/' '_' < in.txt", + `python3 -c "import os; print(os.path.join('a', 'b') + '/c')"`, + "cut -d/ -f2 paths.txt", + "cat < in.txt\n/home/user/x\nEOF\npython3 norm.py in.txt", + ]; + for (const run of fine) assert.equal(stray(run), null, run); + }); + + it("parses words, quotes, substitutions and redirects", () => { + const [c] = parseShell(`x="$(mktemp -d)/a b" 2>/dev/null >> 'o u t' cmd 'it''s' "q\\"x"`); + assert.deepEqual(c!.words.map((w) => w.value), ["x=\u0001/a b", "cmd", "its", 'q"x']); + assert.deepEqual(c!.redirects.map((r) => [r.op, r.target.value]), [[">", "/dev/null"], [">>", "o u t"]]); + assert.deepEqual(c!.words[0]!.substs, ["mktemp -d"]); + assert.deepEqual(parseShell("a && b || c; d | e & f\ng").map((x) => x.words[0]!.value), ["a", "b", "c", "d", "e", "f", "g"]); + assert.deepEqual(absolutePathArgs("cat /a/b 2>&1 >&2 | sed 's/x/y/g' /c/d"), ["/a/b", "/c/d"]); + }); +}); + +describe("a check of a script the work creates is not 'not found'", () => { + it("the trial reads a missing project script as the deliverable missing", async () => { + const ws = workspace(); + try { + mkdirSync(join(ws.dir, "data")); + // bash-backup, 2026-10-07: dropped as "the command was not found". + const run = "BACKUP_NOW=20230101-120000 ./backup.sh data backup_dest 2 && test -f backup_dest/backup-20230101-120000.tar.gz"; + const failed: string[] = []; + const broken = await preflightCriteria([{ name: "backup-script-works", kind: "command", run, expectExit: 0 }], { cwd: ws.dir, failed, stray: { task: "Write backup.sh" } }); + assert.deepEqual(broken, []); + assert.deepEqual(failed, ["backup-script-works"]); + // A tool the machine does not have is still broken. + const tool = await preflightCriteria([{ name: "t", kind: "command", run: "no-such-tool-xyz data", expectExit: 0 }], { cwd: ws.dir }); + assert.deepEqual(tool.map((b) => b.why), ["the command was not found, so nothing ran and nothing was established"]); + // So is a script the project has but cannot run. + writeFileSync(join(ws.dir, "present.sh"), "exit 0\n"); + assert.equal(missingDeliverable("bash: line 1: ./present.sh: No such file or directory\n", ws.dir), null); + } finally { + ws.cleanup(); + } + }); +}); + +describe("the mutation rule", () => { + it("keeps checks that only write into their mktemp directory", () => { + const fine = [ + // bash-rotate, 2026-10-07: touch -d's DATE read as a file the check creates. + `d=$(mktemp -d); for i in 1 2 3 4 5 6 7; do touch -d "2020-01-0$i" "$d/f$i.log"; done; ./rotate.sh "$d" >/dev/null && test $(ls "$d" | wc -l) -eq 5`, + `d=$(mktemp -d); for i in 1 2 3 4 5 6 7; do touch -d "@$((1000000000+i*100))" "$d/f $i.log"; done; touch -d @1 "$d/x.txt"; out=$(./rotate.sh "$d") && [ "$out" = "$(printf 'f 1.log\\nf 2.log')" ]`, + `d=$(mktemp -d); o=$(mktemp); touch -d '2020-01-01' "$d/old one.log"; ./rotate.sh "$d" > "$o"`, + `dir=$(mktemp -d) && for f in $(seq 1 8); do touch -d "-$f days" "$dir/$f.log"; done && ./rotate.sh -n 5 "$dir"`, + "d=$(mktemp -d); touch -t 202001010000 \"$d/a.log\"; touch -r \"$d/a.log\" \"$d/b.log\"; mkdir -m 700 \"$d/sub\"", + // A variable built on the mktemp directory. + `d=$(mktemp -d); for i in 1 2 3 4 5 6 7; do f="$d/f $i.log"; echo x > "$f"; touch -d "@$((1000+i))" "$f"; done; ./rotate.sh "$d"`, + `bash -c 'd=$(mktemp -d); for i in 1 2 3 4; do f="$d/a$i.log"; echo x>"$f"; done; ./rotate.sh -n 2 "$d"'`, + // node-summarize: a redirect straight into a mktemp directory. + `node summarize.js transactions.json > "$(mktemp -d)/out.json" && node summarize.js transactions.json | jq -e 'keys | length > 0'`, + ]; + for (const run of fine) assert.equal(checkMutates(run), null, run); + }); + + it("still refuses the checks that change the work", () => { + const writes: [string, RegExp][] = [ + ["git checkout master && git merge --no-ff -m 'Merge about page changes' about.md", /git checkout/], + ["mkdir -p test_src && echo 'test content' > test_src/file.txt && ./backup.sh test_src test_dest 1", /creates test_src/], + ["make && touch src/x.md && make | grep -q 'out/x.txt'", /creates src\/x\.md/], + ["echo 'not json' > bad.json && node summarize.js bad.json 2>&1 | grep -q 'Error'", /bad\.json/], + ["test -f wc.py && chmod +x wc.py", /permissions on wc\.py/], + ["./rotate.sh -n 3 2>err.txt && exit 1 || test $? -eq 2", /err\.txt/], + ["touch -d 2020-01-01 old.log && ./rotate.sh .", /creates old\.log/], + ["f=out.txt; echo x > \"$f\"", /writes to \$f/], + ["cp items.json /tmp/items.bak && python3 server.py 8080 & sleep 1; cp /tmp/items.bak items.json", /writes items\.json/], + ]; + for (const [run, why] of writes) assert.match(checkMutates(run) ?? "", why, run); + }); +}); + +describe("the cannot-fail rule", () => { + it("keeps a check whose earlier `exit 1` can still fail it", () => { + const fine = [ + // config-migrate, 2026-10-07. + `for f in configs/*.json; do git show HEAD:"$f" 2>/dev/null | jq -e '.version==2' >/dev/null 2>&1 && { git diff --quiet HEAD -- "$f" || exit 1; }; done; exit 0`, + `for f in configs/*.json; do jq -e '.version==1' "$f" >/dev/null 2>&1 && { jq -e '.database.port==5432' "$f" >/dev/null || exit 1; }; done; exit 0`, + // redact-secrets: `cond && exit 1 || exit 0` is `! cond`. + String.raw`find . -type f -not -path './.maat/*' -exec grep -l 'AKIA\|ghp_' {} \; | xargs -I {} sh -c 'grep -q "\[REDACTED\]" {} && echo "OK" || echo "FAIL"' | grep -q "FAIL" && exit 1 || exit 0`, + ]; + for (const run of fine) { + assert.equal(swallowsExit(run), null, run); + assert.equal(cannotFail(run), null, run); + } + }); + + it("still drops the checks that cannot fail", () => { + const never = [ + "git log --oneline -n 1 | grep -q 'master' && ./check.sh || true", + "python3 ledger.py journal.txt 2>&1 | grep -q 'line 10:' || echo 'PASS'", + String.raw`find configs/ -name '*.json' -exec sh -c 'jq -e ".version == 2" {} 2>/dev/null || exit 1' \;`, + "grep -q x out.txt; exit 0", + "grep -q x out.txt && false || true", + "(grep -q x a || exit 1); grep -q y b; exit 0", + "sh -c 'grep -q x a || exit 1'; exit 0", + `python3 -c "import csv; h = next(csv.reader(open('clean.csv'))); print('PASS' if len(h) == 5 else 'FAIL')"`, + ]; + for (const run of never) assert.ok(cannotFail(run), run); + }); +}); + +describe("a check names the project by its absolute path", () => { + it("is tried on the copy before the work, not on the live folder", async () => { + const live = workspace(); + const pre = workspace(); + try { + // The worker has already written out.txt in the live folder; the pre-work copy has not. + writeFileSync(join(live.dir, "out.txt"), "3\n"); + const run = `cd ${live.dir} && grep -qx 3 out.txt`; + const passed: string[] = []; + const failed: string[] = []; + await preflightCriteria([{ name: "counts", kind: "command", run, expectExit: 0 }], { cwd: pre.dir, root: live.dir, passed, failed }); + assert.deepEqual([passed, failed], [[], ["counts"]]); + } finally { + live.cleanup(); + pre.cleanup(); + } + }); + + it("runs in the bar's copy, so it cannot write the work", async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "a.txt"), "a\n"); + const ctx = { cwd: ws.dir, record: [], ledger: [], archivedBatches: 0 } as unknown as BarContext; + const r = await runCheck( + { name: "task:abs", kind: "command", run: `cd ${ws.dir} && touch made-by-check && grep -q a ${ws.dir}/a.txt`, timeoutMs: 5_000, expectExit: 0, tags: ["task"], hidden: true }, + ctx, + ); + assert.equal(r.ok, true, r.output); + assert.equal(existsSync(join(ws.dir, "made-by-check")), false); + } finally { + ws.cleanup(); + } + }); + + it("only a whole path is rewritten", () => { + assert.equal(replacePathPrefix("cd /w/p && cat /w/p/a /w/p2/b /x/w/p", "/w/p", "/tmp/c"), "cd /tmp/c && cat /tmp/c/a /w/p2/b /x/w/p"); + }); +}); + +describe("checks that pass before the work are kept as refuse-only guards", () => { + const drop = (name: string, run: string, rule = "P1-passes-before-work"): LintDrop => ({ name, run, rule, why: "", redraft: false }); + + it("keeps fix-git's invariants and leaves out the checks that pin today's output", () => { + // Dropped by P1 on 2026-10-07 (mmsj-1 fix-git): each must still hold after a correct merge. + const invariants = [ + drop("no-unmerged-branches", `test -z "$(git branch --no-merged master | tr -d ' *')"`), + drop("on-master-clean-tree", `test "$(git rev-parse --abbrev-ref HEAD)" = master && test -z "$(git status --porcelain -- . ':!.maat')"`), + drop("no-conflict-markers", "! grep -qE '^(<<<<<<<|>>>>>>>)' about.md index.md"), + ]; + // Dropped by P1 the same day, and failed on a finished tree the grader accepted. + const pins = [ + drop("script-works", `node summarize.js transactions.json | jq -e '.zoe == "-3.50" and .amy == "0.00" and .max == "12.50"'`), + drop("schema-version-1", "sqlite3 app.db 'PRAGMA user_version;' | grep -q '^1$'"), + drop("cache-module-exists", "test -f cache.py && echo 'cache.py exists'"), + drop("basic", `node csv2json.js < sample.csv | jq -e '[.[] | keys | sort] | unique | length == 1 and .[0] == ["id", "name", "comment"]'`), + ]; + for (const p of pins) assert.ok(pinsCurrentValue(p.run), p.run); + for (const p of invariants) assert.equal(pinsCurrentValue(p.run), false, p.run); + // Invariants that compare with nothing or zero, and inputs the check makes itself, are not pins. + assert.equal(pinsCurrentValue("python3 slugify.py < titles.txt | sort | uniq -d | wc -l | grep -q '^0$'"), false); + assert.equal(pinsCurrentValue(`echo -e 'id,name\\n1,foo' | node csv2json.js | jq -e '.[0].id == "1"'`), false); + const kept = guardsFrom([...invariants, ...pins, drop("lint", "git checkout master", "L15-mutates")], [{ name: "merged", run: "git merge-base --is-ancestor feature master" }]); + assert.deepEqual(kept.map((g) => [g.name, g.guard]), invariants.map((d) => [d.name, true])); + // A duplicate of a sealed check, or of another guard, is not kept twice; a name clash is renamed. + const again = guardsFrom([invariants[0]!, invariants[0]!, drop("merged", "test -z \"$(git status --porcelain)\"")], [{ name: "merged", run: "true" }]); + assert.deepEqual(again.map((g) => g.name), ["no-unmerged-branches", "merged-2"]); + }); + + it("are sealed with the guard tag, beside the checks and outside their cap", () => { + const checks = Array.from({ length: CRITERIA_MAX_CHECKS }, (_, i) => ({ name: `c${i}`, run: `grep -q ${i} out.txt` })); + const d = sanitizeCriteria({ checks: [...checks, { name: "g", run: "! grep -q '<<<<<<<' a.md", guard: true }], notes: [] }); + assert.equal(d.checks.length, CRITERIA_MAX_CHECKS + 1); + const sealed = taskChecksFrom(d, { hidden: true, author: JUDGE }); + assert.deepEqual(sealed.taskChecks.at(-1)!.tags, ["task", "guard"]); + assert.ok(!sealed.taskChecks[0]!.tags.includes("guard")); + }); + + it("never count toward verified", () => { + const value = (name: string, tags = ["task", "value"]) => ({ name, ok: true, kind: "command" as const, hidden: true, tags }); + const authors = new Map([["task:g", JUDGE], ["task:v", JUDGE]]); + const guardOnly = tierOf({ results: [value("task:g", ["task", "value", "guard"])], worker: "w", authors }); + assert.notEqual(guardOnly.tier, "verified"); + assert.equal(tierOf({ results: [value("task:g", ["task", "value", "guard"]), value("task:v")], worker: "w", authors }).tier, "verified"); + }); + + it("come out of the draft when the pre-work screen drops a check", async () => { + const ws = workspace(); + const pre = workspace(); + try { + for (const d of [ws.dir, pre.dir]) writeFileSync(join(d, "about.md"), "about\n"); + const draft = (...cs: [string, string][]) => JSON.stringify({ checks: cs.map(([name, run]) => ({ name, run })), notes: [] }); + const fetchFn = (async (_u: string, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as { messages: { role: string; content: string }[] }; + const critic = body.messages[0]!.content.startsWith("You review acceptance checks"); + const redraft = body.messages.some((m) => m.content.includes("was dropped:")); + const content = critic + ? JSON.stringify({ checks: [], uncovered: [], requirements: [] }) + : redraft + ? draft(["merged", "grep -q merged about.md"]) + : draft(["merged", "grep -q merged about.md"], ["no-markers", "! grep -q '^<<<<<<<' about.md"]); + return { ok: true, status: 200, json: async () => ({ choices: [{ message: { content }, finish_reason: "stop" }] }), text: async () => "" } as unknown as Response; + }) as unknown as typeof fetch; + const r = await draftCriteriaCritiqued({ scripts: [], barChecks: [], baseUrl: "http://p.test/v1", model: "m", task: "Merge the about page", cwd: ws.dir, preWorkDir: pre.dir, fetchFn }); + assert.ok(r.ok); + assert.deepEqual(r.draft.checks.map((c) => [c.name, c.guard ?? false]), [["merged", false], ["no-markers", true]]); + assert.ok(r.critique.some((l) => /refuse-only guards.*no-markers/.test(l))); + } finally { + ws.cleanup(); + pre.cleanup(); + } + }); + + describe("at the bar", () => { + const work: ScriptedTurn[] = [{ calls: [{ name: "write_file", args: { path: "out.txt", content: "hello\n" } }] }, { text: "Done." }]; + const check = (name: string, run: string, guard = false): Check => + ({ name, kind: "command", run, timeoutMs: 5_000, expectExit: 0, tags: ["task", "value", ...(guard ? ["guard"] : [])], hidden: true, author: JUDGE }) as Check; + + async function turn(checks: Check[]) { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "keep.txt"), "keep\n"); + const provider = scriptedProvider([...work, ...work, ...work]); + const journal = new Journal(ws.dir, "guard"); + const engine = new Engine({ baseUrl: "http://provider.test/v1", model: "m", cwd: ws.dir, fetchFn: provider.fetchFn, bar: null, receipts: new Receipts(ws.dir), stream: false, autonomy: "high", journal }); + const events = await drain(engine.run("Write out.txt containing hello.", allowAll, { taskChecks: checks, taskNotes: [] })); + const end = events.find((e) => e.kind === "job_end"); + assert.ok(end && end.kind === "job_end"); + const dir = join(ws.dir, ".maat", "receipts"); + const receipt = readdirSync(dir).filter((f) => f.endsWith(".md")).sort().map((f) => readFileSync(join(dir, f), "utf8")).join("\n"); + return { end, receipt }; + } finally { + ws.cleanup(); + } + } + + it("a guard that holds says so and verifies nothing on its own", async () => { + const { end, receipt } = await turn([check("hello", "grep -qx hello out.txt"), check("kept", "grep -qx keep keep.txt", true)]); + assert.equal(end.outcome, "verified"); + assert.match(receipt, /role: refuse-only guard/); + assert.match(receipt, /`kept: [^`]*` — written by .* · refuse-only guard/); + assert.doesNotMatch(receipt, /`hello: [^`]*` — written by [^\n]*refuse-only/); + const alone = await turn([check("kept", "grep -qx keep keep.txt", true)]); + assert.notEqual(alone.end.outcome, "verified"); + }); + + it("a guard that fails refuses the claim", async () => { + const { end } = await turn([check("hello", "grep -qx hello out.txt"), check("untouched", "test ! -e out.txt", true)]); + assert.notEqual(end.outcome, "verified"); + }); + }); +});