diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ba80e1..25a7d04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,21 @@ - **`opencode://zen`** is the OpenCode endpoint (short name `opencode`). The old `opencode://subscription` still works for this release, with a deprecation notice. +- **Unattended runs stop when they stop advancing** (`MAAT_NO_PROGRESS_CALLS`, + default 30, `0` turns it off; on for every unattended run on an HTTP + backend, not on ACP backends yet). A call advances when it changes a file + in the project, reads something not read before, runs a command not run + before, or gets output from a command that differs from every earlier run + of it (timings aside), wherever the work happens: a build into `dist/`, a + venv, a config under `/etc`. After N calls in a row that advance nothing the + model is told once to finish or stop; after N more the turn ends, the work + on disk is judged, and the receipt says `ended: no progress` (never + verified: the model did not say it was done). +- **While an unattended job with hidden checks runs, the file tools stay in + the task.** `read_file`, `list_dir`, `grep` and `inspect` refuse paths + outside the project and under `.maat/` (except spilled output and + background logs, and paths the task names). bash is not blocked; this is a + budget control, not isolation. ## 0.2.2 — subscription backends removed diff --git a/bench/local/container/Dockerfile b/bench/local/container/Dockerfile index 2816c8f..581650e 100644 --- a/bench/local/container/Dockerfile +++ b/bench/local/container/Dockerfile @@ -14,4 +14,6 @@ ENV BENCH_AGENT_USER=agent WORKDIR /opt # Maat itself is installed per build by run-in-container.sh (FROM maat-bench + npm i -g the packed tarball). -ENV BENCH_WORK=/work RESULTS_DIR=/results +# run-in-container.sh overrides these: tasks run in a container-local /var/lib/bench-work and the +# host folders are mounted under /root, where the agent user cannot reach them. +ENV BENCH_WORK=/var/lib/bench-work BENCH_EXPORT=/root/bench-export RESULTS_DIR=/root/bench-results diff --git a/bench/local/container/run-in-container.sh b/bench/local/container/run-in-container.sh index c41c38c..bace4d6 100755 --- a/bench/local/container/run-in-container.sh +++ b/bench/local/container/run-in-container.sh @@ -4,9 +4,10 @@ # Make the tarball with: npm run pack:cli && npm pack ./out-cli # The image is maat-bench:, built once and reused, so every run # records exactly which Maat it tested. Env passed through: REFERENCE BENCH_MODEL -# BENCH_URL BENCH_LIMIT RESULTS (file name). Results land in -# ~/.cache/maat-bench/container-results/; each task's folder and Maat's log are kept -# in ~/.cache/maat-bench/container-work// (the only host path it can write). +# BENCH_URL BENCH_LIMIT RESULTS (file name; run.py refuses a file holding another lane's rows). Results land in +# ~/.cache/maat-bench/container-results/; each task's folder and Maat's log are copied, +# once the task is graded, to ~/.cache/maat-bench/container-work// (the only host +# path it can write; the agent never sees it, see "Task isolation" below). # SUBSCRIPTION=grok: use the owner's Grok Build login (BENCH_URL=grok-build://subscription # BENCH_MODEL=grok-4.7). Image is maat-bench-grok: (Dockerfile.grok: official grok # installer, linux, pinned). ~/.grok/auth.json is mounted read-only OUTSIDE the container's @@ -59,10 +60,27 @@ url=${url//127.0.0.1/host.docker.internal}; url=${url//localhost/host.docker.int # run ends: on the command line (-e KEY=...) it showed in every `ps` listing. envf=$(mktemp); chmod 600 "$envf"; trap 'rm -f "$envf"' EXIT printf 'OPENROUTER_API_KEY=%s\n' "$key" > "$envf" +# Anthropic API lanes (BENCH_URL=https://api.anthropic.com/v1): the key comes from the +# Keychain item maat-bench-anthropic (paste it with container/set-anthropic-key.sh), +# never from the command line. +if [[ "$url" == *api.anthropic.com* || "${ARMS:-}" == *api.anthropic.com* ]]; then + akey=$(security find-generic-password -s maat-bench-anthropic -w 2>/dev/null) || { echo "no Keychain item maat-bench-anthropic: run bench/local/container/set-anthropic-key.sh" >&2; exit 1; } + printf 'ANTHROPIC_API_KEY=%s\n' "$akey" >> "$envf" +fi # The graders and reference solutions are mounted where only root can reach (/root is 700), # copied to a root-only /opt/bench, and run.py runs from there as root; it runs the agent # as the unprivileged `agent` user (BENCH_AGENT_USER) and grades as root afterwards. -credmount=; startcmd='export MOLT_DIST_ABS=$(npm root -g)/@solvyx/molt/dist; mkdir -p /work && chmod 755 /work && rm -rf /opt/bench && cp -a /root/bench-src /opt/bench && chmod -R go-rwx /opt/bench && cd /opt/bench && python3 run.py "$@"' +# +# Task isolation, in every mode (plain, SUBSCRIPTION=grok, OPENCODE=1). A host bind mount +# ignores chown and does not enforce modes for other users (OrbStack, Docker Desktop), so the +# agent can read anything mounted where it can reach it: on 2026-10-07 a worker spent its whole +# budget reading the other tasks' folders and logs in a shared /work. So nothing from the host +# is mounted where the agent can reach it. The work folder and the results are mounted under +# /root (700, container-local, enforced). Each task runs in a private folder under the +# container-local /var/lib/bench-work (711, random per-task parent, removed after the task); +# Maat's logs are written to a root-only folder there; folder and logs are copied to the export +# only once the task is graded and every process of the agent user is gone (run.py finish_task). +credmount=; startcmd='export MOLT_DIST_ABS=$(npm root -g)/@solvyx/molt/dist BENCH_WORK=/var/lib/bench-work BENCH_EXPORT=/root/bench-export RESULTS_DIR=/root/bench-results; chmod 700 /root && mkdir -p $BENCH_WORK && chmod 711 $BENCH_WORK && rm -rf /opt/bench && cp -a /root/bench-src /opt/bench && chmod -R go-rwx /opt/bench && cd /opt/bench && python3 run.py "$@"' if [ "$SUBSCRIPTION" = grok ]; then credmount="-v $HOME/.grok/auth.json:/root/grok-cred/auth.json:ro" # grok runs as `agent`, so the credential copy lives in ITS home, owned by it, mode 600. @@ -81,9 +99,12 @@ if [ "$OPENCODE" = 1 ]; then startcmd='install -d -o agent -g agent -m 700 /home/agent/.local /home/agent/.local/share /home/agent/.local/share/opencode && install -o agent -g agent -m 600 /root/oc-cred/auth.json /home/agent/.local/share/opencode/auth.json && '$startcmd fi echo "maat $img → $out/${RESULTS:-results.jsonl}" -docker run --rm --name "maat-bench-$name" \ - -v "$local_dir":/root/bench-src:ro -v "$out":/results -v "$work":/work ${=credmount} \ +# Pinned CPU and memory (BENCH_CPUS, BENCH_MEMORY): perf graders have wall-clock limits, and lanes +# often run side by side on the Mac. Recorded in every row's lane. +cpus=${BENCH_CPUS:-2}; mem=${BENCH_MEMORY:-4g} +docker run --rm --name "maat-bench-$name" --cpus "$cpus" --memory "$mem" \ + -v "$local_dir":/root/bench-src:ro -v "$out":/root/bench-results -v "$work":/root/bench-export ${=credmount} \ --env-file "$envf" -e REFERENCE="${REFERENCE:-0}" -e BENCH_MODEL="${BENCH_MODEL:-}" \ -e ARMS="${ARMS:-}" -e BENCH_TASKS="${BENCH_TASKS:-}" -e BENCH_URL="$url" -e BENCH_LIMIT="${BENCH_LIMIT:-600}" -e BENCH_REASONING="${BENCH_REASONING:-}" -e BENCH_GATE="${BENCH_GATE:-}" -e RESULTS="${RESULTS:-results.jsonl}" \ - -e MAAT_BUILD="$sha" -e PYTHONDONTWRITEBYTECODE=1 \ + -e MAAT_BUILD="$sha" -e PYTHONDONTWRITEBYTECODE=1 -e BENCH_CPUS="$cpus" -e BENCH_MEMORY="$mem" \ "$img" sh -c "$startcmd" run "$@" diff --git a/bench/local/container/set-anthropic-key.sh b/bench/local/container/set-anthropic-key.sh new file mode 100755 index 0000000..4cfe830 --- /dev/null +++ b/bench/local/container/set-anthropic-key.sh @@ -0,0 +1,10 @@ +#!/bin/zsh +# Store an Anthropic API key for the bench in the macOS Keychain (item: maat-bench-anthropic). +# The key is read without echo and never appears on a command line or in shell history. +print -n "Paste your Anthropic API key (input hidden): " +read -rs key; print +[[ "$key" == sk-ant-* ]] || { print "That doesn't look like an Anthropic key (sk-ant-...). Nothing saved."; exit 1; } +# Fed to `security -i` on stdin, so the key is never in any process's argument list. +print -r -- "add-generic-password -U -s maat-bench-anthropic -a $USER -w $key" | security -i >/dev/null && + security find-generic-password -s maat-bench-anthropic >/dev/null 2>&1 && print "Saved to Keychain as maat-bench-anthropic (${#key} chars, ends …${key[-4,-1]})." +unset key diff --git a/bench/local/crosscheck.py b/bench/local/crosscheck.py index b2b5f5f..52a8f6d 100644 --- a/bench/local/crosscheck.py +++ b/bench/local/crosscheck.py @@ -83,7 +83,8 @@ def fresh() -> Path: d = fresh() # a check names the folder of the run that drafted it (/work/-molt-); in another # run's tree that path must mean "this tree", or every foreign tree fails it spuriously - run_cmd = re.sub(r"/work/[\w.-]+-molt-\d+", str(dst), cmd) + # (runs since 2026-10-07 have a private folder: /var/lib/bench-work//-molt-) + run_cmd = re.sub(r"(?:/var/lib/bench-work/[0-9a-f]+|/work)/[\w.-]+-molt-\d+", str(dst), cmd) p = subprocess.Popen(run_cmd, shell=True, cwd=d, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, start_new_session=True) try: @@ -212,7 +213,7 @@ def exec_tree(run: dict, cmds: list[str]) -> str: if set(cmds) <= set(old["checks"]): return "cached" else: # results from before the /work/ rewrite: redo only commands that name such a path - cmds = [c for c in cmds if "/work/" in c] + cmds = [c for c in cmds if "/work/" in c or "/var/lib/bench-work/" in c] if not cmds: old["pathfix"] = True out.write_text(json.dumps(old)) @@ -263,7 +264,7 @@ def execute(jobs: int) -> None: # ------------------------------------------------------------------ analysis def sig(rc: int, out: str, run_name: str) -> str: - o = re.sub(r"/work/[\w.-]+", "/work/T", out) + o = re.sub(r"(?:/var/lib/bench-work/[0-9a-f]+|/work)/[\w.-]+", "/work/T", out) o = re.sub(r"0x[0-9a-f]+|\b[0-9a-f]{7,40}\b", "H", o) o = re.sub(r"\d{4,}", "N", o) o = re.sub(r"\s+", " ", o).strip()[:300] diff --git a/bench/local/grading.py b/bench/local/grading.py new file mode 100644 index 0000000..c226c36 --- /dev/null +++ b/bench/local/grading.py @@ -0,0 +1,200 @@ +""" +What every grading path shares (run.py, regrade_today.py, crosscheck.py, the validators). + +- git_env / git_guard: grader git never refuses the worker's repository for its owner, and never + runs the worker's hooks or fsmonitor (B1 of the PR #33 audit). +- child_values / PY_EMIT / emit_js: a grader that has to run the worker's code in a process only + takes computed VALUES back from it, as one JSON record on a dedicated file descriptor; the + grader process compares them with the expected values, which never enter the worker's process + (B2). A last stdout line "OK" was forgeable by an atexit hook in the worker's module. +- safe_grade: a grader that raises records a failed row flagged grader_error instead of stopping + the lane (B4). +- links_into: a symlink in the task folder that resolves into the bench's own folders (graders, + reference solutions, results, export) fails the task before any grader follows it (N1). +""" + +from __future__ import annotations + +import contextlib +import json +import os +import subprocess +import tempfile +from pathlib import Path + +# ---------------------------------------------------------------- B1: git as the grader sees it + +# Applied in the command scope (GIT_CONFIG_COUNT), which outranks the repository's own config. +GIT_GRADE_CONFIG = (("core.hooksPath", os.devnull), ("core.fsmonitor", "false")) + + +def git_env(d: Path | str) -> dict[str, str]: + """ + Environment for every git command a grader runs on task folder `d`: + + - safe.directory= (both spellings, as given and resolved), so a repository owned by the + agent user is not refused as "dubious ownership" when root grades it. Git >= 2.35.2 printed + nothing for every grader call and four git tasks failed correct work. + - core.hooksPath=/dev/null and core.fsmonitor=false: the ownership refusal used to be the only + thing keeping root's git from running the worker's hooks or fsmonitor command. + - GIT_CONFIG_GLOBAL=/dev/null: the agent user's ~/.gitconfig is the worker's to write. + """ + p = Path(d) + dirs = list(dict.fromkeys([str(p), str(p.resolve())])) + pairs = [("safe.directory", x) for x in dirs] + list(GIT_GRADE_CONFIG) + env = {"GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_COUNT": str(len(pairs)), "GIT_TERMINAL_PROMPT": "0"} + for i, (k, v) in enumerate(pairs): + env[f"GIT_CONFIG_KEY_{i}"] = k + env[f"GIT_CONFIG_VALUE_{i}"] = v + return env + + +@contextlib.contextmanager +def git_guard(d: Path | str): + """git_env(d) in os.environ for the duration (graders' children inherit it), then restored.""" + add = git_env(d) + stale = [k for k in os.environ if k.startswith(("GIT_CONFIG_KEY_", "GIT_CONFIG_VALUE_"))] + saved = {k: os.environ.get(k) for k in [*add, *stale]} + for k in stale: + del os.environ[k] + os.environ.update(add) + try: + yield + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +# ---------------------------------------------------------------- B2: values out of process + +VALUES_FD_ENV = "BENCH_VALUES_FD" + +# Prepended to every Python harness: emit(v) writes v as the one record on the values channel and +# ends the process at once (os._exit, so an atexit hook in the worker's module cannot add to it). +PY_EMIT = ( + "import os as _bo, json as _bj\n" + f"_BFD = int(_bo.environ.pop({VALUES_FD_ENV!r}))\n" + "def _btag(v):\n" + " if isinstance(v, tuple):\n" + " return {'$tuple': [_btag(x) for x in v]}\n" + " if isinstance(v, list):\n" + " return [_btag(x) for x in v]\n" + " if isinstance(v, dict):\n" + " return {k: _btag(x) for k, x in v.items()}\n" + " return v\n" + "def emit(v):\n" + " b = (_bj.dumps(_btag(v), default=repr) + '\\n').encode()\n" + " while b:\n" + " b = b[_bo.write(_BFD, b):]\n" + " _bo._exit(0)\n" + "def call(f, *a, **k):\n" + " try:\n" + " return ['ok', f(*a, **k)]\n" + " except Exception as e:\n" + " return ['err', type(e).__name__, str(e)[:300], isinstance(e, ValueError), isinstance(e, TypeError)]\n" +) + +# The same for a Node harness: `emit(v)`. +JS_EMIT = ( + "const __bfs = require('fs');\n" + f"const __bfd = Number(process.env.{VALUES_FD_ENV});\n" + "function emit(v) { __bfs.writeSync(__bfd, JSON.stringify(v) + '\\n'); process.exit(0); }\n" +) + + +def child_values(cmd: list[str], cwd: Path, timeout: float, env: dict | None = None) -> tuple[object | None, str]: + """ + Run `cmd` (a harness that loads the worker's code) and return (values, diagnostic). + + The harness writes exactly one JSON record to the file descriptor named by $BENCH_VALUES_FD. + values is None when there is no record, more than one, or it is not JSON; stdout and stderr + are only ever a diagnostic, never a verdict. + """ + e = dict(os.environ) + e.update(env or {}) + with tempfile.TemporaryFile() as ch: + e[VALUES_FD_ENV] = str(ch.fileno()) + try: + r = subprocess.run(cmd, cwd=cwd, capture_output=True, timeout=timeout, env=e, pass_fds=(ch.fileno(),)) + except subprocess.TimeoutExpired: + return None, f"timed out after {timeout}s" + ch.seek(0) + raw = ch.read() + diag = (r.stdout + b"\n" + r.stderr).decode(errors="replace").strip() + diag = diag.splitlines()[-1][:240] if diag else f"exit {r.returncode}" + if not raw: + return None, diag or "no values" + lines = raw.split(b"\n") + if len(lines) != 2 or lines[1] != b"": + return None, "the values channel held more than one record" + try: + return json.loads(lines[0]), diag + except ValueError: + return None, "the values channel held no valid JSON" + + +def py_values(d: Path, code: str, timeout: float = 60) -> tuple[object | None, str]: + """Run PY_EMIT + code in a fresh python3 inside d (see child_values).""" + return child_values(["python3", "-c", PY_EMIT + code], d, timeout) + + +def tag(v): + """Tuples as {"$tuple": [...]}, as emit() sends them: a tuple and a list stay unequal, as they + were when the comparison ran next to the worker's code.""" + if isinstance(v, tuple): + return {"$tuple": [tag(x) for x in v]} + if isinstance(v, list): + return [tag(x) for x in v] + if isinstance(v, dict): + return {k: tag(x) for k, x in v.items()} + return v + + +def jsonish(v): + """v as it looks after emit()'s JSON round trip, for comparing with child values.""" + return json.loads(json.dumps(tag(v), default=repr)) + + +# ---------------------------------------------------------------- B4: a grader that raises + +def safe_grade(T, d: Path) -> tuple[bool, str, bool]: + """ + (passed, why, grader_error). A grader exception is a failed row with grader_error set, never + an abort: the lane went on to the next task's free retry of this sample. A TimeoutExpired is the + worker's program hanging, an ordinary fail. + """ + try: + with git_guard(d): + ok, why = T.grade(d) + return bool(ok), str(why), False + except subprocess.TimeoutExpired as e: + return False, f"timed out: {' '.join(map(str, e.cmd)) if isinstance(e.cmd, (list, tuple)) else e.cmd}"[:300], False + except (Exception, SystemExit) as e: # noqa: BLE001 - every failure is a verdict + return False, f"grader error: {e!r}"[:2000], True + + +# ---------------------------------------------------------------- N1: links into the bench + +def links_into(d: Path, roots: list[Path]) -> list[str]: + """Symlinks under d (not followed) whose target resolves into one of `roots`. A link to + something inside d itself is fine, wherever d lives.""" + rs = [Path(r).resolve() for r in roots if r] + home = Path(d).resolve() + bad = [] + for dirpath, dirnames, filenames in os.walk(d): + for name in dirnames + filenames: + p = Path(dirpath) / name + if not p.is_symlink(): + continue + try: + t = p.resolve() + except (OSError, RuntimeError): + continue + if t == home or t.is_relative_to(home): + continue + if any(t == r or t.is_relative_to(r) for r in rs): + bad.append(os.path.relpath(p, d)) + return bad diff --git a/bench/local/refs2.py b/bench/local/refs2.py index 933dc40..1c373f5 100644 --- a/bench/local/refs2.py +++ b/bench/local/refs2.py @@ -161,7 +161,7 @@ def solve_bash_rotate(d: Path): i=$((i + 1)) if [ "$i" -gt "$n" ]; then echo "$f"; fi done < <(cd "$dir" && ls -t -- *.log 2>/dev/null | while IFS= read -r x; do [ -f "$x" ] && echo "$x"; done) > "${TMPDIR:-/tmp}/rot.$$" -tail -r "${TMPDIR:-/tmp}/rot.$$" | while IFS= read -r f; do rm -f -- "$dir/$f"; echo "$f"; done +{ tac "${TMPDIR:-/tmp}/rot.$$" 2>/dev/null || tail -r "${TMPDIR:-/tmp}/rot.$$"; } | while IFS= read -r f; do rm -f -- "$dir/$f"; echo "$f"; done rm -f "${TMPDIR:-/tmp}/rot.$$" exit 0 ''') diff --git a/bench/local/regrade.py b/bench/local/regrade.py index fbe0cea..7eaa45d 100644 --- a/bench/local/regrade.py +++ b/bench/local/regrade.py @@ -1,32 +1,45 @@ """ -Regrade finished runs from their saved work folders with the current graders. +Regrade finished runs from their saved task folders with the current graders, on this machine. - RESULTS=results-v4.jsonl python3 regrade.py [--write] + RESULTS=results-v4.jsonl BENCH_EXPORT=~/.cache/maat-bench/container-work/ python3 regrade.py [--write] + +Folders are looked up as the runs export them: /--[-] (BENCH_EXPORT, +else BENCH_WORK). It used to look for /-- only, so after the export +moved and gained the - suffix it silently regraded nothing and reported "0 changed". +Container lanes: use regrade_today.py, which grades inside the bench image as the agent user. Exists because the graders ignored molt's state folder as `.molt/` only; after the rename to `.maat/` they failed correct work for leaving it behind. """ -import json, os, sys +import json +import os +import sys from pathlib import Path + +from grading import safe_grade from tasks import TASKS from tasks2 import TASKS2 from tasks3 import TASKS3 HERE = Path(__file__).resolve().parent out = HERE / os.environ.get("RESULTS", "results-v4.jsonl") -WORK = Path(os.environ.get("BENCH_WORK", Path.home() / ".cache/maat-bench/work")) +EXPORT = Path(os.environ.get("BENCH_EXPORT") or os.environ.get("BENCH_WORK") or Path.home() / ".cache/maat-bench/work").expanduser() by = {T.name: T for T in TASKS + TASKS2 + TASKS3} -rows = [json.loads(l) for l in out.read_text().splitlines()] -changed = 0 +rows = [json.loads(line) for line in out.read_text().splitlines() if line.strip()] +changed = found = 0 for r in rows: - d = WORK / f"{r['task']}-{r['agent']}-{r['rep']}" + tag = f"{r['task']}-{r['agent']}-{r['rep']}" + (f"-{r['arm']}" if r.get("arm") else "") + d = EXPORT / tag if not d.is_dir(): continue - ok, why = by[r["task"]].grade(d) + found += 1 + ok, why, gerr = safe_grade(by[r["task"]], d) if ok != r["passed"] or why != r["why"]: changed += ok != r["passed"] - print(f"{r['task']} {r['agent']} {r['rep']}: {r['passed']} -> {ok} ({r['why'][:60]} -> {why[:60]})") - r["passed"], r["why"] = ok, why -print(f"{changed} changed of {len(rows)}") + print(f"{tag}: {r['passed']} -> {ok} ({r['why'][:60]} -> {why[:60]})") + r["passed"], r["why"], r["grader_error"] = ok, why, gerr +print(f"{changed} changed of {found} regraded ({len(rows)} rows; {len(rows) - found} without a saved folder in {EXPORT})") +if not found: + sys.exit(f"no saved folder found under {EXPORT}") if "--write" in sys.argv: out.write_text("".join(json.dumps(r) + "\n" for r in rows)) diff --git a/bench/local/regrade_today.py b/bench/local/regrade_today.py new file mode 100644 index 0000000..313ebf5 --- /dev/null +++ b/bench/local/regrade_today.py @@ -0,0 +1,306 @@ +""" +Regrade today's finished container lanes with the fixed graders (PR #33 audit, B1 and B2). + + python3 bench/local/regrade_today.py [--date 2026-10-07] [--old ] [-j 3] [--lanes a,b] + +For every lane run today (a folder under ~/.cache/maat-bench/container-work// whose +~/.cache/maat-bench/.log names its results file) that is no longer running, every row of a +git task (B1) or of a task whose grader trusted the worker's process (B2) is graded again from the +task folder the lane saved, inside the bench image (maat-bench:agentu, network off): + + new the graders at this checkout, as the lanes grade now: the folder belongs to the + agent user and run.grade drops to it (git guard, values channel, grader_error) + old_root the graders of (default: the branch head before the fixes), as root, on the + same agent-owned folder: what the lanes graded before c6e3e56 (dubious ownership) + old_guard the same old graders as root with safe.directory set: the old grader minus B1, so a + change between old_guard and new is the B2 rewrite alone + +The originals are never modified. Each results file gets a copy, results-.regraded.jsonl, in +which regraded rows carry the new verdict and a "regrade" record (the recorded verdict, the +three re-runs, the grader hash); rows whose folder was not saved are copied unchanged with +"regrade": {"skipped": ...}. The report says, per lane, how many verdicts changed, and lists every +"verified" row that now fails and every failed row that now passes. +""" + +from __future__ import annotations + +import argparse +import collections +import concurrent.futures as cf +import datetime as dt +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +CACHE = Path.home() / ".cache/maat-bench" +WORKS = CACHE / "container-work" +RESULTS = CACHE / "container-results" +IMAGE = "maat-bench:agentu" +OLD_REV = "b6c5ddc" # claude/no-probing before the audit fixes + +GIT_TASKS = {"fix-git", "git-revert-one", "git-split-lib-history", "git-find-culprit"} +B2_TASKS = {"duration-bug", "refactor-pricing", "perf-pairs", "size-parse-bug", "refactor-invoice", + "vendor-units-workaround", "rename-api-signature", "async-pool-node"} +TASKS = GIT_TASKS | B2_TASKS + + +# ------------------------------------------------------------------ inside the container (root) +def inside(mode: str) -> None: + """mode "new": graders from /bench, run.grade as the agent user. "old": graders from /old, as root.""" + job = json.load(open("/job.json")) + if mode == "new": + sys.path.insert(0, "/bench") + import run # noqa: PLC0415 + run.AGENT_USER = "agent" + else: + sys.path.insert(0, "/old") + from tasks import TASKS as T1 # noqa: PLC0415 + from tasks2 import TASKS2 # noqa: PLC0415 + from tasks3 import TASKS3 # noqa: PLC0415 + by = {T.name: T for T in T1 + TASKS2 + TASKS3} + base = Path("/var/lib/bench-work") + base.mkdir(parents=True, exist_ok=True) + os.chmod(base, 0o711) + out = {} + + def fresh(src: str, tag: str) -> Path: + box = base / "rg" + shutil.rmtree(box, ignore_errors=True) + box.mkdir(mode=0o711) + os.chmod(box, 0o711) + d = box / tag + subprocess.run(["cp", "-a", src, str(d)], check=True) + subprocess.run(["chown", "-R", "agent:agent", str(d)], check=True) + return d + + def timed(fn): + try: + return list(fn()) + except Exception as e: # noqa: BLE001 + return [False, f"grader error: {e!r}"[:300], True] + + for j in job: + T = by[j["task"]] + res = {} + if mode == "new": + d = fresh(j["src"], j["tag"]) + res["new"] = timed(lambda: run.grade(T, d)) + else: + d = fresh(j["src"], j["tag"]) + res["old_root"] = timed(lambda: (*T.grade(d), False)) + d = fresh(j["src"], j["tag"]) + env = {"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "safe.directory", "GIT_CONFIG_VALUE_0": "*"} + os.environ.update(env) + try: + res["old_guard"] = timed(lambda: (*T.grade(d), False)) + finally: + for k in env: + os.environ.pop(k, None) + for k in ("new", "old_root", "old_guard"): + if k in res: + res[k][1] = str(res[k][1])[:300] + out[j["id"]] = res + print("@@ROW@@" + json.dumps({"id": j["id"], **res}), flush=True) + + +# ------------------------------------------------------------------ host +def running() -> set[str]: + p = subprocess.run(["docker", "ps", "--format", "{{.Names}}"], capture_output=True, text=True) + return {n[len("maat-bench-"):] for n in p.stdout.split() if n.startswith("maat-bench-")} + + +def lanes_of(day: str) -> tuple[dict[str, list[Path]], list[str]]: + """{results file name: [work dirs]} for lanes whose work dir changed on `day` and are not running.""" + live = running() + by: dict[str, list[Path]] = collections.defaultdict(list) + notes = [] + for w in sorted(WORKS.iterdir()): + if not w.is_dir() or dt.date.fromtimestamp(w.stat().st_mtime).isoformat() != day: + continue + if w.name in live: + notes.append(f"{w.name}: still running, skipped") + continue + log = CACHE / f"{w.name}.log" + m = re.search(r"results/(\S+\.jsonl)", log.read_text(errors="replace")[:4000]) if log.exists() else None + if not m: + notes.append(f"{w.name}: no results file named in {log.name}, skipped") + continue + by[m.group(1)].append(w) + # a results file shared with a lane that is still running is not finished + for name in list(by): + live_dirs = [n for n in live if (CACHE / f"{n}.log").exists() and f"results/{name}" in (CACHE / f"{n}.log").read_text(errors="replace")[:4000]] + if live_dirs: + notes.append(f"{name}: lane {', '.join(sorted(live_dirs))} still writing it, skipped") + del by[name] + return by, notes + + +def tag_of(r: dict) -> str: + return f"{r['task']}-{r['agent']}-{r['rep']}" + (f"-{r['arm']}" if r.get("arm") else "") + + +def old_tree(rev: str) -> Path: + d = Path(tempfile.mkdtemp(prefix="regrade-old-")) + repo = HERE.parents[1] + arch = subprocess.run(["git", "-C", str(repo), "archive", rev, "bench/local"], capture_output=True, check=True).stdout + subprocess.run(["tar", "-x", "-C", str(d)], input=arch, check=True) + return d / "bench" / "local" + + +def grade_lane(name: str, jobs: list[dict], dirs: list[Path], old: Path) -> dict: + with tempfile.TemporaryDirectory() as td: + jf = Path(td) / "job.json" + jf.write_text(json.dumps(jobs)) + mounts = [] + for w in dirs: + mounts += ["-v", f"{w}:/src/{w.name}:ro"] + cmd = ["docker", "run", "--rm", "--network", "none", "--memory", "4g", "--pids-limit", "1024", + "-e", "PYTHONDONTWRITEBYTECODE=1", "-v", f"{HERE}:/bench:ro", "-v", f"{old}:/old:ro", "-v", f"{jf}:/job.json:ro", + *mounts, IMAGE, "sh", "-c", + "python3 /bench/regrade_today.py --inside old; python3 /bench/regrade_today.py --inside new"] + p = subprocess.run(cmd, capture_output=True, text=True, timeout=3600 + 120 * len(jobs)) + res: dict = collections.defaultdict(dict) + for line in p.stdout.splitlines(): + if line.startswith("@@ROW@@"): + x = json.loads(line[7:]) + res[x.pop("id")].update(x) + if not res: + print(f" ! {name}: no results from the container: {(p.stderr or p.stdout)[-400:]}", flush=True) + return res + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--date", default=dt.date.today().isoformat()) + ap.add_argument("--old", default=OLD_REV, help="git revision of the old graders") + ap.add_argument("-j", type=int, default=3, help="containers at once") + ap.add_argument("--lanes", help="only these results files (comma-separated names)") + a = ap.parse_args(argv) + lanes, notes = lanes_of(a.date) + if a.lanes: + want = set(a.lanes.split(",")) + lanes = {k: v for k, v in lanes.items() if k in want} + for n in notes: + print("note:", n) + old = old_tree(a.old) + sys.path.insert(0, str(HERE)) + import run # noqa: PLC0415 + ghash = run.grader_hash() + + plan = {} + for name, dirs in sorted(lanes.items()): + src = RESULTS / name + if not src.exists(): + print(f"note: {name}: no results file, skipped") + continue + rows = [json.loads(x) for x in src.read_text().splitlines() if x.strip()] + jobs, where, dup = [], {}, 0 + for i, r in enumerate(rows): + if r.get("task") not in TASKS: + continue + cands = [w / tag_of(r) for w in dirs if (w / tag_of(r)).is_dir()] + if not cands: + where[i] = None + continue + dup += len(cands) > 1 + best = max(cands, key=lambda p: p.stat().st_mtime) # a capped run's leftover is older than the recorded one + where[i] = best + jobs.append({"id": i, "task": r["task"], "tag": tag_of(r), "src": f"/src/{best.parent.name}/{best.name}"}) + plan[name] = (rows, jobs, where, dirs, dup) + + print(f"{len(plan)} lanes, {sum(len(p[1]) for p in plan.values())} rows to regrade (old graders: {a.old}, new grader hash {ghash})", flush=True) + results = {} + with cf.ThreadPoolExecutor(a.j) as ex: + futs = {ex.submit(grade_lane, name, jobs, dirs, old): name for name, (rows, jobs, where, dirs, dup) in plan.items() if jobs} + for f in cf.as_completed(futs): + results[futs[f]] = f.result() + print(f" graded {futs[f]}", flush=True) + shutil.rmtree(old.parents[1], ignore_errors=True) + + table = [] + flips = [] + for name, (rows, jobs, where, dirs, dup) in plan.items(): + res = results.get(name, {}) + out_rows = [] + c = collections.Counter() + for i, r in enumerate(rows): + r = dict(r) + if i in where and where[i] is None: + r["regrade"] = {"skipped": "no saved task folder"} + c["missing"] += 1 + elif i in where: + g = res.get(i) or res.get(str(i)) + if not g or "new" not in g: + r["regrade"] = {"skipped": "the container gave no verdict"} + c["missing"] += 1 + else: + new_ok, new_why, gerr = g["new"] + rec = {"recorded": [r.get("passed"), r.get("why")], "new": g["new"], "old_root": g.get("old_root"), + "old_guard": g.get("old_guard"), "grader": ghash, "folder": str(where[i])} + c["regraded"] += 1 + verified = str(r.get("claim") or "").startswith("verified") + if bool(new_ok) != bool(r.get("passed")): + c["changed"] += 1 + kind = "fail->pass" if new_ok else "pass->fail" + c[kind] += 1 + cause = [] + og = g.get("old_guard") or [None] + orr = g.get("old_root") or [None] + if orr[0] is not None and bool(orr[0]) != bool(og[0]): + cause.append("B1 ownership") + if og[0] is not None and bool(og[0]) != bool(new_ok): + cause.append("B2 grader") + if orr[0] is not None and bool(orr[0]) != bool(r.get("passed")): + cause.append("old grader no longer reproduces the row") + flips.append((name, r["task"], tag_of(r), r.get("claim"), r.get("passed"), r.get("why"), new_ok, new_why, cause)) + if verified and not new_ok: + c["verified_now_wrong"] += 1 + if verified and new_ok: + c["false_done_now_right"] += 1 + if not verified and new_ok: + c["failed_now_right"] += 1 + if gerr: + c["grader_error"] += 1 + r["passed"], r["why"], r["grader_error"] = bool(new_ok), new_why, bool(gerr) + r["regrade"] = rec + out_rows.append(r) + dst = RESULTS / name.replace(".jsonl", ".regraded.jsonl") + dst.write_text("".join(json.dumps(x) + "\n" for x in out_rows)) + p0 = sum(1 for x in rows if x.get("passed")) + p1 = sum(1 for x in out_rows if x.get("passed")) + table.append((name, len(rows), c["regraded"], c["missing"], dup, c["changed"], c["fail->pass"], c["pass->fail"], + c["verified_now_wrong"], c["false_done_now_right"], c["failed_now_right"], c["grader_error"], p0, p1)) + + print("\n| lane (results file) | rows | regraded | no folder | dup folders | changed | fail->pass | pass->fail | " + "verified now wrong | false-done now right | failed (unverified) now right | grader_error | pass before -> after |") + print("|---|---|---|---|---|---|---|---|---|---|---|---|---|") + for t in table: + print(f"| {t[0]} | {t[1]} | {t[2]} | {t[3]} | {t[4]} | {t[5]} | {t[6]} | {t[7]} | {t[8]} | {t[9]} | {t[10]} | {t[11]} | {t[12]} -> {t[13]} |") + tot = [sum(t[k] for t in table) for k in range(1, 14)] + print(f"| **all** | {' | '.join(map(str, tot[:11]))} | {tot[11]} -> {tot[12]} |") + allg = [r["regrade"] for name in plan for r in map(json.loads, (RESULTS / name.replace(".jsonl", ".regraded.jsonl")).read_text().splitlines()) + if isinstance(r.get("regrade"), dict) and "new" in r["regrade"]] + same_b2 = sum(1 for g in allg if g["old_guard"] and g["old_guard"][0] == g["new"][0]) + repro = sum(1 for g in allg if g["old_root"] and g["old_root"][0] == g["recorded"][0]) + print(f"\nConsistency: old graders with only the ownership fix agree with the new graders on {same_b2}/{len(allg)} rows " + f"(a difference would be the B2 rewrite); the old graders as root on an agent-owned copy reproduce the recorded " + f"verdict on {repro}/{len(allg)} (the rest come from lanes that did not hand the folder to the agent user before " + f"root graded it, so B1 never hit them).") + if flips: + print("\nChanged verdicts:") + for name, task, tag, claim, was, why, now, nwhy, cause in flips: + print(f"- {name} {tag}: {was} -> {now} [{', '.join(cause) or '?'}]; claim {claim!r}\n was: {str(why)[:110]}\n now: {str(nwhy)[:110]}") + return 0 + + +if __name__ == "__main__": + if len(sys.argv) > 2 and sys.argv[1] == "--inside": + inside(sys.argv[2]) + else: + sys.exit(main(sys.argv[1:])) diff --git a/bench/local/run.py b/bench/local/run.py index 8be41cd..9c699eb 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -3,9 +3,15 @@ python3 run.py molt [repeats] [task,task] -Paired arms: ARMS="base:REFERENCE=0;ref:REFERENCE=1" (or JSON) runs, per task, -arm A then arm B before the next task; rows carry "arm"; resume is by -(task, agent, rep, arm). BENCH_TASKS=a,b restricts the task set. +Paired arms: ARMS="base:REFERENCE=0;ref:REFERENCE=1" (or JSON) runs both arms on a +task before the next task, in ABBA order (rows carry "arm" and "arm_pos"); resume +is by (task, agent, rep, arm). BENCH_TASKS=a,b restricts the task set. + +Every row carries its lane (model, url, reasoning, limit, gate, judge and Maat +settings per arm, build, grader hash) and lane_id; a results file holding another +lane's rows is refused. A run the provider's cap stops prints STOPPED and exits 3. +In the containers nothing the agent user leaves in /tmp, /var/tmp, /dev/shm or its +home survives into the next run (Scrubber). Only molt is run here. Other vendors' agents are not benchmarked from this repository: their terms commonly forbid benchmarking and publishing @@ -17,17 +23,29 @@ import json import os import shutil +import stat import subprocess import sys import time from pathlib import Path +from grading import links_into, safe_grade from tasks import TASKS HERE = Path(__file__).resolve().parent # Outside every git repository: a task folder inside this one let agents' # `git commit` walk up and commit task files into molt-desktop's main. WORK = Path(os.environ.get("BENCH_WORK", Path.home() / ".cache/maat-bench/work")) +# BENCH_EXPORT: where each finished task's folder and logs are copied, right after it is graded. +# Unset: WORK itself. The task RUNS somewhere else (private_dir below), never in the export. +# +# 2026-10-07 (ml4b lanes): a worker spent 90-125 tool calls per task reading the other tasks' +# folders and logs in the shared /work instead of working. /work is a host bind mount, and on +# OrbStack and Docker Desktop a bind mount ignores chown and does not enforce modes for other +# users, so nothing there can be locked away from the agent user. The containers therefore mount +# the export (and the results) under /root (700, container-local, enforced), run each task in a +# private folder under BENCH_WORK, and copy it out once it is finished and the agent is gone. +EXPORT = Path(os.environ.get("BENCH_EXPORT") or WORK) # MOLT_DIST_ABS: an absolute path to the built Maat (the container mounts it at /maat). MOLT = (Path(os.environ["MOLT_DIST_ABS"]) if os.environ.get("MOLT_DIST_ABS") else Path.home() / os.environ.get("MOLT_DIST", "Documents/molt-desktop/dist-compare")) / "cli.js" LIMIT = int(os.environ.get("BENCH_LIMIT", "600")) # seconds per task @@ -78,17 +96,308 @@ def kill_tree(proc: subprocess.Popen) -> None: subprocess.run(["pkill", "-KILL", "-u", AGENT_USER], check=False) +def kill_agent_procs() -> None: + """In a container, end every process of the agent user. Before grading, so nothing the worker + left running (a server on the port the grader wants, a loop rewriting outputs) is still there + while the work is judged; and again when the task is finished.""" + if AGENT_USER and shutil.which("pkill"): + subprocess.run(["pkill", "-KILL", "-u", AGENT_USER], check=False) + + +# Environment variables a grader's children never see: the worker's code runs in them. +SECRET_ENV = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL", "AUTH") +GRADE_LIMIT = int(os.environ.get("BENCH_GRADE_LIMIT", "900")) # seconds, the whole grade of one task + + +def grade(T, d: Path) -> tuple[bool, str, bool]: + """ + (passed, why, grader_error): T.grade(d) through grading.safe_grade, and with BENCH_AGENT_USER + set, as that user. + + A grader that raises is a failed row with grader_error set, never an abort: the lane used to + stop, and on resume the (task, rep) ran again, a free retry for exactly the runs that broke a + grader. A symlink that resolves into the bench's own folders fails the task before any grader + can follow it (the agent cannot read them; root, or a host lane's owner, can). + + Graders run the worker's code: `python3 server.py`, `python3 migrate.py`, `python3 -c 'import + the_module'`. Run as root, that code could append a passing row to the results, edit the task + modules the next tasks are graded by, or read reference_solutions/. So the grade runs in a + forked child that has dropped to the agent user (everything it runs inherits that), with the + secrets stripped from its environment. The child is non-dumpable after the drop, so the + worker's code, though it runs as the same user, cannot attach to it or read its pipe; the + verdict comes back to root over that pipe, and root alone writes the results. Code that kills + the grader, hangs past GRADE_LIMIT or answers with anything but a verdict fails the task. + Mac host lanes (no agent user): in this process, as before. + """ + bad = links_into(d, protected_roots()) + if bad: + return False, f"symlink into the bench's own folders: {bad[:3]}", False + if not AGENT_USER: + return safe_grade(T, d) + import pwd + import select + import signal + pw = pwd.getpwnam(AGENT_USER) + r, w = os.pipe() + pid = os.fork() + if pid == 0: # the grader + code = 0 + try: + os.close(r) + os.setgroups([]) + os.setgid(pw.pw_gid) + os.setuid(pw.pw_uid) + try: + import ctypes + ctypes.CDLL(None, use_errno=True).prctl(4, 0, 0, 0, 0) # PR_SET_DUMPABLE 0 (also the default after setuid) + except Exception: + pass + for k in list(os.environ): + if any(s in k.upper() for s in SECRET_ENV): + del os.environ[k] + os.environ["HOME"] = pw.pw_dir + os.chdir(d) + data = json.dumps(list(safe_grade(T, d))) + except BaseException as e: # noqa: BLE001 - every failure is a verdict + data = json.dumps([False, f"grader error: {e!r}"[:2000], True]) + code = 1 + try: + os.write(w, data.encode()) + finally: + os._exit(code) + os.close(w) + chunks: list[bytes] = [] + deadline = time.time() + GRADE_LIMIT + timed_out = False + try: + while True: + left = deadline - time.time() + if left <= 0: + timed_out = True + break + ready, _, _ = select.select([r], [], [], left) + if not ready: + continue + b = os.read(r, 65536) + if not b: + break + chunks.append(b) + finally: + os.close(r) + if timed_out: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + kill_agent_procs() # whatever the worker's code started while it was being graded + _, status = os.waitpid(pid, 0) + if timed_out: + return False, f"grader timed out after {GRADE_LIMIT}s", False + try: + ok, why, err = json.loads(b"".join(chunks).decode()) + return bool(ok), str(why), bool(err) + except (ValueError, TypeError): + # The worker's code, run by the grader as the same user, can kill it: a fail, not a grader error. + return False, f"grader ended without a verdict (wait status {status})", False + + +def protected_roots() -> list[Path]: + """Where a task folder's symlinks must not lead: the graders and reference solutions, the + results, the export and the other tasks' work.""" + return [HERE, Path(os.environ.get("RESULTS_DIR", HERE)), EXPORT, WORK] + + +def _copy_regular(src: str, dst: str, *, follow_symlinks: bool = True) -> str: + """copytree's copy_function: regular files only. A FIFO or a socket a worker left in its folder + (a server's .sock, or `mkfifo x` on purpose) made copytree raise, and the lane stopped.""" + if stat.S_ISREG(os.lstat(src).st_mode): + return shutil.copy2(src, dst, follow_symlinks=follow_symlinks) + return dst + + +class Scrubber: + """ + Nothing the agent user leaves outside its task folder reaches the next run (B3 of the PR #33 + audit). After every task, and so after every arm and every repeat: + + - every entry the agent user owns in /tmp, /var/tmp and /dev/shm is removed (Maat's own + maat-check-*/ref-* copies of a SIGKILLed run included), looking inside world-writable folders + only, the only ones it could create anything in; + - its home is put back exactly as it was before the first task, from a root-only snapshot, so + notes it left there go and the credentials it was given stay. + + It used to be one long-lived machine: rep 1 read the notes rep 0 left in /tmp and ~, and arm B + of a pair could read arm A's solution. + """ + + DIRS = (Path("/tmp"), Path("/var/tmp"), Path("/dev/shm")) + + def __init__(self, uid: int, home: Path, snapshot: Path, dirs=DIRS): + self.uid, self.home, self.snap, self.dirs = uid, Path(home), Path(snapshot), [Path(x) for x in dirs] + + def take(self) -> None: + shutil.rmtree(self.snap, ignore_errors=True) + self.snap.parent.mkdir(parents=True, exist_ok=True) + os.chmod(self.snap.parent, 0o700) + subprocess.run(["cp", "-a", str(self.home), str(self.snap)], check=True) + + def scrub(self) -> list[str]: + removed: list[str] = [] + + def walk(p: str) -> None: + try: + entries = list(os.scandir(p)) + except OSError: + return + for e in entries: + try: + st = e.stat(follow_symlinks=False) + except OSError: + continue + if st.st_uid == self.uid: + try: + if stat.S_ISDIR(st.st_mode): + shutil.rmtree(e.path) + else: + os.unlink(e.path) + removed.append(e.path) + except OSError as err: + print(f"scrub: could not remove {e.path}: {err}", file=sys.stderr, flush=True) + elif stat.S_ISDIR(st.st_mode) and st.st_mode & 0o002: + walk(e.path) + + for d in self.dirs: + if d.is_dir(): + walk(str(d)) + return removed + + def restore_home(self) -> None: + for e in list(os.scandir(self.home)): + if e.is_dir(follow_symlinks=False): + shutil.rmtree(e.path) + else: + os.unlink(e.path) + subprocess.run(["cp", "-a", f"{self.snap}/.", f"{self.home}/"], check=True) + st = os.stat(self.snap) + os.chmod(self.home, stat.S_IMODE(st.st_mode)) + if os.geteuid() == 0: + os.chown(self.home, st.st_uid, st.st_gid) + + def reset(self) -> None: + """Never raises: reported, so a lane is not stopped by a file it could not remove.""" + try: + self.scrub() + self.restore_home() + except (OSError, subprocess.CalledProcessError) as e: + print(f"scrub after the task incomplete: {e}", file=sys.stderr, flush=True) + + +SCRUBBER: Scrubber | None = None + + +def make_scrubber() -> Scrubber | None: + """In the containers (root, with an agent user): the scrubber, its snapshot taken now.""" + if not AGENT_USER or os.geteuid() != 0: + return None + import pwd + pw = pwd.getpwnam(AGENT_USER) + sc = Scrubber(pw.pw_uid, Path(pw.pw_dir), WORK / ".home-snapshot" / "home") + sc.take() + return sc + + def hand_over(d: Path) -> None: if AGENT_USER: subprocess.run(["chown", "-R", AGENT_USER, str(d)], check=True) -def provider_capped(out: str, steps: int) -> bool: +def private_dir(tag: str) -> Path: + """ + A fresh folder for one task: BENCH_WORK//. + + BENCH_WORK is 711 (made so here when we own it): the agent can reach its own task by the + exact path it is given, but cannot list BENCH_WORK, and the random parent cannot be guessed, + so no task can find another's folder. Finished tasks are removed (finish_task), so there is + nothing left to find anyway. On a Mac host lane (no agent user) the layout is the same; the + agent then runs as the owner, so only the names and the removal keep tasks apart. + """ + import secrets + WORK.mkdir(parents=True, exist_ok=True) + if AGENT_USER: + os.chmod(WORK, 0o711) + box = WORK / secrets.token_hex(8) + box.mkdir(mode=0o711) + os.chmod(box, 0o711) # mkdir's mode is masked by the umask + d = box / tag + d.mkdir() + return d + + +def log_dir() -> Path: + """Where Maat's stdout/stderr are written while a run is live: root-only (700), never in a task folder.""" + logs = WORK / ".logs" + logs.mkdir(parents=True, exist_ok=True) + os.chmod(logs, 0o700) + return logs + + +def finish_task(d: Path, logs: list[Path]) -> None: + """ + The task is graded and its agent is gone: copy the folder and its logs to EXPORT, then remove + the private copies. In a container, every process of the agent user is killed first: a server + a worker left running would otherwise still be there, as that user, during the next task. + + Never raises: the export is for the owner to look at, and a failure to copy it (a special + file, a full disk) is reported, not allowed to stop the lane or leave the private folder. + """ + kill_agent_procs() + try: + EXPORT.mkdir(parents=True, exist_ok=True) + dst = EXPORT / d.name + if dst.resolve() != d.resolve(): + shutil.rmtree(dst, ignore_errors=True) + shutil.copytree(d, dst, symlinks=True, copy_function=_copy_regular) + except (OSError, shutil.Error) as e: + print(f"export of {d.name} incomplete: {e}", file=sys.stderr, flush=True) + for f in logs: + try: + if f.exists() and (EXPORT / f.name).resolve() != f.resolve(): + shutil.copy2(f, EXPORT / f.name) + f.unlink() + except OSError as e: + print(f"export of {f.name} failed: {e}", file=sys.stderr, flush=True) + shutil.rmtree(d.parent, ignore_errors=True) + if SCRUBBER is not None: + SCRUBBER.reset() + + +def maat_said(out: str, err: str = "") -> str: + """ + What Maat itself reported: its `error` events, any stdout line that is not a JSON event, and + stderr. Never a tool event's preview or args, nor model text: the worker's own output (`cat + app.log`, an echo, a source file that quotes the phrase) is in those, and it used to stop the + lane as "the provider's cap" (B5 of the PR #33 audit). + """ + keep = [err or ""] + for line in (out or "").splitlines(): + try: + ev = json.loads(line) if line.startswith("{") else None + except json.JSONDecodeError: + ev = None + if not isinstance(ev, dict): + keep.append(line) + elif ev.get("kind") == "error": + keep.append(str(ev.get("text", "")) + " " + str(ev.get("message", ""))) + return "\n".join(keep) + + +def provider_capped(out: str, steps: int, err: str = "") -> bool: """The provider's cap (OpenRouter daily limit, OpenCode free-usage limit), not the work.""" + said = maat_said(out, err) return ( - "rate limit is reached until" in out - or ("free-models-per-day" in out and steps == 0) - or ("OpenCode rate limit" in out and steps == 0) + "rate limit is reached until" in said + or ("free-models-per-day" in said and steps == 0) + or ("OpenCode rate limit" in said and steps == 0) ) @@ -113,7 +422,9 @@ def claim_of(ev: dict) -> str | None: def run_molt(d: Path, prompt: str, log: Path) -> dict: - key = openrouter_key() if "openrouter.ai" in URL else "local" + key = (openrouter_key() if "openrouter.ai" in URL + else os.environ.get("ANTHROPIC_API_KEY", "") if "api.anthropic.com" in URL + else "local") env = os.environ | {"MOLT_API_KEY": key, "MOLT_JUDGMENT": "0"} # nobody rules on a benchmark run cmd = [ "node", str(MOLT), "run", "--url", URL, "--model", os.environ.get("BENCH_MODEL") or MODEL, @@ -164,7 +475,7 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: extra = {k: ev[k] for k in ("revealed", "deadline", "endedBy", "retired", "build", "tier", "tierReason", "providerStall", "checkAuthors") if k in ev} # The provider's daily cap, not the work: every later task would fail the # same way (2026-10-05: eleven tasks per arm "failed" in 140 s, 0 turns). - capped = provider_capped(out, steps) + capped = provider_capped(out, steps, err or "") return { "provider_capped": capped, "secs": round(secs), "turns": steps, "calls": sum(per), "multi": sum(1 for x in per if x > 1), @@ -216,6 +527,94 @@ def parse_arms(spec: str | None) -> list[tuple[str | None, dict]]: return arms +STOPPED_EXIT = 3 # the provider's cap stopped the lane before it finished + +GRADER_FILES = ("run.py", "grading.py", "tasks.py", "tasks2.py", "tasks3.py") + + +def grader_hash() -> str: + """sha256 (12 hex) of the harness and every grader: rows graded by different code differ.""" + import hashlib + h = hashlib.sha256() + for f in GRADER_FILES: + h.update(f.encode() + b"\0" + (HERE / f).read_bytes() + b"\0") + return h.hexdigest()[:12] + + +def _public_env(env: dict) -> dict: + return {k: v for k, v in sorted(env.items()) if not any(x in k.upper() for x in SECRET_ENV)} + + +def lane_meta(arms: list[tuple[str | None, dict]]) -> dict: + """ + Everything that makes a lane this lane, on every row (B6 of the PR #33 audit): rows did not + say which model, endpoint, reasoning, limit, gate, judge or grader produced them, and a second + lane resumed into the same file as if it were the first. + """ + from urllib.parse import urlsplit, urlunsplit + u = urlsplit(URL) + url = urlunsplit((u.scheme, u.hostname + (f":{u.port}" if u.port else "") if u.hostname else u.netloc, u.path, "", "")) + return { + "model": os.environ.get("BENCH_MODEL") or MODEL, + "url": url, + "reasoning": os.environ.get("BENCH_REASONING") or "low", + "limit": LIMIT, + "gate": "yes" if os.environ.get("BENCH_GATE") == "yes" else "sandbox", + "reference": os.environ.get("REFERENCE", "0"), + # judge (MAAT_JUDGE_*) and every other Maat setting the lane runs with, secrets left out + "maat_env": _public_env({k: v for k, v in os.environ.items() if k.startswith("MAAT_") and k != "MAAT_BUILD"}), + "arms": {name or "": _public_env(env) for name, env in arms}, + "build": os.environ.get("MAAT_BUILD"), + "grader": grader_hash(), + "agent_user": AGENT_USER, + "cpus": os.environ.get("BENCH_CPUS"), + "memory": os.environ.get("BENCH_MEMORY"), + } + + +def lane_id(meta: dict) -> str: + import hashlib + return hashlib.sha256(json.dumps(meta, sort_keys=True).encode()).hexdigest()[:12] + + +def read_results(out: Path) -> list[dict]: + """ + The rows already in a results file. A last line cut short (the lane was killed while writing + it) is moved to .partial and cut from the file, so the resume neither crashes on it nor + glues the next row onto it. A broken line anywhere else is real damage and stops the run. + """ + if not out.exists(): + return [] + raw = out.read_bytes() + lines = raw.split(b"\n") + tail = lines.pop() # b"" when the file ends with a newline + rows = [] + for i, line in enumerate(lines): + if not line.strip(): + continue + try: + rows.append(json.loads(line)) + except ValueError: + sys.exit(f"{out}: line {i + 1} is not JSON; not resuming into a damaged results file") + if tail.strip(): + try: + rows.append(json.loads(tail)) + with out.open("ab") as f: + f.write(b"\n") + except ValueError: + with out.with_name(out.name + ".partial").open("ab") as f: + f.write(tail + b"\n") + with out.open("r+b") as f: + f.truncate(len(raw) - len(tail)) + print(f"{out}: the last line was cut short; moved to {out.name}.partial", file=sys.stderr, flush=True) + return rows + + +def arm_order(arms: list, rep: int, task_index: int) -> list: + """ABBA: arm order alternates per (task, rep), so neither arm always runs first.""" + return arms if len(arms) < 2 or (rep + task_index) % 2 == 0 else arms[::-1] + + def main(which: str, repeats: int, task_filter: str | None) -> None: from tasks2 import TASKS2 # noqa: PLC0415 from tasks3 import TASKS3 # noqa: PLC0415 @@ -229,28 +628,43 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: want = {x.strip() for x in os.environ["BENCH_TASKS"].split(",") if x.strip()} tasks = [T for T in tasks if T.name in want] arms = parse_arms(os.environ.get("ARMS")) + if not AGENT_USER: + print("WARNING: no BENCH_AGENT_USER: the worker runs as this user, so it can read the graders, " + "the export and earlier tasks' logs, and graders run its code with this user's rights. " + "Use the container lanes for numbers that matter.", file=sys.stderr, flush=True) out = Path(os.environ.get("RESULTS_DIR", HERE)) / os.environ.get("RESULTS", f"results-{which}-x{repeats}.jsonl") + out.parent.mkdir(parents=True, exist_ok=True) + meta = lane_meta(arms) + lid = lane_id(meta) done = set() - if out.exists(): # resume: skip runs already recorded - for line in out.read_text().splitlines(): - r = json.loads(line) - done.add((r["task"], r["agent"], r["rep"], r.get("arm"))) + rows = read_results(out) # resume: skip runs already recorded + other = sorted({str(r.get("lane_id")) for r in rows if r.get("lane_id") != lid}) + if other: + sys.exit(f"{out} holds rows of another lane (lane_id {', '.join(other)}; this lane is {lid}): " + "not appending to it. Use another RESULTS file.") + for r in rows: + done.add((r["task"], r["agent"], r["rep"], r.get("arm"))) + global SCRUBBER + SCRUBBER = make_scrubber() for rep in range(repeats): - for T in tasks: - for arm, arm_env in arms: # paired: arm A then arm B on this task before the next task + for ti, T in enumerate(tasks): + # paired: both arms on this task before the next task, in ABBA order + for pos, (arm, arm_env) in enumerate(arm_order(arms, rep, ti)): for a in chosen: if (T.name, a, rep, arm) in done: continue tag = f"{T.name}-{a}-{rep}" + (f"-{arm}" if arm else "") - d = WORK / tag - shutil.rmtree(d, ignore_errors=True) - d.mkdir(parents=True) + d = private_dir(tag) + log = log_dir() / f"{tag}.log" T.setup(d) hand_over(d) saved = {k: os.environ.get(k) for k in arm_env} os.environ.update(arm_env) try: - r = agents[a](d, T.PROMPT, WORK / f"{tag}.log") + r = agents[a](d, T.PROMPT, log) + except BaseException: + finish_task(d, [log, log.with_suffix(".err")]) + raise finally: for k, v in saved.items(): if v is None: @@ -258,18 +672,30 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: else: os.environ[k] = v if r.get("provider_capped"): - # Not recorded, so a resume runs this task again. + finish_task(d, [log, log.with_suffix(".err")]) + # Not recorded, so a resume runs this task again. Non-zero: the lane is not done. print(f"STOPPED: the provider's daily limit / quota is reached ({tag} not recorded)", flush=True) - return - ok, why = T.grade(d) + sys.exit(STOPPED_EXIT) + # Nothing the worker started is still running while it is graded. + kill_agent_procs() + try: + ok, why, gerr = grade(T, d) + except BaseException: + finish_task(d, [log, log.with_suffix(".err")]) + raise final = "" - r.update(task=T.name, agent=a, rep=rep, passed=ok, why=why, final=final[:300]) + r.update(task=T.name, agent=a, rep=rep, passed=ok, why=why, final=final[:300], grader_error=gerr) if arm: r["arm"] = arm + r["arm_pos"] = pos + r["lane_id"] = lid + r["lane"] = meta if os.environ.get("MAAT_BUILD"): r["build"] = os.environ["MAAT_BUILD"] # sha8 of the packed Maat the container ran + # The result is written before the export, so nothing in the export can lose it. with out.open("a") as f: f.write(json.dumps(r) + "\n") + finish_task(d, [log, log.with_suffix(".err")]) print(json.dumps({k: r[k] for k in ("task", "agent", "arm", "rep", "passed", "said_done", "turns", "secs") if k in r}), flush=True) diff --git a/bench/local/scoreboard.py b/bench/local/scoreboard.py index e477a4d..cf38472 100644 --- a/bench/local/scoreboard.py +++ b/bench/local/scoreboard.py @@ -30,7 +30,8 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -from stats import TIERS, is_verified, no_verdict, percentile, rate, sign_test, tier_breakdown # noqa: E402 +from stats import (TIERS, excluded, is_verified, no_verdict, paired_tasks, percentile, rate, sign_test, # noqa: E402 + task_rate, tier_breakdown) LOW_POWER_N = 30 # pairs below this cannot separate a few-task difference from noise @@ -45,6 +46,9 @@ def load(path: str | Path) -> list[dict]: def metrics(rows: list[dict]) -> dict: + # A grader error or a provider stall is neither a pass nor a fail: counted apart, left out of n. + skipped = [r for r in rows if excluded(r)] + rows = [r for r in rows if not excluded(r)] n = len(rows) passed = [r for r in rows if r.get("passed")] ver = [r for r in rows if is_verified(r.get("claim"))] @@ -63,6 +67,9 @@ def metrics(rows: list[dict]) -> dict: "secs_med": percentile(secs, 50), "secs_p90": percentile(secs, 90), "tokens_med": percentile(toks, 50), + "grader_error": sum(1 for r in skipped if excluded(r) == "grader_error"), + "stall": sum(1 for r in skipped if excluded(r) == "stall"), + "task_rate": task_rate(rows), } @@ -86,7 +93,10 @@ def tier_lines(m: dict) -> list[str]: def report_arm(label: str, m: dict) -> list[str]: return [ f"== {label} (n={m['n']})", - f" pass rate {rate(m['pass'], m['n'])}", + f" pass rate {rate(m['pass'], m['n'])} (rows; repeats are not independent)", + *([f" pass rate by task {100 * m['task_rate']['mean']:.0f}% [{100 * m['task_rate']['lo']:.0f}-{100 * m['task_rate']['hi']:.0f}]" + f" over {m['task_rate']['tasks']} tasks (bootstrap over tasks)"] if m.get("task_rate") else []), + *([f" not counted grader_error {m['grader_error']}, provider stall {m['stall']}"] if m.get("grader_error") or m.get("stall") else []), f" verified precision {rate(m['verified_pass'], m['verified'])} P(pass | verified)", f" verified recall {rate(m['verified_pass'], m['pass'])} P(verified | pass)", f" false-done {m['false_done']}", @@ -98,10 +108,23 @@ def report_arm(label: str, m: dict) -> list[str]: ] +class Ambiguous(ValueError): + """Pooled files repeat a (task, agent, rep): which A row pairs with which B row is unknown.""" + + def pair_up(a: list[dict], b: list[dict]) -> list[tuple[dict, dict]]: + """ + (A, B) rows of the same (task, agent, rep), and the same lane when both rows record one. + Every run file starts at rep 0, so pooling two files under one label repeats keys: that used + to pair each B row twice (20 pairs from 10). Repeated keys now raise Ambiguous. + """ key = lambda r: (r.get("task"), r.get("agent"), r.get("rep", 0)) - bi = {key(r): r for r in b} - return [(r, bi[key(r)]) for r in a if key(r) in bi] + for side, rows in (("A", a), ("B", b)): + ks = [key(r) for r in rows if not excluded(r)] + if len(ks) != len(set(ks)): + raise Ambiguous(f"arm {side} repeats a (task, agent, rep): pooled runs cannot be paired row by row") + bi = {key(r): r for r in b if not excluded(r)} + return [(r, bi[key(r)]) for r in a if not excluded(r) and key(r) in bi] TARGETS = ("pass", "precision", "recall", "verified", "secs", "tokens", "no_verdict", "false_done") @@ -125,8 +148,16 @@ def target_gain(target: str, ma: dict, mb: dict) -> float | None: def compare(la: str, lb: str, ra: list[dict], rb: list[dict], target: str, min_gain: float) -> tuple[list[str], str]: - pairs = pair_up(ra, rb) out = [f"== paired: {la} (A) vs {lb} (B)"] + pt = paired_tasks(ra, rb) + if pt: + out.append(f" by task ({pt['tasks']} tasks, repeats averaged): B-A {100 * pt['mean_diff']:+.1f} points " + f"[{100 * pt['lo']:+.1f}, {100 * pt['hi']:+.1f}] (bootstrap over tasks); tasks B better {pt['b_better']}, " + f"A better {pt['a_better']}; sign test p = {pt['sign_p']:.3f}; permutation p = {pt['perm_p']:.3f}") + try: + pairs = pair_up(ra, rb) + except Ambiguous as e: + return out + [f" {e}; only the by-task comparison above applies"], f"VERDICT {lb} vs {la}: NO VERDICT (pooled runs repeat keys)" n = len(pairs) if not n: return out + [" no (task, rep) in both arms"], f"VERDICT {lb} vs {la}: NO VERDICT (no shared tasks)" diff --git a/bench/local/stats.py b/bench/local/stats.py index 2969f9f..a9364cb 100644 --- a/bench/local/stats.py +++ b/bench/local/stats.py @@ -116,3 +116,92 @@ def tier_breakdown(rows: list[dict]) -> dict[str, tuple[int, int]]: n, k = out.get(t, (0, 0)) out[t] = (n + 1, k + (1 if r.get("passed") else 0)) return out + + +# ---------------------------------------------------------------- the task as the unit of analysis +# +# Repeats of one task are correlated (same prompt, same grader, often the same mistake), so treating +# every (task, rep) row as an independent trial overstates confidence: Wilson intervals and sign +# tests over rows are too narrow. The functions below average the repeats within each task first and +# then resample or permute over TASKS. Rows a grader could not judge (grader_error) and runs the +# provider stalled (providerStall) are neither a pass nor a fail and are left out; count them with +# excluded(). + +def excluded(row: dict) -> str | None: + """Why a row is not a pass/fail observation: "grader_error", "stall", or None (it counts).""" + if row.get("grader_error"): + return "grader_error" + if row.get("providerStall"): + return "stall" + return None + + +def per_task(rows: list[dict]) -> dict[str, tuple[int, int]]: + """{task: (passes, runs)} over the rows that count.""" + out: dict[str, tuple[int, int]] = {} + for r in rows: + if excluded(r): + continue + k, n = out.get(r.get("task"), (0, 0)) + out[r.get("task")] = (k + (1 if r.get("passed") else 0), n + 1) + return out + + +def _bootstrap_mean(xs: list[float], b: int, seed: int) -> tuple[float, float] | None: + import random + if not xs: + return None + rnd = random.Random(seed) + n = len(xs) + means = sorted(sum(xs[rnd.randrange(n)] for _ in range(n)) / n for _ in range(b)) + return means[int(0.025 * b)], means[min(b - 1, int(0.975 * b))] + + +def task_rate(rows: list[dict], b: int = 2000, seed: int = 0) -> dict | None: + """ + Mean over tasks of each task's pass fraction, with a 95% percentile-bootstrap interval that + resamples tasks (not rows). None with no countable rows. + """ + pt = per_task(rows) + if not pt: + return None + fr = [k / n for k, n in pt.values()] + lo, hi = _bootstrap_mean(fr, b, seed) + return {"tasks": len(fr), "mean": sum(fr) / len(fr), "lo": lo, "hi": hi} + + +def sign_flip_p(diffs: list[float], b: int = 20000, seed: int = 0) -> float: + """Two-sided paired permutation (sign-flip) test of mean(diffs) == 0; exact up to 16 nonzero + differences, Monte Carlo (seeded) above.""" + import itertools + import random + d = [x for x in diffs if x != 0] + if not d: + return 1.0 + obs = abs(sum(d)) + eps = 1e-12 + if len(d) <= 16: + hits = sum(1 for s in itertools.product((1, -1), repeat=len(d)) if abs(sum(x * y for x, y in zip(d, s))) >= obs - eps) + return hits / 2 ** len(d) + rnd = random.Random(seed) + hits = sum(1 for _ in range(b) if abs(sum(x if rnd.random() < 0.5 else -x for x in d)) >= obs - eps) + return (hits + 1) / (b + 1) + + +def paired_tasks(a: list[dict], b: list[dict], boot: int = 2000, seed: int = 0) -> dict | None: + """ + A/B on the tasks both sides ran: per task, B's pass fraction minus A's. The mean difference + with a bootstrap 95% interval over tasks, a sign test on the tasks that moved, and a sign-flip + permutation p on the differences. None when no task is shared. + """ + pa, pb = per_task(a), per_task(b) + shared = sorted(set(pa) & set(pb), key=str) + if not shared: + return None + diffs = [pb[t][0] / pb[t][1] - pa[t][0] / pa[t][1] for t in shared] + lo, hi = _bootstrap_mean(diffs, boot, seed) + up = sum(1 for x in diffs if x > 0) + down = sum(1 for x in diffs if x < 0) + return {"tasks": len(shared), "mean_diff": sum(diffs) / len(diffs), "lo": lo, "hi": hi, + "b_better": up, "a_better": down, "sign_p": sign_test(up, down), "perm_p": sign_flip_p(diffs, seed=seed), + "diffs": dict(zip(shared, diffs))} diff --git a/bench/local/tasks.py b/bench/local/tasks.py index 65d9c2d..bf87d9f 100644 --- a/bench/local/tasks.py +++ b/bench/local/tasks.py @@ -14,12 +14,19 @@ import os import random import re +import shutil import subprocess +import sys +import tempfile from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from grading import py_values # noqa: E402 -def sh(cmd: str, cwd: Path) -> str: - return subprocess.run(cmd, shell=True, cwd=cwd, capture_output=True, text=True).stdout + +def sh(cmd: str, cwd: Path, timeout: float = 120) -> str: + # errors="replace": one non-UTF-8 byte in a deliverable's output made graders raise. + return subprocess.run(cmd, shell=True, cwd=cwd, capture_output=True, text=True, errors="replace", timeout=timeout).stdout # ---------------------------------------------------------------- 1. lost git work @@ -175,20 +182,27 @@ def setup(d: Path) -> None: "def test_mixed():\n assert parse_duration('1h30m') == 5400\n" ) + CASES = {"90s": 90, "5m": 300, "1h30m": 5400, "2h": 7200, "1h5m10s": 3910, "45": 45, " 7m ": 420} + @staticmethod def grade(d: Path): - cases = {"90s": 90, "5m": 300, "1h30m": 5400, "2h": 7200, "1h5m10s": 3910, "45": 45, " 7m ": 420} + t = d / "test_dur.py" + tests = t.read_text(errors="replace") if t.is_file() else "" + if "1h30m" not in tests or "5400" not in tests: + return False, "the tests were changed" + # The worker's module only computes; the verdict is taken here, out of its process. It used + # to print "BAD [...]" and the grader looked for "BAD []" anywhere in stdout. code = ( "import sys\nsys.path.insert(0, '.')\nfrom dur import parse_duration as p\n" - f"cases={cases!r}\nbad=[k for k,v in cases.items() if p(k)!=v]\nprint('BAD', bad)\n" + f"emit({{k: call(p, k)[:2] for k in {list(DurationBug.CASES)!r}}})\n" ) - out = subprocess.run(["python3", "-c", code], cwd=d, capture_output=True, text=True) - tests = (d / "test_dur.py").read_text() - if "1h30m" not in tests or "5400" not in tests: - return False, "the tests were changed" - if "BAD []" in out.stdout: + got, diag = py_values(d, code, timeout=60) + if not isinstance(got, dict): + return False, diag[-160:] + bad = [k for k, v in DurationBug.CASES.items() if not (isinstance(got.get(k), list) and got[k][:1] == ["ok"] and got[k][1:2] == [v])] + if not bad: return True, "all formats" - return False, (out.stdout + out.stderr).strip()[-160:] + return False, f"BAD {bad}"[-160:] # ---------------------------------------------------------------- 5. small CLI tool @@ -214,11 +228,20 @@ def grade(d: Path): extra = [x.name for x in d.iterdir() if x.name not in ("wc.py", ".molt", ".maat", "__pycache__")] if extra: return False, f"extra files left behind: {extra}" - tmp = Path(os.environ.get("TMPDIR", "/tmp")) / "wc-grade" - tmp.mkdir(exist_ok=True) + # A fresh private folder: a fixed /tmp/wc-grade could be pre-made by the worker with a.txt + # a symlink to any file the grader can write. + tmp = Path(tempfile.mkdtemp(prefix="wc-grade-")) + try: + return Wc._grade(p, tmp) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + @staticmethod + def _grade(p: Path, tmp: Path): a = tmp / "a.txt"; b = tmp / "b.txt" a.write_text("one two\nthree\n"); b.write_text("four five six\n\nseven\n") - run = lambda args, inp=None: subprocess.run(["python3", str(p), *args], capture_output=True, text=True, input=inp).stdout.strip() + run = lambda args, inp=None: subprocess.run(["python3", str(p), *args], capture_output=True, text=True, errors="replace", + input=inp, timeout=30).stdout.strip() want1 = f"2 3 {a.stat().st_size} {a}" got1 = run([str(a)]) got2 = run([str(a), str(b)]).splitlines() diff --git a/bench/local/tasks2.py b/bench/local/tasks2.py index 429064b..47ce109 100644 --- a/bench/local/tasks2.py +++ b/bench/local/tasks2.py @@ -26,19 +26,24 @@ import time import urllib.error import urllib.request +import sys from decimal import Decimal from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from grading import jsonish, py_values # noqa: E402,F401 + # agent bookkeeping that may appear in the task folder and is never judged IGNORED = {".molt", ".maat", "__pycache__", ".DS_Store"} -def sh(cmd: str, cwd: Path) -> str: - return subprocess.run(cmd, shell=True, cwd=cwd, capture_output=True, text=True).stdout +def sh(cmd: str, cwd: Path, timeout: float = 120) -> str: + # errors="replace": one non-UTF-8 byte in a deliverable's output made graders raise. + return subprocess.run(cmd, shell=True, cwd=cwd, capture_output=True, text=True, errors="replace", timeout=timeout).stdout def run(args, cwd=None, inp=None, timeout=30): - return subprocess.run(args, cwd=cwd, input=inp, capture_output=True, text=True, timeout=timeout) + return subprocess.run(args, cwd=cwd, input=inp, capture_output=True, text=True, errors="replace", timeout=timeout) def scratch_copy(d: Path): @@ -59,16 +64,29 @@ def all_files(root: Path) -> set[str]: return out -def py_check(d: Path, code: str, timeout=60): - """Run grader code in a fresh interpreter inside d; it prints OK or a reason.""" - try: - r = run(["python3", "-c", code], cwd=d, timeout=timeout) - except subprocess.TimeoutExpired: - return False, f"timed out after {timeout}s" - out = (r.stdout.strip().splitlines() or [""])[-1] - if out == "OK": - return True, "all checks" - return False, (out or r.stderr.strip()[-200:] or "no output")[:240] +# There is no py_check any more. It ran a harness in the same interpreter as the worker's module +# and passed on a last stdout line "OK", which the module could print itself from an atexit hook +# after a failing check had called sys.exit() (PR #33 audit, B2). Graders that run the worker's +# code take computed values back over grading.py_values and compare them here. + + +def ok_val(r, want, typ=None) -> bool: + """A child `call` result ['ok', value] equal to want (and of JSON type typ, when given).""" + if not (isinstance(r, list) and r[:1] == ["ok"] and len(r) >= 2): + return False + v = r[1] + if typ is not None and not (type(v) is typ): + return False + return v == want + + +def shown(r) -> str: + """A child `call` result as the old in-process harness printed it.""" + if isinstance(r, list) and r[:1] == ["ok"]: + return f"('ok', {r[1]!r})" + if isinstance(r, list) and r[:1] == ["err"]: + return f"('err', {r[1]!r})" + return repr(r) # ---------------------------------------------------------------- 1. HTTP JSON server @@ -348,42 +366,80 @@ def setup(d: Path) -> None: def grade(d: Path): if not (d / "pricing.py").exists(): return False, "no pricing.py" + # The worker's module only computes; every comparison is made here, with the reference + # implementation run in this process (it never enters the worker's). code = ( "import sys, inspect\nsys.path.insert(0, '.')\n" - "ref = {}\nexec(" + repr(RefactorPricing.ORIGINAL) + ", ref)\n" - "try:\n import pricing as m\nexcept Exception as e:\n print('import failed:', repr(e)); sys.exit()\n" + "try:\n import pricing as m\nexcept Exception as e:\n emit({'import': repr(e)})\n" "CODES = ['BOGO', 'BULK', 'FIVEOFF', 'SAVE10']\n" - "def call(f, *a):\n try:\n return ('ok', f(*a))\n except Exception as e:\n return ('err', type(e).__name__)\n" + "def call2(f, *a):\n r = call(f, *a)\n return r[:2]\n" "C = getattr(m, 'COUPONS', None)\n" - "if not isinstance(C, dict) or sorted(C) != CODES or not all(callable(v) for v in C.values()):\n" - " print('COUPONS must be a dict of the 4 codes to functions'); sys.exit()\n" - "for it in ['widget', 'gadget', 'doohickey', 'thing']:\n" - " for q in range(0, 26):\n" - " for c in [None, 'NOPE'] + CODES:\n" - " a, b = call(m.price, it, q, c), call(ref['price'], it, q, c)\n" - " if a != b:\n print(f'price({it!r},{q},{c!r}) = {a} but was {b}'); sys.exit()\n" - "if call(m.price, 'widget', 3) != call(ref['price'], 'widget', 3):\n print('default coupon changed'); sys.exit()\n" - "if C['BULK'](2.5, 10, 25.0) != 20.0 or C['BOGO'](2.5, 3, 7.5) != 5.0:\n" - " print('COUPONS functions do not follow f(base, qty, subtotal)'); sys.exit()\n" - "src = inspect.getsource(m.price)\n" - "if any(c in src for c in CODES):\n print('price() still names coupon codes'); sys.exit()\n" + "V = {'coupons': {'dict': isinstance(C, dict), 'keys': sorted(map(str, C)) if isinstance(C, dict) else None,\n" + " 'callable': isinstance(C, dict) and all(callable(v) for v in C.values())}}\n" + "if not (isinstance(C, dict) and sorted(map(str, C)) == CODES and all(callable(v) for v in C.values())):\n emit(V)\n" + "V['price'] = [[it, q, c, call2(m.price, it, q, c)] for it in ['widget', 'gadget', 'doohickey', 'thing'] for q in range(0, 26) for c in [None, 'NOPE'] + CODES]\n" + "V['default'] = call2(m.price, 'widget', 3)\n" + "V['fns'] = [call2(C['BULK'], 2.5, 10, 25.0), call2(C['BOGO'], 2.5, 3, 7.5)]\n" + "try:\n V['src'] = inspect.getsource(m.price)\nexcept Exception as e:\n V['src'] = None\n" "saved = dict(C)\n" "C['SAVE10'] = lambda b, q, s: 1.0\nC['ZZZ'] = lambda b, q, s: s - 1\n" - "r1, r2 = call(m.price, 'gadget', 3, 'SAVE10'), call(m.price, 'gadget', 1, 'ZZZ')\n" + "V['uses'] = [call2(m.price, 'gadget', 3, 'SAVE10'), call2(m.price, 'gadget', 1, 'ZZZ')]\n" "C.clear(); C.update(saved)\n" - "if r1 != ('ok', 1.0) or r2 != ('ok', 9.0):\n print(f'price() does not use COUPONS: {r1} {r2}'); sys.exit()\n" - "if not callable(getattr(m, 'best_coupon', None)):\n print('no best_coupon'); sys.exit()\n" - "def best(it, q):\n" - " none = ref['price'](it, q)\n" - " p, c = min((ref['price'](it, q, c), c) for c in CODES)\n" - " return c if p < none else None\n" - "for it in ['widget', 'gadget', 'doohickey']:\n" - " for q in range(0, 26):\n" - " a, b = call(m.best_coupon, it, q), ('ok', best(it, q))\n" - " if a != b:\n print(f'best_coupon({it!r},{q}) = {a}, want {b}'); sys.exit()\n" - "print('OK')\n" + "V['has_best'] = callable(getattr(m, 'best_coupon', None))\n" + "if V['has_best']:\n" + " V['best'] = [[it, q, call2(m.best_coupon, it, q)] for it in ['widget', 'gadget', 'doohickey'] for q in range(0, 26)]\n" + "emit(V)\n" ) - return py_check(d, code) + V, diag = py_values(d, code) + if not isinstance(V, dict): + return False, diag + if "import" in V: + return False, f"import failed: {V['import']}"[:240] + ref: dict = {} + exec(RefactorPricing.ORIGINAL, ref) + + def rcall(f, *a): + try: + return ["ok", f(*a)] + except Exception as e: # noqa: BLE001 + return ["err", type(e).__name__] + + CODES = ["BOGO", "BULK", "FIVEOFF", "SAVE10"] + c = V.get("coupons") or {} + if not (c.get("dict") and c.get("keys") == CODES and c.get("callable")): + return False, "COUPONS must be a dict of the 4 codes to functions" + for it, q, cp, a in V.get("price") or []: + b = jsonish(rcall(ref["price"], it, q, cp)) + if a != b: + return False, f"price({it!r},{q},{cp!r}) = {shown(a)} but was {shown(b)}"[:240] + if len(V.get("price") or []) != 4 * 26 * 6: + return False, "the price grid was not computed" + if V.get("default") != jsonish(rcall(ref["price"], "widget", 3)): + return False, "default coupon changed" + fns = V.get("fns") or [None, None] + if not (ok_val(fns[0], 20.0) and ok_val(fns[1], 5.0)): + return False, "COUPONS functions do not follow f(base, qty, subtotal)" + src = V.get("src") + if not isinstance(src, str) or any(x in src for x in CODES): + return False, "price() still names coupon codes" + r1, r2 = (V.get("uses") or [None, None])[:2] + if not (ok_val(r1, 1.0) and ok_val(r2, 9.0)): + return False, f"price() does not use COUPONS: {shown(r1)} {shown(r2)}"[:240] + if not V.get("has_best"): + return False, "no best_coupon" + + def best(it, q): + none = ref["price"](it, q) + p, cc = min((ref["price"](it, q, x), x) for x in CODES) + return cc if p < none else None + + rows = V.get("best") or [] + if len(rows) != 3 * 26: + return False, "best_coupon was not computed" + for it, q, a in rows: + if a != ["ok", best(it, q)]: + return False, f"best_coupon({it!r},{q}) = {shown(a)}, want ('ok', {best(it, q)!r})"[:240] + return True, "all checks" # ---------------------------------------------------------------- 5. bash script with args + exit codes @@ -563,31 +619,73 @@ def setup(d: Path) -> None: def grade(d: Path): if not (d / "pairs.py").exists(): return False, "no pairs.py" + small = PerfPairs.small_cases() + # The worker's module only computes (and times itself); the answers and the time limit are + # checked here. The wall time of the whole child is checked too, so a module that fakes + # time.time() gains nothing. code = ( "import sys, time, random\nsys.path.insert(0, '.')\n" - "ref = {}\nexec(" + repr(PerfPairs.SLOW) + ", ref)\n" - "def fast(nums, t):\n" - " from collections import Counter\n c = Counter(nums); out = set()\n" - " for a in c:\n b = t - a\n" - " if a <= b and b in c and (a != b or c[a] > 1):\n out.add((a, b))\n" - " return sorted(out)\n" "from pairs import find_pairs\n" - "small = [([], 5), ([5], 10), ([5, 5], 10), ([1, 2, 3, 4, 5], 6), ([3, 3, 3], 6), ([-2, 7, 4, 1, 1, 4, 9, -2], 5),\n" - " ([0, 0, 0, 1, -1], 0), ([10, -10, 20, -20, 0], 0), ([2, 4, 6], 100)]\n" - "rnd = random.Random(3)\n" - "for _ in range(40):\n small.append(([rnd.randint(-8, 8) for _ in range(rnd.randint(0, 15))], rnd.randint(-10, 10)))\n" + f"small = {small!r}\n" + "V = {'small': []}\n" "for nums, t in small:\n" - " got, want = find_pairs(list(nums), t), ref['find_pairs'](nums, t)\n" - " if got != want or type(got) is not list or any(type(p) is not tuple for p in got):\n" - " print(f'find_pairs({nums}, {t}) = {got!r}, want {want!r}'); sys.exit()\n" + " got = find_pairs(list(nums), t)\n" + " shape = type(got) is list and all(type(p) is tuple for p in got)\n" + " V['small'].append([got, shape])\n" "rnd = random.Random(5)\nbig = [rnd.randint(-10**6, 10**6) for _ in range(200000)]\n" - "want = fast(big, 1234)\n" - "t0 = time.time(); got = find_pairs(list(big), 1234); el = time.time() - t0\n" - "if got != want:\n print(f'wrong answer on the big input ({len(got)} pairs, want {len(want)})'); sys.exit()\n" - "if el > 5:\n print(f'big input took {el:.1f}s'); sys.exit()\n" - "print('OK')\n" + "t0 = time.time(); got = find_pairs(list(big), 1234); V['el'] = time.time() - t0\n" + "V['big'] = got\n" + "emit(V)\n" ) - return py_check(d, code, timeout=20) + t0 = time.time() + V, diag = py_values(d, code, timeout=20) + wall = time.time() - t0 + if not isinstance(V, dict): + return False, diag + ref: dict = {} + exec(PerfPairs.SLOW, ref) + got_small = V.get("small") or [] + if len(got_small) != len(small): + return False, "find_pairs was not run on every small input" + for (nums, t), (got, shape) in zip(small, got_small): + want = jsonish(ref["find_pairs"](nums, t)) + if got != want or not shape: + return False, f"find_pairs({nums}, {t}) = {got!r}, want {want!r}"[:240] + rnd = random.Random(5) + big = [rnd.randint(-10**6, 10**6) for _ in range(200000)] + want = jsonish(PerfPairs.fast(big, 1234)) + got = V.get("big") + if got != want: + return False, f"wrong answer on the big input ({len(got) if isinstance(got, list) else got!r} pairs, want {len(want)})"[:240] + el = V.get("el") + if not isinstance(el, (int, float)) or el > 5: + return False, f"big input took {el:.1f}s" if isinstance(el, (int, float)) else "no timing" + if wall > 5 + PerfPairs.WALL_SLACK: + return False, f"the check took {wall:.1f}s in all (the big input may take 5s)" + return True, "all checks" + + # Interpreter start, the small cases and making the big input, on top of the 5 s the big input + # may take: a child slower than this lied about its own timing. + WALL_SLACK = 4.0 + + @staticmethod + def small_cases(): + small = [([], 5), ([5], 10), ([5, 5], 10), ([1, 2, 3, 4, 5], 6), ([3, 3, 3], 6), ([-2, 7, 4, 1, 1, 4, 9, -2], 5), + ([0, 0, 0, 1, -1], 0), ([10, -10, 20, -20, 0], 0), ([2, 4, 6], 100)] + rnd = random.Random(3) + for _ in range(40): + small.append(([rnd.randint(-8, 8) for _ in range(rnd.randint(0, 15))], rnd.randint(-10, 10))) + return small + + @staticmethod + def fast(nums, t): + from collections import Counter + c = Counter(nums); out = set() + for a in c: + b = t - a + if a <= b and b in c and (a != b or c[a] > 1): + out.add((a, b)) + return sorted(out) # ---------------------------------------------------------------- 8. git: revert one bad commit @@ -833,18 +931,28 @@ def grade(d: Path): bad = ["", " ", "-1 KB", "KB", "5 XB", "1.2.3 MB", "1,5 KB", "abc", "5 KBB", "-0.5"] code = ( "import sys\nsys.path.insert(0, '.')\n" - "try:\n from sizes import parse_size\nexcept Exception as e:\n print('import failed:', repr(e)); sys.exit()\n" - f"good = {good!r}\nbad = {bad!r}\n" - "for s, v in good.items():\n" - " try:\n r = parse_size(s)\n except Exception as e:\n print(f'parse_size({s!r}) raised {type(e).__name__}, want {v}'); sys.exit()\n" - " if r != v or type(r) is not int:\n print(f'parse_size({s!r}) = {r!r}, want {v}'); sys.exit()\n" - "for s in bad:\n" - " try:\n r = parse_size(s)\n except ValueError:\n continue\n" - " except Exception as e:\n print(f'parse_size({s!r}) raised {type(e).__name__}, want ValueError'); sys.exit()\n" - " print(f'parse_size({s!r}) = {r!r}, want ValueError'); sys.exit()\n" - "print('OK')\n" + "try:\n from sizes import parse_size\nexcept Exception as e:\n emit({'import': repr(e)})\n" + f"emit({{'good': [call(parse_size, s) for s in {list(good)!r}], 'bad': [call(parse_size, s) for s in {bad!r}]}})\n" ) - return py_check(d, code) + V, diag = py_values(d, code) + if not isinstance(V, dict): + return False, diag + if "import" in V: + return False, f"import failed: {V['import']}"[:240] + for (s, v), r in zip(good.items(), V.get("good") or []): + if r[:1] == ["err"]: + return False, f"parse_size({s!r}) raised {r[1]}, want {v}" + if not ok_val(r, v, int): + return False, f"parse_size({s!r}) = {r[1]!r}, want {v}" + for s, r in zip(bad, V.get("bad") or []): + if r[:1] == ["err"] and r[3]: # a ValueError (or a subclass) + continue + if r[:1] == ["err"]: + return False, f"parse_size({s!r}) raised {r[1]}, want ValueError" + return False, f"parse_size({s!r}) = {r[1]!r}, want ValueError" + if len(V.get("good") or []) != len(good) or len(V.get("bad") or []) != len(bad): + return False, "parse_size was not run on every input" + return True, "all checks" # ---------------------------------------------------------------- 12. file organisation diff --git a/bench/local/tasks3.py b/bench/local/tasks3.py index 2171523..4c321e9 100644 --- a/bench/local/tasks3.py +++ b/bench/local/tasks3.py @@ -18,6 +18,7 @@ import hashlib import io import json +import math import os import random import re @@ -32,7 +33,8 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -from tasks2 import IGNORED, all_files, py_check, run, scratch_copy, sh # noqa: E402,F401 +from tasks2 import IGNORED, all_files, ok_val, run, scratch_copy, sh, shown # noqa: E402,F401 +from grading import JS_EMIT, child_values, jsonish, py_values # noqa: E402,F401 VCS_ENV = { "GIT_AUTHOR_NAME": "Dev", "GIT_AUTHOR_EMAIL": "dev@example.com", @@ -58,6 +60,11 @@ def vcs(d: Path, *args, date: str | None = None): return r.stdout.decode(errors="replace") +def _text(p: Path) -> str: + """A file's text, "" when it is missing or not a regular file (a grader must not raise on it).""" + return p.read_text(errors="replace") if p.is_file() else "" + + def write(d: Path, rel: str, text): p = d / rel p.parent.mkdir(parents=True, exist_ok=True) @@ -674,38 +681,70 @@ def _rand_orders(seed, n): return out +# The worker's modules only compute; the old render, the expected tax and money strings and the +# source check all stay in the grader's process (INVOICE_ORIG is run there, never next to them). INVOICE_HARNESS = r''' -import importlib.util, json, sys -def load(name, path): - spec = importlib.util.spec_from_file_location(name, path); m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m +import json, sys sys.path.insert(0, ".") -old = load("_orig_invoice", "_orig_invoice.py") import invoice, tax, format as fmt orders = json.load(open("_orders.json")) -def call(f, o): - try: return ("ok", f(o)) - except Exception as e: return ("err", type(e).__name__, str(e)) -for i, o in enumerate(orders): - a, b = call(old.render, o), call(invoice.render, o) - if a != b: - print(f"order {i} ({o['region']}, coupon={o.get('coupon')!r}): old {a!r:.120} new {b!r:.120}"); sys.exit() -for base, region in [(10000, "US-CA"), (1, "US-CA"), (-1234, "DE"), (-1, "FR"), (0, "US-OR"), (99999, "US-NY"), (6, "US-CA"), (-6, "US-CA"), (13, "FR"), (-13, "FR"), (150, "DE"), (200, "US-CA"), (-150, "DE"), (250, "FR")]: - want = (base * old.REGIONS[region] + 5000) // 10000 - got = tax.compute_tax(base, region) - if got != want or type(got) is not int: - print(f"compute_tax({base}, {region!r}) = {got!r}, want {want}"); sys.exit() -try: - tax.compute_tax(100, "XX"); print("compute_tax accepted an unknown region"); sys.exit() -except ValueError as e: - if str(e) != "unknown region: XX": print(f"wrong message {e}"); sys.exit() -for c, region, want in [(123456, "US-CA", "$1,234.56"), (-5, "US-NY", "-$0.05"), (123456789, "DE", "1.234.567,89 \u20ac"), (-100, "FR", "-1,00 \u20ac"), (0, "US-OR", "$0.00"), (7, "FR", "0,07 \u20ac")]: - got = fmt.money(c, region) - if got != want: print(f"money({c}, {region!r}) = {got!r}, want {want!r}"); sys.exit() -src = open("invoice.py").read() -for bad in ("1900", "725", "\u20ac", "REGIONS", "divmod"): - if bad in src: print(f"invoice.py still contains {bad!r}: the logic was not moved"); sys.exit() -print("OK") +V = {"render": [call(invoice.render, o)[:3] for o in orders]} +V["tax"] = [] +for base, region in TAX_CASES: + r = call(tax.compute_tax, base, region) + V["tax"].append([r[:3], type(r[1]).__name__ if r[0] == "ok" else None]) +V["tax_xx"] = call(tax.compute_tax, 100, "XX") +V["money"] = [call(fmt.money, c, region)[:3] for c, region, _ in MONEY_CASES] +emit(V) ''' +INVOICE_TAX_CASES = [(10000, "US-CA"), (1, "US-CA"), (-1234, "DE"), (-1, "FR"), (0, "US-OR"), (99999, "US-NY"), (6, "US-CA"), (-6, "US-CA"), + (13, "FR"), (-13, "FR"), (150, "DE"), (200, "US-CA"), (-150, "DE"), (250, "FR")] +INVOICE_MONEY_CASES = [(123456, "US-CA", "$1,234.56"), (-5, "US-NY", "-$0.05"), (123456789, "DE", "1.234.567,89 \u20ac"), + (-100, "FR", "-1,00 \u20ac"), (0, "US-OR", "$0.00"), (7, "FR", "0,07 \u20ac")] + + +def _invoice_verdict(w: Path, V: dict, orders: list) -> tuple[bool, str]: + old: dict = {} + exec(compile(INVOICE_ORIG, "orig", "exec"), old) + + def call(f, *a): + try: + return ["ok", f(*a)] + except Exception as e: # noqa: BLE001 + return ["err", type(e).__name__, str(e)[:300]] + + got = V.get("render") or [] + if len(got) != len(orders): + return False, "render was not run on every order" + for i, (o, b) in enumerate(zip(orders, got)): + a = jsonish(call(old["render"], o)) + if a != b: + return False, f"order {i} ({o['region']}, coupon={o.get('coupon')!r}): old {tuple(a)!r:.120} new {tuple(b) if isinstance(b, list) else b!r:.120}" + tx = V.get("tax") or [] + if len(tx) != len(INVOICE_TAX_CASES): + return False, "compute_tax was not run on every case" + for (base, region), (r, typ) in zip(INVOICE_TAX_CASES, tx): + want = (base * old["REGIONS"][region] + 5000) // 10000 + if r[:1] != ["ok"] or r[1] != want or typ != "int": + return False, f"compute_tax({base}, {region!r}) = {r[1] if r[:1] == ['ok'] else tuple(r)!r}, want {want}" + xx = V.get("tax_xx") or [] + if xx[:1] == ["ok"]: + return False, "compute_tax accepted an unknown region" + if not (len(xx) >= 4 and xx[3]): # not a ValueError + return False, f"compute_tax raised {xx[1:2]} for an unknown region, want ValueError" + if xx[2] != "unknown region: XX": + return False, f"wrong message {xx[2]}" + mo = V.get("money") or [] + if len(mo) != len(INVOICE_MONEY_CASES): + return False, "money was not run on every case" + for (c, region, want), r in zip(INVOICE_MONEY_CASES, mo): + if r != ["ok", want]: + return False, f"money({c}, {region!r}) = {r[1] if r[:1] == ['ok'] else tuple(r)!r}, want {want!r}" + src = (w / "invoice.py").read_text(errors="replace") + for bad in ("1900", "725", "\u20ac", "REGIONS", "divmod"): + if bad in src: + return False, f"invoice.py still contains {bad!r}: the logic was not moved" + return True, "all checks" class RefactorInvoice: @@ -741,9 +780,13 @@ def grade(d: Path): for f in ("tax.py", "format.py", "invoice.py"): if not (w / f).exists(): return False, f"no {f}" - write(w, "_orig_invoice.py", INVOICE_ORIG) - write(w, "_orders.json", json.dumps(_rand_orders(5, 500), ensure_ascii=False)) - return py_check(w, INVOICE_HARNESS) + orders = _rand_orders(5, 500) + write(w, "_orders.json", json.dumps(orders, ensure_ascii=False)) + cases = f"TAX_CASES = {INVOICE_TAX_CASES!r}\nMONEY_CASES = {[(c, r, None) for c, r, _ in INVOICE_MONEY_CASES]!r}\n" + V, diag = py_values(w, cases + INVOICE_HARNESS) + if not isinstance(V, dict): + return False, diag + return _invoice_verdict(w, V, orders) finally: shutil.rmtree(tmp, ignore_errors=True) @@ -966,8 +1009,8 @@ def grade(d: Path): first_bad = None for r in revs: x = Path(tempfile.mkdtemp(prefix="co-", dir=tmp)) - subprocess.run(f"git archive {r} | tar -x -C {x}", shell=True, cwd=tmp, capture_output=True) - if subprocess.run(["sh", "check.sh"], cwd=x, capture_output=True).returncode != 0: + subprocess.run(f"git archive {r} | tar -x -C {x}", shell=True, cwd=tmp, capture_output=True, timeout=60) + if subprocess.run(["sh", "check.sh"], cwd=x, capture_output=True, timeout=30).returncode != 0: first_bad = r break if vcs(d, "rev-list", "--reverse", "master").split() != revs: @@ -2001,31 +2044,48 @@ def grade(d: Path): return False, "vendor/ was changed" finally: shutil.rmtree(ref, ignore_errors=True) + json.dump({"a": "2m", "b": "1h 1s"}, open(w / "cfg2.json", "w")) code = ( - "import json, math, sys\n" "from app.retry import delay_seconds\n" "from app.schedule import load_schedule\n" - f"GOOD = {VendorUnits.GOOD!r}\nBAD = {VendorUnits.BAD!r}\n" - "for spec, want in GOOD:\n" - " try: got = delay_seconds(spec)\n" - " except Exception as e: print(f'{spec!r}: raised {e!r}'); sys.exit()\n" - " if not math.isclose(got, want, rel_tol=1e-9, abs_tol=1e-9) or (float(want).is_integer() and not isinstance(got, int)):\n" - " print(f'{spec!r}: got {got!r} want {want!r}'); sys.exit()\n" - "for spec in BAD:\n" - " try: got = delay_seconds(spec)\n" - " except ValueError: continue\n" - " except Exception as e: print(f'{spec!r}: raised {type(e).__name__}, want ValueError'); sys.exit()\n" - " print(f'{spec!r}: accepted as {got!r}'); sys.exit()\n" - "json.dump({'a': '2m', 'b': '1h 1s'}, open('cfg2.json', 'w'))\n" - "if load_schedule('cfg2.json') != {'a': 120, 'b': 3601}: print('load_schedule wrong:', load_schedule('cfg2.json')); sys.exit()\n" - "if load_schedule('config.json') != {'backoff': 300, 'timeout': 90, 'cooldown': 5400, 'report': 5400}: print('config.json schedule wrong:', load_schedule('config.json')); sys.exit()\n" - "print('OK')\n" + f"GOOD = {[g for g, _ in VendorUnits.GOOD]!r}\nBAD = {VendorUnits.BAD!r}\n" + "V = {'good': [], 'bad': [call(delay_seconds, x) for x in BAD]}\n" + "for x in GOOD:\n" + " r = call(delay_seconds, x)\n" + " V['good'].append([r, type(r[1]).__name__ if r[0] == 'ok' else None])\n" + "V['cfg2'] = call(load_schedule, 'cfg2.json')\nV['config'] = call(load_schedule, 'config.json')\n" + "emit(V)\n" ) - ok, why = py_check(w, code) - return ok, why + V, diag = py_values(w, code) + if not isinstance(V, dict): + return False, diag + return VendorUnits.verdict(V) finally: shutil.rmtree(tmp, ignore_errors=True) + @staticmethod + def verdict(V: dict) -> tuple[bool, str]: + good = V.get("good") or [] + if len(good) != len(VendorUnits.GOOD) or len(V.get("bad") or []) != len(VendorUnits.BAD): + return False, "delay_seconds was not run on every spec" + for (spec, want), (r, typ) in zip(VendorUnits.GOOD, good): + if r[0] != "ok": + return False, f"{spec!r}: raised {r[1]}({r[2]!r})" + got = r[1] + if typ not in ("int", "float") or not math.isclose(got, want, rel_tol=1e-9, abs_tol=1e-9) or (float(want).is_integer() and typ != "int"): + return False, f"{spec!r}: got {got!r} want {want!r}" + for spec, r in zip(VendorUnits.BAD, V["bad"]): + if r[0] == "err" and r[3]: + continue + if r[0] == "err": + return False, f"{spec!r}: raised {r[1]}, want ValueError" + return False, f"{spec!r}: accepted as {r[1]!r}" + if V.get("cfg2") != ["ok", {"a": 120, "b": 3601}]: + return False, f"load_schedule wrong: {shown(V.get('cfg2'))}" + if V.get("config") != ["ok", {"backoff": 300, "timeout": 90, "cooldown": 5400, "report": 5400}]: + return False, f"config.json schedule wrong: {shown(V.get('config'))}" + return True, "all checks" + # ================================================================ 19. rename-api-signature SHOP_FILES = { @@ -2197,6 +2257,8 @@ def setup(d: Path) -> None: for rel, txt in SHOP_FILES.items(): write(d, rel, txt) + # The worker's package only computes; what each call returned is compared with the expected + # values here, out of its process. HARNESS = r''' import inspect, runpy, sys sys.path.insert(0, ".") @@ -2204,47 +2266,69 @@ def setup(d: Path) -> None: conn = db.connect() conn.execute("INSERT INTO users VALUES (1, 'Ada', 'ada@x.org')") conn.execute("INSERT INTO users VALUES (2, 'Ben', '')") -if hasattr(db, "get_user"): print("shop.db still has get_user"); sys.exit() -sig = inspect.signature(db.fetch_user) -ps = list(sig.parameters.values()) -if [p.name for p in ps] != ["uid", "db"] or ps[0].kind not in (ps[0].POSITIONAL_OR_KEYWORD, ps[0].POSITIONAL_ONLY) or ps[1].kind != ps[1].KEYWORD_ONLY: - print(f"fetch_user signature is {sig}"); sys.exit() -if db.fetch_user(1, db=conn) != {"id": 1, "name": "Ada", "email": "ada@x.org"} or db.fetch_user(9, db=conn) is not None: - print("fetch_user returns the wrong thing"); sys.exit() +V = {"has_get_user": hasattr(db, "get_user")} try: - db.fetch_user(conn, 1); print("fetch_user(conn, 1) should not be accepted"); sys.exit() -except TypeError: - pass -if db.get_user_by_name(conn, "Ben")["id"] != 2 or list(inspect.signature(db.get_user_by_name).parameters) != ["db", "name"]: - print("get_user_by_name changed"); sys.exit() -checks = [ - ("views.profile", lambda: views.profile(conn, 1) == "Ada "), - ("views.profiles", lambda: views.profiles(conn, [1, 2, 3]) == ["Ada ", "Ben <>"]), - ("views.make_loader", lambda: views.make_loader(conn)(2)["name"] == "Ben"), - ("billing.invoice_owner", lambda: billing.invoice_owner(conn, {"user_id": 1}) == "Ada"), - ("billing.total_for", lambda: billing.total_for(conn, [{"user_id": 1, "cents": 5}, {"user_id": 7, "cents": 9}]) == 5), - ("notify.welcome", lambda: notify.welcome(conn, 1) == "Welcome, Ada!" and notify.welcome(conn, 8) == "Welcome!"), - ("notify.by_keyword", lambda: notify.by_keyword(conn, 2)["name"] == "Ben"), - ("registry user_loader", lambda: registry.load("user_loader")(1, db=conn)["email"] == "ada@x.org"), - ("registry name_loader", lambda: registry.load("name_loader")(conn, "Ada")["id"] == 1), - ("cli show", lambda: cli.main(["show", "1"], conn) == 0), - ("scripts/audit.py", lambda: runpy.run_path("scripts/audit.py")["audit"](conn, [1, 2, 3]) == [2, 3]), + V["sig"] = [[p.name, p.kind.name] for p in inspect.signature(db.fetch_user).parameters.values()] +except Exception as e: + V["sig"] = repr(e) +V["fetch1"] = call(lambda: db.fetch_user(1, db=conn))[:3] +V["fetch9"] = call(lambda: db.fetch_user(9, db=conn))[:3] +V["fetch_pos"] = call(lambda: db.fetch_user(conn, 1)) +V["by_name"] = call(lambda: db.get_user_by_name(conn, "Ben")["id"])[:3] +V["by_name_sig"] = call(lambda: list(inspect.signature(db.get_user_by_name).parameters))[:3] +V["checks"] = [ + call(lambda: views.profile(conn, 1))[:3], + call(lambda: views.profiles(conn, [1, 2, 3]))[:3], + call(lambda: views.make_loader(conn)(2)["name"])[:3], + call(lambda: billing.invoice_owner(conn, {"user_id": 1}))[:3], + call(lambda: billing.total_for(conn, [{"user_id": 1, "cents": 5}, {"user_id": 7, "cents": 9}]))[:3], + call(lambda: [notify.welcome(conn, 1), notify.welcome(conn, 8)])[:3], + call(lambda: notify.by_keyword(conn, 2)["name"])[:3], + call(lambda: registry.load("user_loader")(1, db=conn)["email"])[:3], + call(lambda: registry.load("name_loader")(conn, "Ada")["id"])[:3], + call(lambda: cli.main(["show", "1"], conn))[:3], + call(lambda: runpy.run_path("scripts/audit.py")["audit"](conn, [1, 2, 3]))[:3], ] -for name, fn in checks: - try: - ok = fn() - except Exception as e: - print(f"{name}: raised {type(e).__name__}: {e}"); sys.exit() - if not ok: - print(f"{name}: wrong result"); sys.exit() -print("OK") +emit(V) ''' + CHECKS = [("views.profile", "Ada "), ("views.profiles", ["Ada ", "Ben <>"]), ("views.make_loader", "Ben"), + ("billing.invoice_owner", "Ada"), ("billing.total_for", 5), ("notify.welcome", ["Welcome, Ada!", "Welcome!"]), + ("notify.by_keyword", "Ben"), ("registry user_loader", "ada@x.org"), ("registry name_loader", 1), ("cli show", 0), + ("scripts/audit.py", [2, 3])] + + @staticmethod + def verdict(V: dict) -> tuple[bool, str]: + if V.get("has_get_user") is not False: + return False, "shop.db still has get_user" + sig = V.get("sig") + if not (isinstance(sig, list) and [p[0] for p in sig] == ["uid", "db"] + and sig[0][1] in ("POSITIONAL_OR_KEYWORD", "POSITIONAL_ONLY") and sig[1][1] == "KEYWORD_ONLY"): + return False, f"fetch_user signature is {sig}" + if V.get("fetch1") != ["ok", {"id": 1, "name": "Ada", "email": "ada@x.org"}] or V.get("fetch9") != ["ok", None]: + return False, "fetch_user returns the wrong thing" + fp = V.get("fetch_pos") or [] + if not (fp[:1] == ["err"] and len(fp) >= 5 and fp[4]): + if fp[:1] == ["err"]: + return False, f"fetch_user(conn, 1) raised {fp[1]}, want TypeError" + return False, "fetch_user(conn, 1) should not be accepted" + if V.get("by_name") != ["ok", 2] or V.get("by_name_sig") != ["ok", ["db", "name"]]: + return False, "get_user_by_name changed" + got = V.get("checks") or [] + if len(got) != len(RenameApi.CHECKS): + return False, "not every call site was run" + for (name, want), r in zip(RenameApi.CHECKS, got): + if r[:1] == ["err"]: + return False, f"{name}: raised {r[1]}: {r[2]}"[:240] + if r[1:2] != [want]: + return False, f"{name}: wrong result" + return True, "all checks" @staticmethod def grade(d: Path): tmp, w = scratch_copy(d) try: - if (d / "CHANGELOG.md").read_text() != SHOP_FILES["CHANGELOG.md"]: + cl = d / "CHANGELOG.md" + if not cl.is_file() or cl.read_text(errors="replace") != SHOP_FILES["CHANGELOG.md"]: return False, "CHANGELOG.md was modified" for rel in sorted(all_files(w)): if rel == "CHANGELOG.md" or not rel.endswith((".py", ".md", ".json", ".txt", ".toml", ".cfg", ".sh")): @@ -2252,13 +2336,16 @@ def grade(d: Path): txt = (w / rel).read_text(errors="replace") if re.search(r"\bget_user\b", txt): return False, f"{rel} still mentions get_user" - if "fetch_user" not in (w / "docs/API.md").read_text() or "fetch_user" not in (w / "README.md").read_text(): + if "fetch_user" not in _text(w / "docs/API.md") or "fetch_user" not in _text(w / "README.md"): return False, "docs do not mention fetch_user" - if '"fetch_user"' not in (w / "shop/plugins.json").read_text(): + if '"fetch_user"' not in _text(w / "shop/plugins.json"): return False, "shop/plugins.json does not name fetch_user" - if "get_user_by_name" not in (w / "shop/db.py").read_text(): + if "get_user_by_name" not in _text(w / "shop/db.py"): return False, "get_user_by_name is gone" - ok, why = py_check(w, RenameApi.HARNESS) + V, diag = py_values(w, RenameApi.HARNESS) + if not isinstance(V, dict): + return False, diag + ok, why = RenameApi.verdict(V) if not ok: return False, why r = runb(["python3", "-m", "unittest", "discover", "-s", "tests", "-t", "."], cwd=w, timeout=60) @@ -2550,91 +2637,100 @@ def grade(d: Path): # ================================================================ 22. async-pool-node +# The hidden tests only OBSERVE: each one records what mapLimit did (results, start order, peak +# concurrency, how the promise settled) and the record goes back as one JSON value on the values +# channel. Every assertion is made in AsyncPoolNode.verdict, out of the worker's process: the old +# harness printed "OK" last and a `process.on('exit')` hook in pool.js could print it too. POOL_TESTS = r''' "use strict"; -const assert = require("assert"); let mapLimit; -try { ({ mapLimit } = require("./pool.js")); } catch (e) { console.log("cannot load pool.js: " + e.message); process.exit(); } -if (typeof mapLimit !== "function") { console.log("pool.js must export mapLimit"); process.exit(); } +try { ({ mapLimit } = require("./pool.js")); } catch (e) { emit({ load: "cannot load pool.js: " + e.message }); } +if (typeof mapLimit !== "function") emit({ load: "pool.js must export mapLimit" }); let unhandled = 0; process.on("unhandledRejection", () => { unhandled++; }); const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +const enc = (v) => (v === undefined ? { $undefined: true } : Array.isArray(v) ? v.map(enc) : v); +// assert.rejects took a promise, or a thenable with then and catch; anything else failed the test. +const isPromise = (p) => p instanceof Promise || (!!p && typeof p.then === "function" && typeof p.catch === "function"); +const settle = (p) => Promise.resolve(p).then( + (v) => ({ ok: true, value: enc(v), promise: isPromise(p) }), + (e) => ({ ok: false, str: String(e), range: e instanceof RangeError, type: e instanceof TypeError, promise: isPromise(p) })); const tests = []; const test = (name, fn) => tests.push([name, fn]); -test("results keep input order and index is passed", async () => { +test("order", async () => { const items = [...Array(20).keys()]; - const res = await mapLimit(items, 3, async (x, i) => { await sleep((x * 7) % 11); return x * 2 + i; }); - assert.deepStrictEqual(res, items.map((x, i) => x * 2 + i)); + return { res: enc(await mapLimit(items, 3, async (x, i) => { await sleep((x * 7) % 11); return x * 2 + i; })) }; }); -test("never more than `limit` in flight, and uses all of them", async () => { +test("limit", async () => { let running = 0, max = 0; await mapLimit([...Array(15).keys()], 4, async (x) => { running++; max = Math.max(max, running); await sleep(5 + (x % 3)); running--; }); - assert.strictEqual(max, 4); + return { max }; }); -test("limit larger than the item count", async () => { +test("big_limit", async () => { let running = 0, max = 0; const res = await mapLimit([1, 2, 3], 10, async (x) => { running++; max = Math.max(max, running); await sleep(5); running--; return x; }); - assert.deepStrictEqual(res, [1, 2, 3]); assert.strictEqual(max, 3); -}); -test("empty input", async () => { assert.deepStrictEqual(await mapLimit([], 2, async () => 1), []); }); -test("plain (non-promise) return values and falsy results", async () => { - assert.deepStrictEqual(await mapLimit([1, 2, 3], 2, (x) => (x === 2 ? 0 : x)), [1, 0, 3]); - assert.deepStrictEqual(await mapLimit([1, 2], 1, (x) => (x === 1 ? undefined : null)), [undefined, null]); + return { res: enc(res), max }; }); -test("a free slot is refilled at once (no batches)", async () => { +test("empty", async () => ({ res: enc(await mapLimit([], 2, async () => 1)) })); +test("plain", async () => ({ + a: enc(await mapLimit([1, 2, 3], 2, (x) => (x === 2 ? 0 : x))), + b: enc(await mapLimit([1, 2], 1, (x) => (x === 1 ? undefined : null))), +})); +test("refill", async () => { const start = []; const t0 = Date.now(); await mapLimit([80, 10, 10, 10], 2, async (ms, i) => { start[i] = Date.now() - t0; await sleep(ms); }); - assert.ok(start[2] < 50, "item 2 started at " + start[2] + "ms, should be ~10ms"); - assert.ok(start[3] < 60, "item 3 started at " + start[3] + "ms, should be ~20ms"); + return { start: enc(start) }; }); -test("limit 1 runs strictly one after another, in order", async () => { +test("serial", async () => { const order = []; await mapLimit([3, 1, 2], 1, async (x) => { order.push("s" + x); await sleep(x); order.push("e" + x); }); - assert.deepStrictEqual(order, ["s3", "e3", "s1", "e1", "s2", "e2"]); + return { order }; }); -test("first rejection rejects the result; nothing new is started afterwards", async () => { +test("first_rejection", async () => { const started = []; const p = mapLimit([0, 1, 2, 3, 4, 5], 2, async (i) => { started.push(i); if (i === 0) { await sleep(5); throw new Error("boom0"); } await sleep(30); return i; }); - await assert.rejects(p, /boom0/); + const out = await settle(p); await sleep(100); - assert.deepStrictEqual(started, [0, 1]); + return { out, started: [...started] }; }); -test("a synchronous throw becomes a rejection, not an exception", async () => { - let p; +test("sync_throw", async () => { + let p, threw = null; try { p = mapLimit([0, 1, 2], 2, (i) => { if (i === 1) throw new TypeError("sync"); return sleep(5); }); } - catch (e) { assert.fail("mapLimit threw synchronously: " + e.message); } - await assert.rejects(p, TypeError); + catch (e) { threw = String(e && e.message); } + return { threw, out: threw === null ? await settle(p) : null }; }); -test("later failures after the first do not become unhandled rejections", async () => { +test("later_failures", async () => { const before = unhandled; const p = mapLimit([0, 1, 2], 3, async (i) => { await sleep(5 + i * 10); throw new Error("e" + i); }); - await assert.rejects(p, /e0/); + const out = await settle(p); await sleep(80); - assert.strictEqual(unhandled, before, "unhandled rejections leaked"); + return { out, leaked: unhandled - before }; }); -test("invalid limits reject with RangeError and run nothing", async () => { +test("bad_limits", async () => { + const outs = []; for (const bad of [0, -1, 1.5, NaN, "2", Infinity, undefined, null]) { - let called = 0, p; - try { p = mapLimit([1, 2], bad, async () => { called++; }); } catch (e) { assert.fail("threw synchronously for " + String(bad)); } - await assert.rejects(p, (e) => e instanceof RangeError, "limit " + String(bad)); - assert.strictEqual(called, 0); + let called = 0, p, threw = null; + try { p = mapLimit([1, 2], bad, async () => { called++; }); } catch (e) { threw = String(e && e.message); } + const out = threw === null ? await settle(p) : null; + outs.push({ limit: String(bad), threw, out, called }); } + return { outs }; }); -test("many items, many rounds", async () => { +test("many", async () => { const items = [...Array(500).keys()]; - const res = await mapLimit(items, 7, async (x) => { if (x % 50 === 0) await sleep(1); return x * x; }); - assert.deepStrictEqual(res, items.map((x) => x * x)); + return { res: enc(await mapLimit(items, 7, async (x) => { if (x % 50 === 0) await sleep(1); return x * x; })) }; }); (async () => { + const V = { tests: {} }; for (const [name, fn] of tests) { - try { await fn(); } catch (e) { console.log(name + ": " + String(e.message || e).split("\n")[0].slice(0, 160)); process.exit(); } + try { V.tests[name] = await fn(); } catch (e) { V.tests[name] = { error: String((e && e.message) || e).split("\n")[0].slice(0, 160) }; } } await sleep(20); - if (unhandled) { console.log("unhandled rejections: " + unhandled); process.exit(); } - console.log("OK"); + V.unhandled = unhandled; + emit(V); })(); ''' @@ -2663,19 +2759,66 @@ def grade(d: Path): return False, "no pool.js" tmp, w = scratch_copy(d) try: - (w / "pool_test_hidden.js").write_text(POOL_TESTS.replace('require("./pool.js")', 'require("./pool.js")')) - try: - r = runb(["node", "pool_test_hidden.js"], cwd=w, timeout=60) - except subprocess.TimeoutExpired: - return False, "tests hung" - out = (r.stdout.decode().strip().splitlines() or [""])[-1] - if out == "OK" and r.returncode == 0: - return True, "all pool tests" - return False, (out or r.stderr.decode()[-200:] or f"exit {r.returncode}")[:240] + (w / "pool_test_hidden.js").write_text(JS_EMIT + POOL_TESTS) + V, diag = child_values(["node", "pool_test_hidden.js"], w, timeout=60) + if V is None: + return False, ("tests hung" if diag.startswith("timed out") else diag)[:240] + return AsyncPoolNode.verdict(V) finally: shutil.rmtree(tmp, ignore_errors=True) +def _pool_verdict(V) -> tuple[bool, str]: + """The hidden pool tests' assertions, on what the tests observed (POOL_TESTS).""" + if not isinstance(V, dict): + return False, "no observations" + if V.get("load"): + return False, str(V["load"])[:240] + T = V.get("tests") or {} + U = {"$undefined": True} + + def rej(o, pattern=None, flag=None): + return (isinstance(o, dict) and o.get("ok") is False and o.get("promise") is True + and (pattern is None or re.search(pattern, str(o.get("str", ""))) is not None) + and (flag is None or o.get(flag) is True)) + + checks = [ + ("results keep input order and index is passed", lambda t: t["res"] == [x * 2 + i for i, x in enumerate(range(20))]), + ("never more than `limit` in flight, and uses all of them", lambda t: t["max"] == 4), + ("limit larger than the item count", lambda t: t["res"] == [1, 2, 3] and t["max"] == 3), + ("empty input", lambda t: t["res"] == []), + ("plain (non-promise) return values and falsy results", lambda t: t["a"] == [1, 0, 3] and t["b"] == [U, None]), + ("a free slot is refilled at once (no batches)", lambda t: isinstance(t["start"][2], (int, float)) and t["start"][2] < 50 + and isinstance(t["start"][3], (int, float)) and t["start"][3] < 60), + ("limit 1 runs strictly one after another, in order", lambda t: t["order"] == ["s3", "e3", "s1", "e1", "s2", "e2"]), + ("first rejection rejects the result; nothing new is started afterwards", lambda t: rej(t["out"], "boom0") and t["started"] == [0, 1]), + ("a synchronous throw becomes a rejection, not an exception", lambda t: t["threw"] is None and rej(t["out"], flag="type")), + ("later failures after the first do not become unhandled rejections", lambda t: rej(t["out"], "e0") and t["leaked"] == 0), + ("invalid limits reject with RangeError and run nothing", lambda t: len(t["outs"]) == 8 + and all(o["threw"] is None and rej(o["out"], flag="range") and o["called"] == 0 for o in t["outs"])), + ("many items, many rounds", lambda t: t["res"] == [x * x for x in range(500)]), + ] + keys = ["order", "limit", "big_limit", "empty", "plain", "refill", "serial", "first_rejection", "sync_throw", "later_failures", + "bad_limits", "many"] + for key, (name, ok) in zip(keys, checks): + t = T.get(key) + if not isinstance(t, dict): + return False, f"{name}: not run" + if "error" in t: + return False, f"{name}: {t['error']}"[:240] + try: + good = ok(t) + except (KeyError, IndexError, TypeError): + good = False + if not good: + return False, f"{name}: {json.dumps(t)[:160]}" + if V.get("unhandled"): + return False, f"unhandled rejections: {V['unhandled']}" + return True, "all pool tests" + + +AsyncPoolNode.verdict = staticmethod(_pool_verdict) + TASKS3 = [DedupeContacts, MoneySplit, FixDailyBuckets, SemverSort, JsonDiff, RefactorInvoice, SqlGaps, GitSplitHistory, GitFindCulprit, BackupScript, MakeIncremental, CsvToJsonNode, IniToJson, PerfAsofJoin, SlugifyBatch, TextWrap, SqliteMigrate, VendorUnits, RenameApi, SessionizeLogs, LedgerBalance, AsyncPoolNode] diff --git a/bench/local/tests/isolation-in-container.sh b/bench/local/tests/isolation-in-container.sh new file mode 100755 index 0000000..a1b6d51 --- /dev/null +++ b/bench/local/tests/isolation-in-container.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Run the task-isolation and grading-integrity tests where they mean something: as root in the bench image, with the +# unprivileged `agent` user the bench runs the worker as (see test_task_isolation.py and +# test_grading_integrity.py). +# bench/local/tests/isolation-in-container.sh [image] (default maat-bench:agentu) +set -e +here=$(cd "$(dirname "$0")/.." && pwd) +img=${1:-maat-bench:agentu} +docker image inspect "$img" >/dev/null 2>&1 || docker build -q -t "$img" "$here/container" >/dev/null +exec docker run --rm -e PYTHONDONTWRITEBYTECODE=1 -v "$here":/root/bench-src:ro -w /root/bench-src "$img" \ + python3 -m unittest -v tests.test_task_isolation tests.test_grading_integrity diff --git a/bench/local/tests/test_grading_integrity.py b/bench/local/tests/test_grading_integrity.py new file mode 100644 index 0000000..bcb07b6 --- /dev/null +++ b/bench/local/tests/test_grading_integrity.py @@ -0,0 +1,590 @@ +""" +The PR #33 bench audit (reports/grok-reviews/bench-audit-pr33.md), item by item, as the reviewer +reproduced it, with benign markers in place of anything hostile. + +Most tests run anywhere. The ones that need root and the unprivileged agent user (a repository +another user owns, the real /tmp and ~agent) run inside the bench image: + + bench/local/tests/isolation-in-container.sh +""" + +from __future__ import annotations + +import io +import json +import os +import pwd +import shutil +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +HERE = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import grading # noqa: E402 +import refs2 # noqa: E402 +import run # noqa: E402 +import scoreboard as sb # noqa: E402 +import stats # noqa: E402 +import tasks # noqa: E402 +import tasks2 # noqa: E402 +import tasks3 # noqa: E402 +import validate3 # noqa: E402 + +REFS = HERE / "reference_solutions" + + +def fresh(T) -> Path: + d = Path(tempfile.mkdtemp(prefix=f"gi-{T.name}-")) + T.setup(d) + return d + + +def solve_fix_git(d: Path) -> None: + """The reference for fix-git: merge the commit made on the detached HEAD (found in the reflog).""" + log = subprocess.run(["git", "log", "-g", "--format=%H %s", "HEAD"], cwd=d, capture_output=True, text=True).stdout + sha = next(line.split()[0] for line in log.splitlines() if line.endswith("Move to Stanford")) + subprocess.run(["git", "merge", "-q", "--no-edit", sha], cwd=d, check=True, capture_output=True) + + +def solved(T) -> Path: + d = fresh(T) + if T is tasks.FixGit: + solve_fix_git(d) + elif hasattr(refs2, "solve_" + T.name.replace("-", "_")): + getattr(refs2, "solve_" + T.name.replace("-", "_"))(d) + else: + validate3.apply(REFS / T.name, d, REFS / T.name) + return d + + +# ---------------------------------------------------------------- B1 +class B1GitOwnership(unittest.TestCase): + """Correct work in a repository the grader does not own used to fail: git refused it.""" + + def test_git_graders_pass_correct_work_in_a_repository_owned_by_another_user(self): + # GIT_TEST_ASSUME_DIFFERENT_OWNER is git's own switch for "another user owns this": what + # `chown -R agent` did to every task folder before root graded it. (The tasks3 git graders + # also build a reference repo of their own, which the switch would refuse too; they are + # covered as root in the container, with a real chown.) + for T in (tasks.FixGit, tasks2.GitRevert): + d = solved(T) + try: + self.assertTrue(T.grade(d)[0], f"{T.name}: the reference passes when this user owns the repo") + os.environ["GIT_TEST_ASSUME_DIFFERENT_OWNER"] = "1" + try: + ok_old, why_old = T.grade(d) + ok, why, err = grading.safe_grade(T, d) + finally: + del os.environ["GIT_TEST_ASSUME_DIFFERENT_OWNER"] + self.assertFalse(ok_old, f"{T.name}: reproduces the audit (git refused the repo): {why_old}") + self.assertTrue(ok, f"{T.name}: {why}") + self.assertFalse(err) + finally: + shutil.rmtree(d, ignore_errors=True) + + def test_grader_git_does_not_run_the_workers_fsmonitor_or_hooks(self): + d = solved(tasks.FixGit) + try: + marker = d.parent / (d.name + ".fsmonitor-ran") + hook = d / ".git" / "watch.sh" + hook.write_text(f"#!/bin/sh\ntouch '{marker}'\nexit 1\n") + hook.chmod(0o755) + subprocess.run(["git", "config", "core.fsmonitor", str(hook)], cwd=d, check=True) + subprocess.run(["git", "status", "--porcelain"], cwd=d, capture_output=True) + self.assertTrue(marker.exists(), "without the guard, git status runs the repo's fsmonitor command") + marker.unlink() + ok, why, _ = grading.safe_grade(tasks.FixGit, d) + self.assertFalse(marker.exists(), "the grader ran the worker's fsmonitor command") + env = grading.git_env(d) + pairs = {env[f"GIT_CONFIG_KEY_{i}"]: env[f"GIT_CONFIG_VALUE_{i}"] for i in range(int(env["GIT_CONFIG_COUNT"]))} + self.assertEqual(pairs["core.hooksPath"], os.devnull) + self.assertEqual(pairs["core.fsmonitor"], "false") + self.assertEqual(env["GIT_CONFIG_GLOBAL"], os.devnull, "the agent's ~/.gitconfig is not read") + finally: + shutil.rmtree(d, ignore_errors=True) + + +# ---------------------------------------------------------------- B2 +ATEXIT_OK = "\nimport atexit\natexit.register(print, 'OK')\n" + + +class B2VerdictOutOfProcess(unittest.TestCase): + """The worker's module used to be able to print the grader's "OK" itself.""" + + def assertFails(self, T, d, msg): + ok, why = T.grade(d) + self.assertFalse(ok, f"{T.name} passed ({why}): {msg}") + return why + + def test_size_parse_bug_atexit_ok(self): + d = fresh(tasks2.SizeParse) + (d / "sizes.py").write_text((d / "sizes.py").read_text() + ATEXIT_OK) + self.assertFails(tasks2.SizeParse, d, "the buggy file plus atexit.register(print, 'OK')") + + def test_refactor_pricing_atexit_ok(self): + d = fresh(tasks2.RefactorPricing) + (d / "pricing.py").write_text((d / "pricing.py").read_text() + ATEXIT_OK) + self.assertFails(tasks2.RefactorPricing, d, "the original file plus the hook") + + def test_perf_pairs_ok_and_exit_at_import(self): + d = fresh(tasks2.PerfPairs) + (d / "pairs.py").write_text("import os\nprint('OK', flush=True)\nos._exit(0)\n" + (d / "pairs.py").read_text()) + self.assertFails(tasks2.PerfPairs, d, "print('OK'); os._exit(0) at import") + + def test_duration_bug_bad_marker_in_stdout(self): + d = fresh(tasks.DurationBug) + (d / "dur.py").write_text("print('BAD []')\n" + (d / "dur.py").read_text()) + self.assertFails(tasks.DurationBug, d, "'BAD []' anywhere in stdout passed") + + @unittest.skipUnless(shutil.which("node"), "needs node") + def test_async_pool_node_exit_hook(self): + d = fresh(tasks3.AsyncPoolNode) + (d / "pool.js").write_text( + "process.on('exit', () => console.log('OK'));\n" + "exports.mapLimit = async (items, limit, fn) => Promise.all(items.map(fn));\n") # no limit at all + self.assertFails(tasks3.AsyncPoolNode, d, "process.on('exit', () => console.log('OK'))") + + def test_vendor_units_refactor_invoice_rename_api_atexit_ok(self): + d = fresh(tasks3.VendorUnits) + (d / "app" / "retry.py").write_text((d / "app" / "retry.py").read_text() + ATEXIT_OK) + self.assertFails(tasks3.VendorUnits, d, "unfixed retry.py plus the hook") + + d = fresh(tasks3.RefactorInvoice) + (d / "tax.py").write_text("def compute_tax(b, r):\n return 0\n" + ATEXIT_OK) + (d / "format.py").write_text("def money(c, r):\n return ''\n") + self.assertFails(tasks3.RefactorInvoice, d, "stub modules plus the hook") + + d = fresh(tasks3.RenameApi) + validate3.apply(REFS / "_wrong" / "rename-api-signature" / "1", d, REFS / "rename-api-signature") + init = d / "shop" / "__init__.py" + init.write_text(init.read_text() + ATEXIT_OK) + why = self.assertFails(tasks3.RenameApi, d, "a wrong rename plus the hook") + self.assertIn("make_loader", why) + + def test_a_forged_or_second_values_record_fails(self): + # Writing a record of its own to the values channel at import: the grader then sees two. + d = fresh(tasks2.SizeParse) + (d / "sizes.py").write_text( + "import os, __main__\nos.write(__main__._BFD, b'{\"good\": [], \"bad\": []}\\n')\n" + + (d / "sizes.py").read_text()) + why = self.assertFails(tasks2.SizeParse, d, "a forged record at import") + self.assertIn("more than one record", why) + # A forged record and an immediate exit: the values do not cover the inputs. + (d / "sizes.py").write_text( + "import os\nos.write(int(os.environ['BENCH_VALUES_FD']), b'{\"good\": [], \"bad\": []}\\n')\nos._exit(0)\n") + self.assertFails(tasks2.SizeParse, d, "a forged empty record") + + def test_the_references_still_pass(self): + for T in (tasks2.SizeParse, tasks2.RefactorPricing, tasks.DurationBug): + d = fresh(T) + if T is tasks.DurationBug: + (d / "dur.py").write_text( + "import re\ndef parse_duration(s):\n s = s.strip()\n if s.isdigit():\n return int(s)\n" + " m = re.fullmatch(r'(?:(\\d+)h)?(?:(\\d+)m)?(?:(\\d+)s)?', s)\n" + " return sum(int(x or 0) * k for x, k in zip(m.groups(), (3600, 60, 1)))\n") + else: + getattr(refs2, "solve_" + T.name.replace("-", "_"))(d) + self.assertEqual(T.grade(d), (True, "all formats" if T is tasks.DurationBug else "all checks"), T.name) + + +# ---------------------------------------------------------------- run.main harness +def task(name: str, grade=None): + class T: + PROMPT = f"do {name}" + + @staticmethod + def setup(d: Path) -> None: + (d / "input.txt").write_text(f"{name} input\n") + + T.name = name + T.grade = staticmethod(grade or (lambda d: ((d / "out.txt").exists(), "graded"))) + return T + + +class Lane: + """run.main in a temporary layout with a fake agent; everything it touches is put back.""" + + ENV = ("RESULTS_DIR", "RESULTS", "ARMS", "BENCH_TASKS", "BENCH_MODEL", "MAAT_BUILD", "REFERENCE") + + def __init__(self, tasks_, agent=None, **env): + self.tmp = Path(tempfile.mkdtemp(prefix="gi-lane-")) + self.saved = {k: getattr(run, k) for k in ("WORK", "EXPORT", "TASKS", "run_molt", "AGENT_USER", "make_scrubber", "SCRUBBER")} + self.env = {k: os.environ.get(k) for k in self.ENV} + run.WORK, run.EXPORT, run.AGENT_USER = self.tmp / "work", self.tmp / "export", None + run.TASKS = tasks_ + run.run_molt = agent + for k in self.ENV: + os.environ.pop(k, None) + os.environ.update(RESULTS_DIR=str(self.tmp / "results"), RESULTS="r.jsonl", **env) + self.results = self.tmp / "results" / "r.jsonl" + + def main(self, repeats=1, names=None): + return run.main("molt", repeats, names or ",".join(T.name for T in run.TASKS)) + + def rows(self): + return [json.loads(x) for x in self.results.read_text().splitlines()] + + def close(self): + for k, v in self.saved.items(): + setattr(run, k, v) + for k, v in self.env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + shutil.rmtree(self.tmp, ignore_errors=True) + + +def done(d: Path, *args) -> dict: + """A fake agent that does the work: done(d, prompt, log) as run_molt, or done(d, log).""" + log = args[-1] + (d / "out.txt").write_text("x\n") + log.write_text('{"kind":"job_end"}\n') + return {"secs": 0, "turns": 1, "claim": "verified", "said_done": True} + + +# ---------------------------------------------------------------- B3 +class B3NothingCarriesOver(unittest.TestCase): + def test_notes_left_in_tmp_and_home_do_not_reach_the_next_rep_or_arm(self): + lane = Lane([task("carry-a"), task("carry-b")], ARMS="A:X=1;B:X=2") + faketmp, home = lane.tmp / "tmp", lane.tmp / "home" + faketmp.mkdir() + home.mkdir() + (home / "cred.json").write_text('{"token": "benign"}\n') + seen = [] + + def agent(d, prompt, log): + notes = [p for p in (faketmp / ".bench-notes", home / ".bench-notes") if p.exists()] + seen.append((d.name, [p.read_text() for p in notes], (home / "cred.json").read_text())) + (faketmp / ".bench-notes").write_text(f"solution of {d.name}\n") + (home / ".bench-notes").write_text(f"solution of {d.name}\n") + (faketmp / "maat-check-123").mkdir() + (home / "cred.json").write_text("overwritten\n") + return done(d, log) + + run.run_molt = agent + run.make_scrubber = lambda: (lambda sc: (sc.take(), sc)[1])( + run.Scrubber(os.getuid(), home, lane.tmp / "snap" / "home", dirs=[faketmp])) + try: + lane.main(repeats=2) + self.assertEqual(len(seen), 8, "2 tasks x 2 reps x 2 arms") + for name, notes, cred in seen: + self.assertEqual(notes, [], f"{name} read notes an earlier run left") + self.assertEqual(cred, '{"token": "benign"}\n', f"{name} saw a changed credential") + self.assertEqual(os.listdir(faketmp), []) + self.assertEqual(os.listdir(home), ["cred.json"]) + finally: + lane.close() + + def test_scrub_leaves_other_users_files_and_does_not_follow_links(self): + tmp = Path(tempfile.mkdtemp()) + try: + (tmp / "t").mkdir() + outside = tmp / "outside.txt" + outside.write_text("keep\n") + (tmp / "t" / "link").symlink_to(outside) + (tmp / "t" / "f").write_text("x") + sc = run.Scrubber(os.getuid() + 12345, tmp / "h", tmp / "s", dirs=[tmp / "t"]) + self.assertEqual(sc.scrub(), [], "nothing of another uid is removed") + sc = run.Scrubber(os.getuid(), tmp / "h", tmp / "s", dirs=[tmp / "t"]) + self.assertEqual(sorted(Path(p).name for p in sc.scrub()), ["f", "link"]) + self.assertEqual(outside.read_text(), "keep\n", "the link was removed, not followed") + finally: + shutil.rmtree(tmp) + + def test_arm_order_alternates(self): + order = [] + lane = Lane([task("ab-0"), task("ab-1")], ARMS="A:X=1;B:X=2") + + def agent(d, prompt, log): + order.append(d.name) + return done(d, log) + + run.run_molt = agent + try: + lane.main(repeats=2) + self.assertEqual(order, ["ab-0-molt-0-A", "ab-0-molt-0-B", "ab-1-molt-0-B", "ab-1-molt-0-A", + "ab-0-molt-1-B", "ab-0-molt-1-A", "ab-1-molt-1-A", "ab-1-molt-1-B"]) + self.assertEqual({(r["task"], r["rep"], r["arm"]): r["arm_pos"] for r in lane.rows()}[("ab-1", 0, "B")], 0) + finally: + lane.close() + + +# ---------------------------------------------------------------- B4 +class B4GraderErrors(unittest.TestCase): + def test_a_grader_that_raises_is_a_failed_row_and_the_lane_goes_on(self): + def boom(d): + raise UnicodeDecodeError("utf-8", b"\xff", 0, 1, "benign marker") + + lane = Lane([task("ge-a", boom), task("ge-b")], agent=done) + try: + lane.main() + rows = lane.rows() + self.assertEqual([r["task"] for r in rows], ["ge-a", "ge-b"], "the lane went on to the next task") + self.assertEqual((rows[0]["passed"], rows[0]["grader_error"]), (False, True)) + self.assertIn("grader error", rows[0]["why"]) + self.assertEqual((rows[1]["passed"], rows[1]["grader_error"]), (True, False)) + # resume: the failed row is recorded, so it is not run again + ran = [] + run.run_molt = lambda d, p, log: (ran.append(d.name), done(d, log))[1] + lane.main() + self.assertEqual(ran, []) + finally: + lane.close() + + def test_cheap_end_states_no_longer_raise(self): + d = fresh(tasks.DurationBug) + (d / "test_dur.py").unlink() + self.assertEqual(grading.safe_grade(tasks.DurationBug, d), (False, "the tests were changed", False)) + d = fresh(tasks3.RenameApi) + (d / "CHANGELOG.md").unlink() + self.assertEqual(grading.safe_grade(tasks3.RenameApi, d)[1:], ("CHANGELOG.md was modified", False)) + d = Path(tempfile.mkdtemp()) + (d / "wc.py").write_text("import sys\nsys.stdout.buffer.write(b'\\xff\\n')\n") + ok, why, err = grading.safe_grade(tasks.Wc, d) + self.assertEqual((ok, err), (False, False), why) + + def test_a_hang_is_a_fail_not_a_grader_error(self): + def hang(d): + subprocess.run(["sleep", "5"], timeout=0.1) + + self.assertEqual(grading.safe_grade(task("hang", hang), Path("."))[0::2], (False, False)) + + +# ---------------------------------------------------------------- B5 +class B5ProviderCap(unittest.TestCase): + PHRASE = "the provider's rate limit is reached until 10/7/2026, 12:00:00 AM" + + def test_worker_text_does_not_stop_the_lane(self): + tool = json.dumps({"kind": "tool", "name": "bash", "args": "cat app.log", "preview": self.PHRASE}) + text = json.dumps({"kind": "text", "text": "echo " + self.PHRASE}) + self.assertFalse(run.provider_capped(tool + "\n" + text + "\n", 3)) + self.assertFalse(run.provider_capped(json.dumps({"kind": "tool", "preview": "free-models-per-day"}), 0)) + + def test_maats_error_event_stderr_and_plain_lines_still_do(self): + self.assertTrue(run.provider_capped(json.dumps({"kind": "error", "text": self.PHRASE}) + "\n", 3)) + self.assertTrue(run.provider_capped("", 3, err=self.PHRASE)) + self.assertTrue(run.provider_capped(self.PHRASE + "\n", 3)) + + def test_stopped_exits_non_zero_and_records_nothing(self): + lane = Lane([task("cap-a"), task("cap-b")], agent=lambda d, p, log: {"provider_capped": True}) + try: + with redirect_stdout(io.StringIO()) as out, self.assertRaises(SystemExit) as e: + lane.main() + self.assertEqual(e.exception.code, run.STOPPED_EXIT) + self.assertNotEqual(e.exception.code, 0) + self.assertIn("STOPPED", out.getvalue()) + self.assertFalse(lane.results.exists() and lane.results.read_text().strip()) + finally: + lane.close() + + +# ---------------------------------------------------------------- B6 +class B6LaneMetadata(unittest.TestCase): + def test_rows_carry_the_lane(self): + lane = Lane([task("ln-a")], agent=done, BENCH_MODEL="model-a", ARMS="j:MAAT_JUDGE_MODEL=judge-x,MAAT_JUDGE_KEY=sk-benign", + MAAT_BUILD="abc12345") + try: + lane.main() + r = lane.rows()[0] + L = r["lane"] + self.assertEqual(L["model"], "model-a") + self.assertEqual((L["reasoning"], L["limit"], L["gate"], L["build"]), ("low", run.LIMIT, "sandbox", "abc12345")) + self.assertIn("url", L) + self.assertEqual(L["arms"], {"j": {"MAAT_JUDGE_MODEL": "judge-x"}}, "judge settings recorded, the key left out") + self.assertEqual(L["grader"], run.grader_hash()) + self.assertEqual(r["lane_id"], run.lane_id(L)) + finally: + lane.close() + + def test_another_lane_is_not_appended_to_the_same_file(self): + lane = Lane([task("ln-a"), task("ln-b")], agent=done, BENCH_MODEL="model-a") + try: + lane.main() + before = lane.results.read_text() + os.environ["BENCH_MODEL"] = "model-b" + with self.assertRaises(SystemExit) as e: + lane.main() + self.assertIn("another lane", str(e.exception.code)) + self.assertEqual(lane.results.read_text(), before) + # legacy rows with no lane at all are refused too + lane.results.write_text(json.dumps({"task": "ln-a", "agent": "molt", "rep": 0, "passed": True}) + "\n") + with self.assertRaises(SystemExit): + lane.main() + finally: + lane.close() + + def test_a_truncated_last_line_is_tolerated_on_resume(self): + lane = Lane([task("tr-a"), task("tr-b")], agent=done) + try: + lane.main(names="tr-a") + good = lane.results.read_text() + lane.results.write_text(good + '{"task": "tr-b", "agent": "mo') + with redirect_stdout(io.StringIO()): + lane.main() + rows = lane.rows() + self.assertEqual([r["task"] for r in rows], ["tr-a", "tr-b"]) + self.assertTrue(lane.results.with_name("r.jsonl.partial").read_text().startswith('{"task": "tr-b"')) + finally: + lane.close() + + +# ---------------------------------------------------------------- non-blocking +class N1SymlinksIntoTheBench(unittest.TestCase): + def test_a_link_to_a_reference_solution_fails_before_grading(self): + d = Path(tempfile.mkdtemp()) + try: + (d / "solve.sh").symlink_to(REFS / "git-find-culprit" / "solve.sh") + (d / "own").write_text("x") + (d / "ok-link").symlink_to(d / "own") + ran = [] + T = task("n1", lambda dd: (ran.append(1), (True, "followed"))[1]) + saved, run.AGENT_USER = run.AGENT_USER, None # this process grades (the image sets an agent user) + self.addCleanup(setattr, run, "AGENT_USER", saved) + ok, why, err = run.grade(T, d) + self.assertFalse(ok) + self.assertIn("solve.sh", why) + self.assertEqual(ran, [], "no grader ran") + (d / "solve.sh").unlink() + self.assertTrue(run.grade(T, d)[0], "a link inside the task folder is fine") + finally: + shutil.rmtree(d) + + +class N2WcScratch(unittest.TestCase): + def test_a_premade_wc_grade_folder_is_not_written_through(self): + tmp = Path(tempfile.mkdtemp()) + saved = os.environ.get("TMPDIR") + try: + victim = tmp / "victim.txt" + victim.write_text("benign\n") + (tmp / "wc-grade").mkdir() + (tmp / "wc-grade" / "a.txt").symlink_to(victim) + os.environ["TMPDIR"] = str(tmp) + tempfile.tempdir = None + d = tmp / "task" + d.mkdir() + (d / "wc.py").write_text("print('x')\n") + tasks.Wc.grade(d) + self.assertEqual(victim.read_text(), "benign\n") + finally: + if saved is None: + os.environ.pop("TMPDIR", None) + else: + os.environ["TMPDIR"] = saved + tempfile.tempdir = None + shutil.rmtree(tmp) + + +class N4PooledPairs(unittest.TestCase): + def test_pooled_runs_are_not_paired_twice(self): + rows = lambda arm, passes: [{"task": f"t{i}", "agent": "molt", "rep": 0, "arm": arm, "passed": p} for i, p in enumerate(passes)] + a = rows("A", [1, 0] * 5) + rows("A", [1, 0] * 5) # two base files under one label + b = rows("B", [1, 1] * 5) + rows("B", [1, 1] * 5) + with self.assertRaises(sb.Ambiguous): + sb.pair_up(a, b) + lines, verdict = sb.compare("A", "B", a, b, "pass", 0.0) + self.assertIn("NO VERDICT", verdict) + self.assertTrue(any("by task (10 tasks" in x for x in lines), lines) + + +class N7Stalls(unittest.TestCase): + def test_stalled_and_grader_error_rows_are_counted_apart(self): + rows = [{"task": "a", "passed": True}, {"task": "b", "passed": False, "providerStall": True}, + {"task": "c", "passed": False, "grader_error": True}, {"task": "d", "passed": False}] + m = sb.metrics(rows) + self.assertEqual((m["n"], m["pass"], m["stall"], m["grader_error"]), (2, 1, 1, 1)) + + +class TaskLevelStats(unittest.TestCase): + def test_repeats_are_averaged_within_a_task(self): + # 3 tasks: one passes 3/3, one 0/3, one 1/3 -> mean of fractions 4/9, not rows 4/9 by luck + rows = [{"task": "x", "passed": True}] * 3 + [{"task": "y", "passed": False}] * 3 + \ + [{"task": "z", "passed": p} for p in (True, False, False)] + [{"task": "x", "passed": False, "grader_error": True}] + self.assertEqual(stats.per_task(rows), {"x": (3, 3), "y": (0, 3), "z": (1, 3)}) + tr = stats.task_rate(rows) + self.assertEqual(tr["tasks"], 3) + self.assertAlmostEqual(tr["mean"], 4 / 9) + self.assertLessEqual(tr["lo"], tr["mean"]) + self.assertGreaterEqual(tr["hi"], tr["mean"]) + + def test_paired_by_task(self): + a = [{"task": f"t{i}", "passed": i < 5} for i in range(10) for _ in range(3)] + b = [{"task": f"t{i}", "passed": i < 8} for i in range(10) for _ in range(3)] + pt = stats.paired_tasks(a, b) + self.assertEqual((pt["tasks"], pt["b_better"], pt["a_better"]), (10, 3, 0)) + self.assertAlmostEqual(pt["mean_diff"], 0.3) + self.assertAlmostEqual(pt["perm_p"], 2 / 8) # 3 equal nonzero diffs: only all-same signs reach |sum| + self.assertAlmostEqual(pt["sign_p"], 0.25) + # the 30 row pairs would have said 9 discordant, p = 0.004: the repeats are not independent + self.assertLess(stats.sign_test(0, 9), 0.01) + + +# ---------------------------------------------------------------- as root, in the bench image +def have_agent_user() -> bool: + if os.geteuid() != 0 or not shutil.which("runuser"): + return False + try: + pwd.getpwnam(os.environ.get("BENCH_AGENT_USER") or "agent") + return True + except KeyError: + return False + + +@unittest.skipUnless(have_agent_user(), "needs root and the agent user: run isolation-in-container.sh") +class AsRootInTheImage(unittest.TestCase): + def setUp(self): + self.user = os.environ.get("BENCH_AGENT_USER") or "agent" + self.pw = pwd.getpwnam(self.user) + + def test_b1_all_four_git_tasks_pass_correct_work_owned_by_the_agent(self): + saved = run.AGENT_USER + try: + for T in (tasks.FixGit, tasks2.GitRevert, tasks3.GitSplitHistory, tasks3.GitFindCulprit): + d = solved(T) + os.chmod(d, 0o755) + subprocess.run(["chown", "-R", self.user, str(d)], check=True) + ok_old, why_old = T.grade(d) + self.assertFalse(ok_old, f"{T.name}: reproduces the audit as root without the guard: {why_old}") + self.assertEqual(grading.safe_grade(T, d)[0::2], (True, False), f"{T.name} as root with the guard") + run.AGENT_USER = self.user + ok, why, err = run.grade(T, d) + run.AGENT_USER = saved + self.assertTrue(ok, f"{T.name} as the agent user: {why}") + shutil.rmtree(d) + finally: + run.AGENT_USER = saved + + def test_b3_the_real_tmp_dirs_and_home_are_reset(self): + snap = Path(tempfile.mkdtemp(dir="/root")) / "home" + home = Path(self.pw.pw_dir) + (home / ".keep-me").write_text("cred\n") + subprocess.run(["chown", self.user, str(home / ".keep-me")], check=True) + sc = run.Scrubber(self.pw.pw_uid, home, snap) + sc.take() + mine = Path(tempfile.mkdtemp(dir="/tmp", prefix="root-own-")) + try: + script = ("echo n > /tmp/.bench-notes; echo n > /var/tmp/.bench-notes; mkdir -p /tmp/maat-check-1/x; " + "[ -d /dev/shm ] && echo n > /dev/shm/.bench-notes; echo n > ~/.bench-notes; echo changed > ~/.keep-me; " + f"mkdir ~/.maat; true") + cmd, _ = run.as_agent(["sh", "-c", script]) + subprocess.run(cmd, check=True) + sc.reset() + for p in ("/tmp/.bench-notes", "/var/tmp/.bench-notes", "/tmp/maat-check-1", "/dev/shm/.bench-notes", + str(home / ".bench-notes"), str(home / ".maat")): + self.assertFalse(os.path.lexists(p), p) + self.assertEqual((home / ".keep-me").read_text(), "cred\n") + self.assertEqual(os.stat(home / ".keep-me").st_uid, self.pw.pw_uid) + self.assertTrue(mine.exists(), "root's own files in /tmp are left alone") + finally: + shutil.rmtree(mine, ignore_errors=True) + (home / ".keep-me").unlink(missing_ok=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/bench/local/tests/test_task_isolation.py b/bench/local/tests/test_task_isolation.py new file mode 100644 index 0000000..f95f721 --- /dev/null +++ b/bench/local/tests/test_task_isolation.py @@ -0,0 +1,299 @@ +""" +A task cannot list or read another task's folder or logs. + +2026-10-07, ml4b lanes: a worker spent 90-125 tool calls per task reading the other tasks' +folders and Maat logs in the shared, host-mounted /work instead of working. Each task now runs +in a private folder (BENCH_WORK//), its logs go to a root-only folder, and both are +copied to the export only after the task is graded. + +The first test runs anywhere (a Mac host lane included): it checks the layout. The second needs +the real thing, root and the unprivileged agent user, and runs inside the bench image: + + bench/local/tests/isolation-in-container.sh +""" + +import json +import os +import pwd +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import run # noqa: E402 + + +def task(name: str): + class T: + PROMPT = f"do {name}" + + @staticmethod + def setup(d: Path) -> None: + (d / "input.txt").write_text(f"{name} input\n") + + @staticmethod + def grade(d: Path): + return (d / "out.txt").exists(), "graded" + + T.name = name + return T + + +class Harness: + """run.main over two fake tasks, with a fake agent that records what it could see.""" + + def __init__(self, work: Path, export: Path, results: Path, agent=None): + self.saved = {k: getattr(run, k) for k in ("WORK", "EXPORT", "TASKS", "run_molt", "AGENT_USER")} + self.env = {k: os.environ.get(k) for k in ("RESULTS_DIR", "RESULTS", "ARMS", "BENCH_TASKS")} + run.WORK, run.EXPORT, run.AGENT_USER = work, export, agent + run.TASKS = [task("iso-a"), task("iso-b")] + os.environ.update(RESULTS_DIR=str(results), RESULTS="iso.jsonl") + for k in ("ARMS", "BENCH_TASKS"): + os.environ.pop(k, None) + self.seen: list[dict] = [] + + def restore(self): + for k, v in self.saved.items(): + setattr(run, k, v) + for k, v in self.env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +class Layout(unittest.TestCase): + def test_tasks_run_in_private_folders_and_only_finished_work_is_exported(self): + tmp = Path(tempfile.mkdtemp()) + h = Harness(tmp / "work", tmp / "export", tmp / "results") + try: + def fake(d: Path, prompt: str, log: Path) -> dict: + h.seen.append({ + "dir": d, "log": log, + # what is left of the tasks before this one, where this one runs + "box_listing": sorted(os.listdir(d.parent)), + "earlier_dirs_exist": [s["dir"].exists() for s in h.seen], + "earlier_logs_exist": [s["log"].exists() for s in h.seen], + "exported_so_far": sorted(os.listdir(tmp / "export")) if (tmp / "export").exists() else [], + }) + (d / "out.txt").write_text("x\n") + log.write_text('{"kind":"job_end"}\n') + return {"secs": 0, "turns": 0, "claim": "verified", "said_done": True} + + run.run_molt = fake + run.main("molt", 1, "iso-a,iso-b") + finally: + h.restore() + a, b = h.seen + for s in h.seen: + self.assertFalse(s["dir"].is_relative_to(tmp / "export"), "a task never runs in the export") + self.assertNotEqual(s["log"].parent, s["dir"], "logs are not written into a task folder") + self.assertEqual(s["box_listing"], [s["dir"].name], "the private parent holds this task only") + self.assertNotEqual(a["dir"].parent, b["dir"].parent, "each task has its own random parent") + self.assertEqual(b["earlier_dirs_exist"], [False], "task a's private folder is gone before task b starts") + self.assertEqual(b["earlier_logs_exist"], [False], "task a's private log is gone before task b starts") + self.assertIn("iso-a-molt-0", b["exported_so_far"], "task a was exported when it finished") + self.assertNotIn("iso-b-molt-0", b["exported_so_far"]) + out = tmp / "export" + for t in ("iso-a-molt-0", "iso-b-molt-0"): + self.assertTrue((out / t / "out.txt").exists()) + self.assertTrue((out / f"{t}.log").exists()) + rows = [json.loads(x) for x in (tmp / "results" / "iso.jsonl").read_text().splitlines()] + self.assertEqual([r["passed"] for r in rows], [True, True]) + # nothing of either task is left where tasks run, but the (empty) root-only log folder + self.assertEqual(sorted(os.listdir(tmp / "work")), [".logs"]) + self.assertEqual(os.listdir(tmp / "work" / ".logs"), []) + shutil.rmtree(tmp) + + +class SpecialFilesAndOrder(unittest.TestCase): + def test_a_fifo_or_socket_in_the_task_folder_does_not_stop_the_lane(self): + import socket + tmp = Path(tempfile.mkdtemp()) + h = Harness(tmp / "work", tmp / "export", tmp / "results") + socks = [] + try: + def fake(d: Path, prompt: str, log: Path) -> dict: + os.mkfifo(d / "pipe") + s = socket.socket(socket.AF_UNIX) + # AF_UNIX paths are short; bind from inside the folder. + cwd = os.getcwd() + os.chdir(d) + try: + s.bind("srv.sock") + finally: + os.chdir(cwd) + socks.append(s) + (d / "out.txt").write_text("x\n") + log.write_text('{"kind":"job_end"}\n') + return {"secs": 0, "turns": 0, "claim": "verified", "said_done": True} + + run.run_molt = fake + run.main("molt", 1, "iso-a,iso-b") + finally: + h.restore() + for s in socks: + s.close() + rows = [json.loads(x) for x in (tmp / "results" / "iso.jsonl").read_text().splitlines()] + self.assertEqual([r["task"] for r in rows], ["iso-a", "iso-b"], "both tasks ran and were recorded") + self.assertEqual([r["passed"] for r in rows], [True, True]) + for t in ("iso-a-molt-0", "iso-b-molt-0"): + self.assertTrue((tmp / "export" / t / "out.txt").exists(), "the regular files were exported") + self.assertFalse((tmp / "export" / t / "pipe").exists(), "the FIFO was skipped") + self.assertFalse((tmp / "export" / t / "srv.sock").exists(), "the socket was skipped") + self.assertEqual(sorted(os.listdir(tmp / "work")), [".logs"], "no private folder left behind") + shutil.rmtree(tmp) + + def test_agent_processes_are_killed_before_grading_and_the_result_is_written_before_the_export(self): + tmp = Path(tempfile.mkdtemp()) + h = Harness(tmp / "work", tmp / "export", tmp / "results") + events: list[str] = [] + saved = (run.kill_agent_procs, run.finish_task) + results = tmp / "results" / "iso.jsonl" + try: + def fake(d: Path, prompt: str, log: Path) -> dict: + events.append("agent") + (d / "out.txt").write_text("x\n") + return {"secs": 0, "turns": 0, "claim": "verified", "said_done": True} + + real_finish = run.finish_task + run.run_molt = fake + run.kill_agent_procs = lambda: events.append("kill") + run.finish_task = lambda d, logs: (events.append("export:" + str(len(results.read_text().splitlines()) if results.exists() else 0)), real_finish(d, logs)) + grade_a = run.TASKS[0].grade + run.TASKS[0].grade = staticmethod(lambda d: (events.append("grade"), grade_a(d))[1]) + run.main("molt", 1, "iso-a") + finally: + run.kill_agent_procs, run.finish_task = saved + h.restore() + self.assertEqual(events[:3], ["agent", "kill", "grade"], events) + self.assertIn("export:1", events, "the row was on disk before the export ran") + shutil.rmtree(tmp) + + +def have_agent_user() -> bool: + if os.geteuid() != 0 or not shutil.which("runuser"): + return False + try: + pwd.getpwnam(os.environ.get("BENCH_AGENT_USER") or "agent") + return True + except KeyError: + return False + + +@unittest.skipUnless(have_agent_user(), "needs root and the agent user: run isolation-in-container.sh") +class AsTheAgentUser(unittest.TestCase): + def test_the_agent_cannot_list_or_read_another_tasks_folder_or_logs(self): + user = os.environ.get("BENCH_AGENT_USER") or "agent" + # As in the container: tasks under a container-local folder, the export under /root (700). + os.chmod("/root", 0o700) + work = Path(tempfile.mkdtemp(dir="/var/tmp")) / "bench-work" + os.chmod(work.parent, 0o755) # like /var/lib: the agent may pass through to BENCH_WORK + export = Path(tempfile.mkdtemp(dir="/root")) / "bench-export" + results = Path(tempfile.mkdtemp(dir="/root")) + h = Harness(work, export, results, agent=user) + + def as_agent(script: str) -> subprocess.CompletedProcess: + cmd, _ = run.as_agent(["sh", "-c", script]) + return subprocess.run(cmd, capture_output=True, text=True) + + try: + def fake(d: Path, prompt: str, log: Path) -> dict: + probes = { + "own_input": as_agent(f"cat {d}/input.txt"), + "own_write": as_agent(f"echo x > {d}/out.txt"), + "list_work": as_agent(f"ls {work}"), + "list_logs": as_agent(f"ls {work}/.logs"), + "list_box": as_agent(f"ls {d.parent}"), + "list_export": as_agent(f"ls {export}"), + } + if h.seen: + a = h.seen[0] + probes |= { + "other_dir": as_agent(f"ls {a['dir']}"), + "other_input": as_agent(f"cat {a['dir']}/input.txt"), + "other_log_private": as_agent(f"cat {a['log']}"), + "other_dir_exported": as_agent(f"ls {export}/{a['dir'].name}"), + "other_log_exported": as_agent(f"cat {export}/{a['log'].name}"), + "find_logs": as_agent(f"find / -xdev -name '*.log' -path '*iso-*' 2>/dev/null"), + "find_inputs": as_agent(f"grep -rl 'iso-a input' / --exclude-dir=proc --exclude-dir=sys 2>/dev/null"), + } + h.seen.append({"dir": d, "log": log, "probes": probes}) + log.write_text('{"kind":"job_end","secret":"iso-a log"}\n') + return {"secs": 0, "turns": 0, "claim": "verified", "said_done": True} + + run.run_molt = fake + run.main("molt", 1, "iso-a,iso-b") + finally: + h.restore() + a, b = h.seen + for s in h.seen: + p = s["probes"] + self.assertEqual(p["own_input"].returncode, 0, p["own_input"].stderr) + self.assertEqual(p["own_write"].returncode, 0, p["own_write"].stderr) + for k in ("list_work", "list_logs", "list_box", "list_export"): + self.assertNotEqual(p[k].returncode, 0, f"{k} worked: {p[k].stdout}") + p = b["probes"] + for k in ("other_dir", "other_input", "other_log_private", "other_dir_exported", "other_log_exported"): + self.assertNotEqual(p[k].returncode, 0, f"task b could {k}: {p[k].stdout}") + self.assertEqual(p["find_logs"].stdout.strip(), "", "task b found task a's log") + self.assertEqual(p["find_inputs"].stdout.strip(), "", "task b found task a's files") + # Exported for the owner all the same. + self.assertTrue((export / "iso-a-molt-0" / "out.txt").exists()) + self.assertIn("iso-a log", (export / "iso-a-molt-0.log").read_text()) + + +@unittest.skipUnless(have_agent_user(), "needs root and the agent user: run isolation-in-container.sh") +class GradedAsTheAgentUser(unittest.TestCase): + def test_the_workers_code_runs_unprivileged_during_grading_and_cannot_reach_the_results(self): + user = os.environ.get("BENCH_AGENT_USER") or "agent" + uid = pwd.getpwnam(user).pw_uid + os.chmod("/root", 0o700) + work = Path(tempfile.mkdtemp(dir="/var/tmp")) / "bench-work" + os.chmod(work.parent, 0o755) + export = Path(tempfile.mkdtemp(dir="/root")) / "bench-export" + results = Path(tempfile.mkdtemp(dir="/root")) + h = Harness(work, export, results, agent=user) + os.environ["FAKE_API_KEY"] = "sk-must-not-leak" + seen: dict = {} + try: + def fake(d: Path, prompt: str, log: Path) -> dict: + # A hostile "solution": run by the grader, it tries to forge a result and read secrets. + (d / "server.py").write_text( + "import os, glob\n" + "out = {'uid': os.getuid(), 'key': os.environ.get('FAKE_API_KEY')}\n" + f"try:\n open('{results}/iso.jsonl', 'a').write('forged\\n'); out['forged'] = True\n" + "except OSError:\n out['forged'] = False\n" + "print(repr(out))\n" + ) + (d / "out.txt").write_text("x\n") + os.chown(d / "server.py", uid, -1) + return {"secs": 0, "turns": 0, "claim": "verified", "said_done": True} + + def grade(d: Path): + seen["grader_uid"] = os.getuid() + p = subprocess.run(["python3", "server.py"], cwd=d, capture_output=True, text=True) + seen["worker"] = p.stdout.strip() + return True, "graded" + + run.run_molt = fake + run.TASKS[0].grade = staticmethod(grade) + run.main("molt", 1, "iso-a") + # The grader ran in a child: what it saw comes back only through its verdict, so ask again directly. + ok, why, _ = run.grade(type("T", (), {"grade": staticmethod(lambda d: (True, repr((os.getuid(), subprocess.run(["python3", "-c", "import os;print(os.getuid(), os.environ.get('FAKE_API_KEY'))"], capture_output=True, text=True).stdout.strip()))))}), Path("/tmp")) + finally: + h.restore() + os.environ.pop("FAKE_API_KEY", None) + rows = (results / "iso.jsonl").read_text().splitlines() + self.assertEqual(len(rows), 1, rows) + self.assertNotIn("forged", "\n".join(rows)) + self.assertTrue(json.loads(rows[0])["passed"]) + self.assertTrue(ok) + self.assertEqual(why, repr((uid, f"{uid} None")), "the grader and what it runs are the agent user, without the secrets") + + +if __name__ == "__main__": + unittest.main() diff --git a/src/engine.ts b/src/engine.ts index 3f13970..7592a20 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -133,6 +133,7 @@ import { type Spend, } from "./types.js"; import { stateDir, stateDirName } from "./statedir.js"; +import { bashReach, noProgressCallsFromEnv, outsideTask, ProgressMeter, taskPathsIn, treeStamp } from "./scope.js"; import { env } from "./env.js"; import { Judgments, caseReason, reasonText } from "./judgment.js"; @@ -1568,6 +1569,13 @@ export type EngineConfig = { * case this exists for, and it is off everywhere else. */ sandbox?: boolean; + /** + * Unattended: tool calls in a row that change no file in the project before + * the model is told to finish or stop; as many again and the turn ends, + * judged on disk ("no-progress"). 0 turns the guard off. Default: + * MAAT_NO_PROGRESS_CALLS, else 30 (src/scope.ts). + */ + noProgressCalls?: number; /** * Where to write one JSON file per completion attempt holding the whole * turn: the wire transcript, the ledger, the bar's result, the receipt name. @@ -2272,7 +2280,7 @@ export class Engine { * all because the clock or a failed request stopped the turn before any check * ran; two of them had passing work on disk. */ - private turnEndedBy: "deadline" | "provider" | undefined; + private turnEndedBy: "deadline" | "provider" | "no-progress" | undefined; /** A subprocess backend went silent past the stall allowance this turn (a provider issue). */ private turnProviderStall = false; /** `--revert` put this turn's work back, so a person cannot accept it as it stands. */ @@ -2542,6 +2550,43 @@ export class Engine { * written — when the job's work is over. */ private withheld: { name: string; text: string }[] = []; + /** Absolute paths this turn's task names: part of the task, though outside the project (src/scope.ts). */ + private turnTaskPaths: string[] = []; + /** This turn's checks are hidden, or are being drafted to be (src/scope.ts). */ + private turnHidesChecks = false; + + /** + * Whether the file tools are held to the task: an unattended job with + * hidden checks, while it runs. A person at the keyboard can read what they + * like; a worker alone with checks it cannot see spent whole budgets + * reading Maat's records and other tasks' logs instead (src/scope.ts). + */ + private scoped(): boolean { + return this.cfg.unattended === true && this.turnActive && (this.turnHidesChecks || this.withheld.length > 0); + } + + /** The refusal for a file tool's path outside the task, journalled; null when the path is in scope. */ + private refuseOutsideTask(tool: string, path: string): string | null { + if (!this.scoped()) return null; + const v = outsideTask(this.cwd, path, { tool, taskPaths: this.turnTaskPaths }); + if (!v) return null; + this.cfg.journal?.append("outside_task", { tool, path: path || ".", reach: v.kind, refused: true }); + return v.message; + } + + /** A bash command that names `.maat/` or a path outside the project: not blocked, journalled. */ + private noteBashReach(command: string): void { + if (!this.scoped()) return; + const r = bashReach(this.cwd, command, this.turnTaskPaths); + if (!r.stateDir && r.outside.length === 0) return; + this.cfg.journal?.append("outside_task", { + tool: "bash", + refused: false, + stateDir: r.stateDir, + outside: r.outside, + command: command.slice(0, 500), + }); + } /** The effort the next request carries. */ private get effortNow(): string | undefined { @@ -3266,6 +3311,8 @@ export class Engine { case "inspect": { const rel = String(args.path ?? "."); const abs = resolve(this.cwd, rel); + const off = this.refuseOutsideTask("inspect", rel); + if (off) return off; if (!existsSync(abs)) return `${rel} does not exist`; try { return statSync(abs).isDirectory() @@ -3276,7 +3323,9 @@ export class Engine { } } - case "read_file": + case "read_file": { + const off = this.refuseOutsideTask("read_file", String(args.path ?? "")); + if (off) return off; this.readPaths.add(String(args.path ?? "")); return readPart( await this.readText(resolve(this.cwd, String(args.path ?? ""))), @@ -3285,6 +3334,7 @@ export class Engine { num(args.limit, Number.MAX_SAFE_INTEGER), this.resultBudget(), ); + } case "write_file": { const rel = String(args.path ?? ""); @@ -3343,6 +3393,8 @@ export class Engine { case "list_dir": { const rel = String(args.path ?? "."); const abs = resolve(this.cwd, rel); + const off = this.refuseOutsideTask("list_dir", rel); + if (off) return off; this.mustBeInside(abs, rel); // Bounded and off the main thread: a listing the model asks for can be // pointed at anything, including a home directory. @@ -3359,6 +3411,8 @@ export class Engine { case "grep": { const rel = String(args.path ?? "."); const abs = resolve(this.cwd, rel); + const off = this.refuseOutsideTask("grep", rel); + if (off) return off; this.mustBeInside(abs, rel); const pattern = String(args.pattern ?? ""); return formatMatches( @@ -3417,6 +3471,7 @@ export class Engine { // a caller that reaches runTool directly (a subscription backend). foldCommands(args, this.cwd); const command = String(args.command ?? ""); + this.noteBashReach(command); if (!command.trim() && args.stop_job === undefined) { return "bash needs a command, or commands: a list to run in order"; } @@ -4008,7 +4063,7 @@ export class Engine { result: BarResult, attempts: number, log?: Journal, - endedBy?: "deadline" | "provider", + endedBy?: "deadline" | "provider" | "no-progress", ): AsyncGenerator { const onlyWrites = failedOnlyWriteChecks(result); if (this.cfg.receipts) { @@ -4312,6 +4367,8 @@ export class Engine { this.sealedChecks = []; this.sealedNotes = []; this.turnCalls = new Set(); + this.turnTaskPaths = taskPathsIn(userText); + this.turnHidesChecks = false; // molt's records stay out of the project's `git status` (leftovers.ts). excludeMoltFromGit(this.cwd); // Per turn, or receipt five lists what turn one ran. It did: receipts @@ -4380,7 +4437,7 @@ export class Engine { ? opts.ask && this.turnWrites.length === 0 ? "answered" : "verified" - : answered || this.turnEndedBy === "deadline" + : answered || this.turnEndedBy === "deadline" || this.turnEndedBy === "no-progress" ? "unverified" : "stopped"; @@ -4396,7 +4453,7 @@ export class Engine { outcome = "unverified"; yield { kind: "info", - text: `the checks passed on the work as it stood when the ${this.turnEndedBy === "deadline" ? "clock" : "provider"} stopped the model, but it never said it was done — unverified.`, + text: `the checks passed on the work as it stood when the ${this.turnEndedBy === "deadline" ? "clock" : this.turnEndedBy === "no-progress" ? "no-progress guard" : "provider"} stopped the model, but it never said it was done — unverified.`, }; } @@ -5309,6 +5366,8 @@ export class Engine { */ let pendingCriteria = opts.pendingCriteria; const criteriaSince = Date.now(); + // Drafted checks are hidden; given ones may be. Either way the file tools are held to the task. + this.turnHidesChecks = Boolean(pendingCriteria) || (opts.taskChecks ?? []).some((c) => c.hidden === true); /** * The draft, when the time budget's cut found it with no checks: kept alive * to join at the first claim instead of sealing nothing (joinLateCriteria). @@ -5757,7 +5816,7 @@ export class Engine { * is what it says: verified only if it passed, not proven if it did not, * unverified when nothing was sealed. Returns whether a bar ran. */ - async function* judgeOnDisk(why: "deadline" | "provider", claim: string): AsyncGenerator { + async function* judgeOnDisk(why: "deadline" | "provider" | "no-progress", claim: string): AsyncGenerator { self.turnEndedBy = why; if (opts.ask) return false; let barThis = barNow(); @@ -5873,6 +5932,26 @@ export class Engine { * rather than left to re-measure and perhaps disagree. */ let deadlineInterrupted = false; + /** + * The no-progress guard, unattended only (src/scope.ts). Tool calls in a + * row that advanced nothing: no file in the project changed, and each + * call returned only what it had returned before (ProgressMeter). Reading + * something new, running a command for the first time, or getting a + * different answer from one is progress, wherever the work happens. At + * `idleLimit` the model is told once to finish or stop, at twice that the + * turn ends and the work on disk is judged the way the deadline judges + * it. 2026-10-07: a worker made 90-125 calls a task reading other tasks' + * logs and Maat's records until the clock ran out. HTTP backends only: + * an ACP backend (grok-build://, OpenCode) runs its tools inside one step. + */ + const idleLimit = this.cfg.unattended ? (this.cfg.noProgressCalls ?? noProgressCallsFromEnv()) : 0; + let idleStamp = idleLimit > 0 ? treeStamp(this.cwd) : null; + const progress = new ProgressMeter(); + /** Per call this step: whether it advanced anything (see ProgressMeter). */ + const advancedThisStep: boolean[] = []; + let idleCalls = 0; + let idleNudged = false; + let idleStop = false; for (let step = 0; ; step++) { // Cancelled while nothing was in flight — during a permission question, // or between a tool and the next request. The next step is the next @@ -5903,6 +5982,17 @@ export class Engine { // whatever state the work is in when they are up is the state the judge // sees. Unlike a token ceiling it is not a proxy for anything — it is // the thing the person waiting actually spends. + if (idleStop) { + yield { + kind: "info", + text: + `no progress: ${idleCalls} tool calls in a row changed no file and returned nothing new, after ` + + `being told to finish or stop — no more tool calls. The sealed checks judge the work as it stands.`, + }; + const judged = yield* judgeOnDisk("no-progress", ""); + if (!judged) yield* this.salvage("No file in the project has changed for a long time; the turn is over.", fetchFn, log); + return; + } if (deadlineInterrupted || this.pastDeadline()) { const spentMs = Date.now() - this.turnStartedAt; log?.append("deadline", { limitMs: this.turnDeadlineMs, spentMs }); @@ -7088,8 +7178,20 @@ export class Engine { } } if (pendingCriteria && planned.some((p) => changesSomething(p.name, p.rawArgs))) yield* settleCriteria(false); + advancedThisStep.length = 0; for (const p of planned) { const outcome = yield* this.invokeTool(p, { step, userText, confirm, log, shown, answered }); + if (idleLimit > 0) { + let parsed: Record | undefined; + try { + const v = JSON.parse(p.rawArgs || "{}") as unknown; + if (v && typeof v === "object" && !Array.isArray(v)) parsed = v as Record; + } catch { + /* malformed: its answer is Maat's complaint, which is never progress */ + } + const key = parsed ? callKey(outcome.name, parsed) : `${outcome.name}(${p.rawArgs})`; + advancedThisStep.push(!outcome.repeated && progress.advanced(key, outcome.result)); + } called.push(outcome.name); if (outcome.repeated) repeated += 1; if (outcome.auto) autoRan += 1; @@ -7103,6 +7205,45 @@ export class Engine { } yield summary(called, "tools"); + if (idleLimit > 0) { + const stamp = treeStamp(this.cwd); + const changed = stamp === null || idleStamp === null || stamp !== idleStamp; + idleStamp = stamp; + if (changed) { + // A file in the project changed, or the tree could not be measured + // (never counted as idle). + idleCalls = 0; + idleNudged = false; + } else { + // In order: the run of non-advancing calls is what is counted, and + // one call that advanced anything starts it again. + for (const a of advancedThisStep) { + if (a) { + idleCalls = 0; + idleNudged = false; + } else idleCalls += 1; + } + if (!idleNudged && idleCalls >= idleLimit) { + idleNudged = true; + log?.append("no_progress", { step, calls: idleCalls, limit: idleLimit, action: "nudged" }); + yield { kind: "info", text: `no progress: ${idleCalls} tool calls in a row changed no file and returned nothing new — told the model to finish or stop` }; + this.transcript.push({ + role: "user", + content: + `[molt: your last ${idleCalls} tool calls changed no file and returned nothing they ` + + `had not returned before. Repeating them will not finish the task. If you know what ` + + `to change, change it now and say done. If you cannot finish, say plainly what is ` + + `blocking you and stop. If ${idleLimit} more calls advance nothing, the turn ends ` + + `and the work is judged as it stands.]`, + molt: { nudge: true }, + }); + } else if (idleNudged && idleCalls >= 2 * idleLimit) { + log?.append("no_progress", { step, calls: idleCalls, limit: idleLimit, action: "stopped" }); + idleStop = true; + } + } + } + // A step that mostly repeated itself learned little. Worth saying — // and nothing more than that. // diff --git a/src/journal.ts b/src/journal.ts index 2ba7733..89f1853 100644 --- a/src/journal.ts +++ b/src/journal.ts @@ -58,6 +58,17 @@ export type JournalKind = * unfinished, and a reader must be able to tell those apart later. */ | "deadline" + /** + * Unattended, hidden checks: a file tool refused a path outside the task + * (outside the project, or Maat's own `.maat/`), or a bash command named + * one. Bash is not blocked; it is recorded (src/scope.ts). + */ + | "outside_task" + /** + * Unattended: tool calls that changed no file in the project. The first + * entry is the nudge, the second the stop (`action`). + */ + | "no_progress" /** A verified change was committed. Carries the sha and the receipt. */ | "git_commit" /** An unverified change was put back. Carries what was restored, removed, kept. */ @@ -415,6 +426,16 @@ export class Journal { case "deadline": out.push(`${t} deadline · ${Math.round(Number(d.spentMs ?? 0) / 1000)}s of ${Math.round(Number(d.limitMs ?? 0) / 1000)}s — stopped on the clock, not on a failure`); break; + case "outside_task": + out.push( + d.refused + ? `${t} outside the task · ${d.tool} refused ${d.path} (${d.reach})` + : `${t} outside the task · bash named ${[...(d.stateDir ? [".maat/"] : []), ...((d.outside as unknown as string[] | undefined) ?? [])].join(", ")}`, + ); + break; + case "no_progress": + out.push(`${t} no progress · ${d.calls} tool call(s) changed no file · ${d.action}`); + break; case "git_commit": out.push(`${t} commit ${String(d.sha ?? "").slice(0, 8)} · ${d.files} file(s) · receipt ${d.receipt || "(none)"}`); break; diff --git a/src/receipts.ts b/src/receipts.ts index ebd6489..c73ca9a 100644 --- a/src/receipts.ts +++ b/src/receipts.ts @@ -61,7 +61,7 @@ export type ReceiptRecord = { */ ask?: boolean; /** Set when the model was stopped and the tree judged as it stood. */ - endedBy?: "deadline" | "provider"; + endedBy?: "deadline" | "provider" | "no-progress"; /** True when the cost rests on molt's own token estimate anywhere in the session. */ costEstimated?: boolean; /** @@ -382,7 +382,7 @@ export class Receipts { /** True for a question: the bar ran advisory and could not refuse. */ ask?: boolean; /** The model was stopped (clock, provider) and the tree was judged as it stood. */ - endedBy?: "deadline" | "provider"; + endedBy?: "deadline" | "provider" | "no-progress"; /** * Every file the turn changed, with the hashes that prove it — and which * lines it wrote, so the receipt can show the work rather than describe it. @@ -543,7 +543,9 @@ export class Receipts { ? [ args.endedBy === "deadline" ? "The turn's time budget ran out before the model said it was done; the sealed checks ran on the tree as it stood." - : "The provider failed after work had been done; the sealed checks ran on the tree as it stood.", + : args.endedBy === "no-progress" + ? "ended: no progress — the model kept calling tools without changing any file in the project, past a nudge to finish or stop; the sealed checks ran on the tree as it stood." + : "The provider failed after work had been done; the sealed checks ran on the tree as it stood.", "", ] : []), diff --git a/src/scope.ts b/src/scope.ts new file mode 100644 index 0000000..6224470 --- /dev/null +++ b/src/scope.ts @@ -0,0 +1,213 @@ +/** + * Keeping an unattended worker on its task. + * + * 2026-10-07, the ml4b bench lanes: a worker spent its whole time budget, + * 90 to 125 tool calls a task, reading things that were not the task: the + * other tasks' folders and logs in the shared bench folder, Maat's installed + * source, and Maat's own records of the job under `.maat/` (receipts, the + * journal, spilled output), hunting for the hidden checks. The checks' text + * was withheld (src/withhold.ts); the time was gone all the same. + * + * Three instruments, all for unattended runs: + * + * - While a job with hidden checks runs, read_file, list_dir, grep and + * inspect refuse paths outside the project and paths under its `.maat/`, + * saying plainly that this is outside the task (`outsideTask`). Two + * exceptions: a path the task itself names (a task may say "the config is + * in /etc/app"), and the files Maat itself points the model at under + * `.maat/` (spilled output in `out/`, background logs in `bg/`). + * - bash is not blocked (a shell can reach anything; a regex over a command + * line is not a wall), but a command that names `.maat/` or a path outside + * the project is journalled (`bashReach`). + * - The no-progress guard (engine.ts) counts tool calls in a row that did + * not advance anything (`ProgressMeter`): no file in the project changed + * (`treeStamp`), and the call returned nothing it had not returned before. + * Reading a file not read before, a command not run before, or a command + * whose output differs from every earlier run of it is progress, wherever + * it works: a build into dist/, a venv, a config under /etc. What stops a + * run is the same call coming back with the same answer, over and over. + */ +import { createHash } from "node:crypto"; +import { realpathSync, statSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { TREE_SKIP, walk } from "./files.js"; +import { STATE_DIRS } from "./statedir.js"; + +/** Calls without a change before the nudge; as many again and the turn ends. */ +export const NO_PROGRESS_CALLS = 30; + +/** MAAT_NO_PROGRESS_CALLS when it is a whole number (0 turns the guard off), else the default. */ +export function noProgressCallsFromEnv(env: NodeJS.ProcessEnv = process.env): number { + const raw = env.MAAT_NO_PROGRESS_CALLS?.trim(); + if (raw && /^\d+$/.test(raw)) return Number(raw); + return NO_PROGRESS_CALLS; +} + +/** The path with every existing ancestor's symlinks resolved; the rest kept as written. */ +function real(p: string): string { + let head = resolve(p); + const tail: string[] = []; + for (;;) { + try { + return join(realpathSync(head), ...tail.reverse()); + } catch { + const up = dirname(head); + if (up === head) return resolve(p); + tail.push(head.slice(up.length).replace(/^[/\\]/, "")); + head = up; + } + } +} + +const under = (p: string, root: string) => p === root || p.startsWith(root.endsWith(sep) ? root : root + sep); + +/** + * Absolute paths the task text names, and everything under them. Only the + * path itself: a task that names `/srv/bench/shared.toml` has not opened + * `/srv/bench/`, where the other tasks' folders are. + */ +export function taskPathsIn(task: string): string[] { + const out = new Set(); + for (const m of task.matchAll(/(?:^|[\s"'`(=:,<])((?:\/|~\/)[\w.@+-][\w.@+\-/]*)/g)) { + let p = m[1]!.replace(/[.,:;)]+$/, ""); + if (p.startsWith("~/")) p = join(homedir(), p.slice(2)); + if (p.length < 2) continue; + out.add(real(p)); + } + return [...out]; +} + +export type ScopeVerdict = { kind: "state" | "outside"; message: string } | null; + +/** + * Whether a file tool's path is outside the task, and the refusal to give. + * Null when the path is part of the task. + */ +export function outsideTask( + cwd: string, + path: string, + opts: { tool: string; taskPaths?: readonly string[] }, +): ScopeVerdict { + const root = real(cwd); + const target = real(isAbsolute(path) ? path : resolve(cwd, path)); + const shown = path || "."; + if (!under(target, root)) { + if ((opts.taskPaths ?? []).some((p) => under(target, p))) return null; + return { + kind: "outside", + message: + `refused: ${shown} is outside this task. While this job runs, ${opts.tool} reads only the ` + + `project (${cwd}) and paths the task names. Nothing outside it bears on the task or on how ` + + `the work will be judged; work on the task's own files. (A scratch file you made outside ` + + `the project with bash can still be read with bash.)`, + }; + } + const rel = relative(root, target).split(sep).join("/"); + // On a case-insensitive disk (macOS, Windows) `.MAAT/log` is `.maat/log`. + const fold = process.platform === "darwin" || process.platform === "win32" ? rel.toLowerCase() : rel; + const state = STATE_DIRS.find((d) => fold === d.toLowerCase() || fold.startsWith(`${d.toLowerCase()}/`)); + if (!state) return null; + // What Maat itself points the model at: a long output it spilled, a background job's log. + if (opts.tool === "read_file" && /^[^/]+\/(out|bg)\/[^/]+$/.test(fold)) return null; + return { + kind: "state", + message: + `refused: ${shown} is Maat's own record of this job (${state}/), not part of the task. ` + + `While the job runs it is closed to ${opts.tool}: nothing in it says how to do the task, and ` + + `the checks that will judge the work are not in it. Work on the task's own files.`, + }; +} + +/** System places a command names to run something, not to look at anything. */ +const BENIGN = /^\/(?:dev\/(?:null|zero|u?random|std(?:in|out|err)|fd\/\d+|tty)|proc\/self\/fd\/\d+|(?:usr\/(?:local\/)?)?s?bin(?:\/[\w.+-]+)?)$/; + +/** + * What a bash command reaches outside the task: whether it names the state + * folder, and the paths outside the project it names. A note for the + * journal, not a wall: heuristics over a command line, so it may miss a + * path built at run time, and names a path that is only written to. + */ +export function bashReach( + cwd: string, + command: string, + taskPaths: readonly string[] = [], +): { stateDir: boolean; outside: string[] } { + const stateDir = new RegExp(`(?:^|[\\s"'\`=:(/<>|;&])(?:${STATE_DIRS.map((d) => d.replace(".", "\\.")).join("|")})(?:/|\\b)`).test(command); + const root = real(cwd); + const outside = new Set(); + for (const m of command.matchAll(/(?:^|[\s"'`=:(<>|;&])((?:\/|~\/|\.\.\/|\.\.(?=$|[\s"'`;|&)]))[^\s"'`;|&<>()$*?[\]{}]*)/g)) { + let p = m[1]!; + if (p.startsWith("~/")) p = join(homedir(), p.slice(2)); + if (BENIGN.test(p)) continue; + const abs = real(isAbsolute(p) ? p : resolve(cwd, p)); + if (under(abs, root)) continue; + if (taskPaths.some((t) => under(abs, t))) continue; + outside.add(m[1]!); + } + return { stateDir, outside: [...outside].slice(0, 20) }; +} + +/** + * A fingerprint of the project's files (path, size, modification time), + * skipping what TREE_SKIP skips (`.maat/`, node_modules, build output). Null + * when the walk was cut short: a tree that cannot be measured is never + * counted as one that did not change. + */ +export function treeStamp(root: string): string | null { + const w = walk(root, { depth: 24, limit: 20_000, skip: TREE_SKIP, deadline: Date.now() + 1_500 }); + if (w.truncated || w.timedOut) return null; + const h = createHash("sha256"); + for (const e of w.entries) { + if (e.kind !== "file") { + h.update(`d\0${e.path}\0`); + continue; + } + let mtime = 0; + try { + mtime = statSync(join(root, e.path)).mtimeMs; + } catch { + /* gone between the walk and the stat: its absence is in the next stamp */ + } + h.update(`f\0${e.path}\0${e.bytes ?? 0}\0${mtime}\0`); + } + return h.digest("hex"); +} + +/** + * Output that differs between two runs of the same command for no reason that + * matters: durations, clock times, process ids in a "[1] 12345" job line. Left + * in, every timed test run would count as new output forever. + */ +function settled(result: string): string { + return result + .replace(/\b\d{4}-\d\d-\d\d[T ]\d\d:\d\d(?::\d\d(?:[.,]\d+)?)?(?:Z|[+-]\d\d:?\d\d)?/g, "