From 9bbb4d25c6504f95e4ebd2e8cabc72f780ca5526 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 14:34:29 -0700 Subject: [PATCH 1/8] Bench: each task runs in a private folder; logs and results out of the agent's reach A worker in the ml4b lanes spent 90-125 tool calls per task reading the other tasks' folders and logs in the shared, host-mounted /work. A host bind mount ignores chown and modes for other users, so nothing there can be locked. Every container mode now runs each task in /var/lib/bench-work// (711, unlistable, removed after the task), writes Maat's logs to a root-only folder, and copies folder and logs to the export only after the task is graded and every agent-user process is killed. The work and results folders are mounted under /root (700, enforced). Same layout on a Mac host lane. tests/test_task_isolation.py checks the layout anywhere and, as root with the agent user (tests/isolation-in-container.sh), that a task cannot list or read another task's folder or logs. --- bench/local/container/Dockerfile | 4 +- bench/local/container/run-in-container.sh | 19 +- bench/local/crosscheck.py | 7 +- bench/local/run.py | 77 +++++++- bench/local/tests/isolation-in-container.sh | 10 ++ bench/local/tests/test_task_isolation.py | 185 ++++++++++++++++++++ 6 files changed, 289 insertions(+), 13 deletions(-) create mode 100755 bench/local/tests/isolation-in-container.sh create mode 100644 bench/local/tests/test_task_isolation.py diff --git a/bench/local/container/Dockerfile b/bench/local/container/Dockerfile index 2816c8f..581650e 100644 --- a/bench/local/container/Dockerfile +++ b/bench/local/container/Dockerfile @@ -14,4 +14,6 @@ ENV BENCH_AGENT_USER=agent WORKDIR /opt # Maat itself is installed per build by run-in-container.sh (FROM maat-bench + npm i -g the packed tarball). -ENV BENCH_WORK=/work RESULTS_DIR=/results +# run-in-container.sh overrides these: tasks run in a container-local /var/lib/bench-work and the +# host folders are mounted under /root, where the agent user cannot reach them. +ENV BENCH_WORK=/var/lib/bench-work BENCH_EXPORT=/root/bench-export RESULTS_DIR=/root/bench-results diff --git a/bench/local/container/run-in-container.sh b/bench/local/container/run-in-container.sh index c41c38c..cc271a0 100755 --- a/bench/local/container/run-in-container.sh +++ b/bench/local/container/run-in-container.sh @@ -5,8 +5,9 @@ # The image is maat-bench:, built once and reused, so every run # records exactly which Maat it tested. Env passed through: REFERENCE BENCH_MODEL # BENCH_URL BENCH_LIMIT RESULTS (file name). Results land in -# ~/.cache/maat-bench/container-results/; each task's folder and Maat's log are kept -# in ~/.cache/maat-bench/container-work// (the only host path it can write). +# ~/.cache/maat-bench/container-results/; each task's folder and Maat's log are copied, +# once the task is graded, to ~/.cache/maat-bench/container-work// (the only host +# path it can write; the agent never sees it, see "Task isolation" below). # SUBSCRIPTION=grok: use the owner's Grok Build login (BENCH_URL=grok-build://subscription # BENCH_MODEL=grok-4.7). Image is maat-bench-grok: (Dockerfile.grok: official grok # installer, linux, pinned). ~/.grok/auth.json is mounted read-only OUTSIDE the container's @@ -62,7 +63,17 @@ printf 'OPENROUTER_API_KEY=%s\n' "$key" > "$envf" # The graders and reference solutions are mounted where only root can reach (/root is 700), # copied to a root-only /opt/bench, and run.py runs from there as root; it runs the agent # as the unprivileged `agent` user (BENCH_AGENT_USER) and grades as root afterwards. -credmount=; startcmd='export MOLT_DIST_ABS=$(npm root -g)/@solvyx/molt/dist; mkdir -p /work && chmod 755 /work && rm -rf /opt/bench && cp -a /root/bench-src /opt/bench && chmod -R go-rwx /opt/bench && cd /opt/bench && python3 run.py "$@"' +# +# Task isolation, in every mode (plain, SUBSCRIPTION=grok, OPENCODE=1). A host bind mount +# ignores chown and does not enforce modes for other users (OrbStack, Docker Desktop), so the +# agent can read anything mounted where it can reach it: on 2026-10-07 a worker spent its whole +# budget reading the other tasks' folders and logs in a shared /work. So nothing from the host +# is mounted where the agent can reach it. The work folder and the results are mounted under +# /root (700, container-local, enforced). Each task runs in a private folder under the +# container-local /var/lib/bench-work (711, random per-task parent, removed after the task); +# Maat's logs are written to a root-only folder there; folder and logs are copied to the export +# only once the task is graded and every process of the agent user is gone (run.py finish_task). +credmount=; startcmd='export MOLT_DIST_ABS=$(npm root -g)/@solvyx/molt/dist BENCH_WORK=/var/lib/bench-work BENCH_EXPORT=/root/bench-export RESULTS_DIR=/root/bench-results; chmod 700 /root && mkdir -p $BENCH_WORK && chmod 711 $BENCH_WORK && rm -rf /opt/bench && cp -a /root/bench-src /opt/bench && chmod -R go-rwx /opt/bench && cd /opt/bench && python3 run.py "$@"' if [ "$SUBSCRIPTION" = grok ]; then credmount="-v $HOME/.grok/auth.json:/root/grok-cred/auth.json:ro" # grok runs as `agent`, so the credential copy lives in ITS home, owned by it, mode 600. @@ -82,7 +93,7 @@ if [ "$OPENCODE" = 1 ]; then fi echo "maat $img → $out/${RESULTS:-results.jsonl}" docker run --rm --name "maat-bench-$name" \ - -v "$local_dir":/root/bench-src:ro -v "$out":/results -v "$work":/work ${=credmount} \ + -v "$local_dir":/root/bench-src:ro -v "$out":/root/bench-results -v "$work":/root/bench-export ${=credmount} \ --env-file "$envf" -e REFERENCE="${REFERENCE:-0}" -e BENCH_MODEL="${BENCH_MODEL:-}" \ -e ARMS="${ARMS:-}" -e BENCH_TASKS="${BENCH_TASKS:-}" -e BENCH_URL="$url" -e BENCH_LIMIT="${BENCH_LIMIT:-600}" -e BENCH_REASONING="${BENCH_REASONING:-}" -e BENCH_GATE="${BENCH_GATE:-}" -e RESULTS="${RESULTS:-results.jsonl}" \ -e MAAT_BUILD="$sha" -e PYTHONDONTWRITEBYTECODE=1 \ diff --git a/bench/local/crosscheck.py b/bench/local/crosscheck.py index b2b5f5f..52a8f6d 100644 --- a/bench/local/crosscheck.py +++ b/bench/local/crosscheck.py @@ -83,7 +83,8 @@ def fresh() -> Path: d = fresh() # a check names the folder of the run that drafted it (/work/-molt-); in another # run's tree that path must mean "this tree", or every foreign tree fails it spuriously - run_cmd = re.sub(r"/work/[\w.-]+-molt-\d+", str(dst), cmd) + # (runs since 2026-10-07 have a private folder: /var/lib/bench-work//-molt-) + run_cmd = re.sub(r"(?:/var/lib/bench-work/[0-9a-f]+|/work)/[\w.-]+-molt-\d+", str(dst), cmd) p = subprocess.Popen(run_cmd, shell=True, cwd=d, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, start_new_session=True) try: @@ -212,7 +213,7 @@ def exec_tree(run: dict, cmds: list[str]) -> str: if set(cmds) <= set(old["checks"]): return "cached" else: # results from before the /work/ rewrite: redo only commands that name such a path - cmds = [c for c in cmds if "/work/" in c] + cmds = [c for c in cmds if "/work/" in c or "/var/lib/bench-work/" in c] if not cmds: old["pathfix"] = True out.write_text(json.dumps(old)) @@ -263,7 +264,7 @@ def execute(jobs: int) -> None: # ------------------------------------------------------------------ analysis def sig(rc: int, out: str, run_name: str) -> str: - o = re.sub(r"/work/[\w.-]+", "/work/T", out) + o = re.sub(r"(?:/var/lib/bench-work/[0-9a-f]+|/work)/[\w.-]+", "/work/T", out) o = re.sub(r"0x[0-9a-f]+|\b[0-9a-f]{7,40}\b", "H", o) o = re.sub(r"\d{4,}", "N", o) o = re.sub(r"\s+", " ", o).strip()[:300] diff --git a/bench/local/run.py b/bench/local/run.py index 8be41cd..e7ea59d 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -28,6 +28,16 @@ # Outside every git repository: a task folder inside this one let agents' # `git commit` walk up and commit task files into molt-desktop's main. WORK = Path(os.environ.get("BENCH_WORK", Path.home() / ".cache/maat-bench/work")) +# BENCH_EXPORT: where each finished task's folder and logs are copied, right after it is graded. +# Unset: WORK itself. The task RUNS somewhere else (private_dir below), never in the export. +# +# 2026-10-07 (ml4b lanes): a worker spent 90-125 tool calls per task reading the other tasks' +# folders and logs in the shared /work instead of working. /work is a host bind mount, and on +# OrbStack and Docker Desktop a bind mount ignores chown and does not enforce modes for other +# users, so nothing there can be locked away from the agent user. The containers therefore mount +# the export (and the results) under /root (700, container-local, enforced), run each task in a +# private folder under BENCH_WORK, and copy it out once it is finished and the agent is gone. +EXPORT = Path(os.environ.get("BENCH_EXPORT") or WORK) # MOLT_DIST_ABS: an absolute path to the built Maat (the container mounts it at /maat). MOLT = (Path(os.environ["MOLT_DIST_ABS"]) if os.environ.get("MOLT_DIST_ABS") else Path.home() / os.environ.get("MOLT_DIST", "Documents/molt-desktop/dist-compare")) / "cli.js" LIMIT = int(os.environ.get("BENCH_LIMIT", "600")) # seconds per task @@ -83,6 +93,56 @@ def hand_over(d: Path) -> None: subprocess.run(["chown", "-R", AGENT_USER, str(d)], check=True) +def private_dir(tag: str) -> Path: + """ + A fresh folder for one task: BENCH_WORK//. + + BENCH_WORK is 711 (made so here when we own it): the agent can reach its own task by the + exact path it is given, but cannot list BENCH_WORK, and the random parent cannot be guessed, + so no task can find another's folder. Finished tasks are removed (finish_task), so there is + nothing left to find anyway. On a Mac host lane (no agent user) the layout is the same; the + agent then runs as the owner, so only the names and the removal keep tasks apart. + """ + import secrets + WORK.mkdir(parents=True, exist_ok=True) + if AGENT_USER: + os.chmod(WORK, 0o711) + box = WORK / secrets.token_hex(8) + box.mkdir(mode=0o711) + os.chmod(box, 0o711) # mkdir's mode is masked by the umask + d = box / tag + d.mkdir() + return d + + +def log_dir() -> Path: + """Where Maat's stdout/stderr are written while a run is live: root-only (700), never in a task folder.""" + logs = WORK / ".logs" + logs.mkdir(parents=True, exist_ok=True) + os.chmod(logs, 0o700) + return logs + + +def finish_task(d: Path, logs: list[Path]) -> None: + """ + The task is graded and its agent is gone: copy the folder and its logs to EXPORT, then remove + the private copies. In a container, every process of the agent user is killed first: a server + a worker left running would otherwise still be there, as that user, during the next task. + """ + if AGENT_USER and shutil.which("pkill"): + subprocess.run(["pkill", "-KILL", "-u", AGENT_USER], check=False) + EXPORT.mkdir(parents=True, exist_ok=True) + dst = EXPORT / d.name + if dst.resolve() != d.resolve(): + shutil.rmtree(dst, ignore_errors=True) + shutil.copytree(d, dst, symlinks=True) + for f in logs: + if f.exists() and (EXPORT / f.name).resolve() != f.resolve(): + shutil.copy2(f, EXPORT / f.name) + f.unlink() + shutil.rmtree(d.parent, ignore_errors=True) + + def provider_capped(out: str, steps: int) -> bool: """The provider's cap (OpenRouter daily limit, OpenCode free-usage limit), not the work.""" return ( @@ -230,6 +290,7 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: tasks = [T for T in tasks if T.name in want] arms = parse_arms(os.environ.get("ARMS")) out = Path(os.environ.get("RESULTS_DIR", HERE)) / os.environ.get("RESULTS", f"results-{which}-x{repeats}.jsonl") + out.parent.mkdir(parents=True, exist_ok=True) done = set() if out.exists(): # resume: skip runs already recorded for line in out.read_text().splitlines(): @@ -242,15 +303,17 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: if (T.name, a, rep, arm) in done: continue tag = f"{T.name}-{a}-{rep}" + (f"-{arm}" if arm else "") - d = WORK / tag - shutil.rmtree(d, ignore_errors=True) - d.mkdir(parents=True) + d = private_dir(tag) + log = log_dir() / f"{tag}.log" T.setup(d) hand_over(d) saved = {k: os.environ.get(k) for k in arm_env} os.environ.update(arm_env) try: - r = agents[a](d, T.PROMPT, WORK / f"{tag}.log") + r = agents[a](d, T.PROMPT, log) + except BaseException: + finish_task(d, [log, log.with_suffix(".err")]) + raise finally: for k, v in saved.items(): if v is None: @@ -258,10 +321,14 @@ def main(which: str, repeats: int, task_filter: str | None) -> None: else: os.environ[k] = v if r.get("provider_capped"): + finish_task(d, [log, log.with_suffix(".err")]) # Not recorded, so a resume runs this task again. print(f"STOPPED: the provider's daily limit / quota is reached ({tag} not recorded)", flush=True) return - ok, why = T.grade(d) + try: + ok, why = T.grade(d) + finally: + finish_task(d, [log, log.with_suffix(".err")]) final = "" r.update(task=T.name, agent=a, rep=rep, passed=ok, why=why, final=final[:300]) if arm: diff --git a/bench/local/tests/isolation-in-container.sh b/bench/local/tests/isolation-in-container.sh new file mode 100755 index 0000000..8bd9462 --- /dev/null +++ b/bench/local/tests/isolation-in-container.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Run the task-isolation tests where they mean something: as root in the bench image, with the +# unprivileged `agent` user the bench runs the worker as (see test_task_isolation.py). +# bench/local/tests/isolation-in-container.sh [image] (default maat-bench:agentu) +set -e +here=$(cd "$(dirname "$0")/.." && pwd) +img=${1:-maat-bench:agentu} +docker image inspect "$img" >/dev/null 2>&1 || docker build -q -t "$img" "$here/container" >/dev/null +exec docker run --rm -e PYTHONDONTWRITEBYTECODE=1 -v "$here":/root/bench-src:ro -w /root/bench-src "$img" \ + python3 -m unittest -v tests.test_task_isolation diff --git a/bench/local/tests/test_task_isolation.py b/bench/local/tests/test_task_isolation.py new file mode 100644 index 0000000..8ae0f55 --- /dev/null +++ b/bench/local/tests/test_task_isolation.py @@ -0,0 +1,185 @@ +""" +A task cannot list or read another task's folder or logs. + +2026-10-07, ml4b lanes: a worker spent 90-125 tool calls per task reading the other tasks' +folders and Maat logs in the shared, host-mounted /work instead of working. Each task now runs +in a private folder (BENCH_WORK//), its logs go to a root-only folder, and both are +copied to the export only after the task is graded. + +The first test runs anywhere (a Mac host lane included): it checks the layout. The second needs +the real thing, root and the unprivileged agent user, and runs inside the bench image: + + bench/local/tests/isolation-in-container.sh +""" + +import json +import os +import pwd +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import run # noqa: E402 + + +def task(name: str): + class T: + PROMPT = f"do {name}" + + @staticmethod + def setup(d: Path) -> None: + (d / "input.txt").write_text(f"{name} input\n") + + @staticmethod + def grade(d: Path): + return (d / "out.txt").exists(), "graded" + + T.name = name + return T + + +class Harness: + """run.main over two fake tasks, with a fake agent that records what it could see.""" + + def __init__(self, work: Path, export: Path, results: Path, agent=None): + self.saved = {k: getattr(run, k) for k in ("WORK", "EXPORT", "TASKS", "run_molt", "AGENT_USER")} + self.env = {k: os.environ.get(k) for k in ("RESULTS_DIR", "RESULTS", "ARMS", "BENCH_TASKS")} + run.WORK, run.EXPORT, run.AGENT_USER = work, export, agent + run.TASKS = [task("iso-a"), task("iso-b")] + os.environ.update(RESULTS_DIR=str(results), RESULTS="iso.jsonl") + for k in ("ARMS", "BENCH_TASKS"): + os.environ.pop(k, None) + self.seen: list[dict] = [] + + def restore(self): + for k, v in self.saved.items(): + setattr(run, k, v) + for k, v in self.env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +class Layout(unittest.TestCase): + def test_tasks_run_in_private_folders_and_only_finished_work_is_exported(self): + tmp = Path(tempfile.mkdtemp()) + h = Harness(tmp / "work", tmp / "export", tmp / "results") + try: + def fake(d: Path, prompt: str, log: Path) -> dict: + h.seen.append({ + "dir": d, "log": log, + # what is left of the tasks before this one, where this one runs + "box_listing": sorted(os.listdir(d.parent)), + "earlier_dirs_exist": [s["dir"].exists() for s in h.seen], + "earlier_logs_exist": [s["log"].exists() for s in h.seen], + "exported_so_far": sorted(os.listdir(tmp / "export")) if (tmp / "export").exists() else [], + }) + (d / "out.txt").write_text("x\n") + log.write_text('{"kind":"job_end"}\n') + return {"secs": 0, "turns": 0, "claim": "verified", "said_done": True} + + run.run_molt = fake + run.main("molt", 1, "iso-a,iso-b") + finally: + h.restore() + a, b = h.seen + for s in h.seen: + self.assertFalse(s["dir"].is_relative_to(tmp / "export"), "a task never runs in the export") + self.assertNotEqual(s["log"].parent, s["dir"], "logs are not written into a task folder") + self.assertEqual(s["box_listing"], [s["dir"].name], "the private parent holds this task only") + self.assertNotEqual(a["dir"].parent, b["dir"].parent, "each task has its own random parent") + self.assertEqual(b["earlier_dirs_exist"], [False], "task a's private folder is gone before task b starts") + self.assertEqual(b["earlier_logs_exist"], [False], "task a's private log is gone before task b starts") + self.assertIn("iso-a-molt-0", b["exported_so_far"], "task a was exported when it finished") + self.assertNotIn("iso-b-molt-0", b["exported_so_far"]) + out = tmp / "export" + for t in ("iso-a-molt-0", "iso-b-molt-0"): + self.assertTrue((out / t / "out.txt").exists()) + self.assertTrue((out / f"{t}.log").exists()) + rows = [json.loads(x) for x in (tmp / "results" / "iso.jsonl").read_text().splitlines()] + self.assertEqual([r["passed"] for r in rows], [True, True]) + # nothing of either task is left where tasks run, but the (empty) root-only log folder + self.assertEqual(sorted(os.listdir(tmp / "work")), [".logs"]) + self.assertEqual(os.listdir(tmp / "work" / ".logs"), []) + shutil.rmtree(tmp) + + +def have_agent_user() -> bool: + if os.geteuid() != 0 or not shutil.which("runuser"): + return False + try: + pwd.getpwnam(os.environ.get("BENCH_AGENT_USER") or "agent") + return True + except KeyError: + return False + + +@unittest.skipUnless(have_agent_user(), "needs root and the agent user: run isolation-in-container.sh") +class AsTheAgentUser(unittest.TestCase): + def test_the_agent_cannot_list_or_read_another_tasks_folder_or_logs(self): + user = os.environ.get("BENCH_AGENT_USER") or "agent" + # As in the container: tasks under a container-local folder, the export under /root (700). + os.chmod("/root", 0o700) + work = Path(tempfile.mkdtemp(dir="/var/tmp")) / "bench-work" + os.chmod(work.parent, 0o755) # like /var/lib: the agent may pass through to BENCH_WORK + export = Path(tempfile.mkdtemp(dir="/root")) / "bench-export" + results = Path(tempfile.mkdtemp(dir="/root")) + h = Harness(work, export, results, agent=user) + + def as_agent(script: str) -> subprocess.CompletedProcess: + cmd, _ = run.as_agent(["sh", "-c", script]) + return subprocess.run(cmd, capture_output=True, text=True) + + try: + def fake(d: Path, prompt: str, log: Path) -> dict: + probes = { + "own_input": as_agent(f"cat {d}/input.txt"), + "own_write": as_agent(f"echo x > {d}/out.txt"), + "list_work": as_agent(f"ls {work}"), + "list_logs": as_agent(f"ls {work}/.logs"), + "list_box": as_agent(f"ls {d.parent}"), + "list_export": as_agent(f"ls {export}"), + } + if h.seen: + a = h.seen[0] + probes |= { + "other_dir": as_agent(f"ls {a['dir']}"), + "other_input": as_agent(f"cat {a['dir']}/input.txt"), + "other_log_private": as_agent(f"cat {a['log']}"), + "other_dir_exported": as_agent(f"ls {export}/{a['dir'].name}"), + "other_log_exported": as_agent(f"cat {export}/{a['log'].name}"), + "find_logs": as_agent(f"find / -xdev -name '*.log' -path '*iso-*' 2>/dev/null"), + "find_inputs": as_agent(f"grep -rl 'iso-a input' / --exclude-dir=proc --exclude-dir=sys 2>/dev/null"), + } + h.seen.append({"dir": d, "log": log, "probes": probes}) + log.write_text('{"kind":"job_end","secret":"iso-a log"}\n') + return {"secs": 0, "turns": 0, "claim": "verified", "said_done": True} + + run.run_molt = fake + run.main("molt", 1, "iso-a,iso-b") + finally: + h.restore() + a, b = h.seen + for s in h.seen: + p = s["probes"] + self.assertEqual(p["own_input"].returncode, 0, p["own_input"].stderr) + self.assertEqual(p["own_write"].returncode, 0, p["own_write"].stderr) + for k in ("list_work", "list_logs", "list_box", "list_export"): + self.assertNotEqual(p[k].returncode, 0, f"{k} worked: {p[k].stdout}") + p = b["probes"] + for k in ("other_dir", "other_input", "other_log_private", "other_dir_exported", "other_log_exported"): + self.assertNotEqual(p[k].returncode, 0, f"task b could {k}: {p[k].stdout}") + self.assertEqual(p["find_logs"].stdout.strip(), "", "task b found task a's log") + self.assertEqual(p["find_inputs"].stdout.strip(), "", "task b found task a's files") + # Exported for the owner all the same. + self.assertTrue((export / "iso-a-molt-0" / "out.txt").exists()) + self.assertIn("iso-a log", (export / "iso-a-molt-0.log").read_text()) + + +if __name__ == "__main__": + unittest.main() From 4c7d7e3d3c4d745bbab11bd1d37b626bdcb71a83 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 14:34:29 -0700 Subject: [PATCH 2/8] Hold unattended workers to the task; stop runs that change nothing While an unattended job with hidden checks runs, read_file, list_dir, grep and inspect refuse paths outside the project and under .maat/ (except the spilled output and background logs Maat points at, and paths the task names), saying it is outside the task, and journal each refusal (outside_task). Bash is not blocked; a command naming .maat/ or a path outside the project is journalled. No-progress guard: after N tool calls in a row (MAAT_NO_PROGRESS_CALLS, default 30, 0 = off) that change no file in the project, the model is told once to finish or stop; after N more the turn ends and the work on disk is judged as at the deadline, endedBy no-progress, receipt 'ended: no progress'. --- src/engine.ts | 126 +++++++++++++++++- src/journal.ts | 21 +++ src/receipts.ts | 8 +- src/scope.ts | 167 +++++++++++++++++++++++ src/types.ts | 2 +- test/no-probing.test.ts | 284 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 598 insertions(+), 10 deletions(-) create mode 100644 src/scope.ts create mode 100644 test/no-probing.test.ts diff --git a/src/engine.ts b/src/engine.ts index 3f13970..a39f089 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -133,6 +133,7 @@ import { type Spend, } from "./types.js"; import { stateDir, stateDirName } from "./statedir.js"; +import { bashReach, noProgressCallsFromEnv, outsideTask, taskPathsIn, treeStamp } from "./scope.js"; import { env } from "./env.js"; import { Judgments, caseReason, reasonText } from "./judgment.js"; @@ -1568,6 +1569,13 @@ export type EngineConfig = { * case this exists for, and it is off everywhere else. */ sandbox?: boolean; + /** + * Unattended: tool calls in a row that change no file in the project before + * the model is told to finish or stop; as many again and the turn ends, + * judged on disk ("no-progress"). 0 turns the guard off. Default: + * MAAT_NO_PROGRESS_CALLS, else 30 (src/scope.ts). + */ + noProgressCalls?: number; /** * Where to write one JSON file per completion attempt holding the whole * turn: the wire transcript, the ledger, the bar's result, the receipt name. @@ -2272,7 +2280,7 @@ export class Engine { * all because the clock or a failed request stopped the turn before any check * ran; two of them had passing work on disk. */ - private turnEndedBy: "deadline" | "provider" | undefined; + private turnEndedBy: "deadline" | "provider" | "no-progress" | undefined; /** A subprocess backend went silent past the stall allowance this turn (a provider issue). */ private turnProviderStall = false; /** `--revert` put this turn's work back, so a person cannot accept it as it stands. */ @@ -2542,6 +2550,43 @@ export class Engine { * written — when the job's work is over. */ private withheld: { name: string; text: string }[] = []; + /** Absolute paths this turn's task names: part of the task, though outside the project (src/scope.ts). */ + private turnTaskPaths: string[] = []; + /** This turn's checks are hidden, or are being drafted to be (src/scope.ts). */ + private turnHidesChecks = false; + + /** + * Whether the file tools are held to the task: an unattended job with + * hidden checks, while it runs. A person at the keyboard can read what they + * like; a worker alone with checks it cannot see spent whole budgets + * reading Maat's records and other tasks' logs instead (src/scope.ts). + */ + private scoped(): boolean { + return this.cfg.unattended === true && this.turnActive && (this.turnHidesChecks || this.withheld.length > 0); + } + + /** The refusal for a file tool's path outside the task, journalled; null when the path is in scope. */ + private refuseOutsideTask(tool: string, path: string): string | null { + if (!this.scoped()) return null; + const v = outsideTask(this.cwd, path, { tool, taskPaths: this.turnTaskPaths }); + if (!v) return null; + this.cfg.journal?.append("outside_task", { tool, path: path || ".", reach: v.kind, refused: true }); + return v.message; + } + + /** A bash command that names `.maat/` or a path outside the project: not blocked, journalled. */ + private noteBashReach(command: string): void { + if (!this.scoped()) return; + const r = bashReach(this.cwd, command, this.turnTaskPaths); + if (!r.stateDir && r.outside.length === 0) return; + this.cfg.journal?.append("outside_task", { + tool: "bash", + refused: false, + stateDir: r.stateDir, + outside: r.outside, + command: command.slice(0, 500), + }); + } /** The effort the next request carries. */ private get effortNow(): string | undefined { @@ -3266,6 +3311,8 @@ export class Engine { case "inspect": { const rel = String(args.path ?? "."); const abs = resolve(this.cwd, rel); + const off = this.refuseOutsideTask("inspect", rel); + if (off) return off; if (!existsSync(abs)) return `${rel} does not exist`; try { return statSync(abs).isDirectory() @@ -3276,7 +3323,9 @@ export class Engine { } } - case "read_file": + case "read_file": { + const off = this.refuseOutsideTask("read_file", String(args.path ?? "")); + if (off) return off; this.readPaths.add(String(args.path ?? "")); return readPart( await this.readText(resolve(this.cwd, String(args.path ?? ""))), @@ -3285,6 +3334,7 @@ export class Engine { num(args.limit, Number.MAX_SAFE_INTEGER), this.resultBudget(), ); + } case "write_file": { const rel = String(args.path ?? ""); @@ -3343,6 +3393,8 @@ export class Engine { case "list_dir": { const rel = String(args.path ?? "."); const abs = resolve(this.cwd, rel); + const off = this.refuseOutsideTask("list_dir", rel); + if (off) return off; this.mustBeInside(abs, rel); // Bounded and off the main thread: a listing the model asks for can be // pointed at anything, including a home directory. @@ -3359,6 +3411,8 @@ export class Engine { case "grep": { const rel = String(args.path ?? "."); const abs = resolve(this.cwd, rel); + const off = this.refuseOutsideTask("grep", rel); + if (off) return off; this.mustBeInside(abs, rel); const pattern = String(args.pattern ?? ""); return formatMatches( @@ -3417,6 +3471,7 @@ export class Engine { // a caller that reaches runTool directly (a subscription backend). foldCommands(args, this.cwd); const command = String(args.command ?? ""); + this.noteBashReach(command); if (!command.trim() && args.stop_job === undefined) { return "bash needs a command, or commands: a list to run in order"; } @@ -4008,7 +4063,7 @@ export class Engine { result: BarResult, attempts: number, log?: Journal, - endedBy?: "deadline" | "provider", + endedBy?: "deadline" | "provider" | "no-progress", ): AsyncGenerator { const onlyWrites = failedOnlyWriteChecks(result); if (this.cfg.receipts) { @@ -4312,6 +4367,8 @@ export class Engine { this.sealedChecks = []; this.sealedNotes = []; this.turnCalls = new Set(); + this.turnTaskPaths = taskPathsIn(userText); + this.turnHidesChecks = false; // molt's records stay out of the project's `git status` (leftovers.ts). excludeMoltFromGit(this.cwd); // Per turn, or receipt five lists what turn one ran. It did: receipts @@ -4380,7 +4437,7 @@ export class Engine { ? opts.ask && this.turnWrites.length === 0 ? "answered" : "verified" - : answered || this.turnEndedBy === "deadline" + : answered || this.turnEndedBy === "deadline" || this.turnEndedBy === "no-progress" ? "unverified" : "stopped"; @@ -4396,7 +4453,7 @@ export class Engine { outcome = "unverified"; yield { kind: "info", - text: `the checks passed on the work as it stood when the ${this.turnEndedBy === "deadline" ? "clock" : "provider"} stopped the model, but it never said it was done — unverified.`, + text: `the checks passed on the work as it stood when the ${this.turnEndedBy === "deadline" ? "clock" : this.turnEndedBy === "no-progress" ? "no-progress guard" : "provider"} stopped the model, but it never said it was done — unverified.`, }; } @@ -5309,6 +5366,8 @@ export class Engine { */ let pendingCriteria = opts.pendingCriteria; const criteriaSince = Date.now(); + // Drafted checks are hidden; given ones may be. Either way the file tools are held to the task. + this.turnHidesChecks = Boolean(pendingCriteria) || (opts.taskChecks ?? []).some((c) => c.hidden === true); /** * The draft, when the time budget's cut found it with no checks: kept alive * to join at the first claim instead of sealing nothing (joinLateCriteria). @@ -5757,7 +5816,7 @@ export class Engine { * is what it says: verified only if it passed, not proven if it did not, * unverified when nothing was sealed. Returns whether a bar ran. */ - async function* judgeOnDisk(why: "deadline" | "provider", claim: string): AsyncGenerator { + async function* judgeOnDisk(why: "deadline" | "provider" | "no-progress", claim: string): AsyncGenerator { self.turnEndedBy = why; if (opts.ask) return false; let barThis = barNow(); @@ -5873,6 +5932,19 @@ export class Engine { * rather than left to re-measure and perhaps disagree. */ let deadlineInterrupted = false; + /** + * The no-progress guard, unattended only (src/scope.ts). Tool calls since + * a file in the project last changed; at `idleLimit` the model is told + * once to finish or stop, at twice that the turn ends and the work on + * disk is judged the way the deadline judges it. 2026-10-07: a worker + * made 90-125 calls a task reading other tasks' logs and Maat's records, + * changing nothing, until the clock ran out. + */ + const idleLimit = this.cfg.unattended ? (this.cfg.noProgressCalls ?? noProgressCallsFromEnv()) : 0; + let idleStamp = idleLimit > 0 ? treeStamp(this.cwd) : null; + let idleCalls = 0; + let idleNudged = false; + let idleStop = false; for (let step = 0; ; step++) { // Cancelled while nothing was in flight — during a permission question, // or between a tool and the next request. The next step is the next @@ -5903,6 +5975,17 @@ export class Engine { // whatever state the work is in when they are up is the state the judge // sees. Unlike a token ceiling it is not a proxy for anything — it is // the thing the person waiting actually spends. + if (idleStop) { + yield { + kind: "info", + text: + `no progress: ${idleCalls} tool calls in a row changed no file in the project, after ` + + `being told to finish or stop — no more tool calls. The sealed checks judge the work as it stands.`, + }; + const judged = yield* judgeOnDisk("no-progress", ""); + if (!judged) yield* this.salvage("No file in the project has changed for a long time; the turn is over.", fetchFn, log); + return; + } if (deadlineInterrupted || this.pastDeadline()) { const spentMs = Date.now() - this.turnStartedAt; log?.append("deadline", { limitMs: this.turnDeadlineMs, spentMs }); @@ -7103,6 +7186,37 @@ export class Engine { } yield summary(called, "tools"); + if (idleLimit > 0) { + const stamp = treeStamp(this.cwd); + if (stamp === null || idleStamp === null) { + // Not measurable (the walk was cut short): never counted as idle. + idleStamp = stamp; + } else if (stamp !== idleStamp) { + idleStamp = stamp; + idleCalls = 0; + idleNudged = false; + } else { + idleCalls += called.length; + if (!idleNudged && idleCalls >= idleLimit) { + idleNudged = true; + log?.append("no_progress", { step, calls: idleCalls, limit: idleLimit, action: "nudged" }); + yield { kind: "info", text: `no progress: ${idleCalls} tool calls in a row changed no file in the project — told the model to finish or stop` }; + this.transcript.push({ + role: "user", + content: + `[molt: your last ${idleCalls} tool calls changed no file in the project. Reading ` + + `more will not finish the task. If you know what to change, change it now and ` + + `say done. If you cannot finish, say plainly what is blocking you and stop. If ` + + `${idleLimit} more calls change nothing, the turn ends and the work is judged as it stands.]`, + molt: { nudge: true }, + }); + } else if (idleNudged && idleCalls >= 2 * idleLimit) { + log?.append("no_progress", { step, calls: idleCalls, limit: idleLimit, action: "stopped" }); + idleStop = true; + } + } + } + // A step that mostly repeated itself learned little. Worth saying — // and nothing more than that. // diff --git a/src/journal.ts b/src/journal.ts index 2ba7733..89f1853 100644 --- a/src/journal.ts +++ b/src/journal.ts @@ -58,6 +58,17 @@ export type JournalKind = * unfinished, and a reader must be able to tell those apart later. */ | "deadline" + /** + * Unattended, hidden checks: a file tool refused a path outside the task + * (outside the project, or Maat's own `.maat/`), or a bash command named + * one. Bash is not blocked; it is recorded (src/scope.ts). + */ + | "outside_task" + /** + * Unattended: tool calls that changed no file in the project. The first + * entry is the nudge, the second the stop (`action`). + */ + | "no_progress" /** A verified change was committed. Carries the sha and the receipt. */ | "git_commit" /** An unverified change was put back. Carries what was restored, removed, kept. */ @@ -415,6 +426,16 @@ export class Journal { case "deadline": out.push(`${t} deadline · ${Math.round(Number(d.spentMs ?? 0) / 1000)}s of ${Math.round(Number(d.limitMs ?? 0) / 1000)}s — stopped on the clock, not on a failure`); break; + case "outside_task": + out.push( + d.refused + ? `${t} outside the task · ${d.tool} refused ${d.path} (${d.reach})` + : `${t} outside the task · bash named ${[...(d.stateDir ? [".maat/"] : []), ...((d.outside as unknown as string[] | undefined) ?? [])].join(", ")}`, + ); + break; + case "no_progress": + out.push(`${t} no progress · ${d.calls} tool call(s) changed no file · ${d.action}`); + break; case "git_commit": out.push(`${t} commit ${String(d.sha ?? "").slice(0, 8)} · ${d.files} file(s) · receipt ${d.receipt || "(none)"}`); break; diff --git a/src/receipts.ts b/src/receipts.ts index ebd6489..c73ca9a 100644 --- a/src/receipts.ts +++ b/src/receipts.ts @@ -61,7 +61,7 @@ export type ReceiptRecord = { */ ask?: boolean; /** Set when the model was stopped and the tree judged as it stood. */ - endedBy?: "deadline" | "provider"; + endedBy?: "deadline" | "provider" | "no-progress"; /** True when the cost rests on molt's own token estimate anywhere in the session. */ costEstimated?: boolean; /** @@ -382,7 +382,7 @@ export class Receipts { /** True for a question: the bar ran advisory and could not refuse. */ ask?: boolean; /** The model was stopped (clock, provider) and the tree was judged as it stood. */ - endedBy?: "deadline" | "provider"; + endedBy?: "deadline" | "provider" | "no-progress"; /** * Every file the turn changed, with the hashes that prove it — and which * lines it wrote, so the receipt can show the work rather than describe it. @@ -543,7 +543,9 @@ export class Receipts { ? [ args.endedBy === "deadline" ? "The turn's time budget ran out before the model said it was done; the sealed checks ran on the tree as it stood." - : "The provider failed after work had been done; the sealed checks ran on the tree as it stood.", + : args.endedBy === "no-progress" + ? "ended: no progress — the model kept calling tools without changing any file in the project, past a nudge to finish or stop; the sealed checks ran on the tree as it stood." + : "The provider failed after work had been done; the sealed checks ran on the tree as it stood.", "", ] : []), diff --git a/src/scope.ts b/src/scope.ts new file mode 100644 index 0000000..55716d5 --- /dev/null +++ b/src/scope.ts @@ -0,0 +1,167 @@ +/** + * Keeping an unattended worker on its task. + * + * 2026-10-07, the ml4b bench lanes: a worker spent its whole time budget, + * 90 to 125 tool calls a task, reading things that were not the task: the + * other tasks' folders and logs in the shared bench folder, Maat's installed + * source, and Maat's own records of the job under `.maat/` (receipts, the + * journal, spilled output), hunting for the hidden checks. The checks' text + * was withheld (src/withhold.ts); the time was gone all the same. + * + * Three instruments, all for unattended runs: + * + * - While a job with hidden checks runs, read_file, list_dir, grep and + * inspect refuse paths outside the project and paths under its `.maat/`, + * saying plainly that this is outside the task (`outsideTask`). Two + * exceptions: a path the task itself names (a task may say "the config is + * in /etc/app"), and the files Maat itself points the model at under + * `.maat/` (spilled output in `out/`, background logs in `bg/`). + * - bash is not blocked (a shell can reach anything; a regex over a command + * line is not a wall), but a command that names `.maat/` or a path outside + * the project is journalled (`bashReach`). + * - The no-progress guard (engine.ts) counts tool calls that changed no file + * in the project, measured by `treeStamp`. + */ +import { createHash } from "node:crypto"; +import { realpathSync, statSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { TREE_SKIP, walk } from "./files.js"; +import { STATE_DIRS } from "./statedir.js"; + +/** Calls without a change before the nudge; as many again and the turn ends. */ +export const NO_PROGRESS_CALLS = 30; + +/** MAAT_NO_PROGRESS_CALLS when it is a whole number (0 turns the guard off), else the default. */ +export function noProgressCallsFromEnv(env: NodeJS.ProcessEnv = process.env): number { + const raw = env.MAAT_NO_PROGRESS_CALLS?.trim(); + if (raw && /^\d+$/.test(raw)) return Number(raw); + return NO_PROGRESS_CALLS; +} + +/** The path with every existing ancestor's symlinks resolved; the rest kept as written. */ +function real(p: string): string { + let head = resolve(p); + const tail: string[] = []; + for (;;) { + try { + return join(realpathSync(head), ...tail.reverse()); + } catch { + const up = dirname(head); + if (up === head) return resolve(p); + tail.push(head.slice(up.length).replace(/^[/\\]/, "")); + head = up; + } + } +} + +const under = (p: string, root: string) => p === root || p.startsWith(root.endsWith(sep) ? root : root + sep); + +/** + * Absolute paths the task text names, and everything under them. Only the + * path itself: a task that names `/srv/bench/shared.toml` has not opened + * `/srv/bench/`, where the other tasks' folders are. + */ +export function taskPathsIn(task: string): string[] { + const out = new Set(); + for (const m of task.matchAll(/(?:^|[\s"'`(=:,<])((?:\/|~\/)[\w.@+-][\w.@+\-/]*)/g)) { + let p = m[1]!.replace(/[.,:;)]+$/, ""); + if (p.startsWith("~/")) p = join(homedir(), p.slice(2)); + if (p.length < 2) continue; + out.add(real(p)); + } + return [...out]; +} + +export type ScopeVerdict = { kind: "state" | "outside"; message: string } | null; + +/** + * Whether a file tool's path is outside the task, and the refusal to give. + * Null when the path is part of the task. + */ +export function outsideTask( + cwd: string, + path: string, + opts: { tool: string; taskPaths?: readonly string[] }, +): ScopeVerdict { + const root = real(cwd); + const target = real(isAbsolute(path) ? path : resolve(cwd, path)); + const shown = path || "."; + if (!under(target, root)) { + if ((opts.taskPaths ?? []).some((p) => under(target, p))) return null; + return { + kind: "outside", + message: + `refused: ${shown} is outside this task. While this job runs, ${opts.tool} reads only the ` + + `project (${cwd}) and paths the task names. Nothing outside it bears on the task or on how ` + + `the work will be judged; work on the task's own files.`, + }; + } + const rel = relative(root, target).split(sep).join("/"); + const state = STATE_DIRS.find((d) => rel === d || rel.startsWith(`${d}/`)); + if (!state) return null; + // What Maat itself points the model at: a long output it spilled, a background job's log. + if (opts.tool === "read_file" && /^[^/]+\/(out|bg)\/[^/]+$/.test(rel)) return null; + return { + kind: "state", + message: + `refused: ${shown} is Maat's own record of this job (${state}/), not part of the task. ` + + `While the job runs it is closed to ${opts.tool}: nothing in it says how to do the task, and ` + + `the checks that will judge the work are not in it. Work on the task's own files.`, + }; +} + +/** System places a command names to run something, not to look at anything. */ +const BENIGN = /^\/(?:dev\/(?:null|zero|u?random|std(?:in|out|err)|fd\/\d+|tty)|proc\/self\/fd\/\d+|(?:usr\/(?:local\/)?)?s?bin(?:\/[\w.+-]+)?)$/; + +/** + * What a bash command reaches outside the task: whether it names the state + * folder, and the paths outside the project it names. A note for the + * journal, not a wall: heuristics over a command line, so it may miss a + * path built at run time, and names a path that is only written to. + */ +export function bashReach( + cwd: string, + command: string, + taskPaths: readonly string[] = [], +): { stateDir: boolean; outside: string[] } { + const stateDir = new RegExp(`(?:^|[\\s"'\`=:(/<>|;&])(?:${STATE_DIRS.map((d) => d.replace(".", "\\.")).join("|")})(?:/|\\b)`).test(command); + const root = real(cwd); + const outside = new Set(); + for (const m of command.matchAll(/(?:^|[\s"'`=:(<>|;&])((?:\/|~\/|\.\.\/|\.\.(?=$|[\s"'`;|&)]))[^\s"'`;|&<>()$*?[\]{}]*)/g)) { + let p = m[1]!; + if (p.startsWith("~/")) p = join(homedir(), p.slice(2)); + if (BENIGN.test(p)) continue; + const abs = real(isAbsolute(p) ? p : resolve(cwd, p)); + if (under(abs, root)) continue; + if (taskPaths.some((t) => under(abs, t))) continue; + outside.add(m[1]!); + } + return { stateDir, outside: [...outside].slice(0, 20) }; +} + +/** + * A fingerprint of the project's files (path, size, modification time), + * skipping what TREE_SKIP skips (`.maat/`, node_modules, build output). Null + * when the walk was cut short: a tree that cannot be measured is never + * counted as one that did not change. + */ +export function treeStamp(root: string): string | null { + const w = walk(root, { depth: 24, limit: 20_000, skip: TREE_SKIP, deadline: Date.now() + 1_500 }); + if (w.truncated || w.timedOut) return null; + const h = createHash("sha256"); + for (const e of w.entries) { + if (e.kind !== "file") { + h.update(`d\0${e.path}\0`); + continue; + } + let mtime = 0; + try { + mtime = statSync(join(root, e.path)).mtimeMs; + } catch { + /* gone between the walk and the stat: its absence is in the next stamp */ + } + h.update(`f\0${e.path}\0${e.bytes ?? 0}\0${mtime}\0`); + } + return h.digest("hex"); +} diff --git a/src/types.ts b/src/types.ts index 674acc5..a206152 100644 --- a/src/types.ts +++ b/src/types.ts @@ -659,7 +659,7 @@ export type EngineEvent = * happened. The outcome is what the sealed bar said of the tree as it * stood, never a default. */ - endedBy?: "deadline" | "provider"; + endedBy?: "deadline" | "provider" | "no-progress"; /** The turn's wall-clock budget ended it (endedBy "deadline"). */ deadline?: boolean; /** diff --git a/test/no-probing.test.ts b/test/no-probing.test.ts new file mode 100644 index 0000000..8e72b2a --- /dev/null +++ b/test/no-probing.test.ts @@ -0,0 +1,284 @@ +/** + * An unattended worker is kept on its task. + * + * 2026-10-07, the ml4b bench lanes: a worker spent its whole time budget, + * 90-125 tool calls a task, reading the other tasks' logs in the shared bench + * folder and Maat's own `.maat/` records instead of working. While a job with + * hidden checks runs, the file tools refuse paths outside the project and + * under `.maat/`; bash is not blocked but is journalled when it reaches + * there; and a run of tool calls that changes no file is nudged once, then + * stopped and judged on disk ("ended: no progress"). + */ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { Engine } from "../src/engine.js"; +import { Journal } from "../src/journal.js"; +import { Receipts } from "../src/receipts.js"; +import { bashReach, noProgressCallsFromEnv, outsideTask, taskPathsIn } from "../src/scope.js"; +import type { Check, EngineEvent } from "../src/types.js"; +import { allowAll, drain, scriptedProvider, type ScriptedTurn } from "./helpers.js"; + +const greet = (hidden = true): Check => ({ + name: "greeting", kind: "command", run: "grep -qx hello out.txt", timeoutMs: 5_000, expectExit: 0, tags: ["task", "value"], hidden, + author: { kind: "judge", model: "judge-j" }, +}); + +/** A bench-like layout: this task's folder beside another task's folder and log. */ +function bench() { + const root = mkdtempSync(join(tmpdir(), "maat-probe-")); + const dir = join(root, "this-task"); + const other = join(root, "other-task"); + mkdirSync(dir); + mkdirSync(other); + writeFileSync(join(dir, "in.txt"), "hello\n"); + writeFileSync(join(other, "answer.txt"), "the other task's answer\n"); + writeFileSync(join(root, "other-task.log"), '{"kind":"job_end","claim":"verified"}\n'); + mkdirSync(join(dir, ".maat", "receipts"), { recursive: true }); + writeFileSync(join(dir, ".maat", "receipts", "0000-refused.md"), "# a receipt\n"); + mkdirSync(join(dir, ".maat", "out"), { recursive: true }); + writeFileSync(join(dir, ".maat", "out", "call_9.txt"), "spilled output\n"); + return { root, dir, other, cleanup: () => rmSync(root, { recursive: true, force: true }) }; +} + +function engineIn(dir: string, turns: ScriptedTurn[], extra: Partial[0]> = {}) { + const provider = scriptedProvider(turns); + const journal = new Journal(dir); + const engine = new Engine({ + baseUrl: "http://provider.test/v1", model: "m", cwd: dir, fetchFn: provider.fetchFn, bar: null, stream: false, + autonomy: "high", unattended: true, sandbox: true, journal, receipts: new Receipts(dir), noProgressCalls: 0, ...extra, + }); + return { engine, provider, journal }; +} + +/** What each tool call returned, in order, from the tool events. */ +const toolResults = (events: EngineEvent[]) => + events.filter((e): e is Extract => e.kind === "tool").map((e) => ({ name: e.name, detail: e.detail, preview: String(e.preview ?? "") })); + +const end = (events: EngineEvent[]) => { + const e = events.find((x) => x.kind === "job_end"); + assert.ok(e && e.kind === "job_end"); + return e; +}; + +const call = (name: string, args: Record) => ({ name, args }); + +describe("file tools refuse paths outside the task (unattended, hidden checks)", () => { + it("refuses another task's folder and log, the bench root, and .maat/; reads the project, Maat's spilled output and paths the task names", async () => { + const b = bench(); + try { + const named = join(b.root, "shared-config.toml"); + writeFileSync(named, "port = 1\n"); + const task = `Write out.txt with the greeting from in.txt. The port is set in ${named}.`; + const { engine, journal } = engineIn(b.dir, [ + { calls: [ + call("list_dir", { path: ".." }), + call("read_file", { path: "../other-task/answer.txt" }), + call("read_file", { path: join(b.root, "other-task.log") }), + call("grep", { pattern: "verified", path: b.root }), + call("read_file", { path: ".maat/receipts/0000-refused.md" }), + call("list_dir", { path: ".maat" }), + call("grep", { pattern: "greeting", path: ".maat/receipts" }), + call("inspect", { path: "../other-task/answer.txt" }), + ] }, + { calls: [ + call("read_file", { path: "in.txt" }), + call("read_file", { path: ".maat/out/call_9.txt" }), + call("read_file", { path: named }), + call("list_dir", { path: "." }), + ] }, + { calls: [call("write_file", { path: "out.txt", content: "hello\n" })] }, + { text: "Done." }, + ]); + const events = await drain(engine.run(task, allowAll, { taskChecks: [greet()] })); + const results = toolResults(events); + const first = results.slice(0, 8); + for (const r of first) assert.match(r.preview, /^refused: .* (is outside this task|is Maat's own record of this job)/, `${r.name} ${r.detail}: ${r.preview}`); + assert.ok(!first.some((r) => r.preview.includes("the other task's answer") || r.preview.includes("a receipt")), "nothing outside the task was shown"); + assert.match(first[0]!.preview, /outside this task/); + assert.match(first[4]!.preview, /Maat's own record of this job \(\.maat\/\)/); + const [own, spilled, namedRead] = results.slice(8, 11); + assert.match(own!.preview, /hello/); + assert.match(spilled!.preview, /spilled output/, "the output Maat points the model at stays readable"); + assert.match(namedRead!.preview, /port = 1/, "a path the task names is part of the task"); + assert.equal(end(events).outcome, "verified"); + + const notes = Journal.read(journal.path).filter((r) => r.kind === "outside_task"); + assert.equal(notes.length, 8, "every refusal is journalled"); + assert.ok(notes.every((n) => n.data.refused === true)); + assert.deepEqual(notes.map((n) => n.data.reach), ["outside", "outside", "outside", "outside", "state", "state", "state", "outside"]); + assert.deepEqual(notes.map((n) => n.data.tool), ["list_dir", "read_file", "read_file", "grep", "read_file", "list_dir", "grep", "inspect"]); + } finally { + b.cleanup(); + } + }); + + it("bash is not blocked, but a command naming .maat/ or a path outside the project is journalled", async () => { + const b = bench(); + try { + const { engine, journal } = engineIn(b.dir, [ + { calls: [call("bash", { command: "cat ../other-task/answer.txt > seen.txt; ls .maat/receipts >> seen.txt" })] }, + { calls: [call("bash", { command: "python3 -c 'print(1)' > /dev/null && cp in.txt out.txt" })] }, + { text: "Done." }, + ]); + const events = await drain(engine.run("Write out.txt with the greeting from in.txt.", allowAll, { taskChecks: [greet()] })); + assert.match(readFileSync(join(b.dir, "seen.txt"), "utf8"), /the other task's answer[\s\S]*0000-refused\.md/, "bash ran"); + assert.equal(end(events).outcome, "verified"); + const notes = Journal.read(journal.path).filter((r) => r.kind === "outside_task"); + assert.equal(notes.length, 1, "the plain command is not noted"); + assert.equal(notes[0]!.data.tool, "bash"); + assert.equal(notes[0]!.data.refused, false); + assert.equal(notes[0]!.data.stateDir, true); + assert.deepEqual(notes[0]!.data.outside, ["../other-task/answer.txt"]); + assert.match(String(notes[0]!.data.command), /cat \.\.\/other-task/); + } finally { + b.cleanup(); + } + }); + + it("only while a job with hidden checks runs unattended: a person, or visible checks, read where they like", async () => { + for (const [extra, hidden] of [[{ unattended: false }, true], [{}, false]] as const) { + const b = bench(); + try { + const { engine, journal } = engineIn(b.dir, [ + { calls: [call("read_file", { path: ".maat/receipts/0000-refused.md" }), call("read_file", { path: "../other-task/answer.txt" })] }, + { calls: [call("write_file", { path: "out.txt", content: "hello\n" })] }, + { text: "Done." }, + ], extra); + const events = await drain(engine.run("Write out.txt with the greeting from in.txt.", allowAll, { taskChecks: [greet(hidden)] })); + const [receipt, answer] = toolResults(events); + assert.match(receipt!.preview, /a receipt/); + assert.match(answer!.preview, /the other task's answer/); + assert.equal(Journal.read(journal.path).filter((r) => r.kind === "outside_task").length, 0); + } finally { + b.cleanup(); + } + } + }); +}); + +describe("scope helpers", () => { + it("outsideTask, bashReach and taskPathsIn", () => { + const b = bench(); + try { + assert.equal(outsideTask(b.dir, "in.txt", { tool: "read_file" }), null); + assert.equal(outsideTask(b.dir, ".", { tool: "list_dir" }), null); + assert.equal(outsideTask(b.dir, "../other-task", { tool: "list_dir" })?.kind, "outside"); + assert.equal(outsideTask(b.dir, ".maat", { tool: "list_dir" })?.kind, "state"); + assert.equal(outsideTask(b.dir, ".maat/out/call_9.txt", { tool: "read_file" }), null); + assert.equal(outsideTask(b.dir, ".maat/out", { tool: "list_dir" })?.kind, "state", "the spill folder is not listable"); + assert.equal(outsideTask(b.dir, ".maat/log/x.jsonl", { tool: "read_file" })?.kind, "state"); + assert.equal(outsideTask(b.dir, "/etc/app/config.toml", { tool: "read_file", taskPaths: taskPathsIn("edit /etc/app/config.toml") }), null); + assert.equal(outsideTask(b.dir, "/etc/app/other.toml", { tool: "read_file", taskPaths: taskPathsIn("edit /etc/app/config.toml") })?.kind, "outside", "not its neighbours"); + assert.equal(outsideTask(b.dir, "/etc/app/conf.d/x.conf", { tool: "read_file", taskPaths: taskPathsIn("put it in /etc/app/conf.d.") }), null, "under a named folder"); + assert.equal(outsideTask(b.dir, "/etc/passwd", { tool: "read_file", taskPaths: taskPathsIn("edit /etc/app/config.toml") })?.kind, "outside"); + assert.deepEqual(bashReach(b.dir, "grep -r x /usr/local/lib/node_modules/@solvyx/molt/dist | head"), { stateDir: false, outside: ["/usr/local/lib/node_modules/@solvyx/molt/dist"] }); + assert.deepEqual(bashReach(b.dir, "/usr/bin/env python3 run.py 2>/dev/null; cat ./x | sed 's/a/b/'"), { stateDir: false, outside: [] }); + assert.deepEqual(bashReach(b.dir, "find /work -name '*.log'; cd .. && ls"), { stateDir: false, outside: ["/work", ".."] }); + assert.equal(bashReach(b.dir, "cat .maat/log/abc.jsonl").stateDir, true); + assert.equal(bashReach(b.dir, "cat src/maat.ts").stateDir, false); + assert.equal(noProgressCallsFromEnv({}), 30); + assert.equal(noProgressCallsFromEnv({ MAAT_NO_PROGRESS_CALLS: "12" }), 12); + assert.equal(noProgressCallsFromEnv({ MAAT_NO_PROGRESS_CALLS: "0" }), 0); + assert.equal(noProgressCallsFromEnv({ MAAT_NO_PROGRESS_CALLS: "lots" }), 30); + } finally { + b.cleanup(); + } + }); +}); + +describe("no-progress guard (unattended)", () => { + const reads = (n: number) => Array.from({ length: n }, (_, i) => call("read_file", { path: "in.txt", offset: i % 2 })); + + it("nudges once after N calls that change nothing, then ends the turn after N more and judges the work on disk", async () => { + const b = bench(); + try { + // Two calls a step, forever (the last scripted turn repeats). + const { engine, provider, journal } = engineIn(b.dir, [{ calls: reads(2) }], { noProgressCalls: 4 }); + const events = await drain(engine.run("Write out.txt with the greeting from in.txt.", allowAll, { taskChecks: [greet()] })); + const nudges = provider.requests().map((r) => JSON.stringify(r)).map((s) => s.includes("changed no file in the project")); + // Calls 1-4 (steps 1-2) earn the nudge, sent with request 3; calls 5-8 (steps 3-4) end the turn. + assert.deepEqual(nudges, [false, false, true, true], "the nudge goes out once, after the 4th call, and nothing is asked after the 8th"); + assert.equal(provider.calls, 4); + const e = end(events); + assert.equal(e.endedBy, "no-progress"); + assert.notEqual(e.outcome, "verified"); + assert.ok(events.some((x) => x.kind === "proof_start"), "the sealed checks ran on the tree as it stood"); + assert.ok(events.some((x) => x.kind === "info" && /no progress: 8 tool calls in a row changed no file/.test(x.text))); + const dir = join(b.dir, ".maat", "receipts"); + const receipt = readdirSync(dir).filter((f) => f.endsWith(".md") && f !== "0000-refused.md").map((f) => readFileSync(join(dir, f), "utf8")).join("\n"); + assert.match(receipt, /ended: no progress/); + const rows = Journal.read(journal.path).filter((r) => r.kind === "no_progress"); + assert.deepEqual(rows.map((r) => [r.data.action, r.data.calls]), [["nudged", 4], ["stopped", 8]]); + } finally { + b.cleanup(); + } + }); + + it("a stop on passing work is recorded, but the outcome is unverified: the model never said done", async () => { + const b = bench(); + try { + writeFileSync(join(b.dir, "out.txt"), "hello\n"); + const { engine } = engineIn(b.dir, [{ calls: reads(1) }], { noProgressCalls: 2 }); + const e = end(await drain(engine.run("Write out.txt with the greeting from in.txt.", allowAll, { taskChecks: [greet()] }))); + assert.equal(e.endedBy, "no-progress"); + assert.equal(e.outcome, "unverified"); + assert.equal(e.passedAtEnd, true); + } finally { + b.cleanup(); + } + }); + + it("a change after the nudge resets the count, and the turn finishes normally", async () => { + const b = bench(); + try { + const { engine, provider } = engineIn(b.dir, [ + { calls: reads(3) }, + { calls: reads(2) }, + { calls: [call("write_file", { path: "out.txt", content: "hello\n" })] }, + { calls: reads(3) }, + { text: "Done." }, + ], { noProgressCalls: 3 }); + const events = await drain(engine.run("Write out.txt with the greeting from in.txt.", allowAll, { taskChecks: [greet()] })); + const e = end(events); + assert.equal(e.endedBy, undefined); + assert.equal(e.outcome, "verified"); + const nudged = provider.requests().filter((r) => JSON.stringify(r).includes("changed no file in the project")).length; + assert.ok(nudged >= 1, "nudged before the write"); + assert.equal(events.filter((x) => x.kind === "info" && /told the model to finish or stop/.test(x.text)).length, 2, "and once more after the reset"); + } finally { + b.cleanup(); + } + }); + + it("is off when somebody is watching, and when set to 0", async () => { + for (const extra of [{ unattended: false, noProgressCalls: 2 }, { noProgressCalls: 0 }]) { + const b = bench(); + try { + const { engine } = engineIn(b.dir, [...Array.from({ length: 6 }, () => ({ calls: reads(1) })), { calls: [call("write_file", { path: "out.txt", content: "hello\n" })] }, { text: "Done." }], extra); + const events = await drain(engine.run("Write out.txt with the greeting from in.txt.", allowAll, { taskChecks: [greet()] })); + assert.equal(end(events).outcome, "verified"); + assert.ok(!events.some((x) => x.kind === "info" && /no progress/.test(x.text))); + } finally { + b.cleanup(); + } + } + }); + + it("MAAT_NO_PROGRESS_CALLS sets N when the config does not", async () => { + const b = bench(); + const was = process.env.MAAT_NO_PROGRESS_CALLS; + process.env.MAAT_NO_PROGRESS_CALLS = "1"; + try { + const { engine } = engineIn(b.dir, [{ calls: reads(1) }], { noProgressCalls: undefined }); + const e = end(await drain(engine.run("Write out.txt with the greeting from in.txt.", allowAll, { taskChecks: [greet()] }))); + assert.equal(e.endedBy, "no-progress"); + } finally { + if (was === undefined) delete process.env.MAAT_NO_PROGRESS_CALLS; + else process.env.MAAT_NO_PROGRESS_CALLS = was; + b.cleanup(); + } + }); +}); From c257d7ac7fec65a1edfb33b8966df2ceb4f4ab4c Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 14:53:14 -0700 Subject: [PATCH 3/8] Bench: Anthropic API lanes read the key from the Keychain into the container's private env file --- bench/local/container/run-in-container.sh | 7 +++++++ bench/local/container/set-anthropic-key.sh | 10 ++++++++++ bench/local/run.py | 4 +++- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100755 bench/local/container/set-anthropic-key.sh diff --git a/bench/local/container/run-in-container.sh b/bench/local/container/run-in-container.sh index cc271a0..e49d47f 100755 --- a/bench/local/container/run-in-container.sh +++ b/bench/local/container/run-in-container.sh @@ -60,6 +60,13 @@ url=${url//127.0.0.1/host.docker.internal}; url=${url//localhost/host.docker.int # run ends: on the command line (-e KEY=...) it showed in every `ps` listing. envf=$(mktemp); chmod 600 "$envf"; trap 'rm -f "$envf"' EXIT printf 'OPENROUTER_API_KEY=%s\n' "$key" > "$envf" +# Anthropic API lanes (BENCH_URL=https://api.anthropic.com/v1): the key comes from the +# Keychain item maat-bench-anthropic (paste it with container/set-anthropic-key.sh), +# never from the command line. +if [[ "$url" == *api.anthropic.com* ]]; then + akey=$(security find-generic-password -s maat-bench-anthropic -w 2>/dev/null) || { echo "no Keychain item maat-bench-anthropic: run bench/local/container/set-anthropic-key.sh" >&2; exit 1; } + printf 'ANTHROPIC_API_KEY=%s\n' "$akey" >> "$envf" +fi # The graders and reference solutions are mounted where only root can reach (/root is 700), # copied to a root-only /opt/bench, and run.py runs from there as root; it runs the agent # as the unprivileged `agent` user (BENCH_AGENT_USER) and grades as root afterwards. diff --git a/bench/local/container/set-anthropic-key.sh b/bench/local/container/set-anthropic-key.sh new file mode 100755 index 0000000..4cfe830 --- /dev/null +++ b/bench/local/container/set-anthropic-key.sh @@ -0,0 +1,10 @@ +#!/bin/zsh +# Store an Anthropic API key for the bench in the macOS Keychain (item: maat-bench-anthropic). +# The key is read without echo and never appears on a command line or in shell history. +print -n "Paste your Anthropic API key (input hidden): " +read -rs key; print +[[ "$key" == sk-ant-* ]] || { print "That doesn't look like an Anthropic key (sk-ant-...). Nothing saved."; exit 1; } +# Fed to `security -i` on stdin, so the key is never in any process's argument list. +print -r -- "add-generic-password -U -s maat-bench-anthropic -a $USER -w $key" | security -i >/dev/null && + security find-generic-password -s maat-bench-anthropic >/dev/null 2>&1 && print "Saved to Keychain as maat-bench-anthropic (${#key} chars, ends …${key[-4,-1]})." +unset key diff --git a/bench/local/run.py b/bench/local/run.py index e7ea59d..dbb5d45 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -173,7 +173,9 @@ def claim_of(ev: dict) -> str | None: def run_molt(d: Path, prompt: str, log: Path) -> dict: - key = openrouter_key() if "openrouter.ai" in URL else "local" + key = (openrouter_key() if "openrouter.ai" in URL + else os.environ.get("ANTHROPIC_API_KEY", "") if "api.anthropic.com" in URL + else "local") env = os.environ | {"MOLT_API_KEY": key, "MOLT_JUDGMENT": "0"} # nobody rules on a benchmark run cmd = [ "node", str(MOLT), "run", "--url", URL, "--model", os.environ.get("BENCH_MODEL") or MODEL, From c943cb882bd7ef5fd80c02e14d77ebc61d9e48b8 Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 17:29:16 -0700 Subject: [PATCH 4/8] No-progress guard: reading, running and testing are progress, not only file changes The guard counted every call that left the project's tree stamp unchanged as idle, so a build into dist/, an install into .venv or node_modules, or a config edit under /etc ended a legitimate run as "no progress" after 60 calls. A call now advances when its (call, result) pair is new for the turn: a file read for the first time, a command run for the first time, or a command or read whose output differs from every earlier time it was made (durations, clock times and job ids aside). A change to a file in the project still resets the count. Maat's own pointers ("already shown", "same call as step N"), malformed-call complaints and outside-task refusals are never progress. Only a run of calls that come back with the same answer again is counted. The guard stays on for every unattended HTTP run, documented in the CHANGELOG with MAAT_NO_PROGRESS_CALLS; ACP backends are not covered yet. On a case-insensitive disk the .maat/ check folds case, and the outside-task refusal says bash can still read a scratch file it made. Tests are the review's reproductions: builds into dist/, .venv, node_modules, build/, target/, out/; edits to a config outside the project the task names; the same command with changing output; first reads of new files. None is stopped. The same command with the same output (timing aside), and refused reads of many different outside paths, are. --- CHANGELOG.md | 15 ++++++ src/engine.ts | 65 +++++++++++++++------- src/scope.ts | 56 +++++++++++++++++-- test/no-probing.test.ts | 117 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 221 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ba80e1..25a7d04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,21 @@ - **`opencode://zen`** is the OpenCode endpoint (short name `opencode`). The old `opencode://subscription` still works for this release, with a deprecation notice. +- **Unattended runs stop when they stop advancing** (`MAAT_NO_PROGRESS_CALLS`, + default 30, `0` turns it off; on for every unattended run on an HTTP + backend, not on ACP backends yet). A call advances when it changes a file + in the project, reads something not read before, runs a command not run + before, or gets output from a command that differs from every earlier run + of it (timings aside), wherever the work happens: a build into `dist/`, a + venv, a config under `/etc`. After N calls in a row that advance nothing the + model is told once to finish or stop; after N more the turn ends, the work + on disk is judged, and the receipt says `ended: no progress` (never + verified: the model did not say it was done). +- **While an unattended job with hidden checks runs, the file tools stay in + the task.** `read_file`, `list_dir`, `grep` and `inspect` refuse paths + outside the project and under `.maat/` (except spilled output and + background logs, and paths the task names). bash is not blocked; this is a + budget control, not isolation. ## 0.2.2 — subscription backends removed diff --git a/src/engine.ts b/src/engine.ts index a39f089..7592a20 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -133,7 +133,7 @@ import { type Spend, } from "./types.js"; import { stateDir, stateDirName } from "./statedir.js"; -import { bashReach, noProgressCallsFromEnv, outsideTask, taskPathsIn, treeStamp } from "./scope.js"; +import { bashReach, noProgressCallsFromEnv, outsideTask, ProgressMeter, taskPathsIn, treeStamp } from "./scope.js"; import { env } from "./env.js"; import { Judgments, caseReason, reasonText } from "./judgment.js"; @@ -5933,15 +5933,22 @@ export class Engine { */ let deadlineInterrupted = false; /** - * The no-progress guard, unattended only (src/scope.ts). Tool calls since - * a file in the project last changed; at `idleLimit` the model is told - * once to finish or stop, at twice that the turn ends and the work on - * disk is judged the way the deadline judges it. 2026-10-07: a worker - * made 90-125 calls a task reading other tasks' logs and Maat's records, - * changing nothing, until the clock ran out. + * The no-progress guard, unattended only (src/scope.ts). Tool calls in a + * row that advanced nothing: no file in the project changed, and each + * call returned only what it had returned before (ProgressMeter). Reading + * something new, running a command for the first time, or getting a + * different answer from one is progress, wherever the work happens. At + * `idleLimit` the model is told once to finish or stop, at twice that the + * turn ends and the work on disk is judged the way the deadline judges + * it. 2026-10-07: a worker made 90-125 calls a task reading other tasks' + * logs and Maat's records until the clock ran out. HTTP backends only: + * an ACP backend (grok-build://, OpenCode) runs its tools inside one step. */ const idleLimit = this.cfg.unattended ? (this.cfg.noProgressCalls ?? noProgressCallsFromEnv()) : 0; let idleStamp = idleLimit > 0 ? treeStamp(this.cwd) : null; + const progress = new ProgressMeter(); + /** Per call this step: whether it advanced anything (see ProgressMeter). */ + const advancedThisStep: boolean[] = []; let idleCalls = 0; let idleNudged = false; let idleStop = false; @@ -5979,7 +5986,7 @@ export class Engine { yield { kind: "info", text: - `no progress: ${idleCalls} tool calls in a row changed no file in the project, after ` + + `no progress: ${idleCalls} tool calls in a row changed no file and returned nothing new, after ` + `being told to finish or stop — no more tool calls. The sealed checks judge the work as it stands.`, }; const judged = yield* judgeOnDisk("no-progress", ""); @@ -7171,8 +7178,20 @@ export class Engine { } } if (pendingCriteria && planned.some((p) => changesSomething(p.name, p.rawArgs))) yield* settleCriteria(false); + advancedThisStep.length = 0; for (const p of planned) { const outcome = yield* this.invokeTool(p, { step, userText, confirm, log, shown, answered }); + if (idleLimit > 0) { + let parsed: Record | undefined; + try { + const v = JSON.parse(p.rawArgs || "{}") as unknown; + if (v && typeof v === "object" && !Array.isArray(v)) parsed = v as Record; + } catch { + /* malformed: its answer is Maat's complaint, which is never progress */ + } + const key = parsed ? callKey(outcome.name, parsed) : `${outcome.name}(${p.rawArgs})`; + advancedThisStep.push(!outcome.repeated && progress.advanced(key, outcome.result)); + } called.push(outcome.name); if (outcome.repeated) repeated += 1; if (outcome.auto) autoRan += 1; @@ -7188,26 +7207,34 @@ export class Engine { if (idleLimit > 0) { const stamp = treeStamp(this.cwd); - if (stamp === null || idleStamp === null) { - // Not measurable (the walk was cut short): never counted as idle. - idleStamp = stamp; - } else if (stamp !== idleStamp) { - idleStamp = stamp; + const changed = stamp === null || idleStamp === null || stamp !== idleStamp; + idleStamp = stamp; + if (changed) { + // A file in the project changed, or the tree could not be measured + // (never counted as idle). idleCalls = 0; idleNudged = false; } else { - idleCalls += called.length; + // In order: the run of non-advancing calls is what is counted, and + // one call that advanced anything starts it again. + for (const a of advancedThisStep) { + if (a) { + idleCalls = 0; + idleNudged = false; + } else idleCalls += 1; + } if (!idleNudged && idleCalls >= idleLimit) { idleNudged = true; log?.append("no_progress", { step, calls: idleCalls, limit: idleLimit, action: "nudged" }); - yield { kind: "info", text: `no progress: ${idleCalls} tool calls in a row changed no file in the project — told the model to finish or stop` }; + yield { kind: "info", text: `no progress: ${idleCalls} tool calls in a row changed no file and returned nothing new — told the model to finish or stop` }; this.transcript.push({ role: "user", content: - `[molt: your last ${idleCalls} tool calls changed no file in the project. Reading ` + - `more will not finish the task. If you know what to change, change it now and ` + - `say done. If you cannot finish, say plainly what is blocking you and stop. If ` + - `${idleLimit} more calls change nothing, the turn ends and the work is judged as it stands.]`, + `[molt: your last ${idleCalls} tool calls changed no file and returned nothing they ` + + `had not returned before. Repeating them will not finish the task. If you know what ` + + `to change, change it now and say done. If you cannot finish, say plainly what is ` + + `blocking you and stop. If ${idleLimit} more calls advance nothing, the turn ends ` + + `and the work is judged as it stands.]`, molt: { nudge: true }, }); } else if (idleNudged && idleCalls >= 2 * idleLimit) { diff --git a/src/scope.ts b/src/scope.ts index 55716d5..6224470 100644 --- a/src/scope.ts +++ b/src/scope.ts @@ -19,8 +19,13 @@ * - bash is not blocked (a shell can reach anything; a regex over a command * line is not a wall), but a command that names `.maat/` or a path outside * the project is journalled (`bashReach`). - * - The no-progress guard (engine.ts) counts tool calls that changed no file - * in the project, measured by `treeStamp`. + * - The no-progress guard (engine.ts) counts tool calls in a row that did + * not advance anything (`ProgressMeter`): no file in the project changed + * (`treeStamp`), and the call returned nothing it had not returned before. + * Reading a file not read before, a command not run before, or a command + * whose output differs from every earlier run of it is progress, wherever + * it works: a build into dist/, a venv, a config under /etc. What stops a + * run is the same call coming back with the same answer, over and over. */ import { createHash } from "node:crypto"; import { realpathSync, statSync } from "node:fs"; @@ -94,14 +99,17 @@ export function outsideTask( message: `refused: ${shown} is outside this task. While this job runs, ${opts.tool} reads only the ` + `project (${cwd}) and paths the task names. Nothing outside it bears on the task or on how ` + - `the work will be judged; work on the task's own files.`, + `the work will be judged; work on the task's own files. (A scratch file you made outside ` + + `the project with bash can still be read with bash.)`, }; } const rel = relative(root, target).split(sep).join("/"); - const state = STATE_DIRS.find((d) => rel === d || rel.startsWith(`${d}/`)); + // On a case-insensitive disk (macOS, Windows) `.MAAT/log` is `.maat/log`. + const fold = process.platform === "darwin" || process.platform === "win32" ? rel.toLowerCase() : rel; + const state = STATE_DIRS.find((d) => fold === d.toLowerCase() || fold.startsWith(`${d.toLowerCase()}/`)); if (!state) return null; // What Maat itself points the model at: a long output it spilled, a background job's log. - if (opts.tool === "read_file" && /^[^/]+\/(out|bg)\/[^/]+$/.test(rel)) return null; + if (opts.tool === "read_file" && /^[^/]+\/(out|bg)\/[^/]+$/.test(fold)) return null; return { kind: "state", message: @@ -165,3 +173,41 @@ export function treeStamp(root: string): string | null { } return h.digest("hex"); } + +/** + * Output that differs between two runs of the same command for no reason that + * matters: durations, clock times, process ids in a "[1] 12345" job line. Left + * in, every timed test run would count as new output forever. + */ +function settled(result: string): string { + return result + .replace(/\b\d{4}-\d\d-\d\d[T ]\d\d:\d\d(?::\d\d(?:[.,]\d+)?)?(?:Z|[+-]\d\d:?\d\d)?/g, "