Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 30 additions & 1 deletion .github/workflows/check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: check
on:
push:
branches: [main]
# Stacked work is reviewed as PRs into claude/* branches, not only main.
# Stacked PRs target claude/* branches, not main; both get the checks
# (privsep-linux included).
pull_request:
branches: [main, "claude/**"]

Expand All @@ -30,3 +31,31 @@ jobs:
node dist/cli.js --version
# doctor may exit >1 when default localhost endpoint is down (no Ollama in CI)
node dist/cli.js doctor || true

# Privilege separation (--worker-user, --check-user, --worker-strict) is
# Linux-first and needs root to switch users: its tests skip on macOS and
# run here, in a root container. The test creates its own worker and check
# users. The container gets CAP_SYS_ADMIN, so the worker's PID namespace and
# each check run's PID namespace and private /tmp can be made, and
# MAAT_TEST_REQUIRE_PIDNS=1 makes the namespace tests fail here rather than
# skip when they are not. Docker's default AppArmor profile on Ubuntu hosts
# denies mount(2) (needed for the namespace's own /proc and the check's
# tmpfs /tmp) even with the capability, so it is lifted for this container.
privsep-linux:
runs-on: ubuntu-latest
container:
image: node:22-bookworm
options: --cap-add=SYS_ADMIN --security-opt apparmor=unconfined
env:
ELECTRON_SKIP_BINARY_DOWNLOAD: "1"
MAAT_TEST_REQUIRE_PIDNS: "1"
steps:
- uses: actions/checkout@v4
- run: npm ci
- run: npm run test:build && npm run build
# The job container's HOME is /github/home, a runner bind mount the
# worker can read and root does not own, so Maat (correctly) refuses it
# as the judge's HOME. root's own HOME is private.
- run: node --test --test-reporter=spec dist-test/test/privsep.test.js
env:
HOME: /root
12 changes: 9 additions & 3 deletions bench/local/container/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/* \
&& git config --system user.name "bench" && git config --system user.email "bench@localhost" \
&& git config --system init.defaultBranch master
# The agent under test runs as this user; run-in-container.sh keeps the graders and the
# reference solutions in root-only places it cannot read.
# The worker's tools run as this user: Maat runs as root with --worker-user agent (privilege
# separation, src/privsep.ts), or wholly as this user with BENCH_PRIVSEP=0. run-in-container.sh
# keeps the graders and the reference solutions in root-only places it cannot read; /root (the
# judge's HOME) is 700.
RUN useradd -m -u 1001 -s /bin/bash agent && chmod 700 /root
ENV BENCH_AGENT_USER=agent
# Task checks (hidden, drafted, mission) run as this user (--check-user checker): its own group,
# no home (Maat gives each check run a fresh empty HOME). It can read the reference check's
# files and its own copy of the work; `agent` cannot read the reference check.
RUN useradd -M -U -u 1002 -s /bin/sh checker
ENV BENCH_AGENT_USER=agent BENCH_CHECK_USER=checker
WORKDIR /opt
# Maat itself is installed per build by run-in-container.sh (FROM maat-bench + npm i -g the packed tarball).

Expand Down
2 changes: 1 addition & 1 deletion bench/local/container/Dockerfile.grok
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# Installed with the official script (https://x.ai/cli/install.sh), pinned to the version the
# owner's Mac runs so ACP behaviour matches. No credentials are baked in: run-in-container.sh
# copies them in at start (SUBSCRIPTION=grok).
FROM maat-bench:agentu
FROM maat-bench:agentcu
ARG GROK_VERSION=1.0.46
RUN curl -fsSL https://x.ai/cli/install.sh | HOME=/opt/grok-install GROK_BIN_DIR=/opt/grok-install/.grok/bin bash -s ${GROK_VERSION} \
&& cp -L "$(readlink -f /opt/grok-install/.grok/bin/grok)" /tmp/grok.bin && rm -f /usr/local/bin/grok /usr/local/bin/agent && mv /tmp/grok.bin /usr/local/bin/grok && rm -rf /opt/grok-install \
Expand Down
2 changes: 1 addition & 1 deletion bench/local/container/Dockerfile.opencode
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# (run-in-container.sh passes BASE), so OPENCODE=1 combines with SUBSCRIPTION=grok. Pinned to the
# version the owner's Mac runs. No credentials are baked in: run-in-container.sh copies
# auth.json in at start.
ARG BASE=maat-bench:agentu
ARG BASE=maat-bench:agentcu
FROM ${BASE}
ARG OPENCODE_VERSION=1.18.33
RUN npm install -g --no-fund --no-audit opencode-ai@${OPENCODE_VERSION} && opencode --version
42 changes: 33 additions & 9 deletions bench/local/container/run-in-container.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,23 @@
# there. It is for the judge only:
# ARMS="oc:MAAT_JUDGE_MODEL=opencode/big-pickle,MAAT_JUDGE_URL=opencode://zen"
# The key is never printed.
# Privilege separation (default; BENCH_PRIVSEP=0 turns it off): Maat runs as root and only the
# worker's tools run as `agent` (--worker-user agent); task checks run as `checker`
# (--check-user checker), which can read the reference check and `agent` cannot. Maat's records sit in /var/lib/maat (700,
# root) until each job ends; the judge's logins are in /root (700); finished task folders and
# logs in /work are locked to root. BENCH_PIDNS=1 (default) also gives the container
# CAP_SYS_ADMIN so Maat can put the worker's commands in their own PID namespace with a private
# /proc (the worker then sees none of Maat's processes; it gets no capability itself).
# BENCH_PIDNS=0: no extra capability, and the worker can read the process list.
set -e
name=$1; tgz=${2:A}; shift 2; [ "$1" = "--" ] && shift
here=${0:A:h}; local_dir=${here:h}
# Image tags carry ":agentu": images with the unprivileged `agent` user (older ones lack it).
sha=$(shasum -a 256 "$tgz" | cut -c1-8); base=maat-bench:agentu; img=maat-bench:$sha-u
# Image tags carry ":agentcu": images with the unprivileged `agent` (worker) and `checker` (task
# checks, --check-user) users. Older ":agentu" images lack `checker`.
sha=$(shasum -a 256 "$tgz" | cut -c1-8); base=maat-bench:agentcu; img=maat-bench:$sha-cu
docker image inspect $base >/dev/null 2>&1 || docker build -q -t $base "$here" >/dev/null
if [ "$SUBSCRIPTION" = grok ]; then
base=maat-bench-grok:agentu; img=maat-bench-grok:$sha-u
base=maat-bench-grok:agentcu; img=maat-bench-grok:$sha-cu
docker image inspect $base >/dev/null 2>&1 || docker build -q -t $base -f "$here/Dockerfile.grok" "$here" >/dev/null
[ -f ~/.grok/auth.json ] || { echo "no ~/.grok/auth.json: run 'grok login' on the Mac first" >&2; exit 1; }
# The token is short-lived (~6 h) and a refresh inside the container may rotate the refresh
Expand All @@ -38,7 +47,7 @@ if [ "$SUBSCRIPTION" = grok ]; then
fi
if [ "$OPENCODE" = 1 ]; then
[ -f ~/.local/share/opencode/auth.json ] || { echo "no ~/.local/share/opencode/auth.json: run 'opencode auth login' on the Mac first" >&2; exit 1; }
ocbase=${base%:*}-oc:agentu
ocbase=${base%:*}-oc:agentcu
docker image inspect $ocbase >/dev/null 2>&1 || docker build -q -t $ocbase -f "$here/Dockerfile.opencode" --build-arg BASE=$base "$here" >/dev/null
base=$ocbase; img=${img%:*}-oc:${img##*:}
fi
Expand All @@ -62,11 +71,18 @@ printf 'OPENROUTER_API_KEY=%s\n' "$key" > "$envf"
# The graders and reference solutions are mounted where only root can reach (/root is 700),
# copied to a root-only /opt/bench, and run.py runs from there as root; it runs the agent
# as the unprivileged `agent` user (BENCH_AGENT_USER) and grades as root afterwards.
credmount=; startcmd='export MOLT_DIST_ABS=$(npm root -g)/@solvyx/molt/dist; mkdir -p /work && chmod 755 /work && rm -rf /opt/bench && cp -a /root/bench-src /opt/bench && chmod -R go-rwx /opt/bench && cd /opt/bench && python3 run.py "$@"'
# Tasks run in a container-local /var/lib/bench-work (711: the worker reaches its own task folder
# by name but cannot list the others) and are copied to /work when run.py ends. /work is a host
# bind mount, where chown is ignored and modes are not enforced, so nothing locked there would
# stay locked from a later task's worker.
privsep=${BENCH_PRIVSEP:-1}
credmount=; startcmd='export MOLT_DIST_ABS=$(npm root -g)/@solvyx/molt/dist BENCH_WORK=/var/lib/bench-work BENCH_EXPORT=/work; mkdir -p /work $BENCH_WORK && chmod 711 $BENCH_WORK && rm -rf /opt/bench && cp -a /root/bench-src /opt/bench && chmod -R go-rwx /opt/bench && cd /opt/bench && python3 run.py "$@"'
if [ "$SUBSCRIPTION" = grok ]; then
credmount="-v $HOME/.grok/auth.json:/root/grok-cred/auth.json:ro"
# grok runs as `agent`, so the credential copy lives in ITS home, owned by it, mode 600.
# The worker's grok runs as `agent`, so its credential copy lives in ITS home, owned by it,
# mode 600. A grok judge runs as Maat (root) with HOME=/root, so it gets its own copy there.
startcmd='install -d -o agent -g agent -m 700 /home/agent/.grok && install -o agent -g agent -m 600 /root/grok-cred/auth.json /home/agent/.grok/auth.json && '$startcmd
[ "$privsep" = 0 ] || startcmd='install -d -m 700 /root/.grok && install -m 600 /root/grok-cred/auth.json /root/.grok/auth.json && '$startcmd
# GROK_OWN_TOOLS=1: let Grok Build use its own shell/edit tools (inside the container only).
# Needed because Grok hides Maat's MCP tools behind its search_tool/use_tool, so by default
# the model asks for run_terminal_command, Maat refuses, and the turn ends with no work done.
Expand All @@ -78,12 +94,20 @@ if [ "$OPENCODE" = 1 ]; then
ocauth=$(mktemp); chmod 600 "$ocauth"; trap 'rm -f "$envf" "$ocauth"' EXIT
python3 -c 'import json,sys;a=json.load(open(sys.argv[1]));json.dump({k:v for k,v in a.items() if k=="opencode"},open(sys.argv[2],"w"))' ~/.local/share/opencode/auth.json "$ocauth"
credmount="$credmount -v $ocauth:/root/oc-cred/auth.json:ro"
startcmd='install -d -o agent -g agent -m 700 /home/agent/.local /home/agent/.local/share /home/agent/.local/share/opencode && install -o agent -g agent -m 600 /root/oc-cred/auth.json /home/agent/.local/share/opencode/auth.json && '$startcmd
# The OpenCode judge runs as Maat: with privilege separation that is root, HOME=/root, which
# the worker cannot read; without it, as `agent`.
if [ "$privsep" = 0 ]; then
startcmd='install -d -o agent -g agent -m 700 /home/agent/.local /home/agent/.local/share /home/agent/.local/share/opencode && install -o agent -g agent -m 600 /root/oc-cred/auth.json /home/agent/.local/share/opencode/auth.json && '$startcmd
else
startcmd='install -d -m 700 /root/.local /root/.local/share /root/.local/share/opencode && install -m 600 /root/oc-cred/auth.json /root/.local/share/opencode/auth.json && '$startcmd
fi
fi
caps=
[ "$privsep" = 0 ] || [ "${BENCH_PIDNS:-1}" = 0 ] || caps=--cap-add=SYS_ADMIN
echo "maat $img → $out/${RESULTS:-results.jsonl}"
docker run --rm --name "maat-bench-$name" \
docker run --rm --name "maat-bench-$name" ${=caps} \
-v "$local_dir":/root/bench-src:ro -v "$out":/results -v "$work":/work ${=credmount} \
--env-file "$envf" -e REFERENCE="${REFERENCE:-0}" -e BENCH_MODEL="${BENCH_MODEL:-}" \
-e ARMS="${ARMS:-}" -e BENCH_TASKS="${BENCH_TASKS:-}" -e BENCH_URL="$url" -e BENCH_LIMIT="${BENCH_LIMIT:-600}" -e BENCH_REASONING="${BENCH_REASONING:-}" -e BENCH_GATE="${BENCH_GATE:-}" -e RESULTS="${RESULTS:-results.jsonl}" \
-e MAAT_BUILD="$sha" -e PYTHONDONTWRITEBYTECODE=1 \
-e MAAT_BUILD="$sha" -e PYTHONDONTWRITEBYTECODE=1 -e BENCH_PRIVSEP="$privsep" \
"$img" sh -c "$startcmd" run "$@"
21 changes: 12 additions & 9 deletions bench/local/crosscheck.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,28 +54,31 @@


# ------------------------------------------------------------------ inside the container
def inside() -> None:
def inside(job_path: str = "/job.json", src: str = "/src", work: str = "/work") -> None:
sys.path.insert(0, str(HERE))
job = json.load(open("/job.json"))
job = json.load(open(job_path))
from tasks import TASKS # noqa: PLC0415
from tasks2 import TASKS2 # noqa: PLC0415

import run # noqa: PLC0415 (the grading helpers run.py grades with)

by = {T.name: T for T in TASKS + TASKS2}
os.environ["GIT_CONFIG_COUNT"] = "1"
os.environ["GIT_CONFIG_KEY_0"] = "safe.directory"
os.environ["GIT_CONFIG_VALUE_0"] = "*"
dst = Path("/work") / job["name"]
# No safe.directory=*: the trees are agent-written, and their .git/config (core.fsmonitor,
# hooks, filters, diff drivers) would run as root. The grader grades a sanitized root-owned
# copy exactly as run.py does (run.grade_safely: config cut to the format lines, hooks and
# attributes removed, git started without system or global config). The checks run on a
# root-owned copy whose repositories are sanitized the same way (run.sanitized_copy).
dst = Path(work) / job["name"]

def fresh() -> Path:
shutil.rmtree(dst, ignore_errors=True)
dst.parent.mkdir(parents=True, exist_ok=True)
subprocess.run(["cp", "-R", "--preserve=mode,timestamps", "/src", str(dst)], check=True)
return dst
return run.sanitized_copy(Path(src), dst)

res: dict = {"name": job["name"], "checks": {}}
try:
d = fresh()
ok, why = by[job["task"]].grade(d)
ok, why = run.grade_safely(by[job["task"]], d)
res["grade"] = [bool(ok), str(why)[:300]]
except Exception as e: # noqa: BLE001
res["grade"] = [None, f"grader crashed: {e}"[:300]]
Expand Down
Loading
Loading