Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 24 additions & 5 deletions bench/local/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,18 @@ def as_agent(cmd: list, env: dict | None = None) -> tuple[list, dict | None]:
return ["runuser", "-u", AGENT_USER, "--", "env", f"HOME={home}", *cmd], env


def kill_tree(proc: subprocess.Popen) -> None:
"""Kill the agent's whole process group, and in a container every process of the
unprivileged agent user (a backend CLI may start its own session)."""
import signal
try:
os.killpg(proc.pid, signal.SIGKILL)
except (ProcessLookupError, PermissionError):
pass
if AGENT_USER and shutil.which("pkill"):
subprocess.run(["pkill", "-KILL", "-u", AGENT_USER], check=False)


def hand_over(d: Path) -> None:
if AGENT_USER:
subprocess.run(["chown", "-R", AGENT_USER, str(d)], check=True)
Expand Down Expand Up @@ -93,13 +105,20 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict:
]
t0 = time.time()
cmd, env = as_agent(cmd, env)
# Own session, so the backstop kills the whole tree. subprocess.run's timeout killed only
# the top process (runuser in the containers) and then waited on the pipe Maat and its
# backend still held: one hung Grok turn ran 3570 s against a 600 s limit (2026-10-07).
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env,
start_new_session=True)
try:
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=LIMIT, env=env)
out, err = proc.stdout, proc.stderr
out, err = proc.communicate(timeout=LIMIT)
timed_out = False
except subprocess.TimeoutExpired as e:
out = (e.stdout or b"").decode() if isinstance(e.stdout, bytes) else (e.stdout or "")
err = (e.stderr or b"").decode() if isinstance(e.stderr, bytes) else (e.stderr or "")
except subprocess.TimeoutExpired:
kill_tree(proc)
try:
out, err = proc.communicate(timeout=30)
except subprocess.TimeoutExpired:
out, err = "", ""

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we reach this, a descendant escaped the session (setsid) and still holds the pipe, and its output is lost. Maybe write a short note to err here (e.g. "backstop: pipe still held 30 s after kill") so the .err log explains the blank row.

timed_out = True
secs = time.time() - t0
log.write_text(out)
Expand Down
20 changes: 20 additions & 0 deletions bench/local/tests/test_kill_tree.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import subprocess
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import run # noqa: E402


def test_kill_tree_releases_the_pipe_a_grandchild_holds():
# A grandchild keeps stdout open: killing only the top process would leave communicate() waiting.
proc = subprocess.Popen(["sh", "-c", "sleep 30 & sleep 30"], stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True, start_new_session=True)
t0 = time.time()
try:
proc.communicate(timeout=0.5)
except subprocess.TimeoutExpired:
run.kill_tree(proc)

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This hits the real pkill -u $BENCH_AGENT_USER branch when that env var is set. Suggest pinning run.AGENT_USER = None via monkeypatch so the test never kills a running bench's agent processes.

proc.communicate(timeout=5)
assert time.time() - t0 < 5
Loading