diff --git a/docs/UAT.md b/docs/UAT.md index 569c63e..c13df1b 100644 --- a/docs/UAT.md +++ b/docs/UAT.md @@ -247,11 +247,19 @@ killed if it will not. What a harness leaves running after a normal exit is stop - **A harness group killed hard** (measured with Claude Code and Codex): the browser exits on its own within seconds, because its debugging pipe closes with the server, and the runner reports `harness_exit`. -- **`lyra-uat run` itself killed with SIGKILL** cannot be handled, and nothing below it is - stopped: the harness carries on with its persona, with its browser, until it finishes - or reaches `brain.max_turns` / `brain.max_budget_usd`, then exits and takes the browser - with it. Keep those two limits set. Stop such a run with SIGINT to the harness's process - group. +- **`lyra-uat run` or `lyra-uat batch` itself killed with SIGKILL** cannot be handled by + the process itself, so a small *sweeper* does it: a process of its own session that + holds a pipe from `lyra-uat` and learns, as the run goes, every child group it started + and the browser profile directories in use. When the pipe closes without a goodbye + it terminates, then kills, those groups and every process with a `--flag=` + argument pointing into one of those directories (Chrome's `--user-data-dir`, its crash + handler's `--database`). It never matches by program name, so a Chrome of the + operator's own is never a candidate; a clean exit tells it to leave. Measured with + `scripts/verify_uat_procs_e2e.py` (2026-11-15, Linux, headless, 3 runs per driver): a + SIGKILLed process with an open browser left **0** processes on its profile within 10 s + with the sweeper on the native and the Playwright driver; without it the native + driver's browser stayed (10 of 11 processes still running after 10 s), Playwright's + exited on its own when its driver's pipe closed. This is POSIX code (process groups, signals). It was run on Linux only: where there is no `/proc` (macOS) it falls back to `killpg` alone for the liveness check, which has not been @@ -291,6 +299,15 @@ once. The key under `personas` must equal the `id` inside that persona's file; `preseeded` personas need an entry in `data_dirs`, and two personas of a parallel group may not share one. +Four settings make a long or parallel batch safer to leave alone: + +| key | default | what it does | +|---|---|---| +| `disposable_profiles` | `false` | every attempt runs on a profile of its own under `/profiles/`, deleted when it ends: a copy of the persona's `data_dirs` entry without Chrome's lock files (the original is never written to), or an empty one. Nothing one run leaves in a profile reaches the next | +| `retries` | `0` (max 5) | a persona whose child crashed (no report) or ended on `browser_error`, `brain_error` or `harness_exit` is run again, up to this many more times. What the persona met on the site (budget, wall clock, no `finish`) is a result and is not retried. Each attempt has its own spec and log files (`-attempt2.json`), `attempts` in the summary counts them, and the warnings say why an earlier one failed | +| `retry_delay_s` | `5` | pause before a retry | +| `start_interval_s` | `0` | at least this many seconds between two child starts anywhere in the batch, retries included, so a parallel group does not open every session on the target in the same second | + `summary.json` and `summary.md` list each persona's outcome, run status, steps, finding counts, verdict counts and cost, and lay the earlier findings out as a matrix: which persona judged each one `FIXED`, `STILL_THERE` or `COULD_NOT_CHECK`, diff --git a/scripts/verify_uat_procs_e2e.py b/scripts/verify_uat_procs_e2e.py new file mode 100644 index 0000000..730704a --- /dev/null +++ b/scripts/verify_uat_procs_e2e.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Real-browser gate: a UAT process killed with SIGKILL leaves no Chrome behind. + +A child process opens a real browser the way a run does (``build_tools`` + +``open_browser`` on a throwaway data dir), hands the data dir to the sweeper +(``lyra_browser.uat.procs.sweeper``), says it is ready, and waits. The gate kills it +with SIGKILL, which it cannot handle, and then counts the processes whose command line +points into that data dir. With the sweeper the count must reach 0; the control run +(``--control``: the same child, no sweeper) measures what is left without it. + +Processes are found by the throwaway data dir only, never by program name: the +operator's own Chrome is never counted or touched. Loopback only, nothing is visited. + + .venv/bin/python scripts/verify_uat_procs_e2e.py + LYRA_BROWSER_DRIVER=playwright .venv/bin/python scripts/verify_uat_procs_e2e.py --control +""" + +from __future__ import annotations + +import argparse +import os +import signal +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +SCRIPTS = Path(__file__).resolve().parent +REPO = SCRIPTS.parent + +CHILD = r""" +import asyncio, sys +from pathlib import Path +sys.path.insert(0, sys.argv[3]) +from verify_browser_e2e import build_tools +from lyra_browser.uat import procs + +data_dir, sweep = Path(sys.argv[1]), sys.argv[2] == "sweep" + +async def main(): + if sweep: + procs.sweeper.watch_dir(data_dir) + ctx, tools = await build_tools(data_dir, headless=True) + opened = await tools["open_browser"]() + print("ready" if opened.get("status") == "ok" else f"failed {opened}", flush=True) + await asyncio.sleep(600) + +asyncio.run(main()) +""" + + +def pids_under(directory: Path) -> list[int]: + """Processes with ``directory`` anywhere in their command line (this script excluded).""" + needle = os.fsencode(str(directory)) + found = [] + for name in os.listdir("/proc"): + if not name.isdigit() or int(name) == os.getpid(): + continue + try: + with open(f"/proc/{name}/cmdline", "rb") as handle: + cmdline = handle.read() + except OSError: + continue + if needle in cmdline: + found.append(int(name)) + return found + + +def run_once(sweep: bool, settle_s: float) -> tuple[int, int]: + """(processes on the data dir before the kill, processes left ``settle_s`` after it).""" + with tempfile.TemporaryDirectory(prefix="lyra-uat-procs-", ignore_cleanup_errors=True) as tmp: + data_dir = Path(tmp) + env = {**os.environ, "PYTHONPATH": os.pathsep.join([str(REPO / "src"), *sys.path])} + child = subprocess.Popen( + [ + sys.executable, + "-c", + CHILD, + str(data_dir), + "sweep" if sweep else "none", + str(SCRIPTS), + ], + stdout=subprocess.PIPE, + text=True, + env=env, + ) + try: + line = child.stdout.readline().strip() + if line != "ready": + raise AssertionError(f"the child did not open a browser: {line!r}") + before = len(pids_under(data_dir)) + child.send_signal(signal.SIGKILL) + child.wait(timeout=10) + deadline = time.monotonic() + settle_s + left = pids_under(data_dir) + while left and time.monotonic() < deadline: + time.sleep(0.25) + left = pids_under(data_dir) + return before, len(left) + finally: + if child.poll() is None: + child.kill() + for pid in pids_under(data_dir): # never leave a browser behind + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--runs", type=int, default=3) + parser.add_argument("--settle", type=float, default=10.0, help="seconds to wait after the kill") + parser.add_argument("--control", action="store_true", help="also measure without the sweeper") + args = parser.parse_args() + if not os.path.isdir("/proc"): + print("this gate reads /proc (Linux)", file=sys.stderr) + return 2 + driver = os.environ.get("LYRA_BROWSER_DRIVER", "auto") + failed = False + for i in range(args.runs): + before, left = run_once(True, args.settle) + ok = before > 0 and left == 0 + failed |= not ok + print( + f"{'PASS' if ok else 'FAIL'} run {i + 1} (driver={driver}): {before} processes on the " + f"data dir before SIGKILL, {left} left {args.settle:g}s after it, with the sweeper", + flush=True, + ) + if args.control: + before, left = run_once(False, args.settle) + print( + f"NOTE control (driver={driver}): {before} processes before SIGKILL, {left} left " + f"{args.settle:g}s after it, without the sweeper", + flush=True, + ) + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/lyra_browser/uat/batch.py b/src/lyra_browser/uat/batch.py index c4be353..3bb9471 100644 --- a/src/lyra_browser/uat/batch.py +++ b/src/lyra_browser/uat/batch.py @@ -20,7 +20,9 @@ import asyncio import os +import shutil import sys +import time import uuid from collections import Counter from collections.abc import Callable @@ -30,7 +32,7 @@ from pydantic import Field -from .procs import reap_group, spawn, stop_group +from .procs import reap_group, spawn, stop_group, sweeper from .report import Model, Report from .spec import BatchSpec, RunSpec, load_persona @@ -60,6 +62,7 @@ class PersonaResult(Model): run_dir: str | None = None report_json: str | None = None error: str | None = None + attempts: int = 1 class MatrixRow(Model): @@ -107,6 +110,50 @@ def default_launcher(spec_path: Path) -> list[str]: return [sys.executable, "-m", "lyra_browser.uat", "run", str(spec_path)] +# Chrome's markers of a profile in use; a copy must not carry them, or the copy's +# browser would think another one holds it. +_PROFILE_LOCKS = ("SingletonLock", "SingletonSocket", "SingletonCookie", "profile.lock") + + +def make_disposable_profile(source: Path | None, profiles_dir: Path, name: str) -> Path: + """A profile for one attempt: a copy of ``source`` without its lock files, or empty.""" + target = profiles_dir / name + if source is not None and source.exists(): + shutil.copytree( + source, target, symlinks=True, ignore=shutil.ignore_patterns(*_PROFILE_LOCKS) + ) + else: + target.mkdir(parents=True) + return target + + +class StartLimiter: + """At most one child start per ``interval_s`` across the batch.""" + + def __init__(self, interval_s: float) -> None: + self.interval_s = interval_s + self._lock = asyncio.Lock() + self._last: float | None = None + + async def wait(self) -> None: + if self.interval_s <= 0: + return + async with self._lock: + if self._last is not None: + delay = self._last + self.interval_s - time.monotonic() + if delay > 0: + await asyncio.sleep(delay) + self._last = time.monotonic() + + +RETRIED_EXITS = frozenset({"browser_error", "brain_error", "harness_exit"}) + + +def should_retry(result: PersonaResult) -> bool: + """A crash or an infrastructure failure, not something the persona met on the site.""" + return result.status == "crashed" or result.exit_reason in RETRIED_EXITS + + def build_run_spec(batch: BatchSpec, persona_id: str, runs_dir: Path) -> RunSpec: """One persona's run spec: its own file, the batch's target, brain and limits.""" persona = load_persona(batch.personas[persona_id]) @@ -163,12 +210,18 @@ async def _run_child( return proc.returncode, stdout, stderr, timed_out -def _find_report(stdout: str, runs_dir: Path, persona_id: str) -> Path | None: +def _find_report( + stdout: str, runs_dir: Path, persona_id: str, *, since: float = 0.0 +) -> Path | None: + """The report the child printed, else the newest one of the persona written after + ``since`` (a retry must never read an earlier attempt's report as its own).""" for line in reversed(stdout.strip().splitlines()): candidate = Path(line.strip()) if candidate.name == "report.json" and candidate.is_file(): return candidate - matches = sorted(runs_dir.glob(f"*-{persona_id}-*/report.json")) + matches = sorted( + p for p in runs_dir.glob(f"*-{persona_id}-*/report.json") if p.stat().st_mtime >= since + ) return matches[-1] if matches else None @@ -201,23 +254,75 @@ async def run_persona( batch_dir: Path, launcher: Launcher, semaphore: asyncio.Semaphore, + limiter: StartLimiter | None = None, +) -> tuple[PersonaResult, Report | None]: + """Run one persona, again on a crash or an infrastructure failure (``batch.retries``).""" + attempt = 1 + earlier: list[str] = [] + while True: + result, report = await _attempt( + batch, + persona_id, + group, + attempt=attempt, + batch_dir=batch_dir, + launcher=launcher, + semaphore=semaphore, + limiter=limiter, + ) + result.attempts = attempt + result.warnings[:0] = earlier + if attempt > batch.retries or not should_retry(result): + return result, report + why = result.error or result.exit_reason or result.status + earlier = [*result.warnings, f"attempt {attempt} failed ({result.status}: {why})"] + await asyncio.sleep(batch.retry_delay_s) + attempt += 1 + + +async def _attempt( + batch: BatchSpec, + persona_id: str, + group: str, + *, + attempt: int, + batch_dir: Path, + launcher: Launcher, + semaphore: asyncio.Semaphore, + limiter: StartLimiter | None, ) -> tuple[PersonaResult, Report | None]: runs_dir = batch_dir / "runs" spec = build_run_spec(batch, persona_id, runs_dir) - spec_path = batch_dir / "specs" / f"{persona_id}.json" - spec_path.write_text(spec.model_dump_json(indent=2), encoding="utf-8") - argv = launcher(spec_path) - timeout_s = spec.limits.wall_s + 2 * spec.hooks.timeout_s + CHILD_GRACE_S - env = {**os.environ, "LYRA_UAT_BATCH_DIR": str(batch_dir)} + name = persona_id if attempt == 1 else f"{persona_id}-attempt{attempt}" + profile: Path | None = None async with semaphore: - code, stdout, stderr, timed_out = await _run_child( - argv, - cwd=batch_dir, - log_base=batch_dir / "logs" / persona_id, - timeout_s=timeout_s, - env=env, - ) - report_path = _find_report(stdout, runs_dir, persona_id) + if limiter is not None: + await limiter.wait() + if batch.disposable_profiles: + profile = make_disposable_profile( + batch.data_dirs.get(persona_id), batch_dir / "profiles", name + ) + spec.data_dir = profile + sweeper.watch_dir(profile) + spec_path = batch_dir / "specs" / f"{name}.json" + spec_path.write_text(spec.model_dump_json(indent=2), encoding="utf-8") + argv = launcher(spec_path) + timeout_s = spec.limits.wall_s + 2 * spec.hooks.timeout_s + CHILD_GRACE_S + env = {**os.environ, "LYRA_UAT_BATCH_DIR": str(batch_dir)} + started = time.time() + try: + code, stdout, stderr, timed_out = await _run_child( + argv, + cwd=batch_dir, + log_base=batch_dir / "logs" / name, + timeout_s=timeout_s, + env=env, + ) + finally: + if profile is not None: + sweeper.unwatch_dir(profile) + shutil.rmtree(profile, ignore_errors=True) + report_path = _find_report(stdout, runs_dir, persona_id, since=started - 1) if report_path is not None: try: report = Report.model_validate_json(report_path.read_text(encoding="utf-8")) @@ -342,13 +447,25 @@ async def run_batch( for sub in ("specs", "logs", "runs"): (batch_dir / sub).mkdir(parents=True, exist_ok=False if sub == "specs" else True) semaphore = asyncio.Semaphore(batch.concurrency) + limiter = StartLimiter(batch.start_interval_s) + # Killed outright, the batch leaves its children's browsers to the sweeper: every + # profile a child launches lives under the batch directory unless data_dirs says so. + sweeper.watch_dir(batch_dir) + for directory in batch.data_dirs.values(): + sweeper.watch_dir(directory) results: list[PersonaResult] = [] reports: dict[str, Report] = {} async def one(persona_id: str, group: str) -> None: result, report = await run_persona( - batch, persona_id, group, batch_dir=batch_dir, launcher=launcher, semaphore=semaphore + batch, + persona_id, + group, + batch_dir=batch_dir, + launcher=launcher, + semaphore=semaphore, + limiter=limiter, ) results.append(result) if report is not None: @@ -381,6 +498,9 @@ async def one(persona_id: str, group: str) -> None: summary.path, summary.md_path = str(json_path), str(md_path) _write_atomic(json_path, summary.model_dump_json(indent=2)) _write_atomic(md_path, render_markdown(summary)) + sweeper.unwatch_dir(batch_dir) + for directory in batch.data_dirs.values(): + sweeper.unwatch_dir(directory) return summary diff --git a/src/lyra_browser/uat/procs.py b/src/lyra_browser/uat/procs.py index b8a4d7c..95e1c9b 100644 --- a/src/lyra_browser/uat/procs.py +++ b/src/lyra_browser/uat/procs.py @@ -8,12 +8,14 @@ process group is exactly its descendants, and the group is what gets interrupted, then killed. -POSIX only (``start_new_session``, ``killpg``). +POSIX only (``start_new_session``, ``killpg``). A process killed outright (SIGKILL) cannot +stop anything itself: the sweeper at the end of this module does it for it. """ from __future__ import annotations import asyncio +import atexit import os import signal from typing import Any @@ -26,8 +28,13 @@ async def spawn(argv: list[str], **kwargs: Any) -> asyncio.subprocess.Process: - """Start ``argv`` as the leader of a new session (so its pgid is its pid).""" - return await asyncio.create_subprocess_exec(*argv, start_new_session=True, **kwargs) + """Start ``argv`` as the leader of a new session (so its pgid is its pid). + + The group is handed to the sweeper until ``stop_group`` / ``reap_group`` has seen it + gone, so it is stopped even when this process is killed outright.""" + proc = await asyncio.create_subprocess_exec(*argv, start_new_session=True, **kwargs) + sweeper.watch_group(proc.pid) + return proc _HAVE_PROC = os.path.isdir("/proc/self") @@ -116,7 +123,8 @@ async def stop_group(proc: asyncio.subprocess.Process, *, grace_s: float) -> Non _signal_group(pgid, signal.SIGKILL) if proc.returncode is None: await proc.wait() - await _wait_gone(pgid, KILL_AFTER_S) + if await _wait_gone(pgid, KILL_AFTER_S): + sweeper.unwatch_group(pgid) async def reap_group(pgid: int) -> None: @@ -125,10 +133,145 @@ async def reap_group(pgid: int) -> None: A harness that finished but did not close its server would otherwise leave a browser open on the run's profile. """ - if await _wait_gone(pgid, LINGER_S): - return - _signal_group(pgid, signal.SIGTERM) - if await _wait_gone(pgid, KILL_AFTER_S): - return - _signal_group(pgid, signal.SIGKILL) - await _wait_gone(pgid, KILL_AFTER_S) + if not await _wait_gone(pgid, LINGER_S): + _signal_group(pgid, signal.SIGTERM) + if not await _wait_gone(pgid, KILL_AFTER_S): + _signal_group(pgid, signal.SIGKILL) + if not await _wait_gone(pgid, KILL_AFTER_S): + return # still there: the sweeper keeps it + sweeper.unwatch_group(pgid) + + +# --- the sweeper: what is left when this process is killed outright --------------------- +# +# Every group above is stopped by *this* process. Killed with SIGKILL it can stop nothing, +# and a child started in a session of its own (the point of ``spawn``) survives it, as does +# a browser a driver launched in a group of its own. The sweeper is a small process of its +# own session that holds the read end of a pipe from this one. It learns the groups and +# browser profile directories to clean up as they come and go; when the pipe closes without +# a goodbye — this process died, however — it terminates, then kills, every group it still +# knows and every process whose ``--flag=`` argument points into a known directory +# (Chrome's ``--user-data-dir``, its crash handler's ``--database``). It never matches by +# program name: the operator's own Chrome is never a candidate. + +_SWEEPER_SRC = r""" +import os, signal, sys, time +signal.signal(signal.SIGINT, signal.SIG_IGN) +groups, dirs = set(), set() +for line in sys.stdin: + verb, _, arg = line.rstrip("\n").partition(" ") + if verb == "group": + groups.add(int(arg)) + elif verb == "ungroup": + groups.discard(int(arg)) + elif verb == "dir": + dirs.add(arg.rstrip("/")) + elif verb == "undir": + dirs.discard(arg.rstrip("/")) + elif verb == "bye": + sys.exit(0) + + +def inside(path): + return any(path == d or path.startswith(d + "/") for d in dirs) + + +def browsers(): + if not dirs or not os.path.isdir("/proc"): + return [] + found = [] + for name in os.listdir("/proc"): + if not name.isdigit() or int(name) == os.getpid(): + continue + try: + with open(f"/proc/{name}/cmdline", "rb") as handle: + argv = handle.read().split(b"\0") + except OSError: + continue + for arg in argv: + flag, eq, value = arg.partition(b"=") + if eq and flag.startswith(b"--") and inside(os.fsdecode(value)): + found.append(int(name)) + break + return found + + +for sig, pause in ((signal.SIGTERM, 1.0), (signal.SIGKILL, 0.5), (signal.SIGKILL, 0.0)): + for pgid in groups: + try: + os.killpg(pgid, sig) + except OSError: + pass + for pid in browsers(): + try: + os.kill(pid, sig) + except OSError: + pass + time.sleep(pause) +""" + + +class Sweeper: + """The parent side of the sweeper. Started on first use; every call is best effort + (a sweeper that cannot be started or reached must not fail the run it protects).""" + + def __init__(self) -> None: + self._proc: Any = None + + @property + def pid(self) -> int | None: + return self._proc.pid if self._proc is not None else None + + def _send(self, line: str) -> None: + import subprocess + import sys + + if self._proc is None: + try: + self._proc = subprocess.Popen( + [sys.executable, "-I", "-c", _SWEEPER_SRC], + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + except OSError: + return + try: + self._proc.stdin.write(f"{line}\n".encode()) + self._proc.stdin.flush() + except (OSError, ValueError): + pass + + def watch_group(self, pgid: int) -> None: + self._send(f"group {pgid}") + + def unwatch_group(self, pgid: int) -> None: + if self._proc is not None: + self._send(f"ungroup {pgid}") + + def watch_dir(self, path: os.PathLike[str] | str) -> None: + self._send(f"dir {os.path.abspath(path)}") + + def unwatch_dir(self, path: os.PathLike[str] | str) -> None: + if self._proc is not None: + self._send(f"undir {os.path.abspath(path)}") + + def close(self) -> None: + """A clean exit: the sweeper leaves without touching anything.""" + proc, self._proc = self._proc, None + if proc is None: + return + try: + proc.stdin.write(b"bye\n") + proc.stdin.close() + except (OSError, ValueError): + pass + try: + proc.wait(timeout=5) + except Exception: # noqa: BLE001 - exiting anyway + pass + + +sweeper = Sweeper() +atexit.register(sweeper.close) diff --git a/src/lyra_browser/uat/runner.py b/src/lyra_browser/uat/runner.py index 3970697..8d7d988 100644 --- a/src/lyra_browser/uat/runner.py +++ b/src/lyra_browser/uat/runner.py @@ -17,6 +17,7 @@ from datetime import UTC, datetime from pathlib import Path +from . import procs from .brains import BrainError, BrainSession, HarnessBrain, LoopBrain, ToolSpec, make_brain from .prompt import build_system_prompt, build_task_prompt from .report import BrainInfo, ExitReason, HookResult, Hooks, Report, build_report, write_report @@ -159,11 +160,27 @@ async def run_spec(spec: RunSpec, *, brain: LoopBrain | HarnessBrain | None = No only for a run directory that cannot be created or a spec that names no brain. Interrupted (Ctrl-C, a batch stopping its child) it still writes the report of what - happened so far, marked ``interrupted``, and then lets the cancellation through.""" + happened so far, marked ``interrupted``, and then lets the cancellation through. + Killed outright (SIGKILL), the sweeper stops the browser on the run's profile.""" started = datetime.now(UTC) run_id = new_run_id(spec.persona.id, started) run_dir = (spec.out_dir / run_id).resolve() run_dir.mkdir(parents=True, exist_ok=False) + profile = build_config(spec, run_dir).data_dir + procs.sweeper.watch_dir(profile) + try: + return await _run_spec(spec, brain, started, run_id, run_dir) + finally: + procs.sweeper.unwatch_dir(profile) + + +async def _run_spec( + spec: RunSpec, + brain: LoopBrain | HarnessBrain | None, + started: datetime, + run_id: str, + run_dir: Path, +) -> Report: run_file = write_run_file(spec, run_dir) brain = brain or make_brain(spec.brain) hooks = Hooks() diff --git a/src/lyra_browser/uat/spec.py b/src/lyra_browser/uat/spec.py index e318c1a..52af80a 100644 --- a/src/lyra_browser/uat/spec.py +++ b/src/lyra_browser/uat/spec.py @@ -340,6 +340,19 @@ class BatchSpec(Strict): guard_backend: Literal["route", "cdp"] = "route" driver: Literal["auto", "playwright", "patchright"] = "auto" report_language: str = "English" + # Each attempt gets a profile of its own that is deleted when it ends: a copy of its + # ``data_dirs`` entry (the original is never written to) or an empty one. What one + # persona's run left in a profile (cookies, storage, a half-finished login) cannot + # reach the next attempt or another persona. + disposable_profiles: bool = False + # A persona whose child crashed (no report) or ended on a browser, brain or harness + # error is run again, up to this many more times, ``retry_delay_s`` apart. What the + # persona met on the site (a budget, the wall clock, no finish) is a result, not retried. + retries: int = Field(0, ge=0, le=5) + retry_delay_s: float = Field(5.0, ge=0, le=600) + # At least this many seconds between two child starts anywhere in the batch, retries + # included: a parallel group does not open N sessions on the target in the same second. + start_interval_s: float = Field(0.0, ge=0, le=3600) @field_validator("out_dir", mode="before") @classmethod diff --git a/tests/uat/test_uat_batch.py b/tests/uat/test_uat_batch.py index 668d167..cb6d3e3 100644 --- a/tests/uat/test_uat_batch.py +++ b/tests/uat/test_uat_batch.py @@ -37,6 +37,18 @@ def mark(what): if mode == "crash": print("boom: no browser", file=sys.stderr) sys.exit(3) + if mode == "crash_once": + marker = log.with_name(f"crashed-{pid}") + if not marker.exists(): + marker.write_text("x") + print("boom: first attempt", file=sys.stderr) + sys.exit(3) + if mode == "profile": + # What the run found in its profile, then dirt for the next one to find. + found = sorted(q.name for q in spec.data_dir.iterdir()) if spec.data_dir else None + with log.with_name(f"profile-{pid}").open("a") as fh: + fh.write(json.dumps({"dir": str(spec.data_dir), "found": found}) + "\\n") + (spec.data_dir / "dirt").write_text("left by a run") time.sleep(float(spec.persona.goal)) run_dir = spec.out_dir / f"stub-{pid}-run" run_dir.mkdir(parents=True, exist_ok=True) @@ -231,3 +243,139 @@ async def test_only_restricts_the_run_and_unknown_ids_are_refused(tmp_path, stub assert [r.persona_id for r in summary.results] == ["b"] with pytest.raises(ValueError, match="not in the batch"): await run_batch(batch, only=["zz"], launcher=stub_launcher) + + +@pytest.mark.asyncio +async def test_a_crash_is_retried_and_the_attempts_are_counted(tmp_path, stub_launcher): + batch = make_batch( + tmp_path, + {"a": persona("a", mode="crash_once"), "b": persona("b", mode="crash")}, + [{"name": "g", "personas": ["a", "b"]}], + retries=2, + retry_delay_s=0, + ) + summary = await run_batch(batch, launcher=stub_launcher) + by_id = {r.persona_id: r for r in summary.results} + assert by_id["a"].status == "completed" and by_id["a"].attempts == 2 + assert any("attempt 1 failed" in w for w in by_id["a"].warnings) + # A child that always crashes is tried 1 + retries times, then reported as crashed. + assert by_id["b"].status == "crashed" and by_id["b"].attempts == 3 + batch_dir = Path(summary.dir) + assert (batch_dir / "specs" / "a-attempt2.json").is_file() + assert (batch_dir / "logs" / "b-attempt3.err.log").is_file() + + +@pytest.mark.asyncio +async def test_what_the_persona_met_is_not_retried(tmp_path, stub_launcher): + batch = make_batch( + tmp_path, + {"a": persona("a", mode="unfinished")}, + [{"name": "g", "personas": ["a"]}], + retries=3, + retry_delay_s=0, + ) + summary = await run_batch(batch, launcher=stub_launcher) + (result,) = summary.results + assert result.exit_reason == "no_finish" and result.attempts == 1 + + +def test_should_retry_only_infrastructure_failures(): + from lyra_browser.uat.batch import PersonaResult, should_retry + + def r(status, reason=None): + return PersonaResult(persona_id="a", group="g", status=status, exit_reason=reason) + + assert should_retry(r("crashed")) + assert should_retry(r("error", "browser_error")) + assert should_retry(r("incomplete", "brain_error")) + assert should_retry(r("error", "harness_exit")) + for reason in ("finish", "budget_exhausted", "wall_timeout", "no_finish", "interrupted"): + assert not should_retry(r("incomplete", reason)) + + +@pytest.mark.asyncio +async def test_start_interval_spaces_the_children_of_a_parallel_group(tmp_path, stub_launcher): + # Three children of a parallel group, one start per 0.8 s: the batch cannot finish + # before the third has been let through (a child's own clock is too noisy to compare: + # its interpreter takes a varying time to start). + batch = make_batch( + tmp_path, + {pid: persona(pid) for pid in ("a", "b", "c")}, + [{"name": "anon", "mode": "parallel", "personas": ["a", "b", "c"]}], + start_interval_s=0.8, + ) + summary = await run_batch(batch, launcher=stub_launcher) + assert summary.all_completed and summary.duration_s >= 1.6 + + +@pytest.mark.asyncio +async def test_start_limiter_lets_one_start_through_per_interval(): + import asyncio + import time + + from lyra_browser.uat.batch import StartLimiter + + limiter = StartLimiter(0.2) + times: list[float] = [] + + async def start() -> None: + await limiter.wait() + times.append(time.monotonic()) + + await asyncio.gather(*(start() for _ in range(4))) + gaps = [later - earlier for earlier, later in zip(times, times[1:], strict=False)] + assert len(gaps) == 3 and all(gap >= 0.19 for gap in gaps), gaps + + +@pytest.mark.asyncio +async def test_start_limiter_without_an_interval_does_not_wait(): + import time + + from lyra_browser.uat.batch import StartLimiter + + limiter = StartLimiter(0) + started = time.monotonic() + for _ in range(5): + await limiter.wait() + assert time.monotonic() - started < 0.05 + + +@pytest.mark.asyncio +async def test_disposable_profiles_copy_the_seed_and_leave_nothing_behind(tmp_path, stub_launcher): + seed = tmp_path / "seed" + (seed / "profile" / "Default").mkdir(parents=True) + (seed / "profile" / "Default" / "Cookies").write_text("login") + (seed / "profile" / "SingletonLock").write_text("held") + (seed / "profile.lock").write_text("") + batch = make_batch( + tmp_path, + {"a": persona("a", mode="profile"), "b": persona("b", mode="profile")}, + [{"name": "g", "personas": ["a", "b", "a"]}], + data_dirs={"a": str(seed)}, + disposable_profiles=True, + ) + summary = await run_batch(batch, launcher=stub_launcher) + assert summary.all_completed + seen_a = [json.loads(x) for x in (tmp_path / "profile-a").read_text().splitlines()] + seen_b = [json.loads(x) for x in (tmp_path / "profile-b").read_text().splitlines()] + # Both runs of a got a fresh copy of the seed: no lock files, and no dirt from the first. + assert [s["found"] for s in seen_a] == [["profile"], ["profile"]] + assert all(Path(s["dir"]) != seed for s in seen_a) + # b has no seed: an empty profile of its own. + assert seen_b[0]["found"] == [] + # The seed was never written to, and every copy is gone. + assert sorted(q.name for q in (seed / "profile").iterdir()) == ["Default", "SingletonLock"] + assert not (seed / "dirt").exists() + assert not any((Path(summary.dir) / "profiles").iterdir()) + + +def test_make_disposable_profile_skips_lock_files(tmp_path): + from lyra_browser.uat.batch import make_disposable_profile + + seed = tmp_path / "seed" + (seed / "profile").mkdir(parents=True) + for name in ("SingletonLock", "SingletonSocket", "SingletonCookie", "Local State"): + (seed / "profile" / name).write_text("x") + (seed / "profile.lock").write_text("") + copy = make_disposable_profile(seed, tmp_path / "profiles", "a") + assert sorted(q.name for q in copy.rglob("*")) == ["Local State", "profile"] diff --git a/tests/uat/test_uat_procs.py b/tests/uat/test_uat_procs.py index 6ceb9c2..f08cb6c 100644 --- a/tests/uat/test_uat_procs.py +++ b/tests/uat/test_uat_procs.py @@ -198,3 +198,116 @@ def test_the_cli_cancels_on_sigint_and_sigterm_even_if_sigint_was_inherited_as_i if proc.poll() is None: proc.kill() assert proc.returncode == 130 and "cancelled" in out + + +# The parent a sweeper protects: it starts a stubborn harness group through ``spawn`` and a +# "browser" in a session of its own (as Playwright's driver launches Chrome) whose +# ``--user-data-dir`` points into a watched directory, reports every pid, and waits to be +# killed with SIGKILL, which it cannot handle. +KILLED_PARENT = textwrap.dedent( + """ + import asyncio, os, subprocess, sys + from pathlib import Path + from lyra_browser.uat import procs + + stub, pids_out, profile = sys.argv[1], Path(sys.argv[2]), Path(sys.argv[3]) + group_pids = Path(str(pids_out) + ".group") + + async def main(): + procs.sweeper.watch_dir(profile) + await procs.spawn([sys.executable, stub, str(group_pids)], + stdout=asyncio.subprocess.DEVNULL) + browser = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(300)", + f"--user-data-dir={profile / 'profile'}"], + start_new_session=True) + while not group_pids.exists() or group_pids.read_text().count(" ") != 1: + await asyncio.sleep(0.05) + pids_out.write_text(f"{group_pids.read_text()} {browser.pid} {procs.sweeper.pid}") + await asyncio.sleep(300) + + asyncio.run(main()) + """ +) + + +@needs_proc +@pytest.mark.asyncio +async def test_a_sigkilled_parent_leaves_no_orphans(stub, tmp_path): + """SIGKILL cannot be handled: the sweeper stops the parent's groups and its browsers.""" + import signal + import subprocess + + argv, _ = stub(STUBBORN) + parent_script = tmp_path / "parent.py" + parent_script.write_text(KILLED_PARENT) + pids_file, profile = tmp_path / "all-pids", tmp_path / "run-profile" + env = {**os.environ, "PYTHONPATH": os.pathsep.join(sys.path)} + parent = subprocess.Popen( + [sys.executable, str(parent_script), argv[1], str(pids_file), str(profile)], env=env + ) + try: + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if pids_file.exists() and len(pids_file.read_text().split()) == 4: + break + await asyncio.sleep(0.05) + harness_pid, server_pid, browser_pid, sweeper_pid = map(int, pids_file.read_text().split()) + assert all(running(p) for p in (harness_pid, server_pid, browser_pid, sweeper_pid)) + parent.send_signal(signal.SIGKILL) + parent.wait(timeout=5) + survivors = await wait_dead([harness_pid, server_pid, browser_pid, sweeper_pid], 10.0) + assert survivors == [], f"left running: {survivors}" + finally: + if parent.poll() is None: + parent.kill() + + +@needs_proc +def test_a_clean_exit_tells_the_sweeper_to_leave_everything_alone(tmp_path): + import subprocess + + victim = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(30)", f"--user-data-dir={tmp_path}/p"], + start_new_session=True, + ) + try: + sweeper = procs.Sweeper() + sweeper.watch_dir(tmp_path) + sweeper.watch_group(victim.pid) + proc = sweeper._proc + sweeper.close() + assert proc.returncode == 0 + time.sleep(0.3) + assert running(victim.pid) + finally: + victim.kill() + victim.wait() + + +@needs_proc +def test_the_sweeper_matches_only_flags_that_point_into_a_watched_dir(tmp_path): + """Not by name and not by prefix of a sibling: ``-other`` is someone else's.""" + import subprocess + + watched = tmp_path / "run" + sleep = [sys.executable, "-c", "import time; time.sleep(30)"] + flag = f"--user-data-dir={watched}/profile" + inside = subprocess.Popen([*sleep, flag], start_new_session=True) + sibling = subprocess.Popen([*sleep, f"--user-data-dir={watched}-other"], start_new_session=True) + positional = subprocess.Popen([*sleep, str(watched / "profile")], start_new_session=True) + try: + sweeper = procs.Sweeper() + sweeper.watch_dir(watched) + proc = sweeper._proc + proc.stdin.close() # what a dead parent looks like from the sweeper's side + proc.wait(timeout=10) + deadline = time.monotonic() + 5 + while running(inside.pid) and time.monotonic() < deadline: + time.sleep(0.05) + assert not running(inside.pid) + assert running(sibling.pid) and running(positional.pid) + finally: + for p in (inside, sibling, positional): + p.kill() + p.wait()