From 1a6a8ceb01eac8ca073d9a775ec42fc47d482c88 Mon Sep 17 00:00:00 2001 From: Matt Miller Date: Fri, 9 Oct 2026 06:32:56 +0000 Subject: [PATCH 1/3] fix(cursor-review): retry a panel cell on Cursor's transient resource_exhausted error A cell whose cursor-agent exits non-zero with Cursor's retriable marker (`RetriableError: [resource_exhausted]`) is re-run by stop-on-submit.py after 30s, then 90s, before falling through to the pre-seeded status=error path. Only a cell with no submission and no recorded findings is re-run (the MCP tool appends findings), no retry starts past 300s so the last attempt still fits the 15-minute step cap, and each retry emits one ::warning:: naming the cell, attempt and marker. Any other exit, a signal death or a forwarded signal behaves exactly as before. --- .github/cursor-review/README.md | 2 +- .github/cursor-review/stop-on-submit.py | 197 ++++++++++++++++-- .../tests/test_stop_on_submit.py | 181 +++++++++++++++- .github/workflows/cursor-review.yml | 11 + 4 files changed, 369 insertions(+), 22 deletions(-) diff --git a/.github/cursor-review/README.md b/.github/cursor-review/README.md index 2528bbe8..dec1fa4a 100644 --- a/.github/cursor-review/README.md +++ b/.github/cursor-review/README.md @@ -221,7 +221,7 @@ the consolidated review's panel table. See [Panel integrity](#panel-integrity). | [`prompt-edge-case.md`](prompt-edge-case.md) | Prompt for the correctness/logic review pass. | | [`prompt-judge.md`](prompt-judge.md) | Prompt the judge model uses to consolidate panel findings into one review. | | [`review-output-mcp.py`](review-output-mcp.py) | Dependency-free stdio MCP server. Reviewers record individual findings and finish; the judge submits the final findings array. It validates and atomically writes the normalized records consumed by later jobs. | -| [`stop-on-submit.py`](stop-on-submit.py) | Wraps each panel cell's `cursor-agent` and polls its `findings.json`. Once the file parses as a JSON object with `"status": "ok"`, the agent gets 15s to exit by itself, then SIGTERM to its process group, then SIGKILL 10s later, so a cell that has submitted exits **within 30 seconds** of submitting instead of idling to the 15-minute step cap. It only reads the file, so the artifact is byte-for-byte what the MCP tool wrote, and the stop exits 0, so the leg stays green. A missing, partial, unparseable or non-`ok` file never triggers a stop, so a cell that never submits still runs to the cap and fails its leg as before. The step cap's and a run cancel's signals reach it through the step shell's traps and are forwarded to the agent's group, which is SIGKILLed at most 5s later (exit 128+signal). Every exit path, including an agent that exits by itself, ends with SIGKILL to the group, so nothing it left running can rewrite the findings after the upload. Only a regular file of at most 8 MiB is read (no FIFO/symlink/device). | +| [`stop-on-submit.py`](stop-on-submit.py) | Wraps each panel cell's `cursor-agent` and polls its `findings.json`. Once the file parses as a JSON object with `"status": "ok"`, the agent gets 15s to exit by itself, then SIGTERM to its process group, then SIGKILL 10s later, so a cell that has submitted exits **within 30 seconds** of submitting instead of idling to the 15-minute step cap. It only reads the file, so the artifact is byte-for-byte what the MCP tool wrote, and the stop exits 0, so the leg stays green. A missing, partial, unparseable or non-`ok` file never triggers a stop, so a cell that never submits still runs to the cap and fails its leg as before. The step cap's and a run cancel's signals reach it through the step shell's traps and are forwarded to the agent's group, which is SIGKILLed at most 5s later (exit 128+signal). Every exit path, including an agent that exits by itself, ends with SIGKILL to the group, so nothing it left running can rewrite the findings after the upload. Only a regular file of at most 8 MiB is read (no FIFO/symlink/device). It also retries Cursor's transient capacity error: an agent that exits non-zero with `[resource_exhausted]` / `RetriableError` in the tail of its stdout or stderr is re-run after `--retry-delays` (30s, then 90s in the workflow), each retry logged as one `::warning::` naming the cell. A retry needs the findings file to still be an unsubmitted record with **no** recorded findings (the MCP tool appends, so a re-run would duplicate them) and must start within `--retry-window` (300s), so the last attempt still fits the step cap. Exit 0, any other error, a signal death and a forwarded signal are never retried. | | [`post-review.py`](post-review.py) | Reads the judge's consolidated findings and posts **one** PR review with line-anchored inline comments and severity badges. | | [`auto-approve.py`](auto-approve.py) | Opt-in auto-approve (`approve_max_severity`): `decide` (in `post-review`) approves / requests changes / reaches no decision (withdrawing its own earlier approvals and leaving one standing REQUEST_CHANGES with the reasons and the next step, replaced by the next no-decision round and dismissed by the next approving one; none on a `capped` round, which withdraws the earlier one) per its fail-closed rules, then writes cursor-approve's status card for the outcome (via [`../cursor-approve/card.py`](../cursor-approve/card.py); no card for an author `approve_authors` does not list); `dismiss-stale` (the **Dismiss stale auto-approval** job) withdraws the bot's own approvals that are stale against the PR's live head and base (a push, a reopen, a retarget) — and every one of them while the PR carries `skip-cursor-review` or `needs-human-review` — plus its own standing REQUEST_CHANGES while the PR carries either label (re-read after the reviews are listed) or `approve_max_severity` is empty (then only once none of its approvals still stands), on every event of an open PR, whether or not `approve_max_severity` is still set; `round-cap` (the **Round cap** job, ahead of the panel) enforces `max_rounds`. On an APPROVE that survived its post-write re-check, `decide` also resolves (re-reading each first) and then replies on the posting identity's own at-or-below-threshold threads nobody else commented in (nothing, if any live thread is above the threshold); [`build-ledger.py`](build-ledger.py) never counts that reply as an answer. `decide` also writes the `approve_gate` output, and `round-cap` feeds the `round` and `max_rounds` outputs. Reads open-thread severity through `gate-unresolved.py`'s query. See [`docs/callers/cursor-review.md`](../../docs/callers/cursor-review.md#auto-approve). | | [`gate-unresolved.py`](gate-unresolved.py) | The opt-in blocking gate (`blocking: true` → the **Blocking gate** job): queries the PR's review threads and exits non-zero while any cursor-review finding thread is unresolved and non-outdated. Dropped from `cursor-review.yml` by accident in #31 and restored by BE-4691 — see [the blocking section](#optional-make-the-review-blocking). Double-billed: [`build-ledger.py`](build-ledger.py) also imports it for `CONSOLIDATED_MARKER`, the paging `reviewThreads` GraphQL query and the `iter_threads` / `is_cursor_thread` helpers, so the gate and the ledger can never disagree about which threads are ours. | diff --git a/.github/cursor-review/stop-on-submit.py b/.github/cursor-review/stop-on-submit.py index 3c0767d5..3f72aa8e 100644 --- a/.github/cursor-review/stop-on-submit.py +++ b/.github/cursor-review/stop-on-submit.py @@ -22,28 +22,47 @@ exiting by itself and an exception here, ends with SIGKILL to the group, so nothing the agent left running can rewrite the findings afterwards. +Retry on a transient Cursor capacity error (`--retry-delays`): when the agent +exits non-zero by itself and the tail of its stdout or stderr carries Cursor's +retriable marker (`[resource_exhausted]` or `RetriableError`), the agent is run +again after the next delay, up to one retry per delay. A retry happens only when +the findings file still reads as an untouched, unsubmitted record — a parseable +object, status not "ok", and NO recorded findings — because the MCP tool appends +to that file, so re-running a cell that already recorded findings would +duplicate them. It is also skipped once `--retry-window` seconds (measured from +the wrapper's start, including the delay) would be exceeded, so a retry starts +only while a full review still fits inside the step cap. Anything else — exit 0, +a different error, death by signal, a forwarded signal (the step cap) — is +reported exactly as without retries. Each retry prints one `::warning::` line +naming the `--cell`, the attempt and the matched marker. The agent's output +still reaches this wrapper's stdout/stderr unchanged; it is only teed through +here so its tail can be matched. + A SIGINT/SIGTERM/SIGHUP to this wrapper (the step cap, a run cancel) is passed to the agent's group; the agent then gets at most 5 seconds before SIGKILL — inside the runner's own SIGINT -> SIGTERM -> kill escalation — and the wrapper exits 128+signal whatever the agent's own status was. -Exit status: the agent's own exit code when it exits by itself (before or -after submitting); 0 when this wrapper stopped it after a verified submission; -128+signal when a forwarded signal ended it. +Exit status: the (last attempt's) agent's own exit code when it exits by +itself (before or after submitting); 0 when this wrapper stopped it after a +verified submission; 128+signal when a forwarded signal ended it. Status lines go to stderr with a `stop-on-submit:` prefix. Usage: - stop-on-submit.py --findings /tmp/findings-out/findings.json -- cursor-agent ... + stop-on-submit.py --findings /tmp/findings-out/findings.json \ + [--retry-delays 30,90 --cell adversarial/model] -- cursor-agent ... """ import argparse import json import math import os +import re import signal import stat import subprocess import sys +import threading import time FORWARDED_SIGNALS = (signal.SIGINT, signal.SIGTERM, signal.SIGHUP) @@ -56,14 +75,28 @@ SIGNAL_GRACE = 5.0 # How often a wait re-checks for a forwarded signal. WAIT_SLICE = 0.1 +# Cursor's own label for a transient capacity error, e.g. +# `RetriableError: [resource_exhausted] Error`. Matched on raw bytes; the +# warning names only the fixed marker, never agent-printed text. +# The specific code is preferred over the generic class when both appear. +RETRIABLE = ( + ("resource_exhausted", re.compile(rb"\[resource_exhausted\]")), + ("RetriableError", re.compile(rb"\bRetriableError\b")), +) +# How much of each stream's end is kept for the match. The marker is the last +# thing a failed run prints. +TAIL_BYTES = 64 * 1024 +# How long to wait for a stream to drain after the agent's group is killed. +# Bounded so a stray process that escaped the group cannot hang the wrapper. +DRAIN_TIMEOUT = 5.0 def log(message): print(f"stop-on-submit: {message}", file=sys.stderr, flush=True) -def submitted(path): - """True only for a complete, parseable JSON object with status == "ok". +def read_findings(path): + """The findings record as a dict, or None for anything that is not one. The file is agent-writable (the agent runs `--trust` with a shell), so every failure mode reads as "not submitted" rather than raising or @@ -76,18 +109,81 @@ def submitted(path): try: fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) except OSError: - return False + return None try: with open(fd, "rb") as source: if not stat.S_ISREG(os.fstat(source.fileno()).st_mode): - return False + return None data = source.read(MAX_FINDINGS_BYTES + 1) if len(data) > MAX_FINDINGS_BYTES: - return False + return None record = json.loads(data.decode("utf-8")) except (OSError, ValueError, RecursionError, MemoryError): - return False - return isinstance(record, dict) and record.get("status") == "ok" + return None + return record if isinstance(record, dict) else None + + +def submitted(path): + """True only for a complete, parseable JSON object with status == "ok".""" + record = read_findings(path) + return record is not None and record.get("status") == "ok" + + +def retry_safe(path): + """True only for an unsubmitted record with no findings recorded yet. + + review-output-mcp.py appends each recorded finding to the file, so a re-run + on top of a partial review would duplicate findings; and an unreadable file + is not known to be untouched. Both are left to fail as they always did. + """ + record = read_findings(path) + return (record is not None and record.get("status") != "ok" + and record.get("findings") == []) + + +class Tee: + """Copy a child's pipe to `sink_fd` unchanged, keeping its last TAIL_BYTES.""" + + def __init__(self, source, sink_fd): + self.source = source + self.sink_fd = sink_fd + self.tail = b"" + self.thread = threading.Thread(target=self.pump, daemon=True) + self.thread.start() + + def pump(self): + fd = self.source.fileno() + while True: + try: + chunk = os.read(fd, 65536) + except OSError: + return + if not chunk: + return + self.tail = (self.tail + chunk)[-TAIL_BYTES:] + view = memoryview(chunk) + while view: + try: + view = view[os.write(self.sink_fd, view):] + except OSError: + break + + def drain(self): + self.thread.join(DRAIN_TIMEOUT) + return self.tail + + +def retriable_marker(*tails): + """The fixed marker name Cursor's retriable error carries, or None.""" + for name, pattern in RETRIABLE: + if any(pattern.search(tail) for tail in tails): + return name + return None + + +def safe_label(text): + """A ::warning:: -safe rendering of a caller-supplied cell label.""" + return re.sub(r"[^A-Za-z0-9._/@+-]", "_", text)[:200] def signal_group(pgid, signum): @@ -109,33 +205,77 @@ def wait(child, timeout, interrupted=lambda: False): return None -def run(command, findings, poll, linger, grace): +def run(command, findings, poll, linger, grace, + retry_delays=(), retry_window=0.0, cell="cell"): received = [] - child = None + current = [None] + started = time.monotonic() # The step cap and a run cancel signal THIS process; pass them on, so a cell # that never submits still dies at the cap exactly as it did unwrapped. # Installed BEFORE the spawn, so no signal can kill the wrapper in between - # and orphan the agent; one landing before `child` is set is forwarded by - # stop_on_signal instead. + # and orphan the agent; one landing before a child is set is forwarded by + # stop_on_signal instead, and one landing during a retry backoff ends the + # wrapper before the next attempt starts. def forward(signum, _frame): received.append(signum) - if child is not None: - signal_group(child.pid, signum) + if current[0] is not None: + signal_group(current[0].pid, signum) for signum in FORWARDED_SIGNALS: signal.signal(signum, forward) + attempts = len(retry_delays) + 1 + for attempt in range(1, attempts + 1): + code, marker = attempt_once( + command, findings, poll, linger, grace, received, current) + if attempt == attempts or received or code <= 0 or marker is None: + return code + if not retry_safe(findings): + log(f"agent exited {code} with [{marker}], but findings were already " + "recorded or the file is unreadable; not retrying") + return code + delay = retry_delays[attempt - 1] + if time.monotonic() - started + delay > retry_window: + log(f"agent exited {code} with [{marker}], but a retry after {delay:g}s " + f"would start past the {retry_window:g}s retry window; not retrying") + return code + print(f"::warning::Reviewer cell {safe_label(cell)}: attempt {attempt}/" + f"{attempts} failed with Cursor's retriable error [{marker}] " + f"(exit {code}); retrying in {delay:g}s", + file=sys.stderr, flush=True) + deadline = time.monotonic() + delay + while not received and time.monotonic() < deadline: + time.sleep(min(WAIT_SLICE, max(0.0, deadline - time.monotonic()))) + if received: + log(f"received signal {received[0]} during the retry backoff; not retrying") + return -received[0] + raise AssertionError("unreachable") + + +def attempt_once(command, findings, poll, linger, grace, received, current): + """Run the agent once: (its status, the retriable marker it printed or None).""" # Own process group, so a stop reaches cursor-agent's children too (the # MCP server, any shell it spawned) — not just the top-level process. - child = subprocess.Popen(command, start_new_session=True) + # Both streams are piped through here only so a retriable error can be + # recognised; every byte is copied on to this wrapper's own stdout/stderr. + child = subprocess.Popen(command, start_new_session=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + current[0] = child + tees = (Tee(child.stdout, sys.stdout.fileno()), + Tee(child.stderr, sys.stderr.fileno())) try: - return supervise(child, findings, poll, linger, grace, received) + code = supervise(child, findings, poll, linger, grace, received) finally: # Every path out — the agent exiting by itself, a stop, an exception # here: nothing left in the agent's group may outlive the wrapper and # rewrite the findings after the upload and leg check read them. signal_group(child.pid, signal.SIGKILL) + current[0] = None + tails = [tee.drain() for tee in tees] + for stream in (child.stdout, child.stderr): + stream.close() + return code, retriable_marker(*tails) def stop_on_signal(child, signum, grace): @@ -197,6 +337,14 @@ def main(argv=None): parser.add_argument("--poll", type=float, default=5.0) parser.add_argument("--linger", type=float, default=15.0) parser.add_argument("--grace", type=float, default=10.0) + parser.add_argument("--retry-delays", default="", + help="comma-separated backoff seconds, one retry each " + "(default: none, never retry)") + parser.add_argument("--retry-window", type=float, default=300.0, + help="no retry may start later than this many seconds " + "after the wrapper started") + parser.add_argument("--cell", default="cell", + help="/, named in the retry warning") parser.add_argument("command", nargs=argparse.REMAINDER) args = parser.parse_args(argv) command = args.command[1:] if args.command[:1] == ["--"] else args.command @@ -206,7 +354,16 @@ def main(argv=None): value = getattr(args, name) if not (math.isfinite(value) and value > 0): parser.error(f"--{name} must be a positive finite number") - code = run(command, args.findings, args.poll, args.linger, args.grace) + try: + delays = tuple(float(d) for d in args.retry_delays.split(",") if d.strip()) + except ValueError: + parser.error("--retry-delays must be comma-separated numbers") + if not all(math.isfinite(d) and d >= 0 for d in delays): + parser.error("--retry-delays must be non-negative finite numbers") + if not (math.isfinite(args.retry_window) and args.retry_window >= 0): + parser.error("--retry-window must be a non-negative finite number") + code = run(command, args.findings, args.poll, args.linger, args.grace, + delays, args.retry_window, args.cell) # A signal death reads as the shell would report it (143 for SIGTERM), not # as Python's negative returncode wrapped mod 256. return 128 - code if code < 0 else code diff --git a/.github/cursor-review/tests/test_stop_on_submit.py b/.github/cursor-review/tests/test_stop_on_submit.py index bd2a01f2..68aece35 100644 --- a/.github/cursor-review/tests/test_stop_on_submit.py +++ b/.github/cursor-review/tests/test_stop_on_submit.py @@ -97,7 +97,9 @@ def test_an_oversized_file_is_not_submitted(self): self.assertFalse(stop_on_submit.submitted(self.path)) -class WrapperTest(unittest.TestCase): +class AgentHarness(unittest.TestCase): + """Runs the real wrapper over a stub agent; holds no tests of its own.""" + def setUp(self): self.dir = tempfile.TemporaryDirectory() self.path = os.path.join(self.dir.name, "findings.json") @@ -170,6 +172,7 @@ def assert_gone(self, pid): time.sleep(0.05) self.fail(f"pid {pid} is still running") +class WrapperTest(AgentHarness): def test_ok_cell_is_stopped_early_and_findings_are_untouched(self): proc = self.wrap(f"""\ write({json.dumps(json.dumps(OK))}) @@ -339,6 +342,178 @@ def test_missing_command_is_a_usage_error(self): self.assertEqual(result.returncode, 2) +SEEDED = {"status": "error", "error": "x", "findings": [], "model": "m", + "review_type": "adversarial"} +EXHAUSTED = "RetriableError: [resource_exhausted] Error" +RETRY = ["--retry-delays", "0.2,0.4", "--retry-window", "30", + "--cell", "adversarial/kimi-k3-high"] + + +class RetryTest(AgentHarness): + """A non-zero exit carrying Cursor's retriable marker re-runs the agent.""" + + def setUp(self): + super().setUp() + with open(self.path, "w") as f: + json.dump(SEEDED, f) + + def stub(self, attempt_body): + """An agent that counts its runs; `attempt_body` sees ATTEMPT (1-based).""" + return (f"ATTEMPT = len(open({self.pidfile!r}).read().split())\n" + + textwrap.dedent(attempt_body)) + + def retry(self, attempt_body, extra=None): + proc = self.wrap(self.stub(attempt_body), extra=TIMINGS + (extra or RETRY)) + return self.finish(proc, cap=20) + + def test_resource_exhausted_then_success_is_retried(self): + code, err, _ = self.retry(f"""\ + if ATTEMPT == 1: + print({EXHAUSTED!r}, file=sys.stderr) + sys.exit(1) + write({json.dumps(json.dumps(OK))}) + sys.exit(0) + """) + self.assertEqual(code, 0, err) + self.assertEqual(len(self.pids()), 2) + warnings = [l for l in err.splitlines() if l.startswith("::warning::")] + self.assertEqual(warnings, [ + "::warning::Reviewer cell adversarial/kimi-k3-high: attempt 1/3 failed " + "with Cursor's retriable error [resource_exhausted] (exit 1); retrying in 0.2s"]) + # The agent's own stderr still reaches the wrapper's, unchanged. + self.assertIn(EXHAUSTED + "\n", err) + with open(self.path) as f: + self.assertEqual(json.load(f)["status"], "ok") + + def test_marker_on_stdout_is_retried_and_stdout_passes_through(self): + proc = subprocess.Popen( + [sys.executable, SCRIPT, "--findings", self.path, *TIMINGS, *RETRY, "--", + *self.agent(self.stub(f"""\ + print("out-" + str(ATTEMPT)) + if ATTEMPT == 1: + print("RetriableError: boom") + sys.exit(1) + write({json.dumps(json.dumps(OK))}) + """))], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + out, err = proc.communicate(timeout=20) + self.assertEqual(proc.returncode, 0, err) + self.assertEqual(out, "out-1\nRetriableError: boom\nout-2\n") + self.assertIn("[RetriableError]", err) + + def test_persistent_resource_exhausted_exhausts_retries(self): + code, err, took = self.retry(f"""\ + print({EXHAUSTED!r}, file=sys.stderr) + sys.exit(1) + """) + self.assertEqual(code, 1, err) + self.assertEqual(len(self.pids()), 3) + warnings = [l for l in err.splitlines() if l.startswith("::warning::")] + self.assertEqual(len(warnings), 2, err) + self.assertIn("attempt 2/3", warnings[1]) + self.assertIn("retrying in 0.4s", warnings[1]) + self.assertGreaterEqual(took, 0.6) + with open(self.path) as f: + self.assertEqual(json.load(f), SEEDED) + + def test_other_errors_are_not_retried(self): + code, err, _ = self.retry("""\ + print("Error: invalid model", file=sys.stderr) + sys.exit(1) + """) + self.assertEqual(code, 1, err) + self.assertEqual(len(self.pids()), 1) + self.assertNotIn("::warning::", err) + + def test_marker_with_exit_zero_or_signal_death_is_not_retried(self): + for ending, expected in (("sys.exit(0)", 0), + ("os.kill(os.getpid(), signal.SIGKILL)", 128 + 9)): + with self.subTest(ending=ending): + open(self.pidfile, "w").close() + code, err, _ = self.retry(f"""\ + print({EXHAUSTED!r}, file=sys.stderr, flush=True) + {ending} + """) + self.assertEqual(code, expected, err) + self.assertEqual(len(self.pids()), 1) + self.assertNotIn("::warning::", err) + + def test_no_retry_without_retry_delays(self): + code, err, _ = self.retry(f"""\ + print({EXHAUSTED!r}, file=sys.stderr) + sys.exit(1) + """, extra=["--cell", "a/b"]) + self.assertEqual(code, 1, err) + self.assertEqual(len(self.pids()), 1) + + def test_submitted_or_partially_recorded_cells_are_never_rerun(self): + records = ( + OK, + {"status": "error", "findings": [{"file": "a.py", "line": 1}]}, + "not json", + ) + for record in records: + with self.subTest(record=record): + open(self.pidfile, "w").close() + text = record if isinstance(record, str) else json.dumps(record) + code, err, _ = self.retry(f"""\ + write({text!r}) + print({EXHAUSTED!r}, file=sys.stderr) + sys.exit(1) + """) + self.assertEqual(code, 1, err) + self.assertEqual(len(self.pids()), 1) + self.assertNotIn("::warning::", err) + + def test_retry_that_would_start_past_the_window_is_skipped(self): + code, err, _ = self.retry(f"""\ + print({EXHAUSTED!r}, file=sys.stderr) + sys.exit(1) + """, extra=["--retry-delays", "5", "--retry-window", "1", "--cell", "a/b"]) + self.assertEqual(code, 1, err) + self.assertEqual(len(self.pids()), 1) + self.assertIn("retry window", err) + + def test_signal_during_backoff_ends_the_wrapper_without_retrying(self): + # The step cap landing mid-backoff: no further attempt may start. + proc = self.wrap(self.stub(f"""\ + print({EXHAUSTED!r}, file=sys.stderr) + sys.exit(1) + """), extra=TIMINGS + ["--retry-delays", "30", "--retry-window", "60"]) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline and not ( + os.path.exists(self.pidfile) and self.pids()): + time.sleep(0.05) + time.sleep(1) # the stub exits at once; the wrapper is now backing off + proc.send_signal(signal.SIGTERM) + code, err, _ = self.finish(proc, cap=5) + self.assertEqual(code, 128 + signal.SIGTERM, err) + self.assertIn("::warning::", err) + self.assertIn("during the retry backoff", err) + self.assertEqual(len(self.pids()), 1) + + def test_cell_label_is_sanitized_in_the_warning(self): + code, err, _ = self.retry(f"""\ + if ATTEMPT == 1: + print({EXHAUSTED!r}, file=sys.stderr) + sys.exit(1) + write({json.dumps(json.dumps(OK))}) + """, extra=["--retry-delays", "0", "--retry-window", "30", + "--cell", "a/b\n::error::x"]) + self.assertEqual(code, 0, err) + self.assertIn("::warning::Reviewer cell a/b___error__x:", err) + self.assertNotIn("\n::error::", err) + + def test_bad_retry_delays_are_a_usage_error(self): + for value in ("x", "-1", "nan"): + with self.subTest(value=value): + result = subprocess.run( + [sys.executable, SCRIPT, "--findings", self.path, + "--retry-delays", value, "--", "true"], + capture_output=True, text=True) + self.assertEqual(result.returncode, 2, result.stderr) + + class WorkflowWiringTest(unittest.TestCase): def test_run_step_wraps_the_reviewer_agent(self): workflow = os.path.normpath(os.path.join( @@ -356,6 +531,10 @@ def test_run_step_wraps_the_reviewer_agent(self): self.assertIn(f"trap 'forward_signal {sig}' {sig}", step) self.assertLess(step.index("trap 'forward_signal TERM'"), step.index("stop-on-submit.py\"")) self.assertIn("timeout-minutes: 15", step) + # Transient Cursor capacity errors are retried, named per cell. + self.assertIn("--retry-delays 30,90", step) + self.assertIn('--cell "$REVIEW_TYPE/$MODEL"', step) + self.assertIn("REVIEW_TYPE: ${{ matrix.review_type }}", step) if __name__ == "__main__": diff --git a/.github/workflows/cursor-review.yml b/.github/workflows/cursor-review.yml index 4437bd25..46ef682c 100644 --- a/.github/workflows/cursor-review.yml +++ b/.github/workflows/cursor-review.yml @@ -2593,11 +2593,20 @@ jobs: # artifact is byte-for-byte what the MCP tool wrote; it exits 0 on that # path, and a cell that never reaches `status: ok` is not stopped, so # it still runs to this cap and fails its leg exactly as before. + # + # The wrapper also retries Cursor's own transient capacity error: an + # agent that exits non-zero with `RetriableError: [resource_exhausted]` + # (~25s in, before recording anything) is re-run after 30s, then 90s. + # Only a cell with no recorded findings is re-run, and no retry starts + # later than 300s in, so the last attempt still has ~10 min of this cap + # for a full review. Each retry is a `::warning::` naming the cell; a + # cell that exhausts them fails its leg exactly as before. continue-on-error: true timeout-minutes: 15 env: CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }} MODEL: ${{ matrix.model }} + REVIEW_TYPE: ${{ matrix.review_type }} run: | # --sandbox is intentionally omitted. The hosted runner is already # an ephemeral VM — and a `runs_on` pool must be one too: one job per @@ -2629,6 +2638,8 @@ jobs: trap 'forward_signal TERM' TERM trap 'forward_signal HUP' HUP python3 "$CURSOR_REVIEW_ASSETS/stop-on-submit.py" \ + --retry-delays 30,90 --retry-window 300 \ + --cell "$REVIEW_TYPE/$MODEL" \ --findings /tmp/findings-out/findings.json -- \ cursor-agent \ --print \ From c7a5eebcfb11067b29deee2d6b772e4c266e9e82 Mon Sep 17 00:00:00 2001 From: Matt Miller Date: Fri, 9 Oct 2026 00:15:16 -0700 Subject: [PATCH 2/3] fix(cursor-review): rewind the prompt before a retried attempt Every attempt inherits the wrapper's stdin, the `< /tmp/prompt.txt` redirect, together with its shared file offset. The first attempt reads it to EOF, so a retried cursor-agent read an empty prompt and failed at once ("No prompt provided for print mode"): a retry could never recover a cell. The wrapper now records stdin's offset at startup and seeks back to it before every retry. A stdin that cannot seek (a pipe, a tty) cannot be replayed, so it rules the retry out and the wrapper logs why. Two smaller fixes from the same review: - The retry `::warning::` always starts a line of its own. It was glued onto the agent's last stderr write when that write had no trailing newline, and the runner then did not parse it as a command. - The retriable marker is matched in the agent's stderr only. stdout is the model's own text, which the PR under review can steer, so it is inherited as is again instead of being teed and searched. New tests use a stub agent that reads its whole stdin: every attempt must get the full prompt (from the offset stdin started at), a pipe on stdin must rule the retry out, a marker on stdout must not trigger one, and the warning must start a line after an unterminated stderr. The retry tests now feed the wrapper a prompt file, as the workflow does, instead of inheriting the test runner's stdin. --- .github/cursor-review/README.md | 2 +- .github/cursor-review/stop-on-submit.py | 127 +++++++++++---- .../tests/test_stop_on_submit.py | 151 +++++++++++++++--- .github/workflows/cursor-review.yml | 5 +- 4 files changed, 231 insertions(+), 54 deletions(-) diff --git a/.github/cursor-review/README.md b/.github/cursor-review/README.md index dec1fa4a..6a6eb458 100644 --- a/.github/cursor-review/README.md +++ b/.github/cursor-review/README.md @@ -221,7 +221,7 @@ the consolidated review's panel table. See [Panel integrity](#panel-integrity). | [`prompt-edge-case.md`](prompt-edge-case.md) | Prompt for the correctness/logic review pass. | | [`prompt-judge.md`](prompt-judge.md) | Prompt the judge model uses to consolidate panel findings into one review. | | [`review-output-mcp.py`](review-output-mcp.py) | Dependency-free stdio MCP server. Reviewers record individual findings and finish; the judge submits the final findings array. It validates and atomically writes the normalized records consumed by later jobs. | -| [`stop-on-submit.py`](stop-on-submit.py) | Wraps each panel cell's `cursor-agent` and polls its `findings.json`. Once the file parses as a JSON object with `"status": "ok"`, the agent gets 15s to exit by itself, then SIGTERM to its process group, then SIGKILL 10s later, so a cell that has submitted exits **within 30 seconds** of submitting instead of idling to the 15-minute step cap. It only reads the file, so the artifact is byte-for-byte what the MCP tool wrote, and the stop exits 0, so the leg stays green. A missing, partial, unparseable or non-`ok` file never triggers a stop, so a cell that never submits still runs to the cap and fails its leg as before. The step cap's and a run cancel's signals reach it through the step shell's traps and are forwarded to the agent's group, which is SIGKILLed at most 5s later (exit 128+signal). Every exit path, including an agent that exits by itself, ends with SIGKILL to the group, so nothing it left running can rewrite the findings after the upload. Only a regular file of at most 8 MiB is read (no FIFO/symlink/device). It also retries Cursor's transient capacity error: an agent that exits non-zero with `[resource_exhausted]` / `RetriableError` in the tail of its stdout or stderr is re-run after `--retry-delays` (30s, then 90s in the workflow), each retry logged as one `::warning::` naming the cell. A retry needs the findings file to still be an unsubmitted record with **no** recorded findings (the MCP tool appends, so a re-run would duplicate them) and must start within `--retry-window` (300s), so the last attempt still fits the step cap. Exit 0, any other error, a signal death and a forwarded signal are never retried. | +| [`stop-on-submit.py`](stop-on-submit.py) | Wraps each panel cell's `cursor-agent` and polls its `findings.json`. Once the file parses as a JSON object with `"status": "ok"`, the agent gets 15s to exit by itself, then SIGTERM to its process group, then SIGKILL 10s later, so a cell that has submitted exits **within 30 seconds** of submitting instead of idling to the 15-minute step cap. It only reads the file, so the artifact is byte-for-byte what the MCP tool wrote, and the stop exits 0, so the leg stays green. A missing, partial, unparseable or non-`ok` file never triggers a stop, so a cell that never submits still runs to the cap and fails its leg as before. The step cap's and a run cancel's signals reach it through the step shell's traps and are forwarded to the agent's group, which is SIGKILLed at most 5s later (exit 128+signal). Every exit path, including an agent that exits by itself, ends with SIGKILL to the group, so nothing it left running can rewrite the findings after the upload. Only a regular file of at most 8 MiB is read (no FIFO/symlink/device). It also retries Cursor's transient capacity error: an agent that exits non-zero with `[resource_exhausted]` / `RetriableError` in the tail of its stderr (never its stdout, the model's own text, which the PR under review can steer) is re-run after `--retry-delays` (30s, then 90s in the workflow), each retry logged as one `::warning::` on a line of its own, naming the cell. Every attempt reads the whole prompt: the wrapper puts its stdin (`< /tmp/prompt.txt`) back at its starting offset before each retry, so the prompt must arrive as a file redirect; a stdin that cannot seek (a pipe, a tty) rules retries out. A retry needs the findings file to still be an unsubmitted record with **no** recorded findings (the MCP tool appends, so a re-run would duplicate them) and must start within `--retry-window` (300s), so the last attempt still fits the step cap. Exit 0, any other error, a signal death and a forwarded signal are never retried. | | [`post-review.py`](post-review.py) | Reads the judge's consolidated findings and posts **one** PR review with line-anchored inline comments and severity badges. | | [`auto-approve.py`](auto-approve.py) | Opt-in auto-approve (`approve_max_severity`): `decide` (in `post-review`) approves / requests changes / reaches no decision (withdrawing its own earlier approvals and leaving one standing REQUEST_CHANGES with the reasons and the next step, replaced by the next no-decision round and dismissed by the next approving one; none on a `capped` round, which withdraws the earlier one) per its fail-closed rules, then writes cursor-approve's status card for the outcome (via [`../cursor-approve/card.py`](../cursor-approve/card.py); no card for an author `approve_authors` does not list); `dismiss-stale` (the **Dismiss stale auto-approval** job) withdraws the bot's own approvals that are stale against the PR's live head and base (a push, a reopen, a retarget) — and every one of them while the PR carries `skip-cursor-review` or `needs-human-review` — plus its own standing REQUEST_CHANGES while the PR carries either label (re-read after the reviews are listed) or `approve_max_severity` is empty (then only once none of its approvals still stands), on every event of an open PR, whether or not `approve_max_severity` is still set; `round-cap` (the **Round cap** job, ahead of the panel) enforces `max_rounds`. On an APPROVE that survived its post-write re-check, `decide` also resolves (re-reading each first) and then replies on the posting identity's own at-or-below-threshold threads nobody else commented in (nothing, if any live thread is above the threshold); [`build-ledger.py`](build-ledger.py) never counts that reply as an answer. `decide` also writes the `approve_gate` output, and `round-cap` feeds the `round` and `max_rounds` outputs. Reads open-thread severity through `gate-unresolved.py`'s query. See [`docs/callers/cursor-review.md`](../../docs/callers/cursor-review.md#auto-approve). | | [`gate-unresolved.py`](gate-unresolved.py) | The opt-in blocking gate (`blocking: true` → the **Blocking gate** job): queries the PR's review threads and exits non-zero while any cursor-review finding thread is unresolved and non-outdated. Dropped from `cursor-review.yml` by accident in #31 and restored by BE-4691 — see [the blocking section](#optional-make-the-review-blocking). Double-billed: [`build-ledger.py`](build-ledger.py) also imports it for `CONSOLIDATED_MARKER`, the paging `reviewThreads` GraphQL query and the `iter_threads` / `is_cursor_thread` helpers, so the gate and the ledger can never disagree about which threads are ours. | diff --git a/.github/cursor-review/stop-on-submit.py b/.github/cursor-review/stop-on-submit.py index 3f72aa8e..c92dbe22 100644 --- a/.github/cursor-review/stop-on-submit.py +++ b/.github/cursor-review/stop-on-submit.py @@ -23,20 +23,29 @@ so nothing the agent left running can rewrite the findings afterwards. Retry on a transient Cursor capacity error (`--retry-delays`): when the agent -exits non-zero by itself and the tail of its stdout or stderr carries Cursor's -retriable marker (`[resource_exhausted]` or `RetriableError`), the agent is run -again after the next delay, up to one retry per delay. A retry happens only when -the findings file still reads as an untouched, unsubmitted record — a parseable -object, status not "ok", and NO recorded findings — because the MCP tool appends -to that file, so re-running a cell that already recorded findings would -duplicate them. It is also skipped once `--retry-window` seconds (measured from -the wrapper's start, including the delay) would be exceeded, so a retry starts -only while a full review still fits inside the step cap. Anything else — exit 0, -a different error, death by signal, a forwarded signal (the step cap) — is -reported exactly as without retries. Each retry prints one `::warning::` line -naming the `--cell`, the attempt and the matched marker. The agent's output -still reaches this wrapper's stdout/stderr unchanged; it is only teed through -here so its tail can be matched. +exits non-zero by itself and the tail of its stderr carries Cursor's retriable +marker (`[resource_exhausted]` or `RetriableError`), the agent is run again +after the next delay, up to one retry per delay. Only stderr is matched: stdout +is the model's own text, which the PR under review can steer. A retry happens +only when the findings file still reads as an untouched, unsubmitted record — a +parseable object, status not "ok", and NO recorded findings — because the MCP +tool appends to that file, so re-running a cell that already recorded findings +would duplicate them. It is also skipped once `--retry-window` seconds +(measured from the wrapper's start, including the delay) would be exceeded, so +a retry starts only while a full review still fits inside the step cap. +Anything else — exit 0, a different error, death by signal, a forwarded signal +(the step cap) — is reported exactly as without retries. Each retry prints one +`::warning::` line, on a line of its own, naming the `--cell`, the attempt and +the matched marker. The agent's stdout is inherited untouched; its stderr still +reaches this wrapper's stderr unchanged, only teed through here so its tail can +be matched. + +The prompt is the agent's stdin (`< /tmp/prompt.txt`), one open file whose +offset every attempt shares, and the first attempt reads it to EOF. So the +offset stdin has at startup is recorded, and stdin is put back there before +every retry; without that, the retry would read an empty prompt and fail at +once. A stdin that cannot seek (a pipe, a tty) cannot be replayed, so it rules +the retry out, and the wrapper logs why. A SIGINT/SIGTERM/SIGHUP to this wrapper (the step cap, a run cancel) is passed to the agent's group; the agent then gets at most 5 seconds before SIGKILL — @@ -83,10 +92,10 @@ ("resource_exhausted", re.compile(rb"\[resource_exhausted\]")), ("RetriableError", re.compile(rb"\bRetriableError\b")), ) -# How much of each stream's end is kept for the match. The marker is the last -# thing a failed run prints. +# How much of the end of the agent's stderr is kept for the match. The marker +# is the last thing a failed run prints. TAIL_BYTES = 64 * 1024 -# How long to wait for a stream to drain after the agent's group is killed. +# How long to wait for stderr to drain after the agent's group is killed. # Bounded so a stray process that escaped the group cannot hang the wrapper. DRAIN_TIMEOUT = 5.0 @@ -173,14 +182,59 @@ def drain(self): return self.tail -def retriable_marker(*tails): - """The fixed marker name Cursor's retriable error carries, or None.""" +def retriable_marker(stderr_tail): + """The fixed marker name Cursor's retriable error carries, or None. + + Only the agent's stderr is searched, never its stdout: that is the model's + own output, which text in the PR under review can steer. + """ for name, pattern in RETRIABLE: - if any(pattern.search(tail) for tail in tails): + if pattern.search(stderr_tail): return name return None +def fresh_line(stderr_tail): + """A newline when the agent's stderr ended mid-line, else nothing. + + The agent's stderr is copied into the wrapper's, and its last write may + lack a newline; a `::warning::` glued onto it is not a workflow command + (the runner only parses one at the start of a line). + """ + return "" if not stderr_tail or stderr_tail.endswith(b"\n") else "\n" + + +class Prompt: + """The agent's stdin, put back where it started before each retry. + + Every attempt inherits the same open stdin, offset included, and the first + one reads it to EOF, so a retry left alone reads an empty prompt (and + cursor-agent fails at once with "No prompt provided for print mode"). The + offset is recorded before the first attempt; a stdin that cannot seek — a + pipe, a tty, a closed descriptor — cannot be replayed, and `problem` says + why. + """ + + def __init__(self, fd=0): + self.fd = fd + self.problem = None + try: + self.offset = os.lseek(fd, 0, os.SEEK_CUR) + except OSError as exc: + self.offset, self.problem = None, exc.strerror or repr(exc) + + def rewind(self): + """True once stdin is back at the offset it had at startup.""" + if self.offset is None: + return False + try: + os.lseek(self.fd, self.offset, os.SEEK_SET) + except OSError as exc: + self.problem = exc.strerror or repr(exc) + return False + return True + + def safe_label(text): """A ::warning:: -safe rendering of a caller-supplied cell label.""" return re.sub(r"[^A-Za-z0-9._/@+-]", "_", text)[:200] @@ -210,6 +264,8 @@ def run(command, findings, poll, linger, grace, received = [] current = [None] started = time.monotonic() + # Recorded before the first attempt can move the shared offset. + prompt = Prompt() # The step cap and a run cancel signal THIS process; pass them on, so a cell # that never submits still dies at the cap exactly as it did unwrapped. @@ -227,10 +283,14 @@ def forward(signum, _frame): attempts = len(retry_delays) + 1 for attempt in range(1, attempts + 1): - code, marker = attempt_once( + code, stderr_tail = attempt_once( command, findings, poll, linger, grace, received, current) + marker = retriable_marker(stderr_tail) if attempt == attempts or received or code <= 0 or marker is None: return code + # Everything printed below follows the agent's own stderr, whose last + # write may lack a newline; the `::warning::` must start a line. + print(fresh_line(stderr_tail), end="", file=sys.stderr, flush=True) if not retry_safe(findings): log(f"agent exited {code} with [{marker}], but findings were already " "recorded or the file is unreadable; not retrying") @@ -240,6 +300,12 @@ def forward(signum, _frame): log(f"agent exited {code} with [{marker}], but a retry after {delay:g}s " f"would start past the {retry_window:g}s retry window; not retrying") return code + # Nothing reads stdin between here and the next attempt: the agent's + # whole group is dead, and this wrapper never reads it. + if not prompt.rewind(): + log(f"agent exited {code} with [{marker}], but its stdin cannot be " + f"rewound to replay the prompt ({prompt.problem}); not retrying") + return code print(f"::warning::Reviewer cell {safe_label(cell)}: attempt {attempt}/" f"{attempts} failed with Cursor's retriable error [{marker}] " f"(exit {code}); retrying in {delay:g}s", @@ -254,16 +320,16 @@ def forward(signum, _frame): def attempt_once(command, findings, poll, linger, grace, received, current): - """Run the agent once: (its status, the retriable marker it printed or None).""" + """Run the agent once: (its status, the last TAIL_BYTES of its stderr).""" # Own process group, so a stop reaches cursor-agent's children too (the # MCP server, any shell it spawned) — not just the top-level process. - # Both streams are piped through here only so a retriable error can be - # recognised; every byte is copied on to this wrapper's own stdout/stderr. + # stdin and stdout are inherited as they are. stderr is piped through here + # only so a retriable error can be recognised; every byte is copied on to + # this wrapper's own stderr. child = subprocess.Popen(command, start_new_session=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stderr=subprocess.PIPE) current[0] = child - tees = (Tee(child.stdout, sys.stdout.fileno()), - Tee(child.stderr, sys.stderr.fileno())) + tee = Tee(child.stderr, sys.stderr.fileno()) try: code = supervise(child, findings, poll, linger, grace, received) finally: @@ -272,10 +338,9 @@ def attempt_once(command, findings, poll, linger, grace, received, current): # rewrite the findings after the upload and leg check read them. signal_group(child.pid, signal.SIGKILL) current[0] = None - tails = [tee.drain() for tee in tees] - for stream in (child.stdout, child.stderr): - stream.close() - return code, retriable_marker(*tails) + stderr_tail = tee.drain() + child.stderr.close() + return code, stderr_tail def stop_on_signal(child, signum, grace): diff --git a/.github/cursor-review/tests/test_stop_on_submit.py b/.github/cursor-review/tests/test_stop_on_submit.py index 68aece35..76a45205 100644 --- a/.github/cursor-review/tests/test_stop_on_submit.py +++ b/.github/cursor-review/tests/test_stop_on_submit.py @@ -136,10 +136,12 @@ def write(text): """) return [sys.executable, "-c", prelude + textwrap.dedent(body)] - def wrap(self, body, *, cap=None, extra=TIMINGS): + def wrap(self, body, *, cap=None, extra=TIMINGS, stdin=None, stdout=None): return subprocess.Popen( [sys.executable, SCRIPT, "--findings", self.path, *extra, "--", *self.agent(body)], + stdin=stdin, + stdout=stdout, stderr=subprocess.PIPE, text=True, ) @@ -347,23 +349,64 @@ def test_missing_command_is_a_usage_error(self): EXHAUSTED = "RetriableError: [resource_exhausted] Error" RETRY = ["--retry-delays", "0.2,0.4", "--retry-window", "30", "--cell", "adversarial/kimi-k3-high"] +# A multi-line prompt about the size of a real diff review (~120 KB). +PROMPT = b"Review this diff.\n" + b"+an added line\n" * 8192 +# What cursor-agent prints, then exits 1, when its stdin is empty. +NO_PROMPT = "Error: No prompt provided for print mode" class RetryTest(AgentHarness): - """A non-zero exit carrying Cursor's retriable marker re-runs the agent.""" + """A non-zero exit carrying Cursor's retriable marker re-runs the agent. + + The wrapper's stdin is the prompt, a regular file, as the workflow feeds + it (`< /tmp/prompt.txt`): a retry rewinds it, and a stdin that cannot seek + rules the retry out. Feeding it explicitly also keeps these tests off the + test runner's own stdin, which is a pipe on a GitHub runner. + """ def setUp(self): super().setUp() with open(self.path, "w") as f: json.dump(SEEDED, f) + self.prompt = os.path.join(self.dir.name, "prompt.txt") + with open(self.prompt, "wb") as f: + f.write(PROMPT) + + def prompt_file(self, offset=0): + """The prompt opened for the wrapper's stdin, positioned at `offset`.""" + source = open(self.prompt, "rb", buffering=0) + self.addCleanup(source.close) + source.seek(offset) + return source def stub(self, attempt_body): """An agent that counts its runs; `attempt_body` sees ATTEMPT (1-based).""" return (f"ATTEMPT = len(open({self.pidfile!r}).read().split())\n" + textwrap.dedent(attempt_body)) - def retry(self, attempt_body, extra=None): - proc = self.wrap(self.stub(attempt_body), extra=TIMINGS + (extra or RETRY)) + def reading(self, attempt_body): + """A stub that first reads its whole stdin, as cursor-agent does, keeps + a copy per attempt, and fails like cursor-agent when it is empty.""" + return textwrap.dedent(f"""\ + PROMPT_READ = sys.stdin.buffer.read() + with open(os.path.join({self.dir.name!r}, f"prompt-{{ATTEMPT}}"), "wb") as f: + f.write(PROMPT_READ) + if not PROMPT_READ: + print({NO_PROMPT!r}, file=sys.stderr) + sys.exit(1) + """) + textwrap.dedent(attempt_body) + + def prompts_read(self): + """What each attempt of a `reading` stub read from its stdin, in order.""" + reads = [] + for attempt in range(1, len(self.pids()) + 1): + with open(os.path.join(self.dir.name, f"prompt-{attempt}"), "rb") as f: + reads.append(f.read()) + return reads + + def retry(self, attempt_body, extra=None, stdin=None): + proc = self.wrap(self.stub(attempt_body), extra=TIMINGS + (extra or RETRY), + stdin=self.prompt_file() if stdin is None else stdin) return self.finish(proc, cap=20) def test_resource_exhausted_then_success_is_retried(self): @@ -380,26 +423,87 @@ def test_resource_exhausted_then_success_is_retried(self): self.assertEqual(warnings, [ "::warning::Reviewer cell adversarial/kimi-k3-high: attempt 1/3 failed " "with Cursor's retriable error [resource_exhausted] (exit 1); retrying in 0.2s"]) - # The agent's own stderr still reaches the wrapper's, unchanged. - self.assertIn(EXHAUSTED + "\n", err) + # The agent's own stderr still reaches the wrapper's, unchanged, and + # one that already ended its line gets no blank line before the warning. + self.assertIn(EXHAUSTED + "\n::warning::", err) with open(self.path) as f: self.assertEqual(json.load(f)["status"], "ok") - def test_marker_on_stdout_is_retried_and_stdout_passes_through(self): - proc = subprocess.Popen( - [sys.executable, SCRIPT, "--findings", self.path, *TIMINGS, *RETRY, "--", - *self.agent(self.stub(f"""\ - print("out-" + str(ATTEMPT)) - if ATTEMPT == 1: - print("RetriableError: boom") - sys.exit(1) - write({json.dumps(json.dumps(OK))}) - """))], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + def test_every_retry_reads_the_whole_prompt_again(self): + # Each attempt inherits the same stdin, offset included, and the first + # reads it to EOF: without a rewind every retry would get an empty + # prompt and fail at once, so a retry could never recover a cell. + code, err, _ = self.retry(self.reading(f"""\ + if ATTEMPT < 3: + print({EXHAUSTED!r}, file=sys.stderr) + sys.exit(1) + write({json.dumps(json.dumps(OK))}) + """)) + self.assertEqual(code, 0, err) + self.assertNotIn(NO_PROMPT, err) + self.assertEqual(self.prompts_read(), [PROMPT] * 3) + + def test_the_prompt_is_replayed_from_where_stdin_started(self): + # The rewind goes back to the offset stdin had when the wrapper + # started, not to the start of the file. + start = len(b"Review this diff.\n") + code, err, _ = self.retry(self.reading(f"""\ + if ATTEMPT == 1: + print({EXHAUSTED!r}, file=sys.stderr) + sys.exit(1) + write({json.dumps(json.dumps(OK))}) + """), stdin=self.prompt_file(start)) + self.assertEqual(code, 0, err) + self.assertEqual(self.prompts_read(), [PROMPT[start:]] * 2) + + def test_a_stdin_that_cannot_seek_rules_the_retry_out(self): + # A pipe cannot be replayed, so a retry could only run on an empty + # prompt: the first failure stands, and the wrapper logs why, once. + prompt = PROMPT[:1024] # fits the pipe buffer, so the write cannot block + read_end, write_end = os.pipe() + os.write(write_end, prompt) + os.close(write_end) + try: + code, err, _ = self.retry(self.reading(f"""\ + print({EXHAUSTED!r}, file=sys.stderr) + sys.exit(1) + """), stdin=read_end) + finally: + os.close(read_end) + self.assertEqual(code, 1, err) + self.assertEqual(self.prompts_read(), [prompt]) + self.assertNotIn("::warning::", err) + self.assertEqual(err.count("cannot be rewound to replay the prompt"), 1, err) + self.assertIn("not retrying", err) + + def test_the_warning_starts_a_line_after_an_unterminated_stderr(self): + # The runner only parses `::warning::` at the start of a line; an agent + # whose last stderr write has no newline must not swallow it. + code, err, _ = self.retry(f"""\ + if ATTEMPT == 1: + sys.stderr.write({EXHAUSTED!r}) + sys.exit(1) + write({json.dumps(json.dumps(OK))}) + """) + self.assertEqual(code, 0, err) + self.assertIn(EXHAUSTED + "\n::warning::Reviewer cell adversarial/kimi-k3-high: " + "attempt 1/3 failed", err) + + def test_marker_on_stdout_is_not_retried_and_stdout_passes_through(self): + # stdout is the model's own text, which the PR under review can steer; + # only the agent's stderr may call for a retry. + proc = self.wrap(self.stub(f"""\ + print("out-" + str(ATTEMPT)) + print({EXHAUSTED!r}) + sys.exit(1) + """), extra=TIMINGS + RETRY, stdin=self.prompt_file(), + stdout=subprocess.PIPE) out, err = proc.communicate(timeout=20) - self.assertEqual(proc.returncode, 0, err) - self.assertEqual(out, "out-1\nRetriableError: boom\nout-2\n") - self.assertIn("[RetriableError]", err) + self.assertEqual(proc.returncode, 1, err) + self.assertEqual(out, "out-1\n" + EXHAUSTED + "\n") + self.assertEqual(len(self.pids()), 1) + self.assertNotIn("::warning::", err) + self.assertNotIn("not retrying", err) def test_persistent_resource_exhausted_exhausts_retries(self): code, err, took = self.retry(f"""\ @@ -479,7 +583,8 @@ def test_signal_during_backoff_ends_the_wrapper_without_retrying(self): proc = self.wrap(self.stub(f"""\ print({EXHAUSTED!r}, file=sys.stderr) sys.exit(1) - """), extra=TIMINGS + ["--retry-delays", "30", "--retry-window", "60"]) + """), extra=TIMINGS + ["--retry-delays", "30", "--retry-window", "60"], + stdin=self.prompt_file()) deadline = time.monotonic() + 10 while time.monotonic() < deadline and not ( os.path.exists(self.pidfile) and self.pids()): @@ -535,6 +640,10 @@ def test_run_step_wraps_the_reviewer_agent(self): self.assertIn("--retry-delays 30,90", step) self.assertIn('--cell "$REVIEW_TYPE/$MODEL"', step) self.assertIn("REVIEW_TYPE: ${{ matrix.review_type }}", step) + # A retry replays the prompt by rewinding the wrapper's stdin, so the + # prompt must arrive as a file redirect: a pipe cannot be rewound, and + # would turn every retry off. + self.assertIn("< /tmp/prompt.txt \\\n", step) if __name__ == "__main__": diff --git a/.github/workflows/cursor-review.yml b/.github/workflows/cursor-review.yml index 46ef682c..f6d6607e 100644 --- a/.github/workflows/cursor-review.yml +++ b/.github/workflows/cursor-review.yml @@ -2600,7 +2600,10 @@ jobs: # Only a cell with no recorded findings is re-run, and no retry starts # later than 300s in, so the last attempt still has ~10 min of this cap # for a full review. Each retry is a `::warning::` naming the cell; a - # cell that exhausts them fails its leg exactly as before. + # cell that exhausts them fails its leg exactly as before. A retry + # re-reads the whole prompt because the wrapper rewinds its stdin, so + # keep feeding it as `< /tmp/prompt.txt`: a pipe cannot be rewound, + # and turns the retries off. continue-on-error: true timeout-minutes: 15 env: From 454e25e8092c28e91976d1250bc3be77616ceedb Mon Sep 17 00:00:00 2001 From: Matt Miller Date: Fri, 9 Oct 2026 07:23:19 +0000 Subject: [PATCH 3/3] fix(cursor-review): report a signal that lands while the agent's stderr drains A forwarded signal (the step cap) that arrived after the agent exited by itself, while its stderr was still draining, was recorded but ignored: the wrapper returned the agent's own code, possibly 0, instead of 128+signal, and the drain could still run its full 5s past the signal. The drain now stops as soon as a signal is pending, and run() reports any received signal as the wrapper's status before considering a retry. New test: an agent that exits 0 leaving a process outside its group that holds stderr open; SIGTERM during the drain must exit 143 at once. Co-Authored-By: Claude Opus 5.5 --- .github/cursor-review/stop-on-submit.py | 19 +++++++++++++++---- .../tests/test_stop_on_submit.py | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/cursor-review/stop-on-submit.py b/.github/cursor-review/stop-on-submit.py index c92dbe22..8087d89c 100644 --- a/.github/cursor-review/stop-on-submit.py +++ b/.github/cursor-review/stop-on-submit.py @@ -177,8 +177,15 @@ def pump(self): except OSError: break - def drain(self): - self.thread.join(DRAIN_TIMEOUT) + def drain(self, interrupted=lambda: False): + """The kept tail, once the pipe hits EOF, DRAIN_TIMEOUT passes, or + `interrupted()` turns true — a forwarded signal must not wait it out.""" + deadline = time.monotonic() + DRAIN_TIMEOUT + while self.thread.is_alive() and not interrupted(): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + self.thread.join(min(WAIT_SLICE, remaining)) return self.tail @@ -285,8 +292,12 @@ def forward(signum, _frame): for attempt in range(1, attempts + 1): code, stderr_tail = attempt_once( command, findings, poll, linger, grace, received, current) + # A signal that lands after the agent exited — while its stderr drains + # — still ends the wrapper as signalled, whatever the agent returned. + if received: + return -received[0] marker = retriable_marker(stderr_tail) - if attempt == attempts or received or code <= 0 or marker is None: + if attempt == attempts or code <= 0 or marker is None: return code # Everything printed below follows the agent's own stderr, whose last # write may lack a newline; the `::warning::` must start a line. @@ -338,7 +349,7 @@ def attempt_once(command, findings, poll, linger, grace, received, current): # rewrite the findings after the upload and leg check read them. signal_group(child.pid, signal.SIGKILL) current[0] = None - stderr_tail = tee.drain() + stderr_tail = tee.drain(lambda: bool(received)) child.stderr.close() return code, stderr_tail diff --git a/.github/cursor-review/tests/test_stop_on_submit.py b/.github/cursor-review/tests/test_stop_on_submit.py index 76a45205..271b650f 100644 --- a/.github/cursor-review/tests/test_stop_on_submit.py +++ b/.github/cursor-review/tests/test_stop_on_submit.py @@ -312,6 +312,24 @@ def test_signal_trapping_agent_is_killed_and_reported_as_signalled(self): for pid in self.pids(): self.assert_gone(pid) + def test_signal_while_stderr_drains_is_reported_as_signalled(self): + # A process that escaped the agent's group keeps its stderr open, so + # the wrapper is still draining it after the agent exited 0; the step + # cap landing then must end the wrapper at once, reported as a signal. + proc = self.wrap("""\ + kid = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)"], + start_new_session=True) + record_pid(kid.pid) + sys.exit(0) + """) + self.wait_for_pids(2) + time.sleep(1) # the agent has exited; the wrapper is draining its stderr + started = time.monotonic() + proc.send_signal(signal.SIGTERM) + code, err, _ = self.finish(proc, cap=10) + self.assertEqual(code, 128 + signal.SIGTERM, err) + self.assertLess(time.monotonic() - started, 2, "signal waited out the drain") + def test_agent_that_exits_on_its_own_leaves_nothing_behind(self): # A grandchild still in the agent's group could rewrite findings.json # after the upload; the wrapper reaps the group on this path too.