diff --git a/.github/cursor-approve/card.py b/.github/cursor-approve/card.py index 947362da..7c63db5f 100644 --- a/.github/cursor-approve/card.py +++ b/.github/cursor-approve/card.py @@ -158,6 +158,12 @@ def safe_label(label) -> str: return label if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9 _.:/+-]{0,49}", label) else "" +# Mirrors cursor-review.yml's clamp on `auto_retry_delay_minutes`; a test pins +# the two equal, since the sentence must not promise a wait longer than the +# one the workflow will actually take. +RETRY_DELAY_CAP = 30 + + def relabel(label=DEFAULT_REVIEW_LABEL) -> str: name = safe_label(label) return f"remove and re-add the `{name}` label" if name else "remove and re-add the review label" @@ -181,7 +187,25 @@ def next_push_text(label=DEFAULT_REVIEW_LABEL) -> str: f"skips a head that already carries a review — then re-run the round: {relabel(label)}.") -def next_auto_retry_text(label=DEFAULT_REVIEW_LABEL) -> str: +def retry_delay_phrase(delay_minutes=0) -> str: + """" in about N minutes", or "" — the advance notice for a delayed retry. + + Sanitised here as well as in the workflow that clamps the input, because + this text is posted as the approver: the value arrives as a caller input, + and only a small integer may reach the sentence. Out of range or + unparsable reads as no delay, matching what the workflow would then wait. + """ + try: + mins = int(str(delay_minutes).strip()) + except (TypeError, ValueError): + return "" + if mins < 1: + return "" + mins = min(mins, RETRY_DELAY_CAP) + return f" in about {mins} minute{'s' if mins != 1 else ''}" + + +def next_auto_retry_text(label=DEFAULT_REVIEW_LABEL, delay_minutes=0) -> str: """The relabel the workflow is doing itself (BE-19526). Still ``NEXT_RELABEL``: the next step IS a relabel, and an agent reading the @@ -189,8 +213,15 @@ def next_auto_retry_text(label=DEFAULT_REVIEW_LABEL) -> str: differs, so nobody re-does by hand what already fired. Written before the relabel runs (a later job; it can fail or be cancelled by a push), hence the "if no new round starts" fallback rather than a promise. + + ``delay_minutes`` is the caller's ``auto_retry_delay_minutes``: the wait is + announced BEFORE it starts, which is the point of saying it at all — an + author who reads "being re-run automatically" and sees nothing happen for + five minutes otherwise re-runs it by hand. Zero (every caller that has not + set it) leaves the sentence exactly as it was. """ - return (f"Nothing — the round is being re-run automatically ({relabel(label)}). Once only; " + return (f"Nothing — the round is being re-run automatically{retry_delay_phrase(delay_minutes)} " + f"({relabel(label)}). Once only; " "if no new round starts, or it is wrong again, re-run it by hand.") diff --git a/.github/cursor-review/README.md b/.github/cursor-review/README.md index cdaa3057..ceb3002d 100644 --- a/.github/cursor-review/README.md +++ b/.github/cursor-review/README.md @@ -417,6 +417,8 @@ All optional except `workflows_ref` (required, no default) — pass them under | `defer_approval` | `false` | For a caller that runs cursor-approve after this workflow: an APPROVE outcome still reports `approve_gate` = `pass` but posts no approval and resolves no thread, and withdraws the approver's own earlier auto-approvals and requests for changes, so cursor-approve's decide is the only approver. Pass the workflow's `approve_scope_effective` output (the scope `decide` actually gated under, `full` on any fallback) to cursor-approve's `approve_scope`, so its resolution honours non-gating marks only after a `delta` round. | | `approve_max_failed_reviewers` | `0` | Errored panel reviewers (status `error`) auto-approve tolerates before withholding its decision. `0` → any reviewer that did not complete means no decision. Never tolerates another status, more than N, or a review type with no completed reviewer; tolerated cells are named in the decision. | | `rereview_command` | `''` (off) | A comment command, e.g. `/review`, asking for another round on the **same commit**. Empty leaves the same-commit dedupe absolute. Needs the SAME value on [`cursor-review-auto-label.yml`](../../docs/callers/cursor-review-auto-label.md) plus `issue_comment` in that caller's triggers: that half re-applies the label (removing it first, since adding a label already present raises no event), this half stops the dedupe from skipping the round it starts. Setting one alone is a silent no-op. The command must be the first token of the first line, and the commenter's association must be `OWNER`, `MEMBER` or `COLLABORATOR` — not the opted-in roster. The request must postdate the review it overrides, so one request buys one round; `max_rounds` still applies. Exists because the documented alternative cannot work: the consolidated review is `COMMENTED`, which GitHub will not dismiss. See [the setup guide](../../docs/callers/cursor-review.md#asking-for-another-round-on-the-same-commit). | +| `auto_retry_errored_rounds` | `false` | Opt-in: re-run a round the PANEL broke (reviewer cells `error`) rather than leaving a standing block for a human to relabel. Bounded by the same one-per-PR budget as the moved-head retry (`AUTO_RETRY_MARKER` on decide's own block), by `max_rounds`, and by `can_relabel`. Never fires for a structural cause or for findings above the threshold. Its force signal at the gate is that marked block — decide's own shape, not dismissed, newer (by review id) than the newest consolidated review for this commit, and authored by a Bot or by the identity that applied the label (the APPROVER_TOKEN case) — NOT the `rereview_command` comment path, which stays authorized by comment association alone. No effect without `approve_max_severity`. | +| `auto_retry_delay_minutes` | `0` | Minutes an automatic retry waits before relabelling (`0` = immediately, the prior behaviour). Clamped to 30 in the workflow and again in `card.py`'s sentence, so the card cannot promise a longer wait than the one taken; unparsable → no wait. Costs runner-minutes while it sleeps. No effect without `approve_max_severity` and `auto_retry_errored_rounds` (or a moved-head retry). | | `max_rounds` | `5` | Per-PR review-round cap; `0` → none. Counts the posting identity's consolidated reviews since `needs-human-review` was last removed; at or over the cap the panel is skipped, the PR is labelled `needs-human-review`, one comment lists the latest round's open findings above the threshold, and the `approve_gate` output is `capped`. Fails open on a read error. See [the setup guide](../../docs/callers/cursor-review.md#round-cap-and-the-approve_gate-output). | | `runs_on` | `'"ubuntu-latest"'` | JSON-encoded `runs-on` (same shape as `cursor-review-auto-label.yml`'s) for the four credential-free heavy jobs only: `diff-size`, `preflight`, the `review` panel and `consolidate`. The gate, ledger, Slack notices and every job holding the App token or `pull-requests: write` stay on GitHub-hosted `ubuntu-latest`. Those four run models with unsandboxed shell over PR code, so use a self-hosted pool only if each job gets a fresh pod (fresh filesystem and `$HOME`), holds no cloud identity or token, and can't reach private networks or the metadata server. The image must be linux/x64 with bash, git, curl, jq, python3, gh, tar and GNU coreutils. | diff --git a/.github/cursor-review/auto-approve.py b/.github/cursor-review/auto-approve.py index d5b11502..e6c1ed44 100644 --- a/.github/cursor-review/auto-approve.py +++ b/.github/cursor-review/auto-approve.py @@ -577,6 +577,20 @@ def _cell_label(cell: dict) -> str: return f"{_label_part(cell.get('model'))}:{_label_part(cell.get('review_type'))}" +# The withhold reason a short panel produces is COMPUTED (it carries the +# counts, and may name the review type left with no completed cell), so +# `auto_retry_eligible` cannot match it against a fixed string the way it +# matches the head/base races. This suffix is the stable part, built into the +# reason by `panel_gate` and matched by `is_panel_incomplete_reason` — one +# constant, so the producer and the matcher cannot drift apart. +PANEL_INCOMPLETE_SUFFIX = "panel reviewers did not complete" + + +def is_panel_incomplete_reason(reason) -> bool: + """True when this withhold reason is "reviewers did not complete".""" + return isinstance(reason, str) and PANEL_INCOMPLETE_SUFFIX in reason + + def panel_gate(panel: list, max_failed: int = 0): """(withhold_reason or None, tolerated_cells). Pure. @@ -589,7 +603,7 @@ def panel_gate(panel: list, max_failed: int = 0): bad = [c for c in panel if not isinstance(c, dict) or c.get("status") != "ok"] if not bad and max_failed <= 0: return None, [] - incomplete = f"{len(bad)}/{len(panel)} panel reviewers did not complete" + incomplete = f"{len(bad)}/{len(panel)} {PANEL_INCOMPLETE_SUFFIX}" tolerable = [c for c in bad if isinstance(c, dict) and c.get("status") == TOLERABLE_CELL_STATUS] if len(tolerable) != len(bad) or len(bad) > max_failed: return incomplete, [] @@ -776,6 +790,22 @@ def _trusted_decision(threshold: str, findings: list, open_thread_severities: li return APPROVE, GATE_PASS, [f"every finding is at or below `{threshold}`", *note], [] +def blocking_without_trust(threshold: str, findings: list, open_thread_severities: list, scope) -> bool: + """Would this round block even if it WERE trusted? + + For an untrusted round — decide_gate never got past its trust checks — + so a retry is not spent on a PR whose completed cells already found + something above the threshold, or that has such a thread open. Fails + closed: anything unreadable answers True, which only withholds a retry. + """ + try: + _event, gate, _reasons, _blocking = _trusted_decision( + threshold, findings or [], open_thread_severities or [], scope or {"scope": SCOPE_FULL}, True) + except Exception: # noqa: BLE001 — unknown is "do not retry", never a traceback + return True + return gate == GATE_FAIL + + SCOPE_NOTE_PREFIX = "approve_scope `" @@ -1085,19 +1115,52 @@ def no_decision_next(reasons: list, label: str = ""): DEFAULT_REVIEW_LABEL = "cursor-review" -def auto_retry_eligible(reasons: list) -> bool: - """Would a fresh round on the live head decide what this one could not? +def auto_retry_eligible(reasons: list, retry_errored: bool = False, blocked_anyway: bool = False) -> bool: + """Would a fresh round decide what this one could not? + + TWO shapes, and the second one only exists because the wall moved. + + The head moved. That is what makes the next round read different content, + so the gate's same-SHA dedupe cannot skip it. A base retarget alongside it + rides along (the relabel fixes both); a retarget ALONE does not, because + the head is unchanged and the re-run would be deduped away. + + Reviewers ERRORED, with `retry_errored` on. This is the case the docstring + above used to rule out — "any other reason means a re-run would hit the + same wall" — and the wall was the gate's same-SHA dedupe, which it now + lifts for a marked retry (the bypass #407 built for the human command, + keyed here on the marker this decision writes rather than on a comment). + The content being identical is exactly the point: the round failed on + infrastructure — a capacity error, a cell timeout — so running the SAME + content again is what decides it. Opt-in because it spends a whole panel + on the guess that the failure was transient. + + A reason this cannot fix anywhere in the list still rules out a retry: a + structural cause, or findings above the threshold, means the next round + hits the same wall whatever the panel does. - Only when the head moved — that is what makes the next round read different - content, so the gate's same-SHA dedupe cannot skip it. A base retarget - alongside it rides along (the relabel fixes both); a retarget ALONE does - not, because the head is unchanged and the re-run would be deduped away. - Any other reason in the list means a re-run would hit the same wall. + `blocked_anyway` is that last case for a short panel. decide_gate stops at + the trust checks, so an untrusted round's reasons never name its findings: + the caller has to ask `blocking_without_trust` whether the cells that DID + complete (or an open thread) already block. If they do, a full panel on the + same head would only confirm a block, and spend the PR's one retry doing it. """ reasons = [r for r in (reasons or []) if isinstance(r, str) and r.strip()] - if not reasons or REASON_HEAD_MOVED not in reasons: + if not reasons: return False - return all(r in RETRYABLE_CAUSES for r in reasons) + if REASON_HEAD_MOVED in reasons and all(r in RETRYABLE_CAUSES for r in reasons): + return True + if not retry_errored: + return False + if blocked_anyway and REASON_HEAD_MOVED not in reasons: + # Same content, same block. A moved head re-reviews different content, + # so what the old head's cells found decides nothing about it. + return False + # At least one reason must BE the errored panel: without this, a lone base + # retarget (unfixable by a same-SHA re-run) would pass the `all` below. + if not any(is_panel_incomplete_reason(r) for r in reasons): + return False + return all(r in RETRYABLE_CAUSES or is_panel_incomplete_reason(r) for r in reasons) def auto_retries_spent(reviews: list, approver_login: str) -> int: @@ -1134,7 +1197,7 @@ def round_cap_leaves_room(args) -> bool: return max_rounds <= 0 or this_round < max_rounds -def auto_retry_budget_left(args, reasons: list) -> bool: +def auto_retry_budget_left(args, reasons: list, blocked_anyway: bool = False) -> bool: """auto_retry_eligible() plus the per-PR ceiling, read from live reviews. An unreadable review list spends the budget rather than retrying blind: a @@ -1151,7 +1214,8 @@ def auto_retry_budget_left(args, reasons: list) -> bool: """ if (getattr(args, "can_relabel", "") or "").strip().lower() != "true": return False - if not auto_retry_eligible(reasons): + retry_errored = (getattr(args, "retry_errored_rounds", "") or "").strip().lower() == "true" + if not auto_retry_eligible(reasons, retry_errored=retry_errored, blocked_anyway=blocked_anyway): return False if not (getattr(args, "approver_login", "") or "").strip(): return False @@ -1185,8 +1249,26 @@ def cmd_auto_retry(args) -> int: The POST is tried twice, since a failed one after a successful DELETE leaves the trigger label stripped. Any failure exits 1, so the job goes red; the card's next step already says to re-run by hand if no round starts. + + The PR is read again first, because decide's view of it can be up to + `auto_retry_delay_minutes` old by now: a PR closed or merged during the + wait, or handed to a human (`needs-human-review`), gets no new round. A + failed read does not cancel the retry the card already announced — the + relabel is what was promised, and the round's own gate re-reads the PR. """ label = (args.review_label or "").strip() or DEFAULT_REVIEW_LABEL + try: + pr = read_pr(args.repo, args.pr_number) + except (RuntimeError, ValueError) as e: + print(f"::warning::Could not re-read the PR before re-running the round; re-running anyway " + f"({annotation_cause(e, 'unknown error')}).") + else: + if (pr.get("state") or "") != "open": + emit("⏭️ **Auto-approve: not re-running the round** — the PR is no longer open.") + return 0 + if has_label(pr, HUMAN_REVIEW_LABEL): + emit(f"⏭️ **Auto-approve: not re-running the round** — the PR now carries `{HUMAN_REVIEW_LABEL}`.") + return 0 url = f"repos/{args.repo}/issues/{args.pr_number}/labels" try: gh(["api", "-X", "DELETE", f"{url}/{urllib.parse.quote(label, safe='')}"]) @@ -1207,7 +1289,7 @@ def cmd_auto_retry(args) -> int: f"({annotation_cause(err, 'unknown error')}), so the PR may now be WITHOUT the label. " f"Add `{label}` back by hand to start the round.") return 1 - emit(f"🔁 **Auto-approve: re-running the round** — the head moved, so `{label}` was removed and re-added.") + emit(f"🔁 **Auto-approve: re-running the round** — `{label}` was removed and re-added.") return 0 @@ -1522,12 +1604,17 @@ def render_standing_body(headline: str, reasons: list, next_text: str, threshold CARD_NEXT_PUSH = "push_then_relabel" -def _card_text(name: str, args=None) -> str: +def _card_text(name: str, args=None, **extra) -> str: """A card.py next-step text — a function of the review label, or a constant - — or "" when card.py cannot be loaded (the card is then skipped anyway).""" + — or "" when card.py cannot be loaded (the card is then skipped anyway). + + ``extra`` reaches a text that takes more than the label (the retry's delay). + A card.py without that keyword raises, which is the same "" as an unloadable + one: the call site keeps its own fallback text. + """ try: value = getattr(_load_card(), name) - return value(_review_label(args)) if callable(value) else value + return value(_review_label(args), **extra) if callable(value) else value except Exception: # noqa: BLE001 — reporting only, see write_round_card return "" STANDING_REPLACED_MESSAGE = "Superseded by the latest cursor-review round's request for changes." @@ -1627,7 +1714,8 @@ def _review_label(args) -> str: return getattr(args, "review_label", "") or "" -def card_for_none(args, gate: str, reasons: list, threshold: str, threads: list, scope: dict) -> int: +def card_for_none(args, gate: str, reasons: list, threshold: str, threads: list, scope: dict, + blocked_anyway: bool = False) -> int: """A NONE decision: the card, plus the standing request for changes. `capped` posts none: the label hands the PR to a human, and no round will @@ -1656,10 +1744,15 @@ def card_for_none(args, gate: str, reasons: list, threshold: str, threads: list, # BE-19526: the head outran the round, and this PR has a retry left — # say so on the block and the card, then fire it below. Decided BEFORE # the block is posted so the marker it is counted by is on it. - retry = auto_retry_budget_left(args, reasons) - hand_text = next_text + retry = auto_retry_budget_left(args, reasons, blocked_anyway=blocked_anyway) + hand_nxt, hand_text = nxt, next_text if retry: - next_text = _card_text("next_auto_retry_text", args) or next_text + # The marker as well as the prose: on an unchanged head (the errored + # cause) no_decision_next said `push_then_relabel`, and an agent that + # obeyed it would push — cancelling the retry this card announces. + nxt = CARD_NEXT_RELABEL + next_text = _card_text("next_auto_retry_text", args, + delay_minutes=getattr(args, "retry_delay_minutes", 0)) or next_text posted = {} rc = post_standing_change_request(args, headline, reasons, next_text, threshold, retry=retry, result=posted) if retry and posted.get("id") is None: @@ -1667,7 +1760,7 @@ def card_for_none(args, gate: str, reasons: list, threshold: str, threads: list, # body would not render), so no marker counts this retry: firing it # anyway would re-fire on every later moved head, unbounded. The card # goes back to the hand-recovery too, since nothing is re-running. - retry, next_text = False, hand_text + retry, nxt, next_text = False, hand_nxt, hand_text write_round_card(args, state, nxt, headline, reasons, next_text, threshold, gating) if retry: # cursor-review.yml's `auto-retry` job relabels; see cmd_auto_retry. @@ -1753,7 +1846,7 @@ def cmd_decide(args) -> int: scope = with_snapshot_anchors(scope, getattr(args, "open_anchors", "") or "", args.commit_sha) # The open threads behind `prior`, from the same read: the card links them. - threads = [] + threads, prior = [], [] try: pr = read_pr(args.repo, args.pr_number) live_head, live_base = pr_head_base(pr) @@ -1794,7 +1887,8 @@ def cmd_decide(args) -> int: # The standing request for changes and the card (BE-19489): a NONE # round used to leave nothing on the PR at all. try: - if card_for_none(args, gate, reasons, threshold, threads, scope): + blocked = gate == GATE_UNTRUSTED and blocking_without_trust(threshold, findings, prior, scope) + if card_for_none(args, gate, reasons, threshold, threads, scope, blocked_anyway=blocked): rc = 1 except Exception as e: # noqa: BLE001 — the withdrawal above already ran; report, don't traceback print(f"::error::Could not leave the no-decision block or card: {annotation_cause(e, 'unknown error')}") @@ -3207,6 +3301,12 @@ def main() -> int: # `false` on the GITHUB_TOKEN fallback, which applies labels that fire # nothing. Fails closed: only an explicit `true` retries. d.add_argument("--can-relabel", default="false") + # Whether a round that could not decide because reviewers ERRORED may be + # re-run on the same commit (the caller's `auto_retry_errored_rounds`). + # Fails closed, like --can-relabel: only an explicit `true`. + d.add_argument("--retry-errored-rounds", default="false") + # Announced on the card, not waited for here — the `auto-retry` job waits. + d.add_argument("--retry-delay-minutes", default="0") s = sub.add_parser("dismiss-stale") s.add_argument("--repo", required=True) s.add_argument("--pr-number", required=True) diff --git a/.github/cursor-review/tests/test_auto_approve.py b/.github/cursor-review/tests/test_auto_approve.py index 6a90631e..40ecd2a3 100644 --- a/.github/cursor-review/tests/test_auto_approve.py +++ b/.github/cursor-review/tests/test_auto_approve.py @@ -3348,7 +3348,8 @@ class AutoRetryTest(unittest.TestCase): def run_decide(self, head=OTHER_SHA, reviews=(), judge="ok", delivered="true", base_ref="main", live_base="main", reviews_fail=False, review_post_error="", - can_relabel="true", round_="2", max_rounds="5", approver=APPROVER, omit_can_relabel=False): + can_relabel="true", round_="2", max_rounds="5", approver=APPROVER, omit_can_relabel=False, + panel=PANEL_OK, retry_errored=None, retry_delay=None, findings=(), open_threads=()): self.reviews_posted, self.labels, self.card_writes = [], [], [] def fake_gh(args, payload=None): @@ -3377,7 +3378,7 @@ def fake_gh(args, payload=None): with tempfile.TemporaryDirectory() as d: fpath, dpath, out = (os.path.join(d, n) for n in ("c.json", "pr.patch", "out")) with open(fpath, "w") as f: - json.dump({"findings": [], "panel": list(PANEL_OK)}, f) + json.dump({"findings": list(findings), "panel": list(panel)}, f) with open(dpath, "w") as f: f.write(DIFF) open(out, "w").close() @@ -3390,9 +3391,16 @@ def fake_gh(args, payload=None): run_url="https://github.com/o/r/actions/runs/1") if not omit_can_relabel: args.can_relabel = can_relabel + # Left OFF the namespace unless a test sets them, so every existing + # case here still exercises the defaults a caller that has not + # opted in gets. + if retry_errored is not None: + args.retry_errored_rounds = retry_errored + if retry_delay is not None: + args.retry_delay_minutes = retry_delay with mock.patch.dict(os.environ, {"GITHUB_OUTPUT": out}), \ mock.patch.object(AA, "gh", fake_gh), \ - mock.patch.object(AA, "open_thread_severities", lambda *a, **k: []), \ + mock.patch.object(AA, "open_thread_severities", lambda *a, **k: list(open_threads)), \ mock.patch.object(AA, "resolve_eligible_threads", lambda *a: None), \ mock.patch.object(AA, "emit", lambda *a: None), \ mock.patch("builtins.print", lambda *a, **k: None): @@ -3531,11 +3539,104 @@ def test_only_an_explicit_can_relabel_true_retries(self): self.run_decide(omit_can_relabel=True) self.assertFalse(self.retried()) + # -- the errored-panel opt-in ------------------------------------------ + # The sequel to the above: a round the panel broke, on a head that never + # moved. Retrying it was impossible until the gate learned to re-run a + # commit it had already reviewed, so this whole section is off by default. + + PANEL_ERR = [{"model": "m", "review_type": "adversarial", "status": "error"}] + + def test_an_errored_panel_is_not_retried_by_default(self): + self.run_decide(head=SHA, panel=self.PANEL_ERR) + self.assertFalse(self.retried()) + body = self.reviews_posted[0]["body"] + self.assertNotIn(AA.AUTO_RETRY_MARKER, body) + # The hand recovery for a head that never moved: a relabel alone would + # be deduped away, which is the wall the opt-in lifts. + self.assertIn("Push a commit so the head moves", body) + + def test_an_errored_panel_is_retried_when_the_caller_opts_in(self): + self.run_decide(head=SHA, panel=self.PANEL_ERR, retry_errored="true") + self.assertTrue(self.retried()) + body = self.reviews_posted[0]["body"] + self.assertIn(AA.AUTO_RETRY_MARKER, body) + self.assertIn("re-run automatically", body) + + def test_the_opt_in_does_not_widen_the_other_causes(self): + # Still structural, still a judge failure the panel cannot fix. + self.run_decide(head=SHA, delivered="false", retry_errored="true") + self.assertFalse(self.retried()) + self.run_decide(head=SHA, judge="error", retry_errored="true") + self.assertFalse(self.retried()) + + def test_the_configured_wait_is_announced_before_it_starts(self): + self.run_decide(head=SHA, panel=self.PANEL_ERR, + retry_errored="true", retry_delay="5") + self.assertTrue(self.retried()) + for body in (self.reviews_posted[0]["body"], self.card_writes[0]): + self.assertIn("in about 5 minutes", body) + + def test_no_wait_is_announced_when_none_is_configured(self): + self.run_decide(head=SHA, panel=self.PANEL_ERR, retry_errored="true") + self.assertNotIn("in about", self.reviews_posted[0]["body"]) + + def test_the_card_marker_says_relabel_not_push(self): + # On an unchanged head the hand recovery is `push_then_relabel`; an + # agent obeying that marker would push and cancel the announced retry. + self.run_decide(head=SHA, panel=self.PANEL_ERR, retry_errored="true") + self.assertTrue(self.retried()) + self.assertIn("", self.card_writes[0]) + self.assertNotIn("push_then_relabel", self.card_writes[0]) + + def test_a_failed_block_puts_the_push_marker_back(self): + self.run_decide(head=SHA, panel=self.PANEL_ERR, retry_errored="true", + review_post_error="gh api failed: HTTP 422") + self.assertFalse(self.retried()) + self.assertIn("", self.card_writes[0]) + + def test_completed_cells_above_the_threshold_rule_out_the_retry(self): + # decide_gate stops at the trust checks, so this never appears as a + # reason; the retry has to ask whether the completed cells already block. + self.run_decide(head=SHA, panel=self.PANEL_ERR, retry_errored="true", + findings=[{"severity": "high", "title": "t"}]) + self.assertFalse(self.retried()) + self.assertNotIn(AA.AUTO_RETRY_MARKER, self.reviews_posted[0]["body"]) + + def test_findings_at_or_below_the_threshold_still_retry(self): + self.run_decide(head=SHA, panel=self.PANEL_ERR, retry_errored="true", + findings=[{"severity": "low", "title": "t"}]) + self.assertTrue(self.retried()) + + def test_an_open_thread_above_the_threshold_rules_out_the_retry(self): + self.run_decide(head=SHA, panel=self.PANEL_ERR, retry_errored="true", + open_threads=["high"]) + self.assertFalse(self.retried()) + + def test_findings_do_not_gate_the_moved_head_retry(self): + # The pre-existing cause re-reviews DIFFERENT content, so what the old + # head's cells found says nothing about the next round. + self.run_decide(findings=[{"severity": "high", "title": "t"}]) + self.assertTrue(self.retried()) + self.run_decide(panel=self.PANEL_ERR, retry_errored="true", + findings=[{"severity": "high", "title": "t"}]) + self.assertTrue(self.retried()) + + def test_the_opt_in_still_obeys_the_one_per_pr_budget(self): + self.run_decide(head=SHA, panel=self.PANEL_ERR, + retry_errored="true", reviews=[retried_block(5)]) + self.assertFalse(self.retried()) + # -- the relabel itself (the workflow's auto-retry job) ---------------- - def run_auto_retry(self, label="cursor-review", fail=None): + def run_auto_retry(self, label="cursor-review", fail=None, pr=None, pr_error=None): """`fail` maps a method to the errors its successive calls raise.""" self.calls, fails = [], {k: list(v) for k, v in (fail or {}).items()} + pr = {"state": "open", "labels": []} if pr is None else pr + + def fake_read_pr(repo, number): + if pr_error: + raise RuntimeError(pr_error) + return pr def fake_gh(args, payload=None): self.calls.append((args[2], args[3], payload)) @@ -3545,10 +3646,25 @@ def fake_gh(args, payload=None): return "{}" args = argparse.Namespace(repo="o/r", pr_number="1", review_label=label) - with mock.patch.object(AA, "gh", fake_gh), mock.patch.object(AA, "emit", lambda *a: None), \ + with mock.patch.object(AA, "gh", fake_gh), mock.patch.object(AA, "read_pr", fake_read_pr), \ + mock.patch.object(AA, "emit", lambda *a: None), \ mock.patch("builtins.print", lambda *a, **k: None): return AA.cmd_auto_retry(args) + def test_auto_retry_skips_a_pr_closed_during_the_wait(self): + self.assertEqual(self.run_auto_retry(pr={"state": "closed", "labels": []}), 0) + self.assertEqual(self.calls, []) + + def test_auto_retry_skips_a_pr_handed_to_a_human_during_the_wait(self): + pr = {"state": "open", "labels": [{"name": AA.HUMAN_REVIEW_LABEL}]} + self.assertEqual(self.run_auto_retry(pr=pr), 0) + self.assertEqual(self.calls, []) + + def test_auto_retry_still_relabels_when_the_pr_cannot_be_read(self): + # The card already announced it; the round's own gate re-reads the PR. + self.assertEqual(self.run_auto_retry(pr_error="gh api failed: HTTP 502"), 0) + self.assertEqual([c[0] for c in self.calls], ["DELETE", "POST"]) + def test_auto_retry_removes_and_re_adds_the_label(self): self.assertEqual(self.run_auto_retry(), 0) self.assertEqual(self.calls, [("DELETE", "repos/o/r/issues/1/labels/cursor-review", None), diff --git a/.github/cursor-review/tests/test_auto_retry_errored.py b/.github/cursor-review/tests/test_auto_retry_errored.py new file mode 100644 index 00000000..934738dc --- /dev/null +++ b/.github/cursor-review/tests/test_auto_retry_errored.py @@ -0,0 +1,552 @@ +#!/usr/bin/env python3 +"""Tests for `auto_retry_errored_rounds` — re-running a round the panel broke. + +WHY IT EXISTS. Auto-retry (BE-19526) only ever covered the head/base races, and +`auto_retry_eligible`'s own docstring said why it could not cover anything +else: "a re-run would hit the same wall". The wall was the gate's same-commit +dedupe, and #407 removed it. The case that wall was blocking is the common one +— a round that could not decide because reviewer cells ERRORED, where the diff +is fine and there is nothing to push. + +TWO THINGS ARE EASY TO GET WRONG HERE, so both are pinned hard. + +A retry must not be offered for a cause a retry cannot fix. A structural +failure, or findings above the threshold, means the next round hits the same +wall whatever the panel does — so `EligibilityTest` walks the whole truth table +rather than the happy path. + +The retry must not reach the HUMAN force path. `rereview_command` authorizes by +the comment's association; the retry posts no comment, and if it could satisfy +that path then any App's comment could force rounds. The retry's own signal is +the marked block `decide` already writes, checked for its author (a Bot, or the +identity that fired the relabel — the APPROVER_TOKEN case), its shape, the +commit id, and recency by review id. + +Parsed WITHOUT PyYAML, like its sibling suites: this repo is stdlib-only and CI +installs no requirements for these tests. + +Run: python3 -m unittest discover -s .github/cursor-review/tests -p 'test_*.py' -v +""" + +import importlib.util +import io +import json +import os +import re +import shutil +import subprocess +import unittest + +HERE = os.path.dirname(__file__) +SCRIPT = os.path.normpath(os.path.join(HERE, "..", "auto-approve.py")) +WORKFLOW = os.path.normpath( + os.path.join(HERE, "..", "..", "workflows", "cursor-review.yml") +) + +CARD = os.path.normpath( + os.path.join(HERE, "..", "..", "cursor-approve", "card.py") +) + + +def _load(path, name): + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +auto_approve = _load(SCRIPT, "auto_approve_retry") +card = _load(CARD, "card_retry") + +HEAD_MOVED = auto_approve.REASON_HEAD_MOVED +BASE_CHANGED = auto_approve.REASON_BASE_CHANGED +STRUCTURAL = auto_approve.REASON_NOT_DELIVERED + + +def workflow_text(): + with io.open(WORKFLOW, encoding="utf-8") as f: + return f.read() + + +def step_run(name): + """The `run:` body of the step with this exact name.""" + text, cur, found = workflow_text(), None, [] + for line in text.splitlines(): + if re.match(r"^ - name: ", line): + if cur is not None: + found.append("\n".join(cur)) + cur = [line] if line.strip() == f"- name: {name}" else None + elif cur is not None: + cur.append(line) + if cur is not None: + found.append("\n".join(cur)) + assert len(found) == 1, f"{name!r} matched {len(found)} steps" + m = re.search(r"^ run: \|\n(.*)", found[0], re.S | re.M) + body = [] + for line in m.group(1).splitlines(): + if line.strip() and not line.startswith(" "): + break + body.append(line[10:]) + return found[0], "\n".join(body) + + +class PanelReasonTest(unittest.TestCase): + """The predicate must match what `panel_gate` actually produces.""" + + def test_the_predicate_matches_a_real_short_panel_reason(self): + # Generated, not hand-written: a hand-written reason is how the + # producer and the matcher drift apart. + panel = [ + {"review_type": "adversarial", "status": "ok"}, + {"review_type": "edge-case", "status": "error"}, + ] + reason, _tolerated = auto_approve.panel_gate(panel, max_failed=0) + self.assertIsNotNone(reason) + self.assertTrue(auto_approve.is_panel_incomplete_reason(reason), reason) + + def test_the_variant_that_names_a_missing_review_type_also_matches(self): + panel = [ + {"review_type": "adversarial", "status": "error"}, + {"review_type": "edge-case", "status": "error"}, + ] + reason, _ = auto_approve.panel_gate(panel, max_failed=5) + self.assertIsNotNone(reason) + self.assertIn("leaving no completed", reason) + self.assertTrue(auto_approve.is_panel_incomplete_reason(reason), reason) + + def test_unrelated_reasons_do_not_match(self): + for reason in (HEAD_MOVED, BASE_CHANGED, STRUCTURAL, + "no panel metadata", "", None, 7): + with self.subTest(reason=reason): + self.assertFalse(auto_approve.is_panel_incomplete_reason(reason)) + + +class EligibilityTest(unittest.TestCase): + """The whole truth table, because the dangerous answers are the `True`s.""" + + ERRORED = "2/6 " + auto_approve.PANEL_INCOMPLETE_SUFFIX + + def eligible(self, reasons, on): + return auto_approve.auto_retry_eligible(reasons, retry_errored=on) + + # --- the pre-existing path, which must be untouched --- + + def test_head_moved_is_eligible_with_the_flag_off(self): + self.assertTrue(self.eligible([HEAD_MOVED], False)) + self.assertTrue(self.eligible([HEAD_MOVED, BASE_CHANGED], False)) + + def test_a_base_retarget_alone_is_never_eligible(self): + # The head is unchanged, so the re-run would be deduped away — and the + # errored path must not accidentally admit it either. + self.assertFalse(self.eligible([BASE_CHANGED], False)) + self.assertFalse(self.eligible([BASE_CHANGED], True)) + + # --- the new path --- + + def test_an_errored_panel_needs_the_flag(self): + self.assertFalse(self.eligible([self.ERRORED], False)) + self.assertTrue(self.eligible([self.ERRORED], True)) + + def test_errored_alongside_a_race_is_eligible(self): + self.assertTrue(self.eligible([self.ERRORED, HEAD_MOVED], True)) + self.assertTrue(self.eligible([self.ERRORED, BASE_CHANGED], True)) + + def test_a_cause_a_retry_cannot_fix_rules_it_out(self): + # The important negative: the next round hits the same wall. + self.assertFalse(self.eligible([self.ERRORED, STRUCTURAL], True)) + self.assertFalse(self.eligible([STRUCTURAL], True)) + self.assertFalse( + self.eligible([self.ERRORED, "1 finding above the threshold"], True) + ) + + def test_the_panel_inconsistent_cause_is_not_retried(self): + # A forger forges again; a human reading the red legs can tell. + self.assertFalse( + self.eligible([self.ERRORED, auto_approve.REASON_PANEL_INCONSISTENT], True) + ) + + def test_a_round_that_blocks_anyway_is_not_retried(self): + # The completed cells (or an open thread) already block: a full panel + # on the same head would only confirm it. + self.assertFalse(auto_approve.auto_retry_eligible( + [self.ERRORED], retry_errored=True, blocked_anyway=True)) + # ...but a moved head re-reviews different content. + self.assertTrue(auto_approve.auto_retry_eligible( + [HEAD_MOVED], retry_errored=True, blocked_anyway=True)) + + def test_blocking_without_trust_reads_the_real_findings(self): + # Fed real findings, not a hand-written reason the producer never emits. + block = auto_approve.blocking_without_trust + self.assertTrue(block("low", [{"severity": "high"}], [], None)) + self.assertTrue(block("low", [], ["high"], None)) + self.assertFalse(block("low", [{"severity": "low"}], [], None)) + self.assertFalse(block("low", [], [], None)) + + def test_no_reasons_is_not_eligible(self): + for reasons in ([], None, [""], [" "]): + with self.subTest(reasons=reasons): + self.assertFalse(self.eligible(reasons, True)) + + +class BudgetTest(unittest.TestCase): + """The opt-in has to reach the budget, and the budget still fails closed.""" + + class Args: + def __init__(self, **kw): + self.can_relabel = "true" + self.approver_login = "bot[bot]" + self.max_rounds = "5" + self.round = "1" + self.retry_errored_rounds = "true" + self.repo = "o/r" + self.pr_number = "1" + self.__dict__.update(kw) + + ERRORED = "2/6 " + auto_approve.PANEL_INCOMPLETE_SUFFIX + + def setUp(self): + self.calls = [] + self._real = auto_approve.list_reviews + auto_approve.list_reviews = lambda repo, pr: self.calls.append((repo, pr)) or [] + self.addCleanup(setattr, auto_approve, "list_reviews", self._real) + + def test_the_flag_gates_the_errored_cause(self): + self.assertTrue( + auto_approve.auto_retry_budget_left(self.Args(), [self.ERRORED]) + ) + self.assertFalse( + auto_approve.auto_retry_budget_left( + self.Args(retry_errored_rounds="false"), [self.ERRORED] + ) + ) + + def test_only_an_explicit_true_enables_it(self): + for value in ("", "yes", "1", "TRUE ", None): + with self.subTest(value=value): + expected = str(value).strip().lower() == "true" + self.assertEqual( + expected, + auto_approve.auto_retry_budget_left( + self.Args(retry_errored_rounds=value), [self.ERRORED] + ), + ) + + def test_a_token_that_cannot_relabel_never_retries(self): + self.assertFalse( + auto_approve.auto_retry_budget_left( + self.Args(can_relabel="false"), [self.ERRORED] + ) + ) + + def test_the_last_allowed_round_does_not_retry(self): + # The retry would cap out and spend the budget on nothing. + self.assertFalse( + auto_approve.auto_retry_budget_left( + self.Args(round="5", max_rounds="5"), [self.ERRORED] + ) + ) + + def test_a_spent_budget_does_not_retry(self): + marked = [{"user": {"login": "bot[bot]"}, + "body": "x\n" + auto_approve.AUTO_RETRY_MARKER}] + auto_approve.list_reviews = lambda repo, pr: marked + self.assertFalse( + auto_approve.auto_retry_budget_left(self.Args(), [self.ERRORED]) + ) + + +class ConstantParityTest(unittest.TestCase): + """The gate hardcodes the marker; it must stay equal to the Python one.""" + + def test_the_workflow_uses_the_same_marker(self): + _block, run = step_run("Check for prior review on this commit") + self.assertIn(f"--arg marker '{auto_approve.AUTO_RETRY_MARKER}'", run) + self.assertIn(f"--arg approve '{auto_approve.APPROVE_MARKER}'", run) + + def test_the_marker_is_an_html_comment(self): + # It rides in a review body a human reads; it must not be visible. + self.assertTrue(auto_approve.AUTO_RETRY_MARKER.startswith("")) + + def test_one_retry_per_pr(self): + self.assertEqual(1, auto_approve.MAX_AUTO_RETRIES) + + +class GateWiringTest(unittest.TestCase): + """The two force paths stay separate, and the retry path is opt-in.""" + + @classmethod + def setUpClass(cls): + cls.block, cls.body = step_run("Check for prior review on this commit") + + def test_the_input_reaches_the_step(self): + self.assertIn( + "AUTO_RETRY_ERRORED: ${{ inputs.auto_retry_errored_rounds }}", self.block + ) + + def test_the_retry_path_is_off_unless_enabled(self): + self.assertIn('[ "${AUTO_RETRY_ERRORED:-}" = "true" ]', self.body) + + def test_the_retry_cannot_use_the_human_comment_path(self): + # The human path scans PR comments and authorizes by association. Its + # guard must depend on REREVIEW_COMMAND alone: if enabling auto-retry + # could open that query, any App's comment could force rounds. + lines = self.body.splitlines() + idx = next(i for i, ln in enumerate(lines) if "FORCED=$(gh api" in ln) + guard = next( + ln for ln in reversed(lines[:idx]) + if ln.strip().startswith("if ") and ln.rstrip().endswith("then") + ) + self.assertIn("REREVIEW_COMMAND", guard) + self.assertNotIn("AUTO_RETRY_ERRORED", guard) + + def test_the_marker_check_is_permission_bounded(self): + marker = self.body[self.body.index("RETRIED="):] + self.assertIn('.user.type == "Bot"', marker) + self.assertIn('(.user.login // "") == $sender', marker) + self.assertIn(".commit_id == $sha", marker) + self.assertIn('.state != "DISMISSED"', marker) + + def test_the_sender_is_the_event_sender(self): + self.assertIn("EVENT_SENDER: ${{ github.event.sender.login }}", self.block) + + def test_a_second_marked_block_does_not_suppress_the_retry(self): + # An upper bound here only ever turned a real retry into a dedupe. + self.assertIn('[ "$RETRIED" -gt 0 ]', self.body) + self.assertNotIn('-le 1', self.body) + + def test_a_non_numeric_count_cannot_force(self): + self.assertIn("case \"$RETRIED\" in ''|*[!0-9]*) RETRIED=0 ;; esac", self.body) + + +class DelayTest(unittest.TestCase): + """The wait: clamped, and never a reason the job fails.""" + + @classmethod + def setUpClass(cls): + cls.block, cls.body = step_run("Wait before re-running") + + def test_it_only_runs_when_a_delay_is_configured(self): + self.assertIn("inputs.auto_retry_delay_minutes != 0", self.block) + + def test_it_runs_before_the_relabel(self): + text = workflow_text() + self.assertLess(text.index("- name: Wait before re-running"), + text.index("- name: Remove and re-add the review label")) + + def test_the_job_timeout_covers_the_longest_wait(self): + # The bug this pins: the relabel job budgeted 5 minutes, so any wait + # past that killed the round the card had already promised — the one + # failure mode where announcing the retry makes things worse than not + # retrying at all. + text = workflow_text() + job = text[text.index(" auto-retry:"):] + minutes = int(re.search(r"timeout-minutes: (\d+)", job).group(1)) + self.assertGreaterEqual(minutes, card.RETRY_DELAY_CAP) + + @unittest.skipIf(shutil.which("bash") is None, "bash not installed") + def test_the_clamp(self): + for given, want_sleep in (("0", 0), ("5", 300), ("30", 1800), + ("31", 1800), ("999", 1800), ("1000", 1800), + ("99999999999999999999999", 1800), + ("abc", 0), ("", 0), ("-5", 0), ("1.5", 0)): + with self.subTest(given=given): + # Run the real body with `sleep` stubbed, so the test is fast + # and the computed duration is observable. + script = self.body.replace('sleep "$(( mins * 60 ))"', + 'echo "SLEPT=$(( mins * 60 ))"') + proc = subprocess.run( + ["bash", "-c", script], capture_output=True, text=True, + env={**os.environ, "DELAY_MINUTES": given}, + ) + self.assertEqual(0, proc.returncode, proc.stderr) + if want_sleep: + self.assertIn(f"SLEPT={want_sleep}", proc.stdout) + else: + self.assertNotIn("SLEPT=", proc.stdout) + self.assertIn("immediately", proc.stdout) + + +class CardDelayTest(unittest.TestCase): + """The advance notice: the card says the wait before the wait starts.""" + + LABEL = "cursor-review" + + def test_no_delay_leaves_the_sentence_exactly_as_it_was(self): + # Every caller that has not set the input reads this text, and + # cursor-approve has no such input at all — so the zero case is a + # compatibility assertion, not a formatting preference. + self.assertEqual( + card.next_auto_retry_text(self.LABEL), + card.next_auto_retry_text(self.LABEL, 0), + ) + self.assertNotIn("in about", card.next_auto_retry_text(self.LABEL, 0)) + + def test_a_configured_delay_is_named(self): + text = card.next_auto_retry_text(self.LABEL, 5) + self.assertIn("in about 5 minutes", text) + # Still the same next step, so an agent reading the card does not + # conclude a human has to act. + self.assertIn("being re-run automatically", text) + self.assertIn("remove and re-add the `cursor-review` label", text) + + def test_one_minute_is_singular(self): + self.assertIn("in about 1 minute (", card.next_auto_retry_text(self.LABEL, 1)) + + def test_the_phrase_is_clamped_like_the_workflow(self): + for given in (31, 999, 10 ** 9): + with self.subTest(given=given): + self.assertEqual(" in about 30 minutes", + card.retry_delay_phrase(given)) + + def test_an_unusable_delay_promises_no_wait(self): + # The value is a caller input; it must not reach the sentence raw, and + # a wait the workflow will not take must not be announced. + for given in ("abc", "", None, -5, 0, "1.5", 0.5, [], "5; rm -rf /"): + with self.subTest(given=given): + self.assertEqual("", card.retry_delay_phrase(given)) + + def test_the_card_cap_equals_the_workflow_cap(self): + _block, body = step_run("Wait before re-running") + self.assertIn(f'-gt {card.RETRY_DELAY_CAP} ]; then mins={card.RETRY_DELAY_CAP}', + body) + + def test_the_decide_step_passes_the_input(self): + text = workflow_text() + self.assertIn("RETRY_DELAY_MINUTES: ${{ inputs.auto_retry_delay_minutes }}", text) + self.assertIn('--retry-delay-minutes "$RETRY_DELAY_MINUTES"', text) + + def test_the_text_helper_forwards_the_delay(self): + seen = {} + + class Stub: + @staticmethod + def next_auto_retry_text(label, delay_minutes=0): + seen.update(label=label, delay=delay_minutes) + return "x" + + real = auto_approve._load_card + auto_approve._load_card = lambda: Stub + try: + out = auto_approve._card_text("next_auto_retry_text", None, delay_minutes=7) + finally: + auto_approve._load_card = real + self.assertEqual("x", out) + self.assertEqual(7, seen["delay"]) + + def test_an_older_card_without_the_keyword_degrades_to_empty(self): + # The call site keeps its own fallback text, so a signature mismatch + # must not raise out of the decision. + class Stub: + @staticmethod + def next_auto_retry_text(label): + return "x" + + real = auto_approve._load_card + auto_approve._load_card = lambda: Stub + try: + self.assertEqual( + "", auto_approve._card_text("next_auto_retry_text", None, delay_minutes=7)) + finally: + auto_approve._load_card = real + + +@unittest.skipIf(shutil.which("jq") is None, "jq not installed") +class MarkerFilterTest(unittest.TestCase): + """The real jq program for the retry signal, extracted and executed.""" + + SHA = "abc123" + SINCE = 100 # the newest consolidated review's id + SENDER = "approver-pat-user" + + @classmethod + def setUpClass(cls): + _block, run = step_run("Check for prior review on this commit") + seg = run[run.index("RETRIED=$(printf"):] + start = seg.index("'[.[]") + 1 + cls.filter = seg[start:seg.index("' \\", start)] + + def block(self, extra=""): + """decide's own shape: the auto-approve marker first, the retry marker + on a line of its own (render_standing_body).""" + return (auto_approve.APPROVE_MARKER + "\n" + auto_approve.AUTO_RETRY_MARKER + + "\n### 🤖 Cursor Review — auto-approve" + extra) + + def review(self, body, user_type="Bot", login="app[bot]", sha=None, rid=101, + state="CHANGES_REQUESTED"): + return {"id": rid, "commit_id": sha or self.SHA, "state": state, + "user": {"type": user_type, "login": login}, "body": body} + + def count(self, reviews, sender=SENDER): + proc = subprocess.run( + ["jq", "--arg", "sha", self.SHA, "--argjson", "since", str(self.SINCE), + "--arg", "sender", sender, + "--arg", "approve", auto_approve.APPROVE_MARKER, + "--arg", "marker", auto_approve.AUTO_RETRY_MARKER, self.filter], + input=json.dumps(reviews), capture_output=True, text=True, + ) + self.assertEqual(0, proc.returncode, proc.stderr) + return int(proc.stdout.strip()) + + def test_the_real_standing_block_counts(self): + # Rendered by the producer, so the gate and decide cannot drift. + body = auto_approve.render_standing_body( + "No decision.", ["1/6 " + auto_approve.PANEL_INCOMPLETE_SUFFIX], "Nothing.", + "low", "a" * 40, "main", retry=True) + self.assertEqual(1, self.count([self.review(body)])) + unmarked = auto_approve.render_standing_body( + "No decision.", ["x"], "Nothing.", "low", "a" * 40, "main", retry=False) + self.assertEqual(0, self.count([self.review(unmarked)])) + + def test_an_older_marked_block_does_not(self): + # This is what stops one request re-firing forever. + self.assertEqual(0, self.count([self.review(self.block(), rid=99)])) + + def test_the_same_second_does_not_lose_the_request(self): + # Ordered by id, so a block posted in the consolidated review's own + # second still counts. + self.assertEqual(1, self.count([self.review(self.block(), rid=self.SINCE + 1)])) + + def test_another_commit_does_not_count(self): + self.assertEqual(0, self.count([self.review(self.block(), sha="other")])) + + def test_the_approver_token_identity_counts_when_it_relabelled(self): + # APPROVER_TOKEN is a PAT: decide's block is User-typed, and the + # auto-retry job relabels as that same user. + self.assertEqual(1, self.count([ + self.review(self.block(), user_type="User", login=self.SENDER)])) + + def test_another_human_cannot_post_it(self): + self.assertEqual(0, self.count([ + self.review(self.block(), user_type="User", login="someone-else")])) + self.assertEqual(0, self.count([ + self.review(self.block(), user_type="User", login="")], sender="")) + + def test_a_dismissed_block_does_not_count(self): + self.assertEqual(0, self.count([self.review(self.block(), state="DISMISSED")])) + + def test_a_quoted_marker_does_not_count(self): + # A review that merely mentions the marker — quoting this very diff — + # is not decide's block. + self.assertEqual(0, self.count([self.review( + "## 🔍 Cursor Review — Consolidated panel\nsee `" + auto_approve.AUTO_RETRY_MARKER + "`")])) + self.assertEqual(0, self.count([self.review( + auto_approve.APPROVE_MARKER + "\nquote: " + auto_approve.AUTO_RETRY_MARKER)])) + + def test_two_marked_blocks_still_count(self): + self.assertEqual(2, self.count([self.review(self.block(), rid=101), + self.review(self.block(), rid=102)])) + + def test_an_unmarked_bot_review_does_not_count(self): + self.assertEqual(0, self.count([self.review("an ordinary block")])) + + def test_a_null_body_or_id_is_not_an_error(self): + self.assertEqual(0, self.count([self.review(None)])) + self.assertEqual(0, self.count([self.review(self.block(), rid=None)])) + + def test_an_empty_list_is_not_an_error(self): + self.assertEqual(0, self.count([])) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/.github/workflows/cursor-review.yml b/.github/workflows/cursor-review.yml index b11a3046..95f74b49 100644 --- a/.github/workflows/cursor-review.yml +++ b/.github/workflows/cursor-review.yml @@ -272,6 +272,40 @@ on: type: string required: false default: '' + auto_retry_errored_rounds: + description: >- + Opt-in: re-run a round AUTOMATICALLY when it could not decide because + reviewers errored — a capacity error, a cell timeout — rather than + because of anything in the diff. Default false. The round is re-run + once per PR (the existing auto-retry ceiling), the status card says so + before it happens, and `max_rounds` still applies; a round that failed + for a structural reason, or that found something above the threshold, + is never re-run because the next round would hit the same wall. + Independent of `rereview_command`, which is the HUMAN path: that one + keys on an authorized comment, and the retry posts no comment. The + retry's own force signal is the marked block decide already writes + (`AUTO_RETRY_MARKER`), which the gate accepts when it is newer than + the newest consolidated review for this commit and was posted by a + Bot or by the identity that applied the label — so one marked block + buys exactly one round. No effect without `approve_max_severity`, + and needs `bot_app_id` or APPROVER_TOKEN to relabel. + type: boolean + required: false + default: false + auto_retry_delay_minutes: + description: >- + How long to wait before an automatic retry relabels, in minutes. + Default 0 (immediately), which is the behaviour every existing caller + has. Worth setting for `auto_retry_errored_rounds`, because the + failures it retries are usually provider capacity and a few minutes + is what makes the second attempt different from the first. The wait + happens on a runner, so it costs runner-minutes on a private repo — + that is the whole cost of this feature. Clamped to 30 rather than + holding a runner all day; anything unparsable waits not at all. The + status card names the wait before it starts. + type: number + required: false + default: 0 run_without_label: description: >- Run on plain pull_request events (opened / reopened / ready_for_review @@ -965,6 +999,11 @@ jobs: REPO: ${{ github.repository }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} REREVIEW_COMMAND: ${{ inputs.rereview_command }} + AUTO_RETRY_ERRORED: ${{ inputs.auto_retry_errored_rounds }} + # Who fired this event: on a relabel, the identity that applied the + # label. The auto-retry relabels as the same identity decide posted + # its marked block under, so the two logins match (see below). + EVENT_SENDER: ${{ github.event.sender.login }} run: | # Idempotency: skip the matrix if a Cursor Review consolidated # review already exists for this exact HEAD SHA. Protects against @@ -1038,18 +1077,79 @@ jobs: *[[:space:]]*) echo "::warning::rereview_command contains whitespace, so it can never match a comment's first token — use a single token such as /review." ;; esac - if [ "$EXISTING" -gt 0 ] && [ -n "${REREVIEW_COMMAND:-}" ]; then + # Two force signals, deliberately separate. A HUMAN asks with + # `rereview_command` and is authorized by the comment's association. + # The workflow's own auto-retry posts no comment at all — it writes + # the marked block below — and must not be able to reach the human + # path, or any App comment could force rounds. + LAST_REVIEW="" + if [ "$EXISTING" -gt 0 ] \ + && { [ -n "${REREVIEW_COMMAND:-}" ] || [ "${AUTO_RETRY_ERRORED:-}" = "true" ]; }; then LAST_REVIEW=$(printf '%s' "$REVIEWS" \ | jq -r --arg sha "$HEAD_SHA" '[.[] | select(.commit_id == $sha and .state != "DISMISSED" and (.user.type == "Bot") and (.body | startswith("## 🔍 Cursor Review — Consolidated panel"))) | .submitted_at] | max // ""' \ 2>/dev/null || echo "") - if [ -n "$LAST_REVIEW" ]; then - FORCED=$(gh api --paginate "repos/$REPO/issues/$PR_NUMBER/comments" 2>/dev/null \ - | jq -s --arg cmd "$REREVIEW_COMMAND" --arg since "$LAST_REVIEW" \ - '[.[][] | select(.created_at > $since) - | select(.author_association == "OWNER" or .author_association == "MEMBER" or .author_association == "COLLABORATOR") - | select(((.body // "") | split("\n")[0] // "" | sub("\r$"; "") | sub("^[ \t]+"; "") | sub("[ \t].*$"; "")) == $cmd)] | length' \ - 2>/dev/null || echo 0) - case "$FORCED" in ''|*[!0-9]*) FORCED=0 ;; esac + fi + + if [ -n "$LAST_REVIEW" ] && [ -n "${REREVIEW_COMMAND:-}" ]; then + FORCED=$(gh api --paginate "repos/$REPO/issues/$PR_NUMBER/comments" 2>/dev/null \ + | jq -s --arg cmd "$REREVIEW_COMMAND" --arg since "$LAST_REVIEW" \ + '[.[][] | select(.created_at > $since) + | select(.author_association == "OWNER" or .author_association == "MEMBER" or .author_association == "COLLABORATOR") + | select(((.body // "") | split("\n")[0] // "" | sub("\r$"; "") | sub("^[ \t]+"; "") | sub("[ \t].*$"; "")) == $cmd)] | length' \ + 2>/dev/null || echo 0) + case "$FORCED" in ''|*[!0-9]*) FORCED=0 ;; esac + fi + + # The workflow's own retry. `decide` writes a block carrying + # AUTO_RETRY_MARKER once it has decided to re-run the round, and the + # `auto-retry` job relabels afterwards — so a marked block NEWER than + # the newest consolidated review for this commit IS the request, and + # it buys exactly one round: the round it starts posts a newer + # review, after which the marker is older than that review and counts + # for nothing. + # + # Newer by review `id`, not `submitted_at`: ids increase + # monotonically, while timestamps have one-second precision and decide + # posts its block right after the consolidated review, so the two can + # share a second. + # + # Who may have written it. decide posts under APPROVER_TOKEN when one + # is set — a PAT, so `user.type` is "User" — else under the bot App. + # This job holds neither token, so it cannot learn the approver login + # (the credential split's point). It accepts the block when its author + # is a Bot (the App case, and the residual the dedupe above already + # accepts: another installed App), or when its author is the identity + # that fired THIS event — the PAT case, since `auto-retry` relabels as + # the same identity decide posted under. A human can satisfy the + # second only by relabelling themselves, which already needs label + # write on the repo. The block must also be decide's own shape (it + # starts with the auto-approve marker, and carries the retry marker as + # a line of its own, not quoted inside other text) and not DISMISSED. + # + # What bounds a forged marker is `max_rounds` and who can post one — + # not this check, which counts a request rather than capping retries; + # decide's own one-per-PR budget caps the retries it starts. + # + # The markers are duplicated from auto-approve.py's APPROVE_MARKER + # and AUTO_RETRY_MARKER on purpose — the gate runs before any asset is + # loaded — and a test pins them equal, as the relabel's label constant + # already is. + if [ "$FORCED" -eq 0 ] && [ -n "$LAST_REVIEW" ] && [ "${AUTO_RETRY_ERRORED:-}" = "true" ]; then + LAST_REVIEW_ID=$(printf '%s' "$REVIEWS" \ + | jq -r --arg sha "$HEAD_SHA" '[.[] | select(.commit_id == $sha and .state != "DISMISSED" and (.user.type == "Bot") and (.body | startswith("## 🔍 Cursor Review — Consolidated panel"))) | .id | numbers] | max // 0' \ + 2>/dev/null || echo 0) + case "$LAST_REVIEW_ID" in ''|*[!0-9]*) LAST_REVIEW_ID=0 ;; esac + RETRIED=$(printf '%s' "$REVIEWS" \ + | jq --arg sha "$HEAD_SHA" --argjson since "$LAST_REVIEW_ID" \ + --arg sender "${EVENT_SENDER:-}" \ + --arg approve '' \ + --arg marker '' \ + '[.[] | select(.commit_id == $sha and .state != "DISMISSED" and ((.id // 0) | numbers) > $since and ((.user.type == "Bot") or ($sender != "" and (.user.login // "") == $sender)) and ((.body // "") | startswith($approve)) and ((.body // "") | split("\n") | any(. == $marker)))] | length' \ + 2>/dev/null || echo 0) + case "$RETRIED" in ''|*[!0-9]*) RETRIED=0 ;; esac + if [ "$LAST_REVIEW_ID" -gt 0 ] && [ "$RETRIED" -gt 0 ]; then + FORCED=1 + echo "An automatic retry was requested after the last round — proceeding." fi fi @@ -5796,6 +5896,8 @@ jobs: APPROVE_SCOPE: ${{ inputs.approve_scope }} INCREMENTAL_STATE: ${{ needs.diff-size.outputs.incremental_state }} MAX_FAILED_REVIEWERS: ${{ inputs.approve_max_failed_reviewers }} + RETRY_ERRORED_ROUNDS: ${{ inputs.auto_retry_errored_rounds }} + RETRY_DELAY_MINUTES: ${{ inputs.auto_retry_delay_minutes }} # aggregate-panel.py's step output, written before the judge ran — # not consolidated.json, which `consolidate` builds after the judge # (`--trust`, same job) could have rewritten its inputs. @@ -5864,7 +5966,9 @@ jobs: --max-rounds "$MAX_ROUNDS" \ --run-url "$RUN_URL" \ --review-label "$REVIEW_LABEL" \ - --can-relabel "$CAN_RELABEL" + --can-relabel "$CAN_RELABEL" \ + --retry-errored-rounds "$RETRY_ERRORED_ROUNDS" \ + --retry-delay-minutes "$RETRY_DELAY_MINUTES" dismiss-stale-approval: # Companion to `Auto-approve decision`. A ruleset with @@ -6899,11 +7003,15 @@ jobs: # `!cancelled()` so a skipped or red one does not skip the retry; a push that # cancels this run before it gets here leaves the card's "if no new round # starts, re-run it by hand" as the recovery. - name: Re-run the round (head moved) + name: Re-run the round needs: [post-review, panel-integrity, blocking-gate, notify-complete] if: ${{ !cancelled() && needs.post-review.outputs.auto_retry == 'true' }} runs-on: ubuntu-latest - timeout-minutes: 5 + # 5 for the relabel itself, plus the 30 `auto_retry_delay_minutes` is + # clamped to: a job killed mid-wait would leave the card promising a round + # that never starts. A ceiling, not a reservation — a caller that set no + # delay still finishes in seconds. + timeout-minutes: 35 permissions: contents: read steps: @@ -6980,6 +7088,38 @@ jobs: app-id: ${{ inputs.bot_app_id }} private-key: ${{ secrets.BOT_APP_PRIVATE_KEY }} + - name: Wait before re-running + # The failures this retries are overwhelmingly provider capacity, so a + # few minutes is what makes the second attempt different from the + # first. Default 0 keeps every existing caller's behaviour. + # + # The wait is a sleeping runner, which is this feature's whole cost on + # a private repo. Clamped to 30 minutes, and anything unparsable is + # treated as 0 rather than holding a runner on a typo. + # + # The wait widens the window in which this job can be cancelled: a + # caller whose concurrency group is keyed on the PR alone, with + # `cancel-in-progress: true`, loses the pending retry to ANY event on + # the PR during the sleep — a label, an edit — not only a push. The + # card's "if no new round starts, re-run it by hand" is the recovery; + # docs/callers/cursor-review.md says so. + if: ${{ inputs.auto_retry_delay_minutes != 0 }} + env: + DELAY_MINUTES: ${{ inputs.auto_retry_delay_minutes }} + run: | + mins="${DELAY_MINUTES:-0}" + case "$mins" in ''|*[!0-9]*) mins=0 ;; esac + # More than three digits is over the cap whatever it says, and a long + # enough one overflows bash's integers before the test below can clamp it. + if [ "${#mins}" -gt 3 ]; then mins=30; fi + if [ "$mins" -gt 30 ]; then mins=30; fi + if [ "$mins" -eq 0 ]; then + echo "No usable retry delay — re-running immediately." + exit 0 + fi + echo "Waiting ${mins} minute(s) before re-running the round, so a transient provider failure has time to clear." + sleep "$(( mins * 60 ))" + - name: Remove and re-add the review label env: # The token decide held (same precedence), never GITHUB_TOKEN: a diff --git a/docs/callers/cursor-approve.md b/docs/callers/cursor-approve.md index a8f50a36..21cc4b4b 100644 --- a/docs/callers/cursor-approve.md +++ b/docs/callers/cursor-approve.md @@ -163,14 +163,21 @@ The workflow now does it itself: it removes and re-adds the review label, **once per PR**, and the card says so instead of asking you to. The next step stays `relabel` on the contract marker, so an agent reading it waits for the new round either way. decide only decides; the relabel is the run's LAST job -(`Re-run the round (head moved)`), after the Blocking gate, Panel integrity and +(`Re-run the round`), after the Blocking gate, Panel integrity and completion jobs, because the label events it fires cancel whatever of the run is still going in the caller's `cancel-in-progress` group. -It fires only when a fresh round would actually decide differently — the head -moved (a base retarget alongside it rides along). Every other transient cause -leaves the head where it was, and the gate's same-SHA `dup` check would skip the -re-run, so those still ask for a human. The budget is counted off the PR's own +It fires only when a fresh round would actually decide differently. By default +that means the head moved (a base retarget alongside it rides along): every +other transient cause leaves the head where it was, and the gate's same-SHA +`dup` check would skip the re-run, so those ask for a human. A cursor-review +caller can add one more cause with +[`auto_retry_errored_rounds`](cursor-review.md#retrying-a-round-the-panel-broke) +— a round that could not decide because reviewer cells **errored**, which is +retryable on the same commit because the gate lifts its dedupe for the marked +block (and which can wait first, `auto_retry_delay_minutes`, the card naming +the wait). Both causes share the ONE budget below, so a PR gets one automatic +retry, not one each. The budget is counted off the PR's own reviews — each retry's standing block carries `` and still counts once dismissed — so it holds across rounds and across runs; a retry whose block did not post is never fired, since nothing would count it. An diff --git a/docs/callers/cursor-review.md b/docs/callers/cursor-review.md index c10f37ee..767b1a7d 100644 --- a/docs/callers/cursor-review.md +++ b/docs/callers/cursor-review.md @@ -140,6 +140,8 @@ pull-requests: write # posting the consolidated review (and, at the round cap, | `defer_approval` | `false` | Only for a caller that runs [cursor-approve](cursor-approve.md) after this workflow: a round that would approve still reports `approve_gate` = `pass`, but posts **no** approval and resolves no thread, and withdraws the approver's own earlier auto-approvals and requests for changes instead — so cursor-approve's decide is the only approver. Requests changes as usual. No effect without `approve_max_severity`. | | `approve_max_failed_reviewers` | `0` | How many panel reviewers may **error** (the cell ran but failed, timed out, or never uploaded) before auto-approve withholds its decision. `0` keeps the strict rule: any reviewer that did not complete means no decision. `N` tolerates up to N errored reviewers and names them in the decision (e.g. "approved with 1/6 reviewers errored: `:edge-case`"). See [auto-approve](#auto-approve). No effect without `approve_max_severity`. | | `rereview_command` | `''` (off) | A comment command, e.g. `/review`, that asks for ANOTHER round on the **same commit**. Empty leaves the same-commit dedupe absolute. Set the same value on [`cursor-review-auto-label.yml`](cursor-review-auto-label.md) — that half turns the comment into the label event, this half stops the dedupe from skipping the round it starts; setting only one is a silent no-op. Authorized = an `OWNER` / `MEMBER` / `COLLABORATOR` association on the comment. One request buys one round, and `max_rounds` still applies. See [below](#asking-for-another-round-on-the-same-commit). | +| `auto_retry_errored_rounds` | `false` | Re-run a round AUTOMATICALLY when it could not decide because **reviewers errored** — provider capacity, a cell timeout — rather than because of anything in the diff. The status card and the standing block say so before it happens, naming the wait. Once per PR (the existing auto-retry ceiling), and `max_rounds` still applies. A round that failed for a structural reason, or that found something above the threshold, is never re-run: the next round would hit the same wall. Needs a token that can relabel — the `bot_app_id` App token or `APPROVER_TOKEN`, not `GITHUB_TOKEN`, whose label edits fire no run. Independent of `rereview_command`: the retry posts no comment and is not authorized by one. No effect without `approve_max_severity`. See [below](#retrying-a-round-the-panel-broke). | +| `auto_retry_delay_minutes` | `0` | How long an automatic retry waits before relabelling, in minutes — provider capacity is the usual cause, and a few minutes is what makes the second attempt different from the first. `0` relabels immediately, which is what every caller had before this input. The wait happens on a runner, so it costs runner-minutes on a private repo; that is the whole cost. Clamped to `30`; anything unparsable waits not at all. No effect without `approve_max_severity`. | | `max_rounds` | `5` | Cap on review rounds per PR (`0` → no cap). At or over it, no panel runs: the PR is labelled `needs-human-review`, one comment lists the latest round's open findings above the threshold, and the `approve_gate` output is `capped`. Removing the label resets the count. See [round cap](#round-cap-and-the-approve_gate-output). | | `runs_on` | `'"ubuntu-latest"'` | JSON-encoded `runs-on` for `diff-size`, `preflight`, the `review` panel and `consolidate` only — the jobs that hold no write credential. Every other job stays on GitHub-hosted `ubuntu-latest`. A self-hosted pool is fine only if it is one-job-per-fresh-pod, identity-free and private-network-isolated (those jobs run models with shell over PR code), on linux/x64 with bash, git, curl, jq, python3, gh, tar and GNU coreutils. Empty falls back to the default, so `${{ vars.CURSOR_REVIEW_RUNS_ON }}` is safe while the variable is unset; e.g. `'["self-hosted", "linux", "x64"]'`. | @@ -296,6 +298,86 @@ Details worth knowing: - Nothing from the comment reaches a log: a comment body is attacker-authored text, and these logs may be public. +## Retrying a round the panel broke + +`rereview_command` above is the hand version of this: a maintainer notices the +round failed for a reason that is not about the code, and asks for another. With +`auto_retry_errored_rounds` the workflow asks for itself. + +```yaml +jobs: + cursor-review: + uses: Comfy-Org/github-workflows/.github/workflows/cursor-review.yml@ # v1 + with: + workflows_ref: + approve_max_severity: low # the retry is auto-approve's decision: off without it + bot_app_id: ${{ vars.REVIEW_BOT_APP_ID }} # a token that can relabel (or set APPROVER_TOKEN) + auto_retry_errored_rounds: true + auto_retry_delay_minutes: 5 + secrets: inherit +``` + +What happens, in order: + +1. The round ends with no decision because reviewer cells **errored** — a + capacity error from a provider, a cell that timed out, an artifact that never + arrived. Nothing in the diff caused it. +2. `decide` posts its standing block and status card saying the round is being + re-run automatically, **naming the wait** (`in about 5 minutes`). Said in + advance on purpose: an author who reads "re-run automatically" and then sees + nothing happen for five minutes re-runs it by hand, which is the work this + input exists to remove. +3. The `auto-retry` job waits, then removes and re-adds the review label — the + same relabel the maintainer would have done. +4. The gate lets that round through its same-commit dedupe, because `decide`'s + block carries the auto-retry marker, is newer than the newest consolidated + review for this commit, and was posted by a Bot (the App identity) or by the + identity that applied the label (the `APPROVER_TOKEN` identity, which the + `auto-retry` job relabels as). + +### What bounds it + +- **Once per PR.** The retry is counted by the marker on `decide`'s own block, + the budget the moved-head retry (`BE-19526`) already used — so a PR gets one + automatic retry across both causes, not one each. If the second round fails + too, the card goes back to the hand recovery. +- **`max_rounds` still applies**, and a round on the last allowed number is not + retried at all rather than retried into the cap. +- **Only a cause a retry can fix.** A structural failure (the findings did not + land as threads, the reviewed diff was empty) or findings above the threshold + are never retried: the next round would hit the same wall. That includes + findings above the threshold from the cells that DID complete, and an open + thread above it from an earlier round. Mixed causes follow the strictest one. +- **A token that can actually relabel.** With the `GITHUB_TOKEN` fallback the + retry is skipped, because a label applied by `GITHUB_TOKEN` fires no workflow + run — it would stop the round rather than restart it. Set `bot_app_id` / + `BOT_APP_PRIVATE_KEY`, or `APPROVER_TOKEN`. +- **No block, no retry.** If the standing block does not land, nothing carries + the marker, so nothing counts the retry — and it is skipped rather than fired + uncounted. + +### Notes + +- It is **independent of `rereview_command`**: the retry posts no comment, and + the comment path stays authorized by comment association alone. The retry's + own signal is accepted from a Bot, or from whoever applied the label — so + beyond the Apps installed on the repo, only someone who can already edit + labels can use it. +- Under `APPROVER_TOKEN`, a **hand** relabel on the same commit is still + deduped (the block's author is not the person relabelling); use + `rereview_command` for that. +- The delay burns runner-minutes while it sleeps — the whole cost of the + feature, and the reason the input is off by default. Clamped to 30 minutes. +- A push during the wait cancels the retry along with the rest of the run, which + is correct: the new head starts its own round. But so does **any** event that + lands in your concurrency group while it waits, if that group is keyed on the + PR alone with `cancel-in-progress: true` — a label, a title edit, a thread + event. The card has already said the round is being re-run, so if no new + round starts, re-run it by hand; keep the delay short, or key the group so + unrelated events do not cancel it. +- A PR closed, merged or labelled `needs-human-review` during the wait gets no + retry: the job re-reads the PR before relabelling. + ## Gotchas **The label must be applied by a GitHub App token, not `GITHUB_TOKEN`.** Events