diff --git a/.github/cursor-approve/tests/test_caller_doc_pattern.py b/.github/cursor-approve/tests/test_caller_doc_pattern.py new file mode 100644 index 00000000..841e12d7 --- /dev/null +++ b/.github/cursor-approve/tests/test_caller_doc_pattern.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Pins the `if:` guards on the caller pattern in docs/callers/cursor-approve.md. + +cursor-review's called job goes RED whenever a panel cell errors: the leg fails +by design ("Fail the leg when the cell did not submit") and `Panel integrity` +fails alongside it. Neither is affected by `approve_max_failed_reviewers`, which +governs the `approve_gate` VERDICT and not the job's conclusion. So a caller +that chains the start phase and the axes off that job with a bare `needs:` has +them skipped by GitHub's default rule, and decide — which runs on `always()` — +reports every axis as "no result" and withholds. + +That is not hypothetical: it is what the published pattern did. One errored +`edge-case` cell out of six took the axes out of a real run on a caller that had +set `approve_max_failed_reviewers: 1` precisely to tolerate it, and across the +81 preceding panel runs on that repo the axes had never once run. + +A copied caller is as good as its snippet, and `!cancelled()` reads like a +stylistic flourish to anyone editing it later — a bare `if:` here parses, lints, +and silently withholds every approval. So the guards are pinned. + +The guards are pinned as EXACT top-level conjuncts, not substrings: a `||` in +place of an `&&`, an `approve_gate != 'pass'`, or a bare truthy +`approve_gate` (which `fail`/`untrusted` would satisfy) all keep every +substring and all fail open. Every occurrence of a job across every yaml block +is checked, so a stale second copy cannot hide behind a fixed first one. + +docs/callers/cursor-review.md's `next:` example is pinned too: it is the +canonical "chain a job off approve_gate" snippet and fails the same way. + +Deliberately parsed WITHOUT PyYAML, like its sibling suites: this repo is +stdlib-only and CI installs no requirements for these tests. + +Run: python3 .github/cursor-approve/tests/test_caller_doc_pattern.py +""" + +import os +import re +import unittest + +DOCS_DIR = os.path.normpath( + os.path.join(os.path.dirname(__file__), "..", "..", "..", "docs", "callers") +) +APPROVE_DOC = os.path.join(DOCS_DIR, "cursor-approve.md") +REVIEW_DOC = os.path.join(DOCS_DIR, "cursor-review.md") + +YAML_BLOCK = re.compile(r"^```yaml$(.*?)^```$", re.M | re.S) +JOB_HEADER = re.compile(r"^ ([A-Za-z0-9_-]+):\s*$") +IF_KEY = re.compile(r"^ if:\s*(.*)$") + +# Every job id the pattern documents that must not be skipped by a red +# cursor-review. Named explicitly rather than matched by prefix so a renamed or +# deleted block fails loudly instead of passing vacuously. +START = "approve-start" +AXES = ( + "axis-correctness", + "axis-conformance", + "axis-business", + "axis-design", + "axis-completeness", +) +DECIDE = "cursor-approve" +NEXT = "next" + +GATE = "needs.cursor-review.outputs.approve_gate" +START_CONJUNCTS = [ + "!cancelled()", + f"({GATE} == 'pass' || {GATE} == 'capped')", +] +AXIS_CONJUNCTS = [ + "!cancelled()", + "needs.approve-start.result == 'success'", + f"{GATE} == 'pass'", +] +NEXT_CONJUNCTS = ["!cancelled()", f"{GATE} == 'pass'"] + + +def _jobs(path): + """{job_id: [flattened `if:` expression, ...]} over every yaml block. + + A list per job, one entry per occurrence, so a job documented in two blocks + is checked in both rather than the later copy silently winning. + """ + with open(path, encoding="utf-8") as fh: + text = fh.read() + out = {} + for block in YAML_BLOCK.findall(text): + lines = block.splitlines() + job = None + for i, line in enumerate(lines): + header = JOB_HEADER.match(line) + if header: + job = header.group(1) + continue + if job is None: + continue + key = IF_KEY.match(line) + if not key: + continue + first = key.group(1).strip() + parts = [] + if first in (">-", ">", "|", "|-"): + # A block scalar's content is everything indented deeper than + # the key. A `#` line in there is CONTENT, not a comment, so it + # is kept and then fails the exact-conjunct checks below. + for cont in lines[i + 1:]: + if cont.strip() and len(cont) - len(cont.lstrip()) <= 4: + break + parts.append(cont.strip()) + else: + parts.append(first) + expr = re.sub(r"\s+", " ", " ".join(p for p in parts if p)).strip() + wrapped = re.fullmatch(r"\$\{\{(.*)\}\}", expr) + if wrapped: + expr = wrapped.group(1).strip() + out.setdefault(job, []).append(expr) + return out + + +def _conjuncts(expr): + """Split on top-level `&&`; a top-level `||` is returned as a failure marker. + + Parenthesised groups are kept whole, so `a && (b || c)` is two conjuncts. + """ + out, depth, cur, i = [], 0, "", 0 + while i < len(expr): + ch = expr[i] + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0 and expr.startswith("&&", i): + out.append(cur.strip()) + cur, i = "", i + 2 + continue + if depth == 0 and expr.startswith("||", i): + return None + cur += ch + i += 1 + out.append(cur.strip()) + return out + + +class CallerPattern(unittest.TestCase): + def setUp(self): + self.jobs = _jobs(APPROVE_DOC) + + def _assert_conjuncts(self, job, exprs, expected): + for expr in exprs: + got = _conjuncts(expr) + self.assertIsNotNone( + got, + f"{job}: a top-level `||` in its `if:` lets any one side run it " + f"on its own, which fails open. Got: {expr}", + ) + self.assertEqual( + sorted(got), + sorted(expected), + f"{job}: `if:` must be exactly the conjunction of {expected}. " + f"`!cancelled()` keeps a red panel cell from skipping it; the " + f"rest restate what `!cancelled()` drops and keep the " + f"`approve_gate` comparison exact. Got: {expr}", + ) + + def test_every_documented_job_is_present(self): + for job in (START, DECIDE) + AXES: + self.assertIn( + job, + self.jobs, + f"{job} has no `if:` in any yaml block of {APPROVE_DOC} — the " + "pattern was renamed or deleted, so the guards below assert nothing.", + ) + + def test_start_phase_survives_a_red_cursor_review(self): + # `!cancelled()`, never `always()`: a superseded (cancelled) run must + # never go on to decide. The exact-conjunct check rejects both a + # missing guard and an `always()` in its place. + self._assert_conjuncts(START, self.jobs[START], START_CONJUNCTS) + + def test_axes_survive_a_red_cursor_review_and_still_need_start(self): + # `!cancelled()` also lifts the implicit "approve-start succeeded" + # requirement, so that half has to be restated. A start phase that + # failed to withdraw cursor-review's standing approval must still skip + # the axes, which decide reads as "no result". And `approve_gate` is + # the only authority on whether a round may be decided; dropping or + # loosening it while keeping `!cancelled()` is what would actually + # loosen the gate. + for job in AXES: + self._assert_conjuncts(job, self.jobs[job], AXIS_CONJUNCTS) + + def test_decide_still_runs_on_always_and_gates_on_pass(self): + # Unchanged by this pattern, and the reason a withheld round still + # leaves a card: decide must run even when the axes skipped. + self._assert_conjuncts( + DECIDE, self.jobs[DECIDE], ["always()", f"{GATE} == 'pass'"] + ) + + def test_cursor_review_doc_next_example_survives_a_red_cursor_review(self): + jobs = _jobs(REVIEW_DOC) + self.assertIn( + NEXT, + jobs, + f"the `{NEXT}:` approve_gate example is gone from {REVIEW_DOC}; " + "update this test to follow it rather than letting it pass vacuously.", + ) + self._assert_conjuncts(NEXT, jobs[NEXT], NEXT_CONJUNCTS) + + +class Parser(unittest.TestCase): + """The parser has to see the fail-open shapes the guards exist to catch.""" + + def test_top_level_or_is_rejected(self): + self.assertIsNone(_conjuncts(f"!cancelled() || {GATE} == 'pass'")) + + def test_parenthesised_or_stays_one_conjunct(self): + self.assertEqual( + _conjuncts(START_CONJUNCTS[0] + " && " + START_CONJUNCTS[1]), + START_CONJUNCTS, + ) + + def test_bare_truthy_gate_is_not_the_comparison(self): + self.assertNotEqual( + sorted(_conjuncts(f"!cancelled() && {GATE}")), sorted(NEXT_CONJUNCTS) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/cursor-review.yml b/.github/workflows/cursor-review.yml index 4437bd25..93a2522f 100644 --- a/.github/workflows/cursor-review.yml +++ b/.github/workflows/cursor-review.yml @@ -439,7 +439,15 @@ on: metadata is missing, or a review type (`adversarial` or `edge-case`) has no reviewer that completed. Applies to the decision `defer_approval` hands to cursor-approve too. No effect unless `approve_max_severity` - is set. + is set. A tolerated errored cell still turns THIS WORKFLOW's called job + red (the leg itself, and `Panel integrity` with it), so a caller that + chains jobs off it — the cursor-approve axes above all — must guard + them with `!cancelled()` or GitHub's default `needs:` rule skips them + and the tolerance buys nothing. `!cancelled()` also drops the implicit + success requirement on EVERY other `needs:` entry, so restate those + explicitly (the axes keep `needs.approve-start.result == 'success'`). + See docs/callers/cursor-approve.md ("A red panel cell must not skip + the axes"). type: number required: false default: 0 @@ -4382,9 +4390,20 @@ jobs: # `edge-case (kimi-k3-high)` bricks every PR the day that pin moves. # # ADVISORY by default, exactly like `Blocking gate`: red here fails no other - # job, gates no merge, and does not gate `post-review` — a caller that wants - # it to block marks ` / Panel integrity` a required check in - # its own ruleset. See docs/callers/cursor-review.md. + # job IN THIS WORKFLOW, gates no merge, and does not gate `post-review` — a + # caller that wants it to block marks ` / Panel integrity` a + # required check in its own ruleset. See docs/callers/cursor-review.md. + # + # "Advisory" stops at this workflow's edge, and the distinction is not + # academic. Red here puts the CALLER's `uses:` job in `failure`, so any + # caller job chained off it with a bare `needs:` is skipped by GitHub's + # default rule — which is how one errored cell silently stopped the + # cursor-approve axes from running at all, and withheld every approval, on a + # caller that had set `approve_max_failed_reviewers: 1` precisely to tolerate + # it. A red matrix leg does this on its own too, so dropping the check would + # not have helped. Downstream caller jobs must be guarded with `!cancelled()` + # and gated on `approve_gate`, which already fails closed; see + # docs/callers/cursor-approve.md ("A red panel cell must not skip the axes"). # # `always()`, and the same four gate conditions `consolidate` uses, so this # reports on exactly the runs where a panel was supposed to happen and stays diff --git a/.github/workflows/test-cursor-review-scripts.yml b/.github/workflows/test-cursor-review-scripts.yml index 777bf0d3..41c22e73 100644 --- a/.github/workflows/test-cursor-review-scripts.yml +++ b/.github/workflows/test-cursor-review-scripts.yml @@ -19,10 +19,15 @@ name: Test cursor-review scripts # into the table-driven harness at # .github/workflow-pins/tests/test_inputs_docs_drift.py, which reads BOTH # docs/callers/cursor-review.md and .github/cursor-review/README.md. Both paths -# are in test-workflow-pins.yml's filters, so docs/callers/cursor-review.md is -# deliberately NOT listed below — no suite this workflow runs reads it, and a -# docs-only PR that deletes the `blocking` row (the #31 scenario) still fires -# the alarm, over there. +# are in test-workflow-pins.yml's filters, and a docs-only PR that deletes the +# `blocking` row (the #31 scenario) still fires the alarm, over there. +# +# docs/callers/cursor-approve.md and docs/callers/cursor-review.md ARE listed +# below anyway, because a suite here reads them too: test_caller_doc_pattern.py +# pins the `if:` guards on the cursor-approve caller snippet and on +# cursor-review's `next:` approve_gate example. A copied caller is only as good +# as the snippet, and dropping `!cancelled()` from it silently withholds every +# approval (see that file's docstring). on: pull_request: @@ -36,6 +41,11 @@ on: - '.github/workflows/cursor-approve.yml' - '.github/workflows/cursor-axis-base.yml' - '.github/workflows/axis-*.yml' + # test_caller_doc_pattern.py reads the caller snippets out of these + # pages and pins their `if:` guards; the regression it guards is a + # docs-only edit — dropping `!cancelled()` from a snippet a caller copies. + - 'docs/callers/cursor-approve.md' + - 'docs/callers/cursor-review.md' - '.github/workflows/test-cursor-review-scripts.yml' push: branches: [main] @@ -49,6 +59,11 @@ on: - '.github/workflows/cursor-approve.yml' - '.github/workflows/cursor-axis-base.yml' - '.github/workflows/axis-*.yml' + # test_caller_doc_pattern.py reads the caller snippets out of these + # pages and pins their `if:` guards; the regression it guards is a + # docs-only edit — dropping `!cancelled()` from a snippet a caller copies. + - 'docs/callers/cursor-approve.md' + - 'docs/callers/cursor-review.md' - '.github/workflows/test-cursor-review-scripts.yml' permissions: diff --git a/docs/callers/cursor-approve.md b/docs/callers/cursor-approve.md index f72fcdcb..7828e87f 100644 --- a/docs/callers/cursor-approve.md +++ b/docs/callers/cursor-approve.md @@ -14,9 +14,12 @@ the context axes (business, design, completeness) are not available yet. resolving any thread), and withdraws the approving identity's own earlier approvals and requests for changes. Without it, cursor-review's `decide` APPROVES first, and that severity-only approval satisfies branch protection or auto-merge until the - start phase withdraws it — and stays standing if the cursor-review run fails - or is cancelled after approving, because the start phase and the axes are - then skipped. With it, this workflow's decide phase is the only approver. + start phase withdraws it — and stays standing if the cursor-review run is + cancelled after approving, or if its `approve_gate` comes out `untrusted`, + because the start phase and the axes are then skipped. (A run that merely + went red on an errored panel cell does NOT skip them while the gate reads + `pass` or `capped`; see "A red panel cell must not skip the axes".) With it, + this workflow's decide phase is the only approver. 2. `cursor-approve` with `phase: start` first withdraws the approving identity's own approvals — a backstop: with `defer_approval: true` there is none from this round, but a caller that does not set it has one standing @@ -228,7 +231,13 @@ jobs: approve-start: needs: cursor-review - if: needs.cursor-review.outputs.approve_gate == 'pass' || needs.cursor-review.outputs.approve_gate == 'capped' + # `!cancelled()`, not a bare condition: cursor-review's called job goes red + # on any errored panel cell, which would otherwise skip this job. See + # "A red panel cell must not skip the axes" below. + if: >- + !cancelled() + && (needs.cursor-review.outputs.approve_gate == 'pass' + || needs.cursor-review.outputs.approve_gate == 'capped') uses: Comfy-Org/github-workflows/.github/workflows/cursor-approve.yml@ # v1 with: workflows_ref: @@ -243,7 +252,10 @@ jobs: axis-correctness: needs: [cursor-review, approve-start] - if: needs.cursor-review.outputs.approve_gate == 'pass' + if: >- + !cancelled() + && needs.approve-start.result == 'success' + && needs.cursor-review.outputs.approve_gate == 'pass' uses: Comfy-Org/github-workflows/.github/workflows/axis-correctness.yml@ # v1 with: commit_sha: ${{ github.event.pull_request.head.sha }} @@ -252,7 +264,10 @@ jobs: axis-conformance: needs: [cursor-review, approve-start] - if: needs.cursor-review.outputs.approve_gate == 'pass' + if: >- + !cancelled() + && needs.approve-start.result == 'success' + && needs.cursor-review.outputs.approve_gate == 'pass' uses: Comfy-Org/github-workflows/.github/workflows/axis-conformance.yml@ # v1 with: commit_sha: ${{ github.event.pull_request.head.sha }} @@ -298,6 +313,47 @@ leaving an earlier approval standing. The axes `needs: approve-start` so they never start while cursor-review's approval still stands; a start phase that fails to withdraw it skips them, which decide reads as "no result". +### A red panel cell must not skip the axes + +`!cancelled()` on the start phase and the axes is load-bearing. cursor-review's +called job goes **red whenever a panel cell errors** — the leg fails by design +and `Panel integrity` fails alongside it — and `approve_max_failed_reviewers` +does not change that. That input governs the `approve_gate` verdict, not the +job's conclusion, so a tolerated errored cell still leaves the called job in +`failure`. + +Without `!cancelled()`, GitHub's default `needs:` rule skips the start phase on +that red job; the axes skip with it; and decide, which runs on `always()`, +reports every axis as ⚠️ no result and withholds. A caller that sets +`approve_max_failed_reviewers` above 0 and omits `!cancelled()` therefore never +gets the tolerance it configured — one errored cell out of six withholds every +round, and nothing on the card says why. + +`approve_gate` stays the only authority on whether a round may be decided, and +reading it off a red cursor-review job cannot approve a round the gate itself +rejected: it is already `untrusted` when the judge was degraded, when more cells +errored than the caller tolerates, when the review did not land as threads or +when the head moved, and it falls back to `untrusted` whenever an upstream +decision job did not succeed — which is every Panel integrity failure that is +not simply a short panel. The axes keep an explicit +`needs.approve-start.result == 'success'` rather than a blanket `!cancelled()` +so the invariant above is unchanged: a start phase that failed to withdraw +cursor-review's standing approval still skips them. + +What the guard does give up is an *incidental* backstop. The gate counts each +cell by the `status` that cell's own findings artifact reports, not by its +leg's conclusion — that is how a tolerated errored cell still passes — and +cursor-review documents that status as an availability signal, not an +attestation: a prompt-injected cell can write a clean `ok` record, and because +artifact names are run-global it can claim another cell's name first, so that +cell's real upload fails and its leg goes red while consolidate reads the +forgery. A bare `needs:` used to skip the axes on that red leg. With +`!cancelled()` the round goes on to the axes, which still review the change +independently, and decide still withholds on any red axis (or more yellow +than `max_yellow_axes`). Treat the panel count +as tamper-evident only in the rollup (the red leg stays visible), not in the +gate. + ## Inputs `cursor-approve.yml`: @@ -401,7 +457,10 @@ Business and design fetch the change from the GitHub compare API, which ```yaml axis-business: needs: [cursor-review, approve-start] - if: needs.cursor-review.outputs.approve_gate == 'pass' + if: >- + !cancelled() + && needs.approve-start.result == 'success' + && needs.cursor-review.outputs.approve_gate == 'pass' uses: Comfy-Org/github-workflows/.github/workflows/axis-business.yml@ # v1 with: commit_sha: ${{ github.event.pull_request.head.sha }} @@ -412,7 +471,10 @@ Business and design fetch the change from the GitHub compare API, which axis-design: needs: [cursor-review, approve-start] - if: needs.cursor-review.outputs.approve_gate == 'pass' + if: >- + !cancelled() + && needs.approve-start.result == 'success' + && needs.cursor-review.outputs.approve_gate == 'pass' uses: Comfy-Org/github-workflows/.github/workflows/axis-design.yml@ # v1 with: commit_sha: ${{ github.event.pull_request.head.sha }} @@ -423,7 +485,10 @@ Business and design fetch the change from the GitHub compare API, which axis-completeness: needs: [cursor-review, approve-start] - if: needs.cursor-review.outputs.approve_gate == 'pass' + if: >- + !cancelled() + && needs.approve-start.result == 'success' + && needs.cursor-review.outputs.approve_gate == 'pass' uses: Comfy-Org/github-workflows/.github/workflows/axis-completeness.yml@ # v1 with: commit_sha: ${{ github.event.pull_request.head.sha }} diff --git a/docs/callers/cursor-review.md b/docs/callers/cursor-review.md index 6148a5d7..55a68e35 100644 --- a/docs/callers/cursor-review.md +++ b/docs/callers/cursor-review.md @@ -870,7 +870,10 @@ jobs: secrets: { CURSOR_API_KEY: "${{ secrets.CURSOR_API_KEY }}" } next: needs: cursor-review - if: needs.cursor-review.outputs.approve_gate == 'pass' + # `!cancelled()`: an errored panel cell turns the called job red even when + # `approve_max_failed_reviewers` tolerates it, and a bare `if:` is then + # skipped by GitHub's default `needs:` rule. + if: ${{ !cancelled() && needs.cursor-review.outputs.approve_gate == 'pass' }} ``` | Value | Meaning |