diff --git a/.github/cursor-review/README.md b/.github/cursor-review/README.md index 3d44317c..1c268b6a 100644 --- a/.github/cursor-review/README.md +++ b/.github/cursor-review/README.md @@ -442,6 +442,8 @@ All optional except `workflows_ref` (required, no default) — pass them under | `axes` | `correctness,conformance` | With `axes_alongside_panel`: the start card's axis list (cursor-approve `axes`); a non-empty subset of `correctness` and `conformance`, else the start fails. | | `axis_prior_rounds` | `0` | With `axes_alongside_panel`: `prior_rounds` for the start phase and both axes. | | `axis_direct_model` | `''` | With `axes_alongside_panel`: the axes' `axis_direct_model` (direct Anthropic API, `ANTHROPIC_API_KEY`). | +| `byok` | `false` | Bring-your-own-key: `CURSOR_API_KEY` is the PR author's own key, resolved by the caller; empty = the author has none, so the panel is skipped with a sticky comment. See [the caller guide](../../docs/callers/cursor-review.md#byok). | +| `byok_help` | `''` | With `byok`: markdown appended to the missing-key comment (who to ask, an onboarding link). | ### `workflows_ref` must equal the `uses:` pin diff --git a/.github/cursor-review/tests/test_workflow_byok.py b/.github/cursor-review/tests/test_workflow_byok.py new file mode 100644 index 00000000..c120c86c --- /dev/null +++ b/.github/cursor-review/tests/test_workflow_byok.py @@ -0,0 +1,85 @@ +"""Structural regression tests for cursor-review.yml's `byok` wiring. + +With `byok`, an EMPTY CURSOR_API_KEY means the PR author has no key of their +own. Two things must hold, and neither fails loudly if it regresses: the gate's +`should_run` must be vetoed for that author (or the panel runs and every cell +fails at preflight, with no comment telling them why), and the veto must be +limited to runs that would start a panel (or a push on a keyless author's PR +would skip the dismissal job that withdraws a stale approval). Plus the comment +job must only ever POST on `missing`. + +Stdlib-only, reusing the job splitter from test_workflow_job_isolation. + +Run: python3 .github/cursor-review/tests/test_workflow_byok.py +""" + +import unittest + +from test_workflow_job_isolation import read_workflow, split_jobs + + +def text(lines): + return "\n".join(lines) + + +class WorkflowByokTest(unittest.TestCase): + def setUp(self): + self.jobs = split_jobs(read_workflow()) + self.gate = text(self.jobs["gate"]) + + def test_gate_should_run_is_vetoed_by_a_missing_key(self): + self.assertIn( + "should_run: ${{ steps.check.outputs.should_run == 'true' && " + "steps.byok.outputs.status != 'missing' && 'true' || 'false' }}", + self.gate, + ) + + def test_key_check_runs_only_when_a_panel_would_start(self): + # Not on a same-SHA re-trigger either: that runs no panel, so a + # keyless author must not be told one was skipped. + self.assertIn( + "if: inputs.byok && steps.check.outputs.should_run == 'true' && " + "steps.dup.outputs.already_reviewed != 'true'", + self.gate, + ) + + def test_dedupe_runs_before_the_key_check(self): + self.assertLess( + self.gate.index("name: Check for prior review on this commit"), + self.gate.index("name: Check the BYOK key"), + ) + + def test_key_check_reads_presence_only(self): + self.assertIn("KEY_PRESENT: ${{ secrets.CURSOR_API_KEY != '' }}", self.gate) + + def test_the_anthropic_probe_honours_the_veto(self): + # The probe spends API calls; it must not run for a round the veto + # already cancelled. + self.assertIn( + "if: steps.check.outputs.should_run == 'true' && " + "steps.byok.outputs.status != 'missing' && steps.dup", + self.gate, + ) + + def test_blocking_gate_fails_closed_on_a_missing_key(self): + # The veto folds into should_run, so without this step a keyless + # author's round reads as review-free and greens the required check. + job = text(self.jobs["blocking-gate"]) + self.assertIn("if: ${{ needs.gate.outputs.byok_status == 'missing' }}", job) + step = job[job.index("name: Fail if the author has no BYOK key"):] + step = step[: step.index("- name:", 1)] + self.assertIn("exit 1", step) + + def test_comment_job_runs_only_on_a_byok_verdict_and_posts_only_on_missing(self): + job = text(self.jobs["byok-comment"]) + self.assertIn("inputs.byok", job) + self.assertIn("needs.gate.outputs.byok_status == 'missing'", job) + self.assertIn('elif [ "$BYOK_STATUS" = "missing" ]; then', job) + self.assertEqual(job.count("-X POST"), 1) + + def test_comment_job_checks_out_nothing(self): + self.assertNotIn("actions/checkout@", text(self.jobs["byok-comment"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/cursor-review/tests/test_workflow_job_isolation.py b/.github/cursor-review/tests/test_workflow_job_isolation.py index 9db2c3fb..d9304b09 100644 --- a/.github/cursor-review/tests/test_workflow_job_isolation.py +++ b/.github/cursor-review/tests/test_workflow_job_isolation.py @@ -306,8 +306,9 @@ def test_the_bot_key_is_confined_to_the_checkout_free_posting_jobs(self): # `auto-retry` (BE-19526): the round's relabel, which must fire runs, so # never GITHUB_TOKEN. It checks out only this repo's pinned assets. So # does `auto-rereview` (BE-20262), the relabel a dismissal schedules. + # `byok-comment` is `over-cap-comment`'s twin: checkout-free, comment only. self.assertEqual(holders, {"over-cap-comment", "post-review", "dismiss-stale-approval", "round-cap", - "auto-retry", "auto-rereview"}) + "auto-retry", "auto-rereview", "byok-comment"}) def test_the_anthropic_key_reaches_only_the_gate_and_its_direct_cells(self): # `gate` only resolves a presence boolean; the cells and the direct diff --git a/.github/workflows/cursor-review.yml b/.github/workflows/cursor-review.yml index afed3021..67eb400e 100644 --- a/.github/workflows/cursor-review.yml +++ b/.github/workflows/cursor-review.yml @@ -876,10 +876,38 @@ on: type: string required: false default: '' + byok: + description: >- + OPT-IN. Bring-your-own-key: the caller passes the PR AUTHOR's own + Cursor key as CURSOR_API_KEY (resolved in the caller's `secrets:` + mapping, which GitHub evaluates server-side, so this workflow's jobs + only ever hold that one key — who can read the caller's key roster + is a separate question, see docs/callers/cursor-review.md#byok), and + an EMPTY key means the author has none. Then, on a run that would + start a panel, `gate` skips the panel, a sticky PR comment tells the + author how to add a key (once one exists the next round flips that + comment to ✅), and with `blocking` the Blocking gate stays red. + Events that start no panel need no key. Default false = no key check + here: pass a non-empty key. + type: boolean + required: false + default: false + byok_help: + description: >- + With `byok`: markdown appended to the missing-key comment, e.g. who to + ask or a link to your onboarding doc for adding a key. Default '' = + the generic text only. + type: string + required: false + default: '' secrets: CURSOR_API_KEY: - description: Cursor API key for cursor-agent (the panel + judge models bill through it). - required: true + description: >- + Cursor API key for cursor-agent (the panel + judge models bill through + it). Required in practice; declared optional only so a `byok` caller + can pass an empty value for an author with no key (the panel is then + skipped, see `byok`). + required: false OPENAI_API_KEY: description: >- OpenAI API key for the optional direct-API review cell. Required only @@ -1079,7 +1107,10 @@ jobs: # startup on the pin bump, whether or not it set the command. pull-requests: read outputs: - should_run: ${{ steps.check.outputs.should_run }} + # `check`'s verdict, vetoed by a `byok` author with no key. + should_run: ${{ steps.check.outputs.should_run == 'true' && steps.byok.outputs.status != 'missing' && 'true' || 'false' }} + # ok / missing on a `byok` run that would start a panel, else empty. + byok_status: ${{ steps.byok.outputs.status }} already_reviewed: ${{ steps.dup.outputs.already_reviewed }} approve_author_ok: ${{ steps.author.outputs.approve_author_ok }} openai_key_present: ${{ steps.openai_key.outputs.present }} @@ -1445,6 +1476,29 @@ jobs: echo "already_reviewed=false" >> "$GITHUB_OUTPUT" fi + - name: Check the BYOK key + # With `byok` the caller passes the PR author's own key as + # CURSOR_API_KEY, and an empty one means the author has none. Checked + # only on a run `check` would start a panel on: the other events (a + # push, a retarget) start none and spend nothing, so they need no key. + # After the same-SHA dedupe, so a re-trigger on an already-reviewed + # head (which runs no panel either) is not announced as a skip. + # Presence only, computed on this GitHub-hosted job that checks out + # nothing, like the direct-API key presence steps above. + id: byok + if: inputs.byok && steps.check.outputs.should_run == 'true' && steps.dup.outputs.already_reviewed != 'true' + env: + KEY_PRESENT: ${{ secrets.CURSOR_API_KEY != '' }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + if [ "$KEY_PRESENT" = "true" ]; then + echo "status=ok" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "status=missing" >> "$GITHUB_OUTPUT" + echo "::warning::byok: no Cursor key for the PR author (${PR_AUTHOR}) — skipping the panel." + echo "**Cursor review skipped:** \`byok\` is on and the caller passed no Cursor key for the PR author (\`${PR_AUTHOR}\`)." >> "$GITHUB_STEP_SUMMARY" + - name: Probe the direct Anthropic key # Presence is not usability: a key whose organization hit its usage # limit (an HTTP 400 naming the reset time), ran out of credit, or was @@ -1464,7 +1518,7 @@ jobs: # it); it never prints the key or any header. Best effort: a crash # here leaves the outputs empty, which preflight reads as "not probed". id: anthropic_probe - if: steps.check.outputs.should_run == 'true' && steps.dup.outputs.already_reviewed != 'true' && steps.anthropic_key.outputs.present == 'true' + if: steps.check.outputs.should_run == 'true' && steps.byok.outputs.status != 'missing' && steps.dup.outputs.already_reviewed != 'true' && steps.anthropic_key.outputs.present == 'true' continue-on-error: true # Two models at worst: 2 x (3 x 20 s + 60 s) = 4 minutes; the script # stops retrying at PROBE_BUDGET_SECONDS, and writes every output in @@ -2751,6 +2805,110 @@ jobs: echo "This PR is within diff_size_cap, so there is no skip to announce; nothing was owed on this run." fi + byok-comment: + name: Comment when the author has no BYOK key + needs: gate + # A `byok` run that would have started a panel: `missing` posts (or + # refreshes) the sticky notice, `ok` flips an existing one to ✅ and never + # posts. The same shape as `over-cap-comment` above — bot identity with a + # run-token fallback, author-filtered marker lookup, every copy updated, + # a live-head check — for the same reasons, documented there. + if: ${{ !cancelled() && inputs.byok && (needs.gate.outputs.byok_status == 'missing' || needs.gate.outputs.byok_status == 'ok') }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + # The GITHUB_TOKEN fallback; `post-review` already requires it. + pull-requests: write + env: + BOT_CONFIGURED: ${{ inputs.bot_app_id != '' && secrets.BOT_APP_PRIVATE_KEY != '' }} + steps: + - name: Mint bot token + id: bot + if: env.BOT_CONFIGURED == 'true' + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ inputs.bot_app_id }} + private-key: ${{ secrets.BOT_APP_PRIVATE_KEY }} + permission-pull-requests: write + + - name: Upsert sticky BYOK comment + id: upsert + # Best-effort, like the over-cap upsert: the gate's ::warning:: and step + # summary are the primary signal, and a failed comment must not turn a + # skipped review red. + continue-on-error: true + env: + GH_TOKEN: ${{ steps.bot.outputs.token || secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BOT_LOGIN: ${{ steps.bot.outputs.app-slug && format('{0}[bot]', steps.bot.outputs.app-slug) || '' }} + RUN_LOGIN: github-actions[bot] + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + BYOK_STATUS: ${{ needs.gate.outputs.byok_status }} + BYOK_HELP: ${{ inputs.byok_help }} + run: | + set -euo pipefail + # Fails closed on an unreadable head, as the over-cap upsert does. + if ! LIVE_HEAD="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha')" || [ -z "$LIVE_HEAD" ]; then + echo "Could not read the PR's current head (see gh error above) — refusing to write a verdict that may already be stale." >&2 + exit 1 + fi + if [ "$LIVE_HEAD" != "$EVENT_HEAD_SHA" ]; then + echo "This run's head (${EVENT_HEAD_SHA}) is no longer the PR head (${LIVE_HEAD}) — a newer run owns the verdict. Not writing." + exit 0 + fi + MARKER='' + if [ "$BYOK_STATUS" = "ok" ]; then + { + printf '%s\n\n' "$MARKER" + printf '✅ **A Cursor key is now set up for @%s**, so the review is no longer skipped for a missing key. Check this run for the panel'"'"'s result.\n' "$PR_AUTHOR" + } > body.md + else + { + printf '%s\n\n' "$MARKER" + printf '⚠️ **Cursor review skipped** — this repo runs cursor-review on the PR author'"'"'s own Cursor API key, and none is set up for @%s. **No review panel ran on this PR.**\n\n' "$PR_AUTHOR" + # The backticks are a markdown code span, not a substitution. + # shellcheck disable=SC2016 + printf 'To enable it, create a Cursor API key at https://cursor.com/settings/api-keys (set a spend limit), get it added for your GitHub login, then re-apply the `%s` label.\n' "$REVIEW_LABEL" + if [ -n "$BYOK_HELP" ]; then + printf '\n%s\n' "$BYOK_HELP" + fi + } > body.md + fi + existing="$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" --paginate \ + | jq -r --arg m "$MARKER" --arg login "$BOT_LOGIN" --arg run_login "$RUN_LOGIN" ' + .[] | select(($login != "" and (.user.login // "") == $login) + or (.user.login // "") == $run_login) + | select((.user.type // "") == "Bot") + | select(((.body // "") | startswith($m))) + | .id')" + if [ -n "$existing" ]; then + patched=0 + failed=0 + for id in $existing; do + if gh api -X PATCH "repos/${REPO}/issues/comments/${id}" -F body=@body.md >/dev/null; then + echo "Updated sticky BYOK comment ${id}." + patched=$((patched + 1)) + else + failed=$((failed + 1)) + echo "Failed to update sticky BYOK comment ${id} (see gh error above); continuing." >&2 + fi + done + if [ "$patched" -eq 0 ] && [ "$failed" -gt 0 ]; then + echo "No sticky comment could be updated (${failed} failed); the PR may show a stale notice." >&2 + exit 1 + fi + elif [ "$BYOK_STATUS" = "missing" ]; then + gh api -X POST "repos/${REPO}/issues/${PR_NUMBER}/comments" -F body=@body.md >/dev/null + echo "Posted BYOK missing-key comment." + else + # Never POST on the happy path. + echo "Author has a key and no existing comment — nothing to post." + fi + preflight: # Validate the panel's pinned model ids against the LIVE Cursor catalog # once, before the 8-cell fan-out. When Cursor delists a pinned id the @@ -7213,6 +7371,21 @@ jobs: echo "::error::The cursor-review trigger job did not succeed (gate: $GATE_RESULT), so whether this PR should have been reviewed is unknown — refusing to pass the blocking gate. Re-run the failed job." exit 1 + - name: Fail if the author has no BYOK key + # The `byok` veto folds into `should_run`, so a keyless author's + # panel-starting round reads as review-free to the guard below and + # falls through to a thread query that finds nothing — green over an + # unreviewed PR, the same fail-open the over-cap path was un-exempted + # to close. `missing` is only ever set on a round that WOULD have run a + # panel (not on a review-free event or a same-SHA re-trigger), so this + # holds red exactly until a round with a key lands. + if: ${{ needs.gate.outputs.byok_status == 'missing' }} + env: + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + echo "::error::byok is on and the caller passed no Cursor key for the PR author (${PR_AUTHOR}), so no review ran — an unreviewed PR cannot pass a blocking gate. Add a key for the author, then re-trigger the review." + exit 1 + - name: Fail if the fresh review did not land # SECOND fail-closed guard. A gate that passed while the review it gates # on never landed would be fail-OPEN: no threads exist, the query below diff --git a/docs/callers/cursor-review.md b/docs/callers/cursor-review.md index 6126235a..c1784262 100644 --- a/docs/callers/cursor-review.md +++ b/docs/callers/cursor-review.md @@ -51,7 +51,7 @@ Prompts and scripts live in [`.github/cursor-review/`](../../.github/cursor-revi | | | |---|---| -| `secrets.CURSOR_API_KEY` | **Required.** Org-level in Comfy-Org. | +| `secrets.CURSOR_API_KEY` | **Required.** Org-level in Comfy-Org. With `byok`, the PR author's own key instead — see [BYOK](#byok). | | `secrets.OPENAI_API_KEY` | Optional. Only read when `openai_direct_model` is set; use a project-scoped service-account key with a spend limit. | | `secrets.ANTHROPIC_API_KEY` | Optional. Only used when `anthropic_direct_model` or `judge_direct_model` is set; use a key scoped to its own Anthropic workspace with a spend limit, never a personal key. A key that hits that limit falls back to Cursor for the run (see `anthropic_direct_model`). | | `secrets.SLACK_BOT_TOKEN` | Optional. Without it the review still posts; only the DMs are skipped. | @@ -155,6 +155,78 @@ pull-requests: write # posting the consolidated review (and, at the round cap, | `axes` | `correctness,conformance` | With `axes_alongside_panel`: the axes the status card lists. A non-empty subset of `correctness` and `conformance`, the only axes run here — anything else fails the start phase (`axes_started` `failure`). Pass the same list to your decide job. | | `axis_prior_rounds` | `0` | With `axes_alongside_panel`: cursor-approve's `prior_rounds` (0–3) for the start phase and both axes. Pass the same value to your decide job. | | `axis_direct_model` | `''` | With `axes_alongside_panel`: run the axes on the direct Anthropic API (needs `ANTHROPIC_API_KEY`), as the axis workflows' `axis_direct_model`. | +| `byok` | `false` | Bring-your-own-key: the caller passes the PR **author's** own Cursor key as `CURSOR_API_KEY`, and an empty one means the author has none. On a run that would start a panel, `gate` then skips it (a `::warning::` and a job-summary line) and the `byok-comment` job posts one sticky PR comment telling the author how to add a key; the round after they have one flips it to ✅. Events that start no panel (a push, a retarget, a re-trigger on an already-reviewed head) need no key. With `blocking`, a skipped round holds the Blocking gate red until a round with a key lands. See [BYOK](#byok) for the caller wiring. | +| `byok_help` | `''` | With `byok`: markdown appended to the missing-key comment — who to ask, or a link to your onboarding doc. Empty = the generic text only. | + +## BYOK + +Run the panel on each PR author's **own** Cursor key instead of a shared one. +The reusable never sees anyone else's key: the caller resolves the author's key +in its `secrets:` mapping, which GitHub evaluates server-side for a `uses:` job, +so only that one value reaches the reusable (and is masked there like any +passed secret). That bounds what the *reusable* holds, not who can read the +roster — see [who can read the roster](#who-can-read-the-roster) below. The +keys live in one JSON secret, login → key: + +| | | +|---|---| +| `secrets.CURSOR_BYOK_KEYS` | `{"": "", ...}` — a JSON **object**. Malformed JSON makes `fromJSON` fail while GitHub evaluates `secrets:`, which fails the whole call (every job, the stale-approval dismissal included), so validate the value before writing it. Org-level, so adding a person needs no PR. GitHub never shows a secret again, so keep the source of truth elsewhere (e.g. a password-manager item) and re-write the whole value on each change. Logins are matched exactly as GitHub spells them. | +| `vars.CURSOR_REVIEW_BYOK` | `true` turns BYOK on. Org- or repo-level. | +| `vars.CURSOR_BYOK_ORG_KEY_AUTHORS` | Optional JSON **array** of logins that keep the shared key, e.g. `["my-app[bot]"]` for bot-authored PRs. Never a bare JSON string: `contains` on a string is a substring match, so `"bot"` would exempt every login containing it. Malformed JSON fails the call, as above. | + +```yaml + with: + byok: ${{ vars.CURSOR_REVIEW_BYOK == 'true' }} + byok_help: "Ask in #your-channel to get your key added." + secrets: + # BYOK off → the shared key. On → the shared key for an exempt author, + # else the author's own key, EMPTY when they have none (the reusable then + # skips the panel and comments). Each branch repeats its own guard and + # the chain ends `|| ''`: an `a && b || c` chain falls through to `c` + # whenever `b` is EMPTY, so without the guards an unset shared key would + # silently hand the reusable a personal key. Do not "simplify" it. + CURSOR_API_KEY: ${{ vars.CURSOR_REVIEW_BYOK != 'true' && secrets.CURSOR_API_KEY || vars.CURSOR_REVIEW_BYOK == 'true' && contains(fromJSON(vars.CURSOR_BYOK_ORG_KEY_AUTHORS || '[]'), github.event.pull_request.user.login) && secrets.CURSOR_API_KEY || vars.CURSOR_REVIEW_BYOK == 'true' && !contains(fromJSON(vars.CURSOR_BYOK_ORG_KEY_AUTHORS || '[]'), github.event.pull_request.user.login) && fromJSON(secrets.CURSOR_BYOK_KEYS || '{}')[github.event.pull_request.user.login] || '' }} +``` + +If you also call the [cursor-approve](cursor-approve.md) axis workflows +directly, pass them the **same** expression, with no `|| secrets.CURSOR_API_KEY` +fallback: under `approve_only_on_push` an axis runs on a push with no panel and +no BYOK check, so a fallback would bill the shared key for exactly the keyless +author BYOK is meant to stop. Without a key the axis fails, and a failed axis +uploads no verdict, so decide does not approve. + +### Who can read the roster + +The server-side mapping keeps the *reusable* to one key; it does not keep the +*caller* honest. Under `pull_request` GitHub runs +the caller from the PR's head, so a same-repo PR can rewrite its `secrets:` +mapping — or add a step or a new workflow — and dump `CURSOR_BYOK_KEYS`, every +author's key, not just its own. So: + +1. Call cursor-review from **`pull_request_target`**, as in the [trust + model](#trust-model): the caller then comes from the base branch, so a PR + cannot change which key it is handed. Keep that caller free of any step that + checks out or runs PR code. +2. That still does not fence the secret itself: Actions secrets have no + branch policy for a `uses:` job (it cannot declare `environment:`), so a + workflow pushed on any branch of a repo the secret reaches — `on: push` + needs no PR — can read it. Grant the org secret only to repos where + everyone with write access may be trusted with every listed key, and do not + enable BYOK where that is not true. + +What BYOK does change is whose key is exposed. The panel's agents run with +`--trust` and a shell over PR code, and their shell children inherit +`CURSOR_API_KEY`, exactly as with the shared key today — but that code is not +only the author's: a collaborator's push to their branch, base-branch content +and a label any triager can apply all reach those agents. Prompt-injected +content can therefore exfiltrate the author's key or spend on it. A spend cap +bounds the spend, not the key's account-level access, so ask people for a +**dedicated, spend-capped key** they can revoke, never their everyday one. A +fork PR still skips at the gate, before any key matters. + +Never use `secrets[...]` with a computed name to look keys up instead: that +provisions the whole secrets context to the runner (zizmor +`overprovisioned-secrets`). ## Running a model experiment @@ -776,6 +848,8 @@ review to clear it. It holds the check red on all of: inline payload) — findings a thread query cannot see; * **the diff being over `diff_size_cap`.** No panel ran, so nothing was reviewed. Get the PR under the cap, or raise the cap on the caller; +* **a [`byok`](#byok) round skipped for a missing key.** No panel ran either. + Add a key for the author and re-trigger the review; * being triggered by an event with no pull request in its payload (`merge_group`, `push`, `workflow_dispatch`), which the gate cannot judge — use the trigger shape above.