diff --git a/.github/cursor-review/README.md b/.github/cursor-review/README.md index 355bb1f1..2528bbe8 100644 --- a/.github/cursor-review/README.md +++ b/.github/cursor-review/README.md @@ -300,6 +300,7 @@ jobs: | Kind | Name | Required | Purpose | |---|---|---|---| | Secret | `CURSOR_API_KEY` | **yes** | Bills the panel + judge `cursor-agent` calls. | +| Secret | `OPENAI_API_KEY` | no | Only with `openai_direct_model`: bills the optional direct-OpenAI cell. Use a project-scoped service-account key with a spend limit. | | Secret | `SLACK_BOT_TOKEN` | no | Enables start/complete DMs to the triggerer. | | Variable | `CURSOR_REVIEW_DM_EMAIL_MAP` | no | Maps GitHub logins → emails for Slack DM lookup. | @@ -362,6 +363,8 @@ All optional except `workflows_ref` (required, no default) — pass them under |---|---|---| | `judge_model` | `claude-opus-5-5-xhigh` | Model that consolidates panel findings. | | `panel_models` | `''` (built-in list) | JSON array of Cursor model ids that **replaces** the panel list; each still runs both review types and is validated against the live catalog by preflight. For per-repo experiments (e.g. `-xhigh` vs `-max` tiers). | +| `openai_direct_model` | `''` (off) | OPT-IN. An OpenAI model id reviewed through the OpenAI API directly (pinned Codex CLI, `package.json` here) as ONE **adversarial** cell on GitHub-hosted runners, by default **replacing** the Cursor `gpt-*` cell (next row). Needs the `OPENAI_API_KEY` secret and a plain id (`[A-Za-z0-9._-]`); without either, preflight warns and leaves the cell off and the panel unchanged. Not part of `panel_models` (not in the Cursor catalog); advisory — the leg never fails the run and is not counted in `Panel integrity` / the auto-approve gate. | +| `openai_direct_replaces_cursor_openai` | `true` | When the direct cell is in effect, drop the Cursor `gpt-*` cell(s) so the OpenAI lane is reviewed once. `false` runs both for a side-by-side backend comparison. Preflight fails rather than leave the Cursor panel empty. | | `skip_bot_branch_prefixes` | `ci/bump- chore/refresh- auto/refresh-` | Skip the panel when the PR author is a Bot **and** the head branch starts with one of these prefixes (machine pin bumps / catalog refreshes). `''` reviews every bot PR. | | `diff_size_cap` | `5000` | Max counted changed lines (after generated-file exclusion and comment discounting); larger PRs are skipped. | | `ignore_comments` | `true` | Discount blank/comment-only lines from the size count (count-only; the panel still sees them). | diff --git a/.github/cursor-review/package.json b/.github/cursor-review/package.json new file mode 100644 index 00000000..0918b43e --- /dev/null +++ b/.github/cursor-review/package.json @@ -0,0 +1,8 @@ +{ + "name": "comfy-cursor-review-direct-cli", + "private": true, + "description": "Manifest whose ONLY job is to be the single, Dependabot-visible source of truth for the @openai/codex CLI version that cursor-review.yml's optional direct-API review cell installs. Nothing is ever `npm install`ed from this directory: the job reads the pin out of this file and runs `npm install -g @openai/codex@`. It is a real npm manifest rather than a workflow `env:` constant purely so Dependabot's npm ecosystem can see the pin and open a bump PR — an inline `npm install -g pkg@1.2.3` inside a `run:` step is invisible to every ecosystem, which is how the groom CLI pin silently rotted (see .github/groom/package.json, the precedent this mirrors). Keep the version EXACT (no ^ or ~): the agent CLI is executable supply chain for a step that reads untrusted repo content, and the job rejects a non-exact pin at run time.", + "dependencies": { + "@openai/codex": "0.162.0" + } +} diff --git a/.github/cursor-review/tests/test_workflow_ref_guard_parity.py b/.github/cursor-review/tests/test_workflow_ref_guard_parity.py index d1c0fb0f..ff7feb18 100644 --- a/.github/cursor-review/tests/test_workflow_ref_guard_parity.py +++ b/.github/cursor-review/tests/test_workflow_ref_guard_parity.py @@ -42,7 +42,7 @@ # The guard steps. `.strip()`-compared so the assertion is about the step name, # not about the indentation of whichever job happens to hold it. GUARD_STEP_NAME = "- name: Require a pinned workflows_ref" -EXPECTED_GUARDS = 11 +EXPECTED_GUARDS = 12 # The ledger's exempt resolver, addressed by its step id rather than its name so # a comment rewrite above it cannot move the anchor. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ea1f9486..ab4bba61 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -141,6 +141,21 @@ updates: default-days: 7 semver-major-days: 14 + # Same pattern, same reasons, for the @openai/codex pin behind + # cursor-review.yml's opt-in direct-API cell. Its manifest installs nothing; + # the job reads the version out of it and rejects a non-exact pin at run + # time, hence `increase`. Without this entry the pin rots unwatched. + - package-ecosystem: "npm" + directory: "/.github/cursor-review" + schedule: + interval: "monthly" + labels: ["dependencies"] + versioning-strategy: "increase" + open-pull-requests-limit: 5 + cooldown: + default-days: 7 + semver-major-days: 14 + # The PyYAML + jsonschema pins the .coderabbit.yaml validator installs # (BE-7757). This is the repo's only Python dependency set, and it is # hash-pinned with `--require-hashes` — which is exactly why it needs watching: diff --git a/.github/groom/tests/test_claude_code_pin.py b/.github/groom/tests/test_claude_code_pin.py index b71b35fd..775c164d 100644 --- a/.github/groom/tests/test_claude_code_pin.py +++ b/.github/groom/tests/test_claude_code_pin.py @@ -503,12 +503,23 @@ def test_npm_entry_watches_the_groom_manifest(self): # checks would still pass if the npm block were moved off the manifest # while some other ecosystem entry happened to point at # `/.github/groom`, masking exactly the regression this guards. + self._assert_one_exact_npm_entry("/.github/groom") + + def test_npm_entry_watches_the_cursor_review_manifest(self): + # The same contract for cursor-review.yml's direct-API cell pin + # (`@openai/codex`), whose manifest mirrors groom's. + self._assert_one_exact_npm_entry("/.github/cursor-review") + + def _assert_one_exact_npm_entry(self, directory): entries = _dependabot_entries(_read(_DEPENDABOT)) - npm = [e for e in entries if e.get("package-ecosystem") == "npm"] + npm = [ + e for e in entries + if e.get("package-ecosystem") == "npm" and e.get("directory") == directory + ] self.assertEqual( - 1, len(npm), f"expected exactly one npm entry, found {len(npm)}" + 1, len(npm), + f"expected exactly one npm entry for {directory}, found {len(npm)}", ) - self.assertEqual("/.github/groom", npm[0].get("directory")) # The workflow guard rejects a range at run time, so the updater must be # told to keep an exact requirement exact rather than widening it. self.assertEqual("increase", npm[0].get("versioning-strategy")) diff --git a/.github/workflow-pins/tests/test_check_workflow_pins.py b/.github/workflow-pins/tests/test_check_workflow_pins.py index 927e2aec..6a8f516c 100644 --- a/.github/workflow-pins/tests/test_check_workflow_pins.py +++ b/.github/workflow-pins/tests/test_check_workflow_pins.py @@ -3504,7 +3504,7 @@ def test_this_repos_own_workflows_guard_every_ref_checkout(self): self.assertEqual(cwp.find_unguarded_ref_checkouts(lines), [], name) self.assertEqual( seen, - 22, + 23, "expected the 12 guarded sites BE-5546 fixed + pr-size.yml's (BE-5858) " "+ cursor-review.yml's preflight (hard guard) site picked up merging " "main + cursor-review.yml's diff-size job's check-pr-size-tool " @@ -3535,7 +3535,11 @@ def test_this_repos_own_workflows_guard_every_ref_checkout(self): "byte-identical copy of cursor-review.yml's guard. The 22nd is " "cursor-review.yml's `auto-retry` job (BE-19526): the round's " "relabel runs last, loads auto-approve.py from `workflows_ref` in " - "its own job, and carries its own copy of the guard.", + "its own job, and carries its own copy of the guard. The 23rd is " + "cursor-review.yml's opt-in `review-openai-direct` job (BE-19639): " + "the direct-OpenAI cell loads its prompt, fence-diff.py and " + "review-output-mcp.py from `workflows_ref` in its own job and " + "carries its own copy of the guard.", ) WORKFLOWS_DIR = os.path.normpath( diff --git a/.github/workflows/cursor-review.yml b/.github/workflows/cursor-review.yml index a169ae49..4437bd25 100644 --- a/.github/workflows/cursor-review.yml +++ b/.github/workflows/cursor-review.yml @@ -277,6 +277,39 @@ on: type: string required: false default: '' + openai_direct_model: + description: >- + OPT-IN. An OpenAI model id (e.g. `gpt-6.1-sol`) reviewed through the + OpenAI API directly, via the Codex CLI, as ONE adversarial cell. + Empty (default) = the job is skipped and nothing changes. Requires the + `OPENAI_API_KEY` secret and a plain id (`[A-Za-z0-9._-]`); set + without either, preflight warns, the cell stays off and the panel is + unchanged. + Deliberately NOT part of `panel_models`: that list is validated + against the live CURSOR catalog, which a direct-API id is not in. + Advisory — its findings reach the judge like any other cell's, but the + leg never fails the run and is not counted in `Panel integrity` or + the auto-approve gate. By default it REPLACES the Cursor OpenAI cell + rather than adding to it; see `openai_direct_replaces_cursor_openai`. + type: string + required: false + default: '' + openai_direct_replaces_cursor_openai: + description: >- + When the direct cell is in effect (`openai_direct_model` set, a plain + id, and its key present), drop the Cursor-hosted OpenAI cell + from the panel so the lane is reviewed once rather than twice. Default + true: the direct cell REPLACES it. Matching is by vendor prefix + (`gpt-`) against the resolved panel list, which is how every OpenAI id + in Cursor's catalog is spelled today — the other labs' ids are + untouched. Set false to run both for a side-by-side comparison of the + two backends on identical diffs, which is worth doing once before + trusting the replacement. Refuses to leave the Cursor panel empty: if + every remaining id would be dropped, preflight fails rather than + silently shrinking the panel to a single direct cell. + type: boolean + required: false + default: true skip_bot_branch_prefixes: description: >- Space-separated head-branch prefixes that skip the review when the PR @@ -454,6 +487,14 @@ on: CURSOR_API_KEY: description: Cursor API key for cursor-agent (the panel + judge models bill through it). required: true + OPENAI_API_KEY: + description: >- + OpenAI API key for the optional direct-API review cell. Required only + when the `openai_direct_model` input is set; ignored otherwise. Use a + service-account key scoped to its own project with a spend limit, not + a user key — the cell bills per call against it directly rather than + against a pooled seat. + required: false SLACK_BOT_TOKEN: description: Slack bot token for the start/complete DM notifications. Optional — DMs are skipped if absent. required: false @@ -603,7 +644,20 @@ jobs: should_run: ${{ steps.check.outputs.should_run }} already_reviewed: ${{ steps.dup.outputs.already_reviewed }} approve_author_ok: ${{ steps.author.outputs.approve_author_ok }} + openai_key_present: ${{ steps.openai_key.outputs.present }} steps: + - name: Resolve OPENAI_API_KEY presence + # A boolean for `preflight`, which cannot read the secret itself: it + # follows the caller's `runs_on` pool, and a `secrets.` expression is + # evaluated ON the runner, so even `!= ''` would ship the key there. + # This job is GitHub-hosted and checks out nothing, so the key meets + # no PR-authored code here. Only read when the direct cell is opted in. + id: openai_key + if: inputs.openai_direct_model != '' + env: + OPENAI_KEY_PRESENT: ${{ secrets.OPENAI_API_KEY != '' }} + run: echo "present=${OPENAI_KEY_PRESENT}" >> "$GITHUB_OUTPUT" + - name: Resolve the auto-approve author opt-in # approve_authors, resolved ONCE here so the workflow-level # `approve_gate` output (which must say `off` on every event, round or @@ -1928,6 +1982,12 @@ jobs: contents: read outputs: models: ${{ steps.models.outputs.models }} + # 'true' only when the direct-API cell can actually run: a model id is + # set, it is artifact-name safe, and OPENAI_API_KEY reached this + # workflow. `review-openai-direct` gates on THIS rather than on the raw + # input, so a misconfigured opt-in neither drops the Cursor OpenAI cell + # nor schedules a cell that can only error. + openai_direct: ${{ steps.models.outputs.openai_direct }} steps: - name: Define panel models id: models @@ -1996,6 +2056,15 @@ jobs: # Which of the two won, for the log line below. With two sources the # effective list alone no longer says where to go change it. PANEL_MODELS_INPUT: ${{ inputs.panel_models }} + # Drive the direct-cell replacement below. Read here rather than in + # the `review` job so ONE resolved list feeds the matrix, the judge's + # `expected` set and `Panel integrity` alike — filtering anywhere else + # would leave consolidate expecting a cell that was never scheduled. + OPENAI_DIRECT_MODEL: ${{ inputs.openai_direct_model }} + OPENAI_DIRECT_REPLACES: ${{ inputs.openai_direct_replaces_cursor_openai }} + # Resolved in the hosted `gate` job: this one follows `runs_on`, so + # it must not reference the secret at all. + OPENAI_KEY_PRESENT: ${{ needs.gate.outputs.openai_key_present }} run: | cat > /tmp/models.json <<'JSON' [ @@ -2046,6 +2115,45 @@ jobs: # rounds either side of it look the same. echo "panel_models not set (no input, no CURSOR_PANEL_MODELS variable) — using the built-in list: $(jq -c . /tmp/models.json)" fi + # The direct-API cell reviews the OpenAI lane itself, so by default the + # Cursor-hosted OpenAI cell comes OUT of the panel — otherwise that lane + # is reviewed twice and billed twice. Applied after the override has + # been resolved, so it filters whichever list actually won. + # + # Resolved to ONE verdict first, because replacing the Cursor cell is + # only safe when the direct cell will genuinely run: a set input with + # no key, or an id `upload-artifact` would reject (a fine-tuned + # `ft:...` id carries `:`), would otherwise drop the OpenAI lane AND + # leave nothing reviewing it. Either case warns and leaves the panel + # exactly as it would be without the opt-in. + openai_direct=false + if [ -n "$(printf '%s' "${OPENAI_DIRECT_MODEL:-}" | tr -d '[:space:]')" ]; then + if ! printf '%s' "$OPENAI_DIRECT_MODEL" | grep -Eqx '[A-Za-z0-9._-]{1,100}'; then + echo "::warning::openai_direct_model is not a plain model id (letters, digits, '.', '_', '-'; at most 100), so the direct-API cell is off for this run and the panel is unchanged." + elif [ "${OPENAI_KEY_PRESENT:-false}" != "true" ]; then + echo "::warning::openai_direct_model is set but the OPENAI_API_KEY secret did not reach this workflow, so the direct-API cell is off for this run and the panel is unchanged. Pass the secret through from the caller to enable it." + else + openai_direct=true + fi + fi + echo "openai_direct=${openai_direct}" >> "$GITHUB_OUTPUT" + if [ "$openai_direct" = "true" ] \ + && [ "${OPENAI_DIRECT_REPLACES:-true}" = "true" ]; then + # ascii_downcase so a differently-cased id cannot slip past the + # vendor match and leave a duplicate OpenAI cell running. + dropped="$(jq -c '[.[] | select((ascii_downcase | startswith("gpt-")))]' /tmp/models.json)" + kept="$(jq -c '[.[] | select((ascii_downcase | startswith("gpt-")) | not)]' /tmp/models.json)" + if [ "$(printf '%s' "$dropped" | jq 'length')" -gt 0 ]; then + if [ "$(printf '%s' "$kept" | jq 'length')" -eq 0 ]; then + echo "::error::openai_direct_model is set and openai_direct_replaces_cursor_openai is true, but every id in the panel list is an OpenAI id, so replacing them would leave no Cursor cell at all. Set openai_direct_replaces_cursor_openai: false, or widen panel_models." + exit 1 + fi + printf '%s' "$kept" > /tmp/models.json + echo "direct-API cell is on, so the Cursor OpenAI cell(s) $dropped were dropped from the panel; remaining: $kept" + else + echo "direct-API cell is on, but the panel list carries no Cursor OpenAI id to replace; panel unchanged." + fi + fi echo "models=$(jq -c . /tmp/models.json)" >> "$GITHUB_OUTPUT" - name: Require a pinned workflows_ref @@ -2630,6 +2738,366 @@ jobs: fi echo "Reviewer cell ${REVIEW_TYPE}/${MODEL} submitted its findings." + review-openai-direct: + # An OPT-IN adversarial cell that reaches its model through the OpenAI API + # directly, via the Codex CLI, instead of through Cursor. By default it + # REPLACES the Cursor-hosted OpenAI cell (preflight drops that id from the + # panel list) so the lane is reviewed once, not twice; set + # `openai_direct_replaces_cursor_openai: false` to run both for a + # side-by-side comparison. Default OFF: + # `openai_direct_model` is '' unless a caller sets it, and this job is then + # skipped outright, so every enrolled repo sees exactly today's behaviour + # until it opts in. + # + # Why a SEPARATE job rather than another entry in the `review` matrix: + # that matrix's model list is validated against the live CURSOR catalog in + # `preflight`, and a delisted id there fails the run loud by design + # (BE-2627). A direct-API id is not in Cursor's catalog at all, so feeding + # it to that list would fail preflight for the right reason and the wrong + # cause. Separation also means a failure here can neither cancel nor skip + # a Cursor matrix leg. + # + # How the judge picks it up: `consolidate` downloads `pattern: findings-*` + # and globs `findings-*/findings.json`, so this cell's artifact + # (`findings-direct-...`) is read alongside the others, and `consolidate` + # lists this job in `needs:` so it never downloads before the upload + # lands. The `direct-` prefix is what `Aggregate panel findings` keys on + # to keep this cell OUT of the Cursor panel's `expected` set and its + # ok_count/total — the numbers `Panel integrity` and the auto-approve gate + # read — so an opt-in cell can inform the judge without ever gating. + # + # Credential posture: `contents: read` only, no write-scoped token (see + # SECURITY.md), and the only PR-authored content the agent sees is the + # same nonce-fenced diff text — this job does NOT check out the PR. The + # OpenAI key reaches `codex exec` as CODEX_API_KEY in that ONE step's env + # and is never written to disk (no `codex login`, whose auth.json the + # read-only sandbox could still read), and the sandboxed shell's env is + # cut to the core set so the agent cannot echo it back. + # + # ADVISORY on purpose: `continue-on-error` at the JOB level, so no step in + # here — install, version check, prompt build, upload — can turn the + # caller's run red while this backend is still being characterised. + # + # v1 scope: adversarial only (not edge-case), one model, one round. + name: 'adversarial direct (${{ inputs.openai_direct_model }})' + needs: [gate, diff-size, preflight, ledger] + if: >- + needs.preflight.outputs.openai_direct == 'true' + && needs.gate.outputs.should_run == 'true' + && needs.gate.outputs.already_reviewed != 'true' + && needs.diff-size.outputs.within_cap == 'true' + # GitHub-hosted, NOT the caller's `runs_on` pool: this job holds + # OPENAI_API_KEY, and a pool is where a prompt-injected model cell may + # have run. Only CURSOR_API_KEY-only, read-only jobs may follow `runs_on`. + runs-on: ubuntu-latest + continue-on-error: true + # Matches the `review` job's cap so the judge's budget reasoning (see + # `review`) holds for this cell too; the agent step caps lower, at 15. + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Seed default findings artifact + # Same contract as the `review` job: pre-seed status=error so the cell + # ALWAYS appears in the panel summary even if a later step dies. The + # MCP output tool overwrites this on success. + env: + MODEL: ${{ inputs.openai_direct_model }} + run: | + mkdir -p /tmp/findings-out + python3 - <<'PY' + import json, os + record = { + "model": os.environ["MODEL"], + "review_type": "adversarial", + "status": "error", + "error": "Direct-API cell did not reach the submission step.", + "findings": [], + } + with open("/tmp/findings-out/findings.json", "w", encoding="utf-8") as f: + json.dump(record, f, indent=2) + PY + + - name: Require a pinned workflows_ref + # workflows_ref has no default on purpose. GitHub does NOT enforce + # `required: true` for workflow_call inputs, so an omitted input arrives + # as '' and actions/checkout would silently fall back to this repo's + # default branch — running MUTABLE scripts under a pin that claims + # otherwise. Fail fast instead. (BE-5546) + env: + WORKFLOWS_REF: ${{ inputs.workflows_ref }} + WORKFLOW_SHA: ${{ job.workflow_sha }} + run: | + # actions/checkout reads `ref` through core.getInput, which TRIMS, so a + # whitespace-only value is an empty ref to IT while sailing past a bare + # -z test here. Compare the stripped form, and echo only that: dropping + # newlines also stops a multi-line value from smuggling a ::workflow + # command:: into the log, and from satisfying the line-oriented grep + # below on one 40-hex line among many. + REF="$(printf '%s' "$WORKFLOWS_REF" | tr -d '[:space:]')" + if [ -z "$REF" ]; then + echo "::error::workflows_ref is required; pin it to the same commit SHA as the uses: line (see header example)" + exit 1 + fi + if ! printf '%s' "$REF" | grep -Eq '^[0-9a-f]{40}$'; then + echo "::warning::workflows_ref '$REF' is not a full 40-hex commit SHA — branch and tag refs are mutable and can skew between jobs mid-run" + fi + # Equality with the commit the caller's `uses:` line resolved to — the + # same guard public-repo-hygiene.yml runs, and the check that turns + # "workflows_ref must be the same SHA as uses:" from a header comment + # into a fact. `job.workflow_sha` is computed by the RUNNER from the + # `uses:` pin, so a caller cannot set it. Without this, a caller can + # pin `uses:` to one commit and load this workflow's prompts, scripts + # and model list from another. (`job.workflow_sha`, NOT + # `github.job_workflow_sha` — that one is an OIDC token claim, absent + # from the `github` context, and expands to ''.) Both sides are + # lower-cased: hex is case-insensitive, and unlike hygiene's the + # 40-hex test above only WARNS, so an upper-case ref naming the RIGHT + # commit reaches here and must not fail. $REF itself is left alone so + # the messages quote what the caller wrote. (BE-15927) + REF_LC="$(printf '%s' "$REF" | tr 'A-Z' 'a-z')" + SHA="$(printf '%s' "$WORKFLOW_SHA" | tr -d '[:space:]' | tr 'A-Z' 'a-z')" + if [ -z "$SHA" ]; then + # Cannot check, NOT a mismatch. `job.workflow_sha` is the `job`-context + # accessor added in Actions runner v2.334.0, so only an older runner + # gets here. GitHub stops queueing jobs to a self-hosted runner more + # than 30 days behind the latest release, so on any pool — hosted + # `ubuntu-latest`, or the caller's `runs_on` pool the read-only + # model jobs may use — that is a transient window. Warn and continue: a + # property nothing could EVALUATE is not evidence of a violation, and + # failing would take the whole review down on a caller that changed + # nothing. (public-repo-hygiene.yml fails closed in this branch on + # purpose — it is the checker whose own integrity rests on the + # equality. Here the ledger's warn-and-degrade posture is the fit.) + echo "::warning::job.workflow_sha is empty (Actions runner older than v2.334.0?), so workflows_ref could not be cross-checked against the uses: pin. The empty and 40-hex tests above still apply." + elif [ "$REF_LC" != "$SHA" ]; then + echo "::error::workflows_ref ($REF) is not the commit this workflow was called at ($SHA). The uses: pin and workflows_ref MUST name the same commit — otherwise the workflow is pinned but its prompts and scripts are loaded from a different version. Set both to $SHA. (The bump-callers fleet rewrites both together; Dependabot rewrites only uses:, so a caller that lets Dependabot bump this reusable needs a dependabot.yml ignore entry for Comfy-Org/github-workflows.)" + exit 1 + fi + + - name: Load cursor-review assets + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: Comfy-Org/github-workflows + ref: ${{ inputs.workflows_ref }} + path: _cursor_review_assets + persist-credentials: false + + - name: Download reviewed diff + # Built once in the diff-size job (generated files already excluded), + # shared here, so this cell needs no PR checkout of its own. + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: pr-diff + path: /tmp + + - name: Download prior-review ledger + # Same artifact, same tolerance as the `review` job: absent on round 1. + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: cursor-review-ledger + path: /tmp/ledger + + - name: Check the API key is present + id: key + # Belt and braces: preflight already keeps this job off when the key is + # absent. Should the two ever disagree, this attributes the errored + # record clearly rather than leaving a confusing CLI auth failure. + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + if [ -z "${OPENAI_API_KEY:-}" ]; then + echo "::warning::openai_direct_model is set but the OPENAI_API_KEY secret is absent, so the direct-API cell is skipped. Pass the secret through from the caller to enable it." + python3 -c 'import json; p = "/tmp/findings-out/findings.json"; r = json.load(open(p)); r["error"] = "OPENAI_API_KEY secret was not provided to the reusable workflow."; json.dump(r, open(p, "w"), indent=2)' + echo "present=false" >> "$GITHUB_OUTPUT" + else + echo "present=true" >> "$GITHUB_OUTPUT" + fi + + - name: Install Codex CLI + if: steps.key.outputs.present == 'true' + # The pin lives in .github/cursor-review/package.json in THIS repo and + # arrives with the assets checkout above, so it is resolved at the same + # SHA as the prompts and scripts. Mirrors the groom precedent exactly, + # including rejecting a non-exact pin: an inline `npm install -g + # pkg@1.2.3` is invisible to every dependency ecosystem. + run: | + manifest="$CURSOR_REVIEW_ASSETS/package.json" + if [ ! -f "$manifest" ]; then + echo "::error::$manifest is missing — it is the single source of truth for the direct-API cell's CLI pin." + exit 1 + fi + if ! version=$(jq -r ' + if type != "object" then "" + elif (.dependencies | type) != "object" then "" + else (.dependencies["@openai/codex"] // "") + end' "$manifest"); then + echo "::error::$manifest is not valid JSON — it is the single source of truth for the direct-API cell's CLI pin." + exit 1 + fi + version="$(printf '%s' "$version" | tr -d '\r\n')" + case "$version" in + ""|*[!0-9.]*) + echo "::error::dependencies[\"@openai/codex\"] must be an EXACT version (digits and dots only, no ^ or ~); got a value this job will not install." + exit 1 ;; + esac + echo "CODEX_CLI_VERSION=$version" >> "$GITHUB_ENV" + npm install -g "@openai/codex@${version}" + + - name: Verify Codex CLI version + if: steps.key.outputs.present == 'true' + # Re-asserted on the PATH-resolved binary immediately before the review, + # so a swap between install and use cannot hand the review to unpinned + # bits, and the version lands in the log for forensics. + run: | + # stdout only, first line (`codex-cli 0.162.0`): a stray stderr + # notice must not become the last field compared below. + installed="$(codex --version 2>/dev/null | head -n 1 | tr -d '\r' | awk '{print $NF}')" + if [ "$installed" != "$CODEX_CLI_VERSION" ]; then + echo "::error::codex reports '${installed}', expected the pinned '${CODEX_CLI_VERSION}'." + exit 1 + fi + echo "codex ${installed}" + + - name: Build prompt + if: steps.key.outputs.present == 'true' + # Byte-identical construction to the Cursor cells' `Build prompt`, so a + # finding difference is attributable to the model or the backend rather + # than to the prompt: the same prompt-adversarial.md with the same + # prior-review ledger spliced in, the same marker stripped and + # re-emitted by fence-diff.py, the same new-hunks block from round 2 + # on, and a fresh per-prompt nonce minted into a shell variable (never + # a step `env:`) so a leaked marker from one cell cannot forge a fence + # in another. See that step for the full rationale. + env: + LEDGER_ROUNDS: ${{ needs.ledger.outputs.rounds }} + run: | + python3 "$CURSOR_REVIEW_ASSETS/build-ledger.py" splice \ + --prompt "$CURSOR_REVIEW_ASSETS/prompt-adversarial.md" \ + --marker '=== BEGIN DIFF ===' \ + --insert /tmp/ledger/ledger.md \ + --out /tmp/prompt-head.txt + diff_nonce="$(python3 "$CURSOR_REVIEW_ASSETS/fence-diff.py" mint)" + python3 "$CURSOR_REVIEW_ASSETS/fence-diff.py" strip-marker \ + --file /tmp/prompt-head.txt \ + --marker '=== BEGIN DIFF ===' + { + cat /tmp/prompt-head.txt + python3 "$CURSOR_REVIEW_ASSETS/fence-diff.py" emit \ + --body /tmp/pr-diff.patch --label DIFF --nonce "$diff_nonce" + if [ -s /tmp/pr-diff-new.patch ]; then + echo "" + echo "The block below is the subset of the diff above that changed since" + echo "the last reviewed commit. Prioritize it — but the full diff above" + echo "is still in scope." + python3 "$CURSOR_REVIEW_ASSETS/fence-diff.py" emit \ + --body /tmp/pr-diff-new.patch \ + --label "HUNKS NEW SINCE ROUND ${LEDGER_ROUNDS:-?}" \ + --nonce "$diff_nonce" + fi + } > /tmp/prompt.txt + + - name: Run direct-API review + if: steps.key.outputs.present == 'true' + continue-on-error: true + # Capped well under the job so a hung agent trips the STEP and the + # seeded artifact still uploads below, rather than tripping the job and + # discarding the cell. Same 15 minutes the Cursor cells get. + timeout-minutes: 15 + env: + MODEL: ${{ inputs.openai_direct_model }} + # `codex exec` reads its API key from this variable, so the key lives + # only in this step's process env: never argv (visible in the process + # list), never a file (`codex login` writes ~/.codex/auth.json, which + # the read-only sandbox can still READ, so a prompt-injected agent + # could encode it into a finding). Verified against 0.162.0. + CODEX_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + mkdir -p /tmp/findings-out + # A fresh, empty home under RUNNER_TEMP (emptied after every job): no + # stale credential or config from anything else that ran on this + # runner, nothing persisted for the next, and not under /tmp, where + # 0.162.0 refuses to create its helper binaries. + CODEX_HOME="$(mktemp -d -p "$RUNNER_TEMP")" + export CODEX_HOME + python3 "$CURSOR_REVIEW_ASSETS/review-output-mcp.py" \ + --mode reviewer \ + --out /tmp/findings-out/findings.json \ + --model "$MODEL" \ + --review-type adversarial \ + --init + # The SAME MCP submission tool the Cursor cells use, so the findings + # schema the judge reads is produced by one implementation for both + # backends. `default_tools_approval_mode=approve` is required: under + # `codex exec`'s default policy the tool calls are rejected and the + # cell submits nothing. `shell_environment_policy.inherit=core` hands + # the agent's shell only HOME/PATH/etc., never CODEX_API_KEY. + args_toml="$(python3 - <<'PY' + import json, os + print(json.dumps([ + os.path.join(os.environ["CURSOR_REVIEW_ASSETS"], "review-output-mcp.py"), + "--mode", "reviewer", + "--out", "/tmp/findings-out/findings.json", + "--model", os.environ["MODEL"], + "--review-type", "adversarial", + ])) + PY + )" + codex exec --json --ignore-user-config --skip-git-repo-check \ + -s read-only \ + -c shell_environment_policy.inherit=core \ + -c model="$MODEL" \ + -c model_reasoning_effort=max \ + -c mcp_servers.cursor_review_output.command=python3 \ + -c mcp_servers.cursor_review_output.default_tools_approval_mode=approve \ + -c "mcp_servers.cursor_review_output.args=$args_toml" \ + < /tmp/prompt.txt > /tmp/codex-events.jsonl + + - name: Report cell outcome + if: always() + env: + MODEL: ${{ inputs.openai_direct_model }} + # Status and a count only — NOT the findings verbatim, unlike the Cursor + # cells: the run log is public, and finding bodies are model output + # steered by PR-authored text. The full record still reaches the judge + # through the artifact below. + run: | + read -r status count < <(python3 - <<'PY' + import json, re + try: + with open("/tmp/findings-out/findings.json", encoding="utf-8") as f: + cell = json.load(f) + except (OSError, ValueError): + cell = None + value = cell.get("status") if isinstance(cell, dict) else None + if not isinstance(value, str): + value = "error" + findings = cell.get("findings") if isinstance(cell, dict) else None + print(re.sub(r"[^A-Za-z0-9_.-]", "_", value)[:32] or "error", + len(findings) if isinstance(findings, list) else 0) + PY + ) + echo "Direct-API cell: status=${status}, ${count} finding(s)." + if [ "$status" != "ok" ]; then + echo "::warning::Direct-API cell adversarial/${MODEL} did not submit (status=$status). Its record still reaches the judge; this leg stays green because the cell is advisory." + else + echo "Direct-API cell adversarial/${MODEL} submitted its findings." + fi + + - name: Upload findings artifact + if: always() + # `findings-*` so `consolidate` globs it; `direct-` so it can never + # collide with a Cursor cell's `findings--` (with + # replacement off the two may share an id) and so `consolidate` can + # tell it apart. The id is charset-checked in preflight. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: findings-direct-adversarial-${{ inputs.openai_direct_model }} + path: /tmp/findings-out/findings.json + if-no-files-found: error + retention-days: 7 + consolidate: # Adjudicates the panel cell artifacts into one consolidated review. # Calls cursor-agent one more time as the judge, with prompt-judge.md. @@ -2639,7 +3107,10 @@ jobs: # "Credential isolation" note for why the judge must not share a job with # a write-scoped token. name: Consolidate panel - needs: [gate, diff-size, preflight, review, ledger] + # `review-openai-direct` is here only for ORDERING — its artifact must land + # before `Download panel findings` runs. Its result is never read: when it + # is skipped (the default) or fails, `!cancelled()` below still runs. + needs: [gate, diff-size, preflight, review, review-openai-direct, ledger] # `!cancelled()`, not `always()`: the judge is the most expensive call in # the workflow, and the only job that does anything with its payload, # `post-review`, is itself `!cancelled()`. On a cancelled run — now an @@ -2852,6 +3323,12 @@ jobs: import json, os, glob panel = [] + # The opt-in direct-API cell (`findings-direct-*`). Its findings go to + # the judge like any cell's, but it is NOT a Cursor panel cell: it is + # kept out of `present` (so it cannot mask a missing Cursor cell with + # the same id) and out of ok_count/total, which `Panel integrity` and + # the auto-approve gate read — an advisory cell must not gate. + direct = [] for path in sorted(glob.glob("/tmp/panel/findings-*/findings.json")): try: with open(path, encoding="utf-8") as f: @@ -2859,7 +3336,13 @@ jobs: except (OSError, json.JSONDecodeError): continue if isinstance(cell, dict): - panel.append(cell) + if os.path.basename(os.path.dirname(path)).startswith("findings-direct-"): + direct.append(cell) + else: + # Only THIS step may set the marker: a Cursor cell's own + # record must never be able to excuse itself from the count. + cell.pop("direct_api", None) + panel.append(cell) expected = { (model, review_type) @@ -2879,11 +3362,16 @@ jobs: "findings": [], }) - with open("/tmp/panel.json", "w", encoding="utf-8") as f: - json.dump(panel, f, indent=2) - ok = sum(1 for c in panel if c.get("status") == "ok") print(f"Panel: {ok}/{len(panel)} cells contributed findings.") + for c in direct: + print(f"Direct-API cell (advisory, not counted): status={c.get('status')}.") + # Read by `Build consolidated findings file`, which keeps these + # out of the `panel` metadata auto-approve counts failures from. + c["direct_api"] = True + + with open("/tmp/panel.json", "w", encoding="utf-8") as f: + json.dump(panel + direct, f, indent=2) # Emit metadata for the next step to gate on. with open(os.environ["GITHUB_OUTPUT"], "a") as g: @@ -3010,9 +3498,13 @@ jobs: with open("/tmp/panel.json", encoding="utf-8") as f: panel = json.load(f) + # Cursor panel cells only: auto-approve counts failed reviewers from + # this list, and the advisory direct-API cell must not gate. Its + # findings still reach the degraded fallback below via `panel`. panel_meta = [ {"model": c.get("model"), "review_type": c.get("review_type"), "status": c.get("status")} for c in panel + if c.get("direct_api") is not True ] with open("/tmp/judge-findings.json", encoding="utf-8") as f: judge = json.load(f) diff --git a/docs/callers/cursor-review.md b/docs/callers/cursor-review.md index 2220ea04..6148a5d7 100644 --- a/docs/callers/cursor-review.md +++ b/docs/callers/cursor-review.md @@ -52,6 +52,7 @@ Prompts and scripts live in [`.github/cursor-review/`](../../.github/cursor-revi | | | |---|---| | `secrets.CURSOR_API_KEY` | **Required.** Org-level in Comfy-Org. | +| `secrets.OPENAI_API_KEY` | Optional. Only read when `openai_direct_model` is set; use a project-scoped service-account key with a spend limit. | | `secrets.SLACK_BOT_TOKEN` | Optional. Without it the review still posts; only the DMs are skipped. | | `vars.REVIEW_BOT_APP_ID` + `secrets.BOT_APP_PRIVATE_KEY` | Optional. Posts the review as your App instead of `github-actions[bot]`. | | A review label | Default `cursor-review`. Create it in your repo. | @@ -112,6 +113,8 @@ pull-requests: write # posting the consolidated review (and, at the round cap, |---|---|---| | `judge_model` | `claude-opus-5-5-xhigh` | Consolidates the panel into one review. | | `panel_models` | `''` | JSON array of model ids replacing the built-in panel list (each runs both review types; preflight checks each one exists in the live catalog — existence only, not ZDR). Use for per-repo experiments such as a reasoning-tier A/B — and prefer wiring it to a **variable** rather than a literal, see [Running a model experiment](#running-a-model-experiment). | +| `openai_direct_model` | `''` | Opt-in. An OpenAI model id reviewed directly through the OpenAI API (pinned Codex CLI) as one **adversarial** cell, always on GitHub-hosted `ubuntu-latest`. By default it **replaces** the Cursor-hosted OpenAI (`gpt-*`) panel cell — see the next row. Requires `secrets.OPENAI_API_KEY` and a plain id (`[A-Za-z0-9._-]`); without either, preflight warns, the cell stays off and the panel is unchanged. Advisory: its findings reach the judge, but the leg never fails the run and is not counted in `Panel integrity` or the auto-approve gate. | +| `openai_direct_replaces_cursor_openai` | `true` | Only with `openai_direct_model` (and its key) in effect: drop the Cursor `gpt-*` cell(s) from the panel so the OpenAI lane is reviewed once. `false` runs both, for a side-by-side comparison of the two backends. Preflight fails rather than leave the Cursor panel empty. | | `skip_bot_branch_prefixes` | `ci/bump- chore/refresh- auto/refresh-` | Skip the panel for Bot-authored PRs on these branch prefixes (machine pin bumps / refreshes). `''` to review every bot PR. | | `diff_size_cap` | `5000` | Skip review above this diff size. An over-cap PR is not silent — see the gotcha below. Under `blocking: true` it is also not green: an unreviewed PR cannot pass the gate. | | `ignore_comments` | `true` | Discount blank/comment-only lines from the size count (count-only — the panel still sees them). |