From 911d0e84477eda35857e45b8eacfa9ab42cc0a71 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 1 Nov 2026 05:29:52 -0800 Subject: [PATCH] ci: merge backports only when approved at the current head (#22569) ## Summary The backport sweep used `reviewDecision == APPROVED`, but release-branch protection can preserve that decision after the head changes. On Oct 30 the sweep merged #22422 with its only approval one commit behind the merged head. ## Changes - Require the latest non-comment review from a write-capable reviewer to approve the current head. - Pin each merge attempt with `--match-head-commit`. - Fail closed and report review or permission lookup errors. - Merge only the currently authorized frontend trains: `core/1.56` through `core/1.59` and `cloud/1.56` through `cloud/1.59`. Retired 1.54/1.55 and unknown future trains are rejected until the allowlist is deliberately updated. ## Tests `NODE_OPTIONS=--no-experimental-webstorage pnpm vitest run scripts/cicd/backport-auto-merge.test.ts` The 23 cases cover all eight authorized bases, retired and unknown bases, stale/current/dismissed/comment-only reviews, insufficient reviewer permission, API failures, pagination behavior, and head-pinned merge arguments. ## Review focus Version-bump PRs with the `Release` label follow the same exact-head and active-train rules. --- .github/workflows/backport-auto-merge.yaml | 32 ++- scripts/cicd/backport-auto-merge.test.ts | 282 +++++++++++++++++++++ 2 files changed, 310 insertions(+), 4 deletions(-) create mode 100644 scripts/cicd/backport-auto-merge.test.ts diff --git a/.github/workflows/backport-auto-merge.yaml b/.github/workflows/backport-auto-merge.yaml index c8d9273f442..bc88125e649 100644 --- a/.github/workflows/backport-auto-merge.yaml +++ b/.github/workflows/backport-auto-merge.yaml @@ -117,7 +117,7 @@ jobs: echo "::group::PR #${pr}" info=$(GH_TOKEN="$READ_TOKEN" gh pr view "$pr" --repo "$GH_REPO" \ - --json number,state,isDraft,labels,baseRefName,reviewDecision,mergeStateStatus 2>/dev/null || echo '') + --json number,state,isDraft,labels,baseRefName,reviewDecision,mergeStateStatus,headRefOid 2>/dev/null || echo '') if [ -z "$info" ]; then echo "Could not read PR #${pr} — skipping."; echo "::endgroup::"; continue fi @@ -139,8 +139,8 @@ jobs: echo "::endgroup::"; continue fi case "$base" in - cloud/*|core/*) : ;; - *) echo "Base '$base' is not a release branch — skipping."; echo "::endgroup::"; continue ;; + cloud/1.56|core/1.56|cloud/1.57|core/1.57|cloud/1.58|core/1.58|cloud/1.59|core/1.59) : ;; + *) echo "Base '$base' is not an active release branch — skipping."; echo "::endgroup::"; continue ;; esac # Ready = approved AND GitHub says it's mergeable with required checks green. @@ -157,6 +157,30 @@ jobs: echo "::endgroup::"; continue fi + head=$(echo "$info" | jq -r '.headRefOid') + if ! reviews=$(GH_TOKEN="$READ_TOKEN" gh api "repos/${GH_REPO}/pulls/${pr}/reviews" --paginate 2>&1); then + echo "Could not read reviews for PR #${pr}: ${reviews}" + echo "::endgroup::"; continue + fi + head_approvers=$(echo "$reviews" | jq -rs --arg head "$head" ' + [.[][] | select(.state != "COMMENTED" and .state != "PENDING")] + | group_by(.user.login) | map(last) + | .[] | select(.state == "APPROVED" and .commit_id == $head) | .user.login' 2>/dev/null || true) + approved_at_head=false + for login in $head_approvers; do + if ! permission=$(GH_TOKEN="$READ_TOKEN" gh api "repos/${GH_REPO}/collaborators/${login}/permission" --jq '.permission' 2>&1); then + echo "Could not read permission for reviewer ${login} on PR #${pr}: ${permission}" + continue + fi + case "$permission" in + admin|maintain|write) approved_at_head=true; break ;; + esac + done + if [ "$approved_at_head" != "true" ]; then + echo "Approved, but no approval at head ${head} — waiting for a re-approval." + echo "::endgroup::"; continue + fi + echo "PR #${pr} is ready — attempting squash merge." attempt=0 max=3 @@ -165,7 +189,7 @@ jobs: attempt=$((attempt + 1)) # A concurrent run (or a human) may have merged it already. if is_merged "$pr"; then merged=true; break; fi - if out=$(GH_TOKEN="$MERGE_TOKEN" gh pr merge "$pr" --repo "$GH_REPO" --squash 2>&1); then + if out=$(GH_TOKEN="$MERGE_TOKEN" gh pr merge "$pr" --repo "$GH_REPO" --squash --match-head-commit "$head" 2>&1); then merged=true; break fi echo "Merge attempt ${attempt}/${max} failed: ${out}" diff --git a/scripts/cicd/backport-auto-merge.test.ts b/scripts/cicd/backport-auto-merge.test.ts new file mode 100644 index 00000000000..9de30693c2b --- /dev/null +++ b/scripts/cicd/backport-auto-merge.test.ts @@ -0,0 +1,282 @@ +import { spawnSync } from 'node:child_process' +import { + chmodSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it, onTestFinished } from 'vitest' +import { parse } from 'yaml' + +interface WorkflowStep { + name?: string + run?: string +} + +interface Workflow { + jobs?: Partial> +} + +const workflow = parse( + readFileSync('.github/workflows/backport-auto-merge.yaml', 'utf8') +) as Workflow +const mergeScript = + workflow.jobs?.merge?.steps?.find( + (step) => step.name === 'Merge eligible backport PRs' + )?.run ?? '' + +const HEAD = 'b'.repeat(40) +const OLD = 'a'.repeat(40) +const AUTHORIZED_BASES = [ + 'cloud/1.56', + 'core/1.56', + 'cloud/1.57', + 'core/1.57', + 'cloud/1.58', + 'core/1.58', + 'cloud/1.59', + 'core/1.59' +] +const UNAUTHORIZED_BASES = [ + 'cloud/1.54', + 'core/1.54', + 'cloud/1.55', + 'core/1.55', + 'cloud/1.60', + 'core/1.60' +] + +const fakeGh = `#!/usr/bin/env bash +set -euo pipefail +echo "$*" >> "$FAKE_CALLS" +case "$1 $2" in + "pr view") + jq_filter='' + while [ $# -gt 0 ]; do + case "$1" in --jq) jq_filter="$2"; shift 2 ;; *) shift ;; esac + done + if [ -n "$jq_filter" ]; then jq -r "$jq_filter" "$FAKE_PR"; else cat "$FAKE_PR"; fi ;; + "pr merge") + jq '.state = "MERGED"' "$FAKE_PR" > "$FAKE_PR.tmp" && mv "$FAKE_PR.tmp" "$FAKE_PR" ;; + "pr comment") : ;; + api\\ *) + case "$2" in + */reviews) + [ -z "\${FAKE_REVIEWS_FAIL:-}" ] || { echo 'API rate limit exceeded (HTTP 403)' >&2; exit 1; } + cat "$FAKE_REVIEWS" ;; + */collaborators/*/permission) + [ -z "\${FAKE_PERMISSION_FAIL:-}" ] || { echo 'API rate limit exceeded (HTTP 403)' >&2; exit 1; } + login="\${2%/permission}" login="\${login##*/}" + jq -rn --arg login "$login" --slurpfile reviews "$FAKE_REVIEWS" ' + [$reviews[0][] | select(.user.login == $login)][0].permission // "none"' ;; + */comments) echo '[]' ;; + *) echo "unexpected gh api call: $2" >&2; exit 2 ;; + esac ;; + *) echo "unexpected gh call: $*" >&2; exit 2 ;; +esac +` + +interface Review { + state: string + commit_id: string + login?: string + author_association?: string + permission?: string +} + +function runMerge( + reviews: Review[], + { + baseRefName = 'core/1.59', + permissionFail = false, + reviewsFail = false + } = {} +) { + const directory = mkdtempSync(join(tmpdir(), 'backport-auto-merge-')) + onTestFinished(() => rmSync(directory, { recursive: true, force: true })) + const calls = join(directory, 'calls') + writeFileSync(calls, '') + writeFileSync(join(directory, 'gh'), fakeGh) + chmodSync(join(directory, 'gh'), 0o755) + writeFileSync( + join(directory, 'pr.json'), + JSON.stringify({ + number: 22422, + state: 'OPEN', + isDraft: false, + labels: [{ name: 'backport' }], + baseRefName, + reviewDecision: 'APPROVED', + mergeStateStatus: 'CLEAN', + headRefOid: HEAD + }) + ) + writeFileSync( + join(directory, 'reviews.json'), + JSON.stringify( + reviews.map( + ({ + login = 'reviewer', + author_association = 'MEMBER', + permission = 'write', + ...review + }) => ({ + user: { login }, + author_association, + permission, + ...review + }) + ) + ) + ) + const result = spawnSync('bash', ['-c', mergeScript], { + encoding: 'utf8', + env: { + ...process.env, + CANDIDATES: '22422', + FAKE_CALLS: calls, + FAKE_PR: join(directory, 'pr.json'), + FAKE_PERMISSION_FAIL: permissionFail ? '1' : '', + FAKE_REVIEWS: join(directory, 'reviews.json'), + FAKE_REVIEWS_FAIL: reviewsFail ? '1' : '', + GH_REPO: 'Comfy-Org/ComfyUI_frontend', + MERGE_TOKEN: 'merge-token', + PATH: `${directory}:${process.env.PATH}`, + READ_TOKEN: 'read-token' + } + }) + const merges = readFileSync(calls, 'utf8') + .split('\n') + .filter((line) => line.startsWith('pr merge')) + return { result, merges } +} + +describe('backport auto-merge', () => { + it('merges a ready backport approved at its current head', () => { + const { result, merges } = runMerge([ + { state: 'APPROVED', commit_id: HEAD } + ]) + + expect(result.stderr).toBe('') + expect(result.status).toBe(0) + expect(merges).toHaveLength(1) + expect(merges[0]).toContain(`--match-head-commit ${HEAD}`) + }) + + it.for(AUTHORIZED_BASES)( + 'merges an approved backport to %s', + (baseRefName) => { + const { merges } = runMerge([{ state: 'APPROVED', commit_id: HEAD }], { + baseRefName + }) + + expect(merges).toHaveLength(1) + } + ) + + it.for(UNAUTHORIZED_BASES)( + 'does not merge a backport to unauthorized train %s', + (baseRefName) => { + const { result, merges } = runMerge( + [{ state: 'APPROVED', commit_id: HEAD }], + { baseRefName } + ) + + expect(merges).toHaveLength(0) + expect(result.stdout).toContain('not an active release branch') + } + ) + + it('does not merge when the only approval is on an older commit', () => { + const { result, merges } = runMerge([{ state: 'APPROVED', commit_id: OLD }]) + + expect(result.stderr).toBe('') + expect(result.status).toBe(0) + expect(merges).toHaveLength(0) + expect(result.stdout).toContain('no approval at head') + }) + + it('ignores an approval at head from an account without write access', () => { + const { merges } = runMerge([ + { state: 'APPROVED', commit_id: OLD, login: 'maintainer' }, + { + state: 'APPROVED', + commit_id: HEAD, + login: 'outsider', + author_association: 'NONE', + permission: 'read' + } + ]) + + expect(merges).toHaveLength(0) + }) + + it('uses only the latest review from each reviewer', () => { + const { merges } = runMerge([ + { state: 'APPROVED', commit_id: HEAD, login: 'first' }, + { state: 'DISMISSED', commit_id: HEAD, login: 'first' }, + { state: 'COMMENTED', commit_id: HEAD, login: 'first' }, + { state: 'APPROVED', commit_id: OLD, login: 'second' } + ]) + + expect(merges).toHaveLength(0) + }) + + it('reports a failed review lookup instead of waiting silently', () => { + const { result, merges } = runMerge([], { reviewsFail: true }) + + expect(result.status).toBe(0) + expect(merges).toHaveLength(0) + expect(result.stdout).toContain('Could not read reviews') + expect(result.stdout).toContain('API rate limit exceeded') + }) + + it('reports a failed permission lookup and rejects that reviewer', () => { + const { result, merges } = runMerge( + [{ state: 'APPROVED', commit_id: HEAD }], + { permissionFail: true } + ) + + expect(result.status).toBe(0) + expect(merges).toHaveLength(0) + expect(result.stdout).toContain('Could not read permission for reviewer') + expect(result.stdout).toContain('API rate limit exceeded') + }) + + it('ignores an approval at head from a member with read access only', () => { + const { merges } = runMerge([ + { state: 'APPROVED', commit_id: OLD, login: 'maintainer' }, + { + state: 'APPROVED', + commit_id: HEAD, + login: 'reader', + permission: 'read' + } + ]) + + expect(merges).toHaveLength(0) + }) + + it('does not merge after the head approver later requests changes', () => { + const { merges } = runMerge([ + { state: 'APPROVED', commit_id: OLD, login: 'maintainer' }, + { state: 'APPROVED', commit_id: HEAD, login: 'reviewer' }, + { state: 'CHANGES_REQUESTED', commit_id: HEAD, login: 'reviewer' } + ]) + + expect(merges).toHaveLength(0) + }) + + it('keeps an approval when the same reviewer later comments', () => { + const { merges } = runMerge([ + { state: 'APPROVED', commit_id: HEAD }, + { state: 'COMMENTED', commit_id: HEAD } + ]) + + expect(merges).toHaveLength(1) + }) +})