diff --git a/.github/workflows/ci-feature-flag-policy.yaml b/.github/workflows/ci-feature-flag-policy.yaml new file mode 100644 index 00000000000..6f368a5725e --- /dev/null +++ b/.github/workflows/ci-feature-flag-policy.yaml @@ -0,0 +1,30 @@ +name: 'CI: Feature Flag Policy' + +on: + pull_request: + types: [edited] + +permissions: {} + +concurrency: + group: feature-flag-request-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + request: + name: Request feature flag advice + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Capture request metadata + run: | + jq '[{pr_number: .pull_request.number, head_sha: .pull_request.head.sha}]' \ + "$GITHUB_EVENT_PATH" > "$RUNNER_TEMP/feature-flag-request.json" + + - name: Upload request + uses: actions/upload-artifact@v6 + with: + name: feature-flag-request + path: ${{ runner.temp }}/feature-flag-request.json + retention-days: 1 + if-no-files-found: error diff --git a/.github/workflows/ci-pr-risk.yml b/.github/workflows/ci-pr-risk.yml index 04ed28e7cc3..209523c3c75 100644 --- a/.github/workflows/ci-pr-risk.yml +++ b/.github/workflows/ci-pr-risk.yml @@ -2,7 +2,7 @@ name: CI - PR Risk Grade # Thin caller for the shared ADVISORY PR risk grader (shadow check). Grades # every PR against .github/risk.json (read from the BASE ref) and syncs exactly -# one `risk:low` .. `risk:xhigh` label — nothing gates, routes, or merges on it. +# one `risk:low` .. `risk:xhigh` label used by advisory feature-flag policy. # The existing `risk-dispute` label remains a marker. A named # `risk-dispute:low` .. `risk-dispute:xhigh` label overrides the visible grade. @@ -74,3 +74,42 @@ jobs: label_map: 'R0=risk:low,R1=risk:medium,R2=risk:high,R3=risk:xhigh,unknown=risk:unknown' pr_number: ${{ inputs.pr_number }} pr_numbers: ${{ inputs.pr_numbers }} + + flag-policy-request: + needs: pr-risk + if: ${{ !cancelled() }} + name: Request feature flag advice after grading + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + pull-requests: read + steps: + - name: Capture graded PR targets + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBERS: ${{ inputs.pr_numbers || inputs.pr_number }} + run: | + if [ "$GITHUB_EVENT_NAME" = pull_request ]; then + jq '[{pr_number: .pull_request.number, head_sha: .pull_request.head.sha}]' \ + "$GITHUB_EVENT_PATH" > "$RUNNER_TEMP/feature-flag-request.json" + else + targets=$(jq -en --arg targets "$PR_NUMBERS" \ + '$targets | split(",") | map(gsub("^\\s+|\\s+$"; "")) | + if length > 0 and all(.[]; test("^[1-9][0-9]*$")) + then unique else error("Invalid PR numbers") end') + while IFS= read -r pr; do + gh api "repos/$GITHUB_REPOSITORY/pulls/$pr" \ + --jq '{pr_number: .number, head_sha: .head.sha}' + done < <(jq -r '.[]' <<< "$targets") \ + > "$RUNNER_TEMP/feature-flag-targets.jsonl" + jq -s '.' "$RUNNER_TEMP/feature-flag-targets.jsonl" \ + > "$RUNNER_TEMP/feature-flag-request.json" + fi + + - name: Upload request + uses: actions/upload-artifact@v6 + with: + name: feature-flag-request + path: ${{ runner.temp }}/feature-flag-request.json + retention-days: 1 + if-no-files-found: error diff --git a/.github/workflows/pr-feature-flag-policy.yaml b/.github/workflows/pr-feature-flag-policy.yaml new file mode 100644 index 00000000000..66225a0f11a --- /dev/null +++ b/.github/workflows/pr-feature-flag-policy.yaml @@ -0,0 +1,62 @@ +name: 'PR: Feature Flag Policy' + +on: + workflow_run: + workflows: ['CI: Feature Flag Policy', 'CI - PR Risk Grade'] + types: [completed] + +permissions: + contents: read + pull-requests: read + actions: read + checks: write + +jobs: + policy: + name: Publish feature flag advice + if: > + (github.event.workflow_run.event == 'pull_request' || + (github.event.workflow_run.event == 'workflow_dispatch' && + github.event.workflow_run.name == 'CI - PR Risk Grade' && + github.event.workflow_run.head_branch == github.event.repository.default_branch)) && + (github.event.workflow_run.conclusion == 'success' || + (github.event.workflow_run.name == 'CI - PR Risk Grade' && + github.event.workflow_run.conclusion == 'failure')) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout trusted policy + uses: actions/checkout@v7 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Resolve pull request + if: github.event.workflow_run.event == 'pull_request' + id: pr + uses: ./.github/actions/resolve-pr-from-workflow-run + + - name: Download request + if: steps.pr.outputs.skip != 'true' + uses: actions/download-artifact@v8 + with: + name: feature-flag-request + path: ${{ runner.temp }}/feature-flag-request + github-token: ${{ github.token }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Setup Node.js + if: steps.pr.outputs.skip != 'true' + uses: actions/setup-node@v6 + with: + node-version-file: '.nvmrc' + + - name: Evaluate and publish advice + if: steps.pr.outputs.skip != 'true' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ steps.pr.outputs.number }} + EXPECTED_HEAD_SHA: ${{ steps.pr.outputs.head-sha }} + TRUSTED_DEFAULT_BRANCH_DISPATCH: ${{ github.event.workflow_run.event == 'workflow_dispatch' && github.event.workflow_run.name == 'CI - PR Risk Grade' && github.event.workflow_run.head_branch == github.event.repository.default_branch }} + POLICY_REQUEST_PATH: ${{ runner.temp }}/feature-flag-request/feature-flag-request.json + run: node --experimental-strip-types scripts/cicd/feature-flag-policy.ts diff --git a/docs/FEATURE_FLAGS.md b/docs/FEATURE_FLAGS.md index 17d8e943700..fb39ff2f5bb 100644 --- a/docs/FEATURE_FLAGS.md +++ b/docs/FEATURE_FLAGS.md @@ -300,6 +300,68 @@ max_size = feature_flags.get_connection_feature( ## Adding New Feature Flags +### Feature flag policy (advisory) + +`feature-flag-policy` reports **PASS**, **FAIL**, or **UNGRADED** in its title. +Every result uses GitHub's `neutral` conclusion, so neither PASS nor FAIL blocks +merging or feeds a failed check back into CI-based risk grading. This is a policy +result, not evidence that tests passed. Keep this check out of required checks; +normal CI and review requirements still apply. + +Risk labels are the source of truth. The policy uses `risk:low`, `risk:medium`, +`risk:high`, and `risk:xhigh`, with named `risk-dispute:*` overrides taking +precedence. It follows the grader's existing override order: xhigh, high, +medium, low. A plain `risk-dispute` label does not change the tier. If multiple +risk labels exist, the highest wins. Missing or `risk:unknown` classification +reports UNGRADED until a valid grade or named override is available. + +Fork and Dependabot PRs are not graded automatically. A maintainer can assign +risk labels or dispatch the risk workflow for those PRs; until then, UNGRADED +requires no action to merge. + +| Effective risk | Declaration or exception | Result | +| ---------------------------------------------- | ----------------------------------------------------- | ------ | +| Low / medium | Any | PASS | +| High / xhigh | One valid rollout flag | PASS | +| Downgraded to low / medium by `risk-dispute:*` | No flag needed | PASS | +| High / xhigh | `flag-dispute` and a non-empty rationale | PASS | +| High / xhigh | Missing flag, rationale only, or exception label only | FAIL | + +For a flag, fill in the PR description: + +```markdown +## Feature flag + +- **Flag**: unified_cloud_auth +``` + +For a flag exception, document why a flag is unsuitable and include validation +and rollback evidence. A maintainer acknowledges the exception with +`flag-dispute`: + +```markdown +## Feature flag + +- **Rationale**: Removes obsolete code; regression tests cover remaining behavior, and reverting restores the removed path. +``` + +The rationale may continue onto following lines, up to the next field or heading. + +The policy checks declaration presence, not rollout safety or rationale +quality. Reviewers verify containment, fail-closed defaults, production-OFF +state for every cohort, OFF-path coverage, rollback, and exception evidence. +Client capability flags are not rollout controls. Blank, placeholder, commented, +or fenced declarations do not qualify. `flag-exempt` has no effect. Removing a +named dispute restores evaluation from the remaining labels. + +PR body edits, pushes, label changes, and completed risk grading request fresh +advice. Run manual or batch grading from the default branch. Advice can briefly +be stale after an edit; rerun a failed or cancelled request, or edit the PR body, +to refresh it. Publication errors remain visible in the Actions run. + +See [ADR-CI-ROLLOUT-0031](adr/CI-ROLLOUT-0031-advisory-feature-flag-policy.md) +for the trusted publisher, permissions, and risk-grading tradeoffs. + ### Backend 1. **For server capabilities**, add to `SERVER_FEATURE_FLAGS` in `comfy_api/feature_flags.py`: diff --git a/scripts/cicd/feature-flag-workflow-contract.test.ts b/scripts/cicd/feature-flag-workflow-contract.test.ts new file mode 100644 index 00000000000..d30cebff656 --- /dev/null +++ b/scripts/cicd/feature-flag-workflow-contract.test.ts @@ -0,0 +1,204 @@ +import { execFileSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { parse } from 'yaml' +import { z } from 'zod' + +const workflowSchema = z.object({ + name: z.string(), + permissions: z.record(z.string(), z.string()), + on: z.record( + z.string(), + z.object({ + types: z.array(z.string()).default([]), + workflows: z.array(z.string()).default([]) + }) + ), + jobs: z.record( + z.string(), + z.object({ + if: z.string().optional(), + needs: z.string().optional(), + permissions: z.record(z.string(), z.string()).optional(), + steps: z + .array( + z.object({ + id: z.string().optional(), + if: z.string().optional(), + run: z.string().optional(), + env: z.record(z.string(), z.string()).default({}), + uses: z.string().optional(), + with: z.record(z.string(), z.unknown()).default({}) + }) + ) + .default([]) + }) + ) +}) + +function readWorkflow(file: string) { + return workflowSchema.parse(parse(readFileSync(file, 'utf8'))) +} + +const producer = readWorkflow('.github/workflows/ci-feature-flag-policy.yaml') +const publisher = readWorkflow('.github/workflows/pr-feature-flag-policy.yaml') +const riskWorkflow = readWorkflow('.github/workflows/ci-pr-risk.yml') +const requestJob = producer.jobs.request +const gradingRequestJob = riskWorkflow.jobs['flag-policy-request'] +const publishingJob = publisher.jobs.policy + +it('binds privileged publication to a successful read-only producer run', () => { + const checkout = publishingJob.steps.find((step) => + step.uses?.startsWith('actions/checkout@') + ) + expect(checkout?.with).toMatchObject({ + ref: '${{ github.event.repository.default_branch }}', + 'persist-credentials': false + }) + const upload = requestJob.steps.find((step) => + step.uses?.startsWith('actions/upload-artifact@') + ) + const download = publishingJob.steps.find((step) => + step.uses?.startsWith('actions/download-artifact@') + ) + expect(producer.permissions).toEqual({}) + expect(publisher.permissions).toEqual({ + contents: 'read', + 'pull-requests': 'read', + actions: 'read', + checks: 'write' + }) + expect(publisher.on.workflow_run.workflows).toContain(producer.name) + expect(z.string().parse(download?.with.name)).toBe( + z.string().parse(upload?.with.name) + ) + expect(download?.with['run-id']).toBe('${{ github.event.workflow_run.id }}') + expect(publishingJob.if?.replace(/\s+/g, ' ').trim()).toBe( + "(github.event.workflow_run.event == 'pull_request' || (github.event.workflow_run.event == 'workflow_dispatch' && github.event.workflow_run.name == 'CI - PR Risk Grade' && github.event.workflow_run.head_branch == github.event.repository.default_branch)) && (github.event.workflow_run.conclusion == 'success' || (github.event.workflow_run.name == 'CI - PR Risk Grade' && github.event.workflow_run.conclusion == 'failure'))" + ) +}) + +it('refreshes policy after label edits and automated risk grading without new write permissions', () => { + const requestEvents = producer.on.pull_request.types + const gradingEvents = riskWorkflow.on.pull_request.types + expect([...requestEvents, ...gradingEvents]).toEqual( + expect.arrayContaining([ + 'opened', + 'edited', + 'labeled', + 'unlabeled', + 'synchronize' + ]) + ) + expect( + requestEvents.filter((event) => gradingEvents.includes(event)) + ).toEqual([]) + expect(publisher.on.workflow_run.workflows).toContain(riskWorkflow.name) + expect(gradingRequestJob.needs).toBe('pr-risk') + expect(gradingRequestJob.permissions).toEqual({ 'pull-requests': 'read' }) + const upload = gradingRequestJob.steps.find((step) => + step.uses?.startsWith('actions/upload-artifact@') + ) + expect(upload?.with.name).toBe('feature-flag-request') + expect(publishingJob.steps.find((step) => step.id === 'pr')?.if).toBe( + "github.event.workflow_run.event == 'pull_request'" + ) + expect( + publishingJob.steps.find((step) => 'POLICY_REQUEST_PATH' in step.env)?.env + .TRUSTED_DEFAULT_BRANCH_DISPATCH + ).toBe( + "${{ github.event.workflow_run.event == 'workflow_dispatch' && github.event.workflow_run.name == 'CI - PR Risk Grade' && github.event.workflow_run.head_branch == github.event.repository.default_branch }}" + ) +}) + +describe('request capture at the workflow boundary', () => { + let directory = '' + beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), 'feature-flag-request-')) + writeFileSync( + join(directory, 'event.json'), + JSON.stringify({ + pull_request: { number: 17526, head: { sha: 'a'.repeat(40) } } + }) + ) + writeFileSync( + join(directory, 'gh'), + `#!/bin/sh +case "$2" in + repos/Comfy-Org/ComfyUI_frontend/pulls/17526) printf '%s\\n' '{"pr_number":17526,"head_sha":"${'a'.repeat(40)}"}' ;; + repos/Comfy-Org/ComfyUI_frontend/pulls/17527) printf '%s\\n' '{"pr_number":17527,"head_sha":"${'b'.repeat(40)}"}' ;; + *) exit 42 ;; +esac +`, + { mode: 0o700 } + ) + }) + afterEach(() => rmSync(directory, { recursive: true, force: true })) + + function captureScript(job: typeof requestJob) { + return z + .string() + .parse( + job.steps.find((step) => + step.run?.includes('feature-flag-request.json') + )?.run + ) + } + + function capture(source: string, event: string, targets = '') { + execFileSync('bash', ['-e', '-o', 'pipefail', '-c', source], { + env: { + ...process.env, + PATH: `${directory}:${process.env.PATH}`, + GH_TOKEN: 'test-token', + GITHUB_REPOSITORY: 'Comfy-Org/ComfyUI_frontend', + GITHUB_EVENT_NAME: event, + GITHUB_EVENT_PATH: join(directory, 'event.json'), + RUNNER_TEMP: directory, + PR_NUMBERS: targets + }, + stdio: 'pipe' + }) + const result: unknown = JSON.parse( + readFileSync(join(directory, 'feature-flag-request.json'), 'utf8') + ) + return result + } + + it.for([ + { name: 'body edits', job: requestJob }, + { + name: 'automatic risk grading', + job: gradingRequestJob + } + ])('captures only the triggering PR and head for $name', ({ job }) => { + expect(capture(captureScript(job), 'pull_request')).toEqual([ + { pr_number: 17526, head_sha: 'a'.repeat(40) } + ]) + }) + + it('captures every distinct manually graded target from live PR metadata', () => { + expect( + capture( + captureScript(gradingRequestJob), + 'workflow_dispatch', + ' 17526,17527,17526 ' + ) + ).toEqual([ + { pr_number: 17526, head_sha: 'a'.repeat(40) }, + { pr_number: 17527, head_sha: 'b'.repeat(40) } + ]) + }) + + it.for(['', '17526,invalid', '0', '-1'])( + 'rejects invalid dispatch targets: %s', + (targets) => { + expect(() => + capture(captureScript(gradingRequestJob), 'workflow_dispatch', targets) + ).toThrow() + } + ) +})