diff --git a/.github/workflows/vercel-production.yml b/.github/workflows/vercel-production.yml index 936b76f..3fb7105 100644 --- a/.github/workflows/vercel-production.yml +++ b/.github/workflows/vercel-production.yml @@ -1,43 +1,78 @@ -name: Vercel Production Deploy +name: Vercel Production Release + +run-name: >- + Production ${{ inputs.operation }} · ${{ inputs.source_commit }} · ${{ inputs.ticket || 'read-only verification' }} on: - workflow_run: - workflows: ["CI"] - types: [completed] - branches: [main] workflow_dispatch: + inputs: + operation: + description: Read-only schema verification or an approved production deployment + required: true + type: choice + options: + - verify + - deploy + default: verify + source_commit: + description: Exact 40-character main commit to verify or deploy + required: true + type: string + deployment_authorized: + description: Web and Railway production rollout is approved; ignored by verify + required: true + type: boolean + default: false + ticket: + description: Approved change ticket (required for deploy) + required: false + type: string + rollback_owner: + description: Named rollback decision owner (required for deploy) + required: false + type: string + base_url: + description: Approved production/canary base URL + required: false + type: string + default: https://kaxi.vercel.app + baseline_complete_p95_ms: + description: Approved pre-release completed-response p95 in milliseconds + required: false + type: string + cold_first_progress_budget_ms: + description: Approved cold first-progress budget in milliseconds + required: false + type: string + alert_recipients_acknowledged: + description: Alert recipients expect the production rehearsal + required: true + type: boolean + default: false + canary_start_utc: + description: Exact UTC canary start (YYYY-MM-DDTHH:MM:SSZ) + required: false + type: string + +permissions: + contents: read concurrency: group: kaxi-production cancel-in-progress: false jobs: - verify-manual-deploy: - if: ${{ github.event_name == 'workflow_dispatch' }} + verify-release-ci: uses: ./.github/workflows/ci.yml - deploy: - needs: [verify-manual-deploy] - if: >- - ${{ - always() && - ( - (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || - (github.event_name == 'workflow_dispatch' && needs.verify-manual-deploy.result == 'success') - ) - }} + release: + needs: [verify-release-ci] + if: ${{ needs.verify-release-ci.result == 'success' }} runs-on: ubuntu-latest + environment: production env: - # Every step below used to run `bunx vercel`, which resolves whatever is - # latest on npm at that moment — so an upstream CLI release could change or - # break a production deploy with no commit on our side. It did: release - # 0dab19d deployed cleanly on 58.4.0, and the very next run, on 58.4.4, - # failed twice in a row at `deploy --prebuilt --archive=tgz` with - # `ENOENT ... /vercel/path0/node_modules/@opentelemetry/api/build/src/api/context.js` - # — a path outside the prebuilt output the workflow uploads. Same repo, same - # steps, different CLI. - # - # Bump this deliberately, and only after a green run. + # Pin the Vercel CLI. An upstream patch release previously broke the + # prebuilt deployment path without a repository change. VERCEL_CLI_VERSION: "58.4.0" VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} @@ -45,78 +80,108 @@ jobs: DATABASE_URL: ${{ secrets.DATABASE_URL }} POSTGRES_URL: ${{ secrets.POSTGRES_URL }} PRISMA_DATABASE_URL: ${{ secrets.PRISMA_DATABASE_URL }} + RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }} KAXI_PRISMA_PROVIDER: postgresql - EXPECTED_SHA: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} + EXPECTED_SHA: ${{ inputs.source_commit }} + PRODUCTION_ENV_FILE: .vercel/.env.production.local steps: + - name: Require explicit production deployment authorization + if: ${{ inputs.operation == 'deploy' && !inputs.deployment_authorized }} + run: | + echo "deployment_authorized must be true for a production deployment." + exit 1 - name: Require Vercel deployment secrets if: ${{ env.VERCEL_TOKEN == '' || env.VERCEL_ORG_ID == '' || env.VERCEL_PROJECT_ID == '' }} run: | - echo "VERCEL_TOKEN, VERCEL_ORG_ID, and VERCEL_PROJECT_ID are required for production deployment." + echo "VERCEL_TOKEN, VERCEL_ORG_ID, and VERCEL_PROJECT_ID are required." exit 1 - name: Require production database secrets - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' && env.DATABASE_URL == '' && env.POSTGRES_URL == '' }} + if: ${{ env.DATABASE_URL == '' && env.POSTGRES_URL == '' && env.PRISMA_DATABASE_URL == '' }} run: | - echo "DATABASE_URL or POSTGRES_URL GitHub secret is required for production migrations." + echo "DATABASE_URL, POSTGRES_URL, or PRISMA_DATABASE_URL is required for production verification." exit 1 - - name: Mask production database secrets - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} + - name: Mask production access secrets run: | [ -z "$DATABASE_URL" ] || echo "::add-mask::$DATABASE_URL" [ -z "$POSTGRES_URL" ] || echo "::add-mask::$POSTGRES_URL" [ -z "$PRISMA_DATABASE_URL" ] || echo "::add-mask::$PRISMA_DATABASE_URL" + [ -z "$RAILWAY_TOKEN" ] || echo "::add-mask::$RAILWAY_TOKEN" - uses: actions/checkout@v7 - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} with: - # release:check:source requires an attached main branch; the next step - # still pins the exact CI-verified SHA, so reproducibility holds and a - # racing push simply aborts this run in favor of its own pipeline. ref: main - - name: Verify deployment commit - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} - env: - EXPECTED_SHA: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} - run: test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" + - name: Verify exact approved main commit + run: | + test "$EXPECTED_SHA" = "$(git rev-parse HEAD)" + test "${#EXPECTED_SHA}" = "40" - uses: oven-sh/setup-bun@v2 - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} with: bun-version: 1.3.5 - name: Install dependencies - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} run: bun install --frozen-lockfile - name: Verify reproducible release source - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} run: bun run release:check:source -- --expected-sha "$EXPECTED_SHA" - # Serving-projection drift makes production readiness report 503, which the canary - # gate below rejects — but only after a build, a migration and a canary deployment - # have been paid for. The projection is shared Supabase state that can already be - # drifted before this release starts, so read it from the live production readiness - # endpoint first. Needs no credentials, so it runs ahead of every costly step. - - name: Check RAG serving projection drift - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} - run: bun run rag:check:projection -- --base-url "${{ vars.PRODUCTION_BASE_URL || 'https://kaxi.vercel.app' }}" - name: Pull Vercel production environment - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} run: bunx vercel@$VERCEL_CLI_VERSION pull --yes --environment=production --token="$VERCEL_TOKEN" + - name: Verify production database connectivity + run: bun --env-file="$PRODUCTION_ENV_FILE" run db:check-production + - name: Verify applied production schema + if: ${{ inputs.operation == 'verify' }} + run: bun --env-file="$PRODUCTION_ENV_FILE" run ops:check:schema-parity + - name: Stop after read-only verification + if: ${{ inputs.operation == 'verify' }} + run: echo "PASS read-only production database and schema verification" + - name: Run fail-closed production rollout preflight + if: ${{ inputs.operation == 'deploy' }} + env: + CHANGE_TICKET: ${{ inputs.ticket }} + ROLLBACK_OWNER: ${{ inputs.rollback_owner }} + APPROVED_BASE_URL: ${{ inputs.base_url }} + BASELINE_COMPLETE_P95_MS: ${{ inputs.baseline_complete_p95_ms }} + COLD_FIRST_PROGRESS_BUDGET_MS: ${{ inputs.cold_first_progress_budget_ms }} + CANARY_START_UTC: ${{ inputs.canary_start_utc }} + ALERT_RECIPIENTS_ACKNOWLEDGED: ${{ inputs.alert_recipients_acknowledged }} + run: | + args=( + --phase all + --ticket "$CHANGE_TICKET" + --rollback-owner "$ROLLBACK_OWNER" + --source-commit "$EXPECTED_SHA" + --railway-deploy-authorized + --base-url "$APPROVED_BASE_URL" + --baseline-complete-p95-ms "$BASELINE_COMPLETE_P95_MS" + --cold-first-progress-budget-ms "$COLD_FIRST_PROGRESS_BUDGET_MS" + --canary-start-utc "$CANARY_START_UTC" + ) + if [ "$ALERT_RECIPIENTS_ACKNOWLEDGED" = "true" ]; then + args+=(--alert-recipients-acknowledged) + fi + bun --env-file="$PRODUCTION_ENV_FILE" run ops:check:rollout-readiness -- "${args[@]}" + - name: Check RAG serving projection drift + if: ${{ inputs.operation == 'deploy' }} + run: bun run rag:check:projection -- --base-url "${{ inputs.base_url }}" - name: Build with Vercel - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} + if: ${{ inputs.operation == 'deploy' }} run: bunx vercel@$VERCEL_CLI_VERSION build --prod --token="$VERCEL_TOKEN" - name: Apply PostgreSQL migrations - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} + if: ${{ inputs.operation == 'deploy' }} run: bun run scripts/deploy-postgres-migrations.ts + - name: Verify post-migration schema parity + if: ${{ inputs.operation == 'deploy' }} + run: bun --env-file="$PRODUCTION_ENV_FILE" run ops:check:schema-parity - name: Prune deployment model cache - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} + if: ${{ inputs.operation == 'deploy' }} run: bun run scripts/prune-vercel-output.ts - name: Inject runtime environment into prebuilt functions - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} + if: ${{ inputs.operation == 'deploy' }} run: bun run scripts/inject-vercel-runtime-env.ts - name: Deploy production canary without assigning domains id: deploy - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} + if: ${{ inputs.operation == 'deploy' }} run: | DEPLOYMENT_URL="$(bunx vercel@$VERCEL_CLI_VERSION deploy --prebuilt --archive=tgz --prod --skip-domain --meta "kaxiReleaseSha=$EXPECTED_SHA" --meta "kaxiSourceClean=1" --token="$VERCEL_TOKEN")" echo "url=$DEPLOYMENT_URL" >> "$GITHUB_OUTPUT" - name: Verify production canary end to end - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} + if: ${{ inputs.operation == 'deploy' }} env: CANARY_URL: ${{ steps.deploy.outputs.url }} run: | @@ -128,5 +193,5 @@ jobs: --output /dev/null bun run release:check:backend -- --base-url "$CANARY_URL" --cookie-file "$COOKIE_JAR" - name: Promote verified canary to production domains - if: ${{ env.VERCEL_TOKEN != '' && env.VERCEL_ORG_ID != '' && env.VERCEL_PROJECT_ID != '' }} + if: ${{ inputs.operation == 'deploy' }} run: bunx vercel@$VERCEL_CLI_VERSION promote "${{ steps.deploy.outputs.url }}" --yes --token="$VERCEL_TOKEN" diff --git a/docs/runbooks/production-architecture-rollout.md b/docs/runbooks/production-architecture-rollout.md index 14ba9ba..a2db8e5 100644 --- a/docs/runbooks/production-architecture-rollout.md +++ b/docs/runbooks/production-architecture-rollout.md @@ -46,6 +46,29 @@ Replace every placeholder with the approved evidence. The command must pass from the exact clean release checkout. A failure is a stop-the-line result, not a reason to skip an input or weaken the check. +Production deployment is never triggered by a push or completed CI run. Use the +`Vercel Production Release` workflow manually. Its `verify` operation is read-only +and confirms database connectivity plus migration/schema parity from the exact +approved `main` commit. Its `deploy` operation requires the protected `production` +environment, `deployment_authorized=true`, the approved ticket/rollback owner, +Railway authorization, latency budgets, acknowledged alert recipients and the +exact UTC canary start. It reruns CI and the fail-closed rollout preflight before +build, migration, canary creation or domain promotion. + +Run read-only verification first: + +```sh +gh workflow run vercel-production.yml --ref main \ + -f operation=verify \ + -f source_commit=0000000000000000000000000000000000000000 \ + -f deployment_authorized=false \ + -f alert_recipients_acknowledged=false +``` + +Do not dispatch `operation=deploy` until the local preflight above passes and the +same approved values are ready for the workflow inputs. A verify run never builds, +migrates, creates a deployment or changes production domains. + ## 2. Pre-deploy gates From the exact release commit, require CI, type, lint, schema policy, architecture, diff --git a/package.json b/package.json index 186bda0..86d05eb 100644 --- a/package.json +++ b/package.json @@ -163,6 +163,7 @@ "ops:check:trace-coverage": "bun run scripts/check-production-trace-coverage.ts", "ops:check:tenant-writes": "bun run scripts/check-production-tenant-writes.ts", "ops:check:canary": "bun run scripts/check-production-canary.ts", + "ops:check:schema-parity": "bun run scripts/check-production-schema-parity.ts", "ops:check:rollout-readiness": "bun run scripts/check-production-rollout-readiness.ts", "ops:check:latency": "bun run scripts/check-production-latency.ts", "ops:rehearse:alert": "bun run scripts/rehearse-production-alert.ts", diff --git a/scripts/check-production-schema-parity.ts b/scripts/check-production-schema-parity.ts new file mode 100644 index 0000000..485bd8d --- /dev/null +++ b/scripts/check-production-schema-parity.ts @@ -0,0 +1,20 @@ +import { db } from "../src/lib/db"; +import { checkProductionSchemaParity } from "../src/lib/ops/schema-parity"; + +try { + const result = await checkProductionSchemaParity(); + console.log(JSON.stringify({ + ok: result.ok, + latestMigration: result.latestMigration, + missing: result.missing, + }, null, 2)); + + if (!result.ok) { + console.error(`FAIL production schema parity: ${result.detail}`); + process.exitCode = 1; + } else { + console.log(`PASS production schema parity: ${result.latestMigration}`); + } +} finally { + await db.$disconnect(); +} diff --git a/scripts/test-ci-quality-gates.ts b/scripts/test-ci-quality-gates.ts index 27bc355..4f08fb9 100644 --- a/scripts/test-ci-quality-gates.ts +++ b/scripts/test-ci-quality-gates.ts @@ -61,18 +61,74 @@ assert(workflow.includes("bun run ci:suite -- ${{ matrix.suite }}"), "CI matrix assert(workflow.includes("bun run ci:suite -- e2e"), "CI must execute the e2e manifest suite"); assert(workflow.includes("actions/upload-artifact"), "CI must upload suite, bundle and trace artifacts"); -for (const required of ["release:check:source", "release:check:backend", "release:check:typebot"]) { +for (const required of [ + "release:check:source", + "release:check:backend", + "release:check:typebot", + "ops:check:schema-parity", + "ops:check:rollout-readiness", +]) { assert(scripts[required], `missing release gate script: ${required}`); } -assert(deploymentWorkflow.includes("workflow_run:"), "production deploy must be triggered from a completed CI workflow"); assert( - deploymentWorkflow.includes("github.event.workflow_run.conclusion == 'success'"), - "production deploy must require a successful CI conclusion", + !deploymentWorkflow.includes("workflow_run:"), + "production release must never start automatically after a main CI run", ); assert( - deploymentWorkflow.includes("github.event.workflow_run.head_sha"), - "production deploy must checkout and verify the CI-tested SHA", + deploymentWorkflow.includes("workflow_dispatch:") && deploymentWorkflow.includes("operation:"), + "production release must be manually dispatched with an explicit operation", ); +assert( + deploymentWorkflow.includes("environment: production") && deploymentWorkflow.includes("deployment_authorized"), + "production deploy must require the protected production environment and an explicit authorization input", +); +assert( + deploymentWorkflow.includes("bun --env-file=\"$PRODUCTION_ENV_FILE\" run ops:check:rollout-readiness"), + "production deploy must pass the value-redacting rollout preflight before build, migration or deployment", +); +assert( + deploymentWorkflow.includes("inputs.operation == 'verify'") && deploymentWorkflow.includes("ops:check:schema-parity"), + "production release must provide a read-only migration and schema verification operation", +); +assert( + deploymentWorkflow.includes("EXPECTED_SHA: ${{ inputs.source_commit }}"), + "production release must checkout and verify the manually approved source SHA", +); + +function deploymentStep(name: string): string { + const marker = ` - name: ${name}`; + const start = deploymentWorkflow.indexOf(marker); + assert(start >= 0, `production release is missing step: ${name}`); + const next = deploymentWorkflow.indexOf("\n - name:", start + marker.length); + return deploymentWorkflow.slice(start, next >= 0 ? next : undefined); +} + +for (const stepName of [ + "Run fail-closed production rollout preflight", + "Check RAG serving projection drift", + "Build with Vercel", + "Apply PostgreSQL migrations", + "Verify post-migration schema parity", + "Deploy production canary without assigning domains", + "Verify production canary end to end", + "Promote verified canary to production domains", +]) { + assert( + deploymentStep(stepName).includes("if: ${{ inputs.operation == 'deploy' }}"), + `${stepName} must run only for an explicitly selected deploy operation`, + ); +} +assert( + deploymentStep("Verify applied production schema").includes("if: ${{ inputs.operation == 'verify' }}"), + "read-only schema verification must run only for the verify operation", +); +const preflightIndex = deploymentWorkflow.indexOf("- name: Run fail-closed production rollout preflight"); +for (const stepName of ["Build with Vercel", "Apply PostgreSQL migrations", "Deploy production canary without assigning domains"]) { + assert( + deploymentWorkflow.indexOf(`- name: ${stepName}`) > preflightIndex, + `${stepName} must remain after the fail-closed rollout preflight`, + ); +} assert( deploymentWorkflow.includes("bun run release:check:source"), "production deploy must reject a dirty or mismatched source checkout",