From 705ff0007464bcadc05e7b6404b5a17559690bf2 Mon Sep 17 00:00:00 2001 From: Eric Dahl Date: Tue, 22 Sep 2026 14:01:37 -0400 Subject: [PATCH 1/2] chore: detect schema drift between production and committed migrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migration 0002 was merged in March and never reached production: the Neon PR workflow that applied migrations is disabled (and referenced a DATABASE_URL secret that was never set), so the global route-slug index survived for six months until it was fixed by hand today. Weekly (and on pushes touching db/), CI now rebuilds the schema from the committed migrations in a throwaway Postgres and compares it to production: columns, indexes, and the recorded migration hashes. It fails on any difference, including a migration that is committed but never applied. Production is read through a new least-privilege `drift_ro` Neon role that can read pg_catalog and drizzle.__drizzle_migrations and nothing else: it is denied on every application table, on neon_auth, and on DDL. (Roles created through the Neon API are auto-granted neon_superuser and cannot be restricted, so this role is created in SQL.) Its URL is the DRIFT_CHECK_DATABASE_URL repo secret. drizzle.config.ts: an explicit DATABASE_URL now wins over .env.local. It used to be overridden silently, so `DATABASE_URL=... yarn db:migrate` would migrate whatever .env.local pointed at (the dev branch) while appearing to succeed — a plausible origin of the original drift. --- .github/workflows/schema-drift.yml | 46 ++++++++++++++++++++ drizzle.config.ts | 6 ++- package.json | 1 + scripts/schema-drift/check.ts | 39 +++++++++++++++++ scripts/schema-drift/collect.ts | 53 +++++++++++++++++++++++ scripts/schema-drift/diff.ts | 68 ++++++++++++++++++++++++++++++ tests/scripts/schema-drift.test.ts | 65 ++++++++++++++++++++++++++++ 7 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/schema-drift.yml create mode 100644 scripts/schema-drift/check.ts create mode 100644 scripts/schema-drift/collect.ts create mode 100644 scripts/schema-drift/diff.ts create mode 100644 tests/scripts/schema-drift.test.ts diff --git a/.github/workflows/schema-drift.yml b/.github/workflows/schema-drift.yml new file mode 100644 index 0000000..845425a --- /dev/null +++ b/.github/workflows/schema-drift.yml @@ -0,0 +1,46 @@ +name: Schema drift + +# Migrations are applied by hand (the Neon PR workflow is disabled), so a merged migration can +# silently never reach production — that happened with 0002 for six months. This rebuilds the +# schema from the committed migrations and compares it to production, read-only. + +on: + schedule: + - cron: '0 13 * * 1' # Mondays 13:00 UTC + push: + branches: [master] + paths: + - 'db/**' + - 'scripts/schema-drift/**' + - '.github/workflows/schema-drift.yml' + workflow_dispatch: + +jobs: + drift: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:17 + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: yarn + - run: corepack enable + - run: yarn install --immutable + - name: Build the expected schema from committed migrations + env: + DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres + run: yarn db:migrate + - name: Compare production against it + env: + EXPECTED_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres + DRIFT_CHECK_DATABASE_URL: ${{ secrets.DRIFT_CHECK_DATABASE_URL }} + run: yarn db:drift diff --git a/drizzle.config.ts b/drizzle.config.ts index 34fc0f3..ff39496 100644 --- a/drizzle.config.ts +++ b/drizzle.config.ts @@ -3,10 +3,14 @@ import { defineConfig } from 'drizzle-kit'; import { resolve } from 'node:path'; // Next.js loads .env* automatically; drizzle-kit does not — mirror that for CLI. +// An explicit DATABASE_URL in the environment wins over .env.local: `DATABASE_URL=... yarn +// db:migrate` used to be silently redirected to whatever .env.local pointed at (a dev branch), +// so a migration you believed you had applied never reached the database you named. +const explicitDatabaseUrl = process.env.DATABASE_URL; loadEnv({ path: resolve(process.cwd(), '.env'), quiet: true }); loadEnv({ path: resolve(process.cwd(), '.env.local'), override: true, quiet: true }); -const databaseUrl = process.env.DATABASE_URL; +const databaseUrl = explicitDatabaseUrl ?? process.env.DATABASE_URL; if (!databaseUrl) { throw new Error( 'DATABASE_URL is not set. Define it in .env.local (or .env), or export it before running drizzle-kit.' diff --git a/package.json b/package.json index b895dbe..ab9d79b 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "test:ui": "vitest --ui", "db:generate": "drizzle-kit generate", "db:migrate": "drizzle-kit migrate", + "db:drift": "tsx scripts/schema-drift/check.ts", "db:push": "drizzle-kit push" }, "dependencies": { diff --git a/scripts/schema-drift/check.ts b/scripts/schema-drift/check.ts new file mode 100644 index 0000000..ca0f196 --- /dev/null +++ b/scripts/schema-drift/check.ts @@ -0,0 +1,39 @@ +/** + * Compare production's schema with the schema the committed migrations produce. + * + * EXPECTED_DATABASE_URL: a throwaway database with every migration freshly applied (CI). + * DRIFT_CHECK_DATABASE_URL: production, read-only role. + * Exits 1 on any difference, including a migration that is committed but never applied. + */ +import { collectSnapshot } from './collect'; +import { diffSchemas } from './diff'; + +async function main() { + const expectedUrl = process.env.EXPECTED_DATABASE_URL; + const actualUrl = process.env.DRIFT_CHECK_DATABASE_URL; + if (!expectedUrl || !actualUrl) { + console.error('EXPECTED_DATABASE_URL and DRIFT_CHECK_DATABASE_URL are required'); + process.exit(2); + } + + const [expected, actual] = await Promise.all([collectSnapshot(expectedUrl), collectSnapshot(actualUrl)]); + const problems = diffSchemas(expected, actual); + + if (problems.length === 0) { + console.log( + `✓ production matches the migrated schema (${expected.columns.length} columns, ` + + `${expected.indexes.length} indexes, ${expected.migrations.length} migrations)` + ); + return; + } + + console.error(`✗ ${problems.length} difference(s) between production and the committed migrations:\n`); + for (const problem of problems) console.error(` - ${problem}`); + console.error('\nApply the pending migrations, or generate one that matches production.'); + process.exit(1); +} + +main().catch((error) => { + console.error('[schema-drift] check failed to run:', error); + process.exit(2); +}); diff --git a/scripts/schema-drift/collect.ts b/scripts/schema-drift/collect.ts new file mode 100644 index 0000000..3cb5e30 --- /dev/null +++ b/scripts/schema-drift/collect.ts @@ -0,0 +1,53 @@ +import postgres from 'postgres'; + +import type { ColumnInfo, IndexInfo, SchemaSnapshot } from './diff'; + +/** + * Read the shape of a database from pg_catalog. + * + * pg_catalog rather than information_schema: information_schema filters rows to objects the + * current role has privileges on, so the least-privilege CI role would see an empty schema and + * every table would look like drift. + */ +export async function collectSnapshot(databaseUrl: string): Promise { + const sql = postgres(databaseUrl, { + max: 1, + // Best-effort only: Neon's pooler drops this startup parameter, so it is not a guarantee. + // The real control is the `drift_ro` role, which has no privileges on application tables. + connection: { default_transaction_read_only: true }, + }); + try { + const columns = (await sql` + SELECT c.relname AS table, + a.attname AS column, + format_type(a.atttypid, a.atttypmod) AS type, + NOT a.attnotnull AS nullable, + pg_get_expr(d.adbin, d.adrelid) AS default + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum + WHERE n.nspname = 'public' AND c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY c.relname, a.attname + `) as unknown as ColumnInfo[]; + + const indexes = (await sql` + SELECT tablename AS table, indexname AS name, indexdef AS definition + FROM pg_indexes + WHERE schemaname = 'public' + ORDER BY tablename, indexname + `) as unknown as IndexInfo[]; + + const migrationRows = await sql<{ hash: string }[]>` + SELECT hash FROM drizzle.__drizzle_migrations ORDER BY created_at, id + `; + + return { + columns: columns.map((c) => ({ ...c, default: c.default ?? null })), + indexes, + migrations: migrationRows.map((row) => row.hash), + }; + } finally { + await sql.end({ timeout: 5 }); + } +} diff --git a/scripts/schema-drift/diff.ts b/scripts/schema-drift/diff.ts new file mode 100644 index 0000000..6e70ce0 --- /dev/null +++ b/scripts/schema-drift/diff.ts @@ -0,0 +1,68 @@ +export type ColumnInfo = { + table: string; + column: string; + type: string; + nullable: boolean; + default: string | null; +}; + +export type IndexInfo = { table: string; name: string; definition: string }; + +export type SchemaSnapshot = { + columns: ColumnInfo[]; + indexes: IndexInfo[]; + /** Hashes recorded in drizzle.__drizzle_migrations. */ + migrations: string[]; +}; + +function describeColumn(c: ColumnInfo): string { + return `${c.type} ${c.nullable ? 'NULL' : 'NOT NULL'} default ${c.default ?? 'null'}`; +} + +/** Differences between the schema the migrations produce and what production actually has. */ +export function diffSchemas(expected: SchemaSnapshot, actual: SchemaSnapshot): string[] { + const problems: string[] = []; + + const colKey = (c: ColumnInfo) => `${c.table}.${c.column}`; + const expectedCols = new Map(expected.columns.map((c) => [colKey(c), c])); + const actualCols = new Map(actual.columns.map((c) => [colKey(c), c])); + for (const [key] of expectedCols) { + if (!actualCols.has(key)) problems.push(`missing column ${key}`); + } + for (const [key] of actualCols) { + if (!expectedCols.has(key)) problems.push(`unexpected column ${key}`); + } + for (const [key, exp] of expectedCols) { + const act = actualCols.get(key); + if (act && describeColumn(exp) !== describeColumn(act)) { + problems.push(`changed column ${key}: expected ${describeColumn(exp)}, got ${describeColumn(act)}`); + } + } + + const idxKey = (i: IndexInfo) => `${i.table}.${i.name}`; + const expectedIdx = new Map(expected.indexes.map((i) => [idxKey(i), i])); + const actualIdx = new Map(actual.indexes.map((i) => [idxKey(i), i])); + for (const [key, exp] of expectedIdx) { + if (!actualIdx.has(key)) problems.push(`missing index ${key}: ${exp.definition}`); + } + for (const [key, act] of actualIdx) { + if (!expectedIdx.has(key)) problems.push(`unexpected index ${key}: ${act.definition}`); + } + for (const [key, exp] of expectedIdx) { + const act = actualIdx.get(key); + if (act && act.definition !== exp.definition) { + problems.push(`changed index ${key}: expected ${exp.definition}, got ${act.definition}`); + } + } + + const applied = new Set(actual.migrations); + const known = new Set(expected.migrations); + for (const hash of expected.migrations) { + if (!applied.has(hash)) problems.push(`unapplied migration ${hash}`); + } + for (const hash of actual.migrations) { + if (!known.has(hash)) problems.push(`unknown migration recorded in production ${hash}`); + } + + return problems; +} diff --git a/tests/scripts/schema-drift.test.ts b/tests/scripts/schema-drift.test.ts new file mode 100644 index 0000000..a9e403c --- /dev/null +++ b/tests/scripts/schema-drift.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; + +import { diffSchemas, type SchemaSnapshot } from '@/scripts/schema-drift/diff'; + +const base: SchemaSnapshot = { + columns: [ + { table: 'routes', column: 'slug', type: 'text', nullable: false, default: null }, + { table: 'routes', column: 'user_id', type: 'text', nullable: false, default: null }, + ], + indexes: [ + { table: 'routes', name: 'routes_user_id_slug_key', definition: 'CREATE UNIQUE INDEX routes_user_id_slug_key ON public.routes USING btree (user_id, slug)' }, + ], + migrations: ['hash-0000', 'hash-0001'], +}; + +describe('diffSchemas', () => { + it('reports nothing when production matches the migrated schema', () => { + expect(diffSchemas(base, structuredClone(base))).toEqual([]); + }); + + it('reports an index that production is missing', () => { + const prod = { ...structuredClone(base), indexes: [] }; + expect(diffSchemas(base, prod)).toEqual([ + 'missing index routes.routes_user_id_slug_key: CREATE UNIQUE INDEX routes_user_id_slug_key ON public.routes USING btree (user_id, slug)', + ]); + }); + + it('reports an index that exists only in production', () => { + const prod = structuredClone(base); + prod.indexes.push({ table: 'routes', name: 'routes_slug_key', definition: 'CREATE UNIQUE INDEX routes_slug_key ON public.routes USING btree (slug)' }); + expect(diffSchemas(base, prod)).toEqual([ + 'unexpected index routes.routes_slug_key: CREATE UNIQUE INDEX routes_slug_key ON public.routes USING btree (slug)', + ]); + }); + + it('reports an index whose definition differs', () => { + const prod = structuredClone(base); + prod.indexes[0].definition = 'CREATE UNIQUE INDEX routes_user_id_slug_key ON public.routes USING btree (slug)'; + expect(diffSchemas(base, prod)).toHaveLength(1); + expect(diffSchemas(base, prod)[0]).toMatch(/^changed index routes\.routes_user_id_slug_key/); + }); + + it('reports missing, unexpected, and changed columns', () => { + const prod = structuredClone(base); + prod.columns = [ + { table: 'routes', column: 'slug', type: 'varchar', nullable: false, default: null }, + { table: 'routes', column: 'extra', type: 'text', nullable: true, default: null }, + ]; + expect(diffSchemas(base, prod)).toEqual([ + 'missing column routes.user_id', + 'unexpected column routes.extra', + 'changed column routes.slug: expected text NOT NULL default null, got varchar NOT NULL default null', + ]); + }); + + it('reports migrations in the repo that production has not applied', () => { + const prod = { ...structuredClone(base), migrations: ['hash-0000'] }; + expect(diffSchemas(base, prod)).toEqual(['unapplied migration hash-0001']); + }); + + it('reports migrations production recorded that are not in the repo', () => { + const prod = { ...structuredClone(base), migrations: ['hash-0000', 'hash-0001', 'hash-9999'] }; + expect(diffSchemas(base, prod)).toEqual(['unknown migration recorded in production hash-9999']); + }); +}); From b84df47f390faa99c0b45d9969995df499ca3115 Mon Sep 17 00:00:00 2001 From: Eric Dahl Date: Tue, 22 Sep 2026 14:19:09 -0400 Subject: [PATCH 2/2] chore: pin the drift workflow's actions to commit SHAs Strix flagged that this job receives the production DRIFT_CHECK_DATABASE_URL secret while using mutable v4 tags. Pin both actions to the commit SHAs behind v4.4.0 so a retagged release cannot reach the credential. --- .github/workflows/schema-drift.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/schema-drift.yml b/.github/workflows/schema-drift.yml index 845425a..7ccb7b0 100644 --- a/.github/workflows/schema-drift.yml +++ b/.github/workflows/schema-drift.yml @@ -28,8 +28,8 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 cache: yarn