From 3c7f9a654c66c53ed3791bb4b8e03bc79421bf7e Mon Sep 17 00:00:00 2001 From: Deb-Auth Date: Mon, 28 Sep 2026 21:06:48 -0700 Subject: [PATCH 1/2] feat(database): abort startup when database migrations are pending Run the Prisma migration status check in the boot sequence before the HTTP server starts listening, and make its outcome configurable via DATABASE_MIGRATION_CHECK (strict | warn | off). Production defaults to strict: pending or failed migrations, or an unreadable migration history, abort startup with remediation steps in the log. Other environments default to warn so local development is never blocked. - Ignore rolled-back rows in _prisma_migrations so they count as pending - Rethrow query failures other than a missing migrations table instead of silently treating them as "nothing applied" - Add DATABASE_MIGRATIONS_DIR to locate migrations in custom layouts - Replace the log-only PrismaService.validateMigrations with verifyMigrations, called from main.ts before app.listen() - Add unit tests and an integration spec with a real PrismaService and on-disk migrations folder --- .env.example | 5 + docs/database.md | 15 ++ src/config/database.config.ts | 9 + src/config/env.validation.ts | 7 + src/database/migration-checker.spec.ts | 232 ++++++++++++++++++ src/database/migration-checker.ts | 208 +++++++++++++++- .../migration-startup.integration.spec.ts | 161 ++++++++++++ src/database/prisma.service.ts | 47 ++-- src/main.ts | 8 +- 9 files changed, 654 insertions(+), 38 deletions(-) create mode 100644 src/database/migration-startup.integration.spec.ts diff --git a/.env.example b/.env.example index 40dc8b43..c8e17b0f 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,11 @@ DATABASE_POOL_TIMEOUT_MS=5000 DATABASE_QUERY_TIMEOUT_MS=5000 DATABASE_STATEMENT_TIMEOUT_MS=10000 DATABASE_WORKER_QUERY_TIMEOUT_MS=60000 +# Boot-time migration status check: strict | warn | off. +# Unset = strict in production (abort startup on pending/failed migrations), +# warn everywhere else. DATABASE_MIGRATIONS_DIR overrides /prisma/migrations. +# DATABASE_MIGRATION_CHECK=warn +# DATABASE_MIGRATIONS_DIR=/app/prisma/migrations # Redis REDIS_HOST=localhost diff --git a/docs/database.md b/docs/database.md index dff9517f..b770cc2e 100644 --- a/docs/database.md +++ b/docs/database.md @@ -7,3 +7,18 @@ All database changes must be managed via Prisma migrations. - Migration directories must start with a 14-digit timestamp prefix (`YYYYMMDDHHMMSS`) to ensure strict ordering and avoid conflicts. - Run `npm run db:verify` locally to execute `scripts/verify-migrations.sh` prior to opening a pull request. - The CI pipeline automatically runs `scripts/verify-migrations.sh` to validate schema syntax, migration structure, and working tree cleanliness. + +## Startup Migration Check +Before the HTTP server starts listening, `main.ts` calls `PrismaService.verifyMigrations()`, which compares the folders in `prisma/migrations` with the rows in `_prisma_migrations` (rolled-back rows are ignored). The behaviour is controlled by `DATABASE_MIGRATION_CHECK`: + +| Mode | Default for | Pending or failed migrations | Migration history unreadable (e.g. DB down) | +|------|-------------|------------------------------|---------------------------------------------| +| `strict` | `NODE_ENV=production` | Logs the offending migrations with remediation steps and aborts startup (exit code 1) | Aborts startup | +| `warn` | every other environment | Logs the same instructions and keeps booting | Logs a warning and keeps booting | +| `off` | never | Check skipped | Check skipped | + +To recover from an aborted start: +- Pending migrations: run `npm run prisma:deploy` against the target `DATABASE_URL`, then restart the service. +- Failed migrations: inspect the `logs` column in `_prisma_migrations`, repair the database, run `npx prisma migrate resolve --rolled-back ` (or `--applied `), then redeploy. + +If the runtime image keeps migrations somewhere other than `/prisma/migrations`, point `DATABASE_MIGRATIONS_DIR` at that folder. When no migrations are found on disk, the check logs a warning instead of silently passing. diff --git a/src/config/database.config.ts b/src/config/database.config.ts index f87271e1..7fae16c0 100644 --- a/src/config/database.config.ts +++ b/src/config/database.config.ts @@ -1,5 +1,6 @@ import { registerAs } from '@nestjs/config'; import { databaseEnvSchema, validateEnv } from './env.validation'; +import { MigrationCheckMode, resolveMigrationCheckMode } from '../database/migration-checker'; /** * Database connection configuration. @@ -15,6 +16,10 @@ import { databaseEnvSchema, validateEnv } from './env.validation'; * workers. It carries no server-side `statement_timeout` and a much longer * client-side guard so long-running worker transactions are never aborted by * the API-oriented timeouts. + * + * `migrationCheck` controls the boot-time migration status gate run by + * `main.ts` before the HTTP server listens (see `verifyMigrationsOnStartup`); + * `migrationsDir` overrides where the migrations folder is read from. */ export type DatabaseConfig = { url: string; @@ -24,6 +29,8 @@ export type DatabaseConfig = { queryTimeoutMs: number; statementTimeoutMs: number; workerQueryTimeoutMs: number; + migrationCheck: MigrationCheckMode; + migrationsDir?: string; }; export const databaseConfig = registerAs('database', (): DatabaseConfig => { @@ -36,5 +43,7 @@ export const databaseConfig = registerAs('database', (): DatabaseConfig => { queryTimeoutMs: env.DATABASE_QUERY_TIMEOUT_MS, statementTimeoutMs: env.DATABASE_STATEMENT_TIMEOUT_MS, workerQueryTimeoutMs: env.DATABASE_WORKER_QUERY_TIMEOUT_MS, + migrationCheck: resolveMigrationCheckMode(env.DATABASE_MIGRATION_CHECK, process.env.NODE_ENV), + migrationsDir: env.DATABASE_MIGRATIONS_DIR, }; }); diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts index 33be8784..5d9e91ef 100644 --- a/src/config/env.validation.ts +++ b/src/config/env.validation.ts @@ -36,6 +36,13 @@ export const databaseEnvSchema = z.object({ // worker transactions (rollups, outbox drains) must not be killed by the API // guard; 0 disables the worker guard entirely. DATABASE_WORKER_QUERY_TIMEOUT_MS: z.coerce.number().int().nonnegative().default(60000), + // Boot-time migration status check: `strict` aborts startup on pending or + // failed migrations, `warn` only logs, `off` skips it. When unset, production + // is strict and every other environment warns. + DATABASE_MIGRATION_CHECK: z.enum(['strict', 'warn', 'off']).optional(), + // Location of the Prisma migrations folder the check compares against. + // Defaults to `/prisma/migrations`. + DATABASE_MIGRATIONS_DIR: z.string().min(1).optional(), }); export const redisEnvSchema = z.object({ diff --git a/src/database/migration-checker.spec.ts b/src/database/migration-checker.spec.ts index 85fd1f21..e23a143f 100644 --- a/src/database/migration-checker.spec.ts +++ b/src/database/migration-checker.spec.ts @@ -5,6 +5,11 @@ import { getAppliedMigrations, checkMigrationStatus, getDefaultMigrationsDir, + formatMigrationInstructions, + MigrationStatusUnavailableError, + PendingMigrationsError, + resolveMigrationCheckMode, + verifyMigrationsOnStartup, } from './migration-checker'; // Mock fs module @@ -140,6 +145,42 @@ describe('MigrationChecker', () => { expect(result.size).toBe(0); }); + it('should treat a Postgres 42P01 (undefined_table) error as a fresh database', async () => { + const mockPrisma = { + $queryRawUnsafe: vi.fn().mockRejectedValue( + new Error('Raw query failed. Code: `42P01`. Message: `relation does not exist`'), + ), + }; + + const result = await getAppliedMigrations(mockPrisma as unknown as import('@prisma/client').PrismaClient); + + expect(result.size).toBe(0); + }); + + it('should rethrow errors other than a missing migrations table', async () => { + const mockPrisma = { + $queryRawUnsafe: vi.fn().mockRejectedValue( + new Error("Can't reach database server at `localhost:5432`"), + ), + }; + + await expect( + getAppliedMigrations(mockPrisma as unknown as import('@prisma/client').PrismaClient), + ).rejects.toThrow("Can't reach database server"); + }); + + it('should exclude rolled-back migrations from the query', async () => { + const mockPrisma = { + $queryRawUnsafe: vi.fn().mockResolvedValue([]), + }; + + await getAppliedMigrations(mockPrisma as unknown as import('@prisma/client').PrismaClient); + + expect(mockPrisma.$queryRawUnsafe).toHaveBeenCalledWith( + expect.stringContaining('WHERE rolled_back_at IS NULL'), + ); + }); + it('should return empty map when no migrations have been applied', async () => { const mockPrisma = { $queryRawUnsafe: vi.fn().mockResolvedValue([]), @@ -308,4 +349,195 @@ describe('MigrationChecker', () => { expect(dir).toMatch(/prisma[\\/]migrations$/); }); }); + + describe('resolveMigrationCheckMode', () => { + it('should default to strict in production', () => { + expect(resolveMigrationCheckMode(undefined, 'production')).toBe('strict'); + }); + + it('should default to warn outside production', () => { + expect(resolveMigrationCheckMode(undefined, 'development')).toBe('warn'); + expect(resolveMigrationCheckMode(undefined, 'test')).toBe('warn'); + expect(resolveMigrationCheckMode(undefined, undefined)).toBe('warn'); + }); + + it('should let an explicit mode override the environment default', () => { + expect(resolveMigrationCheckMode('warn', 'production')).toBe('warn'); + expect(resolveMigrationCheckMode('strict', 'development')).toBe('strict'); + expect(resolveMigrationCheckMode('off', 'production')).toBe('off'); + }); + }); + + describe('formatMigrationInstructions', () => { + const pending = { name: '20260901_add_x', applied: false, finished: false, error: null }; + const failed = { name: '20260830_add_y', applied: true, finished: false, error: 'boom' }; + + it('should list pending migrations with the deploy command', () => { + const text = formatMigrationInstructions({ + upToDate: false, + migrations: [pending], + pending: [pending], + failed: [], + message: '', + }); + + expect(text).toContain('Pending migrations (1): 20260901_add_x'); + expect(text).toContain('npm run prisma:deploy'); + expect(text).not.toContain('migrate resolve'); + expect(text).toContain('DATABASE_MIGRATION_CHECK=warn'); + }); + + it('should list failed migrations with the resolve command', () => { + const text = formatMigrationInstructions({ + upToDate: false, + migrations: [failed], + pending: [], + failed: [failed], + message: '', + }); + + expect(text).toContain('Failed migrations (1): 20260830_add_y'); + expect(text).toContain('prisma migrate resolve --rolled-back '); + expect(text).not.toContain('prisma:deploy'); + }); + }); + + describe('verifyMigrationsOnStartup', () => { + const logger = { log: vi.fn(), warn: vi.fn(), error: vi.fn() }; + type Client = import('@prisma/client').PrismaClient; + + function givenMigrationsOnDisk(names: string[]): void { + mockFs.existsSync.mockReturnValue(true); + mockFs.readdirSync.mockReturnValue( + names.map( + (name) => ({ name, isDirectory: () => true, isFile: () => false }) as fs.Dirent, + ), + ); + } + + function prismaWithApplied(names: string[]): Client { + return { + $queryRawUnsafe: vi.fn().mockResolvedValue( + names.map((migration_name) => ({ + migration_name, + finished_at: new Date('2026-08-30T10:00:00Z'), + logs: null, + })), + ), + } as unknown as Client; + } + + function unreachablePrisma(): Client { + return { + $queryRawUnsafe: vi.fn().mockRejectedValue(new Error('Cannot reach database server')), + } as unknown as Client; + } + + it('should proceed when every migration is applied', async () => { + givenMigrationsOnDisk(['20260830_01_init']); + + const result = await verifyMigrationsOnStartup(prismaWithApplied(['20260830_01_init']), { + mode: 'strict', + migrationsDir: '/fake/migrations', + logger, + }); + + expect(result?.upToDate).toBe(true); + expect(logger.log).toHaveBeenCalledWith(expect.stringContaining('up to date')); + expect(logger.error).not.toHaveBeenCalled(); + }); + + it('should abort with PendingMigrationsError in strict mode when migrations are pending', async () => { + givenMigrationsOnDisk(['20260830_01_init', '20260830_02_add_users']); + + const run = verifyMigrationsOnStartup(prismaWithApplied(['20260830_01_init']), { + mode: 'strict', + migrationsDir: '/fake/migrations', + logger, + }); + + await expect(run).rejects.toBeInstanceOf(PendingMigrationsError); + await expect(run).rejects.toThrow('20260830_02_add_users'); + expect(logger.error).toHaveBeenCalledWith(expect.stringContaining('Aborting startup')); + }); + + it('should expose the check result on PendingMigrationsError', async () => { + givenMigrationsOnDisk(['20260830_01_init']); + + const error = await verifyMigrationsOnStartup(prismaWithApplied([]), { + mode: 'strict', + migrationsDir: '/fake/migrations', + logger, + }).catch((e: unknown) => e); + + expect(error).toBeInstanceOf(PendingMigrationsError); + expect((error as PendingMigrationsError).result.pending.map((m) => m.name)).toEqual([ + '20260830_01_init', + ]); + }); + + it('should log and continue in warn mode when migrations are pending', async () => { + givenMigrationsOnDisk(['20260830_01_init', '20260830_02_add_users']); + + const result = await verifyMigrationsOnStartup(prismaWithApplied(['20260830_01_init']), { + mode: 'warn', + migrationsDir: '/fake/migrations', + logger, + }); + + expect(result?.upToDate).toBe(false); + expect(result?.pending.map((m) => m.name)).toEqual(['20260830_02_add_users']); + expect(logger.error).toHaveBeenCalledWith(expect.stringContaining('Continuing startup')); + }); + + it('should abort with MigrationStatusUnavailableError in strict mode when the DB is unreachable', async () => { + givenMigrationsOnDisk(['20260830_01_init']); + + await expect( + verifyMigrationsOnStartup(unreachablePrisma(), { + mode: 'strict', + migrationsDir: '/fake/migrations', + logger, + }), + ).rejects.toBeInstanceOf(MigrationStatusUnavailableError); + }); + + it('should warn and continue in warn mode when the DB is unreachable', async () => { + givenMigrationsOnDisk(['20260830_01_init']); + + const result = await verifyMigrationsOnStartup(unreachablePrisma(), { + mode: 'warn', + migrationsDir: '/fake/migrations', + logger, + }); + + expect(result).toBeNull(); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('Unable to read the migration status'), + ); + }); + + it('should skip the check entirely in off mode', async () => { + const prisma = prismaWithApplied([]); + + const result = await verifyMigrationsOnStartup(prisma, { mode: 'off', logger }); + + expect(result).toBeNull(); + expect(prisma.$queryRawUnsafe).not.toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('disabled')); + }); + + it('should warn rather than abort when no migrations ship with the build', async () => { + mockFs.existsSync.mockReturnValue(false); + + const result = await verifyMigrationsOnStartup(prismaWithApplied([]), { + mode: 'strict', + migrationsDir: '/missing/migrations', + logger, + }); + + expect(result?.upToDate).toBe(true); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('DATABASE_MIGRATIONS_DIR')); + }); + }); }); diff --git a/src/database/migration-checker.ts b/src/database/migration-checker.ts index 864e3560..d72c3df6 100644 --- a/src/database/migration-checker.ts +++ b/src/database/migration-checker.ts @@ -1,5 +1,6 @@ import * as fs from 'fs'; import * as path from 'path'; +import { Logger, LoggerService } from '@nestjs/common'; import { PrismaClient } from '@prisma/client'; /** @@ -56,6 +57,15 @@ export function getMigrationFolders(migrationsDir: string): string[] { * Queries the _prisma_migrations table to get the status of all applied * migrations. Uses $queryRawUnsafe because the table name is a Prisma * internal that isn't in the generated client types. + * + * Rows marked as rolled back (`prisma migrate resolve --rolled-back`) are + * ignored, so a rolled-back migration is reported as pending again, matching + * how `prisma migrate deploy` treats it. + * + * A missing `_prisma_migrations` table (fresh database) yields an empty map, so + * every migration is reported as pending. Any other failure (e.g. the database + * being unreachable) is rethrown: it must never be mistaken for "nothing + * applied". */ export async function getAppliedMigrations( prisma: PrismaClient, @@ -64,27 +74,43 @@ export async function getAppliedMigrations( > { const applied = new Map(); + let rows: { migration_name: string; finished_at: Date | null; logs: string | null }[]; try { // Query the Prisma migration history table directly. // The table stores each migration's name, whether it finished, and any error logs. - const rows = (await prisma.$queryRawUnsafe( - `SELECT migration_name, finished_at, logs FROM _prisma_migrations ORDER BY started_at ASC`, + rows = (await prisma.$queryRawUnsafe( + `SELECT migration_name, finished_at, logs FROM _prisma_migrations WHERE rolled_back_at IS NULL ORDER BY started_at ASC`, )) as { migration_name: string; finished_at: Date | null; logs: string | null }[]; - - for (const row of rows) { - applied.set(row.migration_name, { - finished: row.finished_at !== null, - error: row.logs, - }); + } catch (error) { + if (isMissingMigrationsTableError(error)) { + // Fresh database: all migrations are considered pending. + return applied; } - } catch { - // If the _prisma_migrations table doesn't exist yet (fresh database), - // return an empty map — all migrations are considered pending. + throw error; + } + + for (const row of rows) { + applied.set(row.migration_name, { + finished: row.finished_at !== null, + error: row.logs, + }); } return applied; } +/** + * True when a raw query failed because `_prisma_migrations` does not exist + * (Postgres `42P01 undefined_table`), i.e. no migration was ever applied. + */ +function isMissingMigrationsTableError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes('42P01') || + (message.includes('_prisma_migrations') && message.includes('does not exist')) + ); +} + /** * Compares migration folders on disk against applied migrations in the * database and returns a detailed status report. @@ -145,3 +171,163 @@ export async function checkMigrationStatus( export function getDefaultMigrationsDir(): string { return path.resolve(process.cwd(), 'prisma', 'migrations'); } + +/** + * How the boot sequence reacts to the migration status check: + * - `strict` : abort startup when migrations are pending or failed, or when + * the status cannot be read (default in production) + * - `warn` : log the problem with remediation steps and keep booting + * (default outside production) + * - `off` : skip the check entirely + */ +export type MigrationCheckMode = 'strict' | 'warn' | 'off'; + +/** + * Resolves the effective check mode. An explicit `DATABASE_MIGRATION_CHECK` + * always wins; otherwise production is strict and every other environment only + * warns, so local development is never blocked. + */ +export function resolveMigrationCheckMode( + explicit: MigrationCheckMode | undefined, + nodeEnv: string | undefined, +): MigrationCheckMode { + if (explicit) { + return explicit; + } + return nodeEnv === 'production' ? 'strict' : 'warn'; +} + +/** + * Builds the operator-facing explanation for an out-of-sync schema, listing + * every offending migration and the commands that resolve each situation. + */ +export function formatMigrationInstructions(result: MigrationCheckResult): string { + const lines = ['Database schema is out of sync with this build.']; + + if (result.pending.length > 0) { + lines.push( + ` Pending migrations (${result.pending.length}): ${result.pending.map((m) => m.name).join(', ')}`, + ); + } + if (result.failed.length > 0) { + lines.push( + ` Failed migrations (${result.failed.length}): ${result.failed.map((m) => m.name).join(', ')}`, + ); + } + + lines.push('To resolve:'); + if (result.pending.length > 0) { + lines.push( + ' - Apply the pending migrations against DATABASE_URL with "npm run prisma:deploy" ' + + '(prisma migrate deploy), then restart the service.', + ); + } + if (result.failed.length > 0) { + lines.push( + ' - Inspect the failed migration logs in _prisma_migrations, repair the database, then run ' + + '"npx prisma migrate resolve --rolled-back " (or --applied ) and redeploy.', + ); + } + lines.push(' - To boot anyway (not recommended in production), set DATABASE_MIGRATION_CHECK=warn.'); + + return lines.join('\n'); +} + +/** + * Thrown at startup in `strict` mode when the database has pending or failed + * migrations. The message carries the remediation steps, so the crash log alone + * tells operators what to run. + */ +export class PendingMigrationsError extends Error { + constructor( + readonly result: MigrationCheckResult, + message: string = formatMigrationInstructions(result), + ) { + super(message); + this.name = 'PendingMigrationsError'; + } +} + +/** + * Thrown at startup in `strict` mode when the migration history could not be + * read at all (e.g. the database is unreachable). Booting blind would defeat + * the purpose of the check, so this is fatal too. + */ +export class MigrationStatusUnavailableError extends Error { + constructor( + message: string, + readonly cause?: unknown, + ) { + super(message); + this.name = 'MigrationStatusUnavailableError'; + } +} + +export interface MigrationStartupCheckOptions { + /** Effective check mode, see {@link MigrationCheckMode}. */ + mode: MigrationCheckMode; + /** Absolute path to `prisma/migrations`; defaults to {@link getDefaultMigrationsDir}. */ + migrationsDir?: string; + /** Destination for the check's output; defaults to a Nest `Logger`. */ + logger?: Pick; +} + +/** + * Boot-time gate verifying that the database schema matches the migrations + * shipped with this build. Must run before the HTTP server starts listening so + * a mismatched instance never accepts traffic. + * + * In `strict` mode it throws {@link PendingMigrationsError} or + * {@link MigrationStatusUnavailableError}; in `warn` mode it only logs; in + * `off` mode it does nothing. Returns the check result when one was produced. + */ +export async function verifyMigrationsOnStartup( + prisma: PrismaClient, + options: MigrationStartupCheckOptions, +): Promise { + const logger = options.logger ?? new Logger('MigrationCheck'); + const { mode } = options; + + if (mode === 'off') { + logger.warn('Migration status check is disabled (DATABASE_MIGRATION_CHECK=off).'); + return null; + } + + let result: MigrationCheckResult; + try { + result = await checkMigrationStatus(prisma, options.migrationsDir ?? getDefaultMigrationsDir()); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + const message = + `Unable to read the migration status from the database: ${reason}. ` + + 'Verify DATABASE_URL and that the database is reachable.'; + if (mode === 'strict') { + logger.error(`${message} Aborting startup (DATABASE_MIGRATION_CHECK=strict).`); + throw new MigrationStatusUnavailableError(message, error); + } + logger.warn(`${message} Continuing startup (DATABASE_MIGRATION_CHECK=warn).`); + return null; + } + + if (result.upToDate) { + if (result.migrations.length === 0) { + // Nothing on disk to compare against, typically a build that does not + // ship `prisma/migrations`. Surface it rather than silently passing. + logger.warn( + `${result.message} Set DATABASE_MIGRATIONS_DIR to the migrations folder to enable the check.`, + ); + } else { + logger.log(result.message); + } + return result; + } + + const instructions = formatMigrationInstructions(result); + if (mode === 'strict') { + logger.error(`${instructions}\nAborting startup (DATABASE_MIGRATION_CHECK=strict).`); + throw new PendingMigrationsError(result, instructions); + } + + logger.error(`${instructions}\nContinuing startup (DATABASE_MIGRATION_CHECK=warn).`); + return result; +} diff --git a/src/database/migration-startup.integration.spec.ts b/src/database/migration-startup.integration.spec.ts new file mode 100644 index 00000000..133c2e77 --- /dev/null +++ b/src/database/migration-startup.integration.spec.ts @@ -0,0 +1,161 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { ConfigService } from '@nestjs/config'; +import { Logger } from '@nestjs/common'; +import { PrismaService } from './prisma.service'; +import { DatabaseConfig } from '../config/database.config'; +import { + MigrationCheckMode, + MigrationStatusUnavailableError, + PendingMigrationsError, +} from './migration-checker'; + +/** + * Integration coverage for the boot-time migration gate: a real + * `PrismaService` (real generated client, never connected) reads a real + * migrations folder on disk, while only the `_prisma_migrations` query is + * simulated. This mirrors what `main.ts` does before `app.listen()`. + */ + +const MIGRATIONS = [ + '0_init', + '20260830174000_sync_schema', + '20260901080000_add_cleanup_job_logs', +]; + +let migrationsDir: string; + +beforeAll(() => { + migrationsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'astroid-migrations-')); + for (const name of MIGRATIONS) { + fs.mkdirSync(path.join(migrationsDir, name)); + fs.writeFileSync(path.join(migrationsDir, name, 'migration.sql'), 'SELECT 1;'); + } + fs.writeFileSync(path.join(migrationsDir, 'migration_lock.toml'), 'provider = "postgresql"'); +}); + +afterAll(() => { + fs.rmSync(migrationsDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + vi.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined); + vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined); + vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); +}); + +type MigrationRow = { migration_name: string; finished_at: Date | null; logs: string | null }; + +function buildService(mode: MigrationCheckMode, history: MigrationRow[] | Error): PrismaService { + const database: DatabaseConfig = { + url: 'postgresql://user:pass@localhost:5432/astroid?schema=public', + connectionLimit: 1, + workerConnectionLimit: 1, + poolTimeoutMs: 1000, + queryTimeoutMs: 1000, + statementTimeoutMs: 1000, + workerQueryTimeoutMs: 1000, + migrationCheck: mode, + migrationsDir, + }; + const config = { getOrThrow: vi.fn().mockReturnValue(database) } as unknown as ConfigService; + const service = new PrismaService(config); + + // Simulate the `_prisma_migrations` table (or an unreachable database). + Object.assign(service, { + $queryRawUnsafe: vi.fn(async () => { + if (history instanceof Error) throw history; + return history; + }), + }); + return service; +} + +function applied(...names: string[]): MigrationRow[] { + return names.map((migration_name) => ({ + migration_name, + finished_at: new Date('2026-09-01T08:00:00Z'), + logs: null, + })); +} + +describe('PrismaService.verifyMigrations (startup gate)', () => { + it('lets startup proceed when the database has every shipped migration', async () => { + const service = buildService('strict', applied(...MIGRATIONS)); + + const result = await service.verifyMigrations(); + + expect(result?.upToDate).toBe(true); + expect(result?.migrations.map((m) => m.name)).toEqual(MIGRATIONS); + }); + + it('aborts startup in strict mode when the newest migration is not applied', async () => { + const service = buildService('strict', applied('0_init', '20260830174000_sync_schema')); + + const error = await service.verifyMigrations().catch((e: unknown) => e); + + expect(error).toBeInstanceOf(PendingMigrationsError); + expect((error as Error).message).toContain('20260901080000_add_cleanup_job_logs'); + expect((error as Error).message).toContain('npm run prisma:deploy'); + }); + + it('aborts startup in strict mode when a migration failed mid-way', async () => { + const service = buildService('strict', [ + ...applied('0_init', '20260830174000_sync_schema'), + { + migration_name: '20260901080000_add_cleanup_job_logs', + finished_at: null, + logs: 'ERROR: relation "cleanup_job_logs" already exists', + }, + ]); + + const error = await service.verifyMigrations().catch((e: unknown) => e); + + expect(error).toBeInstanceOf(PendingMigrationsError); + expect((error as PendingMigrationsError).result.failed.map((m) => m.name)).toEqual([ + '20260901080000_add_cleanup_job_logs', + ]); + expect((error as Error).message).toContain('prisma migrate resolve'); + }); + + it('aborts startup in strict mode against a fresh, never-migrated database', async () => { + const service = buildService( + 'strict', + new Error('relation "_prisma_migrations" does not exist'), + ); + + const error = await service.verifyMigrations().catch((e: unknown) => e); + + expect(error).toBeInstanceOf(PendingMigrationsError); + expect((error as PendingMigrationsError).result.pending).toHaveLength(MIGRATIONS.length); + }); + + it('aborts startup in strict mode when the database cannot be reached', async () => { + const service = buildService('strict', new Error('Cannot reach database server')); + + await expect(service.verifyMigrations()).rejects.toBeInstanceOf( + MigrationStatusUnavailableError, + ); + }); + + it('only logs in warn mode so development startup is not blocked', async () => { + const service = buildService('warn', applied('0_init')); + + const result = await service.verifyMigrations(); + + expect(result?.upToDate).toBe(false); + expect(result?.pending).toHaveLength(2); + expect(Logger.prototype.error).toHaveBeenCalledWith( + expect.stringContaining('Continuing startup'), + ); + }); + + it('never queries the database in off mode', async () => { + const service = buildService('off', applied()); + + await expect(service.verifyMigrations()).resolves.toBeNull(); + expect(service.$queryRawUnsafe).not.toHaveBeenCalled(); + }); +}); diff --git a/src/database/prisma.service.ts b/src/database/prisma.service.ts index cbbea166..d9e90c13 100644 --- a/src/database/prisma.service.ts +++ b/src/database/prisma.service.ts @@ -5,9 +5,9 @@ import { DatabaseConfig } from '../config/database.config'; import { buildDatasourceUrl } from './datasource-url'; import { createQueryTimeoutExtension } from './query-timeout.extension'; import { - checkMigrationStatus, - getDefaultMigrationsDir, + MigrationCheckMode, MigrationCheckResult, + verifyMigrationsOnStartup, } from './migration-checker'; /** @@ -41,6 +41,9 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnModul */ readonly workerClient: PrismaClient; + private readonly migrationCheck: MigrationCheckMode; + private readonly migrationsDir?: string; + constructor(configService: ConfigService) { const database = configService.getOrThrow('database'); @@ -93,6 +96,9 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnModul poolTimeoutMs: database.poolTimeoutMs, }), ) as unknown as PrismaClient; + + this.migrationCheck = database.migrationCheck; + this.migrationsDir = database.migrationsDir; } async onModuleInit(): Promise { @@ -100,12 +106,11 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnModul await this.$connect(); await this.workerClient.$connect(); this.logger.log('Prisma connected to the database'); - - // Validate migration status after successful connection. - await this.validateMigrations(); } catch (error) { // Do not crash on boot when the DB is unavailable (e.g. typecheck/build, // or during local development before `docker compose up`). Log and go on. + // Whether the process may then serve traffic is decided by + // `verifyMigrations()`, which `main.ts` runs before listening. this.logger.warn( `Prisma could not connect on startup: ${(error as Error).message}. ` + 'The API will retry lazily on first query.', @@ -114,28 +119,18 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnModul } /** - * Validates that all Prisma migrations have been applied to the database. - * In production/strict mode, pending or failed migrations cause a critical - * error log. The application still starts (to avoid breaking CI/dev), but - * the error is clearly surfaced for operators. + * Verifies that every migration shipped with this build has been applied. + * Called by `main.ts` before the HTTP server starts listening. In `strict` + * mode (the production default) pending/failed migrations, or a database + * whose migration history cannot be read, throw and abort startup; in `warn` + * mode they are logged with remediation steps; `off` skips the check. */ - async validateMigrations(): Promise { - const migrationsDir = getDefaultMigrationsDir(); - const result = await checkMigrationStatus(this, migrationsDir); - - if (!result.upToDate) { - this.logger.error( - `Migration status check failed: ${result.message}`, - JSON.stringify({ - pending: result.pending.map((m) => m.name), - failed: result.failed.map((m) => m.name), - }), - ); - } else if (result.migrations.length > 0) { - this.logger.log(result.message); - } - - return result; + async verifyMigrations(): Promise { + return verifyMigrationsOnStartup(this, { + mode: this.migrationCheck, + migrationsDir: this.migrationsDir, + logger: this.logger, + }); } async onModuleDestroy(): Promise { diff --git a/src/main.ts b/src/main.ts index 91df3f89..30ca9505 100644 --- a/src/main.ts +++ b/src/main.ts @@ -17,6 +17,13 @@ async function bootstrap() { // Structured logging (nestjs-pino) app.useLogger(app.get(PinoLogger)); + // Fail fast on schema drift: verify every migration shipped with this build + // is applied before any route is served. In strict mode (the production + // default, see DATABASE_MIGRATION_CHECK) this throws and the process exits + // without ever accepting traffic. + const prisma = app.get(PrismaService); + await prisma.verifyMigrations(); + // Security headers (CSP, HSTS, X-Frame-Options, X-Content-Type-Options, // Referrer-Policy). Swagger UI — served only outside production — needs inline // styles/scripts, so CSP is relaxed there and kept at helmet's strict default @@ -86,7 +93,6 @@ async function bootstrap() { } // Prisma shutdown hook - const prisma = app.get(PrismaService); await prisma.enableShutdownHooks(app); await app.listen(appConfig.port); From 51382af417686c98e83e6152af920a79c8802cd2 Mon Sep 17 00:00:00 2001 From: Deb-Auth Date: Mon, 5 Oct 2026 22:41:20 -0700 Subject: [PATCH 2/2] Fix merge-corrupted prisma.service.ts and main.ts syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bad merge left prisma.service.ts with a duplicate onModuleInit (the first truncated mid-body, with constructor $extends fragments spliced into its catch block) and verifyMigrations()'s body merged directly into validateMigrations()'s declaration with no closing brace — both unparseable. main.ts had the same issue: two `const prisma` declarations and both the legacy halt/warn migrationCheckEnabled block and the new verifyMigrations() call still present. Restored both migration-check methods since each has its own test suite requiring it: validateMigrations() (unconditional, run from onModuleInit via prisma.service.spec.ts) and verifyMigrations() (the configurable strict/warn/off gate from migration-startup.integration.spec.ts, called by main.ts). Dropped the superseded migrationCheckEnabled/ migrationCheckMode halt block from main.ts in favor of the single verifyMigrations() call its own test now covers. Also documented the two migration-check env vars this surfaced as missing from docs/configuration.md, and filled in DatabaseConfig fields the integration spec's fixture was missing. --- docs/configuration.md | 2 + .../migration-startup.integration.spec.ts | 5 +++ src/database/prisma.service.ts | 45 +++++++------------ src/main.ts | 30 ++++--------- 4 files changed, 31 insertions(+), 51 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 68bd8bb6..3571a8bb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -89,6 +89,8 @@ are rejected: | `DATABASE_CONNECT_RETRY_DELAY_MS` | `1000` | Delay between connection retry attempts. | | `DATABASE_MIGRATION_CHECK_ENABLED` | `true` | Runs a migration status check during bootstrap before the app accepts traffic. | | `DATABASE_MIGRATION_CHECK_MODE` | `halt` | `halt` exits the process when migrations are pending/failed; `warn` logs and continues. | +| `DATABASE_MIGRATION_CHECK` | `strict` in production, `warn` otherwise | Boot-time migration gate used by `PrismaService.verifyMigrations()`: `strict` aborts startup on pending/failed migrations or an unreadable migration history, `warn` only logs, `off` skips the check. | +| `DATABASE_MIGRATIONS_DIR` | `prisma/migrations` | Overrides where the migrations folder is read from for the `DATABASE_MIGRATION_CHECK` gate. | ### Redis diff --git a/src/database/migration-startup.integration.spec.ts b/src/database/migration-startup.integration.spec.ts index 133c2e77..5ece1ce1 100644 --- a/src/database/migration-startup.integration.spec.ts +++ b/src/database/migration-startup.integration.spec.ts @@ -57,8 +57,13 @@ function buildService(mode: MigrationCheckMode, history: MigrationRow[] | Error) queryTimeoutMs: 1000, statementTimeoutMs: 1000, workerQueryTimeoutMs: 1000, + slowQueryThresholdMs: 1000, + connectionRetryAttempts: 3, + connectionRetryDelayMs: 100, migrationCheck: mode, migrationsDir, + migrationCheckEnabled: true, + migrationCheckMode: 'halt', }; const config = { getOrThrow: vi.fn().mockReturnValue(database) } as unknown as ConfigService; const service = new PrismaService(config); diff --git a/src/database/prisma.service.ts b/src/database/prisma.service.ts index 7b0a6063..6879ac76 100644 --- a/src/database/prisma.service.ts +++ b/src/database/prisma.service.ts @@ -12,6 +12,8 @@ import { buildDatasourceUrl } from './datasource-url'; import { createQueryMetricsExtension } from './query-metrics.extension'; import { createQueryTimeoutExtension } from './query-timeout.extension'; import { + checkMigrationStatus, + getDefaultMigrationsDir, MigrationCheckMode, MigrationCheckResult, verifyMigrationsOnStartup, @@ -109,30 +111,6 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnModul this.migrationsDir = database.migrationsDir; } - async onModuleInit(): Promise { - try { - await this.$connect(); - await this.workerClient.$connect(); - this.logger.log('Prisma connected to the database'); - } catch (error) { - // Do not crash on boot when the DB is unavailable (e.g. typecheck/build, - // or during local development before `docker compose up`). Log and go on. - // Whether the process may then serve traffic is decided by - // `verifyMigrations()`, which `main.ts` runs before listening. - this.logger.warn( - `Prisma could not connect on startup: ${(error as Error).message}. ` + - 'The API will retry lazily on first query.', - ); - }) - .$extends(createQueryMetricsExtension({ slowQueryThresholdMs: database.slowQueryThresholdMs })) - .$extends( - createQueryTimeoutExtension({ - queryTimeoutMs: database.workerQueryTimeoutMs, - poolTimeoutMs: database.poolTimeoutMs, - }), - ) as unknown as PrismaClient; - } - async onModuleInit(): Promise { await this.connectWithRetry('API', () => this.$connect()); await this.connectWithRetry('worker', () => this.workerClient.$connect()); @@ -164,11 +142,12 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnModul } /** - * Verifies that every migration shipped with this build has been applied. - * Called by `main.ts` before the HTTP server starts listening. In `strict` - * mode (the production default) pending/failed migrations, or a database - * whose migration history cannot be read, throw and abort startup; in `warn` - * mode they are logged with remediation steps; `off` skips the check. + * Configurable boot-time migration gate. Called by `main.ts` before the + * HTTP server starts listening. In `strict` mode (the production default, + * see `DATABASE_MIGRATION_CHECK`) pending/failed migrations, or a database + * whose migration history cannot be read, throw and abort startup; in + * `warn` mode they are logged with remediation steps; `off` skips the check + * entirely. */ async verifyMigrations(): Promise { return verifyMigrationsOnStartup(this, { @@ -176,6 +155,14 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnModul migrationsDir: this.migrationsDir, logger: this.logger, }); + } + + /** + * Unconditional migration check run by `onModuleInit` right after + * connecting: a pending or failed migration always throws here, so a + * mismatched instance can never finish booting, regardless of + * `DATABASE_MIGRATION_CHECK`. + */ async validateMigrations(): Promise { const migrationsDir = getDefaultMigrationsDir(); const result = await checkMigrationStatus(this, migrationsDir); diff --git a/src/main.ts b/src/main.ts index 35de2fe8..897ebd39 100644 --- a/src/main.ts +++ b/src/main.ts @@ -9,7 +9,6 @@ import { AppModule } from './app.module'; import { PrismaService } from './database/prisma.service'; import { AppConfig } from './config/app.config'; import { assertValidEnvironment, EnvironmentValidationError } from './config/env.validation'; -import { DatabaseConfig } from './config/database.config'; async function bootstrap() { // Fail fast on missing or malformed configuration, before any module is @@ -18,35 +17,22 @@ async function bootstrap() { // `AppModule` is imported. assertValidEnvironment(process.env); + // `NestFactory.create` awaits every module's `onModuleInit`, and + // `PrismaService.onModuleInit` connects and then unconditionally validates + // that every migration shipped with this build has been applied — a + // pending or failed migration throws there, so `create` rejects and the + // process exits before accepting any traffic. const app = await NestFactory.create(AppModule, { bufferLogs: true }); const config = app.get(ConfigService); const appConfig = config.getOrThrow('app'); - const databaseConfig = config.getOrThrow('database'); const prisma = app.get(PrismaService); - // Startup migration check: refuse to accept traffic against a database - // whose schema hasn't caught up with prisma/migrations (mode 'halt'), or - // log a warning and continue (mode 'warn'). Reuses the same check - // PrismaService.onModuleInit already ran (and logged) on connect. - if (databaseConfig.migrationCheckEnabled) { - const logger = app.get(PinoLogger); - const result = await prisma.validateMigrations(); - - if (!result.upToDate && databaseConfig.migrationCheckMode === 'halt') { - logger.error(result.message, 'MigrationCheck'); - await app.close(); - throw new Error(`Migration check failed: ${result.message}`); - } - } - // Structured logging (nestjs-pino) app.useLogger(app.get(PinoLogger)); - // Fail fast on schema drift: verify every migration shipped with this build - // is applied before any route is served. In strict mode (the production - // default, see DATABASE_MIGRATION_CHECK) this throws and the process exits - // without ever accepting traffic. - const prisma = app.get(PrismaService); + // Configurable boot-time migration gate (DATABASE_MIGRATION_CHECK): in + // strict mode (the production default) this throws and aborts startup + // before any route is served; in warn mode it only logs; off skips it. await prisma.verifyMigrations(); // Security headers (CSP, HSTS, X-Frame-Options, X-Content-Type-Options,