feat(shutdown): graceful shutdown for HTTP, BullMQ workers, queues, Redis and Prisma

.env.example

@@ -96,6 +96,12 @@ PUBLIC_RATE_LIMIT_TRUST_PROXY=false
 # Prometheus server outside the private network.
 METRICS_ALLOWED_IPS=127.0.0.1/32,::1/128,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
 
+# Graceful shutdown
+# On SIGTERM/SIGINT, in-flight HTTP requests and BullMQ jobs get this long (ms)
+# to finish before being forcibly terminated. Keep it ~10s below the
+# orchestrator's kill deadline (Kubernetes default: 30s).
+SHUTDOWN_GRACE_PERIOD_MS=20000
+
 # AI Provider (Nvidia NIM API compatible with OpenAI SDK)
 AI_PROVIDER=nvidia
 AI_PROVIDER_KEY=your_nvidia_api_key_starting_with_nvapi

docs/graceful-shutdown.md

@@ -0,0 +1,61 @@
+# Graceful Shutdown
+
+On `SIGTERM` or `SIGINT`, the API drains in-flight work before exiting, so a
+deployment never abandons an HTTP request or a BullMQ job halfway through a
+financial operation.
+
+The sequence is run by `ShutdownCoordinator` (`src/common/shutdown`), installed
+from `src/main.ts`:
+
+| Step | What happens | Bound |
+| --- | --- | --- |
+| 1. HTTP | The server stops accepting connections. Idle keep-alive connections are closed and in-flight responses carry `Connection: close`. | Grace period |
+| 2. Workers | Every BullMQ worker is paused (no new jobs are fetched) and its active jobs finish, then the worker is closed. | Grace period (shared with step 1) |
+| 3. Lifecycle hooks | `app.close()` runs Nest's `onModuleDestroy` / `beforeApplicationShutdown` / `onApplicationShutdown` hooks, so every provider releases what it owns. | 15s |
+| 4. Queues | BullMQ queues are closed. | 3s per resource |
+| 5. Redis | Shared Redis clients are closed with `QUIT`. | 3s per resource |
+| 6. Database | Both Prisma connection pools are disconnected. | 3s per resource |
+
+Steps 4-6 run inside step 3 (from `beforeApplicationShutdown`), so they also
+run when a module is closed outside a signal, for example in tests.
+
+## Exit codes
+
+- `0`: everything drained and closed.
+- `1`: the grace period expired with requests or jobs still in progress, or a
+  resource failed to close. The logs name the step and resource that failed.
+  They include resource names and timings only, never job payloads or
+  connection strings.
+
+When the grace period expires, remaining HTTP connections are destroyed and
+busy workers are force-closed. The interrupted jobs' locks lapse, and BullMQ's
+stalled-job recovery re-queues them for another worker.
+
+Repeated signals while shutdown is in progress are logged and ignored.
+Shutdown runs once and is always bounded, so it cannot hang.
+
+## Configuration
+
+| Variable | Default | Description |
+| --- | --- | --- |
+| `SHUTDOWN_GRACE_PERIOD_MS` | `20000` | Time in-flight HTTP requests and BullMQ jobs get to finish. |
+
+Keep the grace period about 10 seconds below your orchestrator's kill
+deadline, to leave room for steps 3-6. Kubernetes' default
+`terminationGracePeriodSeconds` is 30.
+
+## Adding a resource
+
+Providers keep ownership of the connections they create. They register how to
+close each connection, and the coordinator decides when:
+
+```ts
+shutdown.register({
+  name: 'redis:my-feature', // shown in logs; never a URL or credential
+  phase: 'redis',           // 'queues' | 'redis' | 'database'
+  close: () => closeRedisClient(client),
+});
+```
+
+Workers and queues created with `@nestjs/bullmq` (`@Processor`,
+`BullModule.registerQueue`) are discovered automatically.

src/app.module.ts

@@ -11,6 +11,7 @@ import { RedisThrottlerStorage } from './common/throttler/redis-throttler.storag
 import { DatabaseModule } from './database/database.module';
 import { EventsModule } from './events/events.module';
 import { LocksModule } from './common/locks/locks.module';
+import { ShutdownModule } from './common/shutdown/shutdown.module';
 import { REDIS_CLIENT } from './common/locks/locks.constants';
 import { EncryptionModule } from './common/encryption/encryption.module';
 import { RequestIdMiddleware } from './middleware/request-id.middleware';
@@ -105,6 +106,7 @@ import { RequestIdInterceptor } from './common/interceptors/request-id.intercept
         ),
     }),
 
+    ShutdownModule,
     DatabaseModule,
     EventsModule,
     LocksModule,

src/common/locks/locks.module.ts

@@ -3,6 +3,8 @@ import { APP_INTERCEPTOR } from '@nestjs/core';
 import { ConfigService } from '@nestjs/config';
 import Redis from 'ioredis';
 import { RedisConfig } from '../../config/redis.config';
+import { ShutdownCoordinator } from '../shutdown/shutdown-coordinator.service';
+import { closeRedisClient } from '../shutdown/close-redis-client';
 import { REDIS_CLIENT } from './locks.constants';
 import { RedisLock } from './redis-lock.util';
 import { AgentLockInterceptor } from './agent-lock.interceptor';
@@ -14,7 +16,7 @@ import { TransactionLockInterceptor } from './transaction-lock.interceptor';
  * Global distributed-locking infrastructure.
  *
  * Provides a single shared ioredis client and the {@link RedisLock} service to
- * every module, and registers the {@link AgentLockInterceptor} and
+ * every module (the client is closed by the {@link ShutdownCoordinator}), and registers the {@link AgentLockInterceptor} and
  * {@link BudgetLockInterceptor} that enforce `@UseAgentLock()` and
  * `@UseBudgetLock()` on any decorated controller method.
  */
@@ -23,10 +25,13 @@ import { TransactionLockInterceptor } from './transaction-lock.interceptor';
   providers: [
     {
       provide: REDIS_CLIENT,
-      inject: [ConfigService],
-      useFactory: (config: ConfigService) => {
+      inject: [ConfigService, ShutdownCoordinator],
+      useFactory: (config: ConfigService, shutdown: ShutdownCoordinator) => {
         const { host, port, password, db } = config.getOrThrow<RedisConfig>('redis');
-        return new Redis({ host, port, password, db });
+        const client = new Redis({ host, port, password, db });
+        // Closed after queues and before the database; see ShutdownCoordinator.
+        shutdown.register({ name: 'redis:shared', phase: 'redis', close: () => closeRedisClient(client) });
+        return client;
       },
     },
     RedisLock,

src/common/locks/redis-lock.util.spec.ts

@@ -179,8 +179,8 @@ describe('RedisLock', () => {
     });
   });
 
-  it('disconnects the shared client on module destroy', () => {
-    lock.onModuleDestroy();
-    expect(redis.disconnect).toHaveBeenCalled();
+  it('does not close the shared client it does not own', () => {
+    expect((lock as unknown as Record<string, unknown>).onModuleDestroy).toBeUndefined();
+    expect(redis.disconnect).not.toHaveBeenCalled();
   });
 });

src/common/locks/redis-lock.util.ts

@@ -1,4 +1,4 @@
-import { Inject, Injectable, OnModuleDestroy } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import { Redis } from 'ioredis';
 import { randomUUID } from 'crypto';
 import { LockNotAcquiredException } from '../exceptions/domain.exception';
@@ -42,13 +42,11 @@ export type LockRelease = () => Promise<void>;
  * ```
  */
 @Injectable()
-export class RedisLock implements OnModuleDestroy {
+export class RedisLock {
+  // The shared client is owned by LocksModule and closed in the shutdown
+  // coordinator's `redis` phase, after queues have drained.
   constructor(@Inject(REDIS_CLIENT) private readonly redis: Redis) {}
 
-  onModuleDestroy(): void {
-    this.redis.disconnect();
-  }
-
   /**
    * Acquires a distributed lock with automatic expiration.
    * @param key - Lock key (should be unique per resource).

src/common/shutdown/close-redis-client.spec.ts

@@ -0,0 +1,48 @@
+import { describe, expect, it, vi } from 'vitest';
+import Redis from 'ioredis';
+import { closeRedisClient } from './close-redis-client';
+
+function client(status: string, quit: () => Promise<unknown> = async () => 'OK') {
+  return { status, quit: vi.fn(quit), disconnect: vi.fn() };
+}
+
+describe('closeRedisClient', () => {
+  it('sends QUIT to a connected client so pending replies are delivered', async () => {
+    const redis = client('ready');
+
+    await closeRedisClient(redis as unknown as Redis);
+
+    expect(redis.quit).toHaveBeenCalledTimes(1);
+    expect(redis.disconnect).not.toHaveBeenCalled();
+  });
+
+  it.each(['wait', 'connecting', 'reconnecting'])(
+    'disconnects a client in the %s state without QUIT',
+    async (status) => {
+      const redis = client(status);
+
+      await closeRedisClient(redis as unknown as Redis);
+
+      expect(redis.quit).not.toHaveBeenCalled();
+      expect(redis.disconnect).toHaveBeenCalledTimes(1);
+    },
+  );
+
+  it('is a no-op for a client that is already closed', async () => {
+    const redis = client('end');
+
+    await closeRedisClient(redis as unknown as Redis);
+
+    expect(redis.quit).not.toHaveBeenCalled();
+    expect(redis.disconnect).not.toHaveBeenCalled();
+  });
+
+  it('falls back to disconnect when QUIT fails', async () => {
+    const redis = client('ready', async () => {
+      throw new Error('Connection is closed.');
+    });
+
+    await expect(closeRedisClient(redis as unknown as Redis)).resolves.toBeUndefined();
+    expect(redis.disconnect).toHaveBeenCalledTimes(1);
+  });
+});

src/common/shutdown/close-redis-client.ts

@@ -0,0 +1,21 @@
+import Redis from 'ioredis';
+
+/**
+ * Closes an ioredis client gracefully: `QUIT` lets pending replies arrive
+ * before the socket closes. Clients that never connected (lazyConnect) or are
+ * already closed are simply disconnected, so this is safe to call repeatedly.
+ */
+export async function closeRedisClient(client: Redis): Promise<void> {
+  if (client.status === 'end') {
+    return;
+  }
+  if (client.status !== 'ready') {
+    client.disconnect();
+    return;
+  }
+  try {
+    await client.quit();
+  } catch {
+    client.disconnect();
+  }
+}

src/common/shutdown/index.ts

@@ -0,0 +1,3 @@
+export * from './shutdown.module';
+export * from './shutdown-coordinator.service';
+export * from './close-redis-client';