fix(health): readiness probe returns 503 when the database is down

src/health/health.controller.spec.ts

@@ -1,20 +1,42 @@
 import { Test, TestingModule } from '@nestjs/testing';
+import { INestApplication } from '@nestjs/common';
+import request from 'supertest';
 import { HealthController } from './health.controller';
 import {
+  HealthCheckError,
   HealthCheckService,
   HttpHealthIndicator,
+  TerminusModule,
   TypeOrmHealthIndicator,
 } from '@nestjs/terminus';
 import { MatrixHealthIndicator } from '../matrix/health/matrix.health';
+import { GlobalExceptionFilter } from '../filters/global-exception.filter';
 
 describe('HealthController', () => {
   let controller: HealthController;
   let healthCheckService: HealthCheckService;
+  let dbIndicator: { pingCheck: jest.Mock };
+  let matrixIndicator: { isHealthy: jest.Mock };
 
   beforeEach(async () => {
     healthCheckService = {
       check: jest.fn(),
     } as any;
+    dbIndicator = {
+      pingCheck: jest.fn().mockResolvedValue({ database: { status: 'up' } }),
+    };
+    matrixIndicator = {
+      isHealthy: jest.fn().mockResolvedValue({
+        status: 'up',
+        matrix: {
+          serverAvailable: true,
+          tokenState: 'valid',
+          tokenValid: true,
+          adminPrivilegesValid: true,
+          serverUrl: 'https://matrix.example.org',
+        },
+      }),
+    };
 
     const module: TestingModule = await Test.createTestingModule({
       controllers: [HealthController],
@@ -29,22 +51,11 @@ describe('HealthController', () => {
         },
         {
           provide: TypeOrmHealthIndicator,
-          useValue: {},
+          useValue: dbIndicator,
         },
         {
           provide: MatrixHealthIndicator,
-          useValue: {
-            isHealthy: jest.fn().mockResolvedValue({
-              status: 'up',
-              matrix: {
-                serverAvailable: true,
-                tokenState: 'valid',
-                tokenValid: true,
-                adminPrivilegesValid: true,
-                serverUrl: 'https://matrix.example.org',
-              },
-            }),
-          },
+          useValue: matrixIndicator,
         },
       ],
     }).compile();
@@ -68,16 +79,31 @@ describe('HealthController', () => {
       });
     });
 
-    it('should return status error when database is disconnected', async () => {
-      const error = new Error('Database connection failed');
+    it('should propagate the ServiceUnavailableException when the database is disconnected, so the probe sees a 503', async () => {
+      // health.check() throws ServiceUnavailableException on a failed check;
+      // readiness must let it reach the HTTP layer instead of returning a 200 body.
+      const error = new Error('Health Check has failed!');
       jest.spyOn(healthCheckService, 'check').mockRejectedValue(error);
 
-      const result = await controller.readiness();
-      expect(result).toEqual({
-        status: 'error',
-        database: 'disconnected',
-        error: error.message,
-      });
+      await expect(controller.readiness()).rejects.toBe(error);
+    });
+
+    it('should gate on the database only — matrix is not part of readiness', async () => {
+      // Decision (TS, 2026-08-23): a Matrix outage degrades chat, not the API,
+      // and must not pull api pods out of the Service endpoints.
+      jest
+        .spyOn(healthCheckService, 'check')
+        .mockImplementation(async (checks) => {
+          for (const check of checks) {
+            await check();
+          }
+          return { status: 'ok' } as any;
+        });
+
+      await controller.readiness();
+
+      expect(dbIndicator.pingCheck).toHaveBeenCalledWith('database');
+      expect(matrixIndicator.isHealthy).not.toHaveBeenCalled();
     });
   });
 
@@ -107,3 +133,53 @@ describe('HealthController', () => {
     });
   });
 });
+
+describe('HealthController (HTTP, real Terminus check pipeline)', () => {
+  let app: INestApplication;
+  const dbIndicator = { pingCheck: jest.fn() };
+
+  beforeAll(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      imports: [TerminusModule],
+      controllers: [HealthController],
+      providers: [
+        { provide: HttpHealthIndicator, useValue: {} },
+        { provide: TypeOrmHealthIndicator, useValue: dbIndicator },
+        { provide: MatrixHealthIndicator, useValue: { isHealthy: jest.fn() } },
+      ],
+    }).compile();
+
+    app = module.createNestApplication({ logger: false });
+    // Mirror production: GlobalExceptionFilter (APP_FILTER in InterceptorsModule)
+    // catches the ServiceUnavailableException, keeps its 503, and rewrites the body.
+    app.useGlobalFilters(new GlobalExceptionFilter({ inc: jest.fn() } as any));
+    await app.init();
+  });
+
+  afterAll(async () => {
+    await app.close();
+  });
+
+  it('should answer 503 when the database ping fails', async () => {
+    dbIndicator.pingCheck.mockRejectedValue(
+      new HealthCheckError('TypeOrm health check failed', {
+        database: { status: 'down' },
+      }),
+    );
+
+    const res = await request(app.getHttpServer()).get('/health/readiness');
+
+    expect(res.status).toBe(503);
+    expect(res.body.statusCode).toBe(503);
+    expect(res.body.path).toBe('/health/readiness');
+  });
+
+  it('should answer 200 when the database ping succeeds', async () => {
+    dbIndicator.pingCheck.mockResolvedValue({ database: { status: 'up' } });
+
+    const res = await request(app.getHttpServer()).get('/health/readiness');
+
+    expect(res.status).toBe(200);
+    expect(res.body.status).toBe('ok');
+  });
+});

src/health/health.controller.ts

@@ -38,18 +38,11 @@ export class HealthController {
   @Get('readiness')
   @ApiOperation({ summary: 'Readiness probe' })
   async readiness() {
-    try {
-      return await this.health.check([
-        () => this.db.pingCheck('database'),
-        () => this.matrix.isHealthy('matrix'),
-      ]);
-    } catch (error) {
-      return {
-        status: 'error',
-        database: 'disconnected',
-        error: error.message,
-      };
-    }
+    // Let the ServiceUnavailableException propagate: the 503 is how the probe
+    // learns the pod cannot serve, so Kubernetes pulls it from the Service.
+    // Database only — a Matrix outage degrades chat, not the API, and must not
+    // take every api pod out of rotation (see /health/matrix for that signal).
+    return await this.health.check([() => this.db.pingCheck('database')]);
   }
 
   @HealthCheck()