diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 52029992..9d71b0a9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -115,4 +115,3 @@ jobs: run: npm test --if-present # Workflow run retention settings -retention-days: 30 \ No newline at end of file diff --git a/.github/workflows/contract-release.yml b/.github/workflows/contract-release.yml index 2655fcbc..3f66b1a5 100644 --- a/.github/workflows/contract-release.yml +++ b/.github/workflows/contract-release.yml @@ -29,4 +29,3 @@ jobs: release_token: ${{ secrets.GITHUB_TOKEN }} # Workflow run retention settings -retention-days: 90 \ No newline at end of file diff --git a/.github/workflows/dapp-ipfs.yml b/.github/workflows/dapp-ipfs.yml index 29d16e55..42248532 100644 --- a/.github/workflows/dapp-ipfs.yml +++ b/.github/workflows/dapp-ipfs.yml @@ -67,4 +67,3 @@ jobs: echo "- URL: ${{ steps.storacha.outputs.url }}" >> "$GITHUB_STEP_SUMMARY" # Workflow run retention settings -retention-days: 30 \ No newline at end of file diff --git a/.github/workflows/secrets-check.yml b/.github/workflows/secrets-check.yml index b8d3b7e8..c7a34807 100644 --- a/.github/workflows/secrets-check.yml +++ b/.github/workflows/secrets-check.yml @@ -22,4 +22,3 @@ jobs: run: ./scripts/check-k8s-secrets.sh # Workflow run retention settings -retention-days: 30 \ No newline at end of file diff --git a/backend/src/app.ts b/backend/src/app.ts index e2a3bab4..6f91239b 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -113,6 +113,10 @@ app.use( }) ); +// Modified 2026-10-05: keep public pool monitoring outside DB-dependent rate limiting. +// Bypass-token checks and rate-limit violation logging can await the exhausted pool. +app.get('/health/db', HealthController.getDatabaseHealth); + // Global rate limiting — organization-tier based, always on app.use( tieredOrganizationRateLimit({ diff --git a/backend/src/controllers/__tests__/healthController.test.ts b/backend/src/controllers/__tests__/healthController.test.ts index 0fbb003c..299385c9 100644 --- a/backend/src/controllers/__tests__/healthController.test.ts +++ b/backend/src/controllers/__tests__/healthController.test.ts @@ -10,17 +10,31 @@ jest.mock('../../config/env.js', () => ({ }, })); +jest.mock('../../config/database.js', () => ({ + pool: { + query: jest.fn(), + totalCount: 0, + idleCount: 0, + waitingCount: 0, + options: { max: 10 }, + }, +})); + +jest.mock('../../utils/logger.js', () => ({ + __esModule: true, + default: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }, +})); + import { HealthController, healthConfig } from '../healthController.js'; -import pg from 'pg'; +import { pool } from '../../config/database.js'; +import logger from '../../utils/logger.js'; import { Redis } from 'ioredis'; import { StellarService } from '../../services/stellarService.js'; -// Setup Mock for pg -jest.mock('pg', () => { - const mPool = { query: jest.fn() }; - return { Pool: jest.fn(() => mPool) }; -}); - // Setup Mock for ioredis jest.mock('ioredis', () => { const mRedis = { ping: jest.fn() }; @@ -37,8 +51,10 @@ jest.mock('../../services/stellarService', () => ({ })); const app = express(); +const dbPool = pool as any; app.get('/health', HealthController.getHealthStatus); app.get('/health/live', HealthController.getLiveness); +app.get('/health/db', HealthController.getDatabaseHealth); describe('HealthController GET /health/live', () => { it('returns 200 OK with liveness status', async () => { @@ -55,12 +71,10 @@ describe('HealthController GET /health/live', () => { }); describe('HealthController GET /health', () => { - let pool: any; let redisClient: any; let mockServer: any; beforeEach(() => { - pool = new pg.Pool(); redisClient = new Redis(); mockServer = { feeStats: jest.fn() }; (StellarService.getServer as jest.Mock).mockReturnValue(mockServer); @@ -72,7 +86,7 @@ describe('HealthController GET /health', () => { }); it('returns 200 OK when all dependencies are healthy', async () => { - pool.query.mockResolvedValueOnce({ rows: [] }); + dbPool.query.mockResolvedValueOnce({ rows: [] }); redisClient.ping.mockResolvedValueOnce('PONG'); mockServer.feeStats.mockResolvedValueOnce({}); @@ -86,7 +100,7 @@ describe('HealthController GET /health', () => { }); it('returns 503 Degraded when Postgres goes down', async () => { - pool.query.mockRejectedValueOnce(new Error('Connection forced closed')); + dbPool.query.mockRejectedValueOnce(new Error('Connection forced closed')); redisClient.ping.mockResolvedValueOnce('PONG'); mockServer.feeStats.mockResolvedValueOnce({}); @@ -101,7 +115,7 @@ describe('HealthController GET /health', () => { }); it('returns 503 Degraded when Redis fails', async () => { - pool.query.mockResolvedValueOnce({ rows: [] }); + dbPool.query.mockResolvedValueOnce({ rows: [] }); redisClient.ping.mockRejectedValueOnce(new Error('Redis timeout')); mockServer.feeStats.mockResolvedValueOnce({}); @@ -116,7 +130,7 @@ describe('HealthController GET /health', () => { }); it('returns 503 Degraded when Horizon fails', async () => { - pool.query.mockResolvedValueOnce({ rows: [] }); + dbPool.query.mockResolvedValueOnce({ rows: [] }); redisClient.ping.mockResolvedValueOnce('PONG'); mockServer.feeStats.mockRejectedValueOnce(new Error('Horizon unreachable')); @@ -134,7 +148,7 @@ describe('HealthController GET /health', () => { it('returns 503 Degraded when Postgres query times out', async () => { healthConfig.timeoutMs = 50; // set timeout to 50ms // Mock db query to hang (resolve after 200ms) - pool.query.mockReturnValue(new Promise((resolve) => setTimeout(resolve, 200))); + dbPool.query.mockReturnValue(new Promise((resolve) => setTimeout(resolve, 200))); redisClient.ping.mockResolvedValueOnce('PONG'); mockServer.feeStats.mockResolvedValueOnce({}); @@ -148,7 +162,7 @@ describe('HealthController GET /health', () => { it('returns 503 Degraded when Redis ping times out', async () => { healthConfig.timeoutMs = 50; // set timeout to 50ms - pool.query.mockResolvedValueOnce({ rows: [] }); + dbPool.query.mockResolvedValueOnce({ rows: [] }); // Mock redis ping to hang redisClient.ping.mockReturnValue(new Promise((resolve) => setTimeout(resolve, 200))); mockServer.feeStats.mockResolvedValueOnce({}); @@ -163,7 +177,7 @@ describe('HealthController GET /health', () => { it('returns 503 Degraded when Horizon feeStats times out', async () => { healthConfig.timeoutMs = 50; // set timeout to 50ms - pool.query.mockResolvedValueOnce({ rows: [] }); + dbPool.query.mockResolvedValueOnce({ rows: [] }); redisClient.ping.mockResolvedValueOnce('PONG'); // Mock horizon feeStats to hang mockServer.feeStats.mockReturnValue(new Promise((resolve) => setTimeout(resolve, 200))); @@ -176,3 +190,61 @@ describe('HealthController GET /health', () => { expect(response.body.dependencies.horizon.error).toContain('Horizon feeStats timeout'); }); }); + + +describe('HealthController GET /health/db', () => { + beforeEach(() => { + dbPool.totalCount = 5; + dbPool.idleCount = 2; + dbPool.waitingCount = 0; + dbPool.options.max = 10; + jest.clearAllMocks(); + }); + + it('returns live application pool statistics', async () => { + const response = await request(app).get('/health/db'); + + expect(response.status).toBe(200); + expect(response.body.status).toBe('ok'); + expect(response.body.pool).toEqual({ + active: 3, + idle: 2, + waiting: 0, + total: 5, + max: 10, + utilization: 30, + }); + }); + + it('logs a warning when active pool utilization exceeds 80 percent', async () => { + dbPool.totalCount = 9; + dbPool.idleCount = 0; + + const response = await request(app).get('/health/db'); + + expect(response.status).toBe(200); + expect(response.body.status).toBe('warning'); + expect(response.body.pool.utilization).toBe(90); + expect(logger.warn).toHaveBeenCalledWith( + 'Database connection pool utilization above 80%', + response.body.pool, + ); + }); + + it('returns 503 and logs an error when the pool is exhausted', async () => { + dbPool.totalCount = 10; + dbPool.idleCount = 0; + dbPool.waitingCount = 2; + + const response = await request(app).get('/health/db'); + + expect(response.status).toBe(503); + expect(response.body.status).toBe('exhausted'); + expect(response.body.pool.active).toBe(10); + expect(response.body.pool.waiting).toBe(2); + expect(logger.error).toHaveBeenCalledWith( + 'Database connection pool exhausted', + response.body.pool, + ); + }); +}); diff --git a/backend/src/controllers/healthController.ts b/backend/src/controllers/healthController.ts index 2f11c9b8..77385323 100644 --- a/backend/src/controllers/healthController.ts +++ b/backend/src/controllers/healthController.ts @@ -1,11 +1,10 @@ import { Request, Response } from 'express'; -import pg from 'pg'; import { Redis } from 'ioredis'; import { config } from '../config/env.js'; +import { pool } from '../config/database.js'; +import logger from '../utils/logger.js'; import { StellarService } from '../services/stellarService.js'; -const pool = new pg.Pool({ connectionString: config.DATABASE_URL }); - export const redis: Redis | null = config.REDIS_URL ? new Redis(config.REDIS_URL, { maxRetriesPerRequest: 1, @@ -39,6 +38,19 @@ export interface LivenessReport { environment: string; } +export interface DatabasePoolHealthReport { + status: 'ok' | 'warning' | 'exhausted'; + timestamp: string; + pool: { + active: number; + idle: number; + waiting: number; + total: number; + max: number; + utilization: number; + }; +} + export const healthConfig = { timeoutMs: 5000, }; @@ -59,6 +71,41 @@ function withTimeout(promise: Promise, timeoutMs: number, errorMessage: st } export class HealthController { + static getDatabaseHealth(req: Request, res: Response): void { + const total = pool.totalCount; + const idle = pool.idleCount; + const waiting = pool.waitingCount; + const active = Math.max(0, total - idle); + const max = pool.options.max ?? 10; + const utilization = max > 0 ? (active / max) * 100 : 0; + const exhausted = active >= max || (waiting > 0 && total >= max); + + const report: DatabasePoolHealthReport = { + status: exhausted ? 'exhausted' : utilization > 80 ? 'warning' : 'ok', + timestamp: new Date().toISOString(), + pool: { + active, + idle, + waiting, + total, + max, + utilization: Number(utilization.toFixed(1)), + }, + }; + + if (exhausted) { + logger.error('Database connection pool exhausted', report.pool); + res.status(503).json(report); + return; + } + + if (utilization > 80) { + logger.warn('Database connection pool utilization above 80%', report.pool); + } + + res.status(200).json(report); + } + static getLiveness(req: Request, res: Response): void { const timestamp = new Date().toISOString(); const uptime = process.uptime();