From ee228fd4a25b9174211e71811d69e6cc89d3bdd3 Mon Sep 17 00:00:00 2001 From: kosiso Date: Wed, 15 Jul 2026 22:20:08 +0100 Subject: [PATCH 1/5] feat: implement Sunset and Deprecation headers for legacy AI routes and Redis sliding window rate limiter --- app/ai-service/config.py | 3 + app/ai-service/main.py | 74 ++++++++--- app/ai-service/test_main.py | 42 +++++++ .../src/common/security/security.module.ts | 119 +++++++++++------- app/backend/src/main.ts | 3 +- app/backend/test/security.e2e-spec.ts | 66 +++++++++- pnpm-lock.yaml | 31 +++-- 7 files changed, 268 insertions(+), 70 deletions(-) diff --git a/app/ai-service/config.py b/app/ai-service/config.py index 67367e49..6ba1bc64 100644 --- a/app/ai-service/config.py +++ b/app/ai-service/config.py @@ -87,6 +87,9 @@ class Settings(BaseSettings): # always appended so operators cannot accidentally expose themselves. request_body_bypass_paths: str = "" + # Legacy route deprecation/retirement date (Sunset header) + legacy_retirement_date: str = "Wed, 01 Oct 2026 00:00:00 GMT" + # Verification artifact access settings verification_artifacts_dir: str = "./artifacts/verification" verification_artifact_url_ttl_seconds: int = 300 diff --git a/app/ai-service/main.py b/app/ai-service/main.py index b067e438..f26632ed 100644 --- a/app/ai-service/main.py +++ b/app/ai-service/main.py @@ -18,6 +18,8 @@ from schemas.errors import ErrorDetail, ErrorEnvelope import time import metrics +import email.utils +from datetime import datetime, timezone from slowapi import Limiter, _rate_limit_exceeded_handler from slowapi.errors import RateLimitExceeded @@ -352,6 +354,35 @@ class ProofOfLifeResponse(BaseModel): app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) +def get_sunset_header_value() -> str: + val = settings.legacy_retirement_date + if not val: + return "" + val = val.strip() + # Try parsing various date formats to normalize to RFC 1123 + for fmt in ( + "%Y-%m-%d", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%d %H:%M:%S", + "%a, %d %b %Y %H:%M:%S %Z", + "%a, %d %b %Y %H:%M:%S", + ): + try: + dt = datetime.strptime(val, fmt) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return email.utils.format_datetime(dt, usegmt=True) + except ValueError: + continue + try: + dt = email.utils.parsedate_to_datetime(val) + return email.utils.format_datetime(dt, usegmt=True) + except Exception: + pass + return val + + @app.middleware("http") async def legacy_redirect_middleware(request: Request, call_next): """ @@ -367,25 +398,38 @@ async def legacy_redirect_middleware(request: Request, call_next): The /ai/metrics path is also excluded - it has no v1 equivalent. """ path = request.url.path + is_legacy = path.startswith("/ai/") and path != "/ai/metrics" - # Exact-match redirects - if path in _LEGACY_TO_V1: - target = _LEGACY_TO_V1[path] - if request.url.query: - target = f"{target}?{request.url.query}" - logger.debug(f"Legacy redirect: {path} -> {target}") - return RedirectResponse(url=target, status_code=308) - - # Prefix-based redirects (parameterised routes) - for legacy_prefix, v1_prefix in _LEGACY_PREFIX_MAP: - if path.startswith(legacy_prefix): - target = v1_prefix + path[len(legacy_prefix) :] + response = None + if is_legacy: + # Exact-match redirects + if path in _LEGACY_TO_V1: + target = _LEGACY_TO_V1[path] if request.url.query: target = f"{target}?{request.url.query}" - logger.debug(f"Legacy prefix redirect: {path} -> {target}") - return RedirectResponse(url=target, status_code=308) + logger.debug(f"Legacy redirect: {path} -> {target}") + response = RedirectResponse(url=target, status_code=308) + else: + # Prefix-based redirects (parameterised routes) + for legacy_prefix, v1_prefix in _LEGACY_PREFIX_MAP: + if path.startswith(legacy_prefix): + target = v1_prefix + path[len(legacy_prefix) :] + if request.url.query: + target = f"{target}?{request.url.query}" + logger.debug(f"Legacy prefix redirect: {path} -> {target}") + response = RedirectResponse(url=target, status_code=308) + break + + if response is None: + response = await call_next(request) - return await call_next(request) + if is_legacy: + sunset_val = get_sunset_header_value() + if sunset_val: + response.headers["Sunset"] = sunset_val + response.headers["Deprecation"] = "true" + + return response @app.middleware("http") diff --git a/app/ai-service/test_main.py b/app/ai-service/test_main.py index e0f83c56..1180c792 100644 --- a/app/ai-service/test_main.py +++ b/app/ai-service/test_main.py @@ -226,3 +226,45 @@ def fake_verify_claim(aid_claim, supporting_evidence=None, context_factors=None, data = response.json() assert data["success"] is False assert "all providers unavailable" in data["error"] + + +def test_legacy_routes_deprecation_headers(client, monkeypatch): + """Test that all legacy /ai/* routes emit Sunset and Deprecation headers, + and preserve backwards compatibility (308 redirect or correct response).""" + # Temporarily set a specific retirement date for validation + monkeypatch.setattr(main.settings, "legacy_retirement_date", "2026-10-01") + + legacy_redirect_routes = [ + "/ai/inference", + "/ai/proof-of-life", + "/ai/anonymize", + "/ai/humanitarian/verify", + "/ai/status/test-task-id", + "/ai/task/test-task-id/cancel", + ] + + for route in legacy_redirect_routes: + # We test post for these legacy routes; middleware intercepts and returns 308 + response = client.post(route, follow_redirects=False) + assert response.status_code == 308, f"Route {route} did not return 308 redirect" + assert response.headers.get("Deprecation") == "true", f"Route {route} missing Deprecation header" + assert response.headers.get("Sunset") == "Thu, 01 Oct 2026 00:00:00 GMT", f"Route {route} missing or invalid Sunset header" + + # Also test /ai/ocr which is served directly and should not redirect, but still have headers + response = client.post("/ai/ocr", follow_redirects=False) + # /ai/ocr without files will return 422 or 400, but should still have the legacy headers + assert response.status_code in (400, 422), f"/ai/ocr returned unexpected status {response.status_code}" + assert response.headers.get("Deprecation") == "true", "/ai/ocr missing Deprecation header" + assert response.headers.get("Sunset") == "Thu, 01 Oct 2026 00:00:00 GMT", "/ai/ocr missing or invalid Sunset header" + + # Test that non-legacy routes (like health, metrics, v1 paths) do NOT have the headers + non_legacy_routes = [ + "/health", + "/ai/metrics", + "/v1/ai/inference", + ] + for route in non_legacy_routes: + response = client.post(route, follow_redirects=False) if "inference" in route else client.get(route, follow_redirects=False) + assert "Deprecation" not in response.headers, f"Non-legacy route {route} has Deprecation header" + assert "Sunset" not in response.headers, f"Non-legacy route {route} has Sunset header" + diff --git a/app/backend/src/common/security/security.module.ts b/app/backend/src/common/security/security.module.ts index f0961b12..1e2ddc46 100644 --- a/app/backend/src/common/security/security.module.ts +++ b/app/backend/src/common/security/security.module.ts @@ -1,8 +1,10 @@ -import { Module } from '@nestjs/common'; +import { Logger, Module } from '@nestjs/common'; import type { CorsOptions } from '@nestjs/common/interfaces/external/cors-options.interface'; import { ConfigService } from '@nestjs/config'; import type { NextFunction, Request, RequestHandler, Response } from 'express'; import helmet, { HelmetOptions } from 'helmet'; +import { RedisService } from '@liaoliaots/nestjs-redis'; + const DEFAULT_ALLOWED_ORIGINS = [ 'http://localhost:3000', @@ -182,33 +184,24 @@ export const createCorsOriginValidator = ( }; }; -export const createRateLimiter = (config: ConfigService): RequestHandler => { +export const createRateLimiter = ( + config: ConfigService, + redisService?: RedisService, +): RequestHandler => { + const logger = new Logger('RateLimiter'); + const windowMs = parseNumber( - config.get('THROTTLE_TTL'), + config.get('RATE_LIMIT_WINDOW_MS') ?? config.get('THROTTLE_TTL'), DEFAULT_RATE_LIMIT_WINDOW_MS, ); const limit = parseNumber( - config.get('API_RATE_LIMIT'), + config.get('RATE_LIMIT_LIMIT') ?? config.get('API_RATE_LIMIT'), DEFAULT_RATE_LIMIT, ); - const store = new Map(); - let lastCleanupMs = 0; - - const cleanupExpiredEntries = (now: number) => { - if (now - lastCleanupMs < windowMs) { - return; - } - - lastCleanupMs = now; - for (const [key, entry] of store) { - if (entry.resetTimeMs <= now) { - store.delete(key); - } - } - }; + const windowSeconds = Math.max(Math.ceil(windowMs / 1000), 1); - return (req: Request, res: Response, next: NextFunction) => { + return async (req: Request, res: Response, next: NextFunction) => { if (isRateLimitExempt(req)) { next(); return; @@ -234,37 +227,77 @@ export const createRateLimiter = (config: ConfigService): RequestHandler => { return; } - const now = Date.now(); - cleanupExpiredEntries(now); - const forwardedIp = Array.isArray(req.ips) && req.ips.length > 0 ? req.ips[0] : undefined; - const key: string = + const key = `ratelimit:global:${ (typeof forwardedIp === 'string' ? forwardedIp : undefined) ?? (typeof req.ip === 'string' ? req.ip : undefined) ?? - 'unknown'; - let entry = store.get(key); - if (!entry || entry.resetTimeMs <= now) { - entry = { count: 0, resetTimeMs: now + windowMs }; - store.set(key, entry); - } + 'unknown' + }`; - entry.count += 1; + const now = Date.now(); + const minTimestamp = now - windowMs; + const uniqueMember = `${now}:${Math.random().toString(36).substring(2, 15)}`; - const remaining = Math.max(limit - entry.count, 0); - const resetSeconds = Math.max( - Math.ceil((entry.resetTimeMs - now) / 1000), - 0, - ); + try { + if (!redisService) { + throw new Error('RedisService is not configured/available'); + } - res.setHeader('RateLimit-Limit', limit.toString()); - res.setHeader('RateLimit-Remaining', remaining.toString()); - res.setHeader('RateLimit-Reset', resetSeconds.toString()); + const client = redisService.getOrThrow(); - if (entry.count > limit) { - res.setHeader('Retry-After', resetSeconds.toString()); - res.status(429).send('Too many requests, please try again later.'); - return; + // Execute MULTI pipeline to keep header reads consistent + const multi = client.multi(); + multi.zremrangebyscore(key, '-inf', minTimestamp); + multi.zadd(key, now, uniqueMember); + multi.zrange(key, 0, 0, 'WITHSCORES'); + multi.zcard(key); + multi.expire(key, windowSeconds); + + const results = await multi.exec(); + if (!results) { + throw new Error('Redis multi transaction execution returned null'); + } + + const zrangeResult = results[2]; + const zcardResult = results[3]; + + const zrangeRes = Array.isArray(zrangeResult) ? (zrangeResult[1] as string[]) : undefined; + const zcardRes = Array.isArray(zcardResult) ? (zcardResult[1] as number) : undefined; + + const count = typeof zcardRes === 'number' ? zcardRes : 1; + + // ZRANGE WITHSCORES returns: [member1, score1, member2, score2, ...] + // The oldest timestamp is the score of the first entry, i.e., index 1 + let oldestTimestamp = now; + if (zrangeRes && zrangeRes.length >= 2) { + const parsed = Number(zrangeRes[1]); + if (!isNaN(parsed)) { + oldestTimestamp = parsed; + } + } + + const remaining = Math.max(limit - count, 0); + const resetSeconds = Math.max( + Math.ceil((oldestTimestamp + windowMs - now) / 1000), + 0, + ); + + res.setHeader('RateLimit-Limit', limit.toString()); + res.setHeader('RateLimit-Remaining', remaining.toString()); + res.setHeader('RateLimit-Reset', resetSeconds.toString()); + + if (count > limit) { + res.setHeader('Retry-After', resetSeconds.toString()); + res.status(429).send('Too many requests, please try again later.'); + return; + } + } catch (err) { + logger.warn( + `Redis rate limiter failed, failing open: ${ + err instanceof Error ? err.message : String(err) + }`, + ); } next(); diff --git a/app/backend/src/main.ts b/app/backend/src/main.ts index 9fa25572..3e5ea7c5 100644 --- a/app/backend/src/main.ts +++ b/app/backend/src/main.ts @@ -18,6 +18,7 @@ import { createHelmetMiddleware, createRateLimiter, } from './common/security/security.module'; +import { RedisService } from '@liaoliaots/nestjs-redis'; async function bootstrap() { // Load environment variables @@ -49,7 +50,7 @@ async function bootstrap() { app.use(createHelmetMiddleware(configService)); app.use(createCorsOriginValidator(configService)); app.enableCors(buildCorsOptions(configService)); - app.use(createRateLimiter(configService)); + app.use(createRateLimiter(configService, app.get(RedisService))); app.use(compression()); // Global prefix diff --git a/app/backend/test/security.e2e-spec.ts b/app/backend/test/security.e2e-spec.ts index c76c8e12..129bb504 100644 --- a/app/backend/test/security.e2e-spec.ts +++ b/app/backend/test/security.e2e-spec.ts @@ -1,4 +1,4 @@ -import { INestApplication, VersioningType } from '@nestjs/common'; +import { Logger, INestApplication, VersioningType } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; @@ -10,6 +10,8 @@ import { createHelmetMiddleware, createRateLimiter, } from '../src/common/security/security.module'; +import { RedisService } from '@liaoliaots/nestjs-redis'; +import RedisMock from 'ioredis-mock'; type TestAppOptions = { enableDocs: boolean; @@ -41,7 +43,7 @@ const createTestApp = async ({ enableDocs }: TestAppOptions) => { app.use(createHelmetMiddleware(configService)); app.use(createCorsOriginValidator(configService)); app.enableCors(buildCorsOptions(configService)); - app.use(createRateLimiter(configService)); + app.use(createRateLimiter(configService, app.get(RedisService))); if (enableDocs) { const swaggerConfig = new DocumentBuilder() @@ -173,6 +175,7 @@ describe('Security (e2e)', () => { let now = initialNow; let nowSpy: jest.SpyInstance; let rateLimitApp: INestApplication; + let mockRedisClient: any; beforeEach(async () => { process.env.API_RATE_LIMIT = '2'; @@ -183,6 +186,10 @@ describe('Security (e2e)', () => { now = initialNow; nowSpy = jest.spyOn(Date, 'now').mockImplementation(() => now); rateLimitApp = await createTestApp({ enableDocs: true }); + + mockRedisClient = new RedisMock(); + const redisService = rateLimitApp.get(RedisService); + jest.spyOn(redisService, 'getOrThrow').mockReturnValue(mockRedisClient); }); afterEach(async () => { @@ -193,6 +200,8 @@ describe('Security (e2e)', () => { process.env.THROTTLE_TTL = '60000'; process.env.CORS_ORIGINS = 'http://localhost:3000'; process.env.CORS_ALLOW_CREDENTIALS = 'false'; + delete process.env.RATE_LIMIT_LIMIT; + delete process.env.RATE_LIMIT_WINDOW_MS; }); it('should rate limit, include retry headers, and reset after the window passes', async () => { @@ -231,6 +240,59 @@ describe('Security (e2e)', () => { expect(response.status).toBe(200); } }); + + it('should rate limit 100 hits in 1 s => 80+ return 429, and include correct headers', async () => { + // Create a specific application instance configured for 20 req/s + process.env.RATE_LIMIT_LIMIT = '20'; + process.env.RATE_LIMIT_WINDOW_MS = '1000'; + + const appInstance = await createTestApp({ enableDocs: false }); + const redisService = appInstance.get(RedisService); + const testMockRedis = new RedisMock(); + jest.spyOn(redisService, 'getOrThrow').mockReturnValue(testMockRedis as any); + + const server = appInstance.getHttpServer(); + const results: any[] = []; + + for (let i = 0; i < 100; i += 1) { + results.push(request(server).get('/api/v1/')); + } + + const responses = await Promise.all(results); + const count429 = responses.filter(r => r.status === 429).length; + + expect(count429).toBeGreaterThanOrEqual(80); + + const rateLimitedResponse = responses.find(r => r.status === 429); + expect(rateLimitedResponse).toBeDefined(); + expect(rateLimitedResponse.headers['ratelimit-limit']).toBe('20'); + expect(rateLimitedResponse.headers['ratelimit-remaining']).toBeDefined(); + expect(rateLimitedResponse.headers['ratelimit-reset']).toBeDefined(); + expect(rateLimitedResponse.headers['retry-after']).toBeDefined(); + + await appInstance.close(); + }); + + it('should fail open with a WARN log, not 500, when Redis is down', async () => { + const appInstance = await createTestApp({ enableDocs: false }); + const redisService = appInstance.get(RedisService); + + jest.spyOn(redisService, 'getOrThrow').mockImplementation(() => { + throw new Error('Redis connection down'); + }); + + const warnSpy = jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => {}); + + const server = appInstance.getHttpServer(); + const response = await request(server).get('/api/v1/'); + + expect(response.status).not.toBe(500); + expect(response.status).not.toBe(429); + expect(warnSpy).toHaveBeenCalled(); + + warnSpy.mockRestore(); + await appInstance.close(); + }); }); describe('Docs Endpoint', () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fdd7d856..e393c248 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -136,6 +136,9 @@ importers: class-validator: specifier: ^0.14.3 version: 0.14.4 + compression: + specifier: 1.7.5 + version: 1.7.5 dotenv: specifier: ^17.2.3 version: 17.4.2 @@ -185,6 +188,9 @@ importers: '@nestjs/testing': specifier: ^11.0.1 version: 11.1.17(@nestjs/common@11.1.27(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.17(@nestjs/common@11.1.27(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@11.1.17)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@11.1.17(@nestjs/common@11.1.27(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.17)) + '@types/compression': + specifier: 1.7.5 + version: 1.7.5 '@types/express': specifier: ^5.0.0 version: 5.0.6 @@ -3242,6 +3248,9 @@ packages: '@types/body-parser@1.19.6': resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==} + '@types/compression@1.7.5': + resolution: {integrity: sha512-AAQvK5pxMpaT+nDvhHrsBhLSYG5yQdtkaJE1WYieSNY2mVFKAgmU4ks65rkZD5oqnGCFLyQpUr1CqI4DmUMyDg==} + '@types/connect@3.4.38': resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} @@ -4370,8 +4379,8 @@ packages: resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} engines: {node: '>= 0.6'} - compression@1.8.1: - resolution: {integrity: sha512-9mAqGPHLakhCLeNyxPkK4xVo746zQ/czLH1Ky+vkitMnWfWZps8r0qXuwhwizagCRttsL4lfG4pIOvaWLpAP0w==} + compression@1.7.5: + resolution: {integrity: sha512-bQJ0YRck5ak3LgtnpKkiabX5pNF7tMUh1BSy2ZBOTh0Dim0BUu6aPPwByIns6/A5Prh8PufSPerMDUklpzes2Q==} engines: {node: '>= 0.8.0'} concat-map@0.0.1: @@ -6973,8 +6982,8 @@ packages: resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} engines: {node: '>= 0.8'} - on-headers@1.1.0: - resolution: {integrity: sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==} + on-headers@1.0.2: + resolution: {integrity: sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==} engines: {node: '>= 0.8'} once@1.4.0: @@ -9792,7 +9801,7 @@ snapshots: bplist-parser: 0.3.2 chalk: 4.1.2 ci-info: 3.9.0 - compression: 1.8.1 + compression: 1.7.5 connect: 3.7.0 debug: 4.4.3(supports-color@10.2.2) env-editor: 0.4.2 @@ -12192,6 +12201,10 @@ snapshots: '@types/connect': 3.4.38 '@types/node': 25.9.1 + '@types/compression@1.7.5': + dependencies: + '@types/express': 5.0.6 + '@types/connect@3.4.38': dependencies: '@types/node': 25.9.1 @@ -13661,13 +13674,13 @@ snapshots: dependencies: mime-db: 1.54.0 - compression@1.8.1: + compression@1.7.5: dependencies: bytes: 3.1.2 compressible: 2.0.18 debug: 2.6.9 negotiator: 0.6.4 - on-headers: 1.1.0 + on-headers: 1.0.2 safe-buffer: 5.2.1 vary: 1.1.2 transitivePeerDependencies: @@ -14157,7 +14170,7 @@ snapshots: tinyglobby: 0.2.15 unrs-resolver: 1.11.1 optionalDependencies: - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.57.1(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1)) + eslint-plugin-import: 2.32.0(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1)) transitivePeerDependencies: - supports-color @@ -17178,7 +17191,7 @@ snapshots: dependencies: ee-first: 1.1.1 - on-headers@1.1.0: {} + on-headers@1.0.2: {} once@1.4.0: dependencies: From e27ce568ebbdc6145df3b94feb44923eed668168 Mon Sep 17 00:00:00 2001 From: kosiso Date: Thu, 16 Jul 2026 00:25:53 +0100 Subject: [PATCH 2/5] test: add property-based test for AES envelope round-trip --- app/backend/package.json | 1 + app/backend/test/upload-roundtrip.spec.ts | 75 +++++++++++++++++++++++ pnpm-lock.yaml | 16 +++++ 3 files changed, 92 insertions(+) create mode 100644 app/backend/test/upload-roundtrip.spec.ts diff --git a/app/backend/package.json b/app/backend/package.json index c6a274fb..dbfa8225 100644 --- a/app/backend/package.json +++ b/app/backend/package.json @@ -81,6 +81,7 @@ "eslint": "^9.18.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-prettier": "^5.5.5", + "fast-check": "^4.9.0", "globals": "^16.0.0", "ioredis-mock": "^8.13.1", "jest": "^30.4.2", diff --git a/app/backend/test/upload-roundtrip.spec.ts b/app/backend/test/upload-roundtrip.spec.ts new file mode 100644 index 00000000..874936b6 --- /dev/null +++ b/app/backend/test/upload-roundtrip.spec.ts @@ -0,0 +1,75 @@ +import { ConfigService } from '@nestjs/config'; +import { EncryptionService } from '../src/common/encryption/encryption.service'; +import * as fc from 'fast-check'; +import * as crypto from 'crypto'; + +describe('AES Envelope Round-Trip (Property-Based Test)', () => { + let encryptionService: EncryptionService; + + beforeAll(() => { + const configService = new ConfigService({ + ENCRYPTION_MASTER_KEY: 'test-master-key-value-suitable-for-testing-12345', + }); + encryptionService = new EncryptionService(configService); + }); + + it('should preserve checksum equality and correctly decrypt buffers across all evidence sizes (1 KB to 100 MB)', async () => { + // We generate a float/double between 0 and 1 using fast-check. + // We map this value to a piecewise log-uniform distribution to bias the sizes towards smaller values (e.g. 90% < 1 MB) + // while still ensuring that large values up to 100 MB are covered. + // This allows us to run 1000 iterations in CI in just a few seconds. + await fc.assert( + fc.asyncProperty( + fc.double({ min: 0, max: 1, noNaN: true, noInfinity: true }), + async (d) => { + let size: number; + if (d < 0.1) { + // 10% of tests are large (1 MB to 100 MB) + const logMin = Math.log(1024 * 1024); + const logMax = Math.log(100 * 1024 * 1024); + const logVal = (d / 0.1) * (logMax - logMin) + logMin; + size = Math.floor(Math.exp(logVal)); + } else { + // 90% of tests are small to medium (1 KB to 1 MB) + const logMin = Math.log(1024); + const logMax = Math.log(1024 * 1024); + const logVal = ((d - 0.1) / 0.9) * (logMax - logMin) + logMin; + size = Math.floor(Math.exp(logVal)); + } + + // Construct original buffer of generated size. + // To make this extremely fast and avoid entropy bottlenecks, + // we generate a small random chunk (up to 4KB) and copy it repeatedly. + const patternSize = Math.min(size, 4096); + const pattern = crypto.randomBytes(patternSize); + const originalBuffer = Buffer.alloc(size); + + let offset = 0; + while (offset < size) { + const bytesToWrite = Math.min(patternSize, size - offset); + pattern.copy(originalBuffer, offset, 0, bytesToWrite); + offset += bytesToWrite; + } + + const originalChecksum = crypto + .createHash('sha256') + .update(originalBuffer) + .digest('hex'); + + // Encrypt and decrypt buffer (AES envelope round-trip) + const encrypted = encryptionService.encryptBuffer(originalBuffer); + const decrypted = encryptionService.decryptBuffer(encrypted); + + const decryptedChecksum = crypto + .createHash('sha256') + .update(decrypted) + .digest('hex'); + + expect(decryptedChecksum).toBe(originalChecksum); + expect(decrypted.equals(originalBuffer)).toBe(true); + } + ), + { numRuns: 1000 } + ); + }, 35000); // 35 second timeout to be safe, though it should complete in under 5 seconds +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e393c248..15e5307d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -221,6 +221,9 @@ importers: eslint-plugin-prettier: specifier: ^5.5.5 version: 5.5.5(@types/eslint@9.6.1)(eslint-config-prettier@10.1.8(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1))(prettier@3.8.1) + fast-check: + specifier: ^4.9.0 + version: 4.9.0 globals: specifier: ^16.0.0 version: 16.5.0 @@ -5180,6 +5183,10 @@ packages: resolution: {integrity: sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A==} engines: {node: '>=8.0.0'} + fast-check@4.9.0: + resolution: {integrity: sha512-7ms6T7SybUev/PQITciI0yLM2pOSFy5zpG8Ty7tQofcVaQUvrMXp6CBwqF6fThLCLOrfBtuHAtwq6Yu4XPCllg==} + engines: {node: '>=12.17.0'} + fast-copy@4.0.2: resolution: {integrity: sha512-ybA6PDXIXOXivLJK/z9e+Otk7ve13I4ckBvGO5I2RRmBU1gMHLVDJYEuJYhGwez7YNlYji2M2DvVU+a9mSFDlw==} @@ -7386,6 +7393,9 @@ packages: pure-rand@7.0.1: resolution: {integrity: sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==} + pure-rand@8.4.2: + resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==} + qrcode-terminal@0.11.0: resolution: {integrity: sha512-Uu7ii+FQy4Qf82G4xu7ShHhjhGahEpCWc3x8UavY3CTcWV+ufmmCtwkr7ZKsX42jdL0kr1B5FKUeqJvAn51jzQ==} hasBin: true @@ -14664,6 +14674,10 @@ snapshots: dependencies: pure-rand: 6.1.0 + fast-check@4.9.0: + dependencies: + pure-rand: 8.4.2 + fast-copy@4.0.2: {} fast-deep-equal@3.1.3: {} @@ -17639,6 +17653,8 @@ snapshots: pure-rand@7.0.1: {} + pure-rand@8.4.2: {} + qrcode-terminal@0.11.0: {} qs@6.15.0: From ef1122c0005746d90663877dbb8113c049c95c4e Mon Sep 17 00:00:00 2001 From: kosiso Date: Thu, 16 Jul 2026 05:45:20 +0100 Subject: [PATCH 3/5] test: mock prisma client offline and fix rate limiter e2e tests --- app/backend/test/jest-e2e.json | 4 +- app/backend/test/mocks/prisma-client.mock.ts | 139 +++++++++++++++++++ app/backend/test/security.e2e-spec.ts | 48 ++++++- 3 files changed, 183 insertions(+), 8 deletions(-) create mode 100644 app/backend/test/mocks/prisma-client.mock.ts diff --git a/app/backend/test/jest-e2e.json b/app/backend/test/jest-e2e.json index 988edaeb..47488c21 100644 --- a/app/backend/test/jest-e2e.json +++ b/app/backend/test/jest-e2e.json @@ -8,7 +8,9 @@ }, "moduleNameMapper": { "^src/(.*)$": "/src/$1", + "^cache/(.*)$": "/cache/$1", "^@stellar/stellar-sdk$": "/test/mocks/stellar-sdk.mock.ts", - "^openai$": "/test/mocks/openai.mock.ts" + "^openai$": "/test/mocks/openai.mock.ts", + "^@prisma/client$": "/test/mocks/prisma-client.mock.ts" } } diff --git a/app/backend/test/mocks/prisma-client.mock.ts b/app/backend/test/mocks/prisma-client.mock.ts new file mode 100644 index 00000000..3bb1eb69 --- /dev/null +++ b/app/backend/test/mocks/prisma-client.mock.ts @@ -0,0 +1,139 @@ +const createModelMock = () => ({ + findUnique: jest.fn(), + findFirst: jest.fn(), + findMany: jest.fn(), + create: jest.fn(), + update: jest.fn(), + delete: jest.fn(), + count: jest.fn(), + upsert: jest.fn(), + deleteMany: jest.fn(), + updateMany: jest.fn(), +}); + +export class PrismaClient { + constructor() { + return new Proxy(this, { + get(target, prop) { + if (prop === '$connect') return jest.fn().mockResolvedValue(undefined); + if (prop === '$disconnect') return jest.fn().mockResolvedValue(undefined); + if (prop === '$on') return jest.fn(); + if (prop === '$transaction') { + return jest.fn(async (cb) => (typeof cb === 'function' ? cb(this) : cb)); + } + if (typeof prop === 'symbol' || prop === 'constructor' || prop === 'then') { + return (target as any)[prop]; + } + return createModelMock(); + }, + }); + } +} + +export const Prisma = { + defineExtension: jest.fn(x => x), + sql: jest.fn(), +}; + +// Enums +export enum CampaignStatus { + draft = 'draft', + active = 'active', + paused = 'paused', + completed = 'completed', + archived = 'archived', +} + +export enum ClaimStatus { + requested = 'requested', + verified = 'verified', + approved = 'approved', + disbursed = 'disbursed', + archived = 'archived', + cancelled = 'cancelled', +} + +export enum VerificationChannel { + email = 'email', + phone = 'phone', +} + +export enum VerificationSessionStatus { + pending = 'pending', + completed = 'completed', + expired = 'expired', + failed = 'failed', +} + +export enum SessionType { + otp_verification = 'otp_verification', + claim_verification = 'claim_verification', + multi_step_verification = 'multi_step_verification', +} + +export enum SessionStepStatus { + pending = 'pending', + in_progress = 'in_progress', + completed = 'completed', + failed = 'failed', + skipped = 'skipped', +} + +export enum VerificationStatus { + pending = 'pending', + pending_review = 'pending_review', + approved = 'approved', + rejected = 'rejected', + needs_resubmission = 'needs_resubmission', +} + +export enum PurgeStrategy { + soft_delete = 'soft_delete', + hard_delete = 'hard_delete', + anonymize = 'anonymize', +} + +export enum InviteStatus { + pending = 'pending', + accepted = 'accepted', + revoked = 'revoked', + expired = 'expired', +} + +export enum AppRole { + admin = 'admin', + operator = 'operator', + client = 'client', + ngo = 'ngo', +} + +export enum EvidenceStatus { + pending = 'pending', + uploading = 'uploading', + completed = 'completed', + failed = 'failed', +} + +export enum UploadSessionStatus { + active = 'active', + completed = 'completed', + expired = 'expired', + aborted = 'aborted', +} + +export enum NotificationOutboxStatus { + pending = 'pending', + enqueued = 'enqueued', + sent = 'sent', + failed = 'failed', +} + +export enum RegistryEntityType { + individual = 'individual', + household = 'household', +} + +export enum EntityLinkSourceType { + manual = 'manual', + automatic = 'automatic', +} diff --git a/app/backend/test/security.e2e-spec.ts b/app/backend/test/security.e2e-spec.ts index 129bb504..a9681cf3 100644 --- a/app/backend/test/security.e2e-spec.ts +++ b/app/backend/test/security.e2e-spec.ts @@ -1,9 +1,8 @@ -import { Logger, INestApplication, VersioningType } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; +import { Logger, INestApplication, VersioningType, Controller, Get, HttpCode } from '@nestjs/common'; +import { ConfigModule, ConfigService } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import request from 'supertest'; -import { AppModule } from '../src/app.module'; import { buildCorsOptions, createCorsOriginValidator, @@ -13,6 +12,12 @@ import { import { RedisService } from '@liaoliaots/nestjs-redis'; import RedisMock from 'ioredis-mock'; +jest.mock('ioredis', () => { + const RedisMock = require('ioredis-mock'); + RedisMock.Redis = RedisMock; + return RedisMock; +}); + type TestAppOptions = { enableDocs: boolean; }; @@ -25,12 +30,42 @@ const setEnvValue = (key: string, value: string | undefined) => { } }; +@Controller() +class TestController { + @Get('health') + @HttpCode(200) + getHealth() { + return { status: 'OK' }; + } + + @Get() + @HttpCode(200) + getRoot() { + return { message: 'OK' }; + } +} + const createTestApp = async ({ enableDocs }: TestAppOptions) => { + const mockRedisInstance = new RedisMock(); const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], + imports: [ + ConfigModule.forRoot({ + isGlobal: true, + }), + ], + controllers: [TestController], + providers: [ + { + provide: RedisService, + useValue: { + getOrThrow: () => mockRedisInstance, + }, + }, + ], }).compile(); const app = moduleFixture.createNestApplication(); + app.getHttpAdapter().getInstance().disable('x-powered-by'); app.setGlobalPrefix('api'); app.enableVersioning({ @@ -252,13 +287,12 @@ describe('Security (e2e)', () => { jest.spyOn(redisService, 'getOrThrow').mockReturnValue(testMockRedis as any); const server = appInstance.getHttpServer(); - const results: any[] = []; + const responses: any[] = []; for (let i = 0; i < 100; i += 1) { - results.push(request(server).get('/api/v1/')); + responses.push(await request(server).get('/api/v1/')); } - const responses = await Promise.all(results); const count429 = responses.filter(r => r.status === 429).length; expect(count429).toBeGreaterThanOrEqual(80); From 99a34de8f69d138733abcaef82041a927405696b Mon Sep 17 00:00:00 2001 From: kosiso Date: Sun, 19 Jul 2026 05:41:05 +0100 Subject: [PATCH 4/5] fix(package.json): remove trailing comma in scripts and validate JSON in CI --- .github/workflows/backend-ci.yml | 4 ++++ package.json | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml index fca0f0d8..83bd8ea2 100644 --- a/.github/workflows/backend-ci.yml +++ b/.github/workflows/backend-ci.yml @@ -17,6 +17,10 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Validate package.json files + run: | + node -e "const fs=require('fs'); require('child_process').execSync('git ls-files \"*package.json\"',{encoding:'utf8'}).trim().split('\\n').filter(Boolean).forEach(p=>{JSON.parse(fs.readFileSync(p,'utf8')); console.log('OK',p)})" + - name: Install pnpm uses: pnpm/action-setup@v2 with: diff --git a/package.json b/package.json index df23dfa0..b5a1524d 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,8 @@ "scripts": { "build": "pnpm -r build", "lint": "pnpm -r lint", - "test": "pnpm -r test", - }, + "test": "pnpm -r test" + }, "pnpm": { "overrides": { "jest": "^30.4.2", From 6d226289dd877bb335acd7e65a1b98b4f9e47e1a Mon Sep 17 00:00:00 2001 From: kosiso Date: Sun, 19 Jul 2026 06:07:52 +0100 Subject: [PATCH 5/5] fix(lint): remove unused async declarations to satisfy require-await rule --- app/backend/test/mocks/prisma-client.mock.ts | 2 +- app/backend/test/upload-roundtrip.spec.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/backend/test/mocks/prisma-client.mock.ts b/app/backend/test/mocks/prisma-client.mock.ts index 3bb1eb69..0677700c 100644 --- a/app/backend/test/mocks/prisma-client.mock.ts +++ b/app/backend/test/mocks/prisma-client.mock.ts @@ -19,7 +19,7 @@ export class PrismaClient { if (prop === '$disconnect') return jest.fn().mockResolvedValue(undefined); if (prop === '$on') return jest.fn(); if (prop === '$transaction') { - return jest.fn(async (cb) => (typeof cb === 'function' ? cb(this) : cb)); + return jest.fn((cb) => Promise.resolve(typeof cb === 'function' ? cb(this) : cb)); } if (typeof prop === 'symbol' || prop === 'constructor' || prop === 'then') { return (target as any)[prop]; diff --git a/app/backend/test/upload-roundtrip.spec.ts b/app/backend/test/upload-roundtrip.spec.ts index 874936b6..379bc932 100644 --- a/app/backend/test/upload-roundtrip.spec.ts +++ b/app/backend/test/upload-roundtrip.spec.ts @@ -13,15 +13,15 @@ describe('AES Envelope Round-Trip (Property-Based Test)', () => { encryptionService = new EncryptionService(configService); }); - it('should preserve checksum equality and correctly decrypt buffers across all evidence sizes (1 KB to 100 MB)', async () => { + it('should preserve checksum equality and correctly decrypt buffers across all evidence sizes (1 KB to 100 MB)', () => { // We generate a float/double between 0 and 1 using fast-check. // We map this value to a piecewise log-uniform distribution to bias the sizes towards smaller values (e.g. 90% < 1 MB) // while still ensuring that large values up to 100 MB are covered. // This allows us to run 1000 iterations in CI in just a few seconds. - await fc.assert( - fc.asyncProperty( + fc.assert( + fc.property( fc.double({ min: 0, max: 1, noNaN: true, noInfinity: true }), - async (d) => { + (d) => { let size: number; if (d < 0.1) { // 10% of tests are large (1 MB to 100 MB)