From 033940584792f9d8065f2970463121d1d526593d Mon Sep 17 00:00:00 2001 From: John Chantzigoulas Date: Wed, 9 Sep 2026 11:09:40 +0300 Subject: [PATCH 1/8] feat(database): stream live document updates over admin sockets MongoDB change streams require a replica set, so local Compose now initializes a single-node rs0. Schemas can opt in via realtime.enabled. --- docker/.env | 2 +- docker/docker-compose.standalone.yml | 25 +- docker/docker-compose.yml | 30 +- docker/mongo/init-replica.sh | 20 ++ docker/mongo/keyfile | 1 + libraries/grpc-sdk/src/interfaces/Model.ts | 7 +- libraries/grpc-sdk/src/modules/admin/index.ts | 7 +- modules/database/README.mdx | 18 + modules/database/src/Database.ts | 7 + modules/database/src/admin/index.ts | 3 + modules/database/src/admin/schema.admin.ts | 2 + modules/database/src/config/index.ts | 7 + .../database/src/interfaces/ConduitOptions.ts | 3 + modules/database/src/metrics/index.ts | 15 + .../realtime/MongoChangeStreamCoordinator.ts | 329 ++++++++++++++++++ .../database/src/realtime/RealtimeService.ts | 160 +++++++++ .../src/realtime/__tests__/authorize.test.ts | 82 +++++ .../change-stream.integration.test.ts | 47 +++ .../realtime/__tests__/coordinator.test.ts | 166 +++++++++ .../src/realtime/__tests__/normalize.test.ts | 38 ++ .../src/realtime/__tests__/rooms.test.ts | 30 ++ .../src/realtime/__tests__/status.test.ts | 53 +++ .../realtime/__tests__/subscriptions.test.ts | 55 +++ .../src/realtime/__tests__/topology.test.ts | 16 + modules/database/src/realtime/authorize.ts | 147 ++++++++ modules/database/src/realtime/index.ts | 11 + modules/database/src/realtime/normalize.ts | 70 ++++ modules/database/src/realtime/rooms.ts | 25 ++ modules/database/src/realtime/sockets.ts | 166 +++++++++ modules/database/src/realtime/status.ts | 74 ++++ .../database/src/realtime/subscriptions.ts | 92 +++++ modules/database/src/realtime/topology.ts | 49 +++ modules/database/src/realtime/types.ts | 49 +++ modules/database/src/routes/index.ts | 4 +- modules/database/src/utils/SchemaConverter.ts | 6 + packages/core/package.json | 1 + packages/core/src/admin/AdminModule.ts | 36 ++ .../src/admin/middleware/Admin.middleware.ts | 10 +- .../src/admin/middleware/Auth.middleware.ts | 6 + .../core/src/admin/realtime/handshake.test.ts | 12 + packages/core/src/admin/realtime/handshake.ts | 14 + .../core/src/admin/realtime/namespace.test.ts | 11 + packages/core/src/admin/realtime/namespace.ts | 7 + .../core/src/admin/realtime/ticket.test.ts | 26 ++ packages/core/src/admin/realtime/ticket.ts | 26 ++ .../src/admin/routes/RealtimeTicket.route.ts | 51 +++ packages/core/src/admin/routes/index.ts | 1 + packages/core/src/core.proto | 8 + packages/core/tsconfig.json | 2 +- packages/core/tsconfig.test.json | 17 + 50 files changed, 2033 insertions(+), 11 deletions(-) create mode 100644 docker/mongo/init-replica.sh create mode 100644 docker/mongo/keyfile create mode 100644 modules/database/src/realtime/MongoChangeStreamCoordinator.ts create mode 100644 modules/database/src/realtime/RealtimeService.ts create mode 100644 modules/database/src/realtime/__tests__/authorize.test.ts create mode 100644 modules/database/src/realtime/__tests__/change-stream.integration.test.ts create mode 100644 modules/database/src/realtime/__tests__/coordinator.test.ts create mode 100644 modules/database/src/realtime/__tests__/normalize.test.ts create mode 100644 modules/database/src/realtime/__tests__/rooms.test.ts create mode 100644 modules/database/src/realtime/__tests__/status.test.ts create mode 100644 modules/database/src/realtime/__tests__/subscriptions.test.ts create mode 100644 modules/database/src/realtime/__tests__/topology.test.ts create mode 100644 modules/database/src/realtime/authorize.ts create mode 100644 modules/database/src/realtime/index.ts create mode 100644 modules/database/src/realtime/normalize.ts create mode 100644 modules/database/src/realtime/rooms.ts create mode 100644 modules/database/src/realtime/sockets.ts create mode 100644 modules/database/src/realtime/status.ts create mode 100644 modules/database/src/realtime/subscriptions.ts create mode 100644 modules/database/src/realtime/topology.ts create mode 100644 modules/database/src/realtime/types.ts create mode 100644 packages/core/src/admin/realtime/handshake.test.ts create mode 100644 packages/core/src/admin/realtime/handshake.ts create mode 100644 packages/core/src/admin/realtime/namespace.test.ts create mode 100644 packages/core/src/admin/realtime/namespace.ts create mode 100644 packages/core/src/admin/realtime/ticket.test.ts create mode 100644 packages/core/src/admin/realtime/ticket.ts create mode 100644 packages/core/src/admin/routes/RealtimeTicket.route.ts create mode 100644 packages/core/tsconfig.test.json diff --git a/docker/.env b/docker/.env index 66dad89bd..b71166ea3 100644 --- a/docker/.env +++ b/docker/.env @@ -26,7 +26,7 @@ DB_TYPE="mongodb" DB_USER="conduit" DB_PASS="pass" DB_PORT="27017" -DB_CONN_URI="mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin" # profile: mongodb +DB_CONN_URI="mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin&replicaSet=rs0" # profile: mongodb #DB_CONN_URI="postgres://conduit:pass@conduit-postgres:5432/conduit" # profile: postgres # Security diff --git a/docker/docker-compose.standalone.yml b/docker/docker-compose.standalone.yml index 8cd0b0e9b..e1e5225be 100644 --- a/docker/docker-compose.standalone.yml +++ b/docker/docker-compose.standalone.yml @@ -38,7 +38,7 @@ services: ADMIN_SOCKET_PORT: '${ADMIN_SOCKET_PORT:-3031}' __DEFAULT_HOST_URL: '${ADMIN_DEFAULT_HOST_URL:-http://localhost:3030}' GRPC_KEY: '${GRPC_KEY}' - DB_CONN_URI: '${DB_CONN_URI:-mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin}' + DB_CONN_URI: '${DB_CONN_URI:-mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin&replicaSet=rs0}' networks: default: aliases: @@ -74,12 +74,35 @@ services: MONGO_INITDB_DATABASE: 'conduit' MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' + entrypoint: + - bash + - -c + - | + cp /mongo-keyfile /tmp/keyfile + chmod 400 /tmp/keyfile + chown mongodb:mongodb /tmp/keyfile + exec docker-entrypoint.sh mongod --replSet rs0 --bind_ip_all --keyFile /tmp/keyfile networks: default: aliases: - conduit-mongo volumes: - mongo:/data/db + - ./mongo/keyfile:/mongo-keyfile:ro + + mongo-init-replica: + container_name: 'conduit-mongo-init' + image: 'docker.io/library/mongo:4.4.15' + restart: on-failure + depends_on: + - mongodb + environment: + MONGO_HOST: 'conduit-mongo' + MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' + MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' + volumes: + - ./mongo/init-replica.sh:/init-replica.sh:ro + command: ['bash', '/init-replica.sh'] # Persistent Volumes volumes: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 83b3e9b55..751ea8bc6 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -85,7 +85,7 @@ services: LOKI_URL: 'http://conduit-loki:3100' GRPC_KEY: '${GRPC_KEY}' DB_TYPE: '${DB_TYPE:-mongodb}' - DB_CONN_URI: '${DB_CONN_URI:-mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin}' + DB_CONN_URI: '${DB_CONN_URI:-mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin&replicaSet=rs0}' networks: default: aliases: @@ -271,12 +271,40 @@ services: MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' profiles: ['mongodb'] + entrypoint: + - bash + - -c + - | + cp /mongo-keyfile /tmp/keyfile + chmod 400 /tmp/keyfile + chown mongodb:mongodb /tmp/keyfile + exec docker-entrypoint.sh mongod --replSet rs0 --bind_ip_all --keyFile /tmp/keyfile networks: default: aliases: - conduit-mongo volumes: - mongo:/data/db + - ./mongo/keyfile:/mongo-keyfile:ro + + mongo-init-replica: + container_name: 'conduit-mongo-init' + image: 'docker.io/library/mongo:4.4.15' + restart: on-failure + profiles: ['mongodb'] + depends_on: + - mongodb + environment: + MONGO_HOST: 'conduit-mongo' + MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' + MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' + volumes: + - ./mongo/init-replica.sh:/init-replica.sh:ro + command: ['bash', '/init-replica.sh'] + networks: + default: + aliases: + - conduit-mongo-init postgres: container_name: 'conduit-postgres' diff --git a/docker/mongo/init-replica.sh b/docker/mongo/init-replica.sh new file mode 100644 index 000000000..e4bb536d2 --- /dev/null +++ b/docker/mongo/init-replica.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail +HOST="${MONGO_HOST:-conduit-mongo}" +USER="${MONGO_INITDB_ROOT_USERNAME:-conduit}" +PASS="${MONGO_INITDB_ROOT_PASSWORD:-pass}" + +until mongo --host "$HOST" -u "$USER" -p "$PASS" --authenticationDatabase admin --quiet --eval 'db.adminCommand({ ping: 1 })' >/dev/null 2>&1; do + sleep 2 +done + +mongo --host "$HOST" -u "$USER" -p "$PASS" --authenticationDatabase admin --quiet --eval ' + try { + rs.status(); + } catch (err) { + rs.initiate({ + _id: "rs0", + members: [{ _id: 0, host: "'"$HOST"':27017" }] + }); + } +' diff --git a/docker/mongo/keyfile b/docker/mongo/keyfile new file mode 100644 index 000000000..8d7ffaf78 --- /dev/null +++ b/docker/mongo/keyfile @@ -0,0 +1 @@ +conduit-dev-mongo-replica-keyfile-do-not-use-in-production-replace-me-before-any-real-deployment-0123456789abcdefghijklmnopqrstuvwxyz diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 6b7319a9b..b13fd687b 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -71,9 +71,7 @@ export interface ConduitArrayValidation { } export type ConduitValidationRules = - | ConduitStringValidation - | ConduitNumberValidation - | ConduitArrayValidation; + ConduitStringValidation | ConduitNumberValidation | ConduitArrayValidation; type BaseConduitModelField = { type?: TYPE | TYPE[] | ConduitModel | ArrayConduitModel[]; @@ -190,6 +188,9 @@ export interface ConduitSchemaOptions { authorization?: { enabled: boolean; }; + realtime?: { + enabled: boolean; + }; /** Mongoose read preference for this schema (ignored by SQL); per-query wins. */ readPreference?: string; }; diff --git a/libraries/grpc-sdk/src/modules/admin/index.ts b/libraries/grpc-sdk/src/modules/admin/index.ts index 889ca0ead..57a6ac7bd 100644 --- a/libraries/grpc-sdk/src/modules/admin/index.ts +++ b/libraries/grpc-sdk/src/modules/admin/index.ts @@ -3,7 +3,8 @@ import { AdminDefinition, RegisterAdminRouteRequest, RegisterAdminRouteRequest_PathDefinition, -} from '../../protoUtils/index.js'; + SocketPushRequest, +} from '../../protoUtils/core.js'; import { ConduitRouteActions } from '../../interfaces/index.js'; export class Admin extends ConduitModule { @@ -28,6 +29,10 @@ export class Admin extends ConduitModule { return this.client!.registerAdminRoute(request); } + socketPush(data: SocketPushRequest) { + return this.client!.socketPush(data); + } + patchRouteMiddlewares( path: string, action: ConduitRouteActions, diff --git a/modules/database/README.mdx b/modules/database/README.mdx index c340b86bd..61bc195b0 100644 --- a/modules/database/README.mdx +++ b/modules/database/README.mdx @@ -47,6 +47,23 @@ since the latter need to go through parsers that are otherwise unnecessary for M When using MongoDB with a replica set (e.g., MongoDB Atlas), the database module supports configuring read preference, write concern, and read concern through the admin panel at `PATCH /config/database`. +Live document updates also require a replica set or sharded cluster. Local Compose files initialize a single-node replica set so change streams can be exercised. + +### Live updates + +Enable `realtime.enabled` in database module config, then opt a schema in with `modelOptions.conduit.realtime.enabled`. Clients connect to the `/database/` Socket.IO namespace (path `/realtime`) and emit: + +``` +subscribe({ schema: 'Order', documentId?: string }) +unsubscribe({ schema: 'Order', documentId?: string }) +``` + +Events arrive as `change` with `{ version, operation, schema, documentId, occurredAt, resumeToken }` and contain no document fields. Consumers should refetch through their authorized REST or custom-endpoint path. + +Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; session JWTs and masterkeys must not be sent from browser code. + +Admin sockets must be enabled (`admin.transports.sockets`) and the Admin socket port (`ADMIN_SOCKET_PORT`, default 3031) reachable from the UI. + ### Configuration Options | Setting | Values | Default | Description | @@ -54,6 +71,7 @@ When using MongoDB with a replica set (e.g., MongoDB Atlas), the database module | `readPreference` | `primary`, `primaryPreferred`, `secondary`, `secondaryPreferred`, `nearest` | `primary` | Controls which replica set members receive read queries | | `writeConcern` | `1`, `majority` | `1` | How many members must acknowledge a write | | `readConcern` | `local`, `available`, `majority`, `linearizable`, `snapshot` | `local` | Consistency level for read operations | +| `realtime.enabled` | `true`, `false` | `false` | Enable MongoDB change-stream live updates for opted-in schemas | ### Recommended Production Settings diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index 57286cb1e..e0208f256 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -64,6 +64,7 @@ import { QueueController } from './controllers/queue.controller.js'; import AppConfigSchema, { Config } from './config/index.js'; import { Empty } from './protoTypes/google/protobuf/empty.js'; import { fileURLToPath } from 'node:url'; +import { RealtimeService } from './realtime/index.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -107,6 +108,7 @@ export default class DatabaseModule extends ManagedModule { private customEndpointController?: CustomEndpointController; private _authorizationDefinitionsRegistered = false; private _databaseRouterWatchDispose: (() => void) | null = null; + private realtimeService?: RealtimeService; constructor(dbType: string, dbUri: string, peerManifestRoot?: string) { super('database', peerManifestRoot); @@ -164,6 +166,7 @@ export default class DatabaseModule extends ManagedModule { readConcern: config.readConcern ?? 'local', }); } + void this.realtimeService?.reconcile(); if (!config.viewCleanup.enabled) { try { await QueueController.getInstance().drainViewCleanupQueue(); @@ -1015,13 +1018,16 @@ export default class DatabaseModule extends ManagedModule { this.grpcSdk, this._activeAdapter, ); + this.realtimeService = new RealtimeService(this.grpcSdk, this._activeAdapter); this.adminRouter = new AdminHandlers( this.grpcServer, this.grpcSdk, this._activeAdapter, this.schemaController, this.customEndpointController, + this.realtimeService, ); + void this.realtimeService.reconcile(); this._databaseRouterWatchDispose?.(); this._databaseRouterWatchDispose = this.grpcSdk.watchPeer( 'router', @@ -1031,6 +1037,7 @@ export default class DatabaseModule extends ManagedModule { this.grpcServer, this._activeAdapter, this.grpcSdk, + this.realtimeService, ); this.schemaController?.setRouter(this.userRouter); this.customEndpointController?.setRouter(this.userRouter); diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index a4c1be91f..9742f31eb 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -23,6 +23,7 @@ import { SchemaController } from '../controllers/cms/schema.controller.js'; import { CustomEndpointController } from '../controllers/customEndpoints/customEndpoint.controller.js'; import { CustomEndpoints, DeclaredSchema, PendingSchemas } from '../models/index.js'; import { ConduitOptions, SchemaFieldsRequired } from '../interfaces/index.js'; +import type { RealtimeService } from '../realtime/index.js'; export class AdminHandlers { private readonly schemaAdmin: SchemaAdmin; @@ -36,6 +37,7 @@ export class AdminHandlers { private readonly _activeAdapter: DatabaseAdapter, private readonly schemaController: SchemaController, private readonly customEndpointController: CustomEndpointController, + private readonly realtimeService: RealtimeService, ) { this.schemaAdmin = new SchemaAdmin( this.grpcSdk, @@ -686,6 +688,7 @@ export class AdminHandlers { new ConduitRouteReturnDefinition('getDatabaseType', 'String'), this.schemaAdmin.getDatabaseType.bind(this.schemaAdmin), ); + this.realtimeService.registerAdmin(this.routingManager); this.routingManager.registerRoutes(); } } diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index 9c149d6db..cc8309664 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -250,6 +250,7 @@ export class SchemaAdmin { cms: call.request.params.conduitOptions?.cms, permissions: call.request.params.conduitOptions?.permissions, authorization: call.request.params.conduitOptions?.authorization, + realtime: call.request.params.conduitOptions?.realtime, readPreference: call.request.params.conduitOptions?.readPreference, timestamps: call.request.params.timestamps, }); @@ -343,6 +344,7 @@ export class SchemaAdmin { cms: conduitOptions?.cms, authorization: conduitOptions?.authorization, permissions: conduitOptions?.permissions, + realtime: conduitOptions?.realtime, readPreference: conduitOptions?.readPreference, existingModelOptions: requestedSchema.modelOptions, }); diff --git a/modules/database/src/config/index.ts b/modules/database/src/config/index.ts index 0f5a4fcc8..1ddda80d7 100644 --- a/modules/database/src/config/index.ts +++ b/modules/database/src/config/index.ts @@ -47,6 +47,13 @@ const AppConfigSchema = { default: 0, }, }, + realtime: { + enabled: { + doc: 'Enable MongoDB change-stream live updates for opted-in schemas', + format: 'Boolean', + default: false, + }, + }, }; const config = convict(AppConfigSchema); diff --git a/modules/database/src/interfaces/ConduitOptions.ts b/modules/database/src/interfaces/ConduitOptions.ts index 82ee970fb..f5f1b3b02 100644 --- a/modules/database/src/interfaces/ConduitOptions.ts +++ b/modules/database/src/interfaces/ConduitOptions.ts @@ -30,5 +30,8 @@ export const ConduitOptions = { authorization: { enabled: ConduitBoolean.Optional, }, + realtime: { + enabled: ConduitBoolean.Optional, + }, readPreference: ConduitString.Optional, }; diff --git a/modules/database/src/metrics/index.ts b/modules/database/src/metrics/index.ts index e00ad3a33..3d3ef0256 100644 --- a/modules/database/src/metrics/index.ts +++ b/modules/database/src/metrics/index.ts @@ -31,4 +31,19 @@ export default { help: 'Tracks the total number of custom endpoints', }, }, + realtimeEvents: { + type: MetricType.Counter, + config: { + name: 'database_realtime_events_total', + help: 'Tracks normalized database change-stream events', + labelNames: ['operation'], + }, + }, + realtimeStreamErrors: { + type: MetricType.Counter, + config: { + name: 'database_realtime_stream_errors_total', + help: 'Tracks change-stream errors', + }, + }, }; diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts new file mode 100644 index 000000000..32ff41dc7 --- /dev/null +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -0,0 +1,329 @@ +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { + normalizeChangeEvent, + parseResumeToken, + type RawChangeEvent, +} from './normalize.js'; +import { authorizedDocumentRoom, roomsForPublicChange } from './rooms.js'; +import { + isResumeTokenUnusable, + topologyFromHello, + type TopologyResult, +} from './topology.js'; +import type { + ChangeStreamLike, + DatabaseChangeEvent, + OptedInSchema, + RealtimeStatusCode, +} from './types.js'; +import type { RealtimeSubscriptionTracker } from './subscriptions.js'; +import { canReadDocument, type AuthorizationSdk } from './authorize.js'; + +const LEADER_LOCK = 'realtime:change-stream:leader'; +const RESUME_TOKEN_KEY = 'realtime:resumeToken'; +const LOCK_TTL_MS = 15_000; +const LOCK_RENEW_MS = 5_000; +const RETRY_BASE_MS = 1_000; +const RETRY_MAX_MS = 30_000; + +type LeaderLock = NonNullable< + Awaited['tryAcquireLock']>> +>; + +export type WatchFactory = (options: { resumeAfter?: unknown }) => ChangeStreamLike; + +export type CoordinatorOptions = { + grpcSdk: ConduitGrpcSdk; + watch: WatchFactory; + hello: () => Promise<{ setName?: string; msg?: string } | null>; + getOptedInSchemas: () => OptedInSchema[]; + subscriptions: RealtimeSubscriptionTracker; + enabled: () => boolean; + engine: () => string; + socketsEnabled?: () => Promise; +}; + +export class MongoChangeStreamCoordinator { + private lock: LeaderLock | null = null; + private stream: ChangeStreamLike | null = null; + private renewTimer: NodeJS.Timeout | null = null; + private retryTimer: NodeJS.Timeout | null = null; + private closed = false; + private streamState: RealtimeStatusCode = 'idle'; + private lastEventAt?: string; + private lastError?: string; + private topology: TopologyResult = { supported: false }; + private retryAttempt = 0; + private watching = false; + private ignoreClose = false; + + constructor(private readonly options: CoordinatorOptions) {} + + getState(): RealtimeStatusCode { + return this.streamState; + } + + getLastEventAt(): string | undefined { + return this.lastEventAt; + } + + getLastError(): string | undefined { + return this.lastError; + } + + getTopology(): TopologyResult { + return this.topology; + } + + async reconcile(): Promise { + if (this.closed) return; + const engine = this.options.engine(); + if (engine !== 'MongoDB' || !this.options.enabled()) { + await this.stopStream('idle'); + await this.releaseLeader(); + this.streamState = engine !== 'MongoDB' ? 'unsupported' : 'disabled'; + return; + } + this.topology = topologyFromHello(await this.options.hello().catch(() => null)); + if (!this.topology.supported) { + await this.stopStream('idle'); + await this.releaseLeader(); + this.streamState = 'idle'; + this.lastError = this.topology.message; + return; + } + if (this.options.getOptedInSchemas().length === 0) { + await this.stopStream('idle'); + await this.releaseLeader(); + this.streamState = 'idle'; + return; + } + await this.ensureLeader(); + } + + async shutdown(): Promise { + this.closed = true; + this.clearTimers(); + await this.stopStream('idle'); + await this.releaseLeader(); + } + + private async ensureLeader(): Promise { + if (this.lock) { + if (!this.watching) { + await this.openStream(); + } + return; + } + try { + const acquired = await this.options.grpcSdk.state!.tryAcquireLock( + LEADER_LOCK, + LOCK_TTL_MS, + ); + if (!acquired) { + this.streamState = 'idle'; + return; + } + this.lock = acquired; + this.startRenewal(); + await this.openStream(); + } catch (err) { + this.lastError = err instanceof Error ? err.message : String(err); + this.streamState = 'degraded'; + this.scheduleRetry(); + } + } + + private startRenewal() { + this.clearRenewTimer(); + this.renewTimer = setInterval(() => { + void this.renewLock(); + }, LOCK_RENEW_MS); + } + + private async renewLock() { + if (!this.lock) return; + try { + this.lock = await this.lock.extend(LOCK_TTL_MS); + } catch { + this.lock = null; + await this.stopStream('idle'); + this.scheduleRetry(); + } + } + + private async openStream() { + if (this.watching || this.closed) return; + this.streamState = 'starting'; + this.ignoreClose = false; + const resumeAfter = parseResumeToken( + (await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY)) ?? undefined, + ); + try { + const stream = this.options.watch({ resumeAfter: resumeAfter ?? undefined }); + this.stream = stream; + this.watching = true; + this.streamState = 'live'; + this.retryAttempt = 0; + stream.on('change', (change: unknown) => { + void this.handleChange(change as RawChangeEvent); + }); + stream.on('error', (err: unknown) => { + void this.handleStreamError(err); + }); + stream.on('close', () => { + this.watching = false; + if (!this.closed && !this.ignoreClose && this.lock) { + this.scheduleRetry(); + } + }); + } catch (err) { + this.watching = false; + await this.handleStreamError(err); + } + } + + private async handleChange(change: RawChangeEvent) { + const schema = this.resolveSchema(change.ns?.coll); + if (!schema) return; + const event = normalizeChangeEvent(change, schema.name); + if (!event) return; + this.lastEventAt = event.occurredAt; + this.lastError = undefined; + await this.options.grpcSdk.state!.setKey(RESUME_TOKEN_KEY, event.resumeToken); + this.options.grpcSdk.bus?.publish( + `database:change:${schema.name}`, + JSON.stringify(event), + ); + ConduitGrpcSdk.Metrics?.increment('database_realtime_events_total', 1, { + operation: event.operation, + }); + await this.pushEvent(schema, event); + } + + private async pushEvent(schema: OptedInSchema, event: DatabaseChangeEvent) { + const payload = JSON.stringify(event); + const adminRooms = roomsForPublicChange(schema.name, event.documentId); + await this.safePush('admin', adminRooms, payload); + if (!schema.authorizationEnabled) { + await this.safePush('router', adminRooms, payload); + return; + } + const userIds = await this.options.subscriptions.listUsers( + schema.name, + event.documentId, + ); + const allowedRooms: string[] = []; + for (const userId of userIds) { + const allowed = await canReadDocument( + this.options.grpcSdk as unknown as AuthorizationSdk, + schema.name, + event.documentId, + userId, + ); + if (!allowed) { + await this.options.subscriptions.removeUser( + schema.name, + event.documentId, + userId, + ); + continue; + } + allowedRooms.push(authorizedDocumentRoom(schema.name, event.documentId, userId)); + } + if (allowedRooms.length > 0) { + await this.safePush('router', allowedRooms, payload); + } + } + + private async safePush( + target: 'admin' | 'router', + rooms: string[], + data: string, + ): Promise { + const client = + target === 'admin' ? this.options.grpcSdk.admin : this.options.grpcSdk.router; + if (!client?.socketPush) return; + try { + await client.socketPush({ + event: 'change', + data, + rooms, + receivers: [], + }); + } catch (err) { + ConduitGrpcSdk.Logger.error(err as Error); + } + } + + private resolveSchema(collectionName?: string): OptedInSchema | undefined { + if (!collectionName) return undefined; + return this.options + .getOptedInSchemas() + .find(schema => schema.collectionName === collectionName); + } + + private async handleStreamError(err: unknown) { + this.watching = false; + this.lastError = err instanceof Error ? err.message : String(err); + this.streamState = 'degraded'; + ConduitGrpcSdk.Metrics?.increment('database_realtime_stream_errors_total'); + ConduitGrpcSdk.Logger.error(err as Error); + if (isResumeTokenUnusable(err)) { + await this.options.grpcSdk.state!.clearKey(RESUME_TOKEN_KEY); + } + await this.stopStream('degraded'); + this.scheduleRetry(); + } + + private scheduleRetry() { + if (this.closed || this.retryTimer) return; + const delay = Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** this.retryAttempt); + this.retryAttempt += 1; + this.retryTimer = setTimeout(() => { + this.retryTimer = null; + void this.reconcile(); + }, delay); + } + + private async stopStream(nextState: RealtimeStatusCode) { + const stream = this.stream; + this.stream = null; + this.watching = false; + this.streamState = nextState; + this.ignoreClose = true; + if (stream) { + try { + await stream.close(); + } catch { + // already closed + } + } + } + + private async releaseLeader() { + this.clearRenewTimer(); + if (!this.lock) return; + try { + await this.options.grpcSdk.state!.releaseLock(this.lock); + } catch { + // lock may already have expired + } + this.lock = null; + } + + private clearTimers() { + this.clearRenewTimer(); + if (this.retryTimer) { + clearTimeout(this.retryTimer); + this.retryTimer = null; + } + } + + private clearRenewTimer() { + if (this.renewTimer) { + clearInterval(this.renewTimer); + this.renewTimer = null; + } + } +} diff --git a/modules/database/src/realtime/RealtimeService.ts b/modules/database/src/realtime/RealtimeService.ts new file mode 100644 index 000000000..db6e77d21 --- /dev/null +++ b/modules/database/src/realtime/RealtimeService.ts @@ -0,0 +1,160 @@ +import { + ConduitGrpcSdk, + ConduitRouteActions, + ConduitRouteReturnDefinition, +} from '@conduitplatform/grpc-sdk'; +import { + ConduitNumber, + ConduitString, + ConfigController, + RoutingManager, +} from '@conduitplatform/module-tools'; +import { DatabaseAdapter } from '../adapters/DatabaseAdapter.js'; +import { MongooseAdapter } from '../adapters/mongoose-adapter/index.js'; +import { MongooseSchema } from '../adapters/mongoose-adapter/MongooseSchema.js'; +import { SequelizeSchema } from '../adapters/sequelize-adapter/SequelizeSchema.js'; +import { toOptedInSchema } from './authorize.js'; +import { MongoChangeStreamCoordinator } from './MongoChangeStreamCoordinator.js'; +import { registerDatabaseRealtimeSocket } from './sockets.js'; +import { buildRealtimeStatus } from './status.js'; +import { RealtimeSubscriptionTracker } from './subscriptions.js'; +import type { OptedInSchema, RealtimeStatus } from './types.js'; +import type { ChangeStreamLike } from './types.js'; + +export class RealtimeService { + private readonly subscriptions: RealtimeSubscriptionTracker; + private coordinator?: MongoChangeStreamCoordinator; + + constructor( + private readonly grpcSdk: ConduitGrpcSdk, + private readonly adapter: DatabaseAdapter, + ) { + this.subscriptions = new RealtimeSubscriptionTracker( + grpcSdk.redisManager.getClient(), + ); + this.grpcSdk.bus?.subscribe('database:create:schema', () => { + void this.reconcile(); + }); + this.grpcSdk.bus?.subscribe('database:delete:schema', () => { + void this.reconcile(); + }); + if (adapter instanceof MongooseAdapter) { + this.coordinator = new MongoChangeStreamCoordinator({ + grpcSdk, + watch: options => this.openWatch(adapter, options.resumeAfter), + hello: () => this.hello(adapter), + getOptedInSchemas: () => this.getOptedInSchemas(), + subscriptions: this.subscriptions, + enabled: () => this.isGloballyEnabled(), + engine: () => adapter.getDatabaseType(), + socketsEnabled: () => this.areAdminSocketsEnabled(), + }); + } + } + + registerAdmin(routingManager: RoutingManager) { + routingManager.route( + { + path: '/realtime/status', + action: ConduitRouteActions.GET, + description: `Returns live-update capability and runtime status for the database module.`, + }, + new ConduitRouteReturnDefinition('DatabaseRealtimeStatus', { + status: ConduitString.Required, + engine: ConduitString.Required, + activeSchemaCount: ConduitNumber.Required, + lastEventAt: ConduitString.Optional, + message: ConduitString.Optional, + }), + async () => this.getStatus(), + ); + registerDatabaseRealtimeSocket(routingManager, { + mode: 'admin', + grpcSdk: this.grpcSdk, + schemaLookup: this.adapter, + subscriptions: this.subscriptions, + isGloballyEnabled: () => this.isGloballyEnabled(), + }); + } + + registerClient(routingManager: RoutingManager) { + registerDatabaseRealtimeSocket(routingManager, { + mode: 'client', + grpcSdk: this.grpcSdk, + schemaLookup: this.adapter, + subscriptions: this.subscriptions, + isGloballyEnabled: () => this.isGloballyEnabled(), + }); + } + + async reconcile(): Promise { + if (!this.coordinator) return; + await this.coordinator.reconcile(); + } + + async shutdown(): Promise { + await this.coordinator?.shutdown(); + } + + async getStatus(): Promise { + const engine = this.adapter.getDatabaseType(); + const optedIn = this.getOptedInSchemas(); + return buildRealtimeStatus({ + engine, + enabled: this.isGloballyEnabled(), + topologySupported: this.coordinator?.getTopology().supported ?? false, + topologyMessage: this.coordinator?.getTopology().message, + activeSchemaCount: optedIn.length, + streamState: + this.coordinator?.getState() ?? (engine === 'MongoDB' ? 'idle' : 'unsupported'), + lastEventAt: this.coordinator?.getLastEventAt(), + lastError: this.coordinator?.getLastError(), + socketsEnabled: await this.areAdminSocketsEnabled(), + }); + } + + private isGloballyEnabled(): boolean { + return ConfigController.getInstance().config?.realtime?.enabled === true; + } + + private getOptedInSchemas(): OptedInSchema[] { + const schemas: OptedInSchema[] = []; + for (const schema of this.adapter.registeredSchemas.values()) { + const optedIn = toOptedInSchema({ + name: schema.name, + collectionName: schema.collectionName, + modelOptions: schema.modelOptions, + }); + if (optedIn) schemas.push(optedIn); + } + return schemas; + } + + private openWatch(adapter: MongooseAdapter, resumeAfter?: unknown): ChangeStreamLike { + const db = adapter.mongoose.connection.db; + if (!db) { + throw new Error('MongoDB connection is not ready'); + } + return db.watch( + [], + resumeAfter ? { resumeAfter: resumeAfter as never } : {}, + ) as unknown as ChangeStreamLike; + } + + private async hello( + adapter: MongooseAdapter, + ): Promise<{ setName?: string; msg?: string } | null> { + const db = adapter.mongoose.connection.db; + if (!db) return null; + return db.admin().command({ hello: 1 }); + } + + private async areAdminSocketsEnabled(): Promise { + try { + const adminConfig = await this.grpcSdk.config.get('admin'); + return adminConfig?.transports?.sockets === true; + } catch { + return true; + } + } +} diff --git a/modules/database/src/realtime/__tests__/authorize.test.ts b/modules/database/src/realtime/__tests__/authorize.test.ts new file mode 100644 index 000000000..c441863eb --- /dev/null +++ b/modules/database/src/realtime/__tests__/authorize.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from '@jest/globals'; +import { status } from '@grpc/grpc-js'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { + assertSchemaAvailable, + optionalDocumentId, + parseSubscribeRequest, + requireSchemaName, +} from '../authorize.js'; + +describe('realtime authorization helpers', () => { + it('parses subscribe payloads and required schema names', () => { + expect(parseSubscribeRequest([{ schema: 'Order', documentId: '1' }])).toEqual({ + schema: 'Order', + documentId: '1', + }); + expect(requireSchemaName('Order')).toBe('Order'); + expect(optionalDocumentId(undefined)).toBeUndefined(); + try { + requireSchemaName(''); + throw new Error('expected throw'); + } catch (err: any) { + expect(err.code).toBe(status.INVALID_ARGUMENT); + } + }); + + it('rejects missing schemas, disabled realtime, and CMS-read for clients', () => { + const lookup = { + getSchema: (name: string) => { + if (name === 'Missing') return undefined; + if (name === 'Off') { + return { name, modelOptions: { conduit: { realtime: { enabled: false } } } }; + } + return { + name, + modelOptions: { + conduit: { + realtime: { enabled: true }, + cms: { crudOperations: { read: { enabled: false } } }, + authorization: { enabled: true }, + }, + }, + }; + }, + }; + try { + assertSchemaAvailable(lookup, 'Missing', true); + throw new Error('expected throw'); + } catch (err: any) { + expect(err.code).toBe(status.NOT_FOUND); + } + try { + assertSchemaAvailable(lookup, 'Off', false); + throw new Error('expected throw'); + } catch (err: any) { + expect(err.code).toBe(status.FAILED_PRECONDITION); + } + try { + assertSchemaAvailable(lookup, 'Order', true); + throw new Error('expected throw'); + } catch (err: any) { + expect(err.code).toBe(status.PERMISSION_DENIED); + } + const admin = assertSchemaAvailable(lookup, 'Order', false); + expect(admin.authorizationEnabled).toBe(true); + }); + + it('maps adapter NOT_FOUND throws to a subscription error', () => { + const lookup = { + getSchema: () => { + throw new GrpcError(status.NOT_FOUND, 'Schema Missing not defined yet'); + }, + }; + try { + assertSchemaAvailable(lookup, 'Missing', false); + throw new Error('expected throw'); + } catch (err: any) { + expect(err.code).toBe(status.NOT_FOUND); + expect(err.message).toBe('Schema does not exist'); + } + }); +}); diff --git a/modules/database/src/realtime/__tests__/change-stream.integration.test.ts b/modules/database/src/realtime/__tests__/change-stream.integration.test.ts new file mode 100644 index 000000000..dd74753fc --- /dev/null +++ b/modules/database/src/realtime/__tests__/change-stream.integration.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from '@jest/globals'; +import { MongoClient, ObjectId } from 'mongodb'; +import { normalizeChangeEvent, type RawChangeEvent } from '../normalize.js'; + +const replicaSetUri = process.env.DB_CONN_URI; +const integrationEnabled = Boolean(replicaSetUri?.includes('replicaSet')); +const describeIntegration = integrationEnabled ? describe : describe.skip; + +describeIntegration('MongoDB change stream contract', () => { + it('normalizes insert events from a replica-set watch without document fields', async () => { + const client = new MongoClient(replicaSetUri!); + await client.connect(); + const dbName = `conduit_realtime_${Date.now()}`; + const db = client.db(dbName); + try { + const collection = db.collection('orders'); + const stream = db.watch([]); + const change = await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error('timed out waiting for change')), + 10_000, + ); + stream.on('change', event => { + clearTimeout(timer); + resolve(event as RawChangeEvent); + }); + stream.on('error', err => { + clearTimeout(timer); + reject(err); + }); + void collection.insertOne({ _id: new ObjectId(), secret: 'do-not-leak' }); + }); + await stream.close(); + const event = normalizeChangeEvent(change, 'Order'); + expect(event).toMatchObject({ + version: 1, + operation: 'insert', + schema: 'Order', + }); + expect(event).not.toHaveProperty('fullDocument'); + expect(JSON.stringify(event)).not.toContain('do-not-leak'); + } finally { + await db.dropDatabase().catch(() => undefined); + await client.close(); + } + }); +}); diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts new file mode 100644 index 000000000..5ec3a4af3 --- /dev/null +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -0,0 +1,166 @@ +import { EventEmitter } from 'node:events'; +import { describe, expect, it, jest } from '@jest/globals'; +import { ObjectId } from 'bson'; +import { MongoChangeStreamCoordinator } from '../MongoChangeStreamCoordinator.js'; +import { RealtimeSubscriptionTracker } from '../subscriptions.js'; +import { authorizedDocumentRoom, roomsForPublicChange } from '../rooms.js'; + +class MemoryStore { + private sets = new Map>(); + async sadd(key: string, ...members: string[]) { + const set = this.sets.get(key) ?? new Set(); + members.forEach(member => set.add(member)); + this.sets.set(key, set); + } + async srem(key: string, ...members: string[]) { + members.forEach(member => this.sets.get(key)?.delete(member)); + } + async smembers(key: string) { + return [...(this.sets.get(key) ?? new Set())]; + } + async scard(key: string) { + return this.sets.get(key)?.size ?? 0; + } + async del(...keys: string[]) { + keys.forEach(key => this.sets.delete(key)); + } +} + +function createCoordinator(overrides?: { + allow?: boolean; + schemas?: { name: string; collectionName: string; authorizationEnabled: boolean }[]; +}) { + const stream = new EventEmitter() as EventEmitter & { close: () => Promise }; + stream.close = async () => { + stream.emit('close'); + }; + const state = new Map(); + const lock = { + extend: jest.fn(async () => lock), + release: jest.fn(async () => undefined), + }; + const routerPush = jest.fn(async () => undefined); + const adminPush = jest.fn(async () => undefined); + const publish = jest.fn(); + const subscriptions = new RealtimeSubscriptionTracker(new MemoryStore()); + const grpcSdk = { + state: { + tryAcquireLock: jest.fn(async () => lock), + releaseLock: jest.fn(async () => undefined), + getKey: jest.fn(async (key: string) => state.get(key) ?? null), + setKey: jest.fn(async (key: string, value: string) => { + state.set(key, value); + }), + clearKey: jest.fn(async (key: string) => { + state.delete(key); + }), + }, + bus: { publish }, + router: { socketPush: routerPush }, + admin: { socketPush: adminPush }, + isAvailable: () => true, + authorization: { + can: async () => ({ allow: overrides?.allow !== false }), + }, + }; + const coordinator = new MongoChangeStreamCoordinator({ + grpcSdk: grpcSdk as never, + watch: () => stream as never, + hello: async () => ({ setName: 'rs0' }), + getOptedInSchemas: () => + overrides?.schemas ?? [ + { name: 'Order', collectionName: 'orders', authorizationEnabled: false }, + ], + subscriptions, + enabled: () => true, + engine: () => 'MongoDB', + }); + return { + coordinator, + stream, + routerPush, + adminPush, + publish, + subscriptions, + grpcSdk, + state, + lock, + }; +} + +describe('MongoChangeStreamCoordinator', () => { + it('emits one normalized event to public rooms and ignores other collections', async () => { + const { coordinator, stream, routerPush, adminPush, publish } = createCoordinator(); + await coordinator.reconcile(); + const resume = { _data: 'token' }; + stream.emit('change', { + operationType: 'insert', + ns: { coll: 'orders' }, + documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, + fullDocument: { secret: 'nope' }, + _id: resume, + wallTime: new Date('2026-01-02T00:00:00.000Z'), + }); + stream.emit('change', { + operationType: 'insert', + ns: { coll: 'other' }, + documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4d') }, + _id: resume, + }); + await new Promise(resolve => setImmediate(resolve)); + expect(publish).toHaveBeenCalledTimes(1); + expect(publish.mock.calls[0][0]).toBe('database:change:Order'); + const payload = JSON.parse(publish.mock.calls[0][1] as string); + expect(payload).toMatchObject({ + operation: 'insert', + schema: 'Order', + documentId: '64b64c4c4c4c4c4c4c4c4c4c', + }); + expect(payload).not.toHaveProperty('fullDocument'); + expect(payload).not.toHaveProperty('secret'); + const expectedRooms = roomsForPublicChange('Order', '64b64c4c4c4c4c4c4c4c4c4c'); + expect(routerPush).toHaveBeenCalledWith( + expect.objectContaining({ event: 'change', rooms: expectedRooms }), + ); + expect(adminPush).toHaveBeenCalledWith( + expect.objectContaining({ event: 'change', rooms: expectedRooms }), + ); + await coordinator.shutdown(); + }); + + it('re-checks ReBAC before emission and drops revoked users', async () => { + const { coordinator, stream, routerPush, subscriptions } = createCoordinator({ + allow: false, + schemas: [{ name: 'Order', collectionName: 'orders', authorizationEnabled: true }], + }); + await subscriptions.addAuthorizedDocument( + 'sock-1', + 'Order', + '64b64c4c4c4c4c4c4c4c4c4c', + 'user-1', + ); + await coordinator.reconcile(); + stream.emit('change', { + operationType: 'update', + ns: { coll: 'orders' }, + documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, + _id: { _data: 'token' }, + }); + await new Promise(resolve => setImmediate(resolve)); + expect(routerPush).not.toHaveBeenCalled(); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c')).toEqual([]); + expect(authorizedDocumentRoom('Order', '64b64c4c4c4c4c4c4c4c4c', 'user-1')).toContain( + 'user-1', + ); + await coordinator.shutdown(); + }); + + it('clears an unusable resume token and retries', async () => { + const { coordinator, stream, grpcSdk } = createCoordinator(); + await coordinator.reconcile(); + stream.emit('error', { code: 280, message: 'ChangeStreamHistoryLost' }); + await new Promise(resolve => setImmediate(resolve)); + expect(grpcSdk.state.clearKey).toHaveBeenCalled(); + await coordinator.shutdown(); + }); +}); diff --git a/modules/database/src/realtime/__tests__/normalize.test.ts b/modules/database/src/realtime/__tests__/normalize.test.ts new file mode 100644 index 000000000..d51eaa549 --- /dev/null +++ b/modules/database/src/realtime/__tests__/normalize.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from '@jest/globals'; +import { ObjectId } from 'bson'; +import { normalizeChangeEvent, parseResumeToken } from '../normalize.js'; + +describe('normalizeChangeEvent', () => { + it('normalizes insert/update/replace/delete into metadata-only events', () => { + const resume = { _data: 'token-1' }; + const event = normalizeChangeEvent( + { + operationType: 'insert', + documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, + wallTime: new Date('2026-01-01T00:00:00.000Z'), + _id: resume, + }, + 'Order', + ); + expect(event).toMatchObject({ + version: 1, + operation: 'insert', + schema: 'Order', + documentId: '64b64c4c4c4c4c4c4c4c4c4c', + occurredAt: '2026-01-01T00:00:00.000Z', + }); + expect(event?.resumeToken).toEqual( + parseResumeToken(event!.resumeToken) ? event!.resumeToken : event?.resumeToken, + ); + expect(JSON.parse(JSON.stringify(event))).not.toHaveProperty('fullDocument'); + }); + + it('ignores drop/invalidate and missing document ids', () => { + expect( + normalizeChangeEvent({ operationType: 'drop', _id: { _data: 'x' } }, 'Order'), + ).toBeNull(); + expect( + normalizeChangeEvent({ operationType: 'insert', _id: { _data: 'x' } }, 'Order'), + ).toBeNull(); + }); +}); diff --git a/modules/database/src/realtime/__tests__/rooms.test.ts b/modules/database/src/realtime/__tests__/rooms.test.ts new file mode 100644 index 000000000..dbdd57e1b --- /dev/null +++ b/modules/database/src/realtime/__tests__/rooms.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from '@jest/globals'; +import { + authorizedDocumentRoom, + documentRoom, + roomsForPublicChange, + schemaRoom, +} from '../rooms.js'; + +describe('realtime rooms', () => { + it('isolates schema, document, and per-user rooms', () => { + expect(schemaRoom('Order')).toBe('database:schema:Order'); + expect(documentRoom('Order', 'abc')).toBe('database:doc:Order:abc'); + expect(authorizedDocumentRoom('Order', 'abc', 'user-1')).toBe( + 'database:doc:Order:abc:user:user-1', + ); + expect(roomsForPublicChange('Order', 'abc')).toEqual([ + 'database:schema:Order', + 'database:doc:Order:abc', + ]); + expect(schemaRoom('Order')).not.toBe(schemaRoom('Orders')); + expect(authorizedDocumentRoom('Order', 'abc', 'u1')).not.toBe( + authorizedDocumentRoom('Order', 'abc', 'u2'), + ); + }); + + it('encodes reserved characters so rooms cannot collide', () => { + expect(schemaRoom('a/b')).toBe('database:schema:a%2Fb'); + expect(documentRoom('Order', 'id:1')).toBe('database:doc:Order:id%3A1'); + }); +}); diff --git a/modules/database/src/realtime/__tests__/status.test.ts b/modules/database/src/realtime/__tests__/status.test.ts new file mode 100644 index 000000000..a94790cfb --- /dev/null +++ b/modules/database/src/realtime/__tests__/status.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from '@jest/globals'; +import { buildRealtimeStatus } from '../status.js'; + +describe('buildRealtimeStatus', () => { + it('reports unsupported, disabled, idle, live, and degraded states', () => { + expect( + buildRealtimeStatus({ + engine: 'PostgreSQL', + enabled: true, + topologySupported: true, + activeSchemaCount: 1, + streamState: 'live', + }).status, + ).toBe('unsupported'); + expect( + buildRealtimeStatus({ + engine: 'MongoDB', + enabled: false, + topologySupported: true, + activeSchemaCount: 1, + streamState: 'live', + }).status, + ).toBe('disabled'); + expect( + buildRealtimeStatus({ + engine: 'MongoDB', + enabled: true, + topologySupported: false, + activeSchemaCount: 1, + streamState: 'idle', + }).message, + ).toMatch(/replica set/i); + expect( + buildRealtimeStatus({ + engine: 'MongoDB', + enabled: true, + topologySupported: true, + activeSchemaCount: 2, + streamState: 'live', + }).status, + ).toBe('live'); + expect( + buildRealtimeStatus({ + engine: 'MongoDB', + enabled: true, + topologySupported: true, + activeSchemaCount: 1, + streamState: 'degraded', + lastError: 'boom', + }), + ).toMatchObject({ status: 'degraded', message: 'boom' }); + }); +}); diff --git a/modules/database/src/realtime/__tests__/subscriptions.test.ts b/modules/database/src/realtime/__tests__/subscriptions.test.ts new file mode 100644 index 000000000..45ee7d1c9 --- /dev/null +++ b/modules/database/src/realtime/__tests__/subscriptions.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from '@jest/globals'; +import { RealtimeSubscriptionTracker } from '../subscriptions.js'; + +class MemoryStore { + private sets = new Map>(); + + async sadd(key: string, ...members: string[]) { + const set = this.sets.get(key) ?? new Set(); + members.forEach(member => set.add(member)); + this.sets.set(key, set); + return members.length; + } + + async srem(key: string, ...members: string[]) { + const set = this.sets.get(key); + if (!set) return 0; + members.forEach(member => set.delete(member)); + return members.length; + } + + async smembers(key: string) { + return [...(this.sets.get(key) ?? new Set())]; + } + + async scard(key: string) { + return this.sets.get(key)?.size ?? 0; + } + + async del(...keys: string[]) { + keys.forEach(key => this.sets.delete(key)); + return keys.length; + } +} + +describe('RealtimeSubscriptionTracker', () => { + it('tracks users per document and cleans up on disconnect', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); + await tracker.addAuthorizedDocument('s2', 'Order', 'doc-1', 'user-1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + + await tracker.disconnect('s1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + + await tracker.disconnect('s2'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); + }); + + it('removes revoked users from the document set', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); + await tracker.removeUser('Order', 'doc-1', 'user-1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); + }); +}); diff --git a/modules/database/src/realtime/__tests__/topology.test.ts b/modules/database/src/realtime/__tests__/topology.test.ts new file mode 100644 index 000000000..abf8e8861 --- /dev/null +++ b/modules/database/src/realtime/__tests__/topology.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from '@jest/globals'; +import { isResumeTokenUnusable, topologyFromHello } from '../topology.js'; + +describe('topology helpers', () => { + it('accepts replica sets and mongos, rejects standalone', () => { + expect(topologyFromHello({ setName: 'rs0' })).toEqual({ supported: true }); + expect(topologyFromHello({ msg: 'isdbgrid' })).toEqual({ supported: true }); + expect(topologyFromHello({}).supported).toBe(false); + }); + + it('detects unusable resume tokens', () => { + expect(isResumeTokenUnusable({ code: 280 })).toBe(true); + expect(isResumeTokenUnusable(new Error('cannot resume'))).toBe(true); + expect(isResumeTokenUnusable(new Error('socket hang up'))).toBe(false); + }); +}); diff --git a/modules/database/src/realtime/authorize.ts b/modules/database/src/realtime/authorize.ts new file mode 100644 index 000000000..330992f13 --- /dev/null +++ b/modules/database/src/realtime/authorize.ts @@ -0,0 +1,147 @@ +import { status } from '@grpc/grpc-js'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import type { OptedInSchema, SubscribeRequest } from './types.js'; + +export class RealtimeSubscriptionError extends GrpcError { + constructor(code: number, message: string) { + super(code, message); + this.name = 'RealtimeSubscriptionError'; + } +} + +export type AuthorizationSdk = { + isAvailable: (module: string) => boolean; + authorization?: { + can: (request: { + subject: string; + actions: string[]; + resource: string; + }) => Promise<{ allow: boolean }>; + } | null; +}; + +export type SchemaLookup = { + getSchema(name: string): + | { + name: string; + modelOptions?: { + conduit?: { + realtime?: { enabled?: boolean }; + cms?: { crudOperations?: { read?: { enabled?: boolean } } }; + authorization?: { enabled?: boolean }; + }; + }; + } + | undefined; +}; + +export type ResolvedSubscription = { + schema: string; + documentId?: string; + authorizationEnabled: boolean; + rooms: string[]; +}; + +export function parseSubscribeRequest(params: unknown[]): SubscribeRequest { + const raw = params[0]; + if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + return raw as SubscribeRequest; + } + return {}; +} + +export function requireSchemaName(value: unknown): string { + if (typeof value !== 'string' || value.trim() === '') { + throw new RealtimeSubscriptionError(status.INVALID_ARGUMENT, 'schema is required'); + } + return value.trim(); +} + +export function optionalDocumentId(value: unknown): string | undefined { + if (value === undefined || value === null || value === '') return undefined; + if (typeof value !== 'string') { + throw new RealtimeSubscriptionError( + status.INVALID_ARGUMENT, + 'documentId must be a string', + ); + } + return value; +} + +export async function canReadDocument( + grpcSdk: AuthorizationSdk, + schema: string, + documentId: string, + userId: string, +): Promise { + if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { + return false; + } + try { + const decision = await grpcSdk.authorization.can({ + subject: `User:${userId}`, + actions: ['read'], + resource: `${schema}:${documentId}`, + }); + return decision.allow === true; + } catch { + return false; + } +} + +export function assertSchemaAvailable( + lookup: SchemaLookup, + schemaName: string, + requireCmsRead: boolean, +): { + authorizationEnabled: boolean; + realtimeEnabled: boolean; +} { + let schema: ReturnType; + try { + schema = lookup.getSchema(schemaName); + } catch (err) { + if (err instanceof GrpcError && err.code === status.NOT_FOUND) { + throw new RealtimeSubscriptionError(status.NOT_FOUND, 'Schema does not exist'); + } + throw err; + } + if (!schema) { + throw new RealtimeSubscriptionError(status.NOT_FOUND, 'Schema does not exist'); + } + const conduit = schema.modelOptions?.conduit; + if (!conduit?.realtime?.enabled) { + throw new RealtimeSubscriptionError( + status.FAILED_PRECONDITION, + 'Live updates are not enabled for this schema', + ); + } + if (requireCmsRead && conduit.cms?.crudOperations?.read?.enabled !== true) { + throw new RealtimeSubscriptionError( + status.PERMISSION_DENIED, + 'CMS read is not enabled for this schema', + ); + } + return { + authorizationEnabled: conduit.authorization?.enabled === true, + realtimeEnabled: true, + }; +} + +export function toOptedInSchema(schema: { + name: string; + collectionName: string; + modelOptions?: { + conduit?: { + realtime?: { enabled?: boolean }; + authorization?: { enabled?: boolean }; + }; + }; +}): OptedInSchema | null { + if (!schema.modelOptions?.conduit?.realtime?.enabled) return null; + return { + name: schema.name, + collectionName: schema.collectionName, + authorizationEnabled: schema.modelOptions.conduit.authorization?.enabled === true, + }; +} diff --git a/modules/database/src/realtime/index.ts b/modules/database/src/realtime/index.ts new file mode 100644 index 000000000..655c04bba --- /dev/null +++ b/modules/database/src/realtime/index.ts @@ -0,0 +1,11 @@ +export { RealtimeService } from './RealtimeService.js'; +export { buildRealtimeStatus } from './status.js'; +export { normalizeChangeEvent } from './normalize.js'; +export { + schemaRoom, + documentRoom, + authorizedDocumentRoom, + roomsForPublicChange, +} from './rooms.js'; +export { RealtimeSubscriptionTracker } from './subscriptions.js'; +export type { RealtimeStatus, DatabaseChangeEvent } from './types.js'; diff --git a/modules/database/src/realtime/normalize.ts b/modules/database/src/realtime/normalize.ts new file mode 100644 index 000000000..bdd876f16 --- /dev/null +++ b/modules/database/src/realtime/normalize.ts @@ -0,0 +1,70 @@ +import { EJSON } from 'bson'; +import { + DATABASE_CHANGE_EVENT_VERSION, + DATABASE_CHANGE_OPERATIONS, + type DatabaseChangeEvent, + type DatabaseChangeOperation, +} from './types.js'; + +const OPERATION_SET = new Set(DATABASE_CHANGE_OPERATIONS); + +export type RawChangeEvent = { + operationType?: string; + ns?: { coll?: string }; + documentKey?: { _id?: unknown }; + wallTime?: Date; + clusterTime?: { toString?: () => string }; + _id?: unknown; +}; + +export function normalizeChangeEvent( + change: RawChangeEvent, + schemaName: string, + occurredAt: Date = new Date(), +): DatabaseChangeEvent | null { + const operation = change.operationType; + if (!operation || !OPERATION_SET.has(operation)) { + return null; + } + const documentId = extractDocumentId(change.documentKey?._id); + if (!documentId) { + return null; + } + if (!change._id) { + return null; + } + return { + version: DATABASE_CHANGE_EVENT_VERSION, + operation: operation as DatabaseChangeOperation, + schema: schemaName, + documentId, + occurredAt: (change.wallTime instanceof Date + ? change.wallTime + : occurredAt + ).toISOString(), + resumeToken: EJSON.stringify(change._id), + }; +} + +export function parseResumeToken(token: string | null | undefined): unknown | undefined { + if (!token) return undefined; + try { + return EJSON.parse(token); + } catch { + return undefined; + } +} + +function extractDocumentId(id: unknown): string | null { + if (id === undefined || id === null) return null; + if (typeof id === 'string' || typeof id === 'number') return String(id); + if (typeof id === 'object' && id !== null && 'toHexString' in id) { + const hex = (id as { toHexString: () => string }).toHexString(); + return typeof hex === 'string' && hex.length > 0 ? hex : null; + } + if (typeof id === 'object' && id !== null && 'toString' in id) { + const value = String(id); + return value && value !== '[object Object]' ? value : null; + } + return null; +} diff --git a/modules/database/src/realtime/rooms.ts b/modules/database/src/realtime/rooms.ts new file mode 100644 index 000000000..45a69a7cd --- /dev/null +++ b/modules/database/src/realtime/rooms.ts @@ -0,0 +1,25 @@ +const ROOM_PREFIX = 'database'; + +function encodeSegment(value: string): string { + return encodeURIComponent(value); +} + +export function schemaRoom(schema: string): string { + return `${ROOM_PREFIX}:schema:${encodeSegment(schema)}`; +} + +export function documentRoom(schema: string, documentId: string): string { + return `${ROOM_PREFIX}:doc:${encodeSegment(schema)}:${encodeSegment(documentId)}`; +} + +export function authorizedDocumentRoom( + schema: string, + documentId: string, + userId: string, +): string { + return `${ROOM_PREFIX}:doc:${encodeSegment(schema)}:${encodeSegment(documentId)}:user:${encodeSegment(userId)}`; +} + +export function roomsForPublicChange(schema: string, documentId: string): string[] { + return [schemaRoom(schema), documentRoom(schema, documentId)]; +} diff --git a/modules/database/src/realtime/sockets.ts b/modules/database/src/realtime/sockets.ts new file mode 100644 index 000000000..5cc97957e --- /dev/null +++ b/modules/database/src/realtime/sockets.ts @@ -0,0 +1,166 @@ +import { + ConduitGrpcSdk, + ConduitRouteReturnDefinition, + ParsedSocketRequest, + TYPE, + UnparsedSocketResponse, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { RoutingManager } from '@conduitplatform/module-tools'; +import { + assertSchemaAvailable, + canReadDocument, + optionalDocumentId, + parseSubscribeRequest, + RealtimeSubscriptionError, + requireSchemaName, + type AuthorizationSdk, + type SchemaLookup, +} from './authorize.js'; +import { authorizedDocumentRoom, documentRoom, schemaRoom } from './rooms.js'; +import type { RealtimeSubscriptionTracker } from './subscriptions.js'; + +type SocketMode = 'client' | 'admin'; + +export function registerDatabaseRealtimeSocket( + routingManager: RoutingManager, + options: { + mode: SocketMode; + grpcSdk: ConduitGrpcSdk; + schemaLookup: SchemaLookup; + subscriptions: RealtimeSubscriptionTracker; + isGloballyEnabled: () => boolean; + }, +) { + const handlers = createSocketHandlers(options); + routingManager.socket( + { + path: '/', + middlewares: options.mode === 'client' ? ['authMiddleware'] : undefined, + }, + { + connect: { handler: handlers.connect }, + disconnect: { handler: handlers.disconnect }, + subscribe: { + params: [TYPE.JSON], + handler: handlers.subscribe, + returnType: new ConduitRouteReturnDefinition('DatabaseRealtimeSubscribe', { + rooms: [TYPE.String], + }), + }, + unsubscribe: { + params: [TYPE.JSON], + handler: handlers.unsubscribe, + returnType: new ConduitRouteReturnDefinition('DatabaseRealtimeUnsubscribe', { + rooms: [TYPE.String], + }), + }, + }, + ); +} + +function createSocketHandlers(options: { + mode: SocketMode; + grpcSdk: ConduitGrpcSdk; + schemaLookup: SchemaLookup; + subscriptions: RealtimeSubscriptionTracker; + isGloballyEnabled: () => boolean; +}) { + return { + connect: async (): Promise => { + return { event: 'connected', data: { ok: true } }; + }, + disconnect: async (call: ParsedSocketRequest): Promise => { + await options.subscriptions.disconnect(call.request.socketId); + return { event: 'disconnected', data: { ok: true } }; + }, + subscribe: async (call: ParsedSocketRequest): Promise => { + const rooms = await resolveSubscription(call, options, 'join'); + return { event: 'join-room', rooms }; + }, + unsubscribe: async (call: ParsedSocketRequest): Promise => { + const rooms = await resolveSubscription(call, options, 'leave'); + return { event: 'leave-room', rooms }; + }, + }; +} + +async function resolveSubscription( + call: ParsedSocketRequest, + options: { + mode: SocketMode; + grpcSdk: ConduitGrpcSdk; + schemaLookup: SchemaLookup; + subscriptions: RealtimeSubscriptionTracker; + isGloballyEnabled: () => boolean; + }, + action: 'join' | 'leave', +): Promise { + if (!options.isGloballyEnabled()) { + throw new RealtimeSubscriptionError( + status.FAILED_PRECONDITION, + 'Live updates are disabled', + ); + } + const payload = parseSubscribeRequest(call.request.params ?? []); + const schemaName = requireSchemaName(payload.schema); + const documentId = optionalDocumentId(payload.documentId); + const meta = assertSchemaAvailable( + options.schemaLookup, + schemaName, + options.mode === 'client', + ); + + if (options.mode === 'admin') { + if (documentId) return [documentRoom(schemaName, documentId)]; + return [schemaRoom(schemaName)]; + } + + const userId = call.request.context?.user?._id; + if (!userId) { + throw new RealtimeSubscriptionError( + status.UNAUTHENTICATED, + 'Authentication required', + ); + } + + if (meta.authorizationEnabled) { + if (!documentId) { + throw new RealtimeSubscriptionError( + status.PERMISSION_DENIED, + 'Document ID is required for authorized schemas', + ); + } + const allowed = await canReadDocument( + options.grpcSdk as unknown as AuthorizationSdk, + schemaName, + documentId, + userId, + ); + if (!allowed) { + throw new RealtimeSubscriptionError( + status.PERMISSION_DENIED, + 'Read permission denied', + ); + } + if (action === 'join') { + await options.subscriptions.addAuthorizedDocument( + call.request.socketId, + schemaName, + documentId, + userId, + ); + } else { + await options.subscriptions.removeAuthorizedDocument( + call.request.socketId, + schemaName, + documentId, + userId, + ); + } + return [authorizedDocumentRoom(schemaName, documentId, userId)]; + } + + if (documentId) return [documentRoom(schemaName, documentId)]; + return [schemaRoom(schemaName)]; +} diff --git a/modules/database/src/realtime/status.ts b/modules/database/src/realtime/status.ts new file mode 100644 index 000000000..b1da21b03 --- /dev/null +++ b/modules/database/src/realtime/status.ts @@ -0,0 +1,74 @@ +import type { RealtimeStatus, RealtimeStatusCode } from './types.js'; + +export type RealtimeStatusInput = { + engine: string; + enabled: boolean; + topologySupported: boolean; + topologyMessage?: string; + activeSchemaCount: number; + streamState: RealtimeStatusCode; + lastEventAt?: string; + lastError?: string; + socketsEnabled?: boolean; +}; + +export function buildRealtimeStatus(input: RealtimeStatusInput): RealtimeStatus { + if (input.engine !== 'MongoDB') { + return { + status: 'unsupported', + engine: input.engine, + activeSchemaCount: 0, + message: 'Live updates require MongoDB', + }; + } + if (!input.enabled) { + return { + status: 'disabled', + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + }; + } + if (!input.topologySupported) { + return { + status: 'idle', + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + message: + input.topologyMessage ?? + 'A replica set or sharded MongoDB deployment is required for live updates', + }; + } + if (input.socketsEnabled === false) { + return { + status: 'idle', + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + message: + 'Enable Admin socket transport to consume live updates in the control panel', + lastEventAt: input.lastEventAt, + }; + } + if (input.activeSchemaCount === 0) { + return { + status: 'idle', + engine: input.engine, + activeSchemaCount: 0, + message: 'No schemas have live updates enabled', + }; + } + if (input.streamState === 'degraded') { + return { + status: 'degraded', + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + lastEventAt: input.lastEventAt, + message: input.lastError ?? 'Change stream is retrying after an error', + }; + } + return { + status: input.streamState, + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + lastEventAt: input.lastEventAt, + }; +} diff --git a/modules/database/src/realtime/subscriptions.ts b/modules/database/src/realtime/subscriptions.ts new file mode 100644 index 000000000..da7bfb867 --- /dev/null +++ b/modules/database/src/realtime/subscriptions.ts @@ -0,0 +1,92 @@ +export type SubscriptionStore = { + sadd(key: string, ...members: string[]): Promise; + srem(key: string, ...members: string[]): Promise; + smembers(key: string): Promise; + scard(key: string): Promise; + del(...keys: string[]): Promise; +}; + +function socketKey(socketId: string): string { + return `realtime:socket:${socketId}`; +} + +function docUsersKey(schema: string, documentId: string): string { + return `realtime:doc:${schema}:${documentId}`; +} + +function userDocSocketsKey(schema: string, documentId: string, userId: string): string { + return `realtime:userdoc:${schema}:${documentId}:${userId}`; +} + +function subscriptionRecord(schema: string, documentId: string, userId: string): string { + return JSON.stringify({ schema, documentId, userId }); +} + +export class RealtimeSubscriptionTracker { + constructor(private readonly store: SubscriptionStore) {} + + async addAuthorizedDocument( + socketId: string, + schema: string, + documentId: string, + userId: string, + ): Promise { + await this.store.sadd( + socketKey(socketId), + subscriptionRecord(schema, documentId, userId), + ); + await this.store.sadd(userDocSocketsKey(schema, documentId, userId), socketId); + await this.store.sadd(docUsersKey(schema, documentId), userId); + } + + async removeAuthorizedDocument( + socketId: string, + schema: string, + documentId: string, + userId: string, + ): Promise { + await this.store.srem( + socketKey(socketId), + subscriptionRecord(schema, documentId, userId), + ); + await this.store.srem(userDocSocketsKey(schema, documentId, userId), socketId); + const remaining = await this.store.scard( + userDocSocketsKey(schema, documentId, userId), + ); + if (remaining === 0) { + await this.store.srem(docUsersKey(schema, documentId), userId); + await this.store.del(userDocSocketsKey(schema, documentId, userId)); + } + } + + async listUsers(schema: string, documentId: string): Promise { + return this.store.smembers(docUsersKey(schema, documentId)); + } + + async removeUser(schema: string, documentId: string, userId: string): Promise { + await this.store.srem(docUsersKey(schema, documentId), userId); + await this.store.del(userDocSocketsKey(schema, documentId, userId)); + } + + async disconnect(socketId: string): Promise { + const records = await this.store.smembers(socketKey(socketId)); + for (const record of records) { + try { + const parsed = JSON.parse(record) as { + schema: string; + documentId: string; + userId: string; + }; + await this.removeAuthorizedDocument( + socketId, + parsed.schema, + parsed.documentId, + parsed.userId, + ); + } catch { + // ignore malformed records + } + } + await this.store.del(socketKey(socketId)); + } +} diff --git a/modules/database/src/realtime/topology.ts b/modules/database/src/realtime/topology.ts new file mode 100644 index 000000000..341d49438 --- /dev/null +++ b/modules/database/src/realtime/topology.ts @@ -0,0 +1,49 @@ +const CHANGE_STREAM_ERROR_CODES = new Set([ + 136, // CappedPositionLost + 237, // CursorKilled + 280, // ChangeStreamHistoryLost + 286, // ChangeStreamFatalError +]); + +export type TopologyResult = { + supported: boolean; + message?: string; +}; + +export function topologyFromHello( + hello: + | { + setName?: string; + msg?: string; + } + | null + | undefined, +): TopologyResult { + if (!hello) { + return { supported: false, message: 'Unable to determine MongoDB topology' }; + } + if (hello.msg === 'isdbgrid' || Boolean(hello.setName)) { + return { supported: true }; + } + return { + supported: false, + message: 'A replica set or sharded MongoDB deployment is required for live updates', + }; +} + +export function isResumeTokenUnusable(error: unknown): boolean { + const code = extractErrorCode(error); + if (code !== undefined && CHANGE_STREAM_ERROR_CODES.has(code)) { + return true; + } + const message = error instanceof Error ? error.message : String(error ?? ''); + return /resume token|ChangeStreamHistoryLost|cannot resume/i.test(message); +} + +function extractErrorCode(error: unknown): number | undefined { + if (!error || typeof error !== 'object') return undefined; + const candidate = error as { code?: unknown; errorCode?: unknown }; + if (typeof candidate.code === 'number') return candidate.code; + if (typeof candidate.errorCode === 'number') return candidate.errorCode; + return undefined; +} diff --git a/modules/database/src/realtime/types.ts b/modules/database/src/realtime/types.ts new file mode 100644 index 000000000..9901c2d86 --- /dev/null +++ b/modules/database/src/realtime/types.ts @@ -0,0 +1,49 @@ +export const DATABASE_CHANGE_EVENT_VERSION = 1 as const; + +export const DATABASE_CHANGE_OPERATIONS = [ + 'insert', + 'update', + 'replace', + 'delete', +] as const; + +export type DatabaseChangeOperation = (typeof DATABASE_CHANGE_OPERATIONS)[number]; + +export type DatabaseChangeEvent = { + version: typeof DATABASE_CHANGE_EVENT_VERSION; + operation: DatabaseChangeOperation; + schema: string; + documentId: string; + occurredAt: string; + resumeToken: string; +}; + +export type RealtimeStatusCode = + 'unsupported' | 'disabled' | 'idle' | 'starting' | 'live' | 'degraded'; + +export type RealtimeStatus = { + status: RealtimeStatusCode; + engine: string; + activeSchemaCount: number; + lastEventAt?: string; + message?: string; +}; + +export type OptedInSchema = { + name: string; + collectionName: string; + authorizationEnabled: boolean; +}; + +export type SubscribeRequest = { + schema?: unknown; + documentId?: unknown; +}; + +export type ChangeStreamLike = { + on( + event: 'change' | 'error' | 'close' | 'end', + listener: (...args: unknown[]) => void, + ): void; + close(): Promise | void; +}; diff --git a/modules/database/src/routes/index.ts b/modules/database/src/routes/index.ts index 2c09cf237..1d4002a74 100644 --- a/modules/database/src/routes/index.ts +++ b/modules/database/src/routes/index.ts @@ -13,6 +13,7 @@ import { import { DatabaseAdapter } from '../adapters/DatabaseAdapter.js'; import { MongooseSchema } from '../adapters/mongoose-adapter/MongooseSchema.js'; import { SequelizeSchema } from '../adapters/sequelize-adapter/SequelizeSchema.js'; +import type { RealtimeService } from '../realtime/index.js'; export class DatabaseRoutes { private readonly handlers: CmsHandlers; @@ -34,6 +35,7 @@ export class DatabaseRoutes { readonly server: GrpcServer, private readonly database: DatabaseAdapter, private readonly grpcSdk: ConduitGrpcSdk, + private readonly realtimeService?: RealtimeService, ) { this.handlers = new CmsHandlers(grpcSdk, database); this._routingManager = new RoutingManager(this.grpcSdk.router!, server); @@ -55,7 +57,6 @@ export class DatabaseRoutes { } requestRefresh() { - if (this.crudRoutes.length === 0 && this.customRoutes.length === 0) return; this._scheduleTimeout(); } @@ -86,6 +87,7 @@ export class DatabaseRoutes { this.crudRoutes.concat(this.customRoutes).forEach(route => { this._routingManager.route(route.input, route.returnType, route.handler); }); + this.realtimeService?.registerClient(this._routingManager); this._routingManager .registerRoutes() .then(() => { diff --git a/modules/database/src/utils/SchemaConverter.ts b/modules/database/src/utils/SchemaConverter.ts index 2e099ed57..f482f4936 100644 --- a/modules/database/src/utils/SchemaConverter.ts +++ b/modules/database/src/utils/SchemaConverter.ts @@ -38,6 +38,9 @@ export namespace SchemaConverter { authorization?: { enabled?: boolean; }; + realtime?: { + enabled?: boolean; + }; permissions?: { extendable?: boolean; canCreate?: boolean; @@ -130,6 +133,9 @@ export namespace SchemaConverter { defaults.conduit!.authorization?.enabled ?? false, }; + modelOptions.conduit.realtime = { + enabled: opts.realtime?.enabled ?? existing?.realtime?.enabled ?? false, + }; modelOptions.conduit.permissions = { extendable: explicit.permissions?.extendable !== undefined diff --git a/packages/core/package.json b/packages/core/package.json index 0636f4ce2..764e285ab 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -12,6 +12,7 @@ "start": "node dist/bin/www.js", "start:bundle": "node bundle/index.js", "lint": "eslint src", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/admin/realtime/*.test.js", "prebuild": "npm run generateTypes", "build": "rimraf dist && tsc", "generateTypes": "sh build.sh", diff --git a/packages/core/src/admin/AdminModule.ts b/packages/core/src/admin/AdminModule.ts index 3f18ea069..4f4574571 100644 --- a/packages/core/src/admin/AdminModule.ts +++ b/packages/core/src/admin/AdminModule.ts @@ -14,6 +14,7 @@ import { PatchRouteMiddlewaresRequest, RegisterAdminRouteRequest, RegisterAdminRouteRequest_PathDefinition, + SocketPushRequest, } from '../interfaces/index.js'; import { hashPassword } from './utils/auth.js'; import AdminConfigRawSchema from './config/index.js'; @@ -33,6 +34,7 @@ import { ConduitSocket, grpcToConduitRoute, RouteT, + SocketPush, } from '@conduitplatform/hermes'; import convict from 'convict'; import { NextFunction, Request, Response } from 'express'; @@ -48,6 +50,7 @@ import { getReadinessMiddleware } from '../health/readinessMiddleware.js'; import { ReadinessService } from '../health/ReadinessService.js'; import type { CoreHealthProvider } from '../health/types.js'; import { stripUndeclaredConfigParams } from '../utils/stripUndeclaredConfigParams.js'; +import { adminSocketNamespace } from './realtime/namespace.js'; export default class AdminModule { grpcSdk: ConduitGrpcSdk; @@ -93,6 +96,7 @@ export default class AdminModule { adminRoutes.createApiTokenRoute(), adminRoutes.getApiTokensRoute(), adminRoutes.deleteApiTokenRoute(), + adminRoutes.getRealtimeTicketRoute(), configRoutes.getModulesRoute(), adminRoutes.getStateExportRoute(this), adminRoutes.getStateImportRoute(this), @@ -131,6 +135,7 @@ export default class AdminModule { { registerAdminRoute: this.registerAdminRoute.bind(this), patchRouteMiddlewares: this.patchRouteMiddlewares.bind(this), + socketPush: this.socketPush.bind(this), }, ); } @@ -238,6 +243,27 @@ export default class AdminModule { callback(null, {}); } + async socketPush( + call: GrpcRequest, + callback: GrpcCallback>, + ) { + try { + const moduleName = call.metadata?.get('module-name')?.[0] as string | undefined; + const socketData: SocketPush = { + event: call.request.event, + data: call.request.data ? JSON.parse(call.request.data) : undefined, + receivers: call.request.receivers, + rooms: call.request.rooms, + namespace: adminSocketNamespace(moduleName), + }; + await this._router.socketPush(socketData); + } catch (err) { + ConduitGrpcSdk.Logger.error(err as Error); + return callback({ code: status.INTERNAL, message: 'Well that failed :/' }); + } + callback(null, {}); + } + registerRoute(route: ConduitRoute): void { this._sdkRoutes.push(route); this._router.registerConduitRoute(route); @@ -288,6 +314,16 @@ export default class AdminModule { `New admin route registered: ${r.input.action} ${r.input.path} handler url: ${url}`, ); this._router.registerConduitRoute(r); + } else if (r instanceof ConduitMiddleware) { + ConduitGrpcSdk.Logger.log( + `New admin middleware registered: ${r.input.path} handler url: ${url}`, + ); + this._router.registerRouteMiddleware(r, url); + } else if (r instanceof ConduitSocket) { + ConduitGrpcSdk.Logger.log( + `New admin socket registered: ${r.input.path} handler url: ${url}`, + ); + this._router.registerConduitSocket(r); } }); // @ts-ignore diff --git a/packages/core/src/admin/middleware/Admin.middleware.ts b/packages/core/src/admin/middleware/Admin.middleware.ts index 51c811987..716dae763 100644 --- a/packages/core/src/admin/middleware/Admin.middleware.ts +++ b/packages/core/src/admin/middleware/Admin.middleware.ts @@ -4,6 +4,7 @@ import { isNil } from 'lodash-es'; import { isDev } from '../utils/middleware.js'; import { ConduitRequest } from '@conduitplatform/hermes'; import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { isSocketHandshake } from '../realtime/handshake.js'; export function getAdminMiddleware(configManager: any) { return async function adminMiddleware( @@ -28,9 +29,14 @@ export function getAdminMiddleware(configManager: any) { ) { return next(); } - // Allow API tokens (cdt_*) to bypass masterkey; Auth.middleware will validate the token + // Allow API tokens (cdt_*) and Socket.IO handshakes with a Bearer token to + // skip masterkey. Auth.middleware still validates the credential. const authHeader = req.headers.authorization; - if (authHeader && authHeader.startsWith('Bearer cdt_')) { + if ( + typeof authHeader === 'string' && + authHeader.startsWith('Bearer ') && + (authHeader.startsWith('Bearer cdt_') || isSocketHandshake(req)) + ) { return next(); } const masterKey = req.headers.masterkey; diff --git a/packages/core/src/admin/middleware/Auth.middleware.ts b/packages/core/src/admin/middleware/Auth.middleware.ts index 4c883244d..410ea0fd6 100644 --- a/packages/core/src/admin/middleware/Auth.middleware.ts +++ b/packages/core/src/admin/middleware/Auth.middleware.ts @@ -7,6 +7,8 @@ import { isDev } from '../utils/middleware.js'; import { ConduitRequest } from '@conduitplatform/hermes'; import { gql } from 'graphql-tag'; import { ConfigController } from '@conduitplatform/module-tools'; +import { isSocketHandshake } from '../realtime/handshake.js'; +import { isRealtimeTicket } from '../realtime/ticket.js'; const excludedRestRoutes = ['/ready', '/live', '/login', '/config/modules']; const excludedGqlOperations = [ @@ -113,6 +115,10 @@ async function handleJwtToken( } const { id } = decoded; + if (isRealtimeTicket(decoded) && !isSocketHandshake(req)) { + res.status(401).json({ error: 'Realtime ticket cannot be used for HTTP requests' }); + return; + } if (decoded.twoFaRequired && req.path !== '/verify-twofa') { res.status(401).json({ error: 'Two FA required' }); return; diff --git a/packages/core/src/admin/realtime/handshake.test.ts b/packages/core/src/admin/realtime/handshake.test.ts new file mode 100644 index 000000000..59d17547f --- /dev/null +++ b/packages/core/src/admin/realtime/handshake.test.ts @@ -0,0 +1,12 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { isSocketHandshake } from './handshake.js'; + +describe('isSocketHandshake', () => { + it('detects Socket.IO engine requests', () => { + assert.equal(isSocketHandshake({ url: '/realtime/?EIO=4&transport=polling' }), true); + assert.equal(isSocketHandshake({ originalUrl: '/realtime' }), true); + assert.equal(isSocketHandshake({ path: '/login' }), false); + assert.equal(isSocketHandshake({ url: '/graphql' }), false); + }); +}); diff --git a/packages/core/src/admin/realtime/handshake.ts b/packages/core/src/admin/realtime/handshake.ts new file mode 100644 index 000000000..a4bf06fe1 --- /dev/null +++ b/packages/core/src/admin/realtime/handshake.ts @@ -0,0 +1,14 @@ +export function isSocketHandshake(req: { + url?: string; + originalUrl?: string; + path?: string; +}): boolean { + const parts = [req.url, req.originalUrl, req.path].filter((value): value is string => + Boolean(value), + ); + return parts.some(part => part.includes('EIO=') || isRealtimePath(part)); +} + +function isRealtimePath(value: string): boolean { + return /(?:^|[/?])realtime(?:[/?]|$)/.test(value); +} diff --git a/packages/core/src/admin/realtime/namespace.test.ts b/packages/core/src/admin/realtime/namespace.test.ts new file mode 100644 index 000000000..6bfcb3b64 --- /dev/null +++ b/packages/core/src/admin/realtime/namespace.test.ts @@ -0,0 +1,11 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { adminSocketNamespace } from './namespace.js'; + +describe('adminSocketNamespace', () => { + it('mirrors the Router module namespace contract', () => { + assert.equal(adminSocketNamespace('database'), '/database/'); + assert.throws(() => adminSocketNamespace(''), /module-name/); + assert.throws(() => adminSocketNamespace(undefined), /module-name/); + }); +}); diff --git a/packages/core/src/admin/realtime/namespace.ts b/packages/core/src/admin/realtime/namespace.ts new file mode 100644 index 000000000..ee66e9092 --- /dev/null +++ b/packages/core/src/admin/realtime/namespace.ts @@ -0,0 +1,7 @@ +export function adminSocketNamespace(moduleName: string | undefined | null): string { + const name = typeof moduleName === 'string' ? moduleName.trim() : ''; + if (!name) { + throw new Error('module-name metadata is required for Admin socket push'); + } + return `/${name}/`; +} diff --git a/packages/core/src/admin/realtime/ticket.test.ts b/packages/core/src/admin/realtime/ticket.test.ts new file mode 100644 index 000000000..e0d66fff1 --- /dev/null +++ b/packages/core/src/admin/realtime/ticket.test.ts @@ -0,0 +1,26 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { + ADMIN_REALTIME_AUDIENCE, + buildRealtimeTicketClaims, + isRealtimeTicket, +} from './ticket.js'; + +describe('admin realtime tickets', () => { + it('scopes claims to the realtime audience', () => { + const claims = buildRealtimeTicketClaims('admin-1'); + assert.equal(claims.id, 'admin-1'); + assert.equal(claims.aud, ADMIN_REALTIME_AUDIENCE); + assert.equal(isRealtimeTicket(claims), true); + }); + + it('rejects ordinary admin JWTs and missing payloads', () => { + assert.equal(isRealtimeTicket({ id: 'admin-1' }), false); + assert.equal(isRealtimeTicket({ id: 'admin-1', aud: 'other' }), false); + assert.equal(isRealtimeTicket(null), false); + assert.equal( + isRealtimeTicket({ id: 'admin-1', aud: [ADMIN_REALTIME_AUDIENCE, 'extra'] }), + true, + ); + }); +}); diff --git a/packages/core/src/admin/realtime/ticket.ts b/packages/core/src/admin/realtime/ticket.ts new file mode 100644 index 000000000..154630c0c --- /dev/null +++ b/packages/core/src/admin/realtime/ticket.ts @@ -0,0 +1,26 @@ +export const ADMIN_REALTIME_AUDIENCE = 'admin-realtime'; +export const ADMIN_REALTIME_TICKET_TTL_SECONDS = 30; + +export type RealtimeTicketClaims = { + id: string; + aud?: string | string[]; + twoFaRequired?: boolean; +}; + +export function buildRealtimeTicketClaims(adminId: string): RealtimeTicketClaims { + return { + id: adminId, + aud: ADMIN_REALTIME_AUDIENCE, + }; +} + +export function isRealtimeTicket( + decoded: RealtimeTicketClaims | null | undefined, +): boolean { + if (!decoded) return false; + const audience = decoded.aud; + if (Array.isArray(audience)) { + return audience.includes(ADMIN_REALTIME_AUDIENCE); + } + return audience === ADMIN_REALTIME_AUDIENCE; +} diff --git a/packages/core/src/admin/routes/RealtimeTicket.route.ts b/packages/core/src/admin/routes/RealtimeTicket.route.ts new file mode 100644 index 000000000..4a1b8e11e --- /dev/null +++ b/packages/core/src/admin/routes/RealtimeTicket.route.ts @@ -0,0 +1,51 @@ +import { isNil } from 'lodash-es'; +import { ConduitRoute } from '@conduitplatform/hermes'; +import { + ConduitError, + ConduitRouteActions, + ConduitRouteParameters, + ConduitRouteReturnDefinition, +} from '@conduitplatform/grpc-sdk'; +import { + ConduitNumber, + ConduitString, + ConfigController, +} from '@conduitplatform/module-tools'; +import { signToken } from '../utils/auth.js'; +import { + ADMIN_REALTIME_TICKET_TTL_SECONDS, + buildRealtimeTicketClaims, +} from '../realtime/ticket.js'; + +export function getRealtimeTicketRoute() { + return new ConduitRoute( + { + path: '/realtime/ticket', + action: ConduitRouteActions.POST, + mcp: false, + description: + 'Issue a short-lived token for Admin Socket.IO handshakes. The token cannot be used for REST or GraphQL.', + }, + new ConduitRouteReturnDefinition('RealtimeTicket', { + token: ConduitString.Required, + expiresIn: ConduitNumber.Required, + }), + async (req: ConduitRouteParameters) => { + const admin = req.context?.admin; + if (isNil(admin) || isNil(admin._id)) { + throw new ConduitError('UNAUTHORIZED', 401, 'Authentication required'); + } + const adminId = admin._id.toString(); + const authConfig = ConfigController.getInstance().config.auth; + const token = signToken( + buildRealtimeTicketClaims(adminId), + authConfig.tokenSecret, + ADMIN_REALTIME_TICKET_TTL_SECONDS, + ); + return { + token, + expiresIn: ADMIN_REALTIME_TICKET_TTL_SECONDS, + }; + }, + ); +} diff --git a/packages/core/src/admin/routes/index.ts b/packages/core/src/admin/routes/index.ts index 9bed86a70..f7c422e2c 100644 --- a/packages/core/src/admin/routes/index.ts +++ b/packages/core/src/admin/routes/index.ts @@ -20,3 +20,4 @@ export * from './ToggleTwoFa.route.js'; export * from './VerifyQrCode.route.js'; export * from './VerifyTwoFa.route.js'; export * from './ApiTokens.route.js'; +export * from './RealtimeTicket.route.js'; diff --git a/packages/core/src/core.proto b/packages/core/src/core.proto index 527d24f35..858dfa406 100644 --- a/packages/core/src/core.proto +++ b/packages/core/src/core.proto @@ -21,6 +21,14 @@ service Config { service Admin { rpc RegisterAdminRoute (RegisterAdminRouteRequest) returns (google.protobuf.Empty); rpc PatchRouteMiddlewares(PatchRouteMiddlewaresRequest) returns (google.protobuf.Empty); + rpc SocketPush (SocketPushRequest) returns (google.protobuf.Empty); +} + +message SocketPushRequest { + string event = 1; + optional string data = 2; + repeated string receivers = 3; + repeated string rooms = 4; } message RegisterAdminRouteRequest { diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index f4c4ef779..eb15cc5a9 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -67,5 +67,5 @@ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ }, "include": ["src/**/*"], - "exclude": ["tests", "node_modules", "dist", "bundle", "tsup.config.ts"] + "exclude": ["tests", "node_modules", "dist", "bundle", "tsup.config.ts", "**/*.test.ts"] } diff --git a/packages/core/tsconfig.test.json b/packages/core/tsconfig.test.json new file mode 100644 index 000000000..b0548057f --- /dev/null +++ b/packages/core/tsconfig.test.json @@ -0,0 +1,17 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": [ + "src/admin/realtime/ticket.ts", + "src/admin/realtime/handshake.ts", + "src/admin/realtime/namespace.ts", + "src/admin/realtime/*.test.ts" + ], + "exclude": ["node_modules", "dist", "bundle"] +} From 29bd4ce1e8ac88ec3abfabcf097b74e2e8d62748 Mon Sep 17 00:00:00 2001 From: John Chantzigoulas Date: Wed, 9 Sep 2026 12:04:48 +0300 Subject: [PATCH 2/8] fix(docker): initiate local Mongo replica set before Conduit starts Mongo 4.4 keyfiles reject hyphens, and rs.status() returns ok:0 instead of throwing, so Compose never elected a primary for replicaSet URIs. --- docker/docker-compose.standalone.yml | 8 +++++-- docker/docker-compose.yml | 19 +++++++++++++---- docker/mongo/init-replica.sh | 32 ++++++++++++++++++++-------- docker/mongo/keyfile | 2 +- 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/docker/docker-compose.standalone.yml b/docker/docker-compose.standalone.yml index e1e5225be..1c695c107 100644 --- a/docker/docker-compose.standalone.yml +++ b/docker/docker-compose.standalone.yml @@ -15,8 +15,12 @@ services: image: 'docker.io/conduitplatform/conduit-standalone:${IMAGE_TAG}' restart: unless-stopped depends_on: - - redis - - mongodb + redis: + condition: service_started + mongodb: + condition: service_started + mongo-init-replica: + condition: service_completed_successfully ports: - '${CORE_GRPC_PORT:-55152}:55152' - '${DB_GRPC_PORT:-55160}:55160' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 751ea8bc6..7131604e3 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -71,10 +71,21 @@ services: image: 'docker.io/conduitplatform/database:${IMAGE_TAG}' restart: unless-stopped depends_on: - - core - - ${DB_TYPE:-mongodb} - - prometheus - - loki + core: + condition: service_started + prometheus: + condition: service_started + loki: + condition: service_started + mongodb: + condition: service_started + required: false + postgres: + condition: service_started + required: false + mongo-init-replica: + condition: service_completed_successfully + required: false ports: - '${DB_GRPC_PORT:-55160}:${DB_GRPC_PORT:-55160}' environment: diff --git a/docker/mongo/init-replica.sh b/docker/mongo/init-replica.sh index e4bb536d2..f94f5dbf4 100644 --- a/docker/mongo/init-replica.sh +++ b/docker/mongo/init-replica.sh @@ -4,17 +4,31 @@ HOST="${MONGO_HOST:-conduit-mongo}" USER="${MONGO_INITDB_ROOT_USERNAME:-conduit}" PASS="${MONGO_INITDB_ROOT_PASSWORD:-pass}" -until mongo --host "$HOST" -u "$USER" -p "$PASS" --authenticationDatabase admin --quiet --eval 'db.adminCommand({ ping: 1 })' >/dev/null 2>&1; do +mongo_eval() { + mongo --host "$HOST" -u "$USER" -p "$PASS" --authenticationDatabase admin --quiet --eval "$1" +} + +until mongo_eval 'db.adminCommand({ ping: 1 })' >/dev/null 2>&1; do sleep 2 done -mongo --host "$HOST" -u "$USER" -p "$PASS" --authenticationDatabase admin --quiet --eval ' - try { - rs.status(); - } catch (err) { - rs.initiate({ - _id: "rs0", - members: [{ _id: 0, host: "'"$HOST"':27017" }] - }); +# rs.status() returns { ok: 0 } before initiate; it does not throw. +mongo_eval ' + var status = rs.status(); + if (status.ok === 1) { + quit(0); + } + var result = rs.initiate({ + _id: "rs0", + members: [{ _id: 0, host: "'"$HOST"':27017" }] + }); + if (result.ok !== 1) { + printjson(result); + quit(1); } ' + +# Mongoose with replicaSet=rs0 only selects a PRIMARY (myState === 1). +until mongo_eval 'var s = rs.status(); if (s.ok === 1 && s.myState === 1) { quit(0); } quit(1);' >/dev/null 2>&1; do + sleep 1 +done diff --git a/docker/mongo/keyfile b/docker/mongo/keyfile index 8d7ffaf78..056a007b0 100644 --- a/docker/mongo/keyfile +++ b/docker/mongo/keyfile @@ -1 +1 @@ -conduit-dev-mongo-replica-keyfile-do-not-use-in-production-replace-me-before-any-real-deployment-0123456789abcdefghijklmnopqrstuvwxyz +ConduitLocalDevMongoReplicaSetKeyFileDoNotUseInProductionReplaceBeforeAnyRealDeployment0123456789abcdefghijklmnopqrstuvwxyz From 8f14b2f9d2ad09fa3ddd443ebcecc3effa58bc45 Mon Sep 17 00:00:00 2001 From: John Chantzigoulas Date: Wed, 9 Sep 2026 15:07:33 +0300 Subject: [PATCH 3/8] fix: isolate socket streams and persist realtime schema options Stop duplicate admin/router deliveries and apply handshake auth on connect. Retry change-stream leadership after lock races. --- libraries/hermes/package.json | 1 + libraries/hermes/src/Socket/Socket.ts | 33 ++++++---- .../applySocketGlobalMiddlewares.test.ts | 44 +++++++++++++ .../Socket/applySocketGlobalMiddlewares.ts | 66 +++++++++++++++++++ libraries/hermes/src/index.ts | 21 ++++++ libraries/hermes/tsconfig.json | 3 +- libraries/hermes/tsconfig.test.json | 16 +++++ .../__tests__/schema.admin.put-patch.test.ts | 15 +++++ .../realtime/MongoChangeStreamCoordinator.ts | 25 +++++-- .../realtime/__tests__/coordinator.test.ts | 31 ++++++++- .../__tests__/validate-model-options.test.ts | 37 +++++++++++ modules/database/src/utils/utilities.ts | 5 +- 12 files changed, 275 insertions(+), 22 deletions(-) create mode 100644 libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts create mode 100644 libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts create mode 100644 libraries/hermes/tsconfig.test.json create mode 100644 modules/database/src/utils/__tests__/validate-model-options.test.ts diff --git a/libraries/hermes/package.json b/libraries/hermes/package.json index 62f32ab5a..bfe74ff95 100644 --- a/libraries/hermes/package.json +++ b/libraries/hermes/package.json @@ -11,6 +11,7 @@ "scripts": { "prepublish": "npm run build", "build": "rimraf dist && tsc", + "test": "tsc -p tsconfig.test.json && node --test dist-test/Socket/applySocketGlobalMiddlewares.test.js", "publish": "npm publish", "postbuild": "copyfiles -u 1 src/*.proto src/**/*.json ./dist/" }, diff --git a/libraries/hermes/src/Socket/Socket.ts b/libraries/hermes/src/Socket/Socket.ts index 74ced4e2b..937ead5b9 100644 --- a/libraries/hermes/src/Socket/Socket.ts +++ b/libraries/hermes/src/Socket/Socket.ts @@ -15,6 +15,7 @@ import { } from '../interfaces/index.js'; import ObjectHash from 'object-hash'; import { ConduitError, ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { applySocketGlobalMiddlewares } from './applySocketGlobalMiddlewares.js'; import { buildSocketMiddlewareParams } from './buildSocketMiddlewareParams.js'; export class SocketController extends ConduitRouter { @@ -57,9 +58,17 @@ export class SocketController extends ConduitRouter { }; this.io = new IOServer(this.httpServer, this.options); this.redisClient = grpcSdk.redisManager.getClient(); + // Admin (e.g. :3031) and Router (e.g. :3001) are separate Socket.IO + // servers that share Redis. The adapter defaults would put both on the + // same stream, so a database change pushed to admin *and* router is + // delivered twice to every socket in those rooms. + const adapterKey = `socket.io:${this.port}`; this.io.adapter( createAdapter(this.redisClient, { onlyPlaintext: true, + streamName: adapterKey, + channelPrefix: adapterKey, + sessionKeyPrefix: `sio:session:${this.port}:`, }), ); this.httpServer.listen(this.port); @@ -107,20 +116,18 @@ export class SocketController extends ConduitRouter { this._registeredNamespaces.set(namespace, conduitSocket); const self = this; - this.globalMiddlewares.forEach(middleware => { - self.io.engine.use((req: any, res: any, next: any) => { - req.path = namespace; - middleware(req, res, next); - }); - }); this.io.of(namespace).use((socket, next) => { - const context = buildSocketMiddlewareParams(socket); - self - .checkMiddlewares(context, conduitSocket.input.middlewares) - .then(r => { - Object.assign(context.context, r); - socket.data = context.context; - next(); + applySocketGlobalMiddlewares(socket, self.globalMiddlewares) + .then(() => { + const context = buildSocketMiddlewareParams(socket); + Object.assign(context.context, socket.data); + return self + .checkMiddlewares(context, conduitSocket.input.middlewares) + .then(r => { + Object.assign(context.context, r); + socket.data = context.context; + next(); + }); }) .catch((err: Error | ConduitError) => { next(err); diff --git a/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts b/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts new file mode 100644 index 000000000..3e2834c0e --- /dev/null +++ b/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts @@ -0,0 +1,44 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import type { NextFunction, Request, Response } from 'express'; +import { applySocketGlobalMiddlewares } from './applySocketGlobalMiddlewares.js'; + +function fakeSocket(auth?: Record) { + return { + request: { + headers: {}, + url: '/realtime/?EIO=4&transport=polling', + }, + handshake: { auth: auth ?? {} }, + data: {}, + }; +} + +describe('applySocketGlobalMiddlewares', () => { + it('injects handshake.auth.token and copies conduit onto socket.data', async () => { + const socket = fakeSocket({ token: 'ticket-token' }); + await applySocketGlobalMiddlewares(socket as never, [ + (req: Request, _res: Response, next: NextFunction) => { + assert.equal(req.headers.authorization, 'Bearer ticket-token'); + (req as Request & { conduit: { admin: string } }).conduit = { + admin: 'ok', + }; + next(); + }, + ]); + assert.equal((socket.data as { admin?: string }).admin, 'ok'); + }); + + it('rejects when Express middleware sends 401 json', async () => { + const socket = fakeSocket(); + await assert.rejects( + () => + applySocketGlobalMiddlewares(socket as never, [ + (_req: Request, res: Response) => { + res.status(401).json({ error: 'No token provided' }); + }, + ]), + /No token provided/, + ); + }); +}); diff --git a/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts b/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts new file mode 100644 index 000000000..c3f5bb8a8 --- /dev/null +++ b/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts @@ -0,0 +1,66 @@ +import type { NextFunction, Request, Response } from 'express'; +import type { Socket } from 'socket.io'; +import { buildSocketMiddlewareParams } from './buildSocketMiddlewareParams.js'; + +type ExpressMiddleware = (req: Request, res: Response, next: NextFunction) => void; + +type ConduitRequest = Request & { conduit?: Record }; + +export async function applySocketGlobalMiddlewares( + socket: Socket, + middlewares: ExpressMiddleware[], +): Promise { + const params = buildSocketMiddlewareParams(socket); + const req = Object.assign(socket.request, { + headers: params.headers, + path: '/realtime', + url: socket.request.url ?? '/realtime', + originalUrl: socket.request.url ?? '/realtime', + conduit: { ...params.context }, + }) as ConduitRequest; + + await new Promise((resolve, reject) => { + let settled = false; + const res = { + statusCode: 200, + status(this: { statusCode: number }, code: number) { + this.statusCode = code; + return this; + }, + json(this: unknown, body: { error?: string }) { + if (settled) return this; + settled = true; + reject(new Error(body?.error || 'Unauthorized')); + return this; + }, + setHeader() { + return this; + }, + removeHeader() {}, + } as unknown as Response; + + let index = 0; + const next = (err?: unknown) => { + if (settled) return; + if (err) { + settled = true; + reject(err instanceof Error ? err : new Error(String(err))); + return; + } + const middleware = middlewares[index++]; + if (!middleware) { + settled = true; + socket.data = { ...socket.data, ...(req.conduit ?? {}) }; + resolve(); + return; + } + try { + middleware(req, res, next); + } catch (error) { + settled = true; + reject(error as Error); + } + }; + next(); + }); +} diff --git a/libraries/hermes/src/index.ts b/libraries/hermes/src/index.ts index ab17086b2..ac22e28ba 100644 --- a/libraries/hermes/src/index.ts +++ b/libraries/hermes/src/index.ts @@ -47,6 +47,15 @@ export class ConduitRoutingController { private _cleanupTimeout: NodeJS.Timeout | null = null; /** Routes registered before MCP starts; replayed in initMCP. */ private readonly _conduitRoutesByKey: Map = new Map(); + /** Sockets registered before Socket.IO starts; replayed in initSockets. */ + private readonly _conduitSocketsByPath: Map = new Map(); + private readonly _socketMiddlewares: Array< + (req: ConduitRequest, res: Response, next: NextFunction) => void + > = []; + private readonly _socketRouteMiddlewares: Array<{ + middleware: ConduitMiddleware; + moduleUrl: string; + }> = []; private readonly routeTrie: RouteTrie = new RouteTrie(); readonly expressApp: Express = express(); readonly server = http.createServer(this.expressApp); @@ -143,6 +152,15 @@ export class ConduitRoutingController { this.expressApp, this.metrics, ); + for (const middleware of this._socketMiddlewares) { + this._socketRouter.registerGlobalMiddleware(middleware); + } + for (const { middleware, moduleUrl } of this._socketRouteMiddlewares) { + this._socketRouter.registerMiddleware(middleware, moduleUrl); + } + for (const socket of this._conduitSocketsByPath.values()) { + this._socketRouter.registerConduitSocket(socket); + } } initMCP(config?: { @@ -231,6 +249,7 @@ export class ConduitRoutingController { ) { this._middlewareRouter.use(middleware); if (socketMiddleware) { + this._socketMiddlewares.push(middleware); this._socketRouter?.registerGlobalMiddleware(middleware); } } @@ -238,6 +257,7 @@ export class ConduitRoutingController { registerRouteMiddleware(middleware: ConduitMiddleware, moduleUrl: string) { this._restRouter?.registerMiddleware(middleware, moduleUrl); this._graphQLRouter?.registerMiddleware(middleware, moduleUrl); + this._socketRouteMiddlewares.push({ middleware, moduleUrl }); this._socketRouter?.registerMiddleware(middleware, moduleUrl); this._mcpRouter?.registerMiddleware(middleware, moduleUrl); } @@ -286,6 +306,7 @@ export class ConduitRoutingController { } registerConduitSocket(socket: ConduitSocket) { + this._conduitSocketsByPath.set(socket.input.path, socket); this._socketRouter?.registerConduitSocket(socket); } diff --git a/libraries/hermes/tsconfig.json b/libraries/hermes/tsconfig.json index cd32b1ca8..0420a284b 100644 --- a/libraries/hermes/tsconfig.json +++ b/libraries/hermes/tsconfig.json @@ -65,5 +65,6 @@ /* Advanced Options */ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ - } + }, + "exclude": ["node_modules", "dist", "**/*.test.ts"] } diff --git a/libraries/hermes/tsconfig.test.json b/libraries/hermes/tsconfig.test.json new file mode 100644 index 000000000..8c2d1a100 --- /dev/null +++ b/libraries/hermes/tsconfig.test.json @@ -0,0 +1,16 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": [ + "src/Socket/applySocketGlobalMiddlewares.ts", + "src/Socket/applySocketGlobalMiddlewares.test.ts", + "src/Socket/buildSocketMiddlewareParams.ts" + ], + "exclude": ["node_modules", "dist"] +} diff --git a/modules/database/src/admin/__tests__/schema.admin.put-patch.test.ts b/modules/database/src/admin/__tests__/schema.admin.put-patch.test.ts index 89a7ae8ea..464bf4462 100644 --- a/modules/database/src/admin/__tests__/schema.admin.put-patch.test.ts +++ b/modules/database/src/admin/__tests__/schema.admin.put-patch.test.ts @@ -185,6 +185,21 @@ describe('SchemaAdmin PUT/PATCH split', () => { warnSpy.mockRestore(); }); + it('persists realtime conduitOptions used by live document updates', async () => { + const requestedSchema = makeRequestedSchema(); + const { admin, createSchema } = setup(requestedSchema); + + await admin.patchSchema( + makeCall({ + id: 'schema-1', + conduitOptions: { realtime: { enabled: true } }, + }), + ); + + const [writtenSchema] = createSchema.mock.calls[0]; + expect(writtenSchema.modelOptions.conduit.realtime.enabled).toBe(true); + }); + it('wipes existing documents when enabling authorization via conduitOptions', async () => { const requestedSchema = makeRequestedSchema(); const { admin, deleteMany } = setup(requestedSchema); diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts index 32ff41dc7..7b82797dd 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -55,6 +55,7 @@ export class MongoChangeStreamCoordinator { private topology: TopologyResult = { supported: false }; private retryAttempt = 0; private watching = false; + private opening = false; private ignoreClose = false; constructor(private readonly options: CoordinatorOptions) {} @@ -90,6 +91,14 @@ export class MongoChangeStreamCoordinator { await this.releaseLeader(); this.streamState = 'idle'; this.lastError = this.topology.message; + // Hello can fail during startup before Mongo is ready. Keep retrying + // that case; a confirmed standalone topology will not recover. + if ( + !this.topology.message || + this.topology.message.includes('Unable to determine') + ) { + this.scheduleRetry(); + } return; } if (this.options.getOptedInSchemas().length === 0) { @@ -121,7 +130,11 @@ export class MongoChangeStreamCoordinator { LOCK_TTL_MS, ); if (!acquired) { + // Another instance holds the lock, or a crashed holder has not + // expired yet. Without a retry, a standalone process stays idle + // forever after a restart races the previous TTL. this.streamState = 'idle'; + this.scheduleRetry(); return; } this.lock = acquired; @@ -153,13 +166,15 @@ export class MongoChangeStreamCoordinator { } private async openStream() { - if (this.watching || this.closed) return; + if (this.watching || this.closed || this.opening) return; + this.opening = true; this.streamState = 'starting'; this.ignoreClose = false; - const resumeAfter = parseResumeToken( - (await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY)) ?? undefined, - ); try { + const resumeAfter = parseResumeToken( + (await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY)) ?? undefined, + ); + if (this.watching || this.closed) return; const stream = this.options.watch({ resumeAfter: resumeAfter ?? undefined }); this.stream = stream; this.watching = true; @@ -180,6 +195,8 @@ export class MongoChangeStreamCoordinator { } catch (err) { this.watching = false; await this.handleStreamError(err); + } finally { + this.opening = false; } } diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index 5ec3a4af3..a523919b7 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -29,6 +29,7 @@ class MemoryStore { function createCoordinator(overrides?: { allow?: boolean; schemas?: { name: string; collectionName: string; authorizationEnabled: boolean }[]; + getKeyDelayMs?: number; }) { const stream = new EventEmitter() as EventEmitter & { close: () => Promise }; stream.close = async () => { @@ -42,12 +43,18 @@ function createCoordinator(overrides?: { const routerPush = jest.fn(async () => undefined); const adminPush = jest.fn(async () => undefined); const publish = jest.fn(); + const watch = jest.fn(() => stream as never); const subscriptions = new RealtimeSubscriptionTracker(new MemoryStore()); const grpcSdk = { state: { tryAcquireLock: jest.fn(async () => lock), releaseLock: jest.fn(async () => undefined), - getKey: jest.fn(async (key: string) => state.get(key) ?? null), + getKey: jest.fn(async (key: string) => { + if (overrides?.getKeyDelayMs) { + await new Promise(resolve => setTimeout(resolve, overrides.getKeyDelayMs)); + } + return state.get(key) ?? null; + }), setKey: jest.fn(async (key: string, value: string) => { state.set(key, value); }), @@ -65,7 +72,7 @@ function createCoordinator(overrides?: { }; const coordinator = new MongoChangeStreamCoordinator({ grpcSdk: grpcSdk as never, - watch: () => stream as never, + watch, hello: async () => ({ setName: 'rs0' }), getOptedInSchemas: () => overrides?.schemas ?? [ @@ -78,6 +85,7 @@ function createCoordinator(overrides?: { return { coordinator, stream, + watch, routerPush, adminPush, publish, @@ -155,6 +163,25 @@ describe('MongoChangeStreamCoordinator', () => { await coordinator.shutdown(); }); + it('retries later when the leader lock is held by another instance', async () => { + jest.useFakeTimers(); + const { coordinator, grpcSdk, lock } = createCoordinator(); + grpcSdk.state.tryAcquireLock.mockResolvedValueOnce(null).mockResolvedValue(lock); + await coordinator.reconcile(); + expect(coordinator.getState()).toBe('idle'); + await jest.advanceTimersByTimeAsync(1_000); + expect(coordinator.getState()).toBe('live'); + await coordinator.shutdown(); + jest.useRealTimers(); + }); + + it('opens a single watch when reconcile runs concurrently', async () => { + const { coordinator, watch } = createCoordinator({ getKeyDelayMs: 40 }); + await Promise.all([coordinator.reconcile(), coordinator.reconcile()]); + expect(watch).toHaveBeenCalledTimes(1); + await coordinator.shutdown(); + }); + it('clears an unusable resume token and retries', async () => { const { coordinator, stream, grpcSdk } = createCoordinator(); await coordinator.reconcile(); diff --git a/modules/database/src/utils/__tests__/validate-model-options.test.ts b/modules/database/src/utils/__tests__/validate-model-options.test.ts new file mode 100644 index 000000000..a832255e5 --- /dev/null +++ b/modules/database/src/utils/__tests__/validate-model-options.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from '@jest/globals'; +import { validateSchemaInput } from '../utilities.js'; + +const baseConduit = { + cms: { enabled: true }, + authorization: { enabled: false }, + permissions: { + extendable: true, + canCreate: true, + canModify: 'Everything' as const, + canDelete: true, + }, +}; + +describe('validateSchemaInput modelOptions.conduit', () => { + it('allows conduit.realtime used by live document updates', () => { + expect(() => + validateSchemaInput('TestSchema', undefined, { + conduit: { + ...baseConduit, + realtime: { enabled: true }, + }, + }), + ).not.toThrow(); + }); + + it('rejects an unknown conduit field', () => { + expect(() => + validateSchemaInput('TestSchema', undefined, { + conduit: { + ...baseConduit, + notARealField: true, + }, + }), + ).toThrow(/fields allowed inside 'conduit' field/); + }); +}); diff --git a/modules/database/src/utils/utilities.ts b/modules/database/src/utils/utilities.ts index ce6cd7328..f3e98a292 100644 --- a/modules/database/src/utils/utilities.ts +++ b/modules/database/src/utils/utilities.ts @@ -183,10 +183,11 @@ function validateModelOptions(modelOptions: ConduitSchemaOptions) { conduitKey !== 'cms' && conduitKey !== 'permissions' && conduitKey !== 'authorization' && - conduitKey !== 'imported' + conduitKey !== 'imported' && + conduitKey !== 'realtime' ) throw new Error( - "Only 'cms', 'permissions', 'authorization', 'imported', and 'readPreference' fields allowed inside 'conduit' field", + "Only 'cms', 'permissions', 'authorization', 'imported', 'readPreference', and 'realtime' fields allowed inside 'conduit' field", ); if (conduitKey === 'imported') { if (!isBoolean(modelOptions.conduit!.imported)) From 345841316eb63ce12730dd1e27e72563bbd99d66 Mon Sep 17 00:00:00 2001 From: John Chantzigoulas Date: Fri, 11 Sep 2026 14:15:50 +0300 Subject: [PATCH 4/8] refactor(database): drop unused realtime aliases and tautological tests --- .../realtime/MongoChangeStreamCoordinator.ts | 5 +- .../database/src/realtime/RealtimeService.ts | 4 +- .../src/realtime/__tests__/authorize.test.ts | 49 +++++++------------ .../realtime/__tests__/coordinator.test.ts | 5 +- .../src/realtime/__tests__/normalize.test.ts | 5 +- modules/database/src/realtime/authorize.ts | 13 +---- modules/database/src/realtime/sockets.ts | 32 +++++------- 7 files changed, 36 insertions(+), 77 deletions(-) diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts index 7b82797dd..b41cc2f89 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -40,7 +40,6 @@ export type CoordinatorOptions = { subscriptions: RealtimeSubscriptionTracker; enabled: () => boolean; engine: () => string; - socketsEnabled?: () => Promise; }; export class MongoChangeStreamCoordinator { @@ -172,10 +171,10 @@ export class MongoChangeStreamCoordinator { this.ignoreClose = false; try { const resumeAfter = parseResumeToken( - (await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY)) ?? undefined, + await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY), ); if (this.watching || this.closed) return; - const stream = this.options.watch({ resumeAfter: resumeAfter ?? undefined }); + const stream = this.options.watch({ resumeAfter }); this.stream = stream; this.watching = true; this.streamState = 'live'; diff --git a/modules/database/src/realtime/RealtimeService.ts b/modules/database/src/realtime/RealtimeService.ts index db6e77d21..e17e320ea 100644 --- a/modules/database/src/realtime/RealtimeService.ts +++ b/modules/database/src/realtime/RealtimeService.ts @@ -18,8 +18,7 @@ import { MongoChangeStreamCoordinator } from './MongoChangeStreamCoordinator.js' import { registerDatabaseRealtimeSocket } from './sockets.js'; import { buildRealtimeStatus } from './status.js'; import { RealtimeSubscriptionTracker } from './subscriptions.js'; -import type { OptedInSchema, RealtimeStatus } from './types.js'; -import type { ChangeStreamLike } from './types.js'; +import type { ChangeStreamLike, OptedInSchema, RealtimeStatus } from './types.js'; export class RealtimeService { private readonly subscriptions: RealtimeSubscriptionTracker; @@ -47,7 +46,6 @@ export class RealtimeService { subscriptions: this.subscriptions, enabled: () => this.isGloballyEnabled(), engine: () => adapter.getDatabaseType(), - socketsEnabled: () => this.areAdminSocketsEnabled(), }); } } diff --git a/modules/database/src/realtime/__tests__/authorize.test.ts b/modules/database/src/realtime/__tests__/authorize.test.ts index c441863eb..9c5eba15b 100644 --- a/modules/database/src/realtime/__tests__/authorize.test.ts +++ b/modules/database/src/realtime/__tests__/authorize.test.ts @@ -16,12 +16,9 @@ describe('realtime authorization helpers', () => { }); expect(requireSchemaName('Order')).toBe('Order'); expect(optionalDocumentId(undefined)).toBeUndefined(); - try { - requireSchemaName(''); - throw new Error('expected throw'); - } catch (err: any) { - expect(err.code).toBe(status.INVALID_ARGUMENT); - } + expect(() => requireSchemaName('')).toThrow( + expect.objectContaining({ code: status.INVALID_ARGUMENT }), + ); }); it('rejects missing schemas, disabled realtime, and CMS-read for clients', () => { @@ -43,24 +40,15 @@ describe('realtime authorization helpers', () => { }; }, }; - try { - assertSchemaAvailable(lookup, 'Missing', true); - throw new Error('expected throw'); - } catch (err: any) { - expect(err.code).toBe(status.NOT_FOUND); - } - try { - assertSchemaAvailable(lookup, 'Off', false); - throw new Error('expected throw'); - } catch (err: any) { - expect(err.code).toBe(status.FAILED_PRECONDITION); - } - try { - assertSchemaAvailable(lookup, 'Order', true); - throw new Error('expected throw'); - } catch (err: any) { - expect(err.code).toBe(status.PERMISSION_DENIED); - } + expect(() => assertSchemaAvailable(lookup, 'Missing', true)).toThrow( + expect.objectContaining({ code: status.NOT_FOUND }), + ); + expect(() => assertSchemaAvailable(lookup, 'Off', false)).toThrow( + expect.objectContaining({ code: status.FAILED_PRECONDITION }), + ); + expect(() => assertSchemaAvailable(lookup, 'Order', true)).toThrow( + expect.objectContaining({ code: status.PERMISSION_DENIED }), + ); const admin = assertSchemaAvailable(lookup, 'Order', false); expect(admin.authorizationEnabled).toBe(true); }); @@ -71,12 +59,11 @@ describe('realtime authorization helpers', () => { throw new GrpcError(status.NOT_FOUND, 'Schema Missing not defined yet'); }, }; - try { - assertSchemaAvailable(lookup, 'Missing', false); - throw new Error('expected throw'); - } catch (err: any) { - expect(err.code).toBe(status.NOT_FOUND); - expect(err.message).toBe('Schema does not exist'); - } + expect(() => assertSchemaAvailable(lookup, 'Missing', false)).toThrow( + expect.objectContaining({ + code: status.NOT_FOUND, + message: 'Schema does not exist', + }), + ); }); }); diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index a523919b7..efddfca70 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it, jest } from '@jest/globals'; import { ObjectId } from 'bson'; import { MongoChangeStreamCoordinator } from '../MongoChangeStreamCoordinator.js'; import { RealtimeSubscriptionTracker } from '../subscriptions.js'; -import { authorizedDocumentRoom, roomsForPublicChange } from '../rooms.js'; +import { roomsForPublicChange } from '../rooms.js'; class MemoryStore { private sets = new Map>(); @@ -157,9 +157,6 @@ describe('MongoChangeStreamCoordinator', () => { await new Promise(resolve => setImmediate(resolve)); expect(routerPush).not.toHaveBeenCalled(); expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c')).toEqual([]); - expect(authorizedDocumentRoom('Order', '64b64c4c4c4c4c4c4c4c4c', 'user-1')).toContain( - 'user-1', - ); await coordinator.shutdown(); }); diff --git a/modules/database/src/realtime/__tests__/normalize.test.ts b/modules/database/src/realtime/__tests__/normalize.test.ts index d51eaa549..db2278e29 100644 --- a/modules/database/src/realtime/__tests__/normalize.test.ts +++ b/modules/database/src/realtime/__tests__/normalize.test.ts @@ -9,6 +9,7 @@ describe('normalizeChangeEvent', () => { { operationType: 'insert', documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, + fullDocument: { secret: 'nope' }, wallTime: new Date('2026-01-01T00:00:00.000Z'), _id: resume, }, @@ -21,9 +22,7 @@ describe('normalizeChangeEvent', () => { documentId: '64b64c4c4c4c4c4c4c4c4c4c', occurredAt: '2026-01-01T00:00:00.000Z', }); - expect(event?.resumeToken).toEqual( - parseResumeToken(event!.resumeToken) ? event!.resumeToken : event?.resumeToken, - ); + expect(parseResumeToken(event!.resumeToken)).toEqual(resume); expect(JSON.parse(JSON.stringify(event))).not.toHaveProperty('fullDocument'); }); diff --git a/modules/database/src/realtime/authorize.ts b/modules/database/src/realtime/authorize.ts index 330992f13..cf7ed899c 100644 --- a/modules/database/src/realtime/authorize.ts +++ b/modules/database/src/realtime/authorize.ts @@ -35,13 +35,6 @@ export type SchemaLookup = { | undefined; }; -export type ResolvedSubscription = { - schema: string; - documentId?: string; - authorizationEnabled: boolean; - rooms: string[]; -}; - export function parseSubscribeRequest(params: unknown[]): SubscribeRequest { const raw = params[0]; if (raw && typeof raw === 'object' && !Array.isArray(raw)) { @@ -93,10 +86,7 @@ export function assertSchemaAvailable( lookup: SchemaLookup, schemaName: string, requireCmsRead: boolean, -): { - authorizationEnabled: boolean; - realtimeEnabled: boolean; -} { +): { authorizationEnabled: boolean } { let schema: ReturnType; try { schema = lookup.getSchema(schemaName); @@ -124,7 +114,6 @@ export function assertSchemaAvailable( } return { authorizationEnabled: conduit.authorization?.enabled === true, - realtimeEnabled: true, }; } diff --git a/modules/database/src/realtime/sockets.ts b/modules/database/src/realtime/sockets.ts index 5cc97957e..4be4e1018 100644 --- a/modules/database/src/realtime/sockets.ts +++ b/modules/database/src/realtime/sockets.ts @@ -22,15 +22,17 @@ import type { RealtimeSubscriptionTracker } from './subscriptions.js'; type SocketMode = 'client' | 'admin'; +type RealtimeSocketOptions = { + mode: SocketMode; + grpcSdk: ConduitGrpcSdk; + schemaLookup: SchemaLookup; + subscriptions: RealtimeSubscriptionTracker; + isGloballyEnabled: () => boolean; +}; + export function registerDatabaseRealtimeSocket( routingManager: RoutingManager, - options: { - mode: SocketMode; - grpcSdk: ConduitGrpcSdk; - schemaLookup: SchemaLookup; - subscriptions: RealtimeSubscriptionTracker; - isGloballyEnabled: () => boolean; - }, + options: RealtimeSocketOptions, ) { const handlers = createSocketHandlers(options); routingManager.socket( @@ -59,13 +61,7 @@ export function registerDatabaseRealtimeSocket( ); } -function createSocketHandlers(options: { - mode: SocketMode; - grpcSdk: ConduitGrpcSdk; - schemaLookup: SchemaLookup; - subscriptions: RealtimeSubscriptionTracker; - isGloballyEnabled: () => boolean; -}) { +function createSocketHandlers(options: RealtimeSocketOptions) { return { connect: async (): Promise => { return { event: 'connected', data: { ok: true } }; @@ -87,13 +83,7 @@ function createSocketHandlers(options: { async function resolveSubscription( call: ParsedSocketRequest, - options: { - mode: SocketMode; - grpcSdk: ConduitGrpcSdk; - schemaLookup: SchemaLookup; - subscriptions: RealtimeSubscriptionTracker; - isGloballyEnabled: () => boolean; - }, + options: RealtimeSocketOptions, action: 'join' | 'leave', ): Promise { if (!options.isGloballyEnabled()) { From bc32384069aa4f479d86f83379975d048d5a42b0 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 13 Sep 2026 17:01:11 +0300 Subject: [PATCH 5/8] fix(database): Mongo live-update interventions (watch, token, tickets, recovery) (#1605) * fix(router): Event Relays interventions (EventBus, Hermes, relay manager) (#1604) * fix(router): harden event relays for HA and auth lifetime EventBus uses a single Redis message dispatcher with subscriberId maps, SIGTERM/SIGINT shutdown, and subscribe-after-ACK. Hermes installs engine middleware once, skips empty-room namespace broadcasts, demotes hot-path logs, and re-authenticates recovered /events/ subscriptions. EventRelayManager gains coalesced periodic reconcile, compiled templates, inbound caps, TTL-cached emit-time ReBAC, room eviction, backpressure, preview API, and metrics/docs aligned with the interventions plan. * fix(router): address #1604 re-review P1 and P2 follow-ups Recovery re-auth keeps per-user subscription state across disconnect; emit-time ReBAC distinguishes unavailable vs deny; EventBus drops process signal handlers and adds subscribeAck; manager subscribes only after Redis ACK and broadcasts evictRelayIds on refresh; Hermes backpressure uses local sockets without emit acks; cache is bounded; preview caps sample JSON; handshake matcher rejects ticket/sid/non-namespace paths; socket middleware rebind avoids duplicate registration on first sockets enable. * fix(hermes): restore production tsc for engine middleware chain Type the Socket.IO engine middleware runner as Express NextFunction so recursive callbacks match registerGlobalMiddleware. Use Logger.info for socket trace lines (IConduitLogger has no debug). * fix(router): import Express Response for socket global middleware _rebindSocketGlobalMiddlewares passes handlers typed with fetch Response because Response was not imported from express, failing tsc against Hermes registerSocketGlobalMiddleware. * fix(router): quit EventBus on module shutdown signals Stop event relays and call grpcSdk.bus.quit() from Router.shutdown(), registered on SIGTERM/SIGINT with process exit so Redis teardown is not left to SDK signal handlers. Tighten registerGlobalMiddleware typing and trim handshake helper comment (deslop). * fix(router): pass-3 recovery, scoped emit, and subscription hygiene Recovery re-auth keeps membership on Authorization UNAVAILABLE and only leaves on deny; re-check subscriptions for restored er: rooms via room map. Prune user subscriptions on disconnect when no other socket holds them. Scope relay emits to receivers in the target room; use Engine.IO writeBuffer/writable for backpressure. Drop hot-path Hermes socket info logs. * fix(router): pass-4 recovery map, backpressure, emit tests Store relay subs on socket.data; resolve recovered rooms from context and TTL room map without last-writer userId. Prune room map when unused on disconnect. Backpressure uses writeBuffer depth only. Add Hermes/router tests for room-scoped emit and queue metric. * fix(router): clear eventRelaySubs from socket.data on recovery deny * fix(router): clear socket.data subs on recovery fail-closed leave * fix(router): address CodeFactor findings on event relay validation Split validateEventRelayInput into per-field parsers to reduce complexity. Silence unused-parameter lint in EventBus test FakeRedis stub. * fix(database): Mongo live-update interventions (watch, token, tickets, recovery) Rebase #1601 onto current Event Relays Hermes and close the stack-review P1s: watch $match/$project, serialized handleChange, persist resume token after successful emit, keep token on CursorKilled 237, hermes isSocketHandshake, restore /database/ Redis on recover, and shutdown the watch plus leader lock. Co-authored-by: Konstantinos Kopanidis * style: prettier on Mongo intervention tests * fix(database): stop watch on emit failure; keep ReBAC membership Failed socketPush/bus no longer advances the resume token. ReBAC unavailable no longer removeUser. Auth middleware 401s a realtime ticket on POST /realtime/ticket. Drop/rename/invalidate reopen the watch. --------- --- docker/docker-compose.standalone.yml | 2 + docker/docker-compose.yml | 2 + libraries/grpc-sdk/package.json | 3 +- .../grpc-sdk/src/utilities/EventBus.test.ts | 124 ++++++++ libraries/grpc-sdk/src/utilities/EventBus.ts | 156 +++++++--- libraries/grpc-sdk/tsconfig.test.json | 11 + libraries/hermes/package.json | 2 +- libraries/hermes/src/Socket/Socket.ts | 274 +++++++++++------ .../applySocketGlobalMiddlewares.test.ts | 44 --- .../Socket/applySocketGlobalMiddlewares.ts | 66 ---- .../src/Socket/isSocketHandshake.test.ts | 27 ++ .../hermes/src/Socket/isSocketHandshake.ts | 26 ++ .../src/Socket/resolveEngineNamespacePath.ts | 11 + .../hermes/src/Socket/socketPushUtils.test.ts | 69 +++++ .../hermes/src/Socket/socketPushUtils.ts | 36 +++ libraries/hermes/src/index.ts | 23 +- libraries/hermes/src/interfaces/Socket.ts | 14 + libraries/hermes/src/interfaces/SocketPush.ts | 4 + libraries/hermes/tsconfig.json | 2 +- libraries/hermes/tsconfig.test.json | 9 +- modules/database/README.mdx | 10 +- modules/database/src/Database.ts | 4 + modules/database/src/index.ts | 18 ++ .../realtime/MongoChangeStreamCoordinator.ts | 143 ++++++--- .../database/src/realtime/RealtimeService.ts | 12 +- .../change-stream.integration.test.ts | 3 +- .../realtime/__tests__/coordinator.test.ts | 288 ++++++++++++++++-- .../src/realtime/__tests__/normalize.test.ts | 9 +- .../src/realtime/__tests__/recovery.test.ts | 82 +++++ .../src/realtime/__tests__/rooms.test.ts | 8 + .../src/realtime/__tests__/topology.test.ts | 5 +- .../realtime/__tests__/watchPipeline.test.ts | 28 ++ modules/database/src/realtime/normalize.ts | 10 +- modules/database/src/realtime/rebacCache.ts | 96 ++++++ modules/database/src/realtime/recovery.ts | 147 +++++++++ modules/database/src/realtime/rooms.ts | 22 ++ modules/database/src/realtime/sockets.ts | 43 ++- modules/database/src/realtime/topology.ts | 11 +- modules/database/src/realtime/types.ts | 1 - .../database/src/realtime/watchPipeline.ts | 33 ++ modules/router/README.mdx | 25 +- modules/router/src/Router.ts | 50 ++- modules/router/src/admin/event-relays.ts | 34 ++- modules/router/src/admin/index.ts | 15 + .../src/event-relays/EventRelayManager.ts | 242 ++++++++++++--- .../src/event-relays/EventRelaySockets.ts | 68 ++++- modules/router/src/event-relays/authorize.ts | 2 - modules/router/src/event-relays/compile.ts | 50 +++ modules/router/src/event-relays/constants.ts | 6 + .../router/src/event-relays/follow-up.test.ts | 186 +++++++++++ .../src/event-relays/interventions.test.ts | 23 ++ .../router/src/event-relays/process.test.ts | 3 + modules/router/src/event-relays/process.ts | 24 +- modules/router/src/event-relays/push.ts | 13 +- .../src/event-relays/rebacCache.test.ts | 43 +++ modules/router/src/event-relays/rebacCache.ts | 108 +++++++ modules/router/src/event-relays/recovery.ts | 85 ++++++ .../src/event-relays/relaySocketData.ts | 65 ++++ modules/router/src/event-relays/rooms.ts | 4 + .../router/src/event-relays/subscriptions.ts | 224 ++++++++++++++ modules/router/src/event-relays/validation.ts | 127 +++++--- modules/router/src/index.ts | 18 ++ modules/router/src/metrics/index.ts | 35 +++ modules/router/tsconfig.test.json | 3 + .../src/admin/middleware/Admin.middleware.ts | 3 +- .../src/admin/middleware/Auth.middleware.ts | 5 +- .../core/src/admin/realtime/handshake.test.ts | 12 - packages/core/src/admin/realtime/handshake.ts | 14 - .../src/admin/realtime/ticket-http.test.ts | 97 ++++++ packages/core/src/admin/realtime/ticket.ts | 9 + packages/core/tsconfig.test.json | 1 - 71 files changed, 2996 insertions(+), 476 deletions(-) create mode 100644 libraries/grpc-sdk/src/utilities/EventBus.test.ts create mode 100644 libraries/grpc-sdk/tsconfig.test.json delete mode 100644 libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts delete mode 100644 libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts create mode 100644 libraries/hermes/src/Socket/isSocketHandshake.test.ts create mode 100644 libraries/hermes/src/Socket/isSocketHandshake.ts create mode 100644 libraries/hermes/src/Socket/resolveEngineNamespacePath.ts create mode 100644 libraries/hermes/src/Socket/socketPushUtils.test.ts create mode 100644 libraries/hermes/src/Socket/socketPushUtils.ts create mode 100644 modules/database/src/realtime/__tests__/recovery.test.ts create mode 100644 modules/database/src/realtime/__tests__/watchPipeline.test.ts create mode 100644 modules/database/src/realtime/rebacCache.ts create mode 100644 modules/database/src/realtime/recovery.ts create mode 100644 modules/database/src/realtime/watchPipeline.ts create mode 100644 modules/router/src/event-relays/compile.ts create mode 100644 modules/router/src/event-relays/follow-up.test.ts create mode 100644 modules/router/src/event-relays/interventions.test.ts create mode 100644 modules/router/src/event-relays/rebacCache.test.ts create mode 100644 modules/router/src/event-relays/rebacCache.ts create mode 100644 modules/router/src/event-relays/recovery.ts create mode 100644 modules/router/src/event-relays/relaySocketData.ts create mode 100644 modules/router/src/event-relays/subscriptions.ts delete mode 100644 packages/core/src/admin/realtime/handshake.test.ts delete mode 100644 packages/core/src/admin/realtime/handshake.ts create mode 100644 packages/core/src/admin/realtime/ticket-http.test.ts diff --git a/docker/docker-compose.standalone.yml b/docker/docker-compose.standalone.yml index 1c695c107..bd77d5c47 100644 --- a/docker/docker-compose.standalone.yml +++ b/docker/docker-compose.standalone.yml @@ -78,6 +78,8 @@ services: MONGO_INITDB_DATABASE: 'conduit' MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' + # Existing volumes created before rs0 will not elect a replica set. + # Remove the mongo volume (or start with an empty data dir) if hello stays standalone. entrypoint: - bash - -c diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7131604e3..7567df345 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -282,6 +282,8 @@ services: MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' profiles: ['mongodb'] + # Existing volumes created before rs0 will not elect a replica set. + # Remove the mongo volume (or start with an empty data dir) if hello stays standalone. entrypoint: - bash - -c diff --git a/libraries/grpc-sdk/package.json b/libraries/grpc-sdk/package.json index 25809a993..bbf717933 100644 --- a/libraries/grpc-sdk/package.json +++ b/libraries/grpc-sdk/package.json @@ -25,7 +25,8 @@ "prepublish": "npm run build", "prebuild": "npm run protoc", "build": "rimraf dist && tsup", - "protoc": "sh build.sh" + "protoc": "sh build.sh", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utilities/EventBus.test.js" }, "license": "MIT", "dependencies": { diff --git a/libraries/grpc-sdk/src/utilities/EventBus.test.ts b/libraries/grpc-sdk/src/utilities/EventBus.test.ts new file mode 100644 index 000000000..4d069c244 --- /dev/null +++ b/libraries/grpc-sdk/src/utilities/EventBus.test.ts @@ -0,0 +1,124 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { EventBus } from './EventBus.js'; + +type Listener = (channel: string, message: string) => void; + +class FakeRedis { + handlers: Record = {}; + subscribed = new Set(); + failNext = new Set(); + + on(event: string, listener: Listener) { + this.handlers[event] = this.handlers[event] ?? []; + this.handlers[event].push(listener); + } + + subscribe(channel: string, cb?: (err?: Error | null) => void) { + if (this.failNext.has(channel)) { + this.failNext.delete(channel); + cb?.(new Error('subscribe failed')); + return; + } + this.subscribed.add(channel); + cb?.(null); + } + + unsubscribe(channel: string, cb?: () => void) { + void channel; + cb?.(); + } + + publish(channel: string, message: string) { + void channel; + void message; + } + + quit() {} + + emitMessage(channel: string, message: string) { + for (const listener of this.handlers.message ?? []) { + listener(channel, message); + } + } +} + +function createBus() { + const sub = new FakeRedis(); + const pub = new FakeRedis(); + const manager = { + getClient: () => sub, + }; + const bus = new EventBus(manager as never); + (bus as unknown as { _clientSubscriber: FakeRedis })._clientSubscriber = sub; + (bus as unknown as { _clientPublisher: FakeRedis })._clientPublisher = pub; + return { bus, sub }; +} + +describe('EventBus', () => { + it('fires once after deactivate/reactivate on the same channel', () => { + const { bus, sub } = createBus(); + let count = 0; + bus.subscribe( + 'database:update:Order', + () => { + count += 1; + }, + 'relay-a', + ); + bus.unsubscribe('relay-a'); + bus.subscribe( + 'database:update:Order', + () => { + count += 1; + }, + 'relay-a', + ); + sub.emitMessage('database:update:Order', '{"ok":true}'); + assert.equal(count, 1); + }); + + it('keeps the second subscriber when the first is removed', () => { + const { bus, sub } = createBus(); + let first = 0; + let second = 0; + bus.subscribe( + 'chan', + () => { + first += 1; + }, + 'one', + ); + bus.subscribe( + 'chan', + () => { + second += 1; + }, + 'two', + ); + bus.unsubscribe('one'); + sub.emitMessage('chan', 'x'); + assert.equal(first, 0); + assert.equal(second, 1); + }); + + it('subscribeAck rejects when Redis subscribe fails', async () => { + const { bus, sub } = createBus(); + sub.failNext.add('chan'); + await assert.rejects( + () => bus.subscribeAck('chan', () => {}, 'relay-a'), + /subscribe failed/, + ); + sub.failNext.delete('chan'); + let count = 0; + await bus.subscribeAck( + 'chan', + () => { + count += 1; + }, + 'relay-a', + ); + sub.emitMessage('chan', 'x'); + assert.equal(count, 1); + }); +}); diff --git a/libraries/grpc-sdk/src/utilities/EventBus.ts b/libraries/grpc-sdk/src/utilities/EventBus.ts index 8cd8512f1..9388ad483 100644 --- a/libraries/grpc-sdk/src/utilities/EventBus.ts +++ b/libraries/grpc-sdk/src/utilities/EventBus.ts @@ -3,35 +3,58 @@ import { Cluster, Redis } from 'ioredis'; import crypto from 'crypto'; import { getLogger } from './GrpcSdkContext.js'; +type ChannelCallbacks = Map void>; + export class EventBus { private _clientSubscriber: Redis | Cluster; private _clientPublisher: Redis | Cluster; - private _subscribedChannels: { [listener: string]: ((message: string) => void)[] }; - private _subscribers: { [listener: string]: [string, number] }; + /** channelName -> subscriberId -> callback */ + private _channelCallbacks: Map; + /** subscriberId -> channelName */ + private _subscriberChannels: Map; + /** channels with a successful Redis SUBSCRIBE */ + private _redisSubscribedChannels: Set; + private _subscribeInFlight = new Map>(); private _signature: string; + private _anonymousSubscriberSeq = 0; + private _shuttingDown = false; constructor(redisManager: RedisManager) { - this._subscribedChannels = {}; - this._subscribers = {}; + this._channelCallbacks = new Map(); + this._subscriberChannels = new Map(); + this._redisSubscribedChannels = new Set(); this._clientSubscriber = redisManager.getClient({ keyPrefix: 'bus_' }); this._clientPublisher = redisManager.getClient({ keyPrefix: 'bus_' }); this._signature = crypto.randomBytes(20).toString('hex'); this._clientSubscriber.on('ready', () => { getLogger().log('The Bus is in the station...hehe'); }); + this._clientSubscriber.on('message', (channel: string, message: string) => { + this.dispatch(channel, message); + }); process.on('exit', () => { - this._clientSubscriber.quit(); - this._clientPublisher.quit(); + this.quit(); }); } + quit(): void { + if (this._shuttingDown) return; + this._shuttingDown = true; + void this._clientSubscriber.quit(); + void this._clientPublisher.quit(); + } + unsubscribe(subscriberId: string): void { - if (this._subscribers[subscriberId]) { - const [channelName, index] = this._subscribers[subscriberId]; - this._subscribedChannels[channelName].splice(index, 1); - delete this._subscribers[subscriberId]; - if (this._subscribedChannels[channelName].length === 0) { - delete this._subscribedChannels[channelName]; + const channelName = this._subscriberChannels.get(subscriberId); + if (!channelName) { + return; + } + const callbacks = this._channelCallbacks.get(channelName); + callbacks?.delete(subscriberId); + this._subscriberChannels.delete(subscriberId); + if (callbacks && callbacks.size === 0) { + this._channelCallbacks.delete(channelName); + if (this._redisSubscribedChannels.delete(channelName)) { this._clientSubscriber.unsubscribe(channelName, () => {}); } } @@ -42,43 +65,100 @@ export class EventBus { callback: (message: string) => void, subscriberId?: string, ): void { + void this.subscribeAck(channelName, callback, subscriberId).catch(err => { + getLogger().error( + `EventBus subscribe failed for ${channelName}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }); + } + + async subscribeAck( + channelName: string, + callback: (message: string) => void, + subscriberId?: string, + ): Promise { + if (this._shuttingDown) { + return; + } + const id = + subscriberId ?? + `anon:${channelName}:${++this._anonymousSubscriberSeq}:${crypto.randomBytes(4).toString('hex')}`; if (subscriberId) { - // if subscriberId is provided, and it is already subscribed, unsubscribe it first this.unsubscribe(subscriberId); } - if (this._subscribedChannels[channelName]) { - this._subscribedChannels[channelName].push(callback); - if (subscriberId) { - this._subscribers[subscriberId] = [ - channelName, - this._subscribedChannels[channelName].length - 1, - ]; - } + + let callbacks = this._channelCallbacks.get(channelName); + if (!callbacks) { + callbacks = new Map(); + this._channelCallbacks.set(channelName, callbacks); + } + callbacks.set(id, callback); + this._subscriberChannels.set(id, channelName); + + if (this._redisSubscribedChannels.has(channelName)) { return; } - this._subscribedChannels[channelName] = [callback]; - this._clientSubscriber.subscribe(channelName, () => {}); - const self = this; - this._clientSubscriber.on('message', (channel: string, message: string) => { - if (channel !== channelName) return; - // if the message supports the signature - if (message.indexOf('CND_Signature') !== -1) { - // if the message does not contain this module's signature - if (message.indexOf(self._signature) === -1) { - self._subscribedChannels[channelName].forEach(fn => { - fn(message.split('CND_Signature:')[0]); - }); - } - } else { - self._subscribedChannels[channelName].forEach(fn => { - fn(message); + + let inFlight = this._subscribeInFlight.get(channelName); + if (!inFlight) { + inFlight = new Promise((resolve, reject) => { + this._clientSubscriber.subscribe(channelName, err => { + if (err) { + reject(err); + return; + } + this._redisSubscribedChannels.add(channelName); + resolve(); }); + }).finally(() => { + this._subscribeInFlight.delete(channelName); + }); + this._subscribeInFlight.set(channelName, inFlight); + } + + try { + await inFlight; + } catch (err) { + if (!this._redisSubscribedChannels.has(channelName)) { + this.removeChannelCallbacks(channelName); } - }); + this.unsubscribe(id); + throw err; + } } publish(channelName: string, message: string) { message = message + `CND_Signature:${this._signature}`; this._clientPublisher.publish(channelName, message); } + + private removeChannelCallbacks(channelName: string): void { + const callbacks = this._channelCallbacks.get(channelName); + if (!callbacks) { + return; + } + for (const subId of callbacks.keys()) { + this._subscriberChannels.delete(subId); + } + this._channelCallbacks.delete(channelName); + } + + private dispatch(channel: string, message: string): void { + const callbacks = this._channelCallbacks.get(channel); + if (!callbacks || callbacks.size === 0) { + return; + } + let payload = message; + if (message.indexOf('CND_Signature') !== -1) { + if (message.indexOf(this._signature) !== -1) { + return; + } + payload = message.split('CND_Signature:')[0]; + } + for (const fn of callbacks.values()) { + fn(payload); + } + } } diff --git a/libraries/grpc-sdk/tsconfig.test.json b/libraries/grpc-sdk/tsconfig.test.json new file mode 100644 index 000000000..121161cd7 --- /dev/null +++ b/libraries/grpc-sdk/tsconfig.test.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": ["src/utilities/EventBus.ts", "src/utilities/EventBus.test.ts"] +} diff --git a/libraries/hermes/package.json b/libraries/hermes/package.json index bfe74ff95..3beffb157 100644 --- a/libraries/hermes/package.json +++ b/libraries/hermes/package.json @@ -11,7 +11,7 @@ "scripts": { "prepublish": "npm run build", "build": "rimraf dist && tsc", - "test": "tsc -p tsconfig.test.json && node --test dist-test/Socket/applySocketGlobalMiddlewares.test.js", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/Socket/*.test.js", "publish": "npm publish", "postbuild": "copyfiles -u 1 src/*.proto src/**/*.json ./dist/" }, diff --git a/libraries/hermes/src/Socket/Socket.ts b/libraries/hermes/src/Socket/Socket.ts index 937ead5b9..f1c0b47d1 100644 --- a/libraries/hermes/src/Socket/Socket.ts +++ b/libraries/hermes/src/Socket/Socket.ts @@ -15,8 +15,12 @@ import { } from '../interfaces/index.js'; import ObjectHash from 'object-hash'; import { ConduitError, ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; -import { applySocketGlobalMiddlewares } from './applySocketGlobalMiddlewares.js'; import { buildSocketMiddlewareParams } from './buildSocketMiddlewareParams.js'; +import { resolveEngineNamespacePath } from './resolveEngineNamespacePath.js'; +import { + filterRemoteSocketsByUserAndRooms, + isEngineSocketBackpressured, +} from './socketPushUtils.js'; export class SocketController extends ConduitRouter { private readonly httpServer: httpServer; @@ -87,6 +91,22 @@ export class SocketController extends ConduitRouter { `Socket connection error, context: ${err?.context ?? 'N/A'}`, ); }); + + this.io.engine.use((req: any, res: any, next: NextFunction) => { + req.path = resolveEngineNamespacePath(req); + let index = 0; + const run: NextFunction = err => { + if (err) { + return next(err); + } + const middleware = this.globalMiddlewares[index++]; + if (!middleware) { + return next(); + } + middleware(req, res, run); + }; + run(); + }); } registerGlobalMiddleware( @@ -117,17 +137,13 @@ export class SocketController extends ConduitRouter { const self = this; this.io.of(namespace).use((socket, next) => { - applySocketGlobalMiddlewares(socket, self.globalMiddlewares) - .then(() => { - const context = buildSocketMiddlewareParams(socket); - Object.assign(context.context, socket.data); - return self - .checkMiddlewares(context, conduitSocket.input.middlewares) - .then(r => { - Object.assign(context.context, r); - socket.data = context.context; - next(); - }); + const context = buildSocketMiddlewareParams(socket); + self + .checkMiddlewares(context, conduitSocket.input.middlewares) + .then(r => { + Object.assign(context.context, r); + socket.data = context.context; + next(); }) .catch((err: Error | ConduitError) => { next(err); @@ -136,13 +152,25 @@ export class SocketController extends ConduitRouter { this.io.of(namespace).on('connect', socket => { if (socket.recovered) { - ConduitGrpcSdk.Logger.info( - `Socket recovered: ${socket.id} to namespace: ${namespace}`, + const recoveredRooms = [...socket.rooms].filter( + room => room.startsWith('er:') || room.startsWith('database:'), ); + const recovered = conduitSocket.executeRecovered({ + event: 'recovered', + socketId: socket.id, + context: socket.data, + params: recoveredRooms, + recoveredRooms, + }); + if (recovered) { + recovered + .then(res => this.handleResponse(res, socket, namespace)) + .catch(e => { + ConduitGrpcSdk.Logger.error(e); + socket.emit('conduit_error', e); + }); + } } else { - ConduitGrpcSdk.Logger.info( - `Socket connected: ${socket.id} to namespace: ${namespace}`, - ); conduitSocket .executeRequest({ event: 'connect', @@ -157,7 +185,6 @@ export class SocketController extends ConduitRouter { } socket.onAny((event, ...args) => { - ConduitGrpcSdk.Logger.info(`Socket event: ${event} from socket: ${socket.id}`); conduitSocket .executeRequest({ event, @@ -172,15 +199,13 @@ export class SocketController extends ConduitRouter { }); }); - socket.on('disconnect', () => { - ConduitGrpcSdk.Logger.info( - `Socket disconnected: ${socket.id} from namespace: ${namespace}`, - ); + socket.on('disconnect', (reason: string) => { conduitSocket .executeRequest({ event: 'disconnect', socketId: socket.id, context: socket.data, + params: [reason], }) .then(res => this.handleResponse(res, socket, namespace)) .catch(e => { @@ -191,82 +216,162 @@ export class SocketController extends ConduitRouter { }); } - async handleSocketPush(push: SocketPush) { + async handleSocketPush(push: SocketPush): Promise { const localOnly = push.localOnly === true; if (push.event === 'join-room') { - if (push.rooms.length === 0) return; + if (push.rooms.length === 0) return false; const filteredSockets = await this.findAndFilterSockets( push.receivers, push.namespace, localOnly, ); for (const socket of filteredSockets) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} joining rooms: ${push.rooms.join(', ')} in namespace: ${ - push.namespace - }`, - ); socket.join(push.rooms); } + return true; } else if (push.event === 'leave-room') { if (push.rooms && push.rooms.length !== 0) { - const filteredSockets = await this.findAndFilterSockets( - push.receivers, - push.namespace, - localOnly, - ); + const filteredSockets = await this.socketsForRoomPush(push, localOnly); for (const socket of filteredSockets) { for (const room of push.rooms) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} leaving room: ${room} in namespace: ${push.namespace}`, - ); socket.leave(room); } } } + return true; } else if (isInstanceOfEventResponse(push)) { if ( (isNil(push.receivers) || push.receivers.length === 0) && push.rooms.length === 0 ) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${push.event} to all sockets in namespace: ${push.namespace}`, - ); + return false; + } + if (push.receivers.length !== 0) { const nsp = this.io.of(push.namespace); - if (localOnly) { - nsp.local.emit(push.event, push.data); - } else { - nsp.emit(push.event, push.data); + const filteredSockets = await this.findAndFilterSockets( + push.receivers, + push.namespace, + localOnly, + push.rooms, + ); + if (push.skipEmptyRooms && filteredSockets.length === 0) { + return false; } - } else { - if (push.rooms.length !== 0) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${push.event} to rooms: ${push.rooms.join( - ', ', - )} in namespace: ${push.namespace}`, - ); - const target = this.io.of(push.namespace).to(push.rooms); - if (localOnly) { - target.local.emit(push.event, push.data); - } else { - target.emit(push.event, push.data); + for (const remote of filteredSockets) { + const local = localOnly ? nsp.sockets.get(remote.id) : undefined; + if ( + push.boundedEmit && + localOnly && + local && + this.isLocalSocketBackpressured(local) + ) { + ConduitGrpcSdk.Metrics?.increment('event_relays_emit_dropped_total'); + local.disconnect(true); + continue; } + remote.emit(push.event, push.data); } - if (push.receivers.length !== 0) { - const filteredSockets = await this.findAndFilterSockets( - push.receivers, - push.namespace, - localOnly, - ); - for (const socket of filteredSockets) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${push.event} to socket: ${socket.id} in namespace: ${push.namespace}`, - ); - socket.emit(push.event, push.data); + return true; + } + if (push.rooms.length !== 0) { + const emitted = await this.emitEventToRooms(push, localOnly); + if (!emitted) { + return false; + } + } + return true; + } + return false; + } + + async getLocalRoomUserIds(namespace: string, room: string): Promise { + const sockets = await this.io.of(namespace).in(room).local.fetchSockets(); + const userIds = new Set(); + for (const socket of sockets) { + const userId = socket.data?.user?._id; + if (typeof userId === 'string' && userId.length > 0) { + userIds.add(userId); + } + } + return [...userIds]; + } + + async getLocalRoomsWithPrefix(namespace: string, prefix: string): Promise { + const adapter = this.io.of(namespace).adapter as { rooms?: Map }; + const rooms = adapter.rooms; + if (!rooms) { + return []; + } + return [...rooms.keys()].filter(room => room.startsWith(prefix)); + } + + private async emitEventToRooms(push: SocketPush, localOnly: boolean): Promise { + const nsp = this.io.of(push.namespace); + const localSockets = localOnly ? this.localSocketsInRooms(nsp, push.rooms) : []; + if (push.skipEmptyRooms || push.boundedEmit) { + if (localOnly && localSockets.length === 0) { + return false; + } + if (push.boundedEmit && localOnly) { + for (const socket of localSockets) { + if (this.isLocalSocketBackpressured(socket)) { + ConduitGrpcSdk.Metrics?.increment('event_relays_emit_dropped_total'); + socket.disconnect(true); } } } } + + const target = nsp.to(push.rooms); + if (localOnly) { + target.local.emit(push.event, push.data); + } else { + target.emit(push.event, push.data); + } + return true; + } + + private localSocketsInRooms( + nsp: ReturnType, + rooms: string[], + ): Socket[] { + const sockets: Socket[] = []; + for (const socket of nsp.sockets.values()) { + if (rooms.some(room => socket.rooms.has(room))) { + sockets.push(socket); + } + } + return sockets; + } + + private isLocalSocketBackpressured(socket: Socket): boolean { + return isEngineSocketBackpressured( + socket.conn as unknown as { writeBuffer?: unknown[] }, + ); + } + + private async socketsForRoomPush( + push: SocketPush, + localOnly: boolean, + ): Promise[]> { + if (push.receivers.length > 0) { + return this.findAndFilterSockets(push.receivers, push.namespace, localOnly); + } + const nsp = this.io.of(push.namespace); + const seen = new Set(); + const sockets: RemoteSocket[] = []; + for (const room of push.rooms) { + const inRoom = localOnly + ? await nsp.in(room).local.fetchSockets() + : await nsp.in(room).fetchSockets(); + for (const socket of inRoom) { + if (!seen.has(socket.id)) { + seen.add(socket.id); + sockets.push(socket); + } + } + } + return sockets; } private async handleResponse( @@ -276,19 +381,11 @@ export class SocketController extends ConduitRouter { ) { if (res.event === 'join-room') { if (res.rooms && res.rooms.length !== 0) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} joining rooms: ${res.rooms.join( - ', ', - )} in namespace: ${namespace}`, - ); socket.join(res.rooms); } } else if (res.event === 'leave-room') { if (res.rooms && res.rooms.length !== 0) { for (const room of res.rooms) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} leaving room: ${room} in namespace: ${namespace}`, - ); socket.leave(room); } } @@ -297,17 +394,9 @@ export class SocketController extends ConduitRouter { (!res.receivers || res.receivers.length === 0) && (!res.rooms || res.rooms.length === 0) ) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${res.event} to all sockets in namespace: ${namespace}`, - ); socket.emit(res.event, JSON.parse(res.data)); } else { if (res.rooms && res.rooms.length !== 0) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${res.event} to rooms: ${res.rooms.join( - ', ', - )} in namespace: ${namespace}`, - ); this.io.of(namespace).to(res.rooms).emit(res.event, JSON.parse(res.data)); } if (res.receivers && res.receivers.length !== 0) { @@ -316,9 +405,6 @@ export class SocketController extends ConduitRouter { namespace, ); for (const socket of filteredSockets) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${res.event} to socket: ${socket.id} in namespace: ${namespace}`, - ); socket.emit(res.event, JSON.parse(res.data)); } } @@ -330,15 +416,19 @@ export class SocketController extends ConduitRouter { userIds: string[], namespace: string, localOnly: boolean = false, + rooms: string[] = [], ): Promise[]> { const nsp = this.io.of(namespace); const sockets = localOnly ? await nsp.local.fetchSockets() : await nsp.fetchSockets(); - const userIdSet = new Set(userIds); - return sockets.filter(socket => { - if (socket.data && socket.data.user) { - return userIdSet.has(socket.data.user._id); - } - }); + return filterRemoteSocketsByUserAndRooms( + sockets.map(socket => ({ + id: socket.id, + data: socket.data, + rooms: socket.rooms, + })), + userIds, + rooms, + ).map(filtered => sockets.find(s => s.id === filtered.id)!); } protected _refreshRouter(): void { diff --git a/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts b/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts deleted file mode 100644 index 3e2834c0e..000000000 --- a/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; -import type { NextFunction, Request, Response } from 'express'; -import { applySocketGlobalMiddlewares } from './applySocketGlobalMiddlewares.js'; - -function fakeSocket(auth?: Record) { - return { - request: { - headers: {}, - url: '/realtime/?EIO=4&transport=polling', - }, - handshake: { auth: auth ?? {} }, - data: {}, - }; -} - -describe('applySocketGlobalMiddlewares', () => { - it('injects handshake.auth.token and copies conduit onto socket.data', async () => { - const socket = fakeSocket({ token: 'ticket-token' }); - await applySocketGlobalMiddlewares(socket as never, [ - (req: Request, _res: Response, next: NextFunction) => { - assert.equal(req.headers.authorization, 'Bearer ticket-token'); - (req as Request & { conduit: { admin: string } }).conduit = { - admin: 'ok', - }; - next(); - }, - ]); - assert.equal((socket.data as { admin?: string }).admin, 'ok'); - }); - - it('rejects when Express middleware sends 401 json', async () => { - const socket = fakeSocket(); - await assert.rejects( - () => - applySocketGlobalMiddlewares(socket as never, [ - (_req: Request, res: Response) => { - res.status(401).json({ error: 'No token provided' }); - }, - ]), - /No token provided/, - ); - }); -}); diff --git a/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts b/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts deleted file mode 100644 index c3f5bb8a8..000000000 --- a/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { NextFunction, Request, Response } from 'express'; -import type { Socket } from 'socket.io'; -import { buildSocketMiddlewareParams } from './buildSocketMiddlewareParams.js'; - -type ExpressMiddleware = (req: Request, res: Response, next: NextFunction) => void; - -type ConduitRequest = Request & { conduit?: Record }; - -export async function applySocketGlobalMiddlewares( - socket: Socket, - middlewares: ExpressMiddleware[], -): Promise { - const params = buildSocketMiddlewareParams(socket); - const req = Object.assign(socket.request, { - headers: params.headers, - path: '/realtime', - url: socket.request.url ?? '/realtime', - originalUrl: socket.request.url ?? '/realtime', - conduit: { ...params.context }, - }) as ConduitRequest; - - await new Promise((resolve, reject) => { - let settled = false; - const res = { - statusCode: 200, - status(this: { statusCode: number }, code: number) { - this.statusCode = code; - return this; - }, - json(this: unknown, body: { error?: string }) { - if (settled) return this; - settled = true; - reject(new Error(body?.error || 'Unauthorized')); - return this; - }, - setHeader() { - return this; - }, - removeHeader() {}, - } as unknown as Response; - - let index = 0; - const next = (err?: unknown) => { - if (settled) return; - if (err) { - settled = true; - reject(err instanceof Error ? err : new Error(String(err))); - return; - } - const middleware = middlewares[index++]; - if (!middleware) { - settled = true; - socket.data = { ...socket.data, ...(req.conduit ?? {}) }; - resolve(); - return; - } - try { - middleware(req, res, next); - } catch (error) { - settled = true; - reject(error as Error); - } - }; - next(); - }); -} diff --git a/libraries/hermes/src/Socket/isSocketHandshake.test.ts b/libraries/hermes/src/Socket/isSocketHandshake.test.ts new file mode 100644 index 000000000..a686bdb76 --- /dev/null +++ b/libraries/hermes/src/Socket/isSocketHandshake.test.ts @@ -0,0 +1,27 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { isSocketHandshake } from './isSocketHandshake.js'; + +describe('isSocketHandshake', () => { + it('matches Engine.IO handshake query params only', () => { + assert.equal( + isSocketHandshake({ url: '/events/?EIO=4&transport=polling&t=abc' }), + true, + ); + assert.equal(isSocketHandshake({ url: '/realtime/?EIO=4&transport=websocket' }), true); + }); + + it('rejects HTTP routes that merely mention realtime or established sessions', () => { + assert.equal( + isSocketHandshake({ url: '/realtime/ticket?EIO=4&transport=polling' }), + false, + ); + assert.equal(isSocketHandshake({ url: '/realtime/ticket' }), false); + assert.equal(isSocketHandshake({ originalUrl: '/realtime' }), false); + assert.equal(isSocketHandshake({ url: '/graphql?EIO=4&transport=polling' }), false); + assert.equal( + isSocketHandshake({ url: '/events/?EIO=4&transport=polling&sid=abc' }), + false, + ); + }); +}); diff --git a/libraries/hermes/src/Socket/isSocketHandshake.ts b/libraries/hermes/src/Socket/isSocketHandshake.ts new file mode 100644 index 000000000..aa216d2ac --- /dev/null +++ b/libraries/hermes/src/Socket/isSocketHandshake.ts @@ -0,0 +1,26 @@ +export function isSocketHandshake(req: { url?: string; originalUrl?: string }): boolean { + const raw = req.url ?? req.originalUrl ?? ''; + if (!raw) { + return false; + } + const queryIndex = raw.indexOf('?'); + const pathname = queryIndex === -1 ? raw : raw.slice(0, queryIndex); + if (pathname.includes('ticket')) { + return false; + } + const query = queryIndex === -1 ? '' : raw.slice(queryIndex + 1); + if (!query) { + return false; + } + const params = new URLSearchParams(query); + if (!params.has('EIO') || !params.has('transport')) { + return false; + } + if (params.has('sid')) { + return false; + } + if (pathname !== '/' && !pathname.endsWith('/')) { + return false; + } + return true; +} diff --git a/libraries/hermes/src/Socket/resolveEngineNamespacePath.ts b/libraries/hermes/src/Socket/resolveEngineNamespacePath.ts new file mode 100644 index 000000000..e36aad12b --- /dev/null +++ b/libraries/hermes/src/Socket/resolveEngineNamespacePath.ts @@ -0,0 +1,11 @@ +/** + * Derive the Socket.IO namespace path from an Engine.IO handshake request URL. + */ +export function resolveEngineNamespacePath(req: { url?: string }): string { + const url = req.url ?? '/'; + const pathname = url.split('?')[0] ?? '/'; + if (pathname === '/' || pathname === '') { + return '/'; + } + return pathname.endsWith('/') ? pathname : `${pathname}/`; +} diff --git a/libraries/hermes/src/Socket/socketPushUtils.test.ts b/libraries/hermes/src/Socket/socketPushUtils.test.ts new file mode 100644 index 000000000..7f76e2904 --- /dev/null +++ b/libraries/hermes/src/Socket/socketPushUtils.test.ts @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { + filterRemoteSocketsByUserAndRooms, + isEngineSocketBackpressured, + WRITE_BUFFER_PACKET_HIGH_WATER, +} from './socketPushUtils.js'; + +describe('filterRemoteSocketsByUserAndRooms', () => { + const roomA = 'er:relay-1:aaa'; + const roomB = 'er:relay-1:bbb'; + + it('emits only to sockets in the target room for the same user', () => { + const sockets = [ + { + id: 'tab-a', + data: { user: { _id: 'user-1' } }, + rooms: new Set([roomA]), + }, + { + id: 'tab-b', + data: { user: { _id: 'user-1' } }, + rooms: new Set([roomB]), + }, + ]; + + const filtered = filterRemoteSocketsByUserAndRooms(sockets, ['user-1'], [roomA]); + assert.deepEqual( + filtered.map(s => s.id), + ['tab-a'], + ); + }); +}); + +describe('isEngineSocketBackpressured', () => { + it('uses writeBuffer queue depth only', () => { + assert.equal(isEngineSocketBackpressured(undefined), false); + assert.equal( + isEngineSocketBackpressured({ writeBuffer: new Array(WRITE_BUFFER_PACKET_HIGH_WATER) }), + false, + ); + assert.equal( + isEngineSocketBackpressured({ + writeBuffer: new Array(WRITE_BUFFER_PACKET_HIGH_WATER + 1), + }), + true, + ); + assert.equal( + isEngineSocketBackpressured({ + writeBuffer: [], + }), + false, + ); + }); +}); + +describe('SocketController relay emit', () => { + it('does not reference writableLength or client emit ack callbacks', () => { + const socketSource = readFileSync( + path.join(path.dirname(fileURLToPath(import.meta.url)), '../../src/Socket/Socket.ts'), + 'utf8', + ); + assert.doesNotMatch(socketSource, /writableLength/); + assert.doesNotMatch(socketSource, /\.emit\([^)]*,\s*[^,)]+,\s*\(\)\s*=>/); + }); +}); diff --git a/libraries/hermes/src/Socket/socketPushUtils.ts b/libraries/hermes/src/Socket/socketPushUtils.ts new file mode 100644 index 000000000..6b3e14feb --- /dev/null +++ b/libraries/hermes/src/Socket/socketPushUtils.ts @@ -0,0 +1,36 @@ +export const WRITE_BUFFER_PACKET_HIGH_WATER = 64; + +type SocketLike = { + id: string; + data?: { user?: { _id?: string } }; + rooms: Set; +}; + +export function filterRemoteSocketsByUserAndRooms( + sockets: SocketLike[], + userIds: string[], + rooms: string[], +): SocketLike[] { + const userIdSet = new Set(userIds); + const roomSet = rooms.length > 0 ? new Set(rooms) : null; + return sockets.filter(socket => { + if (!socket.data?.user?._id) { + return false; + } + if (!userIdSet.has(socket.data.user._id)) { + return false; + } + if (roomSet) { + return [...roomSet].some(room => socket.rooms.has(room)); + } + return true; + }); +} + +export function isEngineSocketBackpressured( + conn: { writeBuffer?: unknown[] } | undefined, + highWater = WRITE_BUFFER_PACKET_HIGH_WATER, +): boolean { + const pending = conn?.writeBuffer?.length ?? 0; + return pending > highWater; +} diff --git a/libraries/hermes/src/index.ts b/libraries/hermes/src/index.ts index ac22e28ba..46808add1 100644 --- a/libraries/hermes/src/index.ts +++ b/libraries/hermes/src/index.ts @@ -254,6 +254,12 @@ export class ConduitRoutingController { } } + registerSocketGlobalMiddleware( + middleware: (req: ConduitRequest, res: Response, next: NextFunction) => void, + ) { + this._socketRouter?.registerGlobalMiddleware(middleware); + } + registerRouteMiddleware(middleware: ConduitMiddleware, moduleUrl: string) { this._restRouter?.registerMiddleware(middleware, moduleUrl); this._graphQLRouter?.registerMiddleware(middleware, moduleUrl); @@ -343,7 +349,20 @@ export class ConduitRoutingController { } async socketPush(data: SocketPush) { - await this._socketRouter?.handleSocketPush(data); + return (await this._socketRouter?.handleSocketPush(data)) ?? false; + } + + getLocalRoomUserIds(namespace: string, room: string): Promise { + return ( + this._socketRouter?.getLocalRoomUserIds(namespace, room) ?? Promise.resolve([]) + ); + } + + getLocalRoomsWithPrefix(namespace: string, prefix: string): Promise { + return ( + this._socketRouter?.getLocalRoomsWithPrefix(namespace, prefix) ?? + Promise.resolve([]) + ); } /** True if any enabled transport would change this route (new or definition changed). */ @@ -472,6 +491,8 @@ export class ConduitRoutingController { } } +export { isSocketHandshake } from './Socket/isSocketHandshake.js'; +export { resolveEngineNamespacePath } from './Socket/resolveEngineNamespacePath.js'; export * from './interfaces/index.js'; export * from './types/index.js'; export * from './classes/index.js'; diff --git a/libraries/hermes/src/interfaces/Socket.ts b/libraries/hermes/src/interfaces/Socket.ts index 204707624..2b9787257 100644 --- a/libraries/hermes/src/interfaces/Socket.ts +++ b/libraries/hermes/src/interfaces/Socket.ts @@ -10,6 +10,7 @@ export interface ConduitSocketParameters { socketId: string; params?: UntypedArray; context?: Indexable; + recoveredRooms?: string[]; } export type ConduitSocketParamTypes = (TYPE | ConduitSocketParamTypes)[]; @@ -19,6 +20,7 @@ export interface ConduitSocketOptions { name?: string; description?: string; middlewares?: string[]; + onRecovered?: ConduitSocketEventHandler; } export type EventResponse = { @@ -85,6 +87,18 @@ export class ConduitSocket { } return Promise.reject('no such event registered'); } + + executeRecovered( + request: ConduitSocketParameters, + ): ConduitSocketHandlerResponse | null { + if (this._input.onRecovered) { + return this._input.onRecovered(request); + } + if (this._events.has('recovered')) { + return this._events.get('recovered')!.handler(request); + } + return null; + } } export interface EventsProtoDescription { diff --git a/libraries/hermes/src/interfaces/SocketPush.ts b/libraries/hermes/src/interfaces/SocketPush.ts index 8bbabbc85..7d0f572d3 100644 --- a/libraries/hermes/src/interfaces/SocketPush.ts +++ b/libraries/hermes/src/interfaces/SocketPush.ts @@ -5,4 +5,8 @@ export interface SocketPush { rooms: string[]; namespace: string; localOnly?: boolean; + /** Skip emit when the local room has no connected sockets. */ + skipEmptyRooms?: boolean; + /** Drop or disconnect slow clients instead of blocking the caller. */ + boundedEmit?: boolean; } diff --git a/libraries/hermes/tsconfig.json b/libraries/hermes/tsconfig.json index 0420a284b..225b927d7 100644 --- a/libraries/hermes/tsconfig.json +++ b/libraries/hermes/tsconfig.json @@ -66,5 +66,5 @@ /* Advanced Options */ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ }, - "exclude": ["node_modules", "dist", "**/*.test.ts"] + "exclude": ["src/**/*.test.ts"] } diff --git a/libraries/hermes/tsconfig.test.json b/libraries/hermes/tsconfig.test.json index 8c2d1a100..8aecbdee9 100644 --- a/libraries/hermes/tsconfig.test.json +++ b/libraries/hermes/tsconfig.test.json @@ -8,9 +8,10 @@ "types": ["node"] }, "include": [ - "src/Socket/applySocketGlobalMiddlewares.ts", - "src/Socket/applySocketGlobalMiddlewares.test.ts", - "src/Socket/buildSocketMiddlewareParams.ts" + "src/Socket/isSocketHandshake.ts", + "src/Socket/isSocketHandshake.test.ts", + "src/Socket/socketPushUtils.ts", + "src/Socket/socketPushUtils.test.ts" ], - "exclude": ["node_modules", "dist"] + "exclude": [] } diff --git a/modules/database/README.mdx b/modules/database/README.mdx index 61bc195b0..b84cded13 100644 --- a/modules/database/README.mdx +++ b/modules/database/README.mdx @@ -47,7 +47,9 @@ since the latter need to go through parsers that are otherwise unnecessary for M When using MongoDB with a replica set (e.g., MongoDB Atlas), the database module supports configuring read preference, write concern, and read concern through the admin panel at `PATCH /config/database`. -Live document updates also require a replica set or sharded cluster. Local Compose files initialize a single-node replica set so change streams can be exercised. +Live document updates require a replica set or sharded cluster (Atlas is fine). Helm’s bundled Mongo chart is a **standalone** Deployment (`replicas: 1`, no `--replSet`), so live updates stay `idle` there until you point `DB_CONN_URI` at Atlas or an operator-managed replica set. + +Local Compose files initialize a single-node `rs0`. **Existing Compose Mongo volumes will not become a replica set cleanly** — drop the volume or start from an empty data dir if `hello` still reports standalone. ### Live updates @@ -58,9 +60,11 @@ subscribe({ schema: 'Order', documentId?: string }) unsubscribe({ schema: 'Order', documentId?: string }) ``` -Events arrive as `change` with `{ version, operation, schema, documentId, occurredAt, resumeToken }` and contain no document fields. Consumers should refetch through their authorized REST or custom-endpoint path. +Events arrive as `change` with `{ version, operation, schema, documentId, occurredAt }` and contain no document fields and no resume token. This is **live-tail, not backfill**. After a full reconnect, subscribe again and refetch over authorized REST. + +The leader also publishes `database:change:${schema}` on the Redis bus. **Do not also relay `database:change:*` on `/events/` if the same client is on `/database/`** — that duplicates notifications. Keep the bus for other modules; just do not dual-subscribe. -Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; session JWTs and masterkeys must not be sent from browser code. +Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Client sockets also require CMS `crudOperations.read.enabled`. Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; that token cannot mint another ticket or call REST/GraphQL. Session JWTs and masterkeys must not be sent from browser code. Admin sockets must be enabled (`admin.transports.sockets`) and the Admin socket port (`ADMIN_SOCKET_PORT`, default 3031) reachable from the UI. diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index e0208f256..e2d36e26f 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -1053,4 +1053,8 @@ export default class DatabaseModule extends ManagedModule { ); } } + + async shutdown(): Promise { + await this.realtimeService?.shutdown(); + } } diff --git a/modules/database/src/index.ts b/modules/database/src/index.ts index 3e9466196..12261992a 100644 --- a/modules/database/src/index.ts +++ b/modules/database/src/index.ts @@ -1,5 +1,6 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import DatabaseModule from './Database.js'; const dbType = process.env.DB_TYPE ?? 'mongodb'; @@ -7,4 +8,21 @@ const dbUri = process.env.DB_CONN_URI ?? 'mongodb://localhost:27017'; const peerManifestRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const database = new DatabaseModule(dbType, dbUri, peerManifestRoot); + +function registerShutdownSignals(): void { + const shutdown = (signal: NodeJS.Signals) => { + void database + .shutdown() + .catch(err => { + ConduitGrpcSdk.Logger.error(err as Error); + }) + .finally(() => { + process.exit(signal === 'SIGINT' ? 130 : 0); + }); + }; + process.once('SIGTERM', () => shutdown('SIGTERM')); + process.once('SIGINT', () => shutdown('SIGINT')); +} + +registerShutdownSignals(); database.start(); diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts index b41cc2f89..d29938d15 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -2,6 +2,7 @@ import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import { normalizeChangeEvent, parseResumeToken, + serializeResumeToken, type RawChangeEvent, } from './normalize.js'; import { authorizedDocumentRoom, roomsForPublicChange } from './rooms.js'; @@ -17,7 +18,14 @@ import type { RealtimeStatusCode, } from './types.js'; import type { RealtimeSubscriptionTracker } from './subscriptions.js'; -import { canReadDocument, type AuthorizationSdk } from './authorize.js'; +import { type AuthorizationSdk } from './authorize.js'; +import { checkRebacBatch, RealtimeRebacCache } from './rebacCache.js'; +import { + buildWatchPipeline, + optedInCollectionsKey, + WATCH_RESTART_OPERATIONS, + type WatchPipeline, +} from './watchPipeline.js'; const LEADER_LOCK = 'realtime:change-stream:leader'; const RESUME_TOKEN_KEY = 'realtime:resumeToken'; @@ -30,7 +38,10 @@ type LeaderLock = NonNullable< Awaited['tryAcquireLock']>> >; -export type WatchFactory = (options: { resumeAfter?: unknown }) => ChangeStreamLike; +export type WatchFactory = (options: { + resumeAfter?: unknown; + pipeline: WatchPipeline; +}) => ChangeStreamLike; export type CoordinatorOptions = { grpcSdk: ConduitGrpcSdk; @@ -56,6 +67,9 @@ export class MongoChangeStreamCoordinator { private watching = false; private opening = false; private ignoreClose = false; + private changeQueue: Promise = Promise.resolve(); + private watchedCollectionsKey = ''; + private readonly rebacCache = new RealtimeRebacCache(); constructor(private readonly options: CoordinatorOptions) {} @@ -75,6 +89,10 @@ export class MongoChangeStreamCoordinator { return this.topology; } + async waitForIdle(): Promise { + await this.changeQueue; + } + async reconcile(): Promise { if (this.closed) return; const engine = this.options.engine(); @@ -90,8 +108,6 @@ export class MongoChangeStreamCoordinator { await this.releaseLeader(); this.streamState = 'idle'; this.lastError = this.topology.message; - // Hello can fail during startup before Mongo is ready. Keep retrying - // that case; a confirmed standalone topology will not recover. if ( !this.topology.message || this.topology.message.includes('Unable to determine') @@ -100,20 +116,31 @@ export class MongoChangeStreamCoordinator { } return; } - if (this.options.getOptedInSchemas().length === 0) { + const collections = this.collectionNames(); + if (collections.length === 0) { await this.stopStream('idle'); await this.releaseLeader(); this.streamState = 'idle'; return; } + const nextKey = optedInCollectionsKey(collections); + if (this.watching && nextKey !== this.watchedCollectionsKey) { + await this.stopStream('starting'); + } await this.ensureLeader(); } async shutdown(): Promise { this.closed = true; this.clearTimers(); + await this.changeQueue; await this.stopStream('idle'); await this.releaseLeader(); + this.rebacCache.clear(); + } + + private collectionNames(): string[] { + return this.options.getOptedInSchemas().map(schema => schema.collectionName); } private async ensureLeader(): Promise { @@ -129,9 +156,6 @@ export class MongoChangeStreamCoordinator { LOCK_TTL_MS, ); if (!acquired) { - // Another instance holds the lock, or a crashed holder has not - // expired yet. Without a retry, a standalone process stays idle - // forever after a restart races the previous TTL. this.streamState = 'idle'; this.scheduleRetry(); return; @@ -174,13 +198,16 @@ export class MongoChangeStreamCoordinator { await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY), ); if (this.watching || this.closed) return; - const stream = this.options.watch({ resumeAfter }); + const collections = this.collectionNames(); + const pipeline = buildWatchPipeline(collections); + this.watchedCollectionsKey = optedInCollectionsKey(collections); + const stream = this.options.watch({ resumeAfter, pipeline }); this.stream = stream; this.watching = true; this.streamState = 'live'; this.retryAttempt = 0; stream.on('change', (change: unknown) => { - void this.handleChange(change as RawChangeEvent); + this.enqueueChange(change as RawChangeEvent); }); stream.on('error', (err: unknown) => { void this.handleStreamError(err); @@ -199,26 +226,61 @@ export class MongoChangeStreamCoordinator { } } + private enqueueChange(change: RawChangeEvent) { + this.changeQueue = this.changeQueue.then(async () => { + if (this.closed || !this.watching) return; + try { + await this.handleChange(change); + } catch (err) { + this.lastError = err instanceof Error ? err.message : String(err); + ConduitGrpcSdk.Logger.error(err as Error); + this.watching = false; + await this.stopStream('degraded'); + this.scheduleRetry(); + } + }); + } + private async handleChange(change: RawChangeEvent) { + const token = serializeResumeToken(change._id); const schema = this.resolveSchema(change.ns?.coll); - if (!schema) return; - const event = normalizeChangeEvent(change, schema.name); - if (!event) return; + const event = schema ? normalizeChangeEvent(change, schema.name) : null; + if (!event || !schema) { + if (token) { + await this.persistResumeToken(token); + } + if (change.operationType && WATCH_RESTART_OPERATIONS.has(change.operationType)) { + await this.stopStream('starting'); + this.scheduleRetry(); + } + return; + } this.lastEventAt = event.occurredAt; this.lastError = undefined; - await this.options.grpcSdk.state!.setKey(RESUME_TOKEN_KEY, event.resumeToken); - this.options.grpcSdk.bus?.publish( - `database:change:${schema.name}`, - JSON.stringify(event), - ); + await this.emitChange(schema, event); + if (token) { + await this.persistResumeToken(token); + } + } + + private async persistResumeToken(token: string) { + await this.options.grpcSdk.state!.setKey(RESUME_TOKEN_KEY, token); + } + + private async emitChange(schema: OptedInSchema, event: DatabaseChangeEvent) { + const payload = JSON.stringify(event); + this.options.grpcSdk.bus?.publish(`database:change:${schema.name}`, payload); ConduitGrpcSdk.Metrics?.increment('database_realtime_events_total', 1, { operation: event.operation, }); - await this.pushEvent(schema, event); + await this.pushEvent(schema, event, payload); } - private async pushEvent(schema: OptedInSchema, event: DatabaseChangeEvent) { - const payload = JSON.stringify(event); + private async pushEvent( + schema: OptedInSchema, + event: DatabaseChangeEvent, + payload: string, + ) { const adminRooms = roomsForPublicChange(schema.name, event.documentId); await this.safePush('admin', adminRooms, payload); if (!schema.authorizationEnabled) { @@ -229,23 +291,27 @@ export class MongoChangeStreamCoordinator { schema.name, event.documentId, ); + const decisions = await checkRebacBatch( + this.rebacCache, + this.options.grpcSdk as unknown as AuthorizationSdk, + userIds, + schema.name, + event.documentId, + ); const allowedRooms: string[] = []; for (const userId of userIds) { - const allowed = await canReadDocument( - this.options.grpcSdk as unknown as AuthorizationSdk, - schema.name, - event.documentId, - userId, - ); - if (!allowed) { + const decision = decisions.get(userId) ?? 'unavailable'; + if (decision === 'allow') { + allowedRooms.push(authorizedDocumentRoom(schema.name, event.documentId, userId)); + continue; + } + if (decision === 'deny') { await this.options.subscriptions.removeUser( schema.name, event.documentId, userId, ); - continue; } - allowedRooms.push(authorizedDocumentRoom(schema.name, event.documentId, userId)); } if (allowedRooms.length > 0) { await this.safePush('router', allowedRooms, payload); @@ -260,16 +326,12 @@ export class MongoChangeStreamCoordinator { const client = target === 'admin' ? this.options.grpcSdk.admin : this.options.grpcSdk.router; if (!client?.socketPush) return; - try { - await client.socketPush({ - event: 'change', - data, - rooms, - receivers: [], - }); - } catch (err) { - ConduitGrpcSdk.Logger.error(err as Error); - } + await client.socketPush({ + event: 'change', + data, + rooms, + receivers: [], + }); } private resolveSchema(collectionName?: string): OptedInSchema | undefined { @@ -308,6 +370,7 @@ export class MongoChangeStreamCoordinator { this.watching = false; this.streamState = nextState; this.ignoreClose = true; + this.watchedCollectionsKey = ''; if (stream) { try { await stream.close(); diff --git a/modules/database/src/realtime/RealtimeService.ts b/modules/database/src/realtime/RealtimeService.ts index e17e320ea..96d21f485 100644 --- a/modules/database/src/realtime/RealtimeService.ts +++ b/modules/database/src/realtime/RealtimeService.ts @@ -19,6 +19,7 @@ import { registerDatabaseRealtimeSocket } from './sockets.js'; import { buildRealtimeStatus } from './status.js'; import { RealtimeSubscriptionTracker } from './subscriptions.js'; import type { ChangeStreamLike, OptedInSchema, RealtimeStatus } from './types.js'; +import type { WatchPipeline } from './watchPipeline.js'; export class RealtimeService { private readonly subscriptions: RealtimeSubscriptionTracker; @@ -40,7 +41,7 @@ export class RealtimeService { if (adapter instanceof MongooseAdapter) { this.coordinator = new MongoChangeStreamCoordinator({ grpcSdk, - watch: options => this.openWatch(adapter, options.resumeAfter), + watch: options => this.openWatch(adapter, options), hello: () => this.hello(adapter), getOptedInSchemas: () => this.getOptedInSchemas(), subscriptions: this.subscriptions, @@ -128,14 +129,17 @@ export class RealtimeService { return schemas; } - private openWatch(adapter: MongooseAdapter, resumeAfter?: unknown): ChangeStreamLike { + private openWatch( + adapter: MongooseAdapter, + options: { resumeAfter?: unknown; pipeline: WatchPipeline }, + ): ChangeStreamLike { const db = adapter.mongoose.connection.db; if (!db) { throw new Error('MongoDB connection is not ready'); } return db.watch( - [], - resumeAfter ? { resumeAfter: resumeAfter as never } : {}, + options.pipeline, + options.resumeAfter ? { resumeAfter: options.resumeAfter as never } : {}, ) as unknown as ChangeStreamLike; } diff --git a/modules/database/src/realtime/__tests__/change-stream.integration.test.ts b/modules/database/src/realtime/__tests__/change-stream.integration.test.ts index dd74753fc..b4e313310 100644 --- a/modules/database/src/realtime/__tests__/change-stream.integration.test.ts +++ b/modules/database/src/realtime/__tests__/change-stream.integration.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from '@jest/globals'; import { MongoClient, ObjectId } from 'mongodb'; import { normalizeChangeEvent, type RawChangeEvent } from '../normalize.js'; +import { buildWatchPipeline } from '../watchPipeline.js'; const replicaSetUri = process.env.DB_CONN_URI; const integrationEnabled = Boolean(replicaSetUri?.includes('replicaSet')); @@ -14,7 +15,7 @@ describeIntegration('MongoDB change stream contract', () => { const db = client.db(dbName); try { const collection = db.collection('orders'); - const stream = db.watch([]); + const stream = db.watch(buildWatchPipeline(['orders'])); const change = await new Promise((resolve, reject) => { const timer = setTimeout( () => reject(new Error('timed out waiting for change')), diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index efddfca70..cda5dc0e1 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -1,6 +1,6 @@ import { EventEmitter } from 'node:events'; -import { describe, expect, it, jest } from '@jest/globals'; -import { ObjectId } from 'bson'; +import { afterEach, describe, expect, it, jest } from '@jest/globals'; +import { EJSON, ObjectId } from 'bson'; import { MongoChangeStreamCoordinator } from '../MongoChangeStreamCoordinator.js'; import { RealtimeSubscriptionTracker } from '../subscriptions.js'; import { roomsForPublicChange } from '../rooms.js'; @@ -11,9 +11,13 @@ class MemoryStore { const set = this.sets.get(key) ?? new Set(); members.forEach(member => set.add(member)); this.sets.set(key, set); + return members.length; } async srem(key: string, ...members: string[]) { - members.forEach(member => this.sets.get(key)?.delete(member)); + const set = this.sets.get(key); + if (!set) return 0; + members.forEach(member => set.delete(member)); + return members.length; } async smembers(key: string) { return [...(this.sets.get(key) ?? new Set())]; @@ -23,11 +27,13 @@ class MemoryStore { } async del(...keys: string[]) { keys.forEach(key => this.sets.delete(key)); + return keys.length; } } function createCoordinator(overrides?: { allow?: boolean; + authorizationAvailable?: boolean; schemas?: { name: string; collectionName: string; authorizationEnabled: boolean }[]; getKeyDelayMs?: number; }) { @@ -65,10 +71,13 @@ function createCoordinator(overrides?: { bus: { publish }, router: { socketPush: routerPush }, admin: { socketPush: adminPush }, - isAvailable: () => true, - authorization: { - can: async () => ({ allow: overrides?.allow !== false }), - }, + isAvailable: () => overrides?.authorizationAvailable !== false, + authorization: + overrides?.authorizationAvailable === false + ? null + : { + can: async () => ({ allow: overrides?.allow !== false }), + }, }; const coordinator = new MongoChangeStreamCoordinator({ grpcSdk: grpcSdk as never, @@ -96,26 +105,32 @@ function createCoordinator(overrides?: { }; } +function insertChange(collection: string, id: string, token: unknown) { + return { + operationType: 'insert', + ns: { coll: collection }, + documentKey: { _id: new ObjectId(id) }, + _id: token, + }; +} + describe('MongoChangeStreamCoordinator', () => { + afterEach(() => { + jest.useRealTimers(); + }); + it('emits one normalized event to public rooms and ignores other collections', async () => { - const { coordinator, stream, routerPush, adminPush, publish } = createCoordinator(); + const { coordinator, stream, routerPush, adminPush, publish, state } = + createCoordinator(); await coordinator.reconcile(); const resume = { _data: 'token' }; stream.emit('change', { - operationType: 'insert', - ns: { coll: 'orders' }, - documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, + ...insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', resume), fullDocument: { secret: 'nope' }, - _id: resume, wallTime: new Date('2026-01-02T00:00:00.000Z'), }); - stream.emit('change', { - operationType: 'insert', - ns: { coll: 'other' }, - documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4d') }, - _id: resume, - }); - await new Promise(resolve => setImmediate(resolve)); + stream.emit('change', insertChange('other', '64b64c4c4c4c4c4c4c4c4c4d', resume)); + await coordinator.waitForIdle(); expect(publish).toHaveBeenCalledTimes(1); expect(publish.mock.calls[0][0]).toBe('database:change:Order'); const payload = JSON.parse(publish.mock.calls[0][1] as string); @@ -125,6 +140,7 @@ describe('MongoChangeStreamCoordinator', () => { documentId: '64b64c4c4c4c4c4c4c4c4c4c', }); expect(payload).not.toHaveProperty('fullDocument'); + expect(payload).not.toHaveProperty('resumeToken'); expect(payload).not.toHaveProperty('secret'); const expectedRooms = roomsForPublicChange('Order', '64b64c4c4c4c4c4c4c4c4c4c'); expect(routerPush).toHaveBeenCalledWith( @@ -133,6 +149,102 @@ describe('MongoChangeStreamCoordinator', () => { expect(adminPush).toHaveBeenCalledWith( expect.objectContaining({ event: 'change', rooms: expectedRooms }), ); + expect( + JSON.parse((adminPush.mock.calls[0][0] as { data: string }).data), + ).not.toHaveProperty('resumeToken'); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(resume)); + await coordinator.shutdown(); + }); + + it('advances the resume token for filtered events', async () => { + const { coordinator, stream, publish, state } = createCoordinator(); + await coordinator.reconcile(); + const skip = { _data: 'skip-token' }; + stream.emit('change', insertChange('other', '64b64c4c4c4c4c4c4c4c4c4d', skip)); + await coordinator.waitForIdle(); + expect(publish).not.toHaveBeenCalled(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(skip)); + await coordinator.shutdown(); + }); + + it('serializes overlapping handlers and persists after emit', async () => { + const { coordinator, stream, state, adminPush } = createCoordinator(); + let release!: () => void; + const gate = new Promise(resolve => { + release = resolve; + }); + let first = true; + adminPush.mockImplementation(async () => { + if (first) { + first = false; + await gate; + } + }); + await coordinator.reconcile(); + const tokenA = { _data: 'token-a' }; + const tokenB = { _data: 'token-b' }; + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', tokenB)); + await Promise.resolve(); + await new Promise(resolve => setImmediate(resolve)); + expect(state.get('realtime:resumeToken')).toBeUndefined(); + expect(adminPush).toHaveBeenCalledTimes(1); + release(); + await coordinator.waitForIdle(); + expect(adminPush).toHaveBeenCalledTimes(2); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenB)); + await coordinator.shutdown(); + }); + + it('watches opted-in collections with $match and $project', async () => { + const { coordinator, watch } = createCoordinator(); + await coordinator.reconcile(); + expect(watch).toHaveBeenCalledTimes(1); + const pipeline = (watch.mock.calls[0][0] as { pipeline: Record[] }) + .pipeline; + expect(pipeline[0]).toEqual( + expect.objectContaining({ + $match: expect.objectContaining({ + $or: expect.arrayContaining([ + expect.objectContaining({ + 'ns.coll': { $in: ['orders'] }, + }), + ]), + }), + }), + ); + expect(pipeline[1]).toEqual({ + $project: { + fullDocument: 0, + updateDescription: 0, + fullDocumentBeforeChange: 0, + }, + }); + await coordinator.shutdown(); + }); + + it('reopens the watch when the opt-in set changes', async () => { + const schemas = [ + { name: 'Order', collectionName: 'orders', authorizationEnabled: false }, + ]; + const { coordinator, watch } = createCoordinator({ schemas }); + await coordinator.reconcile(); + expect(watch).toHaveBeenCalledTimes(1); + schemas.push({ + name: 'Item', + collectionName: 'items', + authorizationEnabled: false, + }); + await coordinator.reconcile(); + expect(watch).toHaveBeenCalledTimes(2); + const pipeline = (watch.mock.calls[1][0] as { pipeline: Record[] }) + .pipeline; + const match = pipeline[0] as { + $match: { $or: Array<{ 'ns.coll'?: { $in: string[] } }> }; + }; + expect(match.$match.$or[0]['ns.coll']?.$in).toEqual( + expect.arrayContaining(['orders', 'items']), + ); await coordinator.shutdown(); }); @@ -148,15 +260,15 @@ describe('MongoChangeStreamCoordinator', () => { 'user-1', ); await coordinator.reconcile(); - stream.emit('change', { - operationType: 'update', - ns: { coll: 'orders' }, - documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, - _id: { _data: 'token' }, - }); - await new Promise(resolve => setImmediate(resolve)); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), + ); + await coordinator.waitForIdle(); expect(routerPush).not.toHaveBeenCalled(); - expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c')).toEqual([]); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual( + [], + ); await coordinator.shutdown(); }); @@ -179,7 +291,7 @@ describe('MongoChangeStreamCoordinator', () => { await coordinator.shutdown(); }); - it('clears an unusable resume token and retries', async () => { + it('clears an unusable resume token on 280 and retries', async () => { const { coordinator, stream, grpcSdk } = createCoordinator(); await coordinator.reconcile(); stream.emit('error', { code: 280, message: 'ChangeStreamHistoryLost' }); @@ -187,4 +299,124 @@ describe('MongoChangeStreamCoordinator', () => { expect(grpcSdk.state.clearKey).toHaveBeenCalled(); await coordinator.shutdown(); }); + + it('keeps the resume token on CursorKilled 237', async () => { + const { coordinator, stream, grpcSdk, state } = createCoordinator(); + await coordinator.reconcile(); + const token = { _data: 'keep-me' }; + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', token)); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); + stream.emit('error', { code: 237, message: 'CursorKilled' }); + await new Promise(resolve => setImmediate(resolve)); + expect(grpcSdk.state.clearKey).not.toHaveBeenCalled(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); + await coordinator.shutdown(); + }); + + it('does not persist a later token when emit fails and reopens from the last good token', async () => { + jest.useFakeTimers(); + const streams: Array Promise }> = []; + const { coordinator, adminPush, state, watch } = createCoordinator(); + watch.mockImplementation(() => { + const next = new EventEmitter() as EventEmitter & { close: () => Promise }; + next.close = async () => { + next.emit('close'); + }; + streams.push(next); + return next as never; + }); + adminPush + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error('push failed')) + .mockResolvedValue(undefined); + await coordinator.reconcile(); + const tokenGood = { _data: 'token-good' }; + const tokenA = { _data: 'token-a' }; + const tokenB = { _data: 'token-b' }; + streams[0].emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4b', tokenGood), + ); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenGood)); + streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); + streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', tokenB)); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenGood)); + expect(adminPush).toHaveBeenCalledTimes(2); + await jest.advanceTimersByTimeAsync(1_000); + expect(watch).toHaveBeenCalledTimes(2); + expect((watch.mock.calls[1][0] as { resumeAfter?: unknown }).resumeAfter).toEqual( + tokenGood, + ); + streams[1].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenA)); + expect(adminPush).toHaveBeenCalledTimes(3); + await coordinator.shutdown(); + jest.useRealTimers(); + }); + + it.each(['drop', 'rename', 'invalidate'] as const)( + 'reopens the watch on %s', + async operationType => { + jest.useFakeTimers(); + const streams: Array Promise }> = []; + const { coordinator, watch, state } = createCoordinator(); + watch.mockImplementation(() => { + const next = new EventEmitter() as EventEmitter & { close: () => Promise }; + next.close = async () => { + next.emit('close'); + }; + streams.push(next); + return next as never; + }); + await coordinator.reconcile(); + const token = { _data: `${operationType}-token` }; + streams[0].emit('change', { + operationType, + ns: { coll: 'orders' }, + _id: token, + }); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); + await jest.advanceTimersByTimeAsync(1_000); + expect(watch).toHaveBeenCalledTimes(2); + expect((watch.mock.calls[1][0] as { resumeAfter?: unknown }).resumeAfter).toEqual( + token, + ); + await coordinator.shutdown(); + jest.useRealTimers(); + }, + ); + + it('does not remove users when authorization is unavailable', async () => { + const { coordinator, stream, routerPush, subscriptions } = createCoordinator({ + authorizationAvailable: false, + schemas: [{ name: 'Order', collectionName: 'orders', authorizationEnabled: true }], + }); + const removeUser = jest.spyOn(subscriptions, 'removeUser'); + await subscriptions.addAuthorizedDocument( + 'sock-1', + 'Order', + '64b64c4c4c4c4c4c4c4c4c4c', + 'user-1', + ); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual([ + 'user-1', + ]); + await coordinator.reconcile(); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), + ); + await coordinator.waitForIdle(); + expect(routerPush).not.toHaveBeenCalled(); + expect(removeUser).not.toHaveBeenCalled(); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual([ + 'user-1', + ]); + await coordinator.shutdown(); + }); }); diff --git a/modules/database/src/realtime/__tests__/normalize.test.ts b/modules/database/src/realtime/__tests__/normalize.test.ts index db2278e29..f7f933d33 100644 --- a/modules/database/src/realtime/__tests__/normalize.test.ts +++ b/modules/database/src/realtime/__tests__/normalize.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from '@jest/globals'; import { ObjectId } from 'bson'; -import { normalizeChangeEvent, parseResumeToken } from '../normalize.js'; +import { + normalizeChangeEvent, + parseResumeToken, + serializeResumeToken, +} from '../normalize.js'; describe('normalizeChangeEvent', () => { it('normalizes insert/update/replace/delete into metadata-only events', () => { @@ -22,7 +26,8 @@ describe('normalizeChangeEvent', () => { documentId: '64b64c4c4c4c4c4c4c4c4c4c', occurredAt: '2026-01-01T00:00:00.000Z', }); - expect(parseResumeToken(event!.resumeToken)).toEqual(resume); + expect(parseResumeToken(serializeResumeToken(resume))).toEqual(resume); + expect(event).not.toHaveProperty('resumeToken'); expect(JSON.parse(JSON.stringify(event))).not.toHaveProperty('fullDocument'); }); diff --git a/modules/database/src/realtime/__tests__/recovery.test.ts b/modules/database/src/realtime/__tests__/recovery.test.ts new file mode 100644 index 000000000..eee45b918 --- /dev/null +++ b/modules/database/src/realtime/__tests__/recovery.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from '@jest/globals'; +import { canReadDocument } from '../authorize.js'; +import { authorizedDocumentRoom } from '../rooms.js'; +import { isRecoverableDisconnect, restoreAuthorizedSubscriptions } from '../recovery.js'; +import { createSocketHandlers } from '../sockets.js'; +import { RealtimeSubscriptionTracker } from '../subscriptions.js'; + +class MemoryStore { + private sets = new Map>(); + async sadd(key: string, ...members: string[]) { + const set = this.sets.get(key) ?? new Set(); + members.forEach(member => set.add(member)); + this.sets.set(key, set); + } + async srem(key: string, ...members: string[]) { + members.forEach(member => this.sets.get(key)?.delete(member)); + } + async smembers(key: string) { + return [...(this.sets.get(key) ?? new Set())]; + } + async scard(key: string) { + return this.sets.get(key)?.size ?? 0; + } + async del(...keys: string[]) { + keys.forEach(key => this.sets.delete(key)); + } +} + +describe('database socket recovery', () => { + it('does not treat transport close as a wipe', () => { + expect(isRecoverableDisconnect('transport close')).toBe(true); + expect(isRecoverableDisconnect('client namespace disconnect')).toBe(false); + }); + + it('restores authorized Redis membership after a recoverable disconnect', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + const room = authorizedDocumentRoom('Order', 'doc-1', 'user-1'); + await tracker.addAuthorizedDocument('sock-1', 'Order', 'doc-1', 'user-1'); + await tracker.disconnect('sock-1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); + + await restoreAuthorizedSubscriptions({ + socketId: 'sock-1', + rooms: [room], + contextSubs: [], + subscriptions: tracker, + grpcSdk: { + isAvailable: () => true, + authorization: { can: async () => ({ allow: true }) }, + }, + canRead: canReadDocument, + }); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + }); + + it('keeps recovered /database/ sockets in the authorized list', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + await tracker.addAuthorizedDocument('sock-1', 'Order', 'doc-1', 'user-1'); + const handlers = createSocketHandlers({ + mode: 'client', + grpcSdk: { + isAvailable: () => true, + authorization: { can: async () => ({ allow: true }) }, + } as never, + schemaLookup: { getSchema: () => undefined }, + subscriptions: tracker, + isGloballyEnabled: () => true, + }); + await handlers.disconnect({ + request: { socketId: 'sock-1', params: ['transport close'] }, + } as never); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + await handlers.recovered({ + request: { + socketId: 'sock-1', + params: [authorizedDocumentRoom('Order', 'doc-1', 'user-1')], + context: {}, + }, + } as never); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + }); +}); diff --git a/modules/database/src/realtime/__tests__/rooms.test.ts b/modules/database/src/realtime/__tests__/rooms.test.ts index dbdd57e1b..96de67f73 100644 --- a/modules/database/src/realtime/__tests__/rooms.test.ts +++ b/modules/database/src/realtime/__tests__/rooms.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from '@jest/globals'; import { authorizedDocumentRoom, documentRoom, + parseAuthorizedDocumentRoom, roomsForPublicChange, schemaRoom, } from '../rooms.js'; @@ -21,6 +22,13 @@ describe('realtime rooms', () => { expect(authorizedDocumentRoom('Order', 'abc', 'u1')).not.toBe( authorizedDocumentRoom('Order', 'abc', 'u2'), ); + expect( + parseAuthorizedDocumentRoom(authorizedDocumentRoom('Order', 'abc', 'user-1')), + ).toEqual({ + schema: 'Order', + documentId: 'abc', + userId: 'user-1', + }); }); it('encodes reserved characters so rooms cannot collide', () => { diff --git a/modules/database/src/realtime/__tests__/topology.test.ts b/modules/database/src/realtime/__tests__/topology.test.ts index abf8e8861..79afac87c 100644 --- a/modules/database/src/realtime/__tests__/topology.test.ts +++ b/modules/database/src/realtime/__tests__/topology.test.ts @@ -10,7 +10,10 @@ describe('topology helpers', () => { it('detects unusable resume tokens', () => { expect(isResumeTokenUnusable({ code: 280 })).toBe(true); - expect(isResumeTokenUnusable(new Error('cannot resume'))).toBe(true); + expect(isResumeTokenUnusable({ code: 237 })).toBe(false); + expect(isResumeTokenUnusable({ code: 136 })).toBe(false); + expect(isResumeTokenUnusable(new Error('ChangeStreamHistoryLost'))).toBe(true); + expect(isResumeTokenUnusable(new Error('cannot resume'))).toBe(false); expect(isResumeTokenUnusable(new Error('socket hang up'))).toBe(false); }); }); diff --git a/modules/database/src/realtime/__tests__/watchPipeline.test.ts b/modules/database/src/realtime/__tests__/watchPipeline.test.ts new file mode 100644 index 000000000..7c31a64f5 --- /dev/null +++ b/modules/database/src/realtime/__tests__/watchPipeline.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from '@jest/globals'; +import { buildWatchPipeline } from '../watchPipeline.js'; + +describe('buildWatchPipeline', () => { + it('matches opted-in collections and projects out fullDocument', () => { + const pipeline = buildWatchPipeline(['orders', 'items']); + expect(pipeline[0]).toEqual({ + $match: { + $or: [ + { + operationType: { $in: ['insert', 'update', 'replace', 'delete'] }, + 'ns.coll': { $in: ['orders', 'items'] }, + }, + { + operationType: { $in: ['drop', 'rename', 'invalidate', 'dropDatabase'] }, + }, + ], + }, + }); + expect(pipeline[1]).toEqual({ + $project: { + fullDocument: 0, + updateDescription: 0, + fullDocumentBeforeChange: 0, + }, + }); + }); +}); diff --git a/modules/database/src/realtime/normalize.ts b/modules/database/src/realtime/normalize.ts index bdd876f16..b2f4b8ff3 100644 --- a/modules/database/src/realtime/normalize.ts +++ b/modules/database/src/realtime/normalize.ts @@ -42,10 +42,18 @@ export function normalizeChangeEvent( ? change.wallTime : occurredAt ).toISOString(), - resumeToken: EJSON.stringify(change._id), }; } +export function serializeResumeToken(id: unknown): string | null { + if (id === undefined || id === null) return null; + try { + return EJSON.stringify(id); + } catch { + return null; + } +} + export function parseResumeToken(token: string | null | undefined): unknown | undefined { if (!token) return undefined; try { diff --git a/modules/database/src/realtime/rebacCache.ts b/modules/database/src/realtime/rebacCache.ts new file mode 100644 index 000000000..b33a9438c --- /dev/null +++ b/modules/database/src/realtime/rebacCache.ts @@ -0,0 +1,96 @@ +import type { AuthorizationSdk } from './authorize.js'; + +export type RebacDecision = 'allow' | 'deny' | 'unavailable'; + +const DEFAULT_TTL_MS = 12_000; +const DEFAULT_MAX_ENTRIES = 10_000; + +type CacheEntry = { + decision: 'allow' | 'deny'; + expiresAt: number; +}; + +export class RealtimeRebacCache { + private readonly entries = new Map(); + + constructor( + private readonly ttlMs: number = DEFAULT_TTL_MS, + private readonly maxEntries: number = DEFAULT_MAX_ENTRIES, + ) {} + + async check( + grpcSdk: AuthorizationSdk, + userId: string, + schema: string, + documentId: string, + ): Promise { + const resource = `${schema}:${documentId}`; + const key = `${userId}:read:${resource}`; + const now = Date.now(); + this.sweepExpired(now); + const cached = this.entries.get(key); + if (cached && cached.expiresAt > now) { + return cached.decision; + } + if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { + return 'unavailable'; + } + try { + const decision = await grpcSdk.authorization.can({ + subject: `User:${userId}`, + actions: ['read'], + resource, + }); + const value: 'allow' | 'deny' = decision.allow ? 'allow' : 'deny'; + this.set(key, value, now); + return value; + } catch { + return 'unavailable'; + } + } + + clear(): void { + this.entries.clear(); + } + + private set(key: string, decision: 'allow' | 'deny', now: number): void { + if (this.entries.size >= this.maxEntries && !this.entries.has(key)) { + const firstKey = this.entries.keys().next().value; + if (firstKey) { + this.entries.delete(firstKey); + } + } + this.entries.set(key, { decision, expiresAt: now + this.ttlMs }); + } + + private sweepExpired(now: number): void { + for (const [key, entry] of this.entries) { + if (entry.expiresAt <= now) { + this.entries.delete(key); + } + } + } +} + +export async function checkRebacBatch( + cache: RealtimeRebacCache, + grpcSdk: AuthorizationSdk, + userIds: string[], + schema: string, + documentId: string, + concurrency = 8, +): Promise> { + const results = new Map(); + let index = 0; + async function worker(): Promise { + while (index < userIds.length) { + const userId = userIds[index++]; + results.set(userId, await cache.check(grpcSdk, userId, schema, documentId)); + } + } + const workers = Array.from({ length: Math.min(concurrency, userIds.length) }, () => + worker(), + ); + await Promise.all(workers); + return results; +} diff --git a/modules/database/src/realtime/recovery.ts b/modules/database/src/realtime/recovery.ts new file mode 100644 index 000000000..3bc9c3868 --- /dev/null +++ b/modules/database/src/realtime/recovery.ts @@ -0,0 +1,147 @@ +import type { Indexable, ParsedSocketRequest } from '@conduitplatform/grpc-sdk'; +import type { AuthorizationSdk } from './authorize.js'; +import { authorizedDocumentRoom, parseAuthorizedDocumentRoom } from './rooms.js'; +import type { RealtimeSubscriptionTracker } from './subscriptions.js'; + +const RECOVERABLE_DISCONNECT = new Set([ + 'ping timeout', + 'transport close', + 'transport error', +]); + +const CONTEXT_KEY = 'databaseSubs'; + +export type AuthorizedSub = { + schema: string; + documentId: string; + userId: string; +}; + +export function isRecoverableDisconnect(reason: unknown): boolean { + return typeof reason === 'string' && RECOVERABLE_DISCONNECT.has(reason); +} + +export function authorizedSubsFromContext( + context: Indexable | undefined, +): AuthorizedSub[] { + if (!context) return []; + const raw = context[CONTEXT_KEY]; + if (!Array.isArray(raw)) return []; + const subs: AuthorizedSub[] = []; + for (const item of raw) { + if ( + item && + typeof item === 'object' && + typeof (item as AuthorizedSub).schema === 'string' && + typeof (item as AuthorizedSub).documentId === 'string' && + typeof (item as AuthorizedSub).userId === 'string' + ) { + subs.push({ + schema: (item as AuthorizedSub).schema, + documentId: (item as AuthorizedSub).documentId, + userId: (item as AuthorizedSub).userId, + }); + } + } + return subs; +} + +export function persistAuthorizedSubOnContext( + context: Indexable | undefined, + schema: string, + documentId: string, + userId: string, +): void { + if (!context) return; + const next = authorizedSubsFromContext(context).filter( + sub => + !(sub.schema === schema && sub.documentId === documentId && sub.userId === userId), + ); + next.push({ schema, documentId, userId }); + context[CONTEXT_KEY] = next; +} + +export function removeAuthorizedSubFromContext( + context: Indexable | undefined, + schema: string, + documentId: string, + userId: string, +): void { + if (!context) return; + const next = authorizedSubsFromContext(context).filter( + sub => + !(sub.schema === schema && sub.documentId === documentId && sub.userId === userId), + ); + if (next.length === 0) { + delete context[CONTEXT_KEY]; + } else { + context[CONTEXT_KEY] = next; + } +} + +export function recoveredRoomsFromRequest(call: ParsedSocketRequest): string[] { + const params = call.request.params ?? []; + if (params.every(item => typeof item === 'string')) { + return params as string[]; + } + return []; +} + +export async function restoreAuthorizedSubscriptions(options: { + socketId: string; + rooms: string[]; + contextSubs: AuthorizedSub[]; + subscriptions: RealtimeSubscriptionTracker; + grpcSdk: AuthorizationSdk; + canRead: ( + grpcSdk: AuthorizationSdk, + schema: string, + documentId: string, + userId: string, + ) => Promise; +}): Promise<{ leaveRooms: string[] }> { + const seen = new Set(); + const subs: AuthorizedSub[] = []; + for (const room of options.rooms) { + const parsed = parseAuthorizedDocumentRoom(room); + if (!parsed) continue; + const key = `${parsed.schema}:${parsed.documentId}:${parsed.userId}`; + if (seen.has(key)) continue; + seen.add(key); + subs.push(parsed); + } + for (const sub of options.contextSubs) { + const key = `${sub.schema}:${sub.documentId}:${sub.userId}`; + if (seen.has(key)) continue; + seen.add(key); + subs.push(sub); + } + + const leaveRooms: string[] = []; + for (const sub of subs) { + const allowed = await options.canRead( + options.grpcSdk, + sub.schema, + sub.documentId, + sub.userId, + ); + const room = authorizedDocumentRoom(sub.schema, sub.documentId, sub.userId); + if (!allowed) { + await options.subscriptions.removeAuthorizedDocument( + options.socketId, + sub.schema, + sub.documentId, + sub.userId, + ); + leaveRooms.push(room); + continue; + } + await options.subscriptions.addAuthorizedDocument( + options.socketId, + sub.schema, + sub.documentId, + sub.userId, + ); + } + return { leaveRooms }; +} diff --git a/modules/database/src/realtime/rooms.ts b/modules/database/src/realtime/rooms.ts index 45a69a7cd..5792482f9 100644 --- a/modules/database/src/realtime/rooms.ts +++ b/modules/database/src/realtime/rooms.ts @@ -23,3 +23,25 @@ export function authorizedDocumentRoom( export function roomsForPublicChange(schema: string, documentId: string): string[] { return [schemaRoom(schema), documentRoom(schema, documentId)]; } + +export function parseAuthorizedDocumentRoom(room: string): { + schema: string; + documentId: string; + userId: string; +} | null { + const prefix = `${ROOM_PREFIX}:doc:`; + if (!room.startsWith(prefix)) return null; + const rest = room.slice(prefix.length); + const userMarker = ':user:'; + const userIndex = rest.lastIndexOf(userMarker); + if (userIndex === -1) return null; + const userId = decodeURIComponent(rest.slice(userIndex + userMarker.length)); + const schemaDoc = rest.slice(0, userIndex); + const lastColon = schemaDoc.lastIndexOf(':'); + if (lastColon === -1) return null; + return { + schema: decodeURIComponent(schemaDoc.slice(0, lastColon)), + documentId: decodeURIComponent(schemaDoc.slice(lastColon + 1)), + userId, + }; +} diff --git a/modules/database/src/realtime/sockets.ts b/modules/database/src/realtime/sockets.ts index 4be4e1018..e2a80931c 100644 --- a/modules/database/src/realtime/sockets.ts +++ b/modules/database/src/realtime/sockets.ts @@ -1,6 +1,7 @@ import { ConduitGrpcSdk, ConduitRouteReturnDefinition, + type Indexable, ParsedSocketRequest, TYPE, UnparsedSocketResponse, @@ -19,6 +20,14 @@ import { } from './authorize.js'; import { authorizedDocumentRoom, documentRoom, schemaRoom } from './rooms.js'; import type { RealtimeSubscriptionTracker } from './subscriptions.js'; +import { + authorizedSubsFromContext, + isRecoverableDisconnect, + persistAuthorizedSubOnContext, + recoveredRoomsFromRequest, + removeAuthorizedSubFromContext, + restoreAuthorizedSubscriptions, +} from './recovery.js'; type SocketMode = 'client' | 'admin'; @@ -43,6 +52,7 @@ export function registerDatabaseRealtimeSocket( { connect: { handler: handlers.connect }, disconnect: { handler: handlers.disconnect }, + recovered: { handler: handlers.recovered }, subscribe: { params: [TYPE.JSON], handler: handlers.subscribe, @@ -61,15 +71,32 @@ export function registerDatabaseRealtimeSocket( ); } -function createSocketHandlers(options: RealtimeSocketOptions) { +export function createSocketHandlers(options: RealtimeSocketOptions) { return { connect: async (): Promise => { return { event: 'connected', data: { ok: true } }; }, disconnect: async (call: ParsedSocketRequest): Promise => { - await options.subscriptions.disconnect(call.request.socketId); + const reason = call.request.params?.[0]; + if (!isRecoverableDisconnect(reason)) { + await options.subscriptions.disconnect(call.request.socketId); + } return { event: 'disconnected', data: { ok: true } }; }, + recovered: async (call: ParsedSocketRequest): Promise => { + const { leaveRooms } = await restoreAuthorizedSubscriptions({ + socketId: call.request.socketId, + rooms: recoveredRoomsFromRequest(call), + contextSubs: authorizedSubsFromContext(call.request.context as Indexable), + subscriptions: options.subscriptions, + grpcSdk: options.grpcSdk as unknown as AuthorizationSdk, + canRead: canReadDocument, + }); + if (leaveRooms.length > 0) { + return { event: 'leave-room', rooms: leaveRooms }; + } + return { event: 'join-room', rooms: [] }; + }, subscribe: async (call: ParsedSocketRequest): Promise => { const rooms = await resolveSubscription(call, options, 'join'); return { event: 'join-room', rooms }; @@ -140,6 +167,12 @@ async function resolveSubscription( documentId, userId, ); + persistAuthorizedSubOnContext( + call.request.context as Indexable, + schemaName, + documentId, + userId, + ); } else { await options.subscriptions.removeAuthorizedDocument( call.request.socketId, @@ -147,6 +180,12 @@ async function resolveSubscription( documentId, userId, ); + removeAuthorizedSubFromContext( + call.request.context as Indexable, + schemaName, + documentId, + userId, + ); } return [authorizedDocumentRoom(schemaName, documentId, userId)]; } diff --git a/modules/database/src/realtime/topology.ts b/modules/database/src/realtime/topology.ts index 341d49438..4d62fa3fa 100644 --- a/modules/database/src/realtime/topology.ts +++ b/modules/database/src/realtime/topology.ts @@ -1,6 +1,4 @@ -const CHANGE_STREAM_ERROR_CODES = new Set([ - 136, // CappedPositionLost - 237, // CursorKilled +const UNUSABLE_RESUME_TOKEN_CODES = new Set([ 280, // ChangeStreamHistoryLost 286, // ChangeStreamFatalError ]); @@ -33,11 +31,14 @@ export function topologyFromHello( export function isResumeTokenUnusable(error: unknown): boolean { const code = extractErrorCode(error); - if (code !== undefined && CHANGE_STREAM_ERROR_CODES.has(code)) { + if (code === 237 || code === 136) { + return false; + } + if (code !== undefined && UNUSABLE_RESUME_TOKEN_CODES.has(code)) { return true; } const message = error instanceof Error ? error.message : String(error ?? ''); - return /resume token|ChangeStreamHistoryLost|cannot resume/i.test(message); + return /ChangeStreamHistoryLost/i.test(message); } function extractErrorCode(error: unknown): number | undefined { diff --git a/modules/database/src/realtime/types.ts b/modules/database/src/realtime/types.ts index 9901c2d86..2b6e3e8c7 100644 --- a/modules/database/src/realtime/types.ts +++ b/modules/database/src/realtime/types.ts @@ -15,7 +15,6 @@ export type DatabaseChangeEvent = { schema: string; documentId: string; occurredAt: string; - resumeToken: string; }; export type RealtimeStatusCode = diff --git a/modules/database/src/realtime/watchPipeline.ts b/modules/database/src/realtime/watchPipeline.ts new file mode 100644 index 000000000..8ca888262 --- /dev/null +++ b/modules/database/src/realtime/watchPipeline.ts @@ -0,0 +1,33 @@ +const DATA_OPERATIONS = ['insert', 'update', 'replace', 'delete'] as const; +const CONTROL_OPERATIONS = ['drop', 'rename', 'invalidate', 'dropDatabase'] as const; + +export const WATCH_RESTART_OPERATIONS = new Set(CONTROL_OPERATIONS); + +export type WatchPipeline = Record[]; + +export function buildWatchPipeline(collectionNames: string[]): WatchPipeline { + return [ + { + $match: { + $or: [ + { + operationType: { $in: [...DATA_OPERATIONS] }, + 'ns.coll': { $in: collectionNames }, + }, + { operationType: { $in: [...CONTROL_OPERATIONS] } }, + ], + }, + }, + { + $project: { + fullDocument: 0, + updateDescription: 0, + fullDocumentBeforeChange: 0, + }, + }, + ]; +} + +export function optedInCollectionsKey(collectionNames: string[]): string { + return [...collectionNames].sort().join('\0'); +} diff --git a/modules/router/README.mdx b/modules/router/README.mdx index c896f2cee..a0dad3165 100644 --- a/modules/router/README.mdx +++ b/modules/router/README.mdx @@ -92,12 +92,12 @@ import { io } from 'socket.io-client'; const socket = io(`${SOCKET_BASE_URL}/events/`, { path: '/realtime', - extraHeaders: { - authorization: `Bearer ${accessToken}`, - }, + auth: { token: accessToken }, }); -socket.emit('subscribe', relayId, resourceId); +socket.on('connect', () => { + socket.emit('subscribe', relayId, resourceId); +}); socket.on('order-updated', payload => { // payload is the rendered JSON template }); @@ -108,6 +108,20 @@ socket.emit('unsubscribe', relayId, resourceId); - The outbound event name is the relay `socketEvent`. - Placeholders in `messageTemplate` use `{{payload.path}}` against the bus JSON payload. +**Subscribe-only.** Clients cannot publish on `/events/`; modules publish to the Redis bus and relays forward matching messages to subscribed rooms. + +**Authentication and Authorization** are required (`authMiddleware` on subscribe). Authorization is re-checked on a short TTL during delivery; revoked access triggers `leave-room`. + +**Rooms** are server-assigned: `er::`. Conduit has no tenant model and **team ids are not part of room names** — isolation is ReBAC on `User` × `permission` × `resourceType:resourceId`. + +**HA / `localOnly`:** every Router replica subscribes to the bus and emits with `localOnly`, so clients spread across replicas still receive one copy without sticky sessions for delivery. Engine.IO **polling** transport still benefits from sticky sessions or WebSocket-only clients. + +**EventBus self-publish:** the bus ignores this process’s own publishes (signature filter). A relay on a channel the Router publishes will not deliver to local clients for those self-originated messages (typical for module-originated events). + +**Dual delivery warning:** Database `/database/` `change` events and a relay on overlapping bus channels (for example `database:change:*` or CRUD `database:update:*`) can duplicate notifications. CRUD bus payloads include **full documents** — prefer change-stream style payloads when both paths are enabled. + +Use preview to validate templates: `POST /router/event-relays/preview` with `messageTemplate` and `samplePayload` (same renderer as runtime; sample JSON is capped at 256 KiB like inbound bus payloads). + ### Admin API | Method | Path | Description | @@ -115,5 +129,6 @@ socket.emit('unsubscribe', relayId, resourceId); | `GET` | `/router/event-relays` | Paginated list (`skip`, `limit`, `search`) | | `GET` | `/router/event-relays/:id` | Single relay | | `POST` | `/router/event-relays` | Create | -| `PATCH` | `/router/event-relays/:id` | Update | +| `POST` | `/router/event-relays/preview` | Render template against sample JSON | +| `PATCH` | `/router/event-relays/:id` | Update (deactivating evicts rooms) | | `DELETE` | `/router/event-relays/:id` | Delete | diff --git a/modules/router/src/Router.ts b/modules/router/src/Router.ts index 0a3867ed6..78369f6e7 100644 --- a/modules/router/src/Router.ts +++ b/modules/router/src/Router.ts @@ -1,4 +1,4 @@ -import { NextFunction } from 'express'; +import { NextFunction, Response } from 'express'; import { status } from '@grpc/grpc-js'; import { ConduitGrpcSdk, @@ -66,6 +66,11 @@ export default class ConduitDefaultRouter extends ManagedModule { private adminRouter: AdminHandlers; private readonly _routes: string[]; private readonly _globalMiddlewares: string[]; + private readonly _socketGlobalMiddlewareHandlers: (( + req: ConduitRequest, + res: Response, + next: NextFunction, + ) => void)[]; private _grpcRoutes: { [field: string]: RouteT[]; } = {}; @@ -76,12 +81,15 @@ export default class ConduitDefaultRouter extends ManagedModule { private _haInitialized = false; private eventRelayManager: EventRelayManager; private eventsSocket?: ConduitSocket; + private socketsPreviouslyStopped = false; + private securityMiddlewareInitialized = false; constructor(peerManifestRoot?: string) { super('router', peerManifestRoot); this.updateHealth(HealthCheckStatus.UNKNOWN, true); this._routes = []; this._globalMiddlewares = []; + this._socketGlobalMiddlewareHandlers = []; } async onServerStart() { @@ -112,6 +120,12 @@ export default class ConduitDefaultRouter extends ManagedModule { this.eventRelayManager = new EventRelayManager( this.grpcSdk, createEventRelayPusher(data => this._internalRouter.socketPush(data)), + { + getLocalRoomUserIds: (room: string) => + this._internalRouter.getLocalRoomUserIds('/events/', room), + getLocalRoomsWithPrefix: (prefix: string) => + this._internalRouter.getLocalRoomsWithPrefix('/events/', prefix), + }, ); this.adminRouter = new AdminHandlers( this.grpcServer, @@ -133,6 +147,10 @@ export default class ConduitDefaultRouter extends ManagedModule { async onConfig() { const config = ConfigController.getInstance().config; + const shouldRebindSocketGlobals = + config.transports.sockets && + this.socketsPreviouslyStopped && + this.securityMiddlewareInitialized; let atLeastOne = false; if (config.transports.graphql) { this._internalRouter.initGraphQL(); @@ -148,16 +166,24 @@ export default class ConduitDefaultRouter extends ManagedModule { } if (config.transports.sockets) { this._internalRouter.initSockets(); - this.registerEventsNamespace(); await this.eventRelayManager.start(); atLeastOne = true; } else { await this.eventRelayManager?.stop(); this._internalRouter.stopSockets(); + this.socketsPreviouslyStopped = true; } - if (atLeastOne) { + if (atLeastOne && !this.securityMiddlewareInitialized) { this._security.setupMiddlewares(); + this.securityMiddlewareInitialized = true; + } + if (config.transports.sockets) { + if (shouldRebindSocketGlobals) { + this.rebindSocketGlobalMiddlewares(); + } + this.socketsPreviouslyStopped = false; + this.registerEventsNamespace(); } if (!this._sdkRoutes.some(r => r.path === '/ready')) { this.registerRoute(adminRoutes.getReadyRoute()); @@ -335,13 +361,29 @@ export default class ConduitDefaultRouter extends ManagedModule { registerGlobalMiddleware( name: string, - middleware: any, + middleware: (req: ConduitRequest, res: Response, next: NextFunction) => void, socketMiddleware: boolean = false, ) { this._globalMiddlewares.push(name); + if (socketMiddleware) { + this._socketGlobalMiddlewareHandlers.push(middleware); + } this._internalRouter.registerMiddleware(middleware, socketMiddleware); } + private rebindSocketGlobalMiddlewares() { + for (const middleware of this._socketGlobalMiddlewareHandlers) { + this._internalRouter.registerSocketGlobalMiddleware(middleware); + } + } + + async shutdown(): Promise { + if (this.eventRelayManager) { + await this.eventRelayManager.stop(); + } + this.grpcSdk.bus?.quit(); + } + getRegisteredRoutes() { return this._routes; } diff --git a/modules/router/src/admin/event-relays.ts b/modules/router/src/admin/event-relays.ts index 2e56fbd15..442b3efd7 100644 --- a/modules/router/src/admin/event-relays.ts +++ b/modules/router/src/admin/event-relays.ts @@ -10,7 +10,9 @@ import { EventRelay } from '../models/index.js'; import { EventRelayManager } from '../event-relays/EventRelayManager.js'; import { EventRelayInput, validateEventRelayInput } from '../event-relays/validation.js'; import { EventRelayValidationError } from '../event-relays/validationError.js'; +import { renderMessageTemplate } from '../event-relays/template.js'; import { buildSearchQuery, parsePagination } from '../event-relays/search.js'; +import { assertJsonPayloadSize } from '../event-relays/process.js'; export class EventRelayAdmin { constructor(private readonly manager: EventRelayManager) {} @@ -52,6 +54,23 @@ export class EventRelayAdmin { return relay; } + async previewEventRelay(call: ParsedRouterRequest): Promise { + const { messageTemplate, samplePayload } = call.request.params as { + messageTemplate: unknown; + samplePayload: unknown; + }; + try { + assertJsonPayloadSize(samplePayload); + const rendered = renderMessageTemplate(messageTemplate, samplePayload); + return { rendered }; + } catch (err) { + if (err instanceof EventRelayValidationError) { + throw new GrpcError(status.INVALID_ARGUMENT, err.message); + } + throw err; + } + } + async patchEventRelay(call: ParsedRouterRequest): Promise { const existing = await EventRelay.getInstance().findOne({ _id: call.request.params.id, @@ -89,7 +108,18 @@ export class EventRelayAdmin { ...input, messageTemplate: input.messageTemplate as EventRelay['messageTemplate'], }); - await this.manager.notifyChanged(); + const evictRelayIds: string[] = []; + if (!input.active) { + evictRelayIds.push(existing._id); + } else if ( + input.permission !== existing.permission || + input.resourceType !== existing.resourceType + ) { + evictRelayIds.push(existing._id); + } + await this.manager.notifyChanged({ + evictRelayIds: evictRelayIds.length ? evictRelayIds : undefined, + }); return updated!; } @@ -101,7 +131,7 @@ export class EventRelayAdmin { throw new GrpcError(status.NOT_FOUND, 'Event relay not found'); } await EventRelay.getInstance().deleteOne({ _id: existing._id }); - await this.manager.notifyChanged(); + await this.manager.notifyChanged({ evictRelayIds: [existing._id] }); return { message: 'Event relay deleted' }; } } diff --git a/modules/router/src/admin/index.ts b/modules/router/src/admin/index.ts index 0c3231378..e943c25fa 100644 --- a/modules/router/src/admin/index.ts +++ b/modules/router/src/admin/index.ts @@ -242,6 +242,21 @@ export class AdminHandlers { }), this.eventRelayAdmin.deleteEventRelay.bind(this.eventRelayAdmin), ); + this.routingManager.route( + { + path: '/event-relays/preview', + action: ConduitRouteActions.POST, + description: `Renders a relay message template against sample bus JSON (same engine as runtime).`, + bodyParams: { + messageTemplate: ConduitJson.Required, + samplePayload: ConduitJson.Required, + }, + }, + new ConduitRouteReturnDefinition('PreviewEventRelay', { + rendered: ConduitJson.Required, + }), + this.eventRelayAdmin.previewEventRelay.bind(this.eventRelayAdmin), + ); this.routingManager.registerRoutes(); } } diff --git a/modules/router/src/event-relays/EventRelayManager.ts b/modules/router/src/event-relays/EventRelayManager.ts index 80cb809b7..905b0180e 100644 --- a/modules/router/src/event-relays/EventRelayManager.ts +++ b/modules/router/src/event-relays/EventRelayManager.ts @@ -3,70 +3,170 @@ import { EventRelay } from '../models/index.js'; import { EVENT_RELAY_REFRESH_CHANNEL, EVENT_RELAY_SUBSCRIBER_PREFIX, + RECONCILE_INTERVAL_MS, + RELAY_REBAC_TTL_MS, } from './constants.js'; import { groupRelaysByChannel, planChannelSubscriptions } from './channels.js'; -import { buildRelayEmissions, parseBusPayload } from './process.js'; +import { parseBusPayload } from './process.js'; import { EventRelayPusher } from './push.js'; +import { compileRelay, CompiledRelay } from './compile.js'; +import { checkRebacBatch, RelayRebacCache } from './rebacCache.js'; +import { eventRelayRoomPrefix } from './rooms.js'; +import { removeSubscriptionsForRelay } from './subscriptions.js'; export type { EventRelayPusher } from './push.js'; export { createEventRelayPusher } from './push.js'; +export type EventRelaySocketAccess = { + getLocalRoomUserIds: (room: string) => Promise; + getLocalRoomsWithPrefix: (prefix: string) => Promise; +}; + +type RefreshPayload = { + evictRelayIds?: string[]; +}; + export class EventRelayManager { - private readonly relaysByChannel = new Map(); + private readonly relaysById = new Map(); + private readonly relaysByChannel = new Map(); private readonly subscribedChannels = new Set(); + private readonly rebacCache = new RelayRebacCache(RELAY_REBAC_TTL_MS); private started = false; + private reconcilePending = false; + private reconciling = false; + private reconcileTimer?: NodeJS.Timeout; constructor( private readonly grpcSdk: ConduitGrpcSdk, private readonly push: EventRelayPusher, + private readonly sockets: EventRelaySocketAccess, ) {} async start(): Promise { if (!this.started) { this.grpcSdk.bus?.subscribe( EVENT_RELAY_REFRESH_CHANNEL, - () => { - this.reconcile().catch(err => { - ConduitGrpcSdk.Logger.error(err as Error); - }); + message => { + void this.onRefreshMessage(message); }, 'router-event-relays-refresh', ); + this.reconcileTimer = setInterval(() => { + void this.reconcile(); + }, RECONCILE_INTERVAL_MS); this.started = true; } await this.reconcile(); } async stop(): Promise { + if (this.reconcileTimer) { + clearInterval(this.reconcileTimer); + this.reconcileTimer = undefined; + } for (const channel of [...this.subscribedChannels]) { this.grpcSdk.bus?.unsubscribe(`${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`); this.subscribedChannels.delete(channel); } this.relaysByChannel.clear(); + this.relaysById.clear(); + this.rebacCache.clear(); if (this.started) { this.grpcSdk.bus?.unsubscribe('router-event-relays-refresh'); this.started = false; } + ConduitGrpcSdk.Metrics?.set('event_relays_active_total', 0); + ConduitGrpcSdk.Metrics?.set('event_relays_subscribed_channels_total', 0); } - async notifyChanged(): Promise { + async notifyChanged(options?: { evictRelayIds?: string[] }): Promise { + if (options?.evictRelayIds?.length) { + for (const relayId of options.evictRelayIds) { + await this.evictRelayRooms(relayId); + } + } if (this.started) { await this.reconcile(); } - this.grpcSdk.bus?.publish(EVENT_RELAY_REFRESH_CHANNEL, ''); + const payload: RefreshPayload = { + evictRelayIds: options?.evictRelayIds ?? [], + }; + this.grpcSdk.bus?.publish(EVENT_RELAY_REFRESH_CHANNEL, JSON.stringify(payload)); + } + + async evictRelayRooms(relayId: string): Promise { + const prefix = eventRelayRoomPrefix(relayId); + const rooms = await this.sockets.getLocalRoomsWithPrefix(prefix); + if (rooms.length === 0) { + removeSubscriptionsForRelay(relayId); + return; + } + await this.push('leave-room', undefined, rooms); + removeSubscriptionsForRelay(relayId); + } + + getActiveRelay(id: string): EventRelay | undefined { + return this.relaysById.get(id); + } + + private async onRefreshMessage(raw: string): Promise { + let payload: RefreshPayload = {}; + if (raw.trim()) { + try { + payload = JSON.parse(raw) as RefreshPayload; + } catch { + payload = {}; + } + } + if (payload.evictRelayIds?.length) { + for (const relayId of payload.evictRelayIds) { + await this.evictRelayRooms(relayId); + } + } + await this.reconcile(); } async reconcile(): Promise { + this.reconcilePending = true; + if (this.reconciling) { + return; + } + this.reconciling = true; + try { + while (this.reconcilePending) { + this.reconcilePending = false; + await this.runReconcile(); + } + } finally { + this.reconciling = false; + } + } + + private async runReconcile(): Promise { + const previousIds = new Set(this.relaysById.keys()); const relays = await EventRelay.getInstance().findMany({ active: true }); - const next = groupRelaysByChannel(relays); + const nextByChannel = groupRelaysByChannel(relays); const { toSubscribe, toUnsubscribe } = planChannelSubscriptions( this.subscribedChannels, - next.keys(), + nextByChannel.keys(), ); + this.relaysById.clear(); this.relaysByChannel.clear(); - for (const [channel, list] of next) { - this.relaysByChannel.set(channel, list); + for (const relay of relays) { + this.relaysById.set(relay._id, relay); + } + for (const [channel, channelRelays] of nextByChannel) { + this.relaysByChannel.set( + channel, + channelRelays.map(relay => compileRelay(relay)), + ); + } + + for (const id of previousIds) { + if (!this.relaysById.has(id)) { + await this.evictRelayRooms(id); + } } for (const channel of toUnsubscribe) { @@ -75,21 +175,27 @@ export class EventRelayManager { } for (const channel of toSubscribe) { - this.grpcSdk.bus?.subscribe( - channel, - message => this.onBusMessage(channel, message), - `${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`, - ); - this.subscribedChannels.add(channel); + try { + await this.grpcSdk.bus?.subscribeAck( + channel, + message => this.onBusMessage(channel, message), + `${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`, + ); + this.subscribedChannels.add(channel); + } catch (err) { + ConduitGrpcSdk.Logger.error( + `Event relay failed to subscribe to bus channel ${channel}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } } - } - getActiveRelay(id: string): EventRelay | undefined { - for (const relays of this.relaysByChannel.values()) { - const match = relays.find(relay => relay._id === id); - if (match) return match; - } - return undefined; + ConduitGrpcSdk.Metrics?.set('event_relays_active_total', this.relaysById.size); + ConduitGrpcSdk.Metrics?.set( + 'event_relays_subscribed_channels_total', + this.subscribedChannels.size, + ); } private onBusMessage(channel: string, message: string): void { @@ -103,6 +209,13 @@ export class EventRelayManager { payload = parseBusPayload(message); } catch (err) { ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); + if ( + err instanceof Error && + err.message.includes('exceeds') && + err.message.includes('bytes') + ) { + ConduitGrpcSdk.Metrics?.increment('event_relays_inbound_dropped_total'); + } ConduitGrpcSdk.Logger.error( `Event relay failed to parse payload for ${channel}: ${ err instanceof Error ? err.message : String(err) @@ -111,27 +224,74 @@ export class EventRelayManager { return; } - const { emissions, failures } = buildRelayEmissions(relays, payload); - for (const failure of failures) { + for (const relay of relays) { + void this.emitCompiledRelay(relay, payload, channel); + } + } + + private async emitCompiledRelay( + relay: CompiledRelay, + payload: unknown, + channel: string, + ): Promise { + let room: string; + let data: unknown; + let resourceId: string; + try { + ({ room, data, resourceId } = relay.buildEmission(payload)); + } catch (err) { ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); ConduitGrpcSdk.Logger.warn( - `Event relay ${failure.relayId} skipped on ${failure.busEvent}: ${failure.reason}`, + `Event relay ${relay.relayId} skipped on ${relay.busEvent}: ${ + err instanceof Error ? err.message : String(err) + }`, ); + return; } - for (const emission of emissions) { - this.push(emission.socketEvent, emission.data, [emission.room]).then( - () => { - ConduitGrpcSdk.Metrics?.increment('event_relays_emitted_total'); - }, - err => { - ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); - ConduitGrpcSdk.Logger.error( - `Event relay ${emission.relayId} emit failed on ${channel}: ${ - err instanceof Error ? err.message : String(err) - }`, - ); - }, + const userIds = await this.sockets.getLocalRoomUserIds(room); + if (userIds.length === 0) { + ConduitGrpcSdk.Metrics?.increment('event_relays_empty_room_total'); + return; + } + + const decisions = await checkRebacBatch( + this.rebacCache, + this.grpcSdk, + userIds, + relay.permission, + relay.resourceType, + resourceId, + ); + + const allowedUsers: string[] = []; + for (const userId of userIds) { + const decision = decisions.get(userId) ?? 'unavailable'; + if (decision === 'allow') { + allowedUsers.push(userId); + } else if (decision === 'deny') { + await this.push('leave-room', undefined, [room], [userId]); + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + } + } + + if (allowedUsers.length === 0) { + return; + } + + try { + const emitted = await this.push(relay.socketEvent, data, [room], allowedUsers); + if (emitted) { + ConduitGrpcSdk.Metrics?.increment('event_relays_emitted_total'); + } else { + ConduitGrpcSdk.Metrics?.increment('event_relays_empty_room_total'); + } + } catch (err) { + ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); + ConduitGrpcSdk.Logger.error( + `Event relay ${relay.relayId} emit failed on ${channel}: ${ + err instanceof Error ? err.message : String(err) + }`, ); } } diff --git a/modules/router/src/event-relays/EventRelaySockets.ts b/modules/router/src/event-relays/EventRelaySockets.ts index 32e5fec2a..d9fe6f88c 100644 --- a/modules/router/src/event-relays/EventRelaySockets.ts +++ b/modules/router/src/event-relays/EventRelaySockets.ts @@ -1,11 +1,27 @@ import { status } from '@grpc/grpc-js'; import { ConduitGrpcSdk, GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; import { ConduitSocket, ConduitSocketEvent } from '@conduitplatform/hermes'; -import { EVENTS_NAMESPACE } from './constants.js'; +import { + EVENTS_NAMESPACE, + MAX_SUBSCRIBE_PER_MINUTE, +} from './constants.js'; import { EventRelayManager } from './EventRelayManager.js'; import { eventRelayRoom } from './rooms.js'; import { validateResourceId } from './validation.js'; import { authorizeRelaySubscription, toSubscriptionError } from './authorize.js'; +import { reauthorizeRecoveredSubscriptions } from './recovery.js'; +import { + releaseSocketSubscriptions, + removeSubscription, + trackSubscription, + _clearEventRelaySubscriptionStateForTests, +} from './subscriptions.js'; +import { + persistRelaySubscriptionOnContext, + removeRelaySubscriptionFromContext, +} from './relaySocketData.js'; + +const subscribeTimestamps = new Map(); export function createEventsSocket( grpcSdk: ConduitGrpcSdk, @@ -20,16 +36,40 @@ export function createEventsSocket( events.set('disconnect', { name: 'disconnect', - handler: async () => ({ event: 'leave-room', rooms: [] }), + handler: async request => { + const userId = request.context?.user?._id as string | undefined; + releaseSocketSubscriptions(request.socketId, userId); + subscribeTimestamps.delete(request.socketId); + return { event: 'leave-room', rooms: [] }; + }, }); events.set('subscribe', { name: 'subscribe', params: [TYPE.String, TYPE.String], handler: async request => { + assertSubscribeRateLimit(request.socketId); const userId = request.context?.user?._id as string | undefined; + if (!userId) { + throw new GrpcError(status.UNAUTHENTICATED, 'Authentication required'); + } const [relayId, resourceId] = request.params ?? []; + const validatedResourceId = validateResourceId(resourceId); const room = await authorizeOrThrow(grpcSdk, manager, userId, relayId, resourceId); + try { + trackSubscription( + request.socketId, + userId, + String(relayId), + validatedResourceId, + ); + persistRelaySubscriptionOnContext(request.context, String(relayId), validatedResourceId); + } catch (err) { + throw new GrpcError( + status.RESOURCE_EXHAUSTED, + err instanceof Error ? err.message : 'Subscription limit exceeded', + ); + } return { event: 'join-room', rooms: [room] }; }, }); @@ -38,12 +78,15 @@ export function createEventsSocket( name: 'unsubscribe', params: [TYPE.String, TYPE.String], handler: async request => { + const userId = request.context?.user?._id as string | undefined; const [relayId, resourceId] = request.params ?? []; if (typeof relayId !== 'string' || relayId.trim() === '') { throw new GrpcError(status.INVALID_ARGUMENT, 'Relay ID is required'); } try { const validatedResourceId = validateResourceId(resourceId); + removeSubscription(request.socketId, userId, relayId, validatedResourceId); + removeRelaySubscriptionFromContext(request.context, relayId, validatedResourceId); return { event: 'leave-room', rooms: [eventRelayRoom(relayId, validatedResourceId)], @@ -60,11 +103,26 @@ export function createEventsSocket( name: 'eventRelays', description: 'Declarative bus-to-socket event relays', middlewares: ['authMiddleware'], + onRecovered: async request => + reauthorizeRecoveredSubscriptions(grpcSdk, manager, request), }, events, ); } +function assertSubscribeRateLimit(socketId: string): void { + const now = Date.now(); + const windowStart = now - 60_000; + const timestamps = (subscribeTimestamps.get(socketId) ?? []).filter( + t => t >= windowStart, + ); + if (timestamps.length >= MAX_SUBSCRIBE_PER_MINUTE) { + throw new GrpcError(status.RESOURCE_EXHAUSTED, 'Subscribe rate limit exceeded'); + } + timestamps.push(now); + subscribeTimestamps.set(socketId, timestamps); +} + async function authorizeOrThrow( grpcSdk: ConduitGrpcSdk, manager: EventRelayManager, @@ -94,3 +152,9 @@ function toGrpcError(err: unknown): GrpcError { } export { authorizeRelaySubscription } from './authorize.js'; + +/** @internal test helper */ +export function _clearEventRelaySocketStateForTests(): void { + _clearEventRelaySubscriptionStateForTests(); + subscribeTimestamps.clear(); +} diff --git a/modules/router/src/event-relays/authorize.ts b/modules/router/src/event-relays/authorize.ts index 852cf5595..8de7cc005 100644 --- a/modules/router/src/event-relays/authorize.ts +++ b/modules/router/src/event-relays/authorize.ts @@ -62,7 +62,6 @@ export async function authorizeRelaySubscription( } if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { - onDenied?.(); throw new RelaySubscriptionError(status.UNAVAILABLE, 'Authorization is unavailable'); } @@ -75,7 +74,6 @@ export async function authorizeRelaySubscription( }); allowed = decision.allow; } catch { - onDenied?.(); throw new RelaySubscriptionError(status.UNAVAILABLE, 'Authorization check failed'); } diff --git a/modules/router/src/event-relays/compile.ts b/modules/router/src/event-relays/compile.ts new file mode 100644 index 000000000..bfe636592 --- /dev/null +++ b/modules/router/src/event-relays/compile.ts @@ -0,0 +1,50 @@ +import { requireOwnPath } from './path.js'; +import { eventRelayRoom } from './rooms.js'; +import { renderMessageTemplate } from './template.js'; +import { validateResourceId } from './validation.js'; + +export type RelayCompileInput = { + _id: string; + busEvent: string; + socketEvent: string; + resourceIdPath: string; + messageTemplate: unknown; + permission: string; + resourceType: string; +}; + +export type CompiledRelay = { + relayId: string; + busEvent: string; + socketEvent: string; + permission: string; + resourceType: string; + buildEmission: (payload: unknown) => { + room: string; + data: unknown; + resourceId: string; + }; +}; + +export function compileRelay(relay: RelayCompileInput): CompiledRelay { + const resourceIdPath = relay.resourceIdPath; + const template = relay.messageTemplate; + return { + relayId: relay._id, + busEvent: relay.busEvent, + socketEvent: relay.socketEvent, + permission: relay.permission, + resourceType: relay.resourceType, + buildEmission: (payload: unknown) => { + const resourceId = validateResourceId( + requireOwnPath(payload, resourceIdPath, 'Resource ID path'), + ); + const data = renderMessageTemplate(template, payload); + return { + room: eventRelayRoom(relay._id, resourceId), + data, + resourceId, + }; + }, + }; +} diff --git a/modules/router/src/event-relays/constants.ts b/modules/router/src/event-relays/constants.ts index e283da0dd..ce80f6194 100644 --- a/modules/router/src/event-relays/constants.ts +++ b/modules/router/src/event-relays/constants.ts @@ -15,6 +15,12 @@ export const MAX_RESOURCE_ID_LENGTH = 128; export const MAX_RESOURCE_ID_PATH_LENGTH = 128; export const MAX_PERMISSION_LENGTH = 64; +export const MAX_INBOUND_BUS_BYTES = 256 * 1024; +export const RECONCILE_INTERVAL_MS = 30_000; +export const MAX_SUBSCRIBE_PER_MINUTE = 30; +export const MAX_ROOMS_PER_SOCKET = 32; +export const RELAY_REBAC_TTL_MS = 12_000; + export const FORBIDDEN_PATH_SEGMENTS = new Set(['__proto__', 'constructor', 'prototype']); export const RESERVED_SOCKET_EVENTS = new Set([ diff --git a/modules/router/src/event-relays/follow-up.test.ts b/modules/router/src/event-relays/follow-up.test.ts new file mode 100644 index 000000000..fea6d438e --- /dev/null +++ b/modules/router/src/event-relays/follow-up.test.ts @@ -0,0 +1,186 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { + releaseSocketSubscriptions, + removeSubscriptionsForRelay, + subscriptionsForRecoveredRooms, + subscriptionsForUser, + trackSubscription, + _clearEventRelaySubscriptionStateForTests, +} from './subscriptions.js'; +import { eventRelayRoom } from './rooms.js'; +import { reauthorizeRecoveredSubscriptions } from './recovery.js'; +import { RelayLookup } from './authorize.js'; + +describe('event relay subscription tracking', () => { + it('clears user map entries when the only socket disconnects', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-a', 'user-1'); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); + + it('keeps user map entries when another socket still holds the subscription', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-1', 'relay-1', 'order-1'); + trackSubscription('socket-b', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-a', 'user-1'); + assert.deepEqual(subscriptionsForUser('user-1'), [ + { relayId: 'relay-1', resourceId: 'order-1' }, + ]); + }); + + it('clears relay subscriptions on eviction', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-1', 'relay-1', 'order-1'); + removeSubscriptionsForRelay('relay-1'); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); +}); + +describe('recovery re-authorization', () => { + const room = eventRelayRoom('relay-1', 'order-1'); + const relay = { + _id: 'relay-1', + permission: 'read', + resourceType: 'Order', + }; + + it('keeps restored rooms when Authorization is unavailable', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('old-socket', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('old-socket', 'user-1'); + + const manager = { + getActiveRelay: (id: string) => (id === 'relay-1' ? relay : undefined), + } as unknown as RelayLookup; + + const grpcSdk = { + isAvailable: () => false, + authorization: null, + }; + + const result = await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'new-socket', + context: { + user: { _id: 'user-1' }, + eventRelaySubs: [{ relayId: 'relay-1', resourceId: 'order-1' }], + }, + recoveredRooms: [room], + }); + + assert.deepEqual(result, { event: 'join-room', rooms: [] }); + assert.deepEqual(subscriptionsForUser('user-1'), [ + { relayId: 'relay-1', resourceId: 'order-1' }, + ]); + }); + + it('leaves restored rooms on permission deny', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('old-socket', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('old-socket', 'user-1'); + + const manager = { + getActiveRelay: (id: string) => (id === 'relay-1' ? relay : undefined), + } as unknown as RelayLookup; + + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async () => ({ allow: false }), + }, + }; + + const result = await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'new-socket', + context: { + user: { _id: 'user-1' }, + eventRelaySubs: [{ relayId: 'relay-1', resourceId: 'order-1' }], + }, + recoveredRooms: [room], + }); + + assert.deepEqual(result, { event: 'leave-room', rooms: [room] }); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); + + it('does not re-auth rooms outside the recovered socket session', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('old-socket', 'user-1', 'relay-1', 'order-1'); + trackSubscription('old-socket', 'user-1', 'relay-2', 'order-2'); + releaseSocketSubscriptions('old-socket', 'user-1'); + + const manager = { + getActiveRelay: (id: string) => + id === 'relay-2' + ? { _id: 'relay-2', permission: 'read', resourceType: 'Order' } + : undefined, + } as unknown as RelayLookup; + + let canCalls = 0; + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async () => { + canCalls++; + return { allow: false }; + }, + }, + }; + + const otherRoom = eventRelayRoom('relay-2', 'order-2'); + await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'new-socket', + context: { + user: { _id: 'user-1' }, + eventRelaySubs: [{ relayId: 'relay-2', resourceId: 'order-2' }], + }, + recoveredRooms: [otherRoom], + }); + + assert.equal(canCalls, 1); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); + + it('re-auths the first user after a second user subscribed to the same room', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-a', 'relay-1', 'order-1'); + trackSubscription('socket-b', 'user-b', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-b', 'user-b'); + + let checkedSubject = ''; + const manager = { + getActiveRelay: (id: string) => (id === 'relay-1' ? relay : undefined), + } as unknown as RelayLookup; + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async (request: { subject: string }) => { + checkedSubject = request.subject; + return { allow: true }; + }, + }, + }; + + await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'socket-a-new', + context: { user: { _id: 'user-a' } }, + recoveredRooms: [room], + }); + + assert.equal(checkedSubject, 'User:user-a'); + }); + + it('prefers socket.data subscriptions when the room map was pruned on disconnect', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-a', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-a', 'user-a'); + assert.deepEqual(subscriptionsForRecoveredRooms([room]), []); + assert.deepEqual( + subscriptionsForRecoveredRooms([room], [ + { relayId: 'relay-1', resourceId: 'order-1' }, + ]), + [{ relayId: 'relay-1', resourceId: 'order-1' }], + ); + }); +}); diff --git a/modules/router/src/event-relays/interventions.test.ts b/modules/router/src/event-relays/interventions.test.ts new file mode 100644 index 000000000..f9e37f644 --- /dev/null +++ b/modules/router/src/event-relays/interventions.test.ts @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { MAX_INBOUND_BUS_BYTES } from './constants.js'; +import { parseBusPayload } from './process.js'; +import { renderMessageTemplate } from './template.js'; + +describe('parseBusPayload inbound cap', () => { + it('drops payloads larger than 256KiB before JSON.parse', () => { + const oversized = JSON.stringify({ blob: 'x'.repeat(MAX_INBOUND_BUS_BYTES) }); + assert.throws(() => parseBusPayload(oversized), /exceeds/); + }); +}); + +describe('preview renderer parity', () => { + it('renders templates the same as runtime processing', () => { + const payload = { documentId: 'doc-1', nested: { value: 2 } }; + const template = { id: '{{payload.documentId}}', n: '{{payload.nested.value}}' }; + assert.deepEqual(renderMessageTemplate(template, payload), { + id: 'doc-1', + n: 2, + }); + }); +}); diff --git a/modules/router/src/event-relays/process.test.ts b/modules/router/src/event-relays/process.test.ts index cf3c1c10b..c811bcec5 100644 --- a/modules/router/src/event-relays/process.test.ts +++ b/modules/router/src/event-relays/process.test.ts @@ -69,6 +69,7 @@ describe('createEventRelayPusher', () => { const calls: unknown[] = []; const push = createEventRelayPusher(async data => { calls.push(data); + return true; }); await push('order-updated', { id: '1' }, ['room-1']); assert.deepEqual(calls, [ @@ -79,6 +80,8 @@ describe('createEventRelayPusher', () => { rooms: ['room-1'], namespace: EVENTS_NAMESPACE, localOnly: true, + skipEmptyRooms: true, + boundedEmit: true, }, ]); }); diff --git a/modules/router/src/event-relays/process.ts b/modules/router/src/event-relays/process.ts index 858c9f6d0..ebe6ec1ec 100644 --- a/modules/router/src/event-relays/process.ts +++ b/modules/router/src/event-relays/process.ts @@ -3,6 +3,7 @@ import { eventRelayRoom } from './rooms.js'; import { renderMessageTemplate } from './template.js'; import { validateResourceId } from './validation.js'; import { EventRelayValidationError } from './validationError.js'; +import { MAX_INBOUND_BUS_BYTES } from './constants.js'; export type RelayProcessInput = { _id: string; @@ -31,10 +32,16 @@ export type ProcessResult = { failures: RelayFailure[]; }; -export function parseBusPayload(rawMessage: string): unknown { +export function parseBusPayload( + rawMessage: string, + maxBytes: number = MAX_INBOUND_BUS_BYTES, +): unknown { if (typeof rawMessage !== 'string' || rawMessage.trim() === '') { throw new EventRelayValidationError('Bus payload is empty'); } + if (Buffer.byteLength(rawMessage, 'utf8') > maxBytes) { + throw new EventRelayValidationError(`Bus payload exceeds ${maxBytes} bytes`); + } try { return JSON.parse(rawMessage); } catch { @@ -42,6 +49,21 @@ export function parseBusPayload(rawMessage: string): unknown { } } +export function assertJsonPayloadSize( + payload: unknown, + maxBytes: number = MAX_INBOUND_BUS_BYTES, +): void { + let serialized: string; + try { + serialized = JSON.stringify(payload); + } catch { + throw new EventRelayValidationError('Sample payload must be valid JSON'); + } + if (Buffer.byteLength(serialized, 'utf8') > maxBytes) { + throw new EventRelayValidationError(`Sample payload exceeds ${maxBytes} bytes`); + } +} + export function buildRelayEmissions( relays: RelayProcessInput[], payload: unknown, diff --git a/modules/router/src/event-relays/push.ts b/modules/router/src/event-relays/push.ts index 79d9911b2..e4bed5062 100644 --- a/modules/router/src/event-relays/push.ts +++ b/modules/router/src/event-relays/push.ts @@ -4,7 +4,8 @@ export type EventRelayPusher = ( event: string, data: unknown, rooms: string[], -) => Promise; + receivers?: string[], +) => Promise; export type SocketPushFn = (data: { event: string; @@ -13,16 +14,20 @@ export type SocketPushFn = (data: { rooms: string[]; namespace: string; localOnly?: boolean; -}) => Promise; + skipEmptyRooms?: boolean; + boundedEmit?: boolean; +}) => Promise; export function createEventRelayPusher(socketPush: SocketPushFn): EventRelayPusher { - return (event, data, rooms) => + return async (event, data, rooms, receivers = []) => socketPush({ event, data, - receivers: [], + receivers, rooms, namespace: EVENTS_NAMESPACE, localOnly: true, + skipEmptyRooms: event !== 'leave-room', + boundedEmit: event !== 'leave-room', }); } diff --git a/modules/router/src/event-relays/rebacCache.test.ts b/modules/router/src/event-relays/rebacCache.test.ts new file mode 100644 index 000000000..df20365d5 --- /dev/null +++ b/modules/router/src/event-relays/rebacCache.test.ts @@ -0,0 +1,43 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { RelayRebacCache } from './rebacCache.js'; + +describe('RelayRebacCache', () => { + it('denies after TTL when authorization revokes access', async () => { + let allow = true; + const cache = new RelayRebacCache(10); + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async () => ({ allow }), + }, + }; + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'allow', + ); + allow = false; + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'allow', + ); + await new Promise(resolve => setTimeout(resolve, 15)); + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'deny', + ); + }); + + it('returns unavailable without caching when Authorization is down', async () => { + const cache = new RelayRebacCache(10_000); + const grpcSdk = { isAvailable: () => false, authorization: null }; + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'unavailable', + ); + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'unavailable', + ); + }); +}); diff --git a/modules/router/src/event-relays/rebacCache.ts b/modules/router/src/event-relays/rebacCache.ts new file mode 100644 index 000000000..a2d458300 --- /dev/null +++ b/modules/router/src/event-relays/rebacCache.ts @@ -0,0 +1,108 @@ +type RebacSdk = { + isAvailable: (module: string) => boolean; + authorization?: { + can: (request: { + subject: string; + actions: string[]; + resource: string; + }) => Promise<{ allow: boolean }>; + } | null; +}; + +export type RebacDecision = 'allow' | 'deny' | 'unavailable'; + +const DEFAULT_TTL_MS = 12_000; +const DEFAULT_MAX_ENTRIES = 10_000; + +type CacheEntry = { + decision: 'allow' | 'deny'; + expiresAt: number; +}; + +export class RelayRebacCache { + private readonly entries = new Map(); + constructor( + private readonly ttlMs: number = DEFAULT_TTL_MS, + private readonly maxEntries: number = DEFAULT_MAX_ENTRIES, + ) {} + + async check( + grpcSdk: RebacSdk, + userId: string, + permission: string, + resourceType: string, + resourceId: string, + ): Promise { + const resource = `${resourceType}:${resourceId}`; + const key = `${userId}:${permission}:${resource}`; + const now = Date.now(); + this.sweepExpired(now); + const cached = this.entries.get(key); + if (cached && cached.expiresAt > now) { + return cached.decision; + } + if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { + return 'unavailable'; + } + try { + const decision = await grpcSdk.authorization.can({ + subject: `User:${userId}`, + actions: [permission], + resource, + }); + this.set(key, decision.allow ? 'allow' : 'deny', now); + return decision.allow ? 'allow' : 'deny'; + } catch { + return 'unavailable'; + } + } + + clear(): void { + this.entries.clear(); + } + + private set(key: string, decision: 'allow' | 'deny', now: number): void { + if (this.entries.size >= this.maxEntries && !this.entries.has(key)) { + const firstKey = this.entries.keys().next().value; + if (firstKey) { + this.entries.delete(firstKey); + } + } + this.entries.set(key, { decision, expiresAt: now + this.ttlMs }); + } + + private sweepExpired(now: number): void { + for (const [key, entry] of this.entries) { + if (entry.expiresAt <= now) { + this.entries.delete(key); + } + } + } +} + +export async function checkRebacBatch( + cache: RelayRebacCache, + grpcSdk: RebacSdk, + userIds: string[], + permission: string, + resourceType: string, + resourceId: string, + concurrency = 8, +): Promise> { + const results = new Map(); + let index = 0; + async function worker(): Promise { + while (index < userIds.length) { + const userId = userIds[index++]; + results.set( + userId, + await cache.check(grpcSdk, userId, permission, resourceType, resourceId), + ); + } + } + const workers = Array.from({ length: Math.min(concurrency, userIds.length) }, () => + worker(), + ); + await Promise.all(workers); + return results; +} diff --git a/modules/router/src/event-relays/recovery.ts b/modules/router/src/event-relays/recovery.ts new file mode 100644 index 000000000..ceb6c4217 --- /dev/null +++ b/modules/router/src/event-relays/recovery.ts @@ -0,0 +1,85 @@ +import { status } from '@grpc/grpc-js'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { + authorizeRelaySubscription, + RelaySubscriptionError, + RelayLookup, + toSubscriptionError, +} from './authorize.js'; +import { eventRelayRoom } from './rooms.js'; +import { + removeSubscription, + subscriptionsForRecoveredRooms, + trackSubscription, +} from './subscriptions.js'; +import { relaySubscriptionsFromContext, persistRelaySubscriptionOnContext, removeRelaySubscriptionFromContext } from './relaySocketData.js'; + +type RelayAuthorizationSdk = Parameters[0]; + +export async function reauthorizeRecoveredSubscriptions( + grpcSdk: RelayAuthorizationSdk, + manager: RelayLookup, + request: { + socketId: string; + context?: { user?: { _id?: string } } & Record; + recoveredRooms?: string[]; + }, +): Promise< + { event: 'join-room'; rooms: string[] } | { event: 'leave-room'; rooms: string[] } +> { + const userId = request.context?.user?._id as string | undefined; + if (!userId) { + return { event: 'leave-room', rooms: [] }; + } + + const recoveredRooms = request.recoveredRooms ?? []; + const subs = subscriptionsForRecoveredRooms( + recoveredRooms, + relaySubscriptionsFromContext(request.context), + ); + const leaveRooms: string[] = []; + + for (const sub of subs) { + const room = eventRelayRoom(sub.relayId, sub.resourceId); + try { + await authorizeRelaySubscription( + grpcSdk, + manager, + userId, + sub.relayId, + sub.resourceId, + () => { + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + }, + ); + trackSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + persistRelaySubscriptionOnContext(request.context, sub.relayId, sub.resourceId); + } catch (err) { + const mapped = + err instanceof RelaySubscriptionError ? err : toSubscriptionError(err); + if (mapped.code === status.UNAVAILABLE) { + trackSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + persistRelaySubscriptionOnContext(request.context, sub.relayId, sub.resourceId); + continue; + } + if ( + mapped.code === status.PERMISSION_DENIED || + mapped.code === status.NOT_FOUND + ) { + removeSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + removeRelaySubscriptionFromContext(request.context, sub.relayId, sub.resourceId); + leaveRooms.push(room); + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + continue; + } + removeSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + removeRelaySubscriptionFromContext(request.context, sub.relayId, sub.resourceId); + leaveRooms.push(room); + } + } + + if (leaveRooms.length > 0) { + return { event: 'leave-room', rooms: leaveRooms }; + } + return { event: 'join-room', rooms: [] }; +} diff --git a/modules/router/src/event-relays/relaySocketData.ts b/modules/router/src/event-relays/relaySocketData.ts new file mode 100644 index 000000000..63d0eca9c --- /dev/null +++ b/modules/router/src/event-relays/relaySocketData.ts @@ -0,0 +1,65 @@ +import { Indexable } from '@conduitplatform/grpc-sdk'; +import { RelaySubscription } from './subscriptions.js'; + +const CONTEXT_KEY = 'eventRelaySubs'; + +export function relaySubscriptionsFromContext( + context: Indexable | undefined, +): RelaySubscription[] { + if (!context) { + return []; + } + const raw = context[CONTEXT_KEY]; + if (!Array.isArray(raw)) { + return []; + } + const subs: RelaySubscription[] = []; + for (const item of raw) { + if ( + item && + typeof item === 'object' && + typeof (item as RelaySubscription).relayId === 'string' && + typeof (item as RelaySubscription).resourceId === 'string' + ) { + subs.push({ + relayId: (item as RelaySubscription).relayId, + resourceId: (item as RelaySubscription).resourceId, + }); + } + } + return subs; +} + +export function persistRelaySubscriptionOnContext( + context: Indexable | undefined, + relayId: string, + resourceId: string, +): void { + if (!context) { + return; + } + const subs = relaySubscriptionsFromContext(context); + const withoutDup = subs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + withoutDup.push({ relayId, resourceId }); + context[CONTEXT_KEY] = withoutDup; +} + +export function removeRelaySubscriptionFromContext( + context: Indexable | undefined, + relayId: string, + resourceId: string, +): void { + if (!context) { + return; + } + const next = relaySubscriptionsFromContext(context).filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + if (next.length === 0) { + delete context[CONTEXT_KEY]; + } else { + context[CONTEXT_KEY] = next; + } +} diff --git a/modules/router/src/event-relays/rooms.ts b/modules/router/src/event-relays/rooms.ts index fd5dd54ec..0bf26bf74 100644 --- a/modules/router/src/event-relays/rooms.ts +++ b/modules/router/src/event-relays/rooms.ts @@ -4,3 +4,7 @@ export function eventRelayRoom(relayId: string, resourceId: string): string { const digest = createHash('sha256').update(resourceId).digest('hex'); return `er:${relayId}:${digest}`; } + +export function eventRelayRoomPrefix(relayId: string): string { + return `er:${relayId}:`; +} diff --git a/modules/router/src/event-relays/subscriptions.ts b/modules/router/src/event-relays/subscriptions.ts new file mode 100644 index 000000000..a7a083f56 --- /dev/null +++ b/modules/router/src/event-relays/subscriptions.ts @@ -0,0 +1,224 @@ +import { eventRelayRoom } from './rooms.js'; +import { MAX_ROOMS_PER_SOCKET } from './constants.js'; + +export type RelaySubscription = { + relayId: string; + resourceId: string; +}; + +const RECOVERY_ROOM_MAP_TTL_MS = 120_000; + +const subscriptionsBySocket = new Map(); +const subscriptionsByUser = new Map(); +const subscriptionsByRoom = new Map(); + +function pruneExpiredRoomEntries(now = Date.now()): void { + for (const [room, entry] of subscriptionsByRoom) { + if (entry.expiresAt <= now) { + subscriptionsByRoom.delete(room); + } + } +} + +function touchRoomEntry(relayId: string, resourceId: string): void { + const room = eventRelayRoom(relayId, resourceId); + subscriptionsByRoom.set(room, { + relayId, + resourceId, + expiresAt: Date.now() + RECOVERY_ROOM_MAP_TTL_MS, + }); +} + +function pruneRoomEntryIfUnused(relayId: string, resourceId: string): void { + if (!isSubscriptionTrackedOnAnySocket(relayId, resourceId)) { + subscriptionsByRoom.delete(eventRelayRoom(relayId, resourceId)); + } +} + +export function trackSubscription( + socketId: string, + userId: string, + relayId: string, + resourceId: string, +): void { + const entry = { relayId, resourceId }; + const socketSubs = subscriptionsBySocket.get(socketId) ?? []; + const withoutDup = socketSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + withoutDup.push(entry); + if (withoutDup.length > MAX_ROOMS_PER_SOCKET) { + throw new Error(`Cannot subscribe to more than ${MAX_ROOMS_PER_SOCKET} relay rooms`); + } + subscriptionsBySocket.set(socketId, withoutDup); + + const userSubs = subscriptionsByUser.get(userId) ?? []; + const userWithoutDup = userSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + userWithoutDup.push(entry); + subscriptionsByUser.set(userId, userWithoutDup); + touchRoomEntry(relayId, resourceId); +} + +export function removeSubscription( + socketId: string, + userId: string | undefined, + relayId: string, + resourceId: string, +): void { + const socketSubs = subscriptionsBySocket.get(socketId); + if (socketSubs) { + const next = socketSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + if (next.length === 0) { + subscriptionsBySocket.delete(socketId); + } else { + subscriptionsBySocket.set(socketId, next); + } + } + if (userId && !isSubscriptionTrackedOnOtherSocket(relayId, resourceId, socketId)) { + const userSubs = subscriptionsByUser.get(userId); + if (userSubs) { + const next = userSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + if (next.length === 0) { + subscriptionsByUser.delete(userId); + } else { + subscriptionsByUser.set(userId, next); + } + } + pruneRoomEntryIfUnused(relayId, resourceId); + } +} + +function isSubscriptionTrackedOnOtherSocket( + relayId: string, + resourceId: string, + exceptSocketId: string, +): boolean { + for (const [socketId, subs] of subscriptionsBySocket) { + if (socketId === exceptSocketId) { + continue; + } + if (subs.some(sub => sub.relayId === relayId && sub.resourceId === resourceId)) { + return true; + } + } + return false; +} + +function isSubscriptionTrackedOnAnySocket(relayId: string, resourceId: string): boolean { + for (const subs of subscriptionsBySocket.values()) { + if (subs.some(sub => sub.relayId === relayId && sub.resourceId === resourceId)) { + return true; + } + } + return false; +} + +export function releaseSocketSubscriptions(socketId: string, userId?: string): void { + const subs = subscriptionsBySocket.get(socketId) ?? []; + subscriptionsBySocket.delete(socketId); + if (!userId) { + return; + } + for (const sub of subs) { + if (!isSubscriptionTrackedOnOtherSocket(sub.relayId, sub.resourceId, socketId)) { + const userSubs = subscriptionsByUser.get(userId); + if (!userSubs) { + continue; + } + const next = userSubs.filter( + entry => + !(entry.relayId === sub.relayId && entry.resourceId === sub.resourceId), + ); + if (next.length === 0) { + subscriptionsByUser.delete(userId); + } else { + subscriptionsByUser.set(userId, next); + } + } + pruneRoomEntryIfUnused(sub.relayId, sub.resourceId); + } +} + +export function subscriptionsForRecoveredRooms( + recoveredRooms: string[], + contextSubs: RelaySubscription[] = [], +): RelaySubscription[] { + pruneExpiredRoomEntries(); + const seen = new Set(); + const result: RelaySubscription[] = []; + const recoveredSet = new Set(recoveredRooms.filter(room => room.startsWith('er:'))); + + for (const sub of contextSubs) { + const room = eventRelayRoom(sub.relayId, sub.resourceId); + if (recoveredSet.has(room) && !seen.has(room)) { + seen.add(room); + result.push(sub); + } + } + + for (const room of recoveredSet) { + if (seen.has(room)) { + continue; + } + const entry = subscriptionsByRoom.get(room); + if (entry) { + seen.add(room); + result.push({ relayId: entry.relayId, resourceId: entry.resourceId }); + } + } + + return result; +} + +export function subscriptionsForUser(userId: string): RelaySubscription[] { + return [...(subscriptionsByUser.get(userId) ?? [])]; +} + +export function removeSubscriptionsForRelay(relayId: string): void { + for (const [socketId, subs] of subscriptionsBySocket) { + const next = subs.filter(sub => sub.relayId !== relayId); + if (next.length === 0) { + subscriptionsBySocket.delete(socketId); + } else { + subscriptionsBySocket.set(socketId, next); + } + } + for (const [userId, subs] of subscriptionsByUser) { + const next = subs.filter(sub => sub.relayId !== relayId); + if (next.length === 0) { + subscriptionsByUser.delete(userId); + } else { + subscriptionsByUser.set(userId, next); + } + } + for (const [room, entry] of subscriptionsByRoom) { + if (entry.relayId === relayId) { + subscriptionsByRoom.delete(room); + } + } +} + +export function leaveRoomsForRelay(relayId: string): string[] { + const rooms = new Set(); + for (const subs of subscriptionsByUser.values()) { + for (const sub of subs) { + if (sub.relayId === relayId) { + rooms.add(eventRelayRoom(relayId, sub.resourceId)); + } + } + } + return [...rooms]; +} + +/** @internal test helper */ +export function _clearEventRelaySubscriptionStateForTests(): void { + subscriptionsBySocket.clear(); + subscriptionsByUser.clear(); + subscriptionsByRoom.clear(); +} diff --git a/modules/router/src/event-relays/validation.ts b/modules/router/src/event-relays/validation.ts index d6c6c5eb6..c59976335 100644 --- a/modules/router/src/event-relays/validation.ts +++ b/modules/router/src/event-relays/validation.ts @@ -33,24 +33,75 @@ const PERMISSION_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,63}$/; const RESOURCE_ID_PATTERN = /^[^\s:]{1,128}$/; export function validateEventRelayInput(input: EventRelayInput): EventRelayInput { - const name = requireTrimmed(input.name, 'Name'); + const name = parseName(input.name); + const notes = parseNotes(input.notes); + const busEvent = parseBusEvent(input.busEvent); + const socketEvent = parseSocketEvent(input.socketEvent); + const resourceType = parseResourceType(input.resourceType); + const resourceIdPath = parseResourceIdPath(input.resourceIdPath); + const permission = parsePermission(input.permission); + const messageTemplate = parseMessageTemplate(input.messageTemplate); + + return { + name, + notes, + active: input.active !== false, + busEvent, + socketEvent, + resourceType, + resourceIdPath, + permission, + messageTemplate, + }; +} + +export function validateResourceId(resourceId: unknown): string { + if (typeof resourceId !== 'string' && typeof resourceId !== 'number') { + throw new EventRelayValidationError('Resource ID must be a string'); + } + const value = String(resourceId).trim(); + if ( + !value || + value.length > MAX_RESOURCE_ID_LENGTH || + !RESOURCE_ID_PATTERN.test(value) + ) { + throw new EventRelayValidationError('Resource ID is invalid'); + } + return value; +} + +function requireTrimmed(value: unknown, label: string): string { + if (typeof value !== 'string' || value.trim() === '') { + throw new EventRelayValidationError(`${label} is required`); + } + return value.trim(); +} + +function parseName(raw: unknown): string { + const name = requireTrimmed(raw, 'Name'); if (name.length > MAX_NAME_LENGTH || !NAME_PATTERN.test(name)) { throw new EventRelayValidationError( 'Name must be 1-64 characters and start with a letter or number', ); } + return name; +} - const notes = - input.notes === undefined || input.notes === '' - ? undefined - : requireTrimmed(input.notes, 'Notes'); - if (notes && notes.length > MAX_DESCRIPTION_LENGTH) { +function parseNotes(raw: unknown): string | undefined { + if (raw === undefined || raw === '') { + return undefined; + } + const notes = requireTrimmed(raw, 'Notes'); + if (notes.length > MAX_DESCRIPTION_LENGTH) { throw new EventRelayValidationError( `Notes must be at most ${MAX_DESCRIPTION_LENGTH} characters`, ); } + return notes; +} - const busEvent = requireTrimmed(input.busEvent, 'Bus event'); +function parseBusEvent(raw: unknown): string { + const busEvent = requireTrimmed(raw, 'Bus event'); if ( busEvent.length > MAX_BUS_EVENT_LENGTH || busEvent.includes('*') || @@ -60,8 +111,11 @@ export function validateEventRelayInput(input: EventRelayInput): EventRelayInput 'Bus event must be an exact channel name with no wildcards', ); } + return busEvent; +} - const socketEvent = requireTrimmed(input.socketEvent, 'Socket event'); +function parseSocketEvent(raw: unknown): string { + const socketEvent = requireTrimmed(raw, 'Socket event'); if ( socketEvent.length > MAX_SOCKET_EVENT_LENGTH || !SOCKET_EVENT_PATTERN.test(socketEvent) || @@ -69,62 +123,41 @@ export function validateEventRelayInput(input: EventRelayInput): EventRelayInput ) { throw new EventRelayValidationError('Socket event must be a non-reserved event name'); } + return socketEvent; +} - const resourceType = requireTrimmed(input.resourceType, 'Resource type'); +function parseResourceType(raw: unknown): string { + const resourceType = requireTrimmed(raw, 'Resource type'); if ( resourceType.length > MAX_RESOURCE_TYPE_LENGTH || !RESOURCE_TYPE_PATTERN.test(resourceType) ) { throw new EventRelayValidationError('Resource type is invalid'); } + return resourceType; +} - const resourceIdPath = requireTrimmed(input.resourceIdPath, 'Resource ID path'); +function parseResourceIdPath(raw: unknown): string { + const resourceIdPath = requireTrimmed(raw, 'Resource ID path'); if (resourceIdPath.length > MAX_RESOURCE_ID_PATH_LENGTH) { throw new EventRelayValidationError('Resource ID path is too long'); } parseDotPath(resourceIdPath, 'Resource ID path'); + return resourceIdPath; +} - const permission = requireTrimmed(input.permission, 'Permission'); +function parsePermission(raw: unknown): string { + const permission = requireTrimmed(raw, 'Permission'); if (permission.length > MAX_PERMISSION_LENGTH || !PERMISSION_PATTERN.test(permission)) { throw new EventRelayValidationError('Permission is invalid'); } - - if (input.messageTemplate === undefined) { - throw new EventRelayValidationError('Message template is required'); - } - assertTemplateSize(input.messageTemplate); - - return { - name, - notes, - active: input.active !== false, - busEvent, - socketEvent, - resourceType, - resourceIdPath, - permission, - messageTemplate: input.messageTemplate, - }; -} - -export function validateResourceId(resourceId: unknown): string { - if (typeof resourceId !== 'string' && typeof resourceId !== 'number') { - throw new EventRelayValidationError('Resource ID must be a string'); - } - const value = String(resourceId).trim(); - if ( - !value || - value.length > MAX_RESOURCE_ID_LENGTH || - !RESOURCE_ID_PATTERN.test(value) - ) { - throw new EventRelayValidationError('Resource ID is invalid'); - } - return value; + return permission; } -function requireTrimmed(value: unknown, label: string): string { - if (typeof value !== 'string' || value.trim() === '') { - throw new EventRelayValidationError(`${label} is required`); +function parseMessageTemplate(raw: unknown): unknown { + if (raw === undefined) { + throw new EventRelayValidationError('Message template is required'); } - return value.trim(); + assertTemplateSize(raw); + return raw; } diff --git a/modules/router/src/index.ts b/modules/router/src/index.ts index 20a0c779e..fd1d069ed 100644 --- a/modules/router/src/index.ts +++ b/modules/router/src/index.ts @@ -1,7 +1,25 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import ConduitDefaultRouter from './Router.js'; const peerManifestRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const router = new ConduitDefaultRouter(peerManifestRoot); + +function registerShutdownSignals(): void { + const shutdown = (signal: NodeJS.Signals) => { + void router + .shutdown() + .catch(err => { + ConduitGrpcSdk.Logger.error(err as Error); + }) + .finally(() => { + process.exit(signal === 'SIGINT' ? 130 : 0); + }); + }; + process.once('SIGTERM', () => shutdown('SIGTERM')); + process.once('SIGINT', () => shutdown('SIGINT')); +} + +registerShutdownSignals(); router.start(); diff --git a/modules/router/src/metrics/index.ts b/modules/router/src/metrics/index.ts index d3461c107..2ed45b20f 100644 --- a/modules/router/src/metrics/index.ts +++ b/modules/router/src/metrics/index.ts @@ -38,4 +38,39 @@ export default { help: 'Tracks denied or unavailable event-relay socket subscriptions', }, }, + eventRelaysActive: { + type: MetricType.Gauge, + config: { + name: 'event_relays_active_total', + help: 'Active event relays on this router replica', + }, + }, + eventRelaysSubscribedChannels: { + type: MetricType.Gauge, + config: { + name: 'event_relays_subscribed_channels_total', + help: 'Bus channels this router replica subscribes to for event relays', + }, + }, + eventRelaysEmptyRoom: { + type: MetricType.Counter, + config: { + name: 'event_relays_empty_room_total', + help: 'Skipped emits because no local sockets were in the relay room', + }, + }, + eventRelaysInboundDropped: { + type: MetricType.Counter, + config: { + name: 'event_relays_inbound_dropped_total', + help: 'Inbound bus payloads dropped for exceeding the size cap', + }, + }, + eventRelaysEmitDropped: { + type: MetricType.Counter, + config: { + name: 'event_relays_emit_dropped_total', + help: 'Socket emits dropped or disconnected due to backpressure', + }, + }, }; diff --git a/modules/router/tsconfig.test.json b/modules/router/tsconfig.test.json index 601e5ccee..229f7f155 100644 --- a/modules/router/tsconfig.test.json +++ b/modules/router/tsconfig.test.json @@ -19,6 +19,9 @@ "src/event-relays/search.ts", "src/event-relays/channels.ts", "src/event-relays/authorize.ts", + "src/event-relays/compile.ts", + "src/event-relays/rebacCache.ts", + "src/event-relays/subscriptions.ts", "src/event-relays/*.test.ts" ] } diff --git a/packages/core/src/admin/middleware/Admin.middleware.ts b/packages/core/src/admin/middleware/Admin.middleware.ts index 716dae763..4fb743246 100644 --- a/packages/core/src/admin/middleware/Admin.middleware.ts +++ b/packages/core/src/admin/middleware/Admin.middleware.ts @@ -2,9 +2,8 @@ import { NextFunction, Response } from 'express'; import { isNil } from 'lodash-es'; // Removed ConduitCommons import - now using configManager directly import { isDev } from '../utils/middleware.js'; -import { ConduitRequest } from '@conduitplatform/hermes'; +import { ConduitRequest, isSocketHandshake } from '@conduitplatform/hermes'; import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; -import { isSocketHandshake } from '../realtime/handshake.js'; export function getAdminMiddleware(configManager: any) { return async function adminMiddleware( diff --git a/packages/core/src/admin/middleware/Auth.middleware.ts b/packages/core/src/admin/middleware/Auth.middleware.ts index 410ea0fd6..9bac68d99 100644 --- a/packages/core/src/admin/middleware/Auth.middleware.ts +++ b/packages/core/src/admin/middleware/Auth.middleware.ts @@ -7,8 +7,7 @@ import { isDev } from '../utils/middleware.js'; import { ConduitRequest } from '@conduitplatform/hermes'; import { gql } from 'graphql-tag'; import { ConfigController } from '@conduitplatform/module-tools'; -import { isSocketHandshake } from '../realtime/handshake.js'; -import { isRealtimeTicket } from '../realtime/ticket.js'; +import { realtimeTicketForbiddenOnHttp } from '../realtime/ticket.js'; const excludedRestRoutes = ['/ready', '/live', '/login', '/config/modules']; const excludedGqlOperations = [ @@ -115,7 +114,7 @@ async function handleJwtToken( } const { id } = decoded; - if (isRealtimeTicket(decoded) && !isSocketHandshake(req)) { + if (realtimeTicketForbiddenOnHttp(decoded, req)) { res.status(401).json({ error: 'Realtime ticket cannot be used for HTTP requests' }); return; } diff --git a/packages/core/src/admin/realtime/handshake.test.ts b/packages/core/src/admin/realtime/handshake.test.ts deleted file mode 100644 index 59d17547f..000000000 --- a/packages/core/src/admin/realtime/handshake.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; -import { isSocketHandshake } from './handshake.js'; - -describe('isSocketHandshake', () => { - it('detects Socket.IO engine requests', () => { - assert.equal(isSocketHandshake({ url: '/realtime/?EIO=4&transport=polling' }), true); - assert.equal(isSocketHandshake({ originalUrl: '/realtime' }), true); - assert.equal(isSocketHandshake({ path: '/login' }), false); - assert.equal(isSocketHandshake({ url: '/graphql' }), false); - }); -}); diff --git a/packages/core/src/admin/realtime/handshake.ts b/packages/core/src/admin/realtime/handshake.ts deleted file mode 100644 index a4bf06fe1..000000000 --- a/packages/core/src/admin/realtime/handshake.ts +++ /dev/null @@ -1,14 +0,0 @@ -export function isSocketHandshake(req: { - url?: string; - originalUrl?: string; - path?: string; -}): boolean { - const parts = [req.url, req.originalUrl, req.path].filter((value): value is string => - Boolean(value), - ); - return parts.some(part => part.includes('EIO=') || isRealtimePath(part)); -} - -function isRealtimePath(value: string): boolean { - return /(?:^|[/?])realtime(?:[/?]|$)/.test(value); -} diff --git a/packages/core/src/admin/realtime/ticket-http.test.ts b/packages/core/src/admin/realtime/ticket-http.test.ts new file mode 100644 index 000000000..589975acc --- /dev/null +++ b/packages/core/src/admin/realtime/ticket-http.test.ts @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import jwt from 'jsonwebtoken'; +import { ConfigController } from '@conduitplatform/module-tools'; +import { isSocketHandshake } from '@conduitplatform/hermes'; +import { getAuthMiddleware } from '../middleware/Auth.middleware.js'; +import { + buildRealtimeTicketClaims, + isRealtimeTicket, + realtimeTicketForbiddenOnHttp, +} from './ticket.js'; + +function mockResponse() { + let statusCode = 0; + let body: unknown; + const res = { + status(code: number) { + statusCode = code; + return this; + }, + json(payload: unknown) { + body = payload; + return this; + }, + }; + return { + res, + get statusCode() { + return statusCode; + }, + get body() { + return body; + }, + }; +} + +describe('realtime ticket HTTP guard', () => { + it('does not treat POST /realtime/ticket as a handshake so a ticket cannot mint another', () => { + const ticket = buildRealtimeTicketClaims('admin-1'); + assert.equal(isRealtimeTicket(ticket), true); + assert.equal(isSocketHandshake({ url: '/realtime/ticket' }), false); + assert.equal( + isSocketHandshake({ url: '/realtime/ticket?EIO=4&transport=polling' }), + false, + ); + assert.equal( + realtimeTicketForbiddenOnHttp(ticket, { url: '/realtime/ticket' }), + true, + ); + assert.equal( + realtimeTicketForbiddenOnHttp(ticket, { + url: '/realtime/ticket?EIO=4&transport=polling', + }), + true, + ); + }); + + it('allows a ticket on Engine.IO handshake polling without sid', () => { + const ticket = buildRealtimeTicketClaims('admin-1'); + const handshake = { url: '/realtime/?EIO=4&transport=polling' }; + assert.equal(isSocketHandshake(handshake), true); + assert.equal(realtimeTicketForbiddenOnHttp(ticket, handshake), false); + }); + + it('Auth middleware returns 401 on POST /realtime/ticket with a realtime ticket', async () => { + const secret = 'ticket-http-test-secret'; + ConfigController.getInstance().config = { auth: { tokenSecret: secret } }; + const token = jwt.sign(buildRealtimeTicketClaims('admin-1'), secret, { + algorithm: 'HS256', + expiresIn: 30, + }); + const middleware = getAuthMiddleware({} as never, { + get: async () => ({ env: 'production' }), + }); + const mock = mockResponse(); + let nextCalled = false; + await middleware( + { + path: '/realtime/ticket', + originalUrl: '/realtime/ticket', + url: '/realtime/ticket', + method: 'POST', + headers: { authorization: `Bearer ${token}` }, + conduit: {}, + } as never, + mock.res as never, + () => { + nextCalled = true; + }, + ); + assert.equal(nextCalled, false); + assert.equal(mock.statusCode, 401); + assert.deepEqual(mock.body, { + error: 'Realtime ticket cannot be used for HTTP requests', + }); + }); +}); diff --git a/packages/core/src/admin/realtime/ticket.ts b/packages/core/src/admin/realtime/ticket.ts index 154630c0c..e797f5966 100644 --- a/packages/core/src/admin/realtime/ticket.ts +++ b/packages/core/src/admin/realtime/ticket.ts @@ -1,3 +1,5 @@ +import { isSocketHandshake } from '@conduitplatform/hermes'; + export const ADMIN_REALTIME_AUDIENCE = 'admin-realtime'; export const ADMIN_REALTIME_TICKET_TTL_SECONDS = 30; @@ -24,3 +26,10 @@ export function isRealtimeTicket( } return audience === ADMIN_REALTIME_AUDIENCE; } + +export function realtimeTicketForbiddenOnHttp( + decoded: RealtimeTicketClaims | null | undefined, + req: { url?: string; originalUrl?: string }, +): boolean { + return isRealtimeTicket(decoded) && !isSocketHandshake(req); +} diff --git a/packages/core/tsconfig.test.json b/packages/core/tsconfig.test.json index b0548057f..8559c14bf 100644 --- a/packages/core/tsconfig.test.json +++ b/packages/core/tsconfig.test.json @@ -9,7 +9,6 @@ }, "include": [ "src/admin/realtime/ticket.ts", - "src/admin/realtime/handshake.ts", "src/admin/realtime/namespace.ts", "src/admin/realtime/*.test.ts" ], From 47eccc2335ab7cd00bc77fbd86addb60675c03d1 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 13 Sep 2026 21:54:40 +0300 Subject: [PATCH 6/8] fix(database): restack Mongo live updates onto current Event Relays (#1606) * fix(router): Event Relays interventions (EventBus, Hermes, relay manager) (#1604) * fix(router): harden event relays for HA and auth lifetime EventBus uses a single Redis message dispatcher with subscriberId maps, SIGTERM/SIGINT shutdown, and subscribe-after-ACK. Hermes installs engine middleware once, skips empty-room namespace broadcasts, demotes hot-path logs, and re-authenticates recovered /events/ subscriptions. EventRelayManager gains coalesced periodic reconcile, compiled templates, inbound caps, TTL-cached emit-time ReBAC, room eviction, backpressure, preview API, and metrics/docs aligned with the interventions plan. * fix(router): address #1604 re-review P1 and P2 follow-ups Recovery re-auth keeps per-user subscription state across disconnect; emit-time ReBAC distinguishes unavailable vs deny; EventBus drops process signal handlers and adds subscribeAck; manager subscribes only after Redis ACK and broadcasts evictRelayIds on refresh; Hermes backpressure uses local sockets without emit acks; cache is bounded; preview caps sample JSON; handshake matcher rejects ticket/sid/non-namespace paths; socket middleware rebind avoids duplicate registration on first sockets enable. * fix(hermes): restore production tsc for engine middleware chain Type the Socket.IO engine middleware runner as Express NextFunction so recursive callbacks match registerGlobalMiddleware. Use Logger.info for socket trace lines (IConduitLogger has no debug). * fix(router): import Express Response for socket global middleware _rebindSocketGlobalMiddlewares passes handlers typed with fetch Response because Response was not imported from express, failing tsc against Hermes registerSocketGlobalMiddleware. * fix(router): quit EventBus on module shutdown signals Stop event relays and call grpcSdk.bus.quit() from Router.shutdown(), registered on SIGTERM/SIGINT with process exit so Redis teardown is not left to SDK signal handlers. Tighten registerGlobalMiddleware typing and trim handshake helper comment (deslop). * fix(router): pass-3 recovery, scoped emit, and subscription hygiene Recovery re-auth keeps membership on Authorization UNAVAILABLE and only leaves on deny; re-check subscriptions for restored er: rooms via room map. Prune user subscriptions on disconnect when no other socket holds them. Scope relay emits to receivers in the target room; use Engine.IO writeBuffer/writable for backpressure. Drop hot-path Hermes socket info logs. * fix(router): pass-4 recovery map, backpressure, emit tests Store relay subs on socket.data; resolve recovered rooms from context and TTL room map without last-writer userId. Prune room map when unused on disconnect. Backpressure uses writeBuffer depth only. Add Hermes/router tests for room-scoped emit and queue metric. * fix(router): clear eventRelaySubs from socket.data on recovery deny * fix(router): clear socket.data subs on recovery fail-closed leave * fix(router): address CodeFactor findings on event relay validation Split validateEventRelayInput into per-field parsers to reduce complexity. Silence unused-parameter lint in EventBus test FakeRedis stub. * feat(database): restack Mongo live updates onto current Event Relays Replay the Mongo delta onto feat/router-event-relays (9e6356ef). Keep #1604 Hermes (engine.use, er: recovery, writeBuffer) and graft only the port-keyed adapter plus register-before-initSockets. Do not take old #1601 Socket.ts wholesale. Co-authored-by: Konstantinos Kopanidis * fix(database): fence change-stream leadership and expire recovery Redis keys Lock renew failure bumps a generation so a draining watch cannot emit. Do not open a watch unless the leader lock extends after acquire. Arm TTL on realtime socket/doc keys when a recoverable disconnect never recovers; persist on restore. CMS-read deny fails closed at emit without extra ReBAC round-trips and keeps membership; UNAVAILABLE keeps membership. Cover dropDatabase reopen. Co-authored-by: Konstantinos Kopanidis * fix(database): serialize leader acquire so concurrent reconcile still watches Two overlapping ensureLeader calls both bumped the lock generation while the first openStream was still reading the resume token, so the watch never attached. Hold an acquiring flag so only one opener runs. --------- --- modules/database/README.mdx | 2 +- .../realtime/MongoChangeStreamCoordinator.ts | 99 ++++++++++++++++--- .../realtime/__tests__/coordinator.test.ts | 95 ++++++++++++++++-- .../src/realtime/__tests__/recovery.test.ts | 62 +++++++++++- .../realtime/__tests__/subscriptions.test.ts | 45 ++++++++- modules/database/src/realtime/authorize.ts | 24 +++-- modules/database/src/realtime/rebacCache.ts | 3 +- modules/database/src/realtime/recovery.ts | 12 +-- modules/database/src/realtime/sockets.ts | 5 +- .../database/src/realtime/subscriptions.ts | 56 +++++++++-- modules/database/src/realtime/types.ts | 3 + 11 files changed, 354 insertions(+), 52 deletions(-) diff --git a/modules/database/README.mdx b/modules/database/README.mdx index b84cded13..10703937b 100644 --- a/modules/database/README.mdx +++ b/modules/database/README.mdx @@ -64,7 +64,7 @@ Events arrive as `change` with `{ version, operation, schema, documentId, occurr The leader also publishes `database:change:${schema}` on the Redis bus. **Do not also relay `database:change:*` on `/events/` if the same client is on `/database/`** — that duplicates notifications. Keep the bus for other modules; just do not dual-subscribe. -Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Client sockets also require CMS `crudOperations.read.enabled`. Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; that token cannot mint another ticket or call REST/GraphQL. Session JWTs and masterkeys must not be sent from browser code. +Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Client sockets also require CMS `crudOperations.read.enabled` (checked again at emit: deny skips client delivery and keeps membership; authorization UNAVAILABLE keeps membership without emitting). Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; that token cannot mint another ticket or call REST/GraphQL. Session JWTs and masterkeys must not be sent from browser code. Admin sockets must be enabled (`admin.transports.sockets`) and the Admin socket port (`ADMIN_SOCKET_PORT`, default 3031) reachable from the UI. diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts index d29938d15..45644fa78 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -66,7 +66,9 @@ export class MongoChangeStreamCoordinator { private retryAttempt = 0; private watching = false; private opening = false; + private acquiring = false; private ignoreClose = false; + private lockGeneration = 0; private changeQueue: Promise = Promise.resolve(); private watchedCollectionsKey = ''; private readonly rebacCache = new RealtimeRebacCache(); @@ -150,7 +152,15 @@ export class MongoChangeStreamCoordinator { } return; } + if (this.acquiring) return; + this.acquiring = true; try { + if (this.lock) { + if (!this.watching) { + await this.openStream(); + } + return; + } const acquired = await this.options.grpcSdk.state!.tryAcquireLock( LEADER_LOCK, LOCK_TTL_MS, @@ -160,13 +170,39 @@ export class MongoChangeStreamCoordinator { this.scheduleRetry(); return; } - this.lock = acquired; + if (this.lock) { + try { + await this.options.grpcSdk.state!.releaseLock(acquired); + } catch { + // lock may already have expired + } + if (!this.watching) { + await this.openStream(); + } + return; + } + try { + this.lock = await acquired.extend(LOCK_TTL_MS); + } catch { + try { + await this.options.grpcSdk.state!.releaseLock(acquired); + } catch { + // lock may already have expired + } + this.lock = null; + this.streamState = 'idle'; + this.scheduleRetry(); + return; + } + this.bumpLockGeneration(); this.startRenewal(); await this.openStream(); } catch (err) { this.lastError = err instanceof Error ? err.message : String(err); this.streamState = 'degraded'; this.scheduleRetry(); + } finally { + this.acquiring = false; } } @@ -179,40 +215,59 @@ export class MongoChangeStreamCoordinator { private async renewLock() { if (!this.lock) return; + const generation = this.lockGeneration; try { this.lock = await this.lock.extend(LOCK_TTL_MS); } catch { - this.lock = null; - await this.stopStream('idle'); + if (this.lockGeneration !== generation) return; + await this.fenceLock('idle'); this.scheduleRetry(); } } private async openStream() { - if (this.watching || this.closed || this.opening) return; + if (this.watching || this.closed || this.opening || !this.lock) return; this.opening = true; this.streamState = 'starting'; this.ignoreClose = false; + const generation = this.lockGeneration; try { const resumeAfter = parseResumeToken( await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY), ); - if (this.watching || this.closed) return; + if ( + this.watching || + this.closed || + !this.lock || + generation !== this.lockGeneration + ) { + return; + } const collections = this.collectionNames(); const pipeline = buildWatchPipeline(collections); this.watchedCollectionsKey = optedInCollectionsKey(collections); const stream = this.options.watch({ resumeAfter, pipeline }); + if (generation !== this.lockGeneration || this.closed) { + try { + await stream.close(); + } catch { + // already closed + } + return; + } this.stream = stream; this.watching = true; this.streamState = 'live'; this.retryAttempt = 0; stream.on('change', (change: unknown) => { - this.enqueueChange(change as RawChangeEvent); + this.enqueueChange(change as RawChangeEvent, generation); }); stream.on('error', (err: unknown) => { + if (generation !== this.lockGeneration) return; void this.handleStreamError(err); }); stream.on('close', () => { + if (generation !== this.lockGeneration) return; this.watching = false; if (!this.closed && !this.ignoreClose && this.lock) { this.scheduleRetry(); @@ -226,12 +281,13 @@ export class MongoChangeStreamCoordinator { } } - private enqueueChange(change: RawChangeEvent) { + private enqueueChange(change: RawChangeEvent, generation: number) { this.changeQueue = this.changeQueue.then(async () => { - if (this.closed || !this.watching) return; + if (this.closed || !this.watching || generation !== this.lockGeneration) return; try { - await this.handleChange(change); + await this.handleChange(change, generation); } catch (err) { + if (generation !== this.lockGeneration) return; this.lastError = err instanceof Error ? err.message : String(err); ConduitGrpcSdk.Logger.error(err as Error); this.watching = false; @@ -241,12 +297,13 @@ export class MongoChangeStreamCoordinator { }); } - private async handleChange(change: RawChangeEvent) { + private async handleChange(change: RawChangeEvent, generation: number) { + if (generation !== this.lockGeneration) return; const token = serializeResumeToken(change._id); const schema = this.resolveSchema(change.ns?.coll); const event = schema ? normalizeChangeEvent(change, schema.name) : null; if (!event || !schema) { - if (token) { + if (token && generation === this.lockGeneration) { await this.persistResumeToken(token); } if (change.operationType && WATCH_RESTART_OPERATIONS.has(change.operationType)) { @@ -283,6 +340,9 @@ export class MongoChangeStreamCoordinator { ) { const adminRooms = roomsForPublicChange(schema.name, event.documentId); await this.safePush('admin', adminRooms, payload); + if (!schema.cmsReadEnabled) { + return; + } if (!schema.authorizationEnabled) { await this.safePush('router', adminRooms, payload); return; @@ -381,14 +441,25 @@ export class MongoChangeStreamCoordinator { } private async releaseLeader() { + await this.fenceLock(this.streamState); + } + + private async fenceLock(nextState: RealtimeStatusCode) { + this.bumpLockGeneration(); this.clearRenewTimer(); - if (!this.lock) return; + const lock = this.lock; + this.lock = null; + await this.stopStream(nextState); + if (!lock) return; try { - await this.options.grpcSdk.state!.releaseLock(this.lock); + await this.options.grpcSdk.state!.releaseLock(lock); } catch { // lock may already have expired } - this.lock = null; + } + + private bumpLockGeneration() { + this.lockGeneration += 1; } private clearTimers() { diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index cda5dc0e1..3720d5a41 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -7,6 +7,7 @@ import { roomsForPublicChange } from '../rooms.js'; class MemoryStore { private sets = new Map>(); + readonly ttls = new Map(); async sadd(key: string, ...members: string[]) { const set = this.sets.get(key) ?? new Set(); members.forEach(member => set.add(member)); @@ -26,15 +27,29 @@ class MemoryStore { return this.sets.get(key)?.size ?? 0; } async del(...keys: string[]) { - keys.forEach(key => this.sets.delete(key)); + keys.forEach(key => { + this.sets.delete(key); + this.ttls.delete(key); + }); return keys.length; } + async expire(key: string, seconds: number) { + this.ttls.set(key, seconds); + } + async persist(key: string) { + this.ttls.delete(key); + } } function createCoordinator(overrides?: { allow?: boolean; authorizationAvailable?: boolean; - schemas?: { name: string; collectionName: string; authorizationEnabled: boolean }[]; + schemas?: { + name: string; + collectionName: string; + authorizationEnabled: boolean; + cmsReadEnabled?: boolean; + }[]; getKeyDelayMs?: number; }) { const stream = new EventEmitter() as EventEmitter & { close: () => Promise }; @@ -84,9 +99,14 @@ function createCoordinator(overrides?: { watch, hello: async () => ({ setName: 'rs0' }), getOptedInSchemas: () => - overrides?.schemas ?? [ - { name: 'Order', collectionName: 'orders', authorizationEnabled: false }, - ], + ( + overrides?.schemas ?? [ + { name: 'Order', collectionName: 'orders', authorizationEnabled: false }, + ] + ).map(schema => ({ + cmsReadEnabled: true, + ...schema, + })), subscriptions, enabled: () => true, engine: () => 'MongoDB', @@ -358,7 +378,7 @@ describe('MongoChangeStreamCoordinator', () => { jest.useRealTimers(); }); - it.each(['drop', 'rename', 'invalidate'] as const)( + it.each(['drop', 'rename', 'invalidate', 'dropDatabase'] as const)( 'reopens the watch on %s', async operationType => { jest.useFakeTimers(); @@ -419,4 +439,67 @@ describe('MongoChangeStreamCoordinator', () => { ]); await coordinator.shutdown(); }); + + it('does not open a watch when the lock cannot be extended after acquire', async () => { + const { coordinator, watch, lock } = createCoordinator(); + lock.extend.mockRejectedValueOnce(new Error('extend failed')); + await coordinator.reconcile(); + expect(watch).not.toHaveBeenCalled(); + expect(coordinator.getState()).toBe('idle'); + await coordinator.shutdown(); + }); + + it('ignores draining watch events after lock renew failure', async () => { + jest.useFakeTimers(); + const { coordinator, stream, lock, adminPush, state } = createCoordinator(); + lock.extend.mockResolvedValueOnce(lock).mockRejectedValueOnce(new Error('lost lock')); + await coordinator.reconcile(); + expect(coordinator.getState()).toBe('live'); + await jest.advanceTimersByTimeAsync(5_000); + expect(coordinator.getState()).toBe('idle'); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'stale' }), + ); + await coordinator.waitForIdle(); + expect(adminPush).not.toHaveBeenCalled(); + expect(state.get('realtime:resumeToken')).toBeUndefined(); + await coordinator.shutdown(); + jest.useRealTimers(); + }); + + it('does not emit to clients when CMS read is denied and keeps membership', async () => { + const can = jest.fn(async () => ({ allow: true })); + const { coordinator, stream, routerPush, adminPush, subscriptions, grpcSdk } = + createCoordinator({ + schemas: [ + { + name: 'Order', + collectionName: 'orders', + authorizationEnabled: true, + cmsReadEnabled: false, + }, + ], + }); + grpcSdk.authorization = { can }; + await subscriptions.addAuthorizedDocument( + 'sock-1', + 'Order', + '64b64c4c4c4c4c4c4c4c4c4c', + 'user-1', + ); + await coordinator.reconcile(); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), + ); + await coordinator.waitForIdle(); + expect(adminPush).toHaveBeenCalledTimes(1); + expect(routerPush).not.toHaveBeenCalled(); + expect(can).not.toHaveBeenCalled(); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual([ + 'user-1', + ]); + await coordinator.shutdown(); + }); }); diff --git a/modules/database/src/realtime/__tests__/recovery.test.ts b/modules/database/src/realtime/__tests__/recovery.test.ts index eee45b918..4e5aaee37 100644 --- a/modules/database/src/realtime/__tests__/recovery.test.ts +++ b/modules/database/src/realtime/__tests__/recovery.test.ts @@ -1,12 +1,15 @@ import { describe, expect, it } from '@jest/globals'; -import { canReadDocument } from '../authorize.js'; import { authorizedDocumentRoom } from '../rooms.js'; import { isRecoverableDisconnect, restoreAuthorizedSubscriptions } from '../recovery.js'; import { createSocketHandlers } from '../sockets.js'; -import { RealtimeSubscriptionTracker } from '../subscriptions.js'; +import { + RealtimeSubscriptionTracker, + RECOVERY_REDIS_TTL_SECONDS, +} from '../subscriptions.js'; class MemoryStore { private sets = new Map>(); + readonly ttls = new Map(); async sadd(key: string, ...members: string[]) { const set = this.sets.get(key) ?? new Set(); members.forEach(member => set.add(member)); @@ -22,7 +25,16 @@ class MemoryStore { return this.sets.get(key)?.size ?? 0; } async del(...keys: string[]) { - keys.forEach(key => this.sets.delete(key)); + keys.forEach(key => { + this.sets.delete(key); + this.ttls.delete(key); + }); + } + async expire(key: string, seconds: number) { + this.ttls.set(key, seconds); + } + async persist(key: string) { + this.ttls.delete(key); } } @@ -48,7 +60,6 @@ describe('database socket recovery', () => { isAvailable: () => true, authorization: { can: async () => ({ allow: true }) }, }, - canRead: canReadDocument, }); expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); }); @@ -79,4 +90,47 @@ describe('database socket recovery', () => { } as never); expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); }); + + it('keeps membership when recovered authorization is unavailable', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + const room = authorizedDocumentRoom('Order', 'doc-1', 'user-1'); + await tracker.addAuthorizedDocument('sock-1', 'Order', 'doc-1', 'user-1'); + const { leaveRooms } = await restoreAuthorizedSubscriptions({ + socketId: 'sock-1', + rooms: [room], + contextSubs: [], + subscriptions: tracker, + grpcSdk: { + isAvailable: () => false, + }, + }); + expect(leaveRooms).toEqual([]); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + }); + + it('expires Redis membership when a recoverable disconnect never recovers', async () => { + const store = new MemoryStore(); + const tracker = new RealtimeSubscriptionTracker(store); + const handlers = createSocketHandlers({ + mode: 'client', + grpcSdk: { + isAvailable: () => true, + authorization: { can: async () => ({ allow: true }) }, + } as never, + schemaLookup: { getSchema: () => undefined }, + subscriptions: tracker, + isGloballyEnabled: () => true, + }); + await tracker.addAuthorizedDocument('sock-1', 'Order', 'doc-1', 'user-1'); + expect(store.ttls.size).toBe(0); + await handlers.disconnect({ + request: { socketId: 'sock-1', params: ['transport close'] }, + } as never); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + expect(store.ttls.get('realtime:socket:sock-1')).toBe(RECOVERY_REDIS_TTL_SECONDS); + expect(store.ttls.get('realtime:doc:Order:doc-1')).toBe(RECOVERY_REDIS_TTL_SECONDS); + expect(store.ttls.get('realtime:userdoc:Order:doc-1:user-1')).toBe( + RECOVERY_REDIS_TTL_SECONDS, + ); + }); }); diff --git a/modules/database/src/realtime/__tests__/subscriptions.test.ts b/modules/database/src/realtime/__tests__/subscriptions.test.ts index 45ee7d1c9..13bcc7298 100644 --- a/modules/database/src/realtime/__tests__/subscriptions.test.ts +++ b/modules/database/src/realtime/__tests__/subscriptions.test.ts @@ -1,8 +1,12 @@ import { describe, expect, it } from '@jest/globals'; -import { RealtimeSubscriptionTracker } from '../subscriptions.js'; +import { + RealtimeSubscriptionTracker, + RECOVERY_REDIS_TTL_SECONDS, +} from '../subscriptions.js'; class MemoryStore { private sets = new Map>(); + readonly ttls = new Map(); async sadd(key: string, ...members: string[]) { const set = this.sets.get(key) ?? new Set(); @@ -27,9 +31,20 @@ class MemoryStore { } async del(...keys: string[]) { - keys.forEach(key => this.sets.delete(key)); + keys.forEach(key => { + this.sets.delete(key); + this.ttls.delete(key); + }); return keys.length; } + + async expire(key: string, seconds: number) { + this.ttls.set(key, seconds); + } + + async persist(key: string) { + this.ttls.delete(key); + } } describe('RealtimeSubscriptionTracker', () => { @@ -46,10 +61,36 @@ describe('RealtimeSubscriptionTracker', () => { expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); }); + it('does not expire shared document keys while another socket is still live', async () => { + const store = new MemoryStore(); + const tracker = new RealtimeSubscriptionTracker(store); + await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); + await tracker.addAuthorizedDocument('s2', 'Order', 'doc-1', 'user-1'); + await tracker.armRecoverableTtl('s1'); + expect(store.ttls.get('realtime:socket:s1')).toBe(RECOVERY_REDIS_TTL_SECONDS); + expect(store.ttls.has('realtime:doc:Order:doc-1')).toBe(false); + expect(store.ttls.has('realtime:userdoc:Order:doc-1:user-1')).toBe(false); + }); + it('removes revoked users from the document set', async () => { const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); await tracker.removeUser('Order', 'doc-1', 'user-1'); expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); }); + + it('arms Redis TTL on recoverable disconnect keys and persists on restore', async () => { + const store = new MemoryStore(); + const tracker = new RealtimeSubscriptionTracker(store); + await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); + expect(store.ttls.size).toBe(0); + await tracker.armRecoverableTtl('s1'); + expect(store.ttls.get('realtime:socket:s1')).toBe(RECOVERY_REDIS_TTL_SECONDS); + expect(store.ttls.get('realtime:doc:Order:doc-1')).toBe(RECOVERY_REDIS_TTL_SECONDS); + expect(store.ttls.get('realtime:userdoc:Order:doc-1:user-1')).toBe( + RECOVERY_REDIS_TTL_SECONDS, + ); + await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); + expect(store.ttls.size).toBe(0); + }); }); diff --git a/modules/database/src/realtime/authorize.ts b/modules/database/src/realtime/authorize.ts index cf7ed899c..d2a16e1cf 100644 --- a/modules/database/src/realtime/authorize.ts +++ b/modules/database/src/realtime/authorize.ts @@ -1,6 +1,6 @@ import { status } from '@grpc/grpc-js'; import { GrpcError } from '@conduitplatform/grpc-sdk'; -import type { OptedInSchema, SubscribeRequest } from './types.js'; +import type { OptedInSchema, RebacDecision, SubscribeRequest } from './types.js'; export class RealtimeSubscriptionError extends GrpcError { constructor(code: number, message: string) { @@ -61,14 +61,14 @@ export function optionalDocumentId(value: unknown): string | undefined { return value; } -export async function canReadDocument( +export async function readDocumentDecision( grpcSdk: AuthorizationSdk, schema: string, documentId: string, userId: string, -): Promise { +): Promise { if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { - return false; + return 'unavailable'; } try { const decision = await grpcSdk.authorization.can({ @@ -76,12 +76,21 @@ export async function canReadDocument( actions: ['read'], resource: `${schema}:${documentId}`, }); - return decision.allow === true; + return decision.allow === true ? 'allow' : 'deny'; } catch { - return false; + return 'unavailable'; } } +export async function canReadDocument( + grpcSdk: AuthorizationSdk, + schema: string, + documentId: string, + userId: string, +): Promise { + return (await readDocumentDecision(grpcSdk, schema, documentId, userId)) === 'allow'; +} + export function assertSchemaAvailable( lookup: SchemaLookup, schemaName: string, @@ -123,6 +132,7 @@ export function toOptedInSchema(schema: { modelOptions?: { conduit?: { realtime?: { enabled?: boolean }; + cms?: { crudOperations?: { read?: { enabled?: boolean } } }; authorization?: { enabled?: boolean }; }; }; @@ -132,5 +142,7 @@ export function toOptedInSchema(schema: { name: schema.name, collectionName: schema.collectionName, authorizationEnabled: schema.modelOptions.conduit.authorization?.enabled === true, + cmsReadEnabled: + schema.modelOptions.conduit.cms?.crudOperations?.read?.enabled === true, }; } diff --git a/modules/database/src/realtime/rebacCache.ts b/modules/database/src/realtime/rebacCache.ts index b33a9438c..c57595227 100644 --- a/modules/database/src/realtime/rebacCache.ts +++ b/modules/database/src/realtime/rebacCache.ts @@ -1,7 +1,6 @@ +import type { RebacDecision } from './types.js'; import type { AuthorizationSdk } from './authorize.js'; -export type RebacDecision = 'allow' | 'deny' | 'unavailable'; - const DEFAULT_TTL_MS = 12_000; const DEFAULT_MAX_ENTRIES = 10_000; diff --git a/modules/database/src/realtime/recovery.ts b/modules/database/src/realtime/recovery.ts index 3bc9c3868..7ff79735b 100644 --- a/modules/database/src/realtime/recovery.ts +++ b/modules/database/src/realtime/recovery.ts @@ -1,5 +1,5 @@ import type { Indexable, ParsedSocketRequest } from '@conduitplatform/grpc-sdk'; -import type { AuthorizationSdk } from './authorize.js'; +import { readDocumentDecision, type AuthorizationSdk } from './authorize.js'; import { authorizedDocumentRoom, parseAuthorizedDocumentRoom } from './rooms.js'; import type { RealtimeSubscriptionTracker } from './subscriptions.js'; @@ -93,12 +93,6 @@ export async function restoreAuthorizedSubscriptions(options: { contextSubs: AuthorizedSub[]; subscriptions: RealtimeSubscriptionTracker; grpcSdk: AuthorizationSdk; - canRead: ( - grpcSdk: AuthorizationSdk, - schema: string, - documentId: string, - userId: string, - ) => Promise; }): Promise<{ leaveRooms: string[] }> { const seen = new Set(); const subs: AuthorizedSub[] = []; @@ -119,14 +113,14 @@ export async function restoreAuthorizedSubscriptions(options: { const leaveRooms: string[] = []; for (const sub of subs) { - const allowed = await options.canRead( + const decision = await readDocumentDecision( options.grpcSdk, sub.schema, sub.documentId, sub.userId, ); const room = authorizedDocumentRoom(sub.schema, sub.documentId, sub.userId); - if (!allowed) { + if (decision === 'deny') { await options.subscriptions.removeAuthorizedDocument( options.socketId, sub.schema, diff --git a/modules/database/src/realtime/sockets.ts b/modules/database/src/realtime/sockets.ts index e2a80931c..3a06ab121 100644 --- a/modules/database/src/realtime/sockets.ts +++ b/modules/database/src/realtime/sockets.ts @@ -78,7 +78,9 @@ export function createSocketHandlers(options: RealtimeSocketOptions) { }, disconnect: async (call: ParsedSocketRequest): Promise => { const reason = call.request.params?.[0]; - if (!isRecoverableDisconnect(reason)) { + if (isRecoverableDisconnect(reason)) { + await options.subscriptions.armRecoverableTtl(call.request.socketId); + } else { await options.subscriptions.disconnect(call.request.socketId); } return { event: 'disconnected', data: { ok: true } }; @@ -90,7 +92,6 @@ export function createSocketHandlers(options: RealtimeSocketOptions) { contextSubs: authorizedSubsFromContext(call.request.context as Indexable), subscriptions: options.subscriptions, grpcSdk: options.grpcSdk as unknown as AuthorizationSdk, - canRead: canReadDocument, }); if (leaveRooms.length > 0) { return { event: 'leave-room', rooms: leaveRooms }; diff --git a/modules/database/src/realtime/subscriptions.ts b/modules/database/src/realtime/subscriptions.ts index da7bfb867..bc54e8efd 100644 --- a/modules/database/src/realtime/subscriptions.ts +++ b/modules/database/src/realtime/subscriptions.ts @@ -1,9 +1,13 @@ +export const RECOVERY_REDIS_TTL_SECONDS = 120; + export type SubscriptionStore = { sadd(key: string, ...members: string[]): Promise; srem(key: string, ...members: string[]): Promise; smembers(key: string): Promise; scard(key: string): Promise; del(...keys: string[]): Promise; + expire(key: string, seconds: number): Promise; + persist(key: string): Promise; }; function socketKey(socketId: string): string { @@ -31,12 +35,15 @@ export class RealtimeSubscriptionTracker { documentId: string, userId: string, ): Promise { - await this.store.sadd( - socketKey(socketId), - subscriptionRecord(schema, documentId, userId), - ); - await this.store.sadd(userDocSocketsKey(schema, documentId, userId), socketId); - await this.store.sadd(docUsersKey(schema, documentId), userId); + const socket = socketKey(socketId); + const userDoc = userDocSocketsKey(schema, documentId, userId); + const docUsers = docUsersKey(schema, documentId); + await this.store.sadd(socket, subscriptionRecord(schema, documentId, userId)); + await this.store.sadd(userDoc, socketId); + await this.store.sadd(docUsers, userId); + await this.store.persist(socket); + await this.store.persist(userDoc); + await this.store.persist(docUsers); } async removeAuthorizedDocument( @@ -89,4 +96,41 @@ export class RealtimeSubscriptionTracker { } await this.store.del(socketKey(socketId)); } + + async armRecoverableTtl( + socketId: string, + ttlSeconds: number = RECOVERY_REDIS_TTL_SECONDS, + ): Promise { + const socket = socketKey(socketId); + const records = await this.store.smembers(socket); + await this.store.expire(socket, ttlSeconds); + for (const record of records) { + try { + const parsed = JSON.parse(record) as { + schema: string; + documentId: string; + userId: string; + }; + const userDoc = userDocSocketsKey( + parsed.schema, + parsed.documentId, + parsed.userId, + ); + const others = (await this.store.smembers(userDoc)).filter(id => id !== socketId); + if (others.length > 0) continue; + await this.store.expire(userDoc, ttlSeconds); + const users = await this.store.smembers( + docUsersKey(parsed.schema, parsed.documentId), + ); + if (users.length <= 1) { + await this.store.expire( + docUsersKey(parsed.schema, parsed.documentId), + ttlSeconds, + ); + } + } catch { + // ignore malformed records + } + } + } } diff --git a/modules/database/src/realtime/types.ts b/modules/database/src/realtime/types.ts index 2b6e3e8c7..004523198 100644 --- a/modules/database/src/realtime/types.ts +++ b/modules/database/src/realtime/types.ts @@ -28,10 +28,13 @@ export type RealtimeStatus = { message?: string; }; +export type RebacDecision = 'allow' | 'deny' | 'unavailable'; + export type OptedInSchema = { name: string; collectionName: string; authorizationEnabled: boolean; + cmsReadEnabled: boolean; }; export type SubscribeRequest = { From de988889dd00987d73f55e245409838ad385219b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 19:20:57 +0000 Subject: [PATCH 7/8] fix(database): watch Mongo live updates from now, drop resume tokens Live updates notify only: change streams start at the end of the oplog. Do not persist Redis resume tokens or restore on CursorKilled. Leader restart or cursor drop is a gap; clients refetch. Co-authored-by: Konstantinos Kopanidis --- modules/database/README.mdx | 2 +- .../realtime/MongoChangeStreamCoordinator.ts | 46 +------ .../database/src/realtime/RealtimeService.ts | 7 +- .../realtime/__tests__/coordinator.test.ts | 123 +++++------------- .../src/realtime/__tests__/normalize.test.ts | 11 +- .../src/realtime/__tests__/topology.test.ts | 11 +- modules/database/src/realtime/normalize.ts | 22 ---- modules/database/src/realtime/topology.ts | 25 ---- 8 files changed, 47 insertions(+), 200 deletions(-) diff --git a/modules/database/README.mdx b/modules/database/README.mdx index 10703937b..f573efb1e 100644 --- a/modules/database/README.mdx +++ b/modules/database/README.mdx @@ -60,7 +60,7 @@ subscribe({ schema: 'Order', documentId?: string }) unsubscribe({ schema: 'Order', documentId?: string }) ``` -Events arrive as `change` with `{ version, operation, schema, documentId, occurredAt }` and contain no document fields and no resume token. This is **live-tail, not backfill**. After a full reconnect, subscribe again and refetch over authorized REST. +Events arrive as `change` with `{ version, operation, schema, documentId, occurredAt }` and contain no document fields and no resume token. This is **live-tail, not backfill**: the change stream starts at the end of the oplog. A leader restart or cursor drop does not replay missed events; clients subscribe again and refetch over authorized REST. The leader also publishes `database:change:${schema}` on the Redis bus. **Do not also relay `database:change:*` on `/events/` if the same client is on `/database/`** — that duplicates notifications. Keep the bus for other modules; just do not dual-subscribe. diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts index 45644fa78..785bfe098 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -1,16 +1,7 @@ import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; -import { - normalizeChangeEvent, - parseResumeToken, - serializeResumeToken, - type RawChangeEvent, -} from './normalize.js'; +import { normalizeChangeEvent, type RawChangeEvent } from './normalize.js'; import { authorizedDocumentRoom, roomsForPublicChange } from './rooms.js'; -import { - isResumeTokenUnusable, - topologyFromHello, - type TopologyResult, -} from './topology.js'; +import { topologyFromHello, type TopologyResult } from './topology.js'; import type { ChangeStreamLike, DatabaseChangeEvent, @@ -28,7 +19,6 @@ import { } from './watchPipeline.js'; const LEADER_LOCK = 'realtime:change-stream:leader'; -const RESUME_TOKEN_KEY = 'realtime:resumeToken'; const LOCK_TTL_MS = 15_000; const LOCK_RENEW_MS = 5_000; const RETRY_BASE_MS = 1_000; @@ -38,10 +28,7 @@ type LeaderLock = NonNullable< Awaited['tryAcquireLock']>> >; -export type WatchFactory = (options: { - resumeAfter?: unknown; - pipeline: WatchPipeline; -}) => ChangeStreamLike; +export type WatchFactory = (options: { pipeline: WatchPipeline }) => ChangeStreamLike; export type CoordinatorOptions = { grpcSdk: ConduitGrpcSdk; @@ -232,21 +219,10 @@ export class MongoChangeStreamCoordinator { this.ignoreClose = false; const generation = this.lockGeneration; try { - const resumeAfter = parseResumeToken( - await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY), - ); - if ( - this.watching || - this.closed || - !this.lock || - generation !== this.lockGeneration - ) { - return; - } const collections = this.collectionNames(); const pipeline = buildWatchPipeline(collections); this.watchedCollectionsKey = optedInCollectionsKey(collections); - const stream = this.options.watch({ resumeAfter, pipeline }); + const stream = this.options.watch({ pipeline }); if (generation !== this.lockGeneration || this.closed) { try { await stream.close(); @@ -299,13 +275,9 @@ export class MongoChangeStreamCoordinator { private async handleChange(change: RawChangeEvent, generation: number) { if (generation !== this.lockGeneration) return; - const token = serializeResumeToken(change._id); const schema = this.resolveSchema(change.ns?.coll); const event = schema ? normalizeChangeEvent(change, schema.name) : null; if (!event || !schema) { - if (token && generation === this.lockGeneration) { - await this.persistResumeToken(token); - } if (change.operationType && WATCH_RESTART_OPERATIONS.has(change.operationType)) { await this.stopStream('starting'); this.scheduleRetry(); @@ -315,13 +287,6 @@ export class MongoChangeStreamCoordinator { this.lastEventAt = event.occurredAt; this.lastError = undefined; await this.emitChange(schema, event); - if (token) { - await this.persistResumeToken(token); - } - } - - private async persistResumeToken(token: string) { - await this.options.grpcSdk.state!.setKey(RESUME_TOKEN_KEY, token); } private async emitChange(schema: OptedInSchema, event: DatabaseChangeEvent) { @@ -407,9 +372,6 @@ export class MongoChangeStreamCoordinator { this.streamState = 'degraded'; ConduitGrpcSdk.Metrics?.increment('database_realtime_stream_errors_total'); ConduitGrpcSdk.Logger.error(err as Error); - if (isResumeTokenUnusable(err)) { - await this.options.grpcSdk.state!.clearKey(RESUME_TOKEN_KEY); - } await this.stopStream('degraded'); this.scheduleRetry(); } diff --git a/modules/database/src/realtime/RealtimeService.ts b/modules/database/src/realtime/RealtimeService.ts index 96d21f485..1c1a386a7 100644 --- a/modules/database/src/realtime/RealtimeService.ts +++ b/modules/database/src/realtime/RealtimeService.ts @@ -131,16 +131,13 @@ export class RealtimeService { private openWatch( adapter: MongooseAdapter, - options: { resumeAfter?: unknown; pipeline: WatchPipeline }, + options: { pipeline: WatchPipeline }, ): ChangeStreamLike { const db = adapter.mongoose.connection.db; if (!db) { throw new Error('MongoDB connection is not ready'); } - return db.watch( - options.pipeline, - options.resumeAfter ? { resumeAfter: options.resumeAfter as never } : {}, - ) as unknown as ChangeStreamLike; + return db.watch(options.pipeline) as unknown as ChangeStreamLike; } private async hello( diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index 3720d5a41..94d8275c7 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -1,6 +1,6 @@ import { EventEmitter } from 'node:events'; import { afterEach, describe, expect, it, jest } from '@jest/globals'; -import { EJSON, ObjectId } from 'bson'; +import { ObjectId } from 'bson'; import { MongoChangeStreamCoordinator } from '../MongoChangeStreamCoordinator.js'; import { RealtimeSubscriptionTracker } from '../subscriptions.js'; import { roomsForPublicChange } from '../rooms.js'; @@ -50,13 +50,11 @@ function createCoordinator(overrides?: { authorizationEnabled: boolean; cmsReadEnabled?: boolean; }[]; - getKeyDelayMs?: number; }) { const stream = new EventEmitter() as EventEmitter & { close: () => Promise }; stream.close = async () => { stream.emit('close'); }; - const state = new Map(); const lock = { extend: jest.fn(async () => lock), release: jest.fn(async () => undefined), @@ -70,18 +68,6 @@ function createCoordinator(overrides?: { state: { tryAcquireLock: jest.fn(async () => lock), releaseLock: jest.fn(async () => undefined), - getKey: jest.fn(async (key: string) => { - if (overrides?.getKeyDelayMs) { - await new Promise(resolve => setTimeout(resolve, overrides.getKeyDelayMs)); - } - return state.get(key) ?? null; - }), - setKey: jest.fn(async (key: string, value: string) => { - state.set(key, value); - }), - clearKey: jest.fn(async (key: string) => { - state.delete(key); - }), }, bus: { publish }, router: { socketPush: routerPush }, @@ -120,7 +106,6 @@ function createCoordinator(overrides?: { publish, subscriptions, grpcSdk, - state, lock, }; } @@ -140,8 +125,7 @@ describe('MongoChangeStreamCoordinator', () => { }); it('emits one normalized event to public rooms and ignores other collections', async () => { - const { coordinator, stream, routerPush, adminPush, publish, state } = - createCoordinator(); + const { coordinator, stream, routerPush, adminPush, publish } = createCoordinator(); await coordinator.reconcile(); const resume = { _data: 'token' }; stream.emit('change', { @@ -172,23 +156,11 @@ describe('MongoChangeStreamCoordinator', () => { expect( JSON.parse((adminPush.mock.calls[0][0] as { data: string }).data), ).not.toHaveProperty('resumeToken'); - expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(resume)); await coordinator.shutdown(); }); - it('advances the resume token for filtered events', async () => { - const { coordinator, stream, publish, state } = createCoordinator(); - await coordinator.reconcile(); - const skip = { _data: 'skip-token' }; - stream.emit('change', insertChange('other', '64b64c4c4c4c4c4c4c4c4c4d', skip)); - await coordinator.waitForIdle(); - expect(publish).not.toHaveBeenCalled(); - expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(skip)); - await coordinator.shutdown(); - }); - - it('serializes overlapping handlers and persists after emit', async () => { - const { coordinator, stream, state, adminPush } = createCoordinator(); + it('serializes overlapping handlers', async () => { + const { coordinator, stream, adminPush } = createCoordinator(); let release!: () => void; const gate = new Promise(resolve => { release = resolve; @@ -201,22 +173,24 @@ describe('MongoChangeStreamCoordinator', () => { } }); await coordinator.reconcile(); - const tokenA = { _data: 'token-a' }; - const tokenB = { _data: 'token-b' }; - stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); - stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', tokenB)); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token-a' }), + ); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', { _data: 'token-b' }), + ); await Promise.resolve(); await new Promise(resolve => setImmediate(resolve)); - expect(state.get('realtime:resumeToken')).toBeUndefined(); expect(adminPush).toHaveBeenCalledTimes(1); release(); await coordinator.waitForIdle(); expect(adminPush).toHaveBeenCalledTimes(2); - expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenB)); await coordinator.shutdown(); }); - it('watches opted-in collections with $match and $project', async () => { + it('watches opted-in collections with $match and $project from now', async () => { const { coordinator, watch } = createCoordinator(); await coordinator.reconcile(); expect(watch).toHaveBeenCalledTimes(1); @@ -240,6 +214,7 @@ describe('MongoChangeStreamCoordinator', () => { fullDocumentBeforeChange: 0, }, }); + expect(watch.mock.calls[0][0]).not.toHaveProperty('resumeAfter'); await coordinator.shutdown(); }); @@ -265,6 +240,7 @@ describe('MongoChangeStreamCoordinator', () => { expect(match.$match.$or[0]['ns.coll']?.$in).toEqual( expect.arrayContaining(['orders', 'items']), ); + expect(watch.mock.calls[1][0]).not.toHaveProperty('resumeAfter'); await coordinator.shutdown(); }); @@ -305,39 +281,16 @@ describe('MongoChangeStreamCoordinator', () => { }); it('opens a single watch when reconcile runs concurrently', async () => { - const { coordinator, watch } = createCoordinator({ getKeyDelayMs: 40 }); + const { coordinator, watch } = createCoordinator(); await Promise.all([coordinator.reconcile(), coordinator.reconcile()]); expect(watch).toHaveBeenCalledTimes(1); await coordinator.shutdown(); }); - it('clears an unusable resume token on 280 and retries', async () => { - const { coordinator, stream, grpcSdk } = createCoordinator(); - await coordinator.reconcile(); - stream.emit('error', { code: 280, message: 'ChangeStreamHistoryLost' }); - await new Promise(resolve => setImmediate(resolve)); - expect(grpcSdk.state.clearKey).toHaveBeenCalled(); - await coordinator.shutdown(); - }); - - it('keeps the resume token on CursorKilled 237', async () => { - const { coordinator, stream, grpcSdk, state } = createCoordinator(); - await coordinator.reconcile(); - const token = { _data: 'keep-me' }; - stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', token)); - await coordinator.waitForIdle(); - expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); - stream.emit('error', { code: 237, message: 'CursorKilled' }); - await new Promise(resolve => setImmediate(resolve)); - expect(grpcSdk.state.clearKey).not.toHaveBeenCalled(); - expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); - await coordinator.shutdown(); - }); - - it('does not persist a later token when emit fails and reopens from the last good token', async () => { + it('stops the stream and retries from now when emit fails', async () => { jest.useFakeTimers(); const streams: Array Promise }> = []; - const { coordinator, adminPush, state, watch } = createCoordinator(); + const { coordinator, adminPush, watch } = createCoordinator(); watch.mockImplementation(() => { const next = new EventEmitter() as EventEmitter & { close: () => Promise }; next.close = async () => { @@ -351,39 +304,40 @@ describe('MongoChangeStreamCoordinator', () => { .mockRejectedValueOnce(new Error('push failed')) .mockResolvedValue(undefined); await coordinator.reconcile(); - const tokenGood = { _data: 'token-good' }; - const tokenA = { _data: 'token-a' }; - const tokenB = { _data: 'token-b' }; streams[0].emit( 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4b', tokenGood), + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4b', { _data: 'token-good' }), ); await coordinator.waitForIdle(); - expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenGood)); - streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); - streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', tokenB)); + streams[0].emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token-a' }), + ); + streams[0].emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', { _data: 'token-b' }), + ); await coordinator.waitForIdle(); - expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenGood)); expect(adminPush).toHaveBeenCalledTimes(2); await jest.advanceTimersByTimeAsync(1_000); expect(watch).toHaveBeenCalledTimes(2); - expect((watch.mock.calls[1][0] as { resumeAfter?: unknown }).resumeAfter).toEqual( - tokenGood, + expect(watch.mock.calls[1][0]).not.toHaveProperty('resumeAfter'); + streams[1].emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token-a' }), ); - streams[1].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); await coordinator.waitForIdle(); - expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenA)); expect(adminPush).toHaveBeenCalledTimes(3); await coordinator.shutdown(); jest.useRealTimers(); }); it.each(['drop', 'rename', 'invalidate', 'dropDatabase'] as const)( - 'reopens the watch on %s', + 'reopens the watch on %s from now', async operationType => { jest.useFakeTimers(); const streams: Array Promise }> = []; - const { coordinator, watch, state } = createCoordinator(); + const { coordinator, watch } = createCoordinator(); watch.mockImplementation(() => { const next = new EventEmitter() as EventEmitter & { close: () => Promise }; next.close = async () => { @@ -393,19 +347,15 @@ describe('MongoChangeStreamCoordinator', () => { return next as never; }); await coordinator.reconcile(); - const token = { _data: `${operationType}-token` }; streams[0].emit('change', { operationType, ns: { coll: 'orders' }, - _id: token, + _id: { _data: `${operationType}-token` }, }); await coordinator.waitForIdle(); - expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); await jest.advanceTimersByTimeAsync(1_000); expect(watch).toHaveBeenCalledTimes(2); - expect((watch.mock.calls[1][0] as { resumeAfter?: unknown }).resumeAfter).toEqual( - token, - ); + expect(watch.mock.calls[1][0]).not.toHaveProperty('resumeAfter'); await coordinator.shutdown(); jest.useRealTimers(); }, @@ -451,7 +401,7 @@ describe('MongoChangeStreamCoordinator', () => { it('ignores draining watch events after lock renew failure', async () => { jest.useFakeTimers(); - const { coordinator, stream, lock, adminPush, state } = createCoordinator(); + const { coordinator, stream, lock, adminPush } = createCoordinator(); lock.extend.mockResolvedValueOnce(lock).mockRejectedValueOnce(new Error('lost lock')); await coordinator.reconcile(); expect(coordinator.getState()).toBe('live'); @@ -463,7 +413,6 @@ describe('MongoChangeStreamCoordinator', () => { ); await coordinator.waitForIdle(); expect(adminPush).not.toHaveBeenCalled(); - expect(state.get('realtime:resumeToken')).toBeUndefined(); await coordinator.shutdown(); jest.useRealTimers(); }); diff --git a/modules/database/src/realtime/__tests__/normalize.test.ts b/modules/database/src/realtime/__tests__/normalize.test.ts index f7f933d33..1adcfae1a 100644 --- a/modules/database/src/realtime/__tests__/normalize.test.ts +++ b/modules/database/src/realtime/__tests__/normalize.test.ts @@ -1,21 +1,17 @@ import { describe, expect, it } from '@jest/globals'; import { ObjectId } from 'bson'; -import { - normalizeChangeEvent, - parseResumeToken, - serializeResumeToken, -} from '../normalize.js'; +import { normalizeChangeEvent } from '../normalize.js'; describe('normalizeChangeEvent', () => { it('normalizes insert/update/replace/delete into metadata-only events', () => { - const resume = { _data: 'token-1' }; + const streamId = { _data: 'token-1' }; const event = normalizeChangeEvent( { operationType: 'insert', documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, fullDocument: { secret: 'nope' }, wallTime: new Date('2026-01-01T00:00:00.000Z'), - _id: resume, + _id: streamId, }, 'Order', ); @@ -26,7 +22,6 @@ describe('normalizeChangeEvent', () => { documentId: '64b64c4c4c4c4c4c4c4c4c4c', occurredAt: '2026-01-01T00:00:00.000Z', }); - expect(parseResumeToken(serializeResumeToken(resume))).toEqual(resume); expect(event).not.toHaveProperty('resumeToken'); expect(JSON.parse(JSON.stringify(event))).not.toHaveProperty('fullDocument'); }); diff --git a/modules/database/src/realtime/__tests__/topology.test.ts b/modules/database/src/realtime/__tests__/topology.test.ts index 79afac87c..51b52be54 100644 --- a/modules/database/src/realtime/__tests__/topology.test.ts +++ b/modules/database/src/realtime/__tests__/topology.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from '@jest/globals'; -import { isResumeTokenUnusable, topologyFromHello } from '../topology.js'; +import { topologyFromHello } from '../topology.js'; describe('topology helpers', () => { it('accepts replica sets and mongos, rejects standalone', () => { @@ -7,13 +7,4 @@ describe('topology helpers', () => { expect(topologyFromHello({ msg: 'isdbgrid' })).toEqual({ supported: true }); expect(topologyFromHello({}).supported).toBe(false); }); - - it('detects unusable resume tokens', () => { - expect(isResumeTokenUnusable({ code: 280 })).toBe(true); - expect(isResumeTokenUnusable({ code: 237 })).toBe(false); - expect(isResumeTokenUnusable({ code: 136 })).toBe(false); - expect(isResumeTokenUnusable(new Error('ChangeStreamHistoryLost'))).toBe(true); - expect(isResumeTokenUnusable(new Error('cannot resume'))).toBe(false); - expect(isResumeTokenUnusable(new Error('socket hang up'))).toBe(false); - }); }); diff --git a/modules/database/src/realtime/normalize.ts b/modules/database/src/realtime/normalize.ts index b2f4b8ff3..9ea55bd0f 100644 --- a/modules/database/src/realtime/normalize.ts +++ b/modules/database/src/realtime/normalize.ts @@ -1,4 +1,3 @@ -import { EJSON } from 'bson'; import { DATABASE_CHANGE_EVENT_VERSION, DATABASE_CHANGE_OPERATIONS, @@ -30,9 +29,6 @@ export function normalizeChangeEvent( if (!documentId) { return null; } - if (!change._id) { - return null; - } return { version: DATABASE_CHANGE_EVENT_VERSION, operation: operation as DatabaseChangeOperation, @@ -45,24 +41,6 @@ export function normalizeChangeEvent( }; } -export function serializeResumeToken(id: unknown): string | null { - if (id === undefined || id === null) return null; - try { - return EJSON.stringify(id); - } catch { - return null; - } -} - -export function parseResumeToken(token: string | null | undefined): unknown | undefined { - if (!token) return undefined; - try { - return EJSON.parse(token); - } catch { - return undefined; - } -} - function extractDocumentId(id: unknown): string | null { if (id === undefined || id === null) return null; if (typeof id === 'string' || typeof id === 'number') return String(id); diff --git a/modules/database/src/realtime/topology.ts b/modules/database/src/realtime/topology.ts index 4d62fa3fa..a1ab72d71 100644 --- a/modules/database/src/realtime/topology.ts +++ b/modules/database/src/realtime/topology.ts @@ -1,8 +1,3 @@ -const UNUSABLE_RESUME_TOKEN_CODES = new Set([ - 280, // ChangeStreamHistoryLost - 286, // ChangeStreamFatalError -]); - export type TopologyResult = { supported: boolean; message?: string; @@ -28,23 +23,3 @@ export function topologyFromHello( message: 'A replica set or sharded MongoDB deployment is required for live updates', }; } - -export function isResumeTokenUnusable(error: unknown): boolean { - const code = extractErrorCode(error); - if (code === 237 || code === 136) { - return false; - } - if (code !== undefined && UNUSABLE_RESUME_TOKEN_CODES.has(code)) { - return true; - } - const message = error instanceof Error ? error.message : String(error ?? ''); - return /ChangeStreamHistoryLost/i.test(message); -} - -function extractErrorCode(error: unknown): number | undefined { - if (!error || typeof error !== 'object') return undefined; - const candidate = error as { code?: unknown; errorCode?: unknown }; - if (typeof candidate.code === 'number') return candidate.code; - if (typeof candidate.errorCode === 'number') return candidate.errorCode; - return undefined; -} From 585a76ed242de6fef19cf2e886badb80b307da27 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 19:51:07 +0000 Subject: [PATCH 8/8] fix(database): drop leftover resumeAfter watch options Watch is pipeline-only (db.watch(pipeline)). Strip unused change-stream _id/clusterTime fields and test resume-token fixtures. Co-authored-by: Konstantinos Kopanidis --- .../realtime/MongoChangeStreamCoordinator.ts | 4 +- .../database/src/realtime/RealtimeService.ts | 6 +- .../realtime/__tests__/coordinator.test.ts | 81 +++++++------------ .../src/realtime/__tests__/normalize.test.ts | 10 +-- modules/database/src/realtime/normalize.ts | 2 - 5 files changed, 34 insertions(+), 69 deletions(-) diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts index 785bfe098..b6a8349e2 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -28,7 +28,7 @@ type LeaderLock = NonNullable< Awaited['tryAcquireLock']>> >; -export type WatchFactory = (options: { pipeline: WatchPipeline }) => ChangeStreamLike; +export type WatchFactory = (pipeline: WatchPipeline) => ChangeStreamLike; export type CoordinatorOptions = { grpcSdk: ConduitGrpcSdk; @@ -222,7 +222,7 @@ export class MongoChangeStreamCoordinator { const collections = this.collectionNames(); const pipeline = buildWatchPipeline(collections); this.watchedCollectionsKey = optedInCollectionsKey(collections); - const stream = this.options.watch({ pipeline }); + const stream = this.options.watch(pipeline); if (generation !== this.lockGeneration || this.closed) { try { await stream.close(); diff --git a/modules/database/src/realtime/RealtimeService.ts b/modules/database/src/realtime/RealtimeService.ts index 1c1a386a7..90369690e 100644 --- a/modules/database/src/realtime/RealtimeService.ts +++ b/modules/database/src/realtime/RealtimeService.ts @@ -41,7 +41,7 @@ export class RealtimeService { if (adapter instanceof MongooseAdapter) { this.coordinator = new MongoChangeStreamCoordinator({ grpcSdk, - watch: options => this.openWatch(adapter, options), + watch: pipeline => this.openWatch(adapter, pipeline), hello: () => this.hello(adapter), getOptedInSchemas: () => this.getOptedInSchemas(), subscriptions: this.subscriptions, @@ -131,13 +131,13 @@ export class RealtimeService { private openWatch( adapter: MongooseAdapter, - options: { pipeline: WatchPipeline }, + pipeline: WatchPipeline, ): ChangeStreamLike { const db = adapter.mongoose.connection.db; if (!db) { throw new Error('MongoDB connection is not ready'); } - return db.watch(options.pipeline) as unknown as ChangeStreamLike; + return db.watch(pipeline) as unknown as ChangeStreamLike; } private async hello( diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index 94d8275c7..c789ea94d 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -110,15 +110,22 @@ function createCoordinator(overrides?: { }; } -function insertChange(collection: string, id: string, token: unknown) { +function insertChange(collection: string, id: string) { return { operationType: 'insert', ns: { coll: collection }, documentKey: { _id: new ObjectId(id) }, - _id: token, }; } +function expectWatchFromNow( + watch: { mock: { calls: unknown[][] } }, + callIndex = 0, +) { + expect(watch.mock.calls[callIndex]).toHaveLength(1); + expect(watch.mock.calls[callIndex][0]).not.toHaveProperty('resumeAfter'); +} + describe('MongoChangeStreamCoordinator', () => { afterEach(() => { jest.useRealTimers(); @@ -127,13 +134,12 @@ describe('MongoChangeStreamCoordinator', () => { it('emits one normalized event to public rooms and ignores other collections', async () => { const { coordinator, stream, routerPush, adminPush, publish } = createCoordinator(); await coordinator.reconcile(); - const resume = { _data: 'token' }; stream.emit('change', { - ...insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', resume), + ...insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c'), fullDocument: { secret: 'nope' }, wallTime: new Date('2026-01-02T00:00:00.000Z'), }); - stream.emit('change', insertChange('other', '64b64c4c4c4c4c4c4c4c4c4d', resume)); + stream.emit('change', insertChange('other', '64b64c4c4c4c4c4c4c4c4c4d')); await coordinator.waitForIdle(); expect(publish).toHaveBeenCalledTimes(1); expect(publish.mock.calls[0][0]).toBe('database:change:Order'); @@ -173,14 +179,8 @@ describe('MongoChangeStreamCoordinator', () => { } }); await coordinator.reconcile(); - stream.emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token-a' }), - ); - stream.emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', { _data: 'token-b' }), - ); + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c')); + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d')); await Promise.resolve(); await new Promise(resolve => setImmediate(resolve)); expect(adminPush).toHaveBeenCalledTimes(1); @@ -194,8 +194,8 @@ describe('MongoChangeStreamCoordinator', () => { const { coordinator, watch } = createCoordinator(); await coordinator.reconcile(); expect(watch).toHaveBeenCalledTimes(1); - const pipeline = (watch.mock.calls[0][0] as { pipeline: Record[] }) - .pipeline; + const pipeline = watch.mock.calls[0][0] as Record[]; + expectWatchFromNow(watch); expect(pipeline[0]).toEqual( expect.objectContaining({ $match: expect.objectContaining({ @@ -214,7 +214,6 @@ describe('MongoChangeStreamCoordinator', () => { fullDocumentBeforeChange: 0, }, }); - expect(watch.mock.calls[0][0]).not.toHaveProperty('resumeAfter'); await coordinator.shutdown(); }); @@ -232,15 +231,14 @@ describe('MongoChangeStreamCoordinator', () => { }); await coordinator.reconcile(); expect(watch).toHaveBeenCalledTimes(2); - const pipeline = (watch.mock.calls[1][0] as { pipeline: Record[] }) - .pipeline; + const pipeline = watch.mock.calls[1][0] as Record[]; const match = pipeline[0] as { $match: { $or: Array<{ 'ns.coll'?: { $in: string[] } }> }; }; expect(match.$match.$or[0]['ns.coll']?.$in).toEqual( expect.arrayContaining(['orders', 'items']), ); - expect(watch.mock.calls[1][0]).not.toHaveProperty('resumeAfter'); + expectWatchFromNow(watch, 1); await coordinator.shutdown(); }); @@ -256,10 +254,7 @@ describe('MongoChangeStreamCoordinator', () => { 'user-1', ); await coordinator.reconcile(); - stream.emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), - ); + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c')); await coordinator.waitForIdle(); expect(routerPush).not.toHaveBeenCalled(); expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual( @@ -304,28 +299,16 @@ describe('MongoChangeStreamCoordinator', () => { .mockRejectedValueOnce(new Error('push failed')) .mockResolvedValue(undefined); await coordinator.reconcile(); - streams[0].emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4b', { _data: 'token-good' }), - ); + streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4b')); await coordinator.waitForIdle(); - streams[0].emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token-a' }), - ); - streams[0].emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', { _data: 'token-b' }), - ); + streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c')); + streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d')); await coordinator.waitForIdle(); expect(adminPush).toHaveBeenCalledTimes(2); await jest.advanceTimersByTimeAsync(1_000); expect(watch).toHaveBeenCalledTimes(2); - expect(watch.mock.calls[1][0]).not.toHaveProperty('resumeAfter'); - streams[1].emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token-a' }), - ); + expectWatchFromNow(watch, 1); + streams[1].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c')); await coordinator.waitForIdle(); expect(adminPush).toHaveBeenCalledTimes(3); await coordinator.shutdown(); @@ -350,12 +333,11 @@ describe('MongoChangeStreamCoordinator', () => { streams[0].emit('change', { operationType, ns: { coll: 'orders' }, - _id: { _data: `${operationType}-token` }, }); await coordinator.waitForIdle(); await jest.advanceTimersByTimeAsync(1_000); expect(watch).toHaveBeenCalledTimes(2); - expect(watch.mock.calls[1][0]).not.toHaveProperty('resumeAfter'); + expectWatchFromNow(watch, 1); await coordinator.shutdown(); jest.useRealTimers(); }, @@ -377,10 +359,7 @@ describe('MongoChangeStreamCoordinator', () => { 'user-1', ]); await coordinator.reconcile(); - stream.emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), - ); + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c')); await coordinator.waitForIdle(); expect(routerPush).not.toHaveBeenCalled(); expect(removeUser).not.toHaveBeenCalled(); @@ -407,10 +386,7 @@ describe('MongoChangeStreamCoordinator', () => { expect(coordinator.getState()).toBe('live'); await jest.advanceTimersByTimeAsync(5_000); expect(coordinator.getState()).toBe('idle'); - stream.emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'stale' }), - ); + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c')); await coordinator.waitForIdle(); expect(adminPush).not.toHaveBeenCalled(); await coordinator.shutdown(); @@ -438,10 +414,7 @@ describe('MongoChangeStreamCoordinator', () => { 'user-1', ); await coordinator.reconcile(); - stream.emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), - ); + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c')); await coordinator.waitForIdle(); expect(adminPush).toHaveBeenCalledTimes(1); expect(routerPush).not.toHaveBeenCalled(); diff --git a/modules/database/src/realtime/__tests__/normalize.test.ts b/modules/database/src/realtime/__tests__/normalize.test.ts index 1adcfae1a..223009676 100644 --- a/modules/database/src/realtime/__tests__/normalize.test.ts +++ b/modules/database/src/realtime/__tests__/normalize.test.ts @@ -4,14 +4,12 @@ import { normalizeChangeEvent } from '../normalize.js'; describe('normalizeChangeEvent', () => { it('normalizes insert/update/replace/delete into metadata-only events', () => { - const streamId = { _data: 'token-1' }; const event = normalizeChangeEvent( { operationType: 'insert', documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, fullDocument: { secret: 'nope' }, wallTime: new Date('2026-01-01T00:00:00.000Z'), - _id: streamId, }, 'Order', ); @@ -27,11 +25,7 @@ describe('normalizeChangeEvent', () => { }); it('ignores drop/invalidate and missing document ids', () => { - expect( - normalizeChangeEvent({ operationType: 'drop', _id: { _data: 'x' } }, 'Order'), - ).toBeNull(); - expect( - normalizeChangeEvent({ operationType: 'insert', _id: { _data: 'x' } }, 'Order'), - ).toBeNull(); + expect(normalizeChangeEvent({ operationType: 'drop' }, 'Order')).toBeNull(); + expect(normalizeChangeEvent({ operationType: 'insert' }, 'Order')).toBeNull(); }); }); diff --git a/modules/database/src/realtime/normalize.ts b/modules/database/src/realtime/normalize.ts index 9ea55bd0f..b5055c3d2 100644 --- a/modules/database/src/realtime/normalize.ts +++ b/modules/database/src/realtime/normalize.ts @@ -12,8 +12,6 @@ export type RawChangeEvent = { ns?: { coll?: string }; documentKey?: { _id?: unknown }; wallTime?: Date; - clusterTime?: { toString?: () => string }; - _id?: unknown; }; export function normalizeChangeEvent(