From 9c7af8cfa1470c28998edabd27be3f976769cc4b Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 13 Sep 2026 02:36:16 +0300 Subject: [PATCH 1/5] fix(router): Event Relays interventions (EventBus, Hermes, relay manager) (#1604) * fix(router): harden event relays for HA and auth lifetime EventBus uses a single Redis message dispatcher with subscriberId maps, SIGTERM/SIGINT shutdown, and subscribe-after-ACK. Hermes installs engine middleware once, skips empty-room namespace broadcasts, demotes hot-path logs, and re-authenticates recovered /events/ subscriptions. EventRelayManager gains coalesced periodic reconcile, compiled templates, inbound caps, TTL-cached emit-time ReBAC, room eviction, backpressure, preview API, and metrics/docs aligned with the interventions plan. * fix(router): address #1604 re-review P1 and P2 follow-ups Recovery re-auth keeps per-user subscription state across disconnect; emit-time ReBAC distinguishes unavailable vs deny; EventBus drops process signal handlers and adds subscribeAck; manager subscribes only after Redis ACK and broadcasts evictRelayIds on refresh; Hermes backpressure uses local sockets without emit acks; cache is bounded; preview caps sample JSON; handshake matcher rejects ticket/sid/non-namespace paths; socket middleware rebind avoids duplicate registration on first sockets enable. * fix(hermes): restore production tsc for engine middleware chain Type the Socket.IO engine middleware runner as Express NextFunction so recursive callbacks match registerGlobalMiddleware. Use Logger.info for socket trace lines (IConduitLogger has no debug). * fix(router): import Express Response for socket global middleware _rebindSocketGlobalMiddlewares passes handlers typed with fetch Response because Response was not imported from express, failing tsc against Hermes registerSocketGlobalMiddleware. * fix(router): quit EventBus on module shutdown signals Stop event relays and call grpcSdk.bus.quit() from Router.shutdown(), registered on SIGTERM/SIGINT with process exit so Redis teardown is not left to SDK signal handlers. Tighten registerGlobalMiddleware typing and trim handshake helper comment (deslop). * fix(router): pass-3 recovery, scoped emit, and subscription hygiene Recovery re-auth keeps membership on Authorization UNAVAILABLE and only leaves on deny; re-check subscriptions for restored er: rooms via room map. Prune user subscriptions on disconnect when no other socket holds them. Scope relay emits to receivers in the target room; use Engine.IO writeBuffer/writable for backpressure. Drop hot-path Hermes socket info logs. * fix(router): pass-4 recovery map, backpressure, emit tests Store relay subs on socket.data; resolve recovered rooms from context and TTL room map without last-writer userId. Prune room map when unused on disconnect. Backpressure uses writeBuffer depth only. Add Hermes/router tests for room-scoped emit and queue metric. * fix(router): clear eventRelaySubs from socket.data on recovery deny * fix(router): clear socket.data subs on recovery fail-closed leave --- libraries/grpc-sdk/package.json | 3 +- .../grpc-sdk/src/utilities/EventBus.test.ts | 121 +++++++++ libraries/grpc-sdk/src/utilities/EventBus.ts | 156 ++++++++--- libraries/grpc-sdk/tsconfig.test.json | 11 + libraries/hermes/package.json | 1 + libraries/hermes/src/Socket/Socket.ts | 256 ++++++++++++------ .../src/Socket/isSocketHandshake.test.ts | 26 ++ .../hermes/src/Socket/isSocketHandshake.ts | 26 ++ .../src/Socket/resolveEngineNamespacePath.ts | 11 + .../hermes/src/Socket/socketPushUtils.test.ts | 69 +++++ .../hermes/src/Socket/socketPushUtils.ts | 36 +++ libraries/hermes/src/index.ts | 23 +- libraries/hermes/src/interfaces/Socket.ts | 11 + libraries/hermes/src/interfaces/SocketPush.ts | 4 + libraries/hermes/tsconfig.json | 3 +- libraries/hermes/tsconfig.test.json | 17 ++ modules/router/README.mdx | 25 +- modules/router/src/Router.ts | 50 +++- modules/router/src/admin/event-relays.ts | 34 ++- modules/router/src/admin/index.ts | 15 + .../src/event-relays/EventRelayManager.ts | 242 ++++++++++++++--- .../src/event-relays/EventRelaySockets.ts | 68 ++++- modules/router/src/event-relays/authorize.ts | 2 - modules/router/src/event-relays/compile.ts | 50 ++++ modules/router/src/event-relays/constants.ts | 6 + .../router/src/event-relays/follow-up.test.ts | 186 +++++++++++++ .../src/event-relays/interventions.test.ts | 23 ++ .../router/src/event-relays/process.test.ts | 3 + modules/router/src/event-relays/process.ts | 24 +- modules/router/src/event-relays/push.ts | 13 +- .../src/event-relays/rebacCache.test.ts | 43 +++ modules/router/src/event-relays/rebacCache.ts | 108 ++++++++ modules/router/src/event-relays/recovery.ts | 85 ++++++ .../src/event-relays/relaySocketData.ts | 65 +++++ modules/router/src/event-relays/rooms.ts | 4 + .../router/src/event-relays/subscriptions.ts | 224 +++++++++++++++ modules/router/src/index.ts | 18 ++ modules/router/src/metrics/index.ts | 35 +++ modules/router/tsconfig.test.json | 3 + 39 files changed, 1912 insertions(+), 188 deletions(-) create mode 100644 libraries/grpc-sdk/src/utilities/EventBus.test.ts create mode 100644 libraries/grpc-sdk/tsconfig.test.json create mode 100644 libraries/hermes/src/Socket/isSocketHandshake.test.ts create mode 100644 libraries/hermes/src/Socket/isSocketHandshake.ts create mode 100644 libraries/hermes/src/Socket/resolveEngineNamespacePath.ts create mode 100644 libraries/hermes/src/Socket/socketPushUtils.test.ts create mode 100644 libraries/hermes/src/Socket/socketPushUtils.ts create mode 100644 libraries/hermes/tsconfig.test.json create mode 100644 modules/router/src/event-relays/compile.ts create mode 100644 modules/router/src/event-relays/follow-up.test.ts create mode 100644 modules/router/src/event-relays/interventions.test.ts create mode 100644 modules/router/src/event-relays/rebacCache.test.ts create mode 100644 modules/router/src/event-relays/rebacCache.ts create mode 100644 modules/router/src/event-relays/recovery.ts create mode 100644 modules/router/src/event-relays/relaySocketData.ts create mode 100644 modules/router/src/event-relays/subscriptions.ts diff --git a/libraries/grpc-sdk/package.json b/libraries/grpc-sdk/package.json index 25809a993..bbf717933 100644 --- a/libraries/grpc-sdk/package.json +++ b/libraries/grpc-sdk/package.json @@ -25,7 +25,8 @@ "prepublish": "npm run build", "prebuild": "npm run protoc", "build": "rimraf dist && tsup", - "protoc": "sh build.sh" + "protoc": "sh build.sh", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utilities/EventBus.test.js" }, "license": "MIT", "dependencies": { diff --git a/libraries/grpc-sdk/src/utilities/EventBus.test.ts b/libraries/grpc-sdk/src/utilities/EventBus.test.ts new file mode 100644 index 000000000..c1999901d --- /dev/null +++ b/libraries/grpc-sdk/src/utilities/EventBus.test.ts @@ -0,0 +1,121 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { EventBus } from './EventBus.js'; + +type Listener = (channel: string, message: string) => void; + +class FakeRedis { + handlers: Record = {}; + subscribed = new Set(); + failNext = new Set(); + + on(event: string, listener: Listener) { + this.handlers[event] = this.handlers[event] ?? []; + this.handlers[event].push(listener); + } + + subscribe(channel: string, cb?: (err?: Error | null) => void) { + if (this.failNext.has(channel)) { + this.failNext.delete(channel); + cb?.(new Error('subscribe failed')); + return; + } + this.subscribed.add(channel); + cb?.(null); + } + + unsubscribe(_channel: string, cb?: () => void) { + cb?.(); + } + + publish(_channel: string, _message: string) {} + + quit() {} + + emitMessage(channel: string, message: string) { + for (const listener of this.handlers.message ?? []) { + listener(channel, message); + } + } +} + +function createBus() { + const sub = new FakeRedis(); + const pub = new FakeRedis(); + const manager = { + getClient: () => sub, + }; + const bus = new EventBus(manager as never); + (bus as unknown as { _clientSubscriber: FakeRedis })._clientSubscriber = sub; + (bus as unknown as { _clientPublisher: FakeRedis })._clientPublisher = pub; + return { bus, sub }; +} + +describe('EventBus', () => { + it('fires once after deactivate/reactivate on the same channel', () => { + const { bus, sub } = createBus(); + let count = 0; + bus.subscribe( + 'database:update:Order', + () => { + count += 1; + }, + 'relay-a', + ); + bus.unsubscribe('relay-a'); + bus.subscribe( + 'database:update:Order', + () => { + count += 1; + }, + 'relay-a', + ); + sub.emitMessage('database:update:Order', '{"ok":true}'); + assert.equal(count, 1); + }); + + it('keeps the second subscriber when the first is removed', () => { + const { bus, sub } = createBus(); + let first = 0; + let second = 0; + bus.subscribe( + 'chan', + () => { + first += 1; + }, + 'one', + ); + bus.subscribe( + 'chan', + () => { + second += 1; + }, + 'two', + ); + bus.unsubscribe('one'); + sub.emitMessage('chan', 'x'); + assert.equal(first, 0); + assert.equal(second, 1); + }); + + it('subscribeAck rejects when Redis subscribe fails', async () => { + const { bus, sub } = createBus(); + sub.failNext.add('chan'); + await assert.rejects( + () => + bus.subscribeAck( + 'chan', + () => {}, + 'relay-a', + ), + /subscribe failed/, + ); + sub.failNext.delete('chan'); + let count = 0; + await bus.subscribeAck('chan', () => { + count += 1; + }, 'relay-a'); + sub.emitMessage('chan', 'x'); + assert.equal(count, 1); + }); +}); diff --git a/libraries/grpc-sdk/src/utilities/EventBus.ts b/libraries/grpc-sdk/src/utilities/EventBus.ts index 8cd8512f1..9388ad483 100644 --- a/libraries/grpc-sdk/src/utilities/EventBus.ts +++ b/libraries/grpc-sdk/src/utilities/EventBus.ts @@ -3,35 +3,58 @@ import { Cluster, Redis } from 'ioredis'; import crypto from 'crypto'; import { getLogger } from './GrpcSdkContext.js'; +type ChannelCallbacks = Map void>; + export class EventBus { private _clientSubscriber: Redis | Cluster; private _clientPublisher: Redis | Cluster; - private _subscribedChannels: { [listener: string]: ((message: string) => void)[] }; - private _subscribers: { [listener: string]: [string, number] }; + /** channelName -> subscriberId -> callback */ + private _channelCallbacks: Map; + /** subscriberId -> channelName */ + private _subscriberChannels: Map; + /** channels with a successful Redis SUBSCRIBE */ + private _redisSubscribedChannels: Set; + private _subscribeInFlight = new Map>(); private _signature: string; + private _anonymousSubscriberSeq = 0; + private _shuttingDown = false; constructor(redisManager: RedisManager) { - this._subscribedChannels = {}; - this._subscribers = {}; + this._channelCallbacks = new Map(); + this._subscriberChannels = new Map(); + this._redisSubscribedChannels = new Set(); this._clientSubscriber = redisManager.getClient({ keyPrefix: 'bus_' }); this._clientPublisher = redisManager.getClient({ keyPrefix: 'bus_' }); this._signature = crypto.randomBytes(20).toString('hex'); this._clientSubscriber.on('ready', () => { getLogger().log('The Bus is in the station...hehe'); }); + this._clientSubscriber.on('message', (channel: string, message: string) => { + this.dispatch(channel, message); + }); process.on('exit', () => { - this._clientSubscriber.quit(); - this._clientPublisher.quit(); + this.quit(); }); } + quit(): void { + if (this._shuttingDown) return; + this._shuttingDown = true; + void this._clientSubscriber.quit(); + void this._clientPublisher.quit(); + } + unsubscribe(subscriberId: string): void { - if (this._subscribers[subscriberId]) { - const [channelName, index] = this._subscribers[subscriberId]; - this._subscribedChannels[channelName].splice(index, 1); - delete this._subscribers[subscriberId]; - if (this._subscribedChannels[channelName].length === 0) { - delete this._subscribedChannels[channelName]; + const channelName = this._subscriberChannels.get(subscriberId); + if (!channelName) { + return; + } + const callbacks = this._channelCallbacks.get(channelName); + callbacks?.delete(subscriberId); + this._subscriberChannels.delete(subscriberId); + if (callbacks && callbacks.size === 0) { + this._channelCallbacks.delete(channelName); + if (this._redisSubscribedChannels.delete(channelName)) { this._clientSubscriber.unsubscribe(channelName, () => {}); } } @@ -42,43 +65,100 @@ export class EventBus { callback: (message: string) => void, subscriberId?: string, ): void { + void this.subscribeAck(channelName, callback, subscriberId).catch(err => { + getLogger().error( + `EventBus subscribe failed for ${channelName}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }); + } + + async subscribeAck( + channelName: string, + callback: (message: string) => void, + subscriberId?: string, + ): Promise { + if (this._shuttingDown) { + return; + } + const id = + subscriberId ?? + `anon:${channelName}:${++this._anonymousSubscriberSeq}:${crypto.randomBytes(4).toString('hex')}`; if (subscriberId) { - // if subscriberId is provided, and it is already subscribed, unsubscribe it first this.unsubscribe(subscriberId); } - if (this._subscribedChannels[channelName]) { - this._subscribedChannels[channelName].push(callback); - if (subscriberId) { - this._subscribers[subscriberId] = [ - channelName, - this._subscribedChannels[channelName].length - 1, - ]; - } + + let callbacks = this._channelCallbacks.get(channelName); + if (!callbacks) { + callbacks = new Map(); + this._channelCallbacks.set(channelName, callbacks); + } + callbacks.set(id, callback); + this._subscriberChannels.set(id, channelName); + + if (this._redisSubscribedChannels.has(channelName)) { return; } - this._subscribedChannels[channelName] = [callback]; - this._clientSubscriber.subscribe(channelName, () => {}); - const self = this; - this._clientSubscriber.on('message', (channel: string, message: string) => { - if (channel !== channelName) return; - // if the message supports the signature - if (message.indexOf('CND_Signature') !== -1) { - // if the message does not contain this module's signature - if (message.indexOf(self._signature) === -1) { - self._subscribedChannels[channelName].forEach(fn => { - fn(message.split('CND_Signature:')[0]); - }); - } - } else { - self._subscribedChannels[channelName].forEach(fn => { - fn(message); + + let inFlight = this._subscribeInFlight.get(channelName); + if (!inFlight) { + inFlight = new Promise((resolve, reject) => { + this._clientSubscriber.subscribe(channelName, err => { + if (err) { + reject(err); + return; + } + this._redisSubscribedChannels.add(channelName); + resolve(); }); + }).finally(() => { + this._subscribeInFlight.delete(channelName); + }); + this._subscribeInFlight.set(channelName, inFlight); + } + + try { + await inFlight; + } catch (err) { + if (!this._redisSubscribedChannels.has(channelName)) { + this.removeChannelCallbacks(channelName); } - }); + this.unsubscribe(id); + throw err; + } } publish(channelName: string, message: string) { message = message + `CND_Signature:${this._signature}`; this._clientPublisher.publish(channelName, message); } + + private removeChannelCallbacks(channelName: string): void { + const callbacks = this._channelCallbacks.get(channelName); + if (!callbacks) { + return; + } + for (const subId of callbacks.keys()) { + this._subscriberChannels.delete(subId); + } + this._channelCallbacks.delete(channelName); + } + + private dispatch(channel: string, message: string): void { + const callbacks = this._channelCallbacks.get(channel); + if (!callbacks || callbacks.size === 0) { + return; + } + let payload = message; + if (message.indexOf('CND_Signature') !== -1) { + if (message.indexOf(this._signature) !== -1) { + return; + } + payload = message.split('CND_Signature:')[0]; + } + for (const fn of callbacks.values()) { + fn(payload); + } + } } diff --git a/libraries/grpc-sdk/tsconfig.test.json b/libraries/grpc-sdk/tsconfig.test.json new file mode 100644 index 000000000..121161cd7 --- /dev/null +++ b/libraries/grpc-sdk/tsconfig.test.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": ["src/utilities/EventBus.ts", "src/utilities/EventBus.test.ts"] +} diff --git a/libraries/hermes/package.json b/libraries/hermes/package.json index 62f32ab5a..3beffb157 100644 --- a/libraries/hermes/package.json +++ b/libraries/hermes/package.json @@ -11,6 +11,7 @@ "scripts": { "prepublish": "npm run build", "build": "rimraf dist && tsc", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/Socket/*.test.js", "publish": "npm publish", "postbuild": "copyfiles -u 1 src/*.proto src/**/*.json ./dist/" }, diff --git a/libraries/hermes/src/Socket/Socket.ts b/libraries/hermes/src/Socket/Socket.ts index 74ced4e2b..7084fa857 100644 --- a/libraries/hermes/src/Socket/Socket.ts +++ b/libraries/hermes/src/Socket/Socket.ts @@ -16,6 +16,11 @@ import { import ObjectHash from 'object-hash'; import { ConduitError, ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import { buildSocketMiddlewareParams } from './buildSocketMiddlewareParams.js'; +import { resolveEngineNamespacePath } from './resolveEngineNamespacePath.js'; +import { + filterRemoteSocketsByUserAndRooms, + isEngineSocketBackpressured, +} from './socketPushUtils.js'; export class SocketController extends ConduitRouter { private readonly httpServer: httpServer; @@ -78,6 +83,22 @@ export class SocketController extends ConduitRouter { `Socket connection error, context: ${err?.context ?? 'N/A'}`, ); }); + + this.io.engine.use((req: any, res: any, next: NextFunction) => { + req.path = resolveEngineNamespacePath(req); + let index = 0; + const run: NextFunction = err => { + if (err) { + return next(err); + } + const middleware = this.globalMiddlewares[index++]; + if (!middleware) { + return next(); + } + middleware(req, res, run); + }; + run(); + }); } registerGlobalMiddleware( @@ -107,12 +128,6 @@ export class SocketController extends ConduitRouter { this._registeredNamespaces.set(namespace, conduitSocket); const self = this; - this.globalMiddlewares.forEach(middleware => { - self.io.engine.use((req: any, res: any, next: any) => { - req.path = namespace; - middleware(req, res, next); - }); - }); this.io.of(namespace).use((socket, next) => { const context = buildSocketMiddlewareParams(socket); self @@ -129,13 +144,21 @@ export class SocketController extends ConduitRouter { this.io.of(namespace).on('connect', socket => { if (socket.recovered) { - ConduitGrpcSdk.Logger.info( - `Socket recovered: ${socket.id} to namespace: ${namespace}`, - ); + const recovered = conduitSocket.executeRecovered({ + event: 'recovered', + socketId: socket.id, + context: socket.data, + recoveredRooms: [...socket.rooms].filter(room => room.startsWith('er:')), + }); + if (recovered) { + recovered + .then(res => this.handleResponse(res, socket, namespace)) + .catch(e => { + ConduitGrpcSdk.Logger.error(e); + socket.emit('conduit_error', e); + }); + } } else { - ConduitGrpcSdk.Logger.info( - `Socket connected: ${socket.id} to namespace: ${namespace}`, - ); conduitSocket .executeRequest({ event: 'connect', @@ -150,7 +173,6 @@ export class SocketController extends ConduitRouter { } socket.onAny((event, ...args) => { - ConduitGrpcSdk.Logger.info(`Socket event: ${event} from socket: ${socket.id}`); conduitSocket .executeRequest({ event, @@ -166,9 +188,6 @@ export class SocketController extends ConduitRouter { }); socket.on('disconnect', () => { - ConduitGrpcSdk.Logger.info( - `Socket disconnected: ${socket.id} from namespace: ${namespace}`, - ); conduitSocket .executeRequest({ event: 'disconnect', @@ -184,82 +203,162 @@ export class SocketController extends ConduitRouter { }); } - async handleSocketPush(push: SocketPush) { + async handleSocketPush(push: SocketPush): Promise { const localOnly = push.localOnly === true; if (push.event === 'join-room') { - if (push.rooms.length === 0) return; + if (push.rooms.length === 0) return false; const filteredSockets = await this.findAndFilterSockets( push.receivers, push.namespace, localOnly, ); for (const socket of filteredSockets) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} joining rooms: ${push.rooms.join(', ')} in namespace: ${ - push.namespace - }`, - ); socket.join(push.rooms); } + return true; } else if (push.event === 'leave-room') { if (push.rooms && push.rooms.length !== 0) { - const filteredSockets = await this.findAndFilterSockets( - push.receivers, - push.namespace, - localOnly, - ); + const filteredSockets = await this.socketsForRoomPush(push, localOnly); for (const socket of filteredSockets) { for (const room of push.rooms) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} leaving room: ${room} in namespace: ${push.namespace}`, - ); socket.leave(room); } } } + return true; } else if (isInstanceOfEventResponse(push)) { if ( (isNil(push.receivers) || push.receivers.length === 0) && push.rooms.length === 0 ) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${push.event} to all sockets in namespace: ${push.namespace}`, - ); + return false; + } + if (push.receivers.length !== 0) { const nsp = this.io.of(push.namespace); - if (localOnly) { - nsp.local.emit(push.event, push.data); - } else { - nsp.emit(push.event, push.data); + const filteredSockets = await this.findAndFilterSockets( + push.receivers, + push.namespace, + localOnly, + push.rooms, + ); + if (push.skipEmptyRooms && filteredSockets.length === 0) { + return false; } - } else { - if (push.rooms.length !== 0) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${push.event} to rooms: ${push.rooms.join( - ', ', - )} in namespace: ${push.namespace}`, - ); - const target = this.io.of(push.namespace).to(push.rooms); - if (localOnly) { - target.local.emit(push.event, push.data); - } else { - target.emit(push.event, push.data); + for (const remote of filteredSockets) { + const local = localOnly ? nsp.sockets.get(remote.id) : undefined; + if ( + push.boundedEmit && + localOnly && + local && + this.isLocalSocketBackpressured(local) + ) { + ConduitGrpcSdk.Metrics?.increment('event_relays_emit_dropped_total'); + local.disconnect(true); + continue; } + remote.emit(push.event, push.data); } - if (push.receivers.length !== 0) { - const filteredSockets = await this.findAndFilterSockets( - push.receivers, - push.namespace, - localOnly, - ); - for (const socket of filteredSockets) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${push.event} to socket: ${socket.id} in namespace: ${push.namespace}`, - ); - socket.emit(push.event, push.data); + return true; + } + if (push.rooms.length !== 0) { + const emitted = await this.emitEventToRooms(push, localOnly); + if (!emitted) { + return false; + } + } + return true; + } + return false; + } + + async getLocalRoomUserIds(namespace: string, room: string): Promise { + const sockets = await this.io.of(namespace).in(room).local.fetchSockets(); + const userIds = new Set(); + for (const socket of sockets) { + const userId = socket.data?.user?._id; + if (typeof userId === 'string' && userId.length > 0) { + userIds.add(userId); + } + } + return [...userIds]; + } + + async getLocalRoomsWithPrefix(namespace: string, prefix: string): Promise { + const adapter = this.io.of(namespace).adapter as { rooms?: Map }; + const rooms = adapter.rooms; + if (!rooms) { + return []; + } + return [...rooms.keys()].filter(room => room.startsWith(prefix)); + } + + private async emitEventToRooms(push: SocketPush, localOnly: boolean): Promise { + const nsp = this.io.of(push.namespace); + const localSockets = localOnly ? this.localSocketsInRooms(nsp, push.rooms) : []; + if (push.skipEmptyRooms || push.boundedEmit) { + if (localOnly && localSockets.length === 0) { + return false; + } + if (push.boundedEmit && localOnly) { + for (const socket of localSockets) { + if (this.isLocalSocketBackpressured(socket)) { + ConduitGrpcSdk.Metrics?.increment('event_relays_emit_dropped_total'); + socket.disconnect(true); } } } } + + const target = nsp.to(push.rooms); + if (localOnly) { + target.local.emit(push.event, push.data); + } else { + target.emit(push.event, push.data); + } + return true; + } + + private localSocketsInRooms( + nsp: ReturnType, + rooms: string[], + ): Socket[] { + const sockets: Socket[] = []; + for (const socket of nsp.sockets.values()) { + if (rooms.some(room => socket.rooms.has(room))) { + sockets.push(socket); + } + } + return sockets; + } + + private isLocalSocketBackpressured(socket: Socket): boolean { + return isEngineSocketBackpressured( + socket.conn as unknown as { writeBuffer?: unknown[] }, + ); + } + + private async socketsForRoomPush( + push: SocketPush, + localOnly: boolean, + ): Promise[]> { + if (push.receivers.length > 0) { + return this.findAndFilterSockets(push.receivers, push.namespace, localOnly); + } + const nsp = this.io.of(push.namespace); + const seen = new Set(); + const sockets: RemoteSocket[] = []; + for (const room of push.rooms) { + const inRoom = localOnly + ? await nsp.in(room).local.fetchSockets() + : await nsp.in(room).fetchSockets(); + for (const socket of inRoom) { + if (!seen.has(socket.id)) { + seen.add(socket.id); + sockets.push(socket); + } + } + } + return sockets; } private async handleResponse( @@ -269,19 +368,11 @@ export class SocketController extends ConduitRouter { ) { if (res.event === 'join-room') { if (res.rooms && res.rooms.length !== 0) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} joining rooms: ${res.rooms.join( - ', ', - )} in namespace: ${namespace}`, - ); socket.join(res.rooms); } } else if (res.event === 'leave-room') { if (res.rooms && res.rooms.length !== 0) { for (const room of res.rooms) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} leaving room: ${room} in namespace: ${namespace}`, - ); socket.leave(room); } } @@ -290,17 +381,9 @@ export class SocketController extends ConduitRouter { (!res.receivers || res.receivers.length === 0) && (!res.rooms || res.rooms.length === 0) ) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${res.event} to all sockets in namespace: ${namespace}`, - ); socket.emit(res.event, JSON.parse(res.data)); } else { if (res.rooms && res.rooms.length !== 0) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${res.event} to rooms: ${res.rooms.join( - ', ', - )} in namespace: ${namespace}`, - ); this.io.of(namespace).to(res.rooms).emit(res.event, JSON.parse(res.data)); } if (res.receivers && res.receivers.length !== 0) { @@ -309,9 +392,6 @@ export class SocketController extends ConduitRouter { namespace, ); for (const socket of filteredSockets) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${res.event} to socket: ${socket.id} in namespace: ${namespace}`, - ); socket.emit(res.event, JSON.parse(res.data)); } } @@ -323,15 +403,19 @@ export class SocketController extends ConduitRouter { userIds: string[], namespace: string, localOnly: boolean = false, + rooms: string[] = [], ): Promise[]> { const nsp = this.io.of(namespace); const sockets = localOnly ? await nsp.local.fetchSockets() : await nsp.fetchSockets(); - const userIdSet = new Set(userIds); - return sockets.filter(socket => { - if (socket.data && socket.data.user) { - return userIdSet.has(socket.data.user._id); - } - }); + return filterRemoteSocketsByUserAndRooms( + sockets.map(socket => ({ + id: socket.id, + data: socket.data, + rooms: socket.rooms, + })), + userIds, + rooms, + ).map(filtered => sockets.find(s => s.id === filtered.id)!); } protected _refreshRouter(): void { diff --git a/libraries/hermes/src/Socket/isSocketHandshake.test.ts b/libraries/hermes/src/Socket/isSocketHandshake.test.ts new file mode 100644 index 000000000..267de330f --- /dev/null +++ b/libraries/hermes/src/Socket/isSocketHandshake.test.ts @@ -0,0 +1,26 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { isSocketHandshake } from './isSocketHandshake.js'; + +describe('isSocketHandshake', () => { + it('matches Engine.IO handshake query params only', () => { + assert.equal( + isSocketHandshake({ url: '/events/?EIO=4&transport=polling&t=abc' }), + true, + ); + assert.equal(isSocketHandshake({ url: '/realtime/?EIO=4&transport=websocket' }), true); + }); + + it('rejects HTTP routes that merely mention realtime or established sessions', () => { + assert.equal( + isSocketHandshake({ url: '/realtime/ticket?EIO=4&transport=polling' }), + false, + ); + assert.equal(isSocketHandshake({ originalUrl: '/realtime' }), false); + assert.equal(isSocketHandshake({ url: '/graphql?EIO=4&transport=polling' }), false); + assert.equal( + isSocketHandshake({ url: '/events/?EIO=4&transport=polling&sid=abc' }), + false, + ); + }); +}); diff --git a/libraries/hermes/src/Socket/isSocketHandshake.ts b/libraries/hermes/src/Socket/isSocketHandshake.ts new file mode 100644 index 000000000..aa216d2ac --- /dev/null +++ b/libraries/hermes/src/Socket/isSocketHandshake.ts @@ -0,0 +1,26 @@ +export function isSocketHandshake(req: { url?: string; originalUrl?: string }): boolean { + const raw = req.url ?? req.originalUrl ?? ''; + if (!raw) { + return false; + } + const queryIndex = raw.indexOf('?'); + const pathname = queryIndex === -1 ? raw : raw.slice(0, queryIndex); + if (pathname.includes('ticket')) { + return false; + } + const query = queryIndex === -1 ? '' : raw.slice(queryIndex + 1); + if (!query) { + return false; + } + const params = new URLSearchParams(query); + if (!params.has('EIO') || !params.has('transport')) { + return false; + } + if (params.has('sid')) { + return false; + } + if (pathname !== '/' && !pathname.endsWith('/')) { + return false; + } + return true; +} diff --git a/libraries/hermes/src/Socket/resolveEngineNamespacePath.ts b/libraries/hermes/src/Socket/resolveEngineNamespacePath.ts new file mode 100644 index 000000000..e36aad12b --- /dev/null +++ b/libraries/hermes/src/Socket/resolveEngineNamespacePath.ts @@ -0,0 +1,11 @@ +/** + * Derive the Socket.IO namespace path from an Engine.IO handshake request URL. + */ +export function resolveEngineNamespacePath(req: { url?: string }): string { + const url = req.url ?? '/'; + const pathname = url.split('?')[0] ?? '/'; + if (pathname === '/' || pathname === '') { + return '/'; + } + return pathname.endsWith('/') ? pathname : `${pathname}/`; +} diff --git a/libraries/hermes/src/Socket/socketPushUtils.test.ts b/libraries/hermes/src/Socket/socketPushUtils.test.ts new file mode 100644 index 000000000..7f76e2904 --- /dev/null +++ b/libraries/hermes/src/Socket/socketPushUtils.test.ts @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { + filterRemoteSocketsByUserAndRooms, + isEngineSocketBackpressured, + WRITE_BUFFER_PACKET_HIGH_WATER, +} from './socketPushUtils.js'; + +describe('filterRemoteSocketsByUserAndRooms', () => { + const roomA = 'er:relay-1:aaa'; + const roomB = 'er:relay-1:bbb'; + + it('emits only to sockets in the target room for the same user', () => { + const sockets = [ + { + id: 'tab-a', + data: { user: { _id: 'user-1' } }, + rooms: new Set([roomA]), + }, + { + id: 'tab-b', + data: { user: { _id: 'user-1' } }, + rooms: new Set([roomB]), + }, + ]; + + const filtered = filterRemoteSocketsByUserAndRooms(sockets, ['user-1'], [roomA]); + assert.deepEqual( + filtered.map(s => s.id), + ['tab-a'], + ); + }); +}); + +describe('isEngineSocketBackpressured', () => { + it('uses writeBuffer queue depth only', () => { + assert.equal(isEngineSocketBackpressured(undefined), false); + assert.equal( + isEngineSocketBackpressured({ writeBuffer: new Array(WRITE_BUFFER_PACKET_HIGH_WATER) }), + false, + ); + assert.equal( + isEngineSocketBackpressured({ + writeBuffer: new Array(WRITE_BUFFER_PACKET_HIGH_WATER + 1), + }), + true, + ); + assert.equal( + isEngineSocketBackpressured({ + writeBuffer: [], + }), + false, + ); + }); +}); + +describe('SocketController relay emit', () => { + it('does not reference writableLength or client emit ack callbacks', () => { + const socketSource = readFileSync( + path.join(path.dirname(fileURLToPath(import.meta.url)), '../../src/Socket/Socket.ts'), + 'utf8', + ); + assert.doesNotMatch(socketSource, /writableLength/); + assert.doesNotMatch(socketSource, /\.emit\([^)]*,\s*[^,)]+,\s*\(\)\s*=>/); + }); +}); diff --git a/libraries/hermes/src/Socket/socketPushUtils.ts b/libraries/hermes/src/Socket/socketPushUtils.ts new file mode 100644 index 000000000..6b3e14feb --- /dev/null +++ b/libraries/hermes/src/Socket/socketPushUtils.ts @@ -0,0 +1,36 @@ +export const WRITE_BUFFER_PACKET_HIGH_WATER = 64; + +type SocketLike = { + id: string; + data?: { user?: { _id?: string } }; + rooms: Set; +}; + +export function filterRemoteSocketsByUserAndRooms( + sockets: SocketLike[], + userIds: string[], + rooms: string[], +): SocketLike[] { + const userIdSet = new Set(userIds); + const roomSet = rooms.length > 0 ? new Set(rooms) : null; + return sockets.filter(socket => { + if (!socket.data?.user?._id) { + return false; + } + if (!userIdSet.has(socket.data.user._id)) { + return false; + } + if (roomSet) { + return [...roomSet].some(room => socket.rooms.has(room)); + } + return true; + }); +} + +export function isEngineSocketBackpressured( + conn: { writeBuffer?: unknown[] } | undefined, + highWater = WRITE_BUFFER_PACKET_HIGH_WATER, +): boolean { + const pending = conn?.writeBuffer?.length ?? 0; + return pending > highWater; +} diff --git a/libraries/hermes/src/index.ts b/libraries/hermes/src/index.ts index ab17086b2..ac1a9960b 100644 --- a/libraries/hermes/src/index.ts +++ b/libraries/hermes/src/index.ts @@ -235,6 +235,12 @@ export class ConduitRoutingController { } } + registerSocketGlobalMiddleware( + middleware: (req: ConduitRequest, res: Response, next: NextFunction) => void, + ) { + this._socketRouter?.registerGlobalMiddleware(middleware); + } + registerRouteMiddleware(middleware: ConduitMiddleware, moduleUrl: string) { this._restRouter?.registerMiddleware(middleware, moduleUrl); this._graphQLRouter?.registerMiddleware(middleware, moduleUrl); @@ -322,7 +328,20 @@ export class ConduitRoutingController { } async socketPush(data: SocketPush) { - await this._socketRouter?.handleSocketPush(data); + return (await this._socketRouter?.handleSocketPush(data)) ?? false; + } + + getLocalRoomUserIds(namespace: string, room: string): Promise { + return ( + this._socketRouter?.getLocalRoomUserIds(namespace, room) ?? Promise.resolve([]) + ); + } + + getLocalRoomsWithPrefix(namespace: string, prefix: string): Promise { + return ( + this._socketRouter?.getLocalRoomsWithPrefix(namespace, prefix) ?? + Promise.resolve([]) + ); } /** True if any enabled transport would change this route (new or definition changed). */ @@ -451,6 +470,8 @@ export class ConduitRoutingController { } } +export { isSocketHandshake } from './Socket/isSocketHandshake.js'; +export { resolveEngineNamespacePath } from './Socket/resolveEngineNamespacePath.js'; export * from './interfaces/index.js'; export * from './types/index.js'; export * from './classes/index.js'; diff --git a/libraries/hermes/src/interfaces/Socket.ts b/libraries/hermes/src/interfaces/Socket.ts index 204707624..7c7851094 100644 --- a/libraries/hermes/src/interfaces/Socket.ts +++ b/libraries/hermes/src/interfaces/Socket.ts @@ -10,6 +10,7 @@ export interface ConduitSocketParameters { socketId: string; params?: UntypedArray; context?: Indexable; + recoveredRooms?: string[]; } export type ConduitSocketParamTypes = (TYPE | ConduitSocketParamTypes)[]; @@ -19,6 +20,7 @@ export interface ConduitSocketOptions { name?: string; description?: string; middlewares?: string[]; + onRecovered?: ConduitSocketEventHandler; } export type EventResponse = { @@ -85,6 +87,15 @@ export class ConduitSocket { } return Promise.reject('no such event registered'); } + + executeRecovered( + request: ConduitSocketParameters, + ): ConduitSocketHandlerResponse | null { + if (!this._input.onRecovered) { + return null; + } + return this._input.onRecovered(request); + } } export interface EventsProtoDescription { diff --git a/libraries/hermes/src/interfaces/SocketPush.ts b/libraries/hermes/src/interfaces/SocketPush.ts index 8bbabbc85..7d0f572d3 100644 --- a/libraries/hermes/src/interfaces/SocketPush.ts +++ b/libraries/hermes/src/interfaces/SocketPush.ts @@ -5,4 +5,8 @@ export interface SocketPush { rooms: string[]; namespace: string; localOnly?: boolean; + /** Skip emit when the local room has no connected sockets. */ + skipEmptyRooms?: boolean; + /** Drop or disconnect slow clients instead of blocking the caller. */ + boundedEmit?: boolean; } diff --git a/libraries/hermes/tsconfig.json b/libraries/hermes/tsconfig.json index cd32b1ca8..225b927d7 100644 --- a/libraries/hermes/tsconfig.json +++ b/libraries/hermes/tsconfig.json @@ -65,5 +65,6 @@ /* Advanced Options */ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ - } + }, + "exclude": ["src/**/*.test.ts"] } diff --git a/libraries/hermes/tsconfig.test.json b/libraries/hermes/tsconfig.test.json new file mode 100644 index 000000000..8aecbdee9 --- /dev/null +++ b/libraries/hermes/tsconfig.test.json @@ -0,0 +1,17 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": [ + "src/Socket/isSocketHandshake.ts", + "src/Socket/isSocketHandshake.test.ts", + "src/Socket/socketPushUtils.ts", + "src/Socket/socketPushUtils.test.ts" + ], + "exclude": [] +} diff --git a/modules/router/README.mdx b/modules/router/README.mdx index c896f2cee..a0dad3165 100644 --- a/modules/router/README.mdx +++ b/modules/router/README.mdx @@ -92,12 +92,12 @@ import { io } from 'socket.io-client'; const socket = io(`${SOCKET_BASE_URL}/events/`, { path: '/realtime', - extraHeaders: { - authorization: `Bearer ${accessToken}`, - }, + auth: { token: accessToken }, }); -socket.emit('subscribe', relayId, resourceId); +socket.on('connect', () => { + socket.emit('subscribe', relayId, resourceId); +}); socket.on('order-updated', payload => { // payload is the rendered JSON template }); @@ -108,6 +108,20 @@ socket.emit('unsubscribe', relayId, resourceId); - The outbound event name is the relay `socketEvent`. - Placeholders in `messageTemplate` use `{{payload.path}}` against the bus JSON payload. +**Subscribe-only.** Clients cannot publish on `/events/`; modules publish to the Redis bus and relays forward matching messages to subscribed rooms. + +**Authentication and Authorization** are required (`authMiddleware` on subscribe). Authorization is re-checked on a short TTL during delivery; revoked access triggers `leave-room`. + +**Rooms** are server-assigned: `er::`. Conduit has no tenant model and **team ids are not part of room names** — isolation is ReBAC on `User` × `permission` × `resourceType:resourceId`. + +**HA / `localOnly`:** every Router replica subscribes to the bus and emits with `localOnly`, so clients spread across replicas still receive one copy without sticky sessions for delivery. Engine.IO **polling** transport still benefits from sticky sessions or WebSocket-only clients. + +**EventBus self-publish:** the bus ignores this process’s own publishes (signature filter). A relay on a channel the Router publishes will not deliver to local clients for those self-originated messages (typical for module-originated events). + +**Dual delivery warning:** Database `/database/` `change` events and a relay on overlapping bus channels (for example `database:change:*` or CRUD `database:update:*`) can duplicate notifications. CRUD bus payloads include **full documents** — prefer change-stream style payloads when both paths are enabled. + +Use preview to validate templates: `POST /router/event-relays/preview` with `messageTemplate` and `samplePayload` (same renderer as runtime; sample JSON is capped at 256 KiB like inbound bus payloads). + ### Admin API | Method | Path | Description | @@ -115,5 +129,6 @@ socket.emit('unsubscribe', relayId, resourceId); | `GET` | `/router/event-relays` | Paginated list (`skip`, `limit`, `search`) | | `GET` | `/router/event-relays/:id` | Single relay | | `POST` | `/router/event-relays` | Create | -| `PATCH` | `/router/event-relays/:id` | Update | +| `POST` | `/router/event-relays/preview` | Render template against sample JSON | +| `PATCH` | `/router/event-relays/:id` | Update (deactivating evicts rooms) | | `DELETE` | `/router/event-relays/:id` | Delete | diff --git a/modules/router/src/Router.ts b/modules/router/src/Router.ts index 0a3867ed6..78369f6e7 100644 --- a/modules/router/src/Router.ts +++ b/modules/router/src/Router.ts @@ -1,4 +1,4 @@ -import { NextFunction } from 'express'; +import { NextFunction, Response } from 'express'; import { status } from '@grpc/grpc-js'; import { ConduitGrpcSdk, @@ -66,6 +66,11 @@ export default class ConduitDefaultRouter extends ManagedModule { private adminRouter: AdminHandlers; private readonly _routes: string[]; private readonly _globalMiddlewares: string[]; + private readonly _socketGlobalMiddlewareHandlers: (( + req: ConduitRequest, + res: Response, + next: NextFunction, + ) => void)[]; private _grpcRoutes: { [field: string]: RouteT[]; } = {}; @@ -76,12 +81,15 @@ export default class ConduitDefaultRouter extends ManagedModule { private _haInitialized = false; private eventRelayManager: EventRelayManager; private eventsSocket?: ConduitSocket; + private socketsPreviouslyStopped = false; + private securityMiddlewareInitialized = false; constructor(peerManifestRoot?: string) { super('router', peerManifestRoot); this.updateHealth(HealthCheckStatus.UNKNOWN, true); this._routes = []; this._globalMiddlewares = []; + this._socketGlobalMiddlewareHandlers = []; } async onServerStart() { @@ -112,6 +120,12 @@ export default class ConduitDefaultRouter extends ManagedModule { this.eventRelayManager = new EventRelayManager( this.grpcSdk, createEventRelayPusher(data => this._internalRouter.socketPush(data)), + { + getLocalRoomUserIds: (room: string) => + this._internalRouter.getLocalRoomUserIds('/events/', room), + getLocalRoomsWithPrefix: (prefix: string) => + this._internalRouter.getLocalRoomsWithPrefix('/events/', prefix), + }, ); this.adminRouter = new AdminHandlers( this.grpcServer, @@ -133,6 +147,10 @@ export default class ConduitDefaultRouter extends ManagedModule { async onConfig() { const config = ConfigController.getInstance().config; + const shouldRebindSocketGlobals = + config.transports.sockets && + this.socketsPreviouslyStopped && + this.securityMiddlewareInitialized; let atLeastOne = false; if (config.transports.graphql) { this._internalRouter.initGraphQL(); @@ -148,16 +166,24 @@ export default class ConduitDefaultRouter extends ManagedModule { } if (config.transports.sockets) { this._internalRouter.initSockets(); - this.registerEventsNamespace(); await this.eventRelayManager.start(); atLeastOne = true; } else { await this.eventRelayManager?.stop(); this._internalRouter.stopSockets(); + this.socketsPreviouslyStopped = true; } - if (atLeastOne) { + if (atLeastOne && !this.securityMiddlewareInitialized) { this._security.setupMiddlewares(); + this.securityMiddlewareInitialized = true; + } + if (config.transports.sockets) { + if (shouldRebindSocketGlobals) { + this.rebindSocketGlobalMiddlewares(); + } + this.socketsPreviouslyStopped = false; + this.registerEventsNamespace(); } if (!this._sdkRoutes.some(r => r.path === '/ready')) { this.registerRoute(adminRoutes.getReadyRoute()); @@ -335,13 +361,29 @@ export default class ConduitDefaultRouter extends ManagedModule { registerGlobalMiddleware( name: string, - middleware: any, + middleware: (req: ConduitRequest, res: Response, next: NextFunction) => void, socketMiddleware: boolean = false, ) { this._globalMiddlewares.push(name); + if (socketMiddleware) { + this._socketGlobalMiddlewareHandlers.push(middleware); + } this._internalRouter.registerMiddleware(middleware, socketMiddleware); } + private rebindSocketGlobalMiddlewares() { + for (const middleware of this._socketGlobalMiddlewareHandlers) { + this._internalRouter.registerSocketGlobalMiddleware(middleware); + } + } + + async shutdown(): Promise { + if (this.eventRelayManager) { + await this.eventRelayManager.stop(); + } + this.grpcSdk.bus?.quit(); + } + getRegisteredRoutes() { return this._routes; } diff --git a/modules/router/src/admin/event-relays.ts b/modules/router/src/admin/event-relays.ts index 2e56fbd15..442b3efd7 100644 --- a/modules/router/src/admin/event-relays.ts +++ b/modules/router/src/admin/event-relays.ts @@ -10,7 +10,9 @@ import { EventRelay } from '../models/index.js'; import { EventRelayManager } from '../event-relays/EventRelayManager.js'; import { EventRelayInput, validateEventRelayInput } from '../event-relays/validation.js'; import { EventRelayValidationError } from '../event-relays/validationError.js'; +import { renderMessageTemplate } from '../event-relays/template.js'; import { buildSearchQuery, parsePagination } from '../event-relays/search.js'; +import { assertJsonPayloadSize } from '../event-relays/process.js'; export class EventRelayAdmin { constructor(private readonly manager: EventRelayManager) {} @@ -52,6 +54,23 @@ export class EventRelayAdmin { return relay; } + async previewEventRelay(call: ParsedRouterRequest): Promise { + const { messageTemplate, samplePayload } = call.request.params as { + messageTemplate: unknown; + samplePayload: unknown; + }; + try { + assertJsonPayloadSize(samplePayload); + const rendered = renderMessageTemplate(messageTemplate, samplePayload); + return { rendered }; + } catch (err) { + if (err instanceof EventRelayValidationError) { + throw new GrpcError(status.INVALID_ARGUMENT, err.message); + } + throw err; + } + } + async patchEventRelay(call: ParsedRouterRequest): Promise { const existing = await EventRelay.getInstance().findOne({ _id: call.request.params.id, @@ -89,7 +108,18 @@ export class EventRelayAdmin { ...input, messageTemplate: input.messageTemplate as EventRelay['messageTemplate'], }); - await this.manager.notifyChanged(); + const evictRelayIds: string[] = []; + if (!input.active) { + evictRelayIds.push(existing._id); + } else if ( + input.permission !== existing.permission || + input.resourceType !== existing.resourceType + ) { + evictRelayIds.push(existing._id); + } + await this.manager.notifyChanged({ + evictRelayIds: evictRelayIds.length ? evictRelayIds : undefined, + }); return updated!; } @@ -101,7 +131,7 @@ export class EventRelayAdmin { throw new GrpcError(status.NOT_FOUND, 'Event relay not found'); } await EventRelay.getInstance().deleteOne({ _id: existing._id }); - await this.manager.notifyChanged(); + await this.manager.notifyChanged({ evictRelayIds: [existing._id] }); return { message: 'Event relay deleted' }; } } diff --git a/modules/router/src/admin/index.ts b/modules/router/src/admin/index.ts index 0c3231378..e943c25fa 100644 --- a/modules/router/src/admin/index.ts +++ b/modules/router/src/admin/index.ts @@ -242,6 +242,21 @@ export class AdminHandlers { }), this.eventRelayAdmin.deleteEventRelay.bind(this.eventRelayAdmin), ); + this.routingManager.route( + { + path: '/event-relays/preview', + action: ConduitRouteActions.POST, + description: `Renders a relay message template against sample bus JSON (same engine as runtime).`, + bodyParams: { + messageTemplate: ConduitJson.Required, + samplePayload: ConduitJson.Required, + }, + }, + new ConduitRouteReturnDefinition('PreviewEventRelay', { + rendered: ConduitJson.Required, + }), + this.eventRelayAdmin.previewEventRelay.bind(this.eventRelayAdmin), + ); this.routingManager.registerRoutes(); } } diff --git a/modules/router/src/event-relays/EventRelayManager.ts b/modules/router/src/event-relays/EventRelayManager.ts index 80cb809b7..905b0180e 100644 --- a/modules/router/src/event-relays/EventRelayManager.ts +++ b/modules/router/src/event-relays/EventRelayManager.ts @@ -3,70 +3,170 @@ import { EventRelay } from '../models/index.js'; import { EVENT_RELAY_REFRESH_CHANNEL, EVENT_RELAY_SUBSCRIBER_PREFIX, + RECONCILE_INTERVAL_MS, + RELAY_REBAC_TTL_MS, } from './constants.js'; import { groupRelaysByChannel, planChannelSubscriptions } from './channels.js'; -import { buildRelayEmissions, parseBusPayload } from './process.js'; +import { parseBusPayload } from './process.js'; import { EventRelayPusher } from './push.js'; +import { compileRelay, CompiledRelay } from './compile.js'; +import { checkRebacBatch, RelayRebacCache } from './rebacCache.js'; +import { eventRelayRoomPrefix } from './rooms.js'; +import { removeSubscriptionsForRelay } from './subscriptions.js'; export type { EventRelayPusher } from './push.js'; export { createEventRelayPusher } from './push.js'; +export type EventRelaySocketAccess = { + getLocalRoomUserIds: (room: string) => Promise; + getLocalRoomsWithPrefix: (prefix: string) => Promise; +}; + +type RefreshPayload = { + evictRelayIds?: string[]; +}; + export class EventRelayManager { - private readonly relaysByChannel = new Map(); + private readonly relaysById = new Map(); + private readonly relaysByChannel = new Map(); private readonly subscribedChannels = new Set(); + private readonly rebacCache = new RelayRebacCache(RELAY_REBAC_TTL_MS); private started = false; + private reconcilePending = false; + private reconciling = false; + private reconcileTimer?: NodeJS.Timeout; constructor( private readonly grpcSdk: ConduitGrpcSdk, private readonly push: EventRelayPusher, + private readonly sockets: EventRelaySocketAccess, ) {} async start(): Promise { if (!this.started) { this.grpcSdk.bus?.subscribe( EVENT_RELAY_REFRESH_CHANNEL, - () => { - this.reconcile().catch(err => { - ConduitGrpcSdk.Logger.error(err as Error); - }); + message => { + void this.onRefreshMessage(message); }, 'router-event-relays-refresh', ); + this.reconcileTimer = setInterval(() => { + void this.reconcile(); + }, RECONCILE_INTERVAL_MS); this.started = true; } await this.reconcile(); } async stop(): Promise { + if (this.reconcileTimer) { + clearInterval(this.reconcileTimer); + this.reconcileTimer = undefined; + } for (const channel of [...this.subscribedChannels]) { this.grpcSdk.bus?.unsubscribe(`${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`); this.subscribedChannels.delete(channel); } this.relaysByChannel.clear(); + this.relaysById.clear(); + this.rebacCache.clear(); if (this.started) { this.grpcSdk.bus?.unsubscribe('router-event-relays-refresh'); this.started = false; } + ConduitGrpcSdk.Metrics?.set('event_relays_active_total', 0); + ConduitGrpcSdk.Metrics?.set('event_relays_subscribed_channels_total', 0); } - async notifyChanged(): Promise { + async notifyChanged(options?: { evictRelayIds?: string[] }): Promise { + if (options?.evictRelayIds?.length) { + for (const relayId of options.evictRelayIds) { + await this.evictRelayRooms(relayId); + } + } if (this.started) { await this.reconcile(); } - this.grpcSdk.bus?.publish(EVENT_RELAY_REFRESH_CHANNEL, ''); + const payload: RefreshPayload = { + evictRelayIds: options?.evictRelayIds ?? [], + }; + this.grpcSdk.bus?.publish(EVENT_RELAY_REFRESH_CHANNEL, JSON.stringify(payload)); + } + + async evictRelayRooms(relayId: string): Promise { + const prefix = eventRelayRoomPrefix(relayId); + const rooms = await this.sockets.getLocalRoomsWithPrefix(prefix); + if (rooms.length === 0) { + removeSubscriptionsForRelay(relayId); + return; + } + await this.push('leave-room', undefined, rooms); + removeSubscriptionsForRelay(relayId); + } + + getActiveRelay(id: string): EventRelay | undefined { + return this.relaysById.get(id); + } + + private async onRefreshMessage(raw: string): Promise { + let payload: RefreshPayload = {}; + if (raw.trim()) { + try { + payload = JSON.parse(raw) as RefreshPayload; + } catch { + payload = {}; + } + } + if (payload.evictRelayIds?.length) { + for (const relayId of payload.evictRelayIds) { + await this.evictRelayRooms(relayId); + } + } + await this.reconcile(); } async reconcile(): Promise { + this.reconcilePending = true; + if (this.reconciling) { + return; + } + this.reconciling = true; + try { + while (this.reconcilePending) { + this.reconcilePending = false; + await this.runReconcile(); + } + } finally { + this.reconciling = false; + } + } + + private async runReconcile(): Promise { + const previousIds = new Set(this.relaysById.keys()); const relays = await EventRelay.getInstance().findMany({ active: true }); - const next = groupRelaysByChannel(relays); + const nextByChannel = groupRelaysByChannel(relays); const { toSubscribe, toUnsubscribe } = planChannelSubscriptions( this.subscribedChannels, - next.keys(), + nextByChannel.keys(), ); + this.relaysById.clear(); this.relaysByChannel.clear(); - for (const [channel, list] of next) { - this.relaysByChannel.set(channel, list); + for (const relay of relays) { + this.relaysById.set(relay._id, relay); + } + for (const [channel, channelRelays] of nextByChannel) { + this.relaysByChannel.set( + channel, + channelRelays.map(relay => compileRelay(relay)), + ); + } + + for (const id of previousIds) { + if (!this.relaysById.has(id)) { + await this.evictRelayRooms(id); + } } for (const channel of toUnsubscribe) { @@ -75,21 +175,27 @@ export class EventRelayManager { } for (const channel of toSubscribe) { - this.grpcSdk.bus?.subscribe( - channel, - message => this.onBusMessage(channel, message), - `${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`, - ); - this.subscribedChannels.add(channel); + try { + await this.grpcSdk.bus?.subscribeAck( + channel, + message => this.onBusMessage(channel, message), + `${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`, + ); + this.subscribedChannels.add(channel); + } catch (err) { + ConduitGrpcSdk.Logger.error( + `Event relay failed to subscribe to bus channel ${channel}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } } - } - getActiveRelay(id: string): EventRelay | undefined { - for (const relays of this.relaysByChannel.values()) { - const match = relays.find(relay => relay._id === id); - if (match) return match; - } - return undefined; + ConduitGrpcSdk.Metrics?.set('event_relays_active_total', this.relaysById.size); + ConduitGrpcSdk.Metrics?.set( + 'event_relays_subscribed_channels_total', + this.subscribedChannels.size, + ); } private onBusMessage(channel: string, message: string): void { @@ -103,6 +209,13 @@ export class EventRelayManager { payload = parseBusPayload(message); } catch (err) { ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); + if ( + err instanceof Error && + err.message.includes('exceeds') && + err.message.includes('bytes') + ) { + ConduitGrpcSdk.Metrics?.increment('event_relays_inbound_dropped_total'); + } ConduitGrpcSdk.Logger.error( `Event relay failed to parse payload for ${channel}: ${ err instanceof Error ? err.message : String(err) @@ -111,27 +224,74 @@ export class EventRelayManager { return; } - const { emissions, failures } = buildRelayEmissions(relays, payload); - for (const failure of failures) { + for (const relay of relays) { + void this.emitCompiledRelay(relay, payload, channel); + } + } + + private async emitCompiledRelay( + relay: CompiledRelay, + payload: unknown, + channel: string, + ): Promise { + let room: string; + let data: unknown; + let resourceId: string; + try { + ({ room, data, resourceId } = relay.buildEmission(payload)); + } catch (err) { ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); ConduitGrpcSdk.Logger.warn( - `Event relay ${failure.relayId} skipped on ${failure.busEvent}: ${failure.reason}`, + `Event relay ${relay.relayId} skipped on ${relay.busEvent}: ${ + err instanceof Error ? err.message : String(err) + }`, ); + return; } - for (const emission of emissions) { - this.push(emission.socketEvent, emission.data, [emission.room]).then( - () => { - ConduitGrpcSdk.Metrics?.increment('event_relays_emitted_total'); - }, - err => { - ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); - ConduitGrpcSdk.Logger.error( - `Event relay ${emission.relayId} emit failed on ${channel}: ${ - err instanceof Error ? err.message : String(err) - }`, - ); - }, + const userIds = await this.sockets.getLocalRoomUserIds(room); + if (userIds.length === 0) { + ConduitGrpcSdk.Metrics?.increment('event_relays_empty_room_total'); + return; + } + + const decisions = await checkRebacBatch( + this.rebacCache, + this.grpcSdk, + userIds, + relay.permission, + relay.resourceType, + resourceId, + ); + + const allowedUsers: string[] = []; + for (const userId of userIds) { + const decision = decisions.get(userId) ?? 'unavailable'; + if (decision === 'allow') { + allowedUsers.push(userId); + } else if (decision === 'deny') { + await this.push('leave-room', undefined, [room], [userId]); + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + } + } + + if (allowedUsers.length === 0) { + return; + } + + try { + const emitted = await this.push(relay.socketEvent, data, [room], allowedUsers); + if (emitted) { + ConduitGrpcSdk.Metrics?.increment('event_relays_emitted_total'); + } else { + ConduitGrpcSdk.Metrics?.increment('event_relays_empty_room_total'); + } + } catch (err) { + ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); + ConduitGrpcSdk.Logger.error( + `Event relay ${relay.relayId} emit failed on ${channel}: ${ + err instanceof Error ? err.message : String(err) + }`, ); } } diff --git a/modules/router/src/event-relays/EventRelaySockets.ts b/modules/router/src/event-relays/EventRelaySockets.ts index 32e5fec2a..d9fe6f88c 100644 --- a/modules/router/src/event-relays/EventRelaySockets.ts +++ b/modules/router/src/event-relays/EventRelaySockets.ts @@ -1,11 +1,27 @@ import { status } from '@grpc/grpc-js'; import { ConduitGrpcSdk, GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; import { ConduitSocket, ConduitSocketEvent } from '@conduitplatform/hermes'; -import { EVENTS_NAMESPACE } from './constants.js'; +import { + EVENTS_NAMESPACE, + MAX_SUBSCRIBE_PER_MINUTE, +} from './constants.js'; import { EventRelayManager } from './EventRelayManager.js'; import { eventRelayRoom } from './rooms.js'; import { validateResourceId } from './validation.js'; import { authorizeRelaySubscription, toSubscriptionError } from './authorize.js'; +import { reauthorizeRecoveredSubscriptions } from './recovery.js'; +import { + releaseSocketSubscriptions, + removeSubscription, + trackSubscription, + _clearEventRelaySubscriptionStateForTests, +} from './subscriptions.js'; +import { + persistRelaySubscriptionOnContext, + removeRelaySubscriptionFromContext, +} from './relaySocketData.js'; + +const subscribeTimestamps = new Map(); export function createEventsSocket( grpcSdk: ConduitGrpcSdk, @@ -20,16 +36,40 @@ export function createEventsSocket( events.set('disconnect', { name: 'disconnect', - handler: async () => ({ event: 'leave-room', rooms: [] }), + handler: async request => { + const userId = request.context?.user?._id as string | undefined; + releaseSocketSubscriptions(request.socketId, userId); + subscribeTimestamps.delete(request.socketId); + return { event: 'leave-room', rooms: [] }; + }, }); events.set('subscribe', { name: 'subscribe', params: [TYPE.String, TYPE.String], handler: async request => { + assertSubscribeRateLimit(request.socketId); const userId = request.context?.user?._id as string | undefined; + if (!userId) { + throw new GrpcError(status.UNAUTHENTICATED, 'Authentication required'); + } const [relayId, resourceId] = request.params ?? []; + const validatedResourceId = validateResourceId(resourceId); const room = await authorizeOrThrow(grpcSdk, manager, userId, relayId, resourceId); + try { + trackSubscription( + request.socketId, + userId, + String(relayId), + validatedResourceId, + ); + persistRelaySubscriptionOnContext(request.context, String(relayId), validatedResourceId); + } catch (err) { + throw new GrpcError( + status.RESOURCE_EXHAUSTED, + err instanceof Error ? err.message : 'Subscription limit exceeded', + ); + } return { event: 'join-room', rooms: [room] }; }, }); @@ -38,12 +78,15 @@ export function createEventsSocket( name: 'unsubscribe', params: [TYPE.String, TYPE.String], handler: async request => { + const userId = request.context?.user?._id as string | undefined; const [relayId, resourceId] = request.params ?? []; if (typeof relayId !== 'string' || relayId.trim() === '') { throw new GrpcError(status.INVALID_ARGUMENT, 'Relay ID is required'); } try { const validatedResourceId = validateResourceId(resourceId); + removeSubscription(request.socketId, userId, relayId, validatedResourceId); + removeRelaySubscriptionFromContext(request.context, relayId, validatedResourceId); return { event: 'leave-room', rooms: [eventRelayRoom(relayId, validatedResourceId)], @@ -60,11 +103,26 @@ export function createEventsSocket( name: 'eventRelays', description: 'Declarative bus-to-socket event relays', middlewares: ['authMiddleware'], + onRecovered: async request => + reauthorizeRecoveredSubscriptions(grpcSdk, manager, request), }, events, ); } +function assertSubscribeRateLimit(socketId: string): void { + const now = Date.now(); + const windowStart = now - 60_000; + const timestamps = (subscribeTimestamps.get(socketId) ?? []).filter( + t => t >= windowStart, + ); + if (timestamps.length >= MAX_SUBSCRIBE_PER_MINUTE) { + throw new GrpcError(status.RESOURCE_EXHAUSTED, 'Subscribe rate limit exceeded'); + } + timestamps.push(now); + subscribeTimestamps.set(socketId, timestamps); +} + async function authorizeOrThrow( grpcSdk: ConduitGrpcSdk, manager: EventRelayManager, @@ -94,3 +152,9 @@ function toGrpcError(err: unknown): GrpcError { } export { authorizeRelaySubscription } from './authorize.js'; + +/** @internal test helper */ +export function _clearEventRelaySocketStateForTests(): void { + _clearEventRelaySubscriptionStateForTests(); + subscribeTimestamps.clear(); +} diff --git a/modules/router/src/event-relays/authorize.ts b/modules/router/src/event-relays/authorize.ts index 852cf5595..8de7cc005 100644 --- a/modules/router/src/event-relays/authorize.ts +++ b/modules/router/src/event-relays/authorize.ts @@ -62,7 +62,6 @@ export async function authorizeRelaySubscription( } if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { - onDenied?.(); throw new RelaySubscriptionError(status.UNAVAILABLE, 'Authorization is unavailable'); } @@ -75,7 +74,6 @@ export async function authorizeRelaySubscription( }); allowed = decision.allow; } catch { - onDenied?.(); throw new RelaySubscriptionError(status.UNAVAILABLE, 'Authorization check failed'); } diff --git a/modules/router/src/event-relays/compile.ts b/modules/router/src/event-relays/compile.ts new file mode 100644 index 000000000..bfe636592 --- /dev/null +++ b/modules/router/src/event-relays/compile.ts @@ -0,0 +1,50 @@ +import { requireOwnPath } from './path.js'; +import { eventRelayRoom } from './rooms.js'; +import { renderMessageTemplate } from './template.js'; +import { validateResourceId } from './validation.js'; + +export type RelayCompileInput = { + _id: string; + busEvent: string; + socketEvent: string; + resourceIdPath: string; + messageTemplate: unknown; + permission: string; + resourceType: string; +}; + +export type CompiledRelay = { + relayId: string; + busEvent: string; + socketEvent: string; + permission: string; + resourceType: string; + buildEmission: (payload: unknown) => { + room: string; + data: unknown; + resourceId: string; + }; +}; + +export function compileRelay(relay: RelayCompileInput): CompiledRelay { + const resourceIdPath = relay.resourceIdPath; + const template = relay.messageTemplate; + return { + relayId: relay._id, + busEvent: relay.busEvent, + socketEvent: relay.socketEvent, + permission: relay.permission, + resourceType: relay.resourceType, + buildEmission: (payload: unknown) => { + const resourceId = validateResourceId( + requireOwnPath(payload, resourceIdPath, 'Resource ID path'), + ); + const data = renderMessageTemplate(template, payload); + return { + room: eventRelayRoom(relay._id, resourceId), + data, + resourceId, + }; + }, + }; +} diff --git a/modules/router/src/event-relays/constants.ts b/modules/router/src/event-relays/constants.ts index e283da0dd..ce80f6194 100644 --- a/modules/router/src/event-relays/constants.ts +++ b/modules/router/src/event-relays/constants.ts @@ -15,6 +15,12 @@ export const MAX_RESOURCE_ID_LENGTH = 128; export const MAX_RESOURCE_ID_PATH_LENGTH = 128; export const MAX_PERMISSION_LENGTH = 64; +export const MAX_INBOUND_BUS_BYTES = 256 * 1024; +export const RECONCILE_INTERVAL_MS = 30_000; +export const MAX_SUBSCRIBE_PER_MINUTE = 30; +export const MAX_ROOMS_PER_SOCKET = 32; +export const RELAY_REBAC_TTL_MS = 12_000; + export const FORBIDDEN_PATH_SEGMENTS = new Set(['__proto__', 'constructor', 'prototype']); export const RESERVED_SOCKET_EVENTS = new Set([ diff --git a/modules/router/src/event-relays/follow-up.test.ts b/modules/router/src/event-relays/follow-up.test.ts new file mode 100644 index 000000000..fea6d438e --- /dev/null +++ b/modules/router/src/event-relays/follow-up.test.ts @@ -0,0 +1,186 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { + releaseSocketSubscriptions, + removeSubscriptionsForRelay, + subscriptionsForRecoveredRooms, + subscriptionsForUser, + trackSubscription, + _clearEventRelaySubscriptionStateForTests, +} from './subscriptions.js'; +import { eventRelayRoom } from './rooms.js'; +import { reauthorizeRecoveredSubscriptions } from './recovery.js'; +import { RelayLookup } from './authorize.js'; + +describe('event relay subscription tracking', () => { + it('clears user map entries when the only socket disconnects', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-a', 'user-1'); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); + + it('keeps user map entries when another socket still holds the subscription', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-1', 'relay-1', 'order-1'); + trackSubscription('socket-b', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-a', 'user-1'); + assert.deepEqual(subscriptionsForUser('user-1'), [ + { relayId: 'relay-1', resourceId: 'order-1' }, + ]); + }); + + it('clears relay subscriptions on eviction', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-1', 'relay-1', 'order-1'); + removeSubscriptionsForRelay('relay-1'); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); +}); + +describe('recovery re-authorization', () => { + const room = eventRelayRoom('relay-1', 'order-1'); + const relay = { + _id: 'relay-1', + permission: 'read', + resourceType: 'Order', + }; + + it('keeps restored rooms when Authorization is unavailable', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('old-socket', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('old-socket', 'user-1'); + + const manager = { + getActiveRelay: (id: string) => (id === 'relay-1' ? relay : undefined), + } as unknown as RelayLookup; + + const grpcSdk = { + isAvailable: () => false, + authorization: null, + }; + + const result = await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'new-socket', + context: { + user: { _id: 'user-1' }, + eventRelaySubs: [{ relayId: 'relay-1', resourceId: 'order-1' }], + }, + recoveredRooms: [room], + }); + + assert.deepEqual(result, { event: 'join-room', rooms: [] }); + assert.deepEqual(subscriptionsForUser('user-1'), [ + { relayId: 'relay-1', resourceId: 'order-1' }, + ]); + }); + + it('leaves restored rooms on permission deny', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('old-socket', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('old-socket', 'user-1'); + + const manager = { + getActiveRelay: (id: string) => (id === 'relay-1' ? relay : undefined), + } as unknown as RelayLookup; + + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async () => ({ allow: false }), + }, + }; + + const result = await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'new-socket', + context: { + user: { _id: 'user-1' }, + eventRelaySubs: [{ relayId: 'relay-1', resourceId: 'order-1' }], + }, + recoveredRooms: [room], + }); + + assert.deepEqual(result, { event: 'leave-room', rooms: [room] }); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); + + it('does not re-auth rooms outside the recovered socket session', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('old-socket', 'user-1', 'relay-1', 'order-1'); + trackSubscription('old-socket', 'user-1', 'relay-2', 'order-2'); + releaseSocketSubscriptions('old-socket', 'user-1'); + + const manager = { + getActiveRelay: (id: string) => + id === 'relay-2' + ? { _id: 'relay-2', permission: 'read', resourceType: 'Order' } + : undefined, + } as unknown as RelayLookup; + + let canCalls = 0; + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async () => { + canCalls++; + return { allow: false }; + }, + }, + }; + + const otherRoom = eventRelayRoom('relay-2', 'order-2'); + await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'new-socket', + context: { + user: { _id: 'user-1' }, + eventRelaySubs: [{ relayId: 'relay-2', resourceId: 'order-2' }], + }, + recoveredRooms: [otherRoom], + }); + + assert.equal(canCalls, 1); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); + + it('re-auths the first user after a second user subscribed to the same room', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-a', 'relay-1', 'order-1'); + trackSubscription('socket-b', 'user-b', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-b', 'user-b'); + + let checkedSubject = ''; + const manager = { + getActiveRelay: (id: string) => (id === 'relay-1' ? relay : undefined), + } as unknown as RelayLookup; + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async (request: { subject: string }) => { + checkedSubject = request.subject; + return { allow: true }; + }, + }, + }; + + await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'socket-a-new', + context: { user: { _id: 'user-a' } }, + recoveredRooms: [room], + }); + + assert.equal(checkedSubject, 'User:user-a'); + }); + + it('prefers socket.data subscriptions when the room map was pruned on disconnect', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-a', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-a', 'user-a'); + assert.deepEqual(subscriptionsForRecoveredRooms([room]), []); + assert.deepEqual( + subscriptionsForRecoveredRooms([room], [ + { relayId: 'relay-1', resourceId: 'order-1' }, + ]), + [{ relayId: 'relay-1', resourceId: 'order-1' }], + ); + }); +}); diff --git a/modules/router/src/event-relays/interventions.test.ts b/modules/router/src/event-relays/interventions.test.ts new file mode 100644 index 000000000..f9e37f644 --- /dev/null +++ b/modules/router/src/event-relays/interventions.test.ts @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { MAX_INBOUND_BUS_BYTES } from './constants.js'; +import { parseBusPayload } from './process.js'; +import { renderMessageTemplate } from './template.js'; + +describe('parseBusPayload inbound cap', () => { + it('drops payloads larger than 256KiB before JSON.parse', () => { + const oversized = JSON.stringify({ blob: 'x'.repeat(MAX_INBOUND_BUS_BYTES) }); + assert.throws(() => parseBusPayload(oversized), /exceeds/); + }); +}); + +describe('preview renderer parity', () => { + it('renders templates the same as runtime processing', () => { + const payload = { documentId: 'doc-1', nested: { value: 2 } }; + const template = { id: '{{payload.documentId}}', n: '{{payload.nested.value}}' }; + assert.deepEqual(renderMessageTemplate(template, payload), { + id: 'doc-1', + n: 2, + }); + }); +}); diff --git a/modules/router/src/event-relays/process.test.ts b/modules/router/src/event-relays/process.test.ts index cf3c1c10b..c811bcec5 100644 --- a/modules/router/src/event-relays/process.test.ts +++ b/modules/router/src/event-relays/process.test.ts @@ -69,6 +69,7 @@ describe('createEventRelayPusher', () => { const calls: unknown[] = []; const push = createEventRelayPusher(async data => { calls.push(data); + return true; }); await push('order-updated', { id: '1' }, ['room-1']); assert.deepEqual(calls, [ @@ -79,6 +80,8 @@ describe('createEventRelayPusher', () => { rooms: ['room-1'], namespace: EVENTS_NAMESPACE, localOnly: true, + skipEmptyRooms: true, + boundedEmit: true, }, ]); }); diff --git a/modules/router/src/event-relays/process.ts b/modules/router/src/event-relays/process.ts index 858c9f6d0..ebe6ec1ec 100644 --- a/modules/router/src/event-relays/process.ts +++ b/modules/router/src/event-relays/process.ts @@ -3,6 +3,7 @@ import { eventRelayRoom } from './rooms.js'; import { renderMessageTemplate } from './template.js'; import { validateResourceId } from './validation.js'; import { EventRelayValidationError } from './validationError.js'; +import { MAX_INBOUND_BUS_BYTES } from './constants.js'; export type RelayProcessInput = { _id: string; @@ -31,10 +32,16 @@ export type ProcessResult = { failures: RelayFailure[]; }; -export function parseBusPayload(rawMessage: string): unknown { +export function parseBusPayload( + rawMessage: string, + maxBytes: number = MAX_INBOUND_BUS_BYTES, +): unknown { if (typeof rawMessage !== 'string' || rawMessage.trim() === '') { throw new EventRelayValidationError('Bus payload is empty'); } + if (Buffer.byteLength(rawMessage, 'utf8') > maxBytes) { + throw new EventRelayValidationError(`Bus payload exceeds ${maxBytes} bytes`); + } try { return JSON.parse(rawMessage); } catch { @@ -42,6 +49,21 @@ export function parseBusPayload(rawMessage: string): unknown { } } +export function assertJsonPayloadSize( + payload: unknown, + maxBytes: number = MAX_INBOUND_BUS_BYTES, +): void { + let serialized: string; + try { + serialized = JSON.stringify(payload); + } catch { + throw new EventRelayValidationError('Sample payload must be valid JSON'); + } + if (Buffer.byteLength(serialized, 'utf8') > maxBytes) { + throw new EventRelayValidationError(`Sample payload exceeds ${maxBytes} bytes`); + } +} + export function buildRelayEmissions( relays: RelayProcessInput[], payload: unknown, diff --git a/modules/router/src/event-relays/push.ts b/modules/router/src/event-relays/push.ts index 79d9911b2..e4bed5062 100644 --- a/modules/router/src/event-relays/push.ts +++ b/modules/router/src/event-relays/push.ts @@ -4,7 +4,8 @@ export type EventRelayPusher = ( event: string, data: unknown, rooms: string[], -) => Promise; + receivers?: string[], +) => Promise; export type SocketPushFn = (data: { event: string; @@ -13,16 +14,20 @@ export type SocketPushFn = (data: { rooms: string[]; namespace: string; localOnly?: boolean; -}) => Promise; + skipEmptyRooms?: boolean; + boundedEmit?: boolean; +}) => Promise; export function createEventRelayPusher(socketPush: SocketPushFn): EventRelayPusher { - return (event, data, rooms) => + return async (event, data, rooms, receivers = []) => socketPush({ event, data, - receivers: [], + receivers, rooms, namespace: EVENTS_NAMESPACE, localOnly: true, + skipEmptyRooms: event !== 'leave-room', + boundedEmit: event !== 'leave-room', }); } diff --git a/modules/router/src/event-relays/rebacCache.test.ts b/modules/router/src/event-relays/rebacCache.test.ts new file mode 100644 index 000000000..df20365d5 --- /dev/null +++ b/modules/router/src/event-relays/rebacCache.test.ts @@ -0,0 +1,43 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { RelayRebacCache } from './rebacCache.js'; + +describe('RelayRebacCache', () => { + it('denies after TTL when authorization revokes access', async () => { + let allow = true; + const cache = new RelayRebacCache(10); + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async () => ({ allow }), + }, + }; + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'allow', + ); + allow = false; + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'allow', + ); + await new Promise(resolve => setTimeout(resolve, 15)); + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'deny', + ); + }); + + it('returns unavailable without caching when Authorization is down', async () => { + const cache = new RelayRebacCache(10_000); + const grpcSdk = { isAvailable: () => false, authorization: null }; + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'unavailable', + ); + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'unavailable', + ); + }); +}); diff --git a/modules/router/src/event-relays/rebacCache.ts b/modules/router/src/event-relays/rebacCache.ts new file mode 100644 index 000000000..a2d458300 --- /dev/null +++ b/modules/router/src/event-relays/rebacCache.ts @@ -0,0 +1,108 @@ +type RebacSdk = { + isAvailable: (module: string) => boolean; + authorization?: { + can: (request: { + subject: string; + actions: string[]; + resource: string; + }) => Promise<{ allow: boolean }>; + } | null; +}; + +export type RebacDecision = 'allow' | 'deny' | 'unavailable'; + +const DEFAULT_TTL_MS = 12_000; +const DEFAULT_MAX_ENTRIES = 10_000; + +type CacheEntry = { + decision: 'allow' | 'deny'; + expiresAt: number; +}; + +export class RelayRebacCache { + private readonly entries = new Map(); + constructor( + private readonly ttlMs: number = DEFAULT_TTL_MS, + private readonly maxEntries: number = DEFAULT_MAX_ENTRIES, + ) {} + + async check( + grpcSdk: RebacSdk, + userId: string, + permission: string, + resourceType: string, + resourceId: string, + ): Promise { + const resource = `${resourceType}:${resourceId}`; + const key = `${userId}:${permission}:${resource}`; + const now = Date.now(); + this.sweepExpired(now); + const cached = this.entries.get(key); + if (cached && cached.expiresAt > now) { + return cached.decision; + } + if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { + return 'unavailable'; + } + try { + const decision = await grpcSdk.authorization.can({ + subject: `User:${userId}`, + actions: [permission], + resource, + }); + this.set(key, decision.allow ? 'allow' : 'deny', now); + return decision.allow ? 'allow' : 'deny'; + } catch { + return 'unavailable'; + } + } + + clear(): void { + this.entries.clear(); + } + + private set(key: string, decision: 'allow' | 'deny', now: number): void { + if (this.entries.size >= this.maxEntries && !this.entries.has(key)) { + const firstKey = this.entries.keys().next().value; + if (firstKey) { + this.entries.delete(firstKey); + } + } + this.entries.set(key, { decision, expiresAt: now + this.ttlMs }); + } + + private sweepExpired(now: number): void { + for (const [key, entry] of this.entries) { + if (entry.expiresAt <= now) { + this.entries.delete(key); + } + } + } +} + +export async function checkRebacBatch( + cache: RelayRebacCache, + grpcSdk: RebacSdk, + userIds: string[], + permission: string, + resourceType: string, + resourceId: string, + concurrency = 8, +): Promise> { + const results = new Map(); + let index = 0; + async function worker(): Promise { + while (index < userIds.length) { + const userId = userIds[index++]; + results.set( + userId, + await cache.check(grpcSdk, userId, permission, resourceType, resourceId), + ); + } + } + const workers = Array.from({ length: Math.min(concurrency, userIds.length) }, () => + worker(), + ); + await Promise.all(workers); + return results; +} diff --git a/modules/router/src/event-relays/recovery.ts b/modules/router/src/event-relays/recovery.ts new file mode 100644 index 000000000..ceb6c4217 --- /dev/null +++ b/modules/router/src/event-relays/recovery.ts @@ -0,0 +1,85 @@ +import { status } from '@grpc/grpc-js'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { + authorizeRelaySubscription, + RelaySubscriptionError, + RelayLookup, + toSubscriptionError, +} from './authorize.js'; +import { eventRelayRoom } from './rooms.js'; +import { + removeSubscription, + subscriptionsForRecoveredRooms, + trackSubscription, +} from './subscriptions.js'; +import { relaySubscriptionsFromContext, persistRelaySubscriptionOnContext, removeRelaySubscriptionFromContext } from './relaySocketData.js'; + +type RelayAuthorizationSdk = Parameters[0]; + +export async function reauthorizeRecoveredSubscriptions( + grpcSdk: RelayAuthorizationSdk, + manager: RelayLookup, + request: { + socketId: string; + context?: { user?: { _id?: string } } & Record; + recoveredRooms?: string[]; + }, +): Promise< + { event: 'join-room'; rooms: string[] } | { event: 'leave-room'; rooms: string[] } +> { + const userId = request.context?.user?._id as string | undefined; + if (!userId) { + return { event: 'leave-room', rooms: [] }; + } + + const recoveredRooms = request.recoveredRooms ?? []; + const subs = subscriptionsForRecoveredRooms( + recoveredRooms, + relaySubscriptionsFromContext(request.context), + ); + const leaveRooms: string[] = []; + + for (const sub of subs) { + const room = eventRelayRoom(sub.relayId, sub.resourceId); + try { + await authorizeRelaySubscription( + grpcSdk, + manager, + userId, + sub.relayId, + sub.resourceId, + () => { + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + }, + ); + trackSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + persistRelaySubscriptionOnContext(request.context, sub.relayId, sub.resourceId); + } catch (err) { + const mapped = + err instanceof RelaySubscriptionError ? err : toSubscriptionError(err); + if (mapped.code === status.UNAVAILABLE) { + trackSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + persistRelaySubscriptionOnContext(request.context, sub.relayId, sub.resourceId); + continue; + } + if ( + mapped.code === status.PERMISSION_DENIED || + mapped.code === status.NOT_FOUND + ) { + removeSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + removeRelaySubscriptionFromContext(request.context, sub.relayId, sub.resourceId); + leaveRooms.push(room); + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + continue; + } + removeSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + removeRelaySubscriptionFromContext(request.context, sub.relayId, sub.resourceId); + leaveRooms.push(room); + } + } + + if (leaveRooms.length > 0) { + return { event: 'leave-room', rooms: leaveRooms }; + } + return { event: 'join-room', rooms: [] }; +} diff --git a/modules/router/src/event-relays/relaySocketData.ts b/modules/router/src/event-relays/relaySocketData.ts new file mode 100644 index 000000000..63d0eca9c --- /dev/null +++ b/modules/router/src/event-relays/relaySocketData.ts @@ -0,0 +1,65 @@ +import { Indexable } from '@conduitplatform/grpc-sdk'; +import { RelaySubscription } from './subscriptions.js'; + +const CONTEXT_KEY = 'eventRelaySubs'; + +export function relaySubscriptionsFromContext( + context: Indexable | undefined, +): RelaySubscription[] { + if (!context) { + return []; + } + const raw = context[CONTEXT_KEY]; + if (!Array.isArray(raw)) { + return []; + } + const subs: RelaySubscription[] = []; + for (const item of raw) { + if ( + item && + typeof item === 'object' && + typeof (item as RelaySubscription).relayId === 'string' && + typeof (item as RelaySubscription).resourceId === 'string' + ) { + subs.push({ + relayId: (item as RelaySubscription).relayId, + resourceId: (item as RelaySubscription).resourceId, + }); + } + } + return subs; +} + +export function persistRelaySubscriptionOnContext( + context: Indexable | undefined, + relayId: string, + resourceId: string, +): void { + if (!context) { + return; + } + const subs = relaySubscriptionsFromContext(context); + const withoutDup = subs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + withoutDup.push({ relayId, resourceId }); + context[CONTEXT_KEY] = withoutDup; +} + +export function removeRelaySubscriptionFromContext( + context: Indexable | undefined, + relayId: string, + resourceId: string, +): void { + if (!context) { + return; + } + const next = relaySubscriptionsFromContext(context).filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + if (next.length === 0) { + delete context[CONTEXT_KEY]; + } else { + context[CONTEXT_KEY] = next; + } +} diff --git a/modules/router/src/event-relays/rooms.ts b/modules/router/src/event-relays/rooms.ts index fd5dd54ec..0bf26bf74 100644 --- a/modules/router/src/event-relays/rooms.ts +++ b/modules/router/src/event-relays/rooms.ts @@ -4,3 +4,7 @@ export function eventRelayRoom(relayId: string, resourceId: string): string { const digest = createHash('sha256').update(resourceId).digest('hex'); return `er:${relayId}:${digest}`; } + +export function eventRelayRoomPrefix(relayId: string): string { + return `er:${relayId}:`; +} diff --git a/modules/router/src/event-relays/subscriptions.ts b/modules/router/src/event-relays/subscriptions.ts new file mode 100644 index 000000000..a7a083f56 --- /dev/null +++ b/modules/router/src/event-relays/subscriptions.ts @@ -0,0 +1,224 @@ +import { eventRelayRoom } from './rooms.js'; +import { MAX_ROOMS_PER_SOCKET } from './constants.js'; + +export type RelaySubscription = { + relayId: string; + resourceId: string; +}; + +const RECOVERY_ROOM_MAP_TTL_MS = 120_000; + +const subscriptionsBySocket = new Map(); +const subscriptionsByUser = new Map(); +const subscriptionsByRoom = new Map(); + +function pruneExpiredRoomEntries(now = Date.now()): void { + for (const [room, entry] of subscriptionsByRoom) { + if (entry.expiresAt <= now) { + subscriptionsByRoom.delete(room); + } + } +} + +function touchRoomEntry(relayId: string, resourceId: string): void { + const room = eventRelayRoom(relayId, resourceId); + subscriptionsByRoom.set(room, { + relayId, + resourceId, + expiresAt: Date.now() + RECOVERY_ROOM_MAP_TTL_MS, + }); +} + +function pruneRoomEntryIfUnused(relayId: string, resourceId: string): void { + if (!isSubscriptionTrackedOnAnySocket(relayId, resourceId)) { + subscriptionsByRoom.delete(eventRelayRoom(relayId, resourceId)); + } +} + +export function trackSubscription( + socketId: string, + userId: string, + relayId: string, + resourceId: string, +): void { + const entry = { relayId, resourceId }; + const socketSubs = subscriptionsBySocket.get(socketId) ?? []; + const withoutDup = socketSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + withoutDup.push(entry); + if (withoutDup.length > MAX_ROOMS_PER_SOCKET) { + throw new Error(`Cannot subscribe to more than ${MAX_ROOMS_PER_SOCKET} relay rooms`); + } + subscriptionsBySocket.set(socketId, withoutDup); + + const userSubs = subscriptionsByUser.get(userId) ?? []; + const userWithoutDup = userSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + userWithoutDup.push(entry); + subscriptionsByUser.set(userId, userWithoutDup); + touchRoomEntry(relayId, resourceId); +} + +export function removeSubscription( + socketId: string, + userId: string | undefined, + relayId: string, + resourceId: string, +): void { + const socketSubs = subscriptionsBySocket.get(socketId); + if (socketSubs) { + const next = socketSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + if (next.length === 0) { + subscriptionsBySocket.delete(socketId); + } else { + subscriptionsBySocket.set(socketId, next); + } + } + if (userId && !isSubscriptionTrackedOnOtherSocket(relayId, resourceId, socketId)) { + const userSubs = subscriptionsByUser.get(userId); + if (userSubs) { + const next = userSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + if (next.length === 0) { + subscriptionsByUser.delete(userId); + } else { + subscriptionsByUser.set(userId, next); + } + } + pruneRoomEntryIfUnused(relayId, resourceId); + } +} + +function isSubscriptionTrackedOnOtherSocket( + relayId: string, + resourceId: string, + exceptSocketId: string, +): boolean { + for (const [socketId, subs] of subscriptionsBySocket) { + if (socketId === exceptSocketId) { + continue; + } + if (subs.some(sub => sub.relayId === relayId && sub.resourceId === resourceId)) { + return true; + } + } + return false; +} + +function isSubscriptionTrackedOnAnySocket(relayId: string, resourceId: string): boolean { + for (const subs of subscriptionsBySocket.values()) { + if (subs.some(sub => sub.relayId === relayId && sub.resourceId === resourceId)) { + return true; + } + } + return false; +} + +export function releaseSocketSubscriptions(socketId: string, userId?: string): void { + const subs = subscriptionsBySocket.get(socketId) ?? []; + subscriptionsBySocket.delete(socketId); + if (!userId) { + return; + } + for (const sub of subs) { + if (!isSubscriptionTrackedOnOtherSocket(sub.relayId, sub.resourceId, socketId)) { + const userSubs = subscriptionsByUser.get(userId); + if (!userSubs) { + continue; + } + const next = userSubs.filter( + entry => + !(entry.relayId === sub.relayId && entry.resourceId === sub.resourceId), + ); + if (next.length === 0) { + subscriptionsByUser.delete(userId); + } else { + subscriptionsByUser.set(userId, next); + } + } + pruneRoomEntryIfUnused(sub.relayId, sub.resourceId); + } +} + +export function subscriptionsForRecoveredRooms( + recoveredRooms: string[], + contextSubs: RelaySubscription[] = [], +): RelaySubscription[] { + pruneExpiredRoomEntries(); + const seen = new Set(); + const result: RelaySubscription[] = []; + const recoveredSet = new Set(recoveredRooms.filter(room => room.startsWith('er:'))); + + for (const sub of contextSubs) { + const room = eventRelayRoom(sub.relayId, sub.resourceId); + if (recoveredSet.has(room) && !seen.has(room)) { + seen.add(room); + result.push(sub); + } + } + + for (const room of recoveredSet) { + if (seen.has(room)) { + continue; + } + const entry = subscriptionsByRoom.get(room); + if (entry) { + seen.add(room); + result.push({ relayId: entry.relayId, resourceId: entry.resourceId }); + } + } + + return result; +} + +export function subscriptionsForUser(userId: string): RelaySubscription[] { + return [...(subscriptionsByUser.get(userId) ?? [])]; +} + +export function removeSubscriptionsForRelay(relayId: string): void { + for (const [socketId, subs] of subscriptionsBySocket) { + const next = subs.filter(sub => sub.relayId !== relayId); + if (next.length === 0) { + subscriptionsBySocket.delete(socketId); + } else { + subscriptionsBySocket.set(socketId, next); + } + } + for (const [userId, subs] of subscriptionsByUser) { + const next = subs.filter(sub => sub.relayId !== relayId); + if (next.length === 0) { + subscriptionsByUser.delete(userId); + } else { + subscriptionsByUser.set(userId, next); + } + } + for (const [room, entry] of subscriptionsByRoom) { + if (entry.relayId === relayId) { + subscriptionsByRoom.delete(room); + } + } +} + +export function leaveRoomsForRelay(relayId: string): string[] { + const rooms = new Set(); + for (const subs of subscriptionsByUser.values()) { + for (const sub of subs) { + if (sub.relayId === relayId) { + rooms.add(eventRelayRoom(relayId, sub.resourceId)); + } + } + } + return [...rooms]; +} + +/** @internal test helper */ +export function _clearEventRelaySubscriptionStateForTests(): void { + subscriptionsBySocket.clear(); + subscriptionsByUser.clear(); + subscriptionsByRoom.clear(); +} diff --git a/modules/router/src/index.ts b/modules/router/src/index.ts index 20a0c779e..fd1d069ed 100644 --- a/modules/router/src/index.ts +++ b/modules/router/src/index.ts @@ -1,7 +1,25 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import ConduitDefaultRouter from './Router.js'; const peerManifestRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const router = new ConduitDefaultRouter(peerManifestRoot); + +function registerShutdownSignals(): void { + const shutdown = (signal: NodeJS.Signals) => { + void router + .shutdown() + .catch(err => { + ConduitGrpcSdk.Logger.error(err as Error); + }) + .finally(() => { + process.exit(signal === 'SIGINT' ? 130 : 0); + }); + }; + process.once('SIGTERM', () => shutdown('SIGTERM')); + process.once('SIGINT', () => shutdown('SIGINT')); +} + +registerShutdownSignals(); router.start(); diff --git a/modules/router/src/metrics/index.ts b/modules/router/src/metrics/index.ts index d3461c107..2ed45b20f 100644 --- a/modules/router/src/metrics/index.ts +++ b/modules/router/src/metrics/index.ts @@ -38,4 +38,39 @@ export default { help: 'Tracks denied or unavailable event-relay socket subscriptions', }, }, + eventRelaysActive: { + type: MetricType.Gauge, + config: { + name: 'event_relays_active_total', + help: 'Active event relays on this router replica', + }, + }, + eventRelaysSubscribedChannels: { + type: MetricType.Gauge, + config: { + name: 'event_relays_subscribed_channels_total', + help: 'Bus channels this router replica subscribes to for event relays', + }, + }, + eventRelaysEmptyRoom: { + type: MetricType.Counter, + config: { + name: 'event_relays_empty_room_total', + help: 'Skipped emits because no local sockets were in the relay room', + }, + }, + eventRelaysInboundDropped: { + type: MetricType.Counter, + config: { + name: 'event_relays_inbound_dropped_total', + help: 'Inbound bus payloads dropped for exceeding the size cap', + }, + }, + eventRelaysEmitDropped: { + type: MetricType.Counter, + config: { + name: 'event_relays_emit_dropped_total', + help: 'Socket emits dropped or disconnected due to backpressure', + }, + }, }; diff --git a/modules/router/tsconfig.test.json b/modules/router/tsconfig.test.json index 601e5ccee..229f7f155 100644 --- a/modules/router/tsconfig.test.json +++ b/modules/router/tsconfig.test.json @@ -19,6 +19,9 @@ "src/event-relays/search.ts", "src/event-relays/channels.ts", "src/event-relays/authorize.ts", + "src/event-relays/compile.ts", + "src/event-relays/rebacCache.ts", + "src/event-relays/subscriptions.ts", "src/event-relays/*.test.ts" ] } From 9e6356efccae71fd154e2ea411730a4c892bccf5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 11:48:02 +0000 Subject: [PATCH 2/5] fix(router): address CodeFactor findings on event relay validation Split validateEventRelayInput into per-field parsers to reduce complexity. Silence unused-parameter lint in EventBus test FakeRedis stub. --- .../grpc-sdk/src/utilities/EventBus.test.ts | 25 ++-- modules/router/src/event-relays/validation.ts | 127 +++++++++++------- 2 files changed, 94 insertions(+), 58 deletions(-) diff --git a/libraries/grpc-sdk/src/utilities/EventBus.test.ts b/libraries/grpc-sdk/src/utilities/EventBus.test.ts index c1999901d..4d069c244 100644 --- a/libraries/grpc-sdk/src/utilities/EventBus.test.ts +++ b/libraries/grpc-sdk/src/utilities/EventBus.test.ts @@ -24,11 +24,15 @@ class FakeRedis { cb?.(null); } - unsubscribe(_channel: string, cb?: () => void) { + unsubscribe(channel: string, cb?: () => void) { + void channel; cb?.(); } - publish(_channel: string, _message: string) {} + publish(channel: string, message: string) { + void channel; + void message; + } quit() {} @@ -102,19 +106,18 @@ describe('EventBus', () => { const { bus, sub } = createBus(); sub.failNext.add('chan'); await assert.rejects( - () => - bus.subscribeAck( - 'chan', - () => {}, - 'relay-a', - ), + () => bus.subscribeAck('chan', () => {}, 'relay-a'), /subscribe failed/, ); sub.failNext.delete('chan'); let count = 0; - await bus.subscribeAck('chan', () => { - count += 1; - }, 'relay-a'); + await bus.subscribeAck( + 'chan', + () => { + count += 1; + }, + 'relay-a', + ); sub.emitMessage('chan', 'x'); assert.equal(count, 1); }); diff --git a/modules/router/src/event-relays/validation.ts b/modules/router/src/event-relays/validation.ts index d6c6c5eb6..c59976335 100644 --- a/modules/router/src/event-relays/validation.ts +++ b/modules/router/src/event-relays/validation.ts @@ -33,24 +33,75 @@ const PERMISSION_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,63}$/; const RESOURCE_ID_PATTERN = /^[^\s:]{1,128}$/; export function validateEventRelayInput(input: EventRelayInput): EventRelayInput { - const name = requireTrimmed(input.name, 'Name'); + const name = parseName(input.name); + const notes = parseNotes(input.notes); + const busEvent = parseBusEvent(input.busEvent); + const socketEvent = parseSocketEvent(input.socketEvent); + const resourceType = parseResourceType(input.resourceType); + const resourceIdPath = parseResourceIdPath(input.resourceIdPath); + const permission = parsePermission(input.permission); + const messageTemplate = parseMessageTemplate(input.messageTemplate); + + return { + name, + notes, + active: input.active !== false, + busEvent, + socketEvent, + resourceType, + resourceIdPath, + permission, + messageTemplate, + }; +} + +export function validateResourceId(resourceId: unknown): string { + if (typeof resourceId !== 'string' && typeof resourceId !== 'number') { + throw new EventRelayValidationError('Resource ID must be a string'); + } + const value = String(resourceId).trim(); + if ( + !value || + value.length > MAX_RESOURCE_ID_LENGTH || + !RESOURCE_ID_PATTERN.test(value) + ) { + throw new EventRelayValidationError('Resource ID is invalid'); + } + return value; +} + +function requireTrimmed(value: unknown, label: string): string { + if (typeof value !== 'string' || value.trim() === '') { + throw new EventRelayValidationError(`${label} is required`); + } + return value.trim(); +} + +function parseName(raw: unknown): string { + const name = requireTrimmed(raw, 'Name'); if (name.length > MAX_NAME_LENGTH || !NAME_PATTERN.test(name)) { throw new EventRelayValidationError( 'Name must be 1-64 characters and start with a letter or number', ); } + return name; +} - const notes = - input.notes === undefined || input.notes === '' - ? undefined - : requireTrimmed(input.notes, 'Notes'); - if (notes && notes.length > MAX_DESCRIPTION_LENGTH) { +function parseNotes(raw: unknown): string | undefined { + if (raw === undefined || raw === '') { + return undefined; + } + const notes = requireTrimmed(raw, 'Notes'); + if (notes.length > MAX_DESCRIPTION_LENGTH) { throw new EventRelayValidationError( `Notes must be at most ${MAX_DESCRIPTION_LENGTH} characters`, ); } + return notes; +} - const busEvent = requireTrimmed(input.busEvent, 'Bus event'); +function parseBusEvent(raw: unknown): string { + const busEvent = requireTrimmed(raw, 'Bus event'); if ( busEvent.length > MAX_BUS_EVENT_LENGTH || busEvent.includes('*') || @@ -60,8 +111,11 @@ export function validateEventRelayInput(input: EventRelayInput): EventRelayInput 'Bus event must be an exact channel name with no wildcards', ); } + return busEvent; +} - const socketEvent = requireTrimmed(input.socketEvent, 'Socket event'); +function parseSocketEvent(raw: unknown): string { + const socketEvent = requireTrimmed(raw, 'Socket event'); if ( socketEvent.length > MAX_SOCKET_EVENT_LENGTH || !SOCKET_EVENT_PATTERN.test(socketEvent) || @@ -69,62 +123,41 @@ export function validateEventRelayInput(input: EventRelayInput): EventRelayInput ) { throw new EventRelayValidationError('Socket event must be a non-reserved event name'); } + return socketEvent; +} - const resourceType = requireTrimmed(input.resourceType, 'Resource type'); +function parseResourceType(raw: unknown): string { + const resourceType = requireTrimmed(raw, 'Resource type'); if ( resourceType.length > MAX_RESOURCE_TYPE_LENGTH || !RESOURCE_TYPE_PATTERN.test(resourceType) ) { throw new EventRelayValidationError('Resource type is invalid'); } + return resourceType; +} - const resourceIdPath = requireTrimmed(input.resourceIdPath, 'Resource ID path'); +function parseResourceIdPath(raw: unknown): string { + const resourceIdPath = requireTrimmed(raw, 'Resource ID path'); if (resourceIdPath.length > MAX_RESOURCE_ID_PATH_LENGTH) { throw new EventRelayValidationError('Resource ID path is too long'); } parseDotPath(resourceIdPath, 'Resource ID path'); + return resourceIdPath; +} - const permission = requireTrimmed(input.permission, 'Permission'); +function parsePermission(raw: unknown): string { + const permission = requireTrimmed(raw, 'Permission'); if (permission.length > MAX_PERMISSION_LENGTH || !PERMISSION_PATTERN.test(permission)) { throw new EventRelayValidationError('Permission is invalid'); } - - if (input.messageTemplate === undefined) { - throw new EventRelayValidationError('Message template is required'); - } - assertTemplateSize(input.messageTemplate); - - return { - name, - notes, - active: input.active !== false, - busEvent, - socketEvent, - resourceType, - resourceIdPath, - permission, - messageTemplate: input.messageTemplate, - }; -} - -export function validateResourceId(resourceId: unknown): string { - if (typeof resourceId !== 'string' && typeof resourceId !== 'number') { - throw new EventRelayValidationError('Resource ID must be a string'); - } - const value = String(resourceId).trim(); - if ( - !value || - value.length > MAX_RESOURCE_ID_LENGTH || - !RESOURCE_ID_PATTERN.test(value) - ) { - throw new EventRelayValidationError('Resource ID is invalid'); - } - return value; + return permission; } -function requireTrimmed(value: unknown, label: string): string { - if (typeof value !== 'string' || value.trim() === '') { - throw new EventRelayValidationError(`${label} is required`); +function parseMessageTemplate(raw: unknown): unknown { + if (raw === undefined) { + throw new EventRelayValidationError('Message template is required'); } - return value.trim(); + assertTemplateSize(raw); + return raw; } From a94c885f35f8d9ba3225d091023266c13fe5c873 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 14:31:50 +0000 Subject: [PATCH 3/5] feat(database): restack Mongo live updates onto current Event Relays Replay the Mongo delta onto feat/router-event-relays (9e6356ef). Keep #1604 Hermes (engine.use, er: recovery, writeBuffer) and graft only the port-keyed adapter plus register-before-initSockets. Do not take old #1601 Socket.ts wholesale. Co-authored-by: Konstantinos Kopanidis --- docker/.env | 2 +- docker/docker-compose.standalone.yml | 35 +- docker/docker-compose.yml | 51 ++- docker/mongo/init-replica.sh | 34 ++ docker/mongo/keyfile | 1 + libraries/grpc-sdk/src/interfaces/Model.ts | 7 +- libraries/grpc-sdk/src/modules/admin/index.ts | 7 +- libraries/hermes/src/Socket/Socket.ts | 17 +- .../src/Socket/isSocketHandshake.test.ts | 1 + libraries/hermes/src/index.ts | 21 + libraries/hermes/src/interfaces/Socket.ts | 9 +- modules/database/README.mdx | 22 + modules/database/src/Database.ts | 11 + .../__tests__/schema.admin.put-patch.test.ts | 15 + modules/database/src/admin/index.ts | 3 + modules/database/src/admin/schema.admin.ts | 2 + modules/database/src/config/index.ts | 7 + modules/database/src/index.ts | 18 + .../database/src/interfaces/ConduitOptions.ts | 3 + modules/database/src/metrics/index.ts | 15 + .../realtime/MongoChangeStreamCoordinator.ts | 408 +++++++++++++++++ .../database/src/realtime/RealtimeService.ts | 162 +++++++ .../src/realtime/__tests__/authorize.test.ts | 69 +++ .../change-stream.integration.test.ts | 48 ++ .../realtime/__tests__/coordinator.test.ts | 422 ++++++++++++++++++ .../src/realtime/__tests__/normalize.test.ts | 42 ++ .../src/realtime/__tests__/recovery.test.ts | 82 ++++ .../src/realtime/__tests__/rooms.test.ts | 38 ++ .../src/realtime/__tests__/status.test.ts | 53 +++ .../realtime/__tests__/subscriptions.test.ts | 55 +++ .../src/realtime/__tests__/topology.test.ts | 19 + .../realtime/__tests__/watchPipeline.test.ts | 28 ++ modules/database/src/realtime/authorize.ts | 136 ++++++ modules/database/src/realtime/index.ts | 11 + modules/database/src/realtime/normalize.ts | 78 ++++ modules/database/src/realtime/rebacCache.ts | 96 ++++ modules/database/src/realtime/recovery.ts | 147 ++++++ modules/database/src/realtime/rooms.ts | 47 ++ modules/database/src/realtime/sockets.ts | 195 ++++++++ modules/database/src/realtime/status.ts | 74 +++ .../database/src/realtime/subscriptions.ts | 92 ++++ modules/database/src/realtime/topology.ts | 50 +++ modules/database/src/realtime/types.ts | 48 ++ .../database/src/realtime/watchPipeline.ts | 33 ++ modules/database/src/routes/index.ts | 4 +- modules/database/src/utils/SchemaConverter.ts | 6 + .../__tests__/validate-model-options.test.ts | 37 ++ modules/database/src/utils/utilities.ts | 5 +- packages/core/package.json | 1 + packages/core/src/admin/AdminModule.ts | 36 ++ .../src/admin/middleware/Admin.middleware.ts | 11 +- .../src/admin/middleware/Auth.middleware.ts | 5 + .../core/src/admin/realtime/namespace.test.ts | 11 + packages/core/src/admin/realtime/namespace.ts | 7 + .../src/admin/realtime/ticket-http.test.ts | 97 ++++ .../core/src/admin/realtime/ticket.test.ts | 26 ++ packages/core/src/admin/realtime/ticket.ts | 35 ++ .../src/admin/routes/RealtimeTicket.route.ts | 51 +++ packages/core/src/admin/routes/index.ts | 1 + packages/core/src/core.proto | 8 + packages/core/tsconfig.json | 2 +- packages/core/tsconfig.test.json | 16 + 62 files changed, 3048 insertions(+), 25 deletions(-) create mode 100644 docker/mongo/init-replica.sh create mode 100644 docker/mongo/keyfile create mode 100644 modules/database/src/realtime/MongoChangeStreamCoordinator.ts create mode 100644 modules/database/src/realtime/RealtimeService.ts create mode 100644 modules/database/src/realtime/__tests__/authorize.test.ts create mode 100644 modules/database/src/realtime/__tests__/change-stream.integration.test.ts create mode 100644 modules/database/src/realtime/__tests__/coordinator.test.ts create mode 100644 modules/database/src/realtime/__tests__/normalize.test.ts create mode 100644 modules/database/src/realtime/__tests__/recovery.test.ts create mode 100644 modules/database/src/realtime/__tests__/rooms.test.ts create mode 100644 modules/database/src/realtime/__tests__/status.test.ts create mode 100644 modules/database/src/realtime/__tests__/subscriptions.test.ts create mode 100644 modules/database/src/realtime/__tests__/topology.test.ts create mode 100644 modules/database/src/realtime/__tests__/watchPipeline.test.ts create mode 100644 modules/database/src/realtime/authorize.ts create mode 100644 modules/database/src/realtime/index.ts create mode 100644 modules/database/src/realtime/normalize.ts create mode 100644 modules/database/src/realtime/rebacCache.ts create mode 100644 modules/database/src/realtime/recovery.ts create mode 100644 modules/database/src/realtime/rooms.ts create mode 100644 modules/database/src/realtime/sockets.ts create mode 100644 modules/database/src/realtime/status.ts create mode 100644 modules/database/src/realtime/subscriptions.ts create mode 100644 modules/database/src/realtime/topology.ts create mode 100644 modules/database/src/realtime/types.ts create mode 100644 modules/database/src/realtime/watchPipeline.ts create mode 100644 modules/database/src/utils/__tests__/validate-model-options.test.ts create mode 100644 packages/core/src/admin/realtime/namespace.test.ts create mode 100644 packages/core/src/admin/realtime/namespace.ts create mode 100644 packages/core/src/admin/realtime/ticket-http.test.ts create mode 100644 packages/core/src/admin/realtime/ticket.test.ts create mode 100644 packages/core/src/admin/realtime/ticket.ts create mode 100644 packages/core/src/admin/routes/RealtimeTicket.route.ts create mode 100644 packages/core/tsconfig.test.json diff --git a/docker/.env b/docker/.env index 66dad89bd..b71166ea3 100644 --- a/docker/.env +++ b/docker/.env @@ -26,7 +26,7 @@ DB_TYPE="mongodb" DB_USER="conduit" DB_PASS="pass" DB_PORT="27017" -DB_CONN_URI="mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin" # profile: mongodb +DB_CONN_URI="mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin&replicaSet=rs0" # profile: mongodb #DB_CONN_URI="postgres://conduit:pass@conduit-postgres:5432/conduit" # profile: postgres # Security diff --git a/docker/docker-compose.standalone.yml b/docker/docker-compose.standalone.yml index 8cd0b0e9b..bd77d5c47 100644 --- a/docker/docker-compose.standalone.yml +++ b/docker/docker-compose.standalone.yml @@ -15,8 +15,12 @@ services: image: 'docker.io/conduitplatform/conduit-standalone:${IMAGE_TAG}' restart: unless-stopped depends_on: - - redis - - mongodb + redis: + condition: service_started + mongodb: + condition: service_started + mongo-init-replica: + condition: service_completed_successfully ports: - '${CORE_GRPC_PORT:-55152}:55152' - '${DB_GRPC_PORT:-55160}:55160' @@ -38,7 +42,7 @@ services: ADMIN_SOCKET_PORT: '${ADMIN_SOCKET_PORT:-3031}' __DEFAULT_HOST_URL: '${ADMIN_DEFAULT_HOST_URL:-http://localhost:3030}' GRPC_KEY: '${GRPC_KEY}' - DB_CONN_URI: '${DB_CONN_URI:-mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin}' + DB_CONN_URI: '${DB_CONN_URI:-mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin&replicaSet=rs0}' networks: default: aliases: @@ -74,12 +78,37 @@ services: MONGO_INITDB_DATABASE: 'conduit' MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' + # Existing volumes created before rs0 will not elect a replica set. + # Remove the mongo volume (or start with an empty data dir) if hello stays standalone. + entrypoint: + - bash + - -c + - | + cp /mongo-keyfile /tmp/keyfile + chmod 400 /tmp/keyfile + chown mongodb:mongodb /tmp/keyfile + exec docker-entrypoint.sh mongod --replSet rs0 --bind_ip_all --keyFile /tmp/keyfile networks: default: aliases: - conduit-mongo volumes: - mongo:/data/db + - ./mongo/keyfile:/mongo-keyfile:ro + + mongo-init-replica: + container_name: 'conduit-mongo-init' + image: 'docker.io/library/mongo:4.4.15' + restart: on-failure + depends_on: + - mongodb + environment: + MONGO_HOST: 'conduit-mongo' + MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' + MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' + volumes: + - ./mongo/init-replica.sh:/init-replica.sh:ro + command: ['bash', '/init-replica.sh'] # Persistent Volumes volumes: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 83b3e9b55..7567df345 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -71,10 +71,21 @@ services: image: 'docker.io/conduitplatform/database:${IMAGE_TAG}' restart: unless-stopped depends_on: - - core - - ${DB_TYPE:-mongodb} - - prometheus - - loki + core: + condition: service_started + prometheus: + condition: service_started + loki: + condition: service_started + mongodb: + condition: service_started + required: false + postgres: + condition: service_started + required: false + mongo-init-replica: + condition: service_completed_successfully + required: false ports: - '${DB_GRPC_PORT:-55160}:${DB_GRPC_PORT:-55160}' environment: @@ -85,7 +96,7 @@ services: LOKI_URL: 'http://conduit-loki:3100' GRPC_KEY: '${GRPC_KEY}' DB_TYPE: '${DB_TYPE:-mongodb}' - DB_CONN_URI: '${DB_CONN_URI:-mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin}' + DB_CONN_URI: '${DB_CONN_URI:-mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin&replicaSet=rs0}' networks: default: aliases: @@ -271,12 +282,42 @@ services: MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' profiles: ['mongodb'] + # Existing volumes created before rs0 will not elect a replica set. + # Remove the mongo volume (or start with an empty data dir) if hello stays standalone. + entrypoint: + - bash + - -c + - | + cp /mongo-keyfile /tmp/keyfile + chmod 400 /tmp/keyfile + chown mongodb:mongodb /tmp/keyfile + exec docker-entrypoint.sh mongod --replSet rs0 --bind_ip_all --keyFile /tmp/keyfile networks: default: aliases: - conduit-mongo volumes: - mongo:/data/db + - ./mongo/keyfile:/mongo-keyfile:ro + + mongo-init-replica: + container_name: 'conduit-mongo-init' + image: 'docker.io/library/mongo:4.4.15' + restart: on-failure + profiles: ['mongodb'] + depends_on: + - mongodb + environment: + MONGO_HOST: 'conduit-mongo' + MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' + MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' + volumes: + - ./mongo/init-replica.sh:/init-replica.sh:ro + command: ['bash', '/init-replica.sh'] + networks: + default: + aliases: + - conduit-mongo-init postgres: container_name: 'conduit-postgres' diff --git a/docker/mongo/init-replica.sh b/docker/mongo/init-replica.sh new file mode 100644 index 000000000..f94f5dbf4 --- /dev/null +++ b/docker/mongo/init-replica.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail +HOST="${MONGO_HOST:-conduit-mongo}" +USER="${MONGO_INITDB_ROOT_USERNAME:-conduit}" +PASS="${MONGO_INITDB_ROOT_PASSWORD:-pass}" + +mongo_eval() { + mongo --host "$HOST" -u "$USER" -p "$PASS" --authenticationDatabase admin --quiet --eval "$1" +} + +until mongo_eval 'db.adminCommand({ ping: 1 })' >/dev/null 2>&1; do + sleep 2 +done + +# rs.status() returns { ok: 0 } before initiate; it does not throw. +mongo_eval ' + var status = rs.status(); + if (status.ok === 1) { + quit(0); + } + var result = rs.initiate({ + _id: "rs0", + members: [{ _id: 0, host: "'"$HOST"':27017" }] + }); + if (result.ok !== 1) { + printjson(result); + quit(1); + } +' + +# Mongoose with replicaSet=rs0 only selects a PRIMARY (myState === 1). +until mongo_eval 'var s = rs.status(); if (s.ok === 1 && s.myState === 1) { quit(0); } quit(1);' >/dev/null 2>&1; do + sleep 1 +done diff --git a/docker/mongo/keyfile b/docker/mongo/keyfile new file mode 100644 index 000000000..056a007b0 --- /dev/null +++ b/docker/mongo/keyfile @@ -0,0 +1 @@ +ConduitLocalDevMongoReplicaSetKeyFileDoNotUseInProductionReplaceBeforeAnyRealDeployment0123456789abcdefghijklmnopqrstuvwxyz diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 6b7319a9b..b13fd687b 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -71,9 +71,7 @@ export interface ConduitArrayValidation { } export type ConduitValidationRules = - | ConduitStringValidation - | ConduitNumberValidation - | ConduitArrayValidation; + ConduitStringValidation | ConduitNumberValidation | ConduitArrayValidation; type BaseConduitModelField = { type?: TYPE | TYPE[] | ConduitModel | ArrayConduitModel[]; @@ -190,6 +188,9 @@ export interface ConduitSchemaOptions { authorization?: { enabled: boolean; }; + realtime?: { + enabled: boolean; + }; /** Mongoose read preference for this schema (ignored by SQL); per-query wins. */ readPreference?: string; }; diff --git a/libraries/grpc-sdk/src/modules/admin/index.ts b/libraries/grpc-sdk/src/modules/admin/index.ts index 889ca0ead..57a6ac7bd 100644 --- a/libraries/grpc-sdk/src/modules/admin/index.ts +++ b/libraries/grpc-sdk/src/modules/admin/index.ts @@ -3,7 +3,8 @@ import { AdminDefinition, RegisterAdminRouteRequest, RegisterAdminRouteRequest_PathDefinition, -} from '../../protoUtils/index.js'; + SocketPushRequest, +} from '../../protoUtils/core.js'; import { ConduitRouteActions } from '../../interfaces/index.js'; export class Admin extends ConduitModule { @@ -28,6 +29,10 @@ export class Admin extends ConduitModule { return this.client!.registerAdminRoute(request); } + socketPush(data: SocketPushRequest) { + return this.client!.socketPush(data); + } + patchRouteMiddlewares( path: string, action: ConduitRouteActions, diff --git a/libraries/hermes/src/Socket/Socket.ts b/libraries/hermes/src/Socket/Socket.ts index 7084fa857..f1c0b47d1 100644 --- a/libraries/hermes/src/Socket/Socket.ts +++ b/libraries/hermes/src/Socket/Socket.ts @@ -62,9 +62,17 @@ export class SocketController extends ConduitRouter { }; this.io = new IOServer(this.httpServer, this.options); this.redisClient = grpcSdk.redisManager.getClient(); + // Admin (e.g. :3031) and Router (e.g. :3001) are separate Socket.IO + // servers that share Redis. The adapter defaults would put both on the + // same stream, so a database change pushed to admin *and* router is + // delivered twice to every socket in those rooms. + const adapterKey = `socket.io:${this.port}`; this.io.adapter( createAdapter(this.redisClient, { onlyPlaintext: true, + streamName: adapterKey, + channelPrefix: adapterKey, + sessionKeyPrefix: `sio:session:${this.port}:`, }), ); this.httpServer.listen(this.port); @@ -144,11 +152,15 @@ export class SocketController extends ConduitRouter { this.io.of(namespace).on('connect', socket => { if (socket.recovered) { + const recoveredRooms = [...socket.rooms].filter( + room => room.startsWith('er:') || room.startsWith('database:'), + ); const recovered = conduitSocket.executeRecovered({ event: 'recovered', socketId: socket.id, context: socket.data, - recoveredRooms: [...socket.rooms].filter(room => room.startsWith('er:')), + params: recoveredRooms, + recoveredRooms, }); if (recovered) { recovered @@ -187,12 +199,13 @@ export class SocketController extends ConduitRouter { }); }); - socket.on('disconnect', () => { + socket.on('disconnect', (reason: string) => { conduitSocket .executeRequest({ event: 'disconnect', socketId: socket.id, context: socket.data, + params: [reason], }) .then(res => this.handleResponse(res, socket, namespace)) .catch(e => { diff --git a/libraries/hermes/src/Socket/isSocketHandshake.test.ts b/libraries/hermes/src/Socket/isSocketHandshake.test.ts index 267de330f..a686bdb76 100644 --- a/libraries/hermes/src/Socket/isSocketHandshake.test.ts +++ b/libraries/hermes/src/Socket/isSocketHandshake.test.ts @@ -16,6 +16,7 @@ describe('isSocketHandshake', () => { isSocketHandshake({ url: '/realtime/ticket?EIO=4&transport=polling' }), false, ); + assert.equal(isSocketHandshake({ url: '/realtime/ticket' }), false); assert.equal(isSocketHandshake({ originalUrl: '/realtime' }), false); assert.equal(isSocketHandshake({ url: '/graphql?EIO=4&transport=polling' }), false); assert.equal( diff --git a/libraries/hermes/src/index.ts b/libraries/hermes/src/index.ts index ac1a9960b..46808add1 100644 --- a/libraries/hermes/src/index.ts +++ b/libraries/hermes/src/index.ts @@ -47,6 +47,15 @@ export class ConduitRoutingController { private _cleanupTimeout: NodeJS.Timeout | null = null; /** Routes registered before MCP starts; replayed in initMCP. */ private readonly _conduitRoutesByKey: Map = new Map(); + /** Sockets registered before Socket.IO starts; replayed in initSockets. */ + private readonly _conduitSocketsByPath: Map = new Map(); + private readonly _socketMiddlewares: Array< + (req: ConduitRequest, res: Response, next: NextFunction) => void + > = []; + private readonly _socketRouteMiddlewares: Array<{ + middleware: ConduitMiddleware; + moduleUrl: string; + }> = []; private readonly routeTrie: RouteTrie = new RouteTrie(); readonly expressApp: Express = express(); readonly server = http.createServer(this.expressApp); @@ -143,6 +152,15 @@ export class ConduitRoutingController { this.expressApp, this.metrics, ); + for (const middleware of this._socketMiddlewares) { + this._socketRouter.registerGlobalMiddleware(middleware); + } + for (const { middleware, moduleUrl } of this._socketRouteMiddlewares) { + this._socketRouter.registerMiddleware(middleware, moduleUrl); + } + for (const socket of this._conduitSocketsByPath.values()) { + this._socketRouter.registerConduitSocket(socket); + } } initMCP(config?: { @@ -231,6 +249,7 @@ export class ConduitRoutingController { ) { this._middlewareRouter.use(middleware); if (socketMiddleware) { + this._socketMiddlewares.push(middleware); this._socketRouter?.registerGlobalMiddleware(middleware); } } @@ -244,6 +263,7 @@ export class ConduitRoutingController { registerRouteMiddleware(middleware: ConduitMiddleware, moduleUrl: string) { this._restRouter?.registerMiddleware(middleware, moduleUrl); this._graphQLRouter?.registerMiddleware(middleware, moduleUrl); + this._socketRouteMiddlewares.push({ middleware, moduleUrl }); this._socketRouter?.registerMiddleware(middleware, moduleUrl); this._mcpRouter?.registerMiddleware(middleware, moduleUrl); } @@ -292,6 +312,7 @@ export class ConduitRoutingController { } registerConduitSocket(socket: ConduitSocket) { + this._conduitSocketsByPath.set(socket.input.path, socket); this._socketRouter?.registerConduitSocket(socket); } diff --git a/libraries/hermes/src/interfaces/Socket.ts b/libraries/hermes/src/interfaces/Socket.ts index 7c7851094..2b9787257 100644 --- a/libraries/hermes/src/interfaces/Socket.ts +++ b/libraries/hermes/src/interfaces/Socket.ts @@ -91,10 +91,13 @@ export class ConduitSocket { executeRecovered( request: ConduitSocketParameters, ): ConduitSocketHandlerResponse | null { - if (!this._input.onRecovered) { - return null; + if (this._input.onRecovered) { + return this._input.onRecovered(request); } - return this._input.onRecovered(request); + if (this._events.has('recovered')) { + return this._events.get('recovered')!.handler(request); + } + return null; } } diff --git a/modules/database/README.mdx b/modules/database/README.mdx index c340b86bd..b84cded13 100644 --- a/modules/database/README.mdx +++ b/modules/database/README.mdx @@ -47,6 +47,27 @@ since the latter need to go through parsers that are otherwise unnecessary for M When using MongoDB with a replica set (e.g., MongoDB Atlas), the database module supports configuring read preference, write concern, and read concern through the admin panel at `PATCH /config/database`. +Live document updates require a replica set or sharded cluster (Atlas is fine). Helm’s bundled Mongo chart is a **standalone** Deployment (`replicas: 1`, no `--replSet`), so live updates stay `idle` there until you point `DB_CONN_URI` at Atlas or an operator-managed replica set. + +Local Compose files initialize a single-node `rs0`. **Existing Compose Mongo volumes will not become a replica set cleanly** — drop the volume or start from an empty data dir if `hello` still reports standalone. + +### Live updates + +Enable `realtime.enabled` in database module config, then opt a schema in with `modelOptions.conduit.realtime.enabled`. Clients connect to the `/database/` Socket.IO namespace (path `/realtime`) and emit: + +``` +subscribe({ schema: 'Order', documentId?: string }) +unsubscribe({ schema: 'Order', documentId?: string }) +``` + +Events arrive as `change` with `{ version, operation, schema, documentId, occurredAt }` and contain no document fields and no resume token. This is **live-tail, not backfill**. After a full reconnect, subscribe again and refetch over authorized REST. + +The leader also publishes `database:change:${schema}` on the Redis bus. **Do not also relay `database:change:*` on `/events/` if the same client is on `/database/`** — that duplicates notifications. Keep the bus for other modules; just do not dual-subscribe. + +Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Client sockets also require CMS `crudOperations.read.enabled`. Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; that token cannot mint another ticket or call REST/GraphQL. Session JWTs and masterkeys must not be sent from browser code. + +Admin sockets must be enabled (`admin.transports.sockets`) and the Admin socket port (`ADMIN_SOCKET_PORT`, default 3031) reachable from the UI. + ### Configuration Options | Setting | Values | Default | Description | @@ -54,6 +75,7 @@ When using MongoDB with a replica set (e.g., MongoDB Atlas), the database module | `readPreference` | `primary`, `primaryPreferred`, `secondary`, `secondaryPreferred`, `nearest` | `primary` | Controls which replica set members receive read queries | | `writeConcern` | `1`, `majority` | `1` | How many members must acknowledge a write | | `readConcern` | `local`, `available`, `majority`, `linearizable`, `snapshot` | `local` | Consistency level for read operations | +| `realtime.enabled` | `true`, `false` | `false` | Enable MongoDB change-stream live updates for opted-in schemas | ### Recommended Production Settings diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index 57286cb1e..e2d36e26f 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -64,6 +64,7 @@ import { QueueController } from './controllers/queue.controller.js'; import AppConfigSchema, { Config } from './config/index.js'; import { Empty } from './protoTypes/google/protobuf/empty.js'; import { fileURLToPath } from 'node:url'; +import { RealtimeService } from './realtime/index.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -107,6 +108,7 @@ export default class DatabaseModule extends ManagedModule { private customEndpointController?: CustomEndpointController; private _authorizationDefinitionsRegistered = false; private _databaseRouterWatchDispose: (() => void) | null = null; + private realtimeService?: RealtimeService; constructor(dbType: string, dbUri: string, peerManifestRoot?: string) { super('database', peerManifestRoot); @@ -164,6 +166,7 @@ export default class DatabaseModule extends ManagedModule { readConcern: config.readConcern ?? 'local', }); } + void this.realtimeService?.reconcile(); if (!config.viewCleanup.enabled) { try { await QueueController.getInstance().drainViewCleanupQueue(); @@ -1015,13 +1018,16 @@ export default class DatabaseModule extends ManagedModule { this.grpcSdk, this._activeAdapter, ); + this.realtimeService = new RealtimeService(this.grpcSdk, this._activeAdapter); this.adminRouter = new AdminHandlers( this.grpcServer, this.grpcSdk, this._activeAdapter, this.schemaController, this.customEndpointController, + this.realtimeService, ); + void this.realtimeService.reconcile(); this._databaseRouterWatchDispose?.(); this._databaseRouterWatchDispose = this.grpcSdk.watchPeer( 'router', @@ -1031,6 +1037,7 @@ export default class DatabaseModule extends ManagedModule { this.grpcServer, this._activeAdapter, this.grpcSdk, + this.realtimeService, ); this.schemaController?.setRouter(this.userRouter); this.customEndpointController?.setRouter(this.userRouter); @@ -1046,4 +1053,8 @@ export default class DatabaseModule extends ManagedModule { ); } } + + async shutdown(): Promise { + await this.realtimeService?.shutdown(); + } } diff --git a/modules/database/src/admin/__tests__/schema.admin.put-patch.test.ts b/modules/database/src/admin/__tests__/schema.admin.put-patch.test.ts index 89a7ae8ea..464bf4462 100644 --- a/modules/database/src/admin/__tests__/schema.admin.put-patch.test.ts +++ b/modules/database/src/admin/__tests__/schema.admin.put-patch.test.ts @@ -185,6 +185,21 @@ describe('SchemaAdmin PUT/PATCH split', () => { warnSpy.mockRestore(); }); + it('persists realtime conduitOptions used by live document updates', async () => { + const requestedSchema = makeRequestedSchema(); + const { admin, createSchema } = setup(requestedSchema); + + await admin.patchSchema( + makeCall({ + id: 'schema-1', + conduitOptions: { realtime: { enabled: true } }, + }), + ); + + const [writtenSchema] = createSchema.mock.calls[0]; + expect(writtenSchema.modelOptions.conduit.realtime.enabled).toBe(true); + }); + it('wipes existing documents when enabling authorization via conduitOptions', async () => { const requestedSchema = makeRequestedSchema(); const { admin, deleteMany } = setup(requestedSchema); diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index a4c1be91f..9742f31eb 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -23,6 +23,7 @@ import { SchemaController } from '../controllers/cms/schema.controller.js'; import { CustomEndpointController } from '../controllers/customEndpoints/customEndpoint.controller.js'; import { CustomEndpoints, DeclaredSchema, PendingSchemas } from '../models/index.js'; import { ConduitOptions, SchemaFieldsRequired } from '../interfaces/index.js'; +import type { RealtimeService } from '../realtime/index.js'; export class AdminHandlers { private readonly schemaAdmin: SchemaAdmin; @@ -36,6 +37,7 @@ export class AdminHandlers { private readonly _activeAdapter: DatabaseAdapter, private readonly schemaController: SchemaController, private readonly customEndpointController: CustomEndpointController, + private readonly realtimeService: RealtimeService, ) { this.schemaAdmin = new SchemaAdmin( this.grpcSdk, @@ -686,6 +688,7 @@ export class AdminHandlers { new ConduitRouteReturnDefinition('getDatabaseType', 'String'), this.schemaAdmin.getDatabaseType.bind(this.schemaAdmin), ); + this.realtimeService.registerAdmin(this.routingManager); this.routingManager.registerRoutes(); } } diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index 9c149d6db..cc8309664 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -250,6 +250,7 @@ export class SchemaAdmin { cms: call.request.params.conduitOptions?.cms, permissions: call.request.params.conduitOptions?.permissions, authorization: call.request.params.conduitOptions?.authorization, + realtime: call.request.params.conduitOptions?.realtime, readPreference: call.request.params.conduitOptions?.readPreference, timestamps: call.request.params.timestamps, }); @@ -343,6 +344,7 @@ export class SchemaAdmin { cms: conduitOptions?.cms, authorization: conduitOptions?.authorization, permissions: conduitOptions?.permissions, + realtime: conduitOptions?.realtime, readPreference: conduitOptions?.readPreference, existingModelOptions: requestedSchema.modelOptions, }); diff --git a/modules/database/src/config/index.ts b/modules/database/src/config/index.ts index 0f5a4fcc8..1ddda80d7 100644 --- a/modules/database/src/config/index.ts +++ b/modules/database/src/config/index.ts @@ -47,6 +47,13 @@ const AppConfigSchema = { default: 0, }, }, + realtime: { + enabled: { + doc: 'Enable MongoDB change-stream live updates for opted-in schemas', + format: 'Boolean', + default: false, + }, + }, }; const config = convict(AppConfigSchema); diff --git a/modules/database/src/index.ts b/modules/database/src/index.ts index 3e9466196..12261992a 100644 --- a/modules/database/src/index.ts +++ b/modules/database/src/index.ts @@ -1,5 +1,6 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import DatabaseModule from './Database.js'; const dbType = process.env.DB_TYPE ?? 'mongodb'; @@ -7,4 +8,21 @@ const dbUri = process.env.DB_CONN_URI ?? 'mongodb://localhost:27017'; const peerManifestRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const database = new DatabaseModule(dbType, dbUri, peerManifestRoot); + +function registerShutdownSignals(): void { + const shutdown = (signal: NodeJS.Signals) => { + void database + .shutdown() + .catch(err => { + ConduitGrpcSdk.Logger.error(err as Error); + }) + .finally(() => { + process.exit(signal === 'SIGINT' ? 130 : 0); + }); + }; + process.once('SIGTERM', () => shutdown('SIGTERM')); + process.once('SIGINT', () => shutdown('SIGINT')); +} + +registerShutdownSignals(); database.start(); diff --git a/modules/database/src/interfaces/ConduitOptions.ts b/modules/database/src/interfaces/ConduitOptions.ts index 82ee970fb..f5f1b3b02 100644 --- a/modules/database/src/interfaces/ConduitOptions.ts +++ b/modules/database/src/interfaces/ConduitOptions.ts @@ -30,5 +30,8 @@ export const ConduitOptions = { authorization: { enabled: ConduitBoolean.Optional, }, + realtime: { + enabled: ConduitBoolean.Optional, + }, readPreference: ConduitString.Optional, }; diff --git a/modules/database/src/metrics/index.ts b/modules/database/src/metrics/index.ts index e00ad3a33..3d3ef0256 100644 --- a/modules/database/src/metrics/index.ts +++ b/modules/database/src/metrics/index.ts @@ -31,4 +31,19 @@ export default { help: 'Tracks the total number of custom endpoints', }, }, + realtimeEvents: { + type: MetricType.Counter, + config: { + name: 'database_realtime_events_total', + help: 'Tracks normalized database change-stream events', + labelNames: ['operation'], + }, + }, + realtimeStreamErrors: { + type: MetricType.Counter, + config: { + name: 'database_realtime_stream_errors_total', + help: 'Tracks change-stream errors', + }, + }, }; diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts new file mode 100644 index 000000000..d29938d15 --- /dev/null +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -0,0 +1,408 @@ +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { + normalizeChangeEvent, + parseResumeToken, + serializeResumeToken, + type RawChangeEvent, +} from './normalize.js'; +import { authorizedDocumentRoom, roomsForPublicChange } from './rooms.js'; +import { + isResumeTokenUnusable, + topologyFromHello, + type TopologyResult, +} from './topology.js'; +import type { + ChangeStreamLike, + DatabaseChangeEvent, + OptedInSchema, + RealtimeStatusCode, +} from './types.js'; +import type { RealtimeSubscriptionTracker } from './subscriptions.js'; +import { type AuthorizationSdk } from './authorize.js'; +import { checkRebacBatch, RealtimeRebacCache } from './rebacCache.js'; +import { + buildWatchPipeline, + optedInCollectionsKey, + WATCH_RESTART_OPERATIONS, + type WatchPipeline, +} from './watchPipeline.js'; + +const LEADER_LOCK = 'realtime:change-stream:leader'; +const RESUME_TOKEN_KEY = 'realtime:resumeToken'; +const LOCK_TTL_MS = 15_000; +const LOCK_RENEW_MS = 5_000; +const RETRY_BASE_MS = 1_000; +const RETRY_MAX_MS = 30_000; + +type LeaderLock = NonNullable< + Awaited['tryAcquireLock']>> +>; + +export type WatchFactory = (options: { + resumeAfter?: unknown; + pipeline: WatchPipeline; +}) => ChangeStreamLike; + +export type CoordinatorOptions = { + grpcSdk: ConduitGrpcSdk; + watch: WatchFactory; + hello: () => Promise<{ setName?: string; msg?: string } | null>; + getOptedInSchemas: () => OptedInSchema[]; + subscriptions: RealtimeSubscriptionTracker; + enabled: () => boolean; + engine: () => string; +}; + +export class MongoChangeStreamCoordinator { + private lock: LeaderLock | null = null; + private stream: ChangeStreamLike | null = null; + private renewTimer: NodeJS.Timeout | null = null; + private retryTimer: NodeJS.Timeout | null = null; + private closed = false; + private streamState: RealtimeStatusCode = 'idle'; + private lastEventAt?: string; + private lastError?: string; + private topology: TopologyResult = { supported: false }; + private retryAttempt = 0; + private watching = false; + private opening = false; + private ignoreClose = false; + private changeQueue: Promise = Promise.resolve(); + private watchedCollectionsKey = ''; + private readonly rebacCache = new RealtimeRebacCache(); + + constructor(private readonly options: CoordinatorOptions) {} + + getState(): RealtimeStatusCode { + return this.streamState; + } + + getLastEventAt(): string | undefined { + return this.lastEventAt; + } + + getLastError(): string | undefined { + return this.lastError; + } + + getTopology(): TopologyResult { + return this.topology; + } + + async waitForIdle(): Promise { + await this.changeQueue; + } + + async reconcile(): Promise { + if (this.closed) return; + const engine = this.options.engine(); + if (engine !== 'MongoDB' || !this.options.enabled()) { + await this.stopStream('idle'); + await this.releaseLeader(); + this.streamState = engine !== 'MongoDB' ? 'unsupported' : 'disabled'; + return; + } + this.topology = topologyFromHello(await this.options.hello().catch(() => null)); + if (!this.topology.supported) { + await this.stopStream('idle'); + await this.releaseLeader(); + this.streamState = 'idle'; + this.lastError = this.topology.message; + if ( + !this.topology.message || + this.topology.message.includes('Unable to determine') + ) { + this.scheduleRetry(); + } + return; + } + const collections = this.collectionNames(); + if (collections.length === 0) { + await this.stopStream('idle'); + await this.releaseLeader(); + this.streamState = 'idle'; + return; + } + const nextKey = optedInCollectionsKey(collections); + if (this.watching && nextKey !== this.watchedCollectionsKey) { + await this.stopStream('starting'); + } + await this.ensureLeader(); + } + + async shutdown(): Promise { + this.closed = true; + this.clearTimers(); + await this.changeQueue; + await this.stopStream('idle'); + await this.releaseLeader(); + this.rebacCache.clear(); + } + + private collectionNames(): string[] { + return this.options.getOptedInSchemas().map(schema => schema.collectionName); + } + + private async ensureLeader(): Promise { + if (this.lock) { + if (!this.watching) { + await this.openStream(); + } + return; + } + try { + const acquired = await this.options.grpcSdk.state!.tryAcquireLock( + LEADER_LOCK, + LOCK_TTL_MS, + ); + if (!acquired) { + this.streamState = 'idle'; + this.scheduleRetry(); + return; + } + this.lock = acquired; + this.startRenewal(); + await this.openStream(); + } catch (err) { + this.lastError = err instanceof Error ? err.message : String(err); + this.streamState = 'degraded'; + this.scheduleRetry(); + } + } + + private startRenewal() { + this.clearRenewTimer(); + this.renewTimer = setInterval(() => { + void this.renewLock(); + }, LOCK_RENEW_MS); + } + + private async renewLock() { + if (!this.lock) return; + try { + this.lock = await this.lock.extend(LOCK_TTL_MS); + } catch { + this.lock = null; + await this.stopStream('idle'); + this.scheduleRetry(); + } + } + + private async openStream() { + if (this.watching || this.closed || this.opening) return; + this.opening = true; + this.streamState = 'starting'; + this.ignoreClose = false; + try { + const resumeAfter = parseResumeToken( + await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY), + ); + if (this.watching || this.closed) return; + const collections = this.collectionNames(); + const pipeline = buildWatchPipeline(collections); + this.watchedCollectionsKey = optedInCollectionsKey(collections); + const stream = this.options.watch({ resumeAfter, pipeline }); + this.stream = stream; + this.watching = true; + this.streamState = 'live'; + this.retryAttempt = 0; + stream.on('change', (change: unknown) => { + this.enqueueChange(change as RawChangeEvent); + }); + stream.on('error', (err: unknown) => { + void this.handleStreamError(err); + }); + stream.on('close', () => { + this.watching = false; + if (!this.closed && !this.ignoreClose && this.lock) { + this.scheduleRetry(); + } + }); + } catch (err) { + this.watching = false; + await this.handleStreamError(err); + } finally { + this.opening = false; + } + } + + private enqueueChange(change: RawChangeEvent) { + this.changeQueue = this.changeQueue.then(async () => { + if (this.closed || !this.watching) return; + try { + await this.handleChange(change); + } catch (err) { + this.lastError = err instanceof Error ? err.message : String(err); + ConduitGrpcSdk.Logger.error(err as Error); + this.watching = false; + await this.stopStream('degraded'); + this.scheduleRetry(); + } + }); + } + + private async handleChange(change: RawChangeEvent) { + const token = serializeResumeToken(change._id); + const schema = this.resolveSchema(change.ns?.coll); + const event = schema ? normalizeChangeEvent(change, schema.name) : null; + if (!event || !schema) { + if (token) { + await this.persistResumeToken(token); + } + if (change.operationType && WATCH_RESTART_OPERATIONS.has(change.operationType)) { + await this.stopStream('starting'); + this.scheduleRetry(); + } + return; + } + this.lastEventAt = event.occurredAt; + this.lastError = undefined; + await this.emitChange(schema, event); + if (token) { + await this.persistResumeToken(token); + } + } + + private async persistResumeToken(token: string) { + await this.options.grpcSdk.state!.setKey(RESUME_TOKEN_KEY, token); + } + + private async emitChange(schema: OptedInSchema, event: DatabaseChangeEvent) { + const payload = JSON.stringify(event); + this.options.grpcSdk.bus?.publish(`database:change:${schema.name}`, payload); + ConduitGrpcSdk.Metrics?.increment('database_realtime_events_total', 1, { + operation: event.operation, + }); + await this.pushEvent(schema, event, payload); + } + + private async pushEvent( + schema: OptedInSchema, + event: DatabaseChangeEvent, + payload: string, + ) { + const adminRooms = roomsForPublicChange(schema.name, event.documentId); + await this.safePush('admin', adminRooms, payload); + if (!schema.authorizationEnabled) { + await this.safePush('router', adminRooms, payload); + return; + } + const userIds = await this.options.subscriptions.listUsers( + schema.name, + event.documentId, + ); + const decisions = await checkRebacBatch( + this.rebacCache, + this.options.grpcSdk as unknown as AuthorizationSdk, + userIds, + schema.name, + event.documentId, + ); + const allowedRooms: string[] = []; + for (const userId of userIds) { + const decision = decisions.get(userId) ?? 'unavailable'; + if (decision === 'allow') { + allowedRooms.push(authorizedDocumentRoom(schema.name, event.documentId, userId)); + continue; + } + if (decision === 'deny') { + await this.options.subscriptions.removeUser( + schema.name, + event.documentId, + userId, + ); + } + } + if (allowedRooms.length > 0) { + await this.safePush('router', allowedRooms, payload); + } + } + + private async safePush( + target: 'admin' | 'router', + rooms: string[], + data: string, + ): Promise { + const client = + target === 'admin' ? this.options.grpcSdk.admin : this.options.grpcSdk.router; + if (!client?.socketPush) return; + await client.socketPush({ + event: 'change', + data, + rooms, + receivers: [], + }); + } + + private resolveSchema(collectionName?: string): OptedInSchema | undefined { + if (!collectionName) return undefined; + return this.options + .getOptedInSchemas() + .find(schema => schema.collectionName === collectionName); + } + + private async handleStreamError(err: unknown) { + this.watching = false; + this.lastError = err instanceof Error ? err.message : String(err); + this.streamState = 'degraded'; + ConduitGrpcSdk.Metrics?.increment('database_realtime_stream_errors_total'); + ConduitGrpcSdk.Logger.error(err as Error); + if (isResumeTokenUnusable(err)) { + await this.options.grpcSdk.state!.clearKey(RESUME_TOKEN_KEY); + } + await this.stopStream('degraded'); + this.scheduleRetry(); + } + + private scheduleRetry() { + if (this.closed || this.retryTimer) return; + const delay = Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** this.retryAttempt); + this.retryAttempt += 1; + this.retryTimer = setTimeout(() => { + this.retryTimer = null; + void this.reconcile(); + }, delay); + } + + private async stopStream(nextState: RealtimeStatusCode) { + const stream = this.stream; + this.stream = null; + this.watching = false; + this.streamState = nextState; + this.ignoreClose = true; + this.watchedCollectionsKey = ''; + if (stream) { + try { + await stream.close(); + } catch { + // already closed + } + } + } + + private async releaseLeader() { + this.clearRenewTimer(); + if (!this.lock) return; + try { + await this.options.grpcSdk.state!.releaseLock(this.lock); + } catch { + // lock may already have expired + } + this.lock = null; + } + + private clearTimers() { + this.clearRenewTimer(); + if (this.retryTimer) { + clearTimeout(this.retryTimer); + this.retryTimer = null; + } + } + + private clearRenewTimer() { + if (this.renewTimer) { + clearInterval(this.renewTimer); + this.renewTimer = null; + } + } +} diff --git a/modules/database/src/realtime/RealtimeService.ts b/modules/database/src/realtime/RealtimeService.ts new file mode 100644 index 000000000..96d21f485 --- /dev/null +++ b/modules/database/src/realtime/RealtimeService.ts @@ -0,0 +1,162 @@ +import { + ConduitGrpcSdk, + ConduitRouteActions, + ConduitRouteReturnDefinition, +} from '@conduitplatform/grpc-sdk'; +import { + ConduitNumber, + ConduitString, + ConfigController, + RoutingManager, +} from '@conduitplatform/module-tools'; +import { DatabaseAdapter } from '../adapters/DatabaseAdapter.js'; +import { MongooseAdapter } from '../adapters/mongoose-adapter/index.js'; +import { MongooseSchema } from '../adapters/mongoose-adapter/MongooseSchema.js'; +import { SequelizeSchema } from '../adapters/sequelize-adapter/SequelizeSchema.js'; +import { toOptedInSchema } from './authorize.js'; +import { MongoChangeStreamCoordinator } from './MongoChangeStreamCoordinator.js'; +import { registerDatabaseRealtimeSocket } from './sockets.js'; +import { buildRealtimeStatus } from './status.js'; +import { RealtimeSubscriptionTracker } from './subscriptions.js'; +import type { ChangeStreamLike, OptedInSchema, RealtimeStatus } from './types.js'; +import type { WatchPipeline } from './watchPipeline.js'; + +export class RealtimeService { + private readonly subscriptions: RealtimeSubscriptionTracker; + private coordinator?: MongoChangeStreamCoordinator; + + constructor( + private readonly grpcSdk: ConduitGrpcSdk, + private readonly adapter: DatabaseAdapter, + ) { + this.subscriptions = new RealtimeSubscriptionTracker( + grpcSdk.redisManager.getClient(), + ); + this.grpcSdk.bus?.subscribe('database:create:schema', () => { + void this.reconcile(); + }); + this.grpcSdk.bus?.subscribe('database:delete:schema', () => { + void this.reconcile(); + }); + if (adapter instanceof MongooseAdapter) { + this.coordinator = new MongoChangeStreamCoordinator({ + grpcSdk, + watch: options => this.openWatch(adapter, options), + hello: () => this.hello(adapter), + getOptedInSchemas: () => this.getOptedInSchemas(), + subscriptions: this.subscriptions, + enabled: () => this.isGloballyEnabled(), + engine: () => adapter.getDatabaseType(), + }); + } + } + + registerAdmin(routingManager: RoutingManager) { + routingManager.route( + { + path: '/realtime/status', + action: ConduitRouteActions.GET, + description: `Returns live-update capability and runtime status for the database module.`, + }, + new ConduitRouteReturnDefinition('DatabaseRealtimeStatus', { + status: ConduitString.Required, + engine: ConduitString.Required, + activeSchemaCount: ConduitNumber.Required, + lastEventAt: ConduitString.Optional, + message: ConduitString.Optional, + }), + async () => this.getStatus(), + ); + registerDatabaseRealtimeSocket(routingManager, { + mode: 'admin', + grpcSdk: this.grpcSdk, + schemaLookup: this.adapter, + subscriptions: this.subscriptions, + isGloballyEnabled: () => this.isGloballyEnabled(), + }); + } + + registerClient(routingManager: RoutingManager) { + registerDatabaseRealtimeSocket(routingManager, { + mode: 'client', + grpcSdk: this.grpcSdk, + schemaLookup: this.adapter, + subscriptions: this.subscriptions, + isGloballyEnabled: () => this.isGloballyEnabled(), + }); + } + + async reconcile(): Promise { + if (!this.coordinator) return; + await this.coordinator.reconcile(); + } + + async shutdown(): Promise { + await this.coordinator?.shutdown(); + } + + async getStatus(): Promise { + const engine = this.adapter.getDatabaseType(); + const optedIn = this.getOptedInSchemas(); + return buildRealtimeStatus({ + engine, + enabled: this.isGloballyEnabled(), + topologySupported: this.coordinator?.getTopology().supported ?? false, + topologyMessage: this.coordinator?.getTopology().message, + activeSchemaCount: optedIn.length, + streamState: + this.coordinator?.getState() ?? (engine === 'MongoDB' ? 'idle' : 'unsupported'), + lastEventAt: this.coordinator?.getLastEventAt(), + lastError: this.coordinator?.getLastError(), + socketsEnabled: await this.areAdminSocketsEnabled(), + }); + } + + private isGloballyEnabled(): boolean { + return ConfigController.getInstance().config?.realtime?.enabled === true; + } + + private getOptedInSchemas(): OptedInSchema[] { + const schemas: OptedInSchema[] = []; + for (const schema of this.adapter.registeredSchemas.values()) { + const optedIn = toOptedInSchema({ + name: schema.name, + collectionName: schema.collectionName, + modelOptions: schema.modelOptions, + }); + if (optedIn) schemas.push(optedIn); + } + return schemas; + } + + private openWatch( + adapter: MongooseAdapter, + options: { resumeAfter?: unknown; pipeline: WatchPipeline }, + ): ChangeStreamLike { + const db = adapter.mongoose.connection.db; + if (!db) { + throw new Error('MongoDB connection is not ready'); + } + return db.watch( + options.pipeline, + options.resumeAfter ? { resumeAfter: options.resumeAfter as never } : {}, + ) as unknown as ChangeStreamLike; + } + + private async hello( + adapter: MongooseAdapter, + ): Promise<{ setName?: string; msg?: string } | null> { + const db = adapter.mongoose.connection.db; + if (!db) return null; + return db.admin().command({ hello: 1 }); + } + + private async areAdminSocketsEnabled(): Promise { + try { + const adminConfig = await this.grpcSdk.config.get('admin'); + return adminConfig?.transports?.sockets === true; + } catch { + return true; + } + } +} diff --git a/modules/database/src/realtime/__tests__/authorize.test.ts b/modules/database/src/realtime/__tests__/authorize.test.ts new file mode 100644 index 000000000..9c5eba15b --- /dev/null +++ b/modules/database/src/realtime/__tests__/authorize.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from '@jest/globals'; +import { status } from '@grpc/grpc-js'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { + assertSchemaAvailable, + optionalDocumentId, + parseSubscribeRequest, + requireSchemaName, +} from '../authorize.js'; + +describe('realtime authorization helpers', () => { + it('parses subscribe payloads and required schema names', () => { + expect(parseSubscribeRequest([{ schema: 'Order', documentId: '1' }])).toEqual({ + schema: 'Order', + documentId: '1', + }); + expect(requireSchemaName('Order')).toBe('Order'); + expect(optionalDocumentId(undefined)).toBeUndefined(); + expect(() => requireSchemaName('')).toThrow( + expect.objectContaining({ code: status.INVALID_ARGUMENT }), + ); + }); + + it('rejects missing schemas, disabled realtime, and CMS-read for clients', () => { + const lookup = { + getSchema: (name: string) => { + if (name === 'Missing') return undefined; + if (name === 'Off') { + return { name, modelOptions: { conduit: { realtime: { enabled: false } } } }; + } + return { + name, + modelOptions: { + conduit: { + realtime: { enabled: true }, + cms: { crudOperations: { read: { enabled: false } } }, + authorization: { enabled: true }, + }, + }, + }; + }, + }; + expect(() => assertSchemaAvailable(lookup, 'Missing', true)).toThrow( + expect.objectContaining({ code: status.NOT_FOUND }), + ); + expect(() => assertSchemaAvailable(lookup, 'Off', false)).toThrow( + expect.objectContaining({ code: status.FAILED_PRECONDITION }), + ); + expect(() => assertSchemaAvailable(lookup, 'Order', true)).toThrow( + expect.objectContaining({ code: status.PERMISSION_DENIED }), + ); + const admin = assertSchemaAvailable(lookup, 'Order', false); + expect(admin.authorizationEnabled).toBe(true); + }); + + it('maps adapter NOT_FOUND throws to a subscription error', () => { + const lookup = { + getSchema: () => { + throw new GrpcError(status.NOT_FOUND, 'Schema Missing not defined yet'); + }, + }; + expect(() => assertSchemaAvailable(lookup, 'Missing', false)).toThrow( + expect.objectContaining({ + code: status.NOT_FOUND, + message: 'Schema does not exist', + }), + ); + }); +}); diff --git a/modules/database/src/realtime/__tests__/change-stream.integration.test.ts b/modules/database/src/realtime/__tests__/change-stream.integration.test.ts new file mode 100644 index 000000000..b4e313310 --- /dev/null +++ b/modules/database/src/realtime/__tests__/change-stream.integration.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from '@jest/globals'; +import { MongoClient, ObjectId } from 'mongodb'; +import { normalizeChangeEvent, type RawChangeEvent } from '../normalize.js'; +import { buildWatchPipeline } from '../watchPipeline.js'; + +const replicaSetUri = process.env.DB_CONN_URI; +const integrationEnabled = Boolean(replicaSetUri?.includes('replicaSet')); +const describeIntegration = integrationEnabled ? describe : describe.skip; + +describeIntegration('MongoDB change stream contract', () => { + it('normalizes insert events from a replica-set watch without document fields', async () => { + const client = new MongoClient(replicaSetUri!); + await client.connect(); + const dbName = `conduit_realtime_${Date.now()}`; + const db = client.db(dbName); + try { + const collection = db.collection('orders'); + const stream = db.watch(buildWatchPipeline(['orders'])); + const change = await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error('timed out waiting for change')), + 10_000, + ); + stream.on('change', event => { + clearTimeout(timer); + resolve(event as RawChangeEvent); + }); + stream.on('error', err => { + clearTimeout(timer); + reject(err); + }); + void collection.insertOne({ _id: new ObjectId(), secret: 'do-not-leak' }); + }); + await stream.close(); + const event = normalizeChangeEvent(change, 'Order'); + expect(event).toMatchObject({ + version: 1, + operation: 'insert', + schema: 'Order', + }); + expect(event).not.toHaveProperty('fullDocument'); + expect(JSON.stringify(event)).not.toContain('do-not-leak'); + } finally { + await db.dropDatabase().catch(() => undefined); + await client.close(); + } + }); +}); diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts new file mode 100644 index 000000000..cda5dc0e1 --- /dev/null +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -0,0 +1,422 @@ +import { EventEmitter } from 'node:events'; +import { afterEach, describe, expect, it, jest } from '@jest/globals'; +import { EJSON, ObjectId } from 'bson'; +import { MongoChangeStreamCoordinator } from '../MongoChangeStreamCoordinator.js'; +import { RealtimeSubscriptionTracker } from '../subscriptions.js'; +import { roomsForPublicChange } from '../rooms.js'; + +class MemoryStore { + private sets = new Map>(); + async sadd(key: string, ...members: string[]) { + const set = this.sets.get(key) ?? new Set(); + members.forEach(member => set.add(member)); + this.sets.set(key, set); + return members.length; + } + async srem(key: string, ...members: string[]) { + const set = this.sets.get(key); + if (!set) return 0; + members.forEach(member => set.delete(member)); + return members.length; + } + async smembers(key: string) { + return [...(this.sets.get(key) ?? new Set())]; + } + async scard(key: string) { + return this.sets.get(key)?.size ?? 0; + } + async del(...keys: string[]) { + keys.forEach(key => this.sets.delete(key)); + return keys.length; + } +} + +function createCoordinator(overrides?: { + allow?: boolean; + authorizationAvailable?: boolean; + schemas?: { name: string; collectionName: string; authorizationEnabled: boolean }[]; + getKeyDelayMs?: number; +}) { + const stream = new EventEmitter() as EventEmitter & { close: () => Promise }; + stream.close = async () => { + stream.emit('close'); + }; + const state = new Map(); + const lock = { + extend: jest.fn(async () => lock), + release: jest.fn(async () => undefined), + }; + const routerPush = jest.fn(async () => undefined); + const adminPush = jest.fn(async () => undefined); + const publish = jest.fn(); + const watch = jest.fn(() => stream as never); + const subscriptions = new RealtimeSubscriptionTracker(new MemoryStore()); + const grpcSdk = { + state: { + tryAcquireLock: jest.fn(async () => lock), + releaseLock: jest.fn(async () => undefined), + getKey: jest.fn(async (key: string) => { + if (overrides?.getKeyDelayMs) { + await new Promise(resolve => setTimeout(resolve, overrides.getKeyDelayMs)); + } + return state.get(key) ?? null; + }), + setKey: jest.fn(async (key: string, value: string) => { + state.set(key, value); + }), + clearKey: jest.fn(async (key: string) => { + state.delete(key); + }), + }, + bus: { publish }, + router: { socketPush: routerPush }, + admin: { socketPush: adminPush }, + isAvailable: () => overrides?.authorizationAvailable !== false, + authorization: + overrides?.authorizationAvailable === false + ? null + : { + can: async () => ({ allow: overrides?.allow !== false }), + }, + }; + const coordinator = new MongoChangeStreamCoordinator({ + grpcSdk: grpcSdk as never, + watch, + hello: async () => ({ setName: 'rs0' }), + getOptedInSchemas: () => + overrides?.schemas ?? [ + { name: 'Order', collectionName: 'orders', authorizationEnabled: false }, + ], + subscriptions, + enabled: () => true, + engine: () => 'MongoDB', + }); + return { + coordinator, + stream, + watch, + routerPush, + adminPush, + publish, + subscriptions, + grpcSdk, + state, + lock, + }; +} + +function insertChange(collection: string, id: string, token: unknown) { + return { + operationType: 'insert', + ns: { coll: collection }, + documentKey: { _id: new ObjectId(id) }, + _id: token, + }; +} + +describe('MongoChangeStreamCoordinator', () => { + afterEach(() => { + jest.useRealTimers(); + }); + + it('emits one normalized event to public rooms and ignores other collections', async () => { + const { coordinator, stream, routerPush, adminPush, publish, state } = + createCoordinator(); + await coordinator.reconcile(); + const resume = { _data: 'token' }; + stream.emit('change', { + ...insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', resume), + fullDocument: { secret: 'nope' }, + wallTime: new Date('2026-01-02T00:00:00.000Z'), + }); + stream.emit('change', insertChange('other', '64b64c4c4c4c4c4c4c4c4c4d', resume)); + await coordinator.waitForIdle(); + expect(publish).toHaveBeenCalledTimes(1); + expect(publish.mock.calls[0][0]).toBe('database:change:Order'); + const payload = JSON.parse(publish.mock.calls[0][1] as string); + expect(payload).toMatchObject({ + operation: 'insert', + schema: 'Order', + documentId: '64b64c4c4c4c4c4c4c4c4c4c', + }); + expect(payload).not.toHaveProperty('fullDocument'); + expect(payload).not.toHaveProperty('resumeToken'); + expect(payload).not.toHaveProperty('secret'); + const expectedRooms = roomsForPublicChange('Order', '64b64c4c4c4c4c4c4c4c4c4c'); + expect(routerPush).toHaveBeenCalledWith( + expect.objectContaining({ event: 'change', rooms: expectedRooms }), + ); + expect(adminPush).toHaveBeenCalledWith( + expect.objectContaining({ event: 'change', rooms: expectedRooms }), + ); + expect( + JSON.parse((adminPush.mock.calls[0][0] as { data: string }).data), + ).not.toHaveProperty('resumeToken'); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(resume)); + await coordinator.shutdown(); + }); + + it('advances the resume token for filtered events', async () => { + const { coordinator, stream, publish, state } = createCoordinator(); + await coordinator.reconcile(); + const skip = { _data: 'skip-token' }; + stream.emit('change', insertChange('other', '64b64c4c4c4c4c4c4c4c4c4d', skip)); + await coordinator.waitForIdle(); + expect(publish).not.toHaveBeenCalled(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(skip)); + await coordinator.shutdown(); + }); + + it('serializes overlapping handlers and persists after emit', async () => { + const { coordinator, stream, state, adminPush } = createCoordinator(); + let release!: () => void; + const gate = new Promise(resolve => { + release = resolve; + }); + let first = true; + adminPush.mockImplementation(async () => { + if (first) { + first = false; + await gate; + } + }); + await coordinator.reconcile(); + const tokenA = { _data: 'token-a' }; + const tokenB = { _data: 'token-b' }; + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', tokenB)); + await Promise.resolve(); + await new Promise(resolve => setImmediate(resolve)); + expect(state.get('realtime:resumeToken')).toBeUndefined(); + expect(adminPush).toHaveBeenCalledTimes(1); + release(); + await coordinator.waitForIdle(); + expect(adminPush).toHaveBeenCalledTimes(2); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenB)); + await coordinator.shutdown(); + }); + + it('watches opted-in collections with $match and $project', async () => { + const { coordinator, watch } = createCoordinator(); + await coordinator.reconcile(); + expect(watch).toHaveBeenCalledTimes(1); + const pipeline = (watch.mock.calls[0][0] as { pipeline: Record[] }) + .pipeline; + expect(pipeline[0]).toEqual( + expect.objectContaining({ + $match: expect.objectContaining({ + $or: expect.arrayContaining([ + expect.objectContaining({ + 'ns.coll': { $in: ['orders'] }, + }), + ]), + }), + }), + ); + expect(pipeline[1]).toEqual({ + $project: { + fullDocument: 0, + updateDescription: 0, + fullDocumentBeforeChange: 0, + }, + }); + await coordinator.shutdown(); + }); + + it('reopens the watch when the opt-in set changes', async () => { + const schemas = [ + { name: 'Order', collectionName: 'orders', authorizationEnabled: false }, + ]; + const { coordinator, watch } = createCoordinator({ schemas }); + await coordinator.reconcile(); + expect(watch).toHaveBeenCalledTimes(1); + schemas.push({ + name: 'Item', + collectionName: 'items', + authorizationEnabled: false, + }); + await coordinator.reconcile(); + expect(watch).toHaveBeenCalledTimes(2); + const pipeline = (watch.mock.calls[1][0] as { pipeline: Record[] }) + .pipeline; + const match = pipeline[0] as { + $match: { $or: Array<{ 'ns.coll'?: { $in: string[] } }> }; + }; + expect(match.$match.$or[0]['ns.coll']?.$in).toEqual( + expect.arrayContaining(['orders', 'items']), + ); + await coordinator.shutdown(); + }); + + it('re-checks ReBAC before emission and drops revoked users', async () => { + const { coordinator, stream, routerPush, subscriptions } = createCoordinator({ + allow: false, + schemas: [{ name: 'Order', collectionName: 'orders', authorizationEnabled: true }], + }); + await subscriptions.addAuthorizedDocument( + 'sock-1', + 'Order', + '64b64c4c4c4c4c4c4c4c4c4c', + 'user-1', + ); + await coordinator.reconcile(); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), + ); + await coordinator.waitForIdle(); + expect(routerPush).not.toHaveBeenCalled(); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual( + [], + ); + await coordinator.shutdown(); + }); + + it('retries later when the leader lock is held by another instance', async () => { + jest.useFakeTimers(); + const { coordinator, grpcSdk, lock } = createCoordinator(); + grpcSdk.state.tryAcquireLock.mockResolvedValueOnce(null).mockResolvedValue(lock); + await coordinator.reconcile(); + expect(coordinator.getState()).toBe('idle'); + await jest.advanceTimersByTimeAsync(1_000); + expect(coordinator.getState()).toBe('live'); + await coordinator.shutdown(); + jest.useRealTimers(); + }); + + it('opens a single watch when reconcile runs concurrently', async () => { + const { coordinator, watch } = createCoordinator({ getKeyDelayMs: 40 }); + await Promise.all([coordinator.reconcile(), coordinator.reconcile()]); + expect(watch).toHaveBeenCalledTimes(1); + await coordinator.shutdown(); + }); + + it('clears an unusable resume token on 280 and retries', async () => { + const { coordinator, stream, grpcSdk } = createCoordinator(); + await coordinator.reconcile(); + stream.emit('error', { code: 280, message: 'ChangeStreamHistoryLost' }); + await new Promise(resolve => setImmediate(resolve)); + expect(grpcSdk.state.clearKey).toHaveBeenCalled(); + await coordinator.shutdown(); + }); + + it('keeps the resume token on CursorKilled 237', async () => { + const { coordinator, stream, grpcSdk, state } = createCoordinator(); + await coordinator.reconcile(); + const token = { _data: 'keep-me' }; + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', token)); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); + stream.emit('error', { code: 237, message: 'CursorKilled' }); + await new Promise(resolve => setImmediate(resolve)); + expect(grpcSdk.state.clearKey).not.toHaveBeenCalled(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); + await coordinator.shutdown(); + }); + + it('does not persist a later token when emit fails and reopens from the last good token', async () => { + jest.useFakeTimers(); + const streams: Array Promise }> = []; + const { coordinator, adminPush, state, watch } = createCoordinator(); + watch.mockImplementation(() => { + const next = new EventEmitter() as EventEmitter & { close: () => Promise }; + next.close = async () => { + next.emit('close'); + }; + streams.push(next); + return next as never; + }); + adminPush + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error('push failed')) + .mockResolvedValue(undefined); + await coordinator.reconcile(); + const tokenGood = { _data: 'token-good' }; + const tokenA = { _data: 'token-a' }; + const tokenB = { _data: 'token-b' }; + streams[0].emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4b', tokenGood), + ); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenGood)); + streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); + streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', tokenB)); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenGood)); + expect(adminPush).toHaveBeenCalledTimes(2); + await jest.advanceTimersByTimeAsync(1_000); + expect(watch).toHaveBeenCalledTimes(2); + expect((watch.mock.calls[1][0] as { resumeAfter?: unknown }).resumeAfter).toEqual( + tokenGood, + ); + streams[1].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenA)); + expect(adminPush).toHaveBeenCalledTimes(3); + await coordinator.shutdown(); + jest.useRealTimers(); + }); + + it.each(['drop', 'rename', 'invalidate'] as const)( + 'reopens the watch on %s', + async operationType => { + jest.useFakeTimers(); + const streams: Array Promise }> = []; + const { coordinator, watch, state } = createCoordinator(); + watch.mockImplementation(() => { + const next = new EventEmitter() as EventEmitter & { close: () => Promise }; + next.close = async () => { + next.emit('close'); + }; + streams.push(next); + return next as never; + }); + await coordinator.reconcile(); + const token = { _data: `${operationType}-token` }; + streams[0].emit('change', { + operationType, + ns: { coll: 'orders' }, + _id: token, + }); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); + await jest.advanceTimersByTimeAsync(1_000); + expect(watch).toHaveBeenCalledTimes(2); + expect((watch.mock.calls[1][0] as { resumeAfter?: unknown }).resumeAfter).toEqual( + token, + ); + await coordinator.shutdown(); + jest.useRealTimers(); + }, + ); + + it('does not remove users when authorization is unavailable', async () => { + const { coordinator, stream, routerPush, subscriptions } = createCoordinator({ + authorizationAvailable: false, + schemas: [{ name: 'Order', collectionName: 'orders', authorizationEnabled: true }], + }); + const removeUser = jest.spyOn(subscriptions, 'removeUser'); + await subscriptions.addAuthorizedDocument( + 'sock-1', + 'Order', + '64b64c4c4c4c4c4c4c4c4c4c', + 'user-1', + ); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual([ + 'user-1', + ]); + await coordinator.reconcile(); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), + ); + await coordinator.waitForIdle(); + expect(routerPush).not.toHaveBeenCalled(); + expect(removeUser).not.toHaveBeenCalled(); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual([ + 'user-1', + ]); + await coordinator.shutdown(); + }); +}); diff --git a/modules/database/src/realtime/__tests__/normalize.test.ts b/modules/database/src/realtime/__tests__/normalize.test.ts new file mode 100644 index 000000000..f7f933d33 --- /dev/null +++ b/modules/database/src/realtime/__tests__/normalize.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from '@jest/globals'; +import { ObjectId } from 'bson'; +import { + normalizeChangeEvent, + parseResumeToken, + serializeResumeToken, +} from '../normalize.js'; + +describe('normalizeChangeEvent', () => { + it('normalizes insert/update/replace/delete into metadata-only events', () => { + const resume = { _data: 'token-1' }; + const event = normalizeChangeEvent( + { + operationType: 'insert', + documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, + fullDocument: { secret: 'nope' }, + wallTime: new Date('2026-01-01T00:00:00.000Z'), + _id: resume, + }, + 'Order', + ); + expect(event).toMatchObject({ + version: 1, + operation: 'insert', + schema: 'Order', + documentId: '64b64c4c4c4c4c4c4c4c4c4c', + occurredAt: '2026-01-01T00:00:00.000Z', + }); + expect(parseResumeToken(serializeResumeToken(resume))).toEqual(resume); + expect(event).not.toHaveProperty('resumeToken'); + expect(JSON.parse(JSON.stringify(event))).not.toHaveProperty('fullDocument'); + }); + + it('ignores drop/invalidate and missing document ids', () => { + expect( + normalizeChangeEvent({ operationType: 'drop', _id: { _data: 'x' } }, 'Order'), + ).toBeNull(); + expect( + normalizeChangeEvent({ operationType: 'insert', _id: { _data: 'x' } }, 'Order'), + ).toBeNull(); + }); +}); diff --git a/modules/database/src/realtime/__tests__/recovery.test.ts b/modules/database/src/realtime/__tests__/recovery.test.ts new file mode 100644 index 000000000..eee45b918 --- /dev/null +++ b/modules/database/src/realtime/__tests__/recovery.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from '@jest/globals'; +import { canReadDocument } from '../authorize.js'; +import { authorizedDocumentRoom } from '../rooms.js'; +import { isRecoverableDisconnect, restoreAuthorizedSubscriptions } from '../recovery.js'; +import { createSocketHandlers } from '../sockets.js'; +import { RealtimeSubscriptionTracker } from '../subscriptions.js'; + +class MemoryStore { + private sets = new Map>(); + async sadd(key: string, ...members: string[]) { + const set = this.sets.get(key) ?? new Set(); + members.forEach(member => set.add(member)); + this.sets.set(key, set); + } + async srem(key: string, ...members: string[]) { + members.forEach(member => this.sets.get(key)?.delete(member)); + } + async smembers(key: string) { + return [...(this.sets.get(key) ?? new Set())]; + } + async scard(key: string) { + return this.sets.get(key)?.size ?? 0; + } + async del(...keys: string[]) { + keys.forEach(key => this.sets.delete(key)); + } +} + +describe('database socket recovery', () => { + it('does not treat transport close as a wipe', () => { + expect(isRecoverableDisconnect('transport close')).toBe(true); + expect(isRecoverableDisconnect('client namespace disconnect')).toBe(false); + }); + + it('restores authorized Redis membership after a recoverable disconnect', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + const room = authorizedDocumentRoom('Order', 'doc-1', 'user-1'); + await tracker.addAuthorizedDocument('sock-1', 'Order', 'doc-1', 'user-1'); + await tracker.disconnect('sock-1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); + + await restoreAuthorizedSubscriptions({ + socketId: 'sock-1', + rooms: [room], + contextSubs: [], + subscriptions: tracker, + grpcSdk: { + isAvailable: () => true, + authorization: { can: async () => ({ allow: true }) }, + }, + canRead: canReadDocument, + }); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + }); + + it('keeps recovered /database/ sockets in the authorized list', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + await tracker.addAuthorizedDocument('sock-1', 'Order', 'doc-1', 'user-1'); + const handlers = createSocketHandlers({ + mode: 'client', + grpcSdk: { + isAvailable: () => true, + authorization: { can: async () => ({ allow: true }) }, + } as never, + schemaLookup: { getSchema: () => undefined }, + subscriptions: tracker, + isGloballyEnabled: () => true, + }); + await handlers.disconnect({ + request: { socketId: 'sock-1', params: ['transport close'] }, + } as never); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + await handlers.recovered({ + request: { + socketId: 'sock-1', + params: [authorizedDocumentRoom('Order', 'doc-1', 'user-1')], + context: {}, + }, + } as never); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + }); +}); diff --git a/modules/database/src/realtime/__tests__/rooms.test.ts b/modules/database/src/realtime/__tests__/rooms.test.ts new file mode 100644 index 000000000..96de67f73 --- /dev/null +++ b/modules/database/src/realtime/__tests__/rooms.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from '@jest/globals'; +import { + authorizedDocumentRoom, + documentRoom, + parseAuthorizedDocumentRoom, + roomsForPublicChange, + schemaRoom, +} from '../rooms.js'; + +describe('realtime rooms', () => { + it('isolates schema, document, and per-user rooms', () => { + expect(schemaRoom('Order')).toBe('database:schema:Order'); + expect(documentRoom('Order', 'abc')).toBe('database:doc:Order:abc'); + expect(authorizedDocumentRoom('Order', 'abc', 'user-1')).toBe( + 'database:doc:Order:abc:user:user-1', + ); + expect(roomsForPublicChange('Order', 'abc')).toEqual([ + 'database:schema:Order', + 'database:doc:Order:abc', + ]); + expect(schemaRoom('Order')).not.toBe(schemaRoom('Orders')); + expect(authorizedDocumentRoom('Order', 'abc', 'u1')).not.toBe( + authorizedDocumentRoom('Order', 'abc', 'u2'), + ); + expect( + parseAuthorizedDocumentRoom(authorizedDocumentRoom('Order', 'abc', 'user-1')), + ).toEqual({ + schema: 'Order', + documentId: 'abc', + userId: 'user-1', + }); + }); + + it('encodes reserved characters so rooms cannot collide', () => { + expect(schemaRoom('a/b')).toBe('database:schema:a%2Fb'); + expect(documentRoom('Order', 'id:1')).toBe('database:doc:Order:id%3A1'); + }); +}); diff --git a/modules/database/src/realtime/__tests__/status.test.ts b/modules/database/src/realtime/__tests__/status.test.ts new file mode 100644 index 000000000..a94790cfb --- /dev/null +++ b/modules/database/src/realtime/__tests__/status.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from '@jest/globals'; +import { buildRealtimeStatus } from '../status.js'; + +describe('buildRealtimeStatus', () => { + it('reports unsupported, disabled, idle, live, and degraded states', () => { + expect( + buildRealtimeStatus({ + engine: 'PostgreSQL', + enabled: true, + topologySupported: true, + activeSchemaCount: 1, + streamState: 'live', + }).status, + ).toBe('unsupported'); + expect( + buildRealtimeStatus({ + engine: 'MongoDB', + enabled: false, + topologySupported: true, + activeSchemaCount: 1, + streamState: 'live', + }).status, + ).toBe('disabled'); + expect( + buildRealtimeStatus({ + engine: 'MongoDB', + enabled: true, + topologySupported: false, + activeSchemaCount: 1, + streamState: 'idle', + }).message, + ).toMatch(/replica set/i); + expect( + buildRealtimeStatus({ + engine: 'MongoDB', + enabled: true, + topologySupported: true, + activeSchemaCount: 2, + streamState: 'live', + }).status, + ).toBe('live'); + expect( + buildRealtimeStatus({ + engine: 'MongoDB', + enabled: true, + topologySupported: true, + activeSchemaCount: 1, + streamState: 'degraded', + lastError: 'boom', + }), + ).toMatchObject({ status: 'degraded', message: 'boom' }); + }); +}); diff --git a/modules/database/src/realtime/__tests__/subscriptions.test.ts b/modules/database/src/realtime/__tests__/subscriptions.test.ts new file mode 100644 index 000000000..45ee7d1c9 --- /dev/null +++ b/modules/database/src/realtime/__tests__/subscriptions.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from '@jest/globals'; +import { RealtimeSubscriptionTracker } from '../subscriptions.js'; + +class MemoryStore { + private sets = new Map>(); + + async sadd(key: string, ...members: string[]) { + const set = this.sets.get(key) ?? new Set(); + members.forEach(member => set.add(member)); + this.sets.set(key, set); + return members.length; + } + + async srem(key: string, ...members: string[]) { + const set = this.sets.get(key); + if (!set) return 0; + members.forEach(member => set.delete(member)); + return members.length; + } + + async smembers(key: string) { + return [...(this.sets.get(key) ?? new Set())]; + } + + async scard(key: string) { + return this.sets.get(key)?.size ?? 0; + } + + async del(...keys: string[]) { + keys.forEach(key => this.sets.delete(key)); + return keys.length; + } +} + +describe('RealtimeSubscriptionTracker', () => { + it('tracks users per document and cleans up on disconnect', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); + await tracker.addAuthorizedDocument('s2', 'Order', 'doc-1', 'user-1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + + await tracker.disconnect('s1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + + await tracker.disconnect('s2'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); + }); + + it('removes revoked users from the document set', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); + await tracker.removeUser('Order', 'doc-1', 'user-1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); + }); +}); diff --git a/modules/database/src/realtime/__tests__/topology.test.ts b/modules/database/src/realtime/__tests__/topology.test.ts new file mode 100644 index 000000000..79afac87c --- /dev/null +++ b/modules/database/src/realtime/__tests__/topology.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from '@jest/globals'; +import { isResumeTokenUnusable, topologyFromHello } from '../topology.js'; + +describe('topology helpers', () => { + it('accepts replica sets and mongos, rejects standalone', () => { + expect(topologyFromHello({ setName: 'rs0' })).toEqual({ supported: true }); + expect(topologyFromHello({ msg: 'isdbgrid' })).toEqual({ supported: true }); + expect(topologyFromHello({}).supported).toBe(false); + }); + + it('detects unusable resume tokens', () => { + expect(isResumeTokenUnusable({ code: 280 })).toBe(true); + expect(isResumeTokenUnusable({ code: 237 })).toBe(false); + expect(isResumeTokenUnusable({ code: 136 })).toBe(false); + expect(isResumeTokenUnusable(new Error('ChangeStreamHistoryLost'))).toBe(true); + expect(isResumeTokenUnusable(new Error('cannot resume'))).toBe(false); + expect(isResumeTokenUnusable(new Error('socket hang up'))).toBe(false); + }); +}); diff --git a/modules/database/src/realtime/__tests__/watchPipeline.test.ts b/modules/database/src/realtime/__tests__/watchPipeline.test.ts new file mode 100644 index 000000000..7c31a64f5 --- /dev/null +++ b/modules/database/src/realtime/__tests__/watchPipeline.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from '@jest/globals'; +import { buildWatchPipeline } from '../watchPipeline.js'; + +describe('buildWatchPipeline', () => { + it('matches opted-in collections and projects out fullDocument', () => { + const pipeline = buildWatchPipeline(['orders', 'items']); + expect(pipeline[0]).toEqual({ + $match: { + $or: [ + { + operationType: { $in: ['insert', 'update', 'replace', 'delete'] }, + 'ns.coll': { $in: ['orders', 'items'] }, + }, + { + operationType: { $in: ['drop', 'rename', 'invalidate', 'dropDatabase'] }, + }, + ], + }, + }); + expect(pipeline[1]).toEqual({ + $project: { + fullDocument: 0, + updateDescription: 0, + fullDocumentBeforeChange: 0, + }, + }); + }); +}); diff --git a/modules/database/src/realtime/authorize.ts b/modules/database/src/realtime/authorize.ts new file mode 100644 index 000000000..cf7ed899c --- /dev/null +++ b/modules/database/src/realtime/authorize.ts @@ -0,0 +1,136 @@ +import { status } from '@grpc/grpc-js'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import type { OptedInSchema, SubscribeRequest } from './types.js'; + +export class RealtimeSubscriptionError extends GrpcError { + constructor(code: number, message: string) { + super(code, message); + this.name = 'RealtimeSubscriptionError'; + } +} + +export type AuthorizationSdk = { + isAvailable: (module: string) => boolean; + authorization?: { + can: (request: { + subject: string; + actions: string[]; + resource: string; + }) => Promise<{ allow: boolean }>; + } | null; +}; + +export type SchemaLookup = { + getSchema(name: string): + | { + name: string; + modelOptions?: { + conduit?: { + realtime?: { enabled?: boolean }; + cms?: { crudOperations?: { read?: { enabled?: boolean } } }; + authorization?: { enabled?: boolean }; + }; + }; + } + | undefined; +}; + +export function parseSubscribeRequest(params: unknown[]): SubscribeRequest { + const raw = params[0]; + if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + return raw as SubscribeRequest; + } + return {}; +} + +export function requireSchemaName(value: unknown): string { + if (typeof value !== 'string' || value.trim() === '') { + throw new RealtimeSubscriptionError(status.INVALID_ARGUMENT, 'schema is required'); + } + return value.trim(); +} + +export function optionalDocumentId(value: unknown): string | undefined { + if (value === undefined || value === null || value === '') return undefined; + if (typeof value !== 'string') { + throw new RealtimeSubscriptionError( + status.INVALID_ARGUMENT, + 'documentId must be a string', + ); + } + return value; +} + +export async function canReadDocument( + grpcSdk: AuthorizationSdk, + schema: string, + documentId: string, + userId: string, +): Promise { + if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { + return false; + } + try { + const decision = await grpcSdk.authorization.can({ + subject: `User:${userId}`, + actions: ['read'], + resource: `${schema}:${documentId}`, + }); + return decision.allow === true; + } catch { + return false; + } +} + +export function assertSchemaAvailable( + lookup: SchemaLookup, + schemaName: string, + requireCmsRead: boolean, +): { authorizationEnabled: boolean } { + let schema: ReturnType; + try { + schema = lookup.getSchema(schemaName); + } catch (err) { + if (err instanceof GrpcError && err.code === status.NOT_FOUND) { + throw new RealtimeSubscriptionError(status.NOT_FOUND, 'Schema does not exist'); + } + throw err; + } + if (!schema) { + throw new RealtimeSubscriptionError(status.NOT_FOUND, 'Schema does not exist'); + } + const conduit = schema.modelOptions?.conduit; + if (!conduit?.realtime?.enabled) { + throw new RealtimeSubscriptionError( + status.FAILED_PRECONDITION, + 'Live updates are not enabled for this schema', + ); + } + if (requireCmsRead && conduit.cms?.crudOperations?.read?.enabled !== true) { + throw new RealtimeSubscriptionError( + status.PERMISSION_DENIED, + 'CMS read is not enabled for this schema', + ); + } + return { + authorizationEnabled: conduit.authorization?.enabled === true, + }; +} + +export function toOptedInSchema(schema: { + name: string; + collectionName: string; + modelOptions?: { + conduit?: { + realtime?: { enabled?: boolean }; + authorization?: { enabled?: boolean }; + }; + }; +}): OptedInSchema | null { + if (!schema.modelOptions?.conduit?.realtime?.enabled) return null; + return { + name: schema.name, + collectionName: schema.collectionName, + authorizationEnabled: schema.modelOptions.conduit.authorization?.enabled === true, + }; +} diff --git a/modules/database/src/realtime/index.ts b/modules/database/src/realtime/index.ts new file mode 100644 index 000000000..655c04bba --- /dev/null +++ b/modules/database/src/realtime/index.ts @@ -0,0 +1,11 @@ +export { RealtimeService } from './RealtimeService.js'; +export { buildRealtimeStatus } from './status.js'; +export { normalizeChangeEvent } from './normalize.js'; +export { + schemaRoom, + documentRoom, + authorizedDocumentRoom, + roomsForPublicChange, +} from './rooms.js'; +export { RealtimeSubscriptionTracker } from './subscriptions.js'; +export type { RealtimeStatus, DatabaseChangeEvent } from './types.js'; diff --git a/modules/database/src/realtime/normalize.ts b/modules/database/src/realtime/normalize.ts new file mode 100644 index 000000000..b2f4b8ff3 --- /dev/null +++ b/modules/database/src/realtime/normalize.ts @@ -0,0 +1,78 @@ +import { EJSON } from 'bson'; +import { + DATABASE_CHANGE_EVENT_VERSION, + DATABASE_CHANGE_OPERATIONS, + type DatabaseChangeEvent, + type DatabaseChangeOperation, +} from './types.js'; + +const OPERATION_SET = new Set(DATABASE_CHANGE_OPERATIONS); + +export type RawChangeEvent = { + operationType?: string; + ns?: { coll?: string }; + documentKey?: { _id?: unknown }; + wallTime?: Date; + clusterTime?: { toString?: () => string }; + _id?: unknown; +}; + +export function normalizeChangeEvent( + change: RawChangeEvent, + schemaName: string, + occurredAt: Date = new Date(), +): DatabaseChangeEvent | null { + const operation = change.operationType; + if (!operation || !OPERATION_SET.has(operation)) { + return null; + } + const documentId = extractDocumentId(change.documentKey?._id); + if (!documentId) { + return null; + } + if (!change._id) { + return null; + } + return { + version: DATABASE_CHANGE_EVENT_VERSION, + operation: operation as DatabaseChangeOperation, + schema: schemaName, + documentId, + occurredAt: (change.wallTime instanceof Date + ? change.wallTime + : occurredAt + ).toISOString(), + }; +} + +export function serializeResumeToken(id: unknown): string | null { + if (id === undefined || id === null) return null; + try { + return EJSON.stringify(id); + } catch { + return null; + } +} + +export function parseResumeToken(token: string | null | undefined): unknown | undefined { + if (!token) return undefined; + try { + return EJSON.parse(token); + } catch { + return undefined; + } +} + +function extractDocumentId(id: unknown): string | null { + if (id === undefined || id === null) return null; + if (typeof id === 'string' || typeof id === 'number') return String(id); + if (typeof id === 'object' && id !== null && 'toHexString' in id) { + const hex = (id as { toHexString: () => string }).toHexString(); + return typeof hex === 'string' && hex.length > 0 ? hex : null; + } + if (typeof id === 'object' && id !== null && 'toString' in id) { + const value = String(id); + return value && value !== '[object Object]' ? value : null; + } + return null; +} diff --git a/modules/database/src/realtime/rebacCache.ts b/modules/database/src/realtime/rebacCache.ts new file mode 100644 index 000000000..b33a9438c --- /dev/null +++ b/modules/database/src/realtime/rebacCache.ts @@ -0,0 +1,96 @@ +import type { AuthorizationSdk } from './authorize.js'; + +export type RebacDecision = 'allow' | 'deny' | 'unavailable'; + +const DEFAULT_TTL_MS = 12_000; +const DEFAULT_MAX_ENTRIES = 10_000; + +type CacheEntry = { + decision: 'allow' | 'deny'; + expiresAt: number; +}; + +export class RealtimeRebacCache { + private readonly entries = new Map(); + + constructor( + private readonly ttlMs: number = DEFAULT_TTL_MS, + private readonly maxEntries: number = DEFAULT_MAX_ENTRIES, + ) {} + + async check( + grpcSdk: AuthorizationSdk, + userId: string, + schema: string, + documentId: string, + ): Promise { + const resource = `${schema}:${documentId}`; + const key = `${userId}:read:${resource}`; + const now = Date.now(); + this.sweepExpired(now); + const cached = this.entries.get(key); + if (cached && cached.expiresAt > now) { + return cached.decision; + } + if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { + return 'unavailable'; + } + try { + const decision = await grpcSdk.authorization.can({ + subject: `User:${userId}`, + actions: ['read'], + resource, + }); + const value: 'allow' | 'deny' = decision.allow ? 'allow' : 'deny'; + this.set(key, value, now); + return value; + } catch { + return 'unavailable'; + } + } + + clear(): void { + this.entries.clear(); + } + + private set(key: string, decision: 'allow' | 'deny', now: number): void { + if (this.entries.size >= this.maxEntries && !this.entries.has(key)) { + const firstKey = this.entries.keys().next().value; + if (firstKey) { + this.entries.delete(firstKey); + } + } + this.entries.set(key, { decision, expiresAt: now + this.ttlMs }); + } + + private sweepExpired(now: number): void { + for (const [key, entry] of this.entries) { + if (entry.expiresAt <= now) { + this.entries.delete(key); + } + } + } +} + +export async function checkRebacBatch( + cache: RealtimeRebacCache, + grpcSdk: AuthorizationSdk, + userIds: string[], + schema: string, + documentId: string, + concurrency = 8, +): Promise> { + const results = new Map(); + let index = 0; + async function worker(): Promise { + while (index < userIds.length) { + const userId = userIds[index++]; + results.set(userId, await cache.check(grpcSdk, userId, schema, documentId)); + } + } + const workers = Array.from({ length: Math.min(concurrency, userIds.length) }, () => + worker(), + ); + await Promise.all(workers); + return results; +} diff --git a/modules/database/src/realtime/recovery.ts b/modules/database/src/realtime/recovery.ts new file mode 100644 index 000000000..3bc9c3868 --- /dev/null +++ b/modules/database/src/realtime/recovery.ts @@ -0,0 +1,147 @@ +import type { Indexable, ParsedSocketRequest } from '@conduitplatform/grpc-sdk'; +import type { AuthorizationSdk } from './authorize.js'; +import { authorizedDocumentRoom, parseAuthorizedDocumentRoom } from './rooms.js'; +import type { RealtimeSubscriptionTracker } from './subscriptions.js'; + +const RECOVERABLE_DISCONNECT = new Set([ + 'ping timeout', + 'transport close', + 'transport error', +]); + +const CONTEXT_KEY = 'databaseSubs'; + +export type AuthorizedSub = { + schema: string; + documentId: string; + userId: string; +}; + +export function isRecoverableDisconnect(reason: unknown): boolean { + return typeof reason === 'string' && RECOVERABLE_DISCONNECT.has(reason); +} + +export function authorizedSubsFromContext( + context: Indexable | undefined, +): AuthorizedSub[] { + if (!context) return []; + const raw = context[CONTEXT_KEY]; + if (!Array.isArray(raw)) return []; + const subs: AuthorizedSub[] = []; + for (const item of raw) { + if ( + item && + typeof item === 'object' && + typeof (item as AuthorizedSub).schema === 'string' && + typeof (item as AuthorizedSub).documentId === 'string' && + typeof (item as AuthorizedSub).userId === 'string' + ) { + subs.push({ + schema: (item as AuthorizedSub).schema, + documentId: (item as AuthorizedSub).documentId, + userId: (item as AuthorizedSub).userId, + }); + } + } + return subs; +} + +export function persistAuthorizedSubOnContext( + context: Indexable | undefined, + schema: string, + documentId: string, + userId: string, +): void { + if (!context) return; + const next = authorizedSubsFromContext(context).filter( + sub => + !(sub.schema === schema && sub.documentId === documentId && sub.userId === userId), + ); + next.push({ schema, documentId, userId }); + context[CONTEXT_KEY] = next; +} + +export function removeAuthorizedSubFromContext( + context: Indexable | undefined, + schema: string, + documentId: string, + userId: string, +): void { + if (!context) return; + const next = authorizedSubsFromContext(context).filter( + sub => + !(sub.schema === schema && sub.documentId === documentId && sub.userId === userId), + ); + if (next.length === 0) { + delete context[CONTEXT_KEY]; + } else { + context[CONTEXT_KEY] = next; + } +} + +export function recoveredRoomsFromRequest(call: ParsedSocketRequest): string[] { + const params = call.request.params ?? []; + if (params.every(item => typeof item === 'string')) { + return params as string[]; + } + return []; +} + +export async function restoreAuthorizedSubscriptions(options: { + socketId: string; + rooms: string[]; + contextSubs: AuthorizedSub[]; + subscriptions: RealtimeSubscriptionTracker; + grpcSdk: AuthorizationSdk; + canRead: ( + grpcSdk: AuthorizationSdk, + schema: string, + documentId: string, + userId: string, + ) => Promise; +}): Promise<{ leaveRooms: string[] }> { + const seen = new Set(); + const subs: AuthorizedSub[] = []; + for (const room of options.rooms) { + const parsed = parseAuthorizedDocumentRoom(room); + if (!parsed) continue; + const key = `${parsed.schema}:${parsed.documentId}:${parsed.userId}`; + if (seen.has(key)) continue; + seen.add(key); + subs.push(parsed); + } + for (const sub of options.contextSubs) { + const key = `${sub.schema}:${sub.documentId}:${sub.userId}`; + if (seen.has(key)) continue; + seen.add(key); + subs.push(sub); + } + + const leaveRooms: string[] = []; + for (const sub of subs) { + const allowed = await options.canRead( + options.grpcSdk, + sub.schema, + sub.documentId, + sub.userId, + ); + const room = authorizedDocumentRoom(sub.schema, sub.documentId, sub.userId); + if (!allowed) { + await options.subscriptions.removeAuthorizedDocument( + options.socketId, + sub.schema, + sub.documentId, + sub.userId, + ); + leaveRooms.push(room); + continue; + } + await options.subscriptions.addAuthorizedDocument( + options.socketId, + sub.schema, + sub.documentId, + sub.userId, + ); + } + return { leaveRooms }; +} diff --git a/modules/database/src/realtime/rooms.ts b/modules/database/src/realtime/rooms.ts new file mode 100644 index 000000000..5792482f9 --- /dev/null +++ b/modules/database/src/realtime/rooms.ts @@ -0,0 +1,47 @@ +const ROOM_PREFIX = 'database'; + +function encodeSegment(value: string): string { + return encodeURIComponent(value); +} + +export function schemaRoom(schema: string): string { + return `${ROOM_PREFIX}:schema:${encodeSegment(schema)}`; +} + +export function documentRoom(schema: string, documentId: string): string { + return `${ROOM_PREFIX}:doc:${encodeSegment(schema)}:${encodeSegment(documentId)}`; +} + +export function authorizedDocumentRoom( + schema: string, + documentId: string, + userId: string, +): string { + return `${ROOM_PREFIX}:doc:${encodeSegment(schema)}:${encodeSegment(documentId)}:user:${encodeSegment(userId)}`; +} + +export function roomsForPublicChange(schema: string, documentId: string): string[] { + return [schemaRoom(schema), documentRoom(schema, documentId)]; +} + +export function parseAuthorizedDocumentRoom(room: string): { + schema: string; + documentId: string; + userId: string; +} | null { + const prefix = `${ROOM_PREFIX}:doc:`; + if (!room.startsWith(prefix)) return null; + const rest = room.slice(prefix.length); + const userMarker = ':user:'; + const userIndex = rest.lastIndexOf(userMarker); + if (userIndex === -1) return null; + const userId = decodeURIComponent(rest.slice(userIndex + userMarker.length)); + const schemaDoc = rest.slice(0, userIndex); + const lastColon = schemaDoc.lastIndexOf(':'); + if (lastColon === -1) return null; + return { + schema: decodeURIComponent(schemaDoc.slice(0, lastColon)), + documentId: decodeURIComponent(schemaDoc.slice(lastColon + 1)), + userId, + }; +} diff --git a/modules/database/src/realtime/sockets.ts b/modules/database/src/realtime/sockets.ts new file mode 100644 index 000000000..e2a80931c --- /dev/null +++ b/modules/database/src/realtime/sockets.ts @@ -0,0 +1,195 @@ +import { + ConduitGrpcSdk, + ConduitRouteReturnDefinition, + type Indexable, + ParsedSocketRequest, + TYPE, + UnparsedSocketResponse, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { RoutingManager } from '@conduitplatform/module-tools'; +import { + assertSchemaAvailable, + canReadDocument, + optionalDocumentId, + parseSubscribeRequest, + RealtimeSubscriptionError, + requireSchemaName, + type AuthorizationSdk, + type SchemaLookup, +} from './authorize.js'; +import { authorizedDocumentRoom, documentRoom, schemaRoom } from './rooms.js'; +import type { RealtimeSubscriptionTracker } from './subscriptions.js'; +import { + authorizedSubsFromContext, + isRecoverableDisconnect, + persistAuthorizedSubOnContext, + recoveredRoomsFromRequest, + removeAuthorizedSubFromContext, + restoreAuthorizedSubscriptions, +} from './recovery.js'; + +type SocketMode = 'client' | 'admin'; + +type RealtimeSocketOptions = { + mode: SocketMode; + grpcSdk: ConduitGrpcSdk; + schemaLookup: SchemaLookup; + subscriptions: RealtimeSubscriptionTracker; + isGloballyEnabled: () => boolean; +}; + +export function registerDatabaseRealtimeSocket( + routingManager: RoutingManager, + options: RealtimeSocketOptions, +) { + const handlers = createSocketHandlers(options); + routingManager.socket( + { + path: '/', + middlewares: options.mode === 'client' ? ['authMiddleware'] : undefined, + }, + { + connect: { handler: handlers.connect }, + disconnect: { handler: handlers.disconnect }, + recovered: { handler: handlers.recovered }, + subscribe: { + params: [TYPE.JSON], + handler: handlers.subscribe, + returnType: new ConduitRouteReturnDefinition('DatabaseRealtimeSubscribe', { + rooms: [TYPE.String], + }), + }, + unsubscribe: { + params: [TYPE.JSON], + handler: handlers.unsubscribe, + returnType: new ConduitRouteReturnDefinition('DatabaseRealtimeUnsubscribe', { + rooms: [TYPE.String], + }), + }, + }, + ); +} + +export function createSocketHandlers(options: RealtimeSocketOptions) { + return { + connect: async (): Promise => { + return { event: 'connected', data: { ok: true } }; + }, + disconnect: async (call: ParsedSocketRequest): Promise => { + const reason = call.request.params?.[0]; + if (!isRecoverableDisconnect(reason)) { + await options.subscriptions.disconnect(call.request.socketId); + } + return { event: 'disconnected', data: { ok: true } }; + }, + recovered: async (call: ParsedSocketRequest): Promise => { + const { leaveRooms } = await restoreAuthorizedSubscriptions({ + socketId: call.request.socketId, + rooms: recoveredRoomsFromRequest(call), + contextSubs: authorizedSubsFromContext(call.request.context as Indexable), + subscriptions: options.subscriptions, + grpcSdk: options.grpcSdk as unknown as AuthorizationSdk, + canRead: canReadDocument, + }); + if (leaveRooms.length > 0) { + return { event: 'leave-room', rooms: leaveRooms }; + } + return { event: 'join-room', rooms: [] }; + }, + subscribe: async (call: ParsedSocketRequest): Promise => { + const rooms = await resolveSubscription(call, options, 'join'); + return { event: 'join-room', rooms }; + }, + unsubscribe: async (call: ParsedSocketRequest): Promise => { + const rooms = await resolveSubscription(call, options, 'leave'); + return { event: 'leave-room', rooms }; + }, + }; +} + +async function resolveSubscription( + call: ParsedSocketRequest, + options: RealtimeSocketOptions, + action: 'join' | 'leave', +): Promise { + if (!options.isGloballyEnabled()) { + throw new RealtimeSubscriptionError( + status.FAILED_PRECONDITION, + 'Live updates are disabled', + ); + } + const payload = parseSubscribeRequest(call.request.params ?? []); + const schemaName = requireSchemaName(payload.schema); + const documentId = optionalDocumentId(payload.documentId); + const meta = assertSchemaAvailable( + options.schemaLookup, + schemaName, + options.mode === 'client', + ); + + if (options.mode === 'admin') { + if (documentId) return [documentRoom(schemaName, documentId)]; + return [schemaRoom(schemaName)]; + } + + const userId = call.request.context?.user?._id; + if (!userId) { + throw new RealtimeSubscriptionError( + status.UNAUTHENTICATED, + 'Authentication required', + ); + } + + if (meta.authorizationEnabled) { + if (!documentId) { + throw new RealtimeSubscriptionError( + status.PERMISSION_DENIED, + 'Document ID is required for authorized schemas', + ); + } + const allowed = await canReadDocument( + options.grpcSdk as unknown as AuthorizationSdk, + schemaName, + documentId, + userId, + ); + if (!allowed) { + throw new RealtimeSubscriptionError( + status.PERMISSION_DENIED, + 'Read permission denied', + ); + } + if (action === 'join') { + await options.subscriptions.addAuthorizedDocument( + call.request.socketId, + schemaName, + documentId, + userId, + ); + persistAuthorizedSubOnContext( + call.request.context as Indexable, + schemaName, + documentId, + userId, + ); + } else { + await options.subscriptions.removeAuthorizedDocument( + call.request.socketId, + schemaName, + documentId, + userId, + ); + removeAuthorizedSubFromContext( + call.request.context as Indexable, + schemaName, + documentId, + userId, + ); + } + return [authorizedDocumentRoom(schemaName, documentId, userId)]; + } + + if (documentId) return [documentRoom(schemaName, documentId)]; + return [schemaRoom(schemaName)]; +} diff --git a/modules/database/src/realtime/status.ts b/modules/database/src/realtime/status.ts new file mode 100644 index 000000000..b1da21b03 --- /dev/null +++ b/modules/database/src/realtime/status.ts @@ -0,0 +1,74 @@ +import type { RealtimeStatus, RealtimeStatusCode } from './types.js'; + +export type RealtimeStatusInput = { + engine: string; + enabled: boolean; + topologySupported: boolean; + topologyMessage?: string; + activeSchemaCount: number; + streamState: RealtimeStatusCode; + lastEventAt?: string; + lastError?: string; + socketsEnabled?: boolean; +}; + +export function buildRealtimeStatus(input: RealtimeStatusInput): RealtimeStatus { + if (input.engine !== 'MongoDB') { + return { + status: 'unsupported', + engine: input.engine, + activeSchemaCount: 0, + message: 'Live updates require MongoDB', + }; + } + if (!input.enabled) { + return { + status: 'disabled', + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + }; + } + if (!input.topologySupported) { + return { + status: 'idle', + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + message: + input.topologyMessage ?? + 'A replica set or sharded MongoDB deployment is required for live updates', + }; + } + if (input.socketsEnabled === false) { + return { + status: 'idle', + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + message: + 'Enable Admin socket transport to consume live updates in the control panel', + lastEventAt: input.lastEventAt, + }; + } + if (input.activeSchemaCount === 0) { + return { + status: 'idle', + engine: input.engine, + activeSchemaCount: 0, + message: 'No schemas have live updates enabled', + }; + } + if (input.streamState === 'degraded') { + return { + status: 'degraded', + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + lastEventAt: input.lastEventAt, + message: input.lastError ?? 'Change stream is retrying after an error', + }; + } + return { + status: input.streamState, + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + lastEventAt: input.lastEventAt, + }; +} diff --git a/modules/database/src/realtime/subscriptions.ts b/modules/database/src/realtime/subscriptions.ts new file mode 100644 index 000000000..da7bfb867 --- /dev/null +++ b/modules/database/src/realtime/subscriptions.ts @@ -0,0 +1,92 @@ +export type SubscriptionStore = { + sadd(key: string, ...members: string[]): Promise; + srem(key: string, ...members: string[]): Promise; + smembers(key: string): Promise; + scard(key: string): Promise; + del(...keys: string[]): Promise; +}; + +function socketKey(socketId: string): string { + return `realtime:socket:${socketId}`; +} + +function docUsersKey(schema: string, documentId: string): string { + return `realtime:doc:${schema}:${documentId}`; +} + +function userDocSocketsKey(schema: string, documentId: string, userId: string): string { + return `realtime:userdoc:${schema}:${documentId}:${userId}`; +} + +function subscriptionRecord(schema: string, documentId: string, userId: string): string { + return JSON.stringify({ schema, documentId, userId }); +} + +export class RealtimeSubscriptionTracker { + constructor(private readonly store: SubscriptionStore) {} + + async addAuthorizedDocument( + socketId: string, + schema: string, + documentId: string, + userId: string, + ): Promise { + await this.store.sadd( + socketKey(socketId), + subscriptionRecord(schema, documentId, userId), + ); + await this.store.sadd(userDocSocketsKey(schema, documentId, userId), socketId); + await this.store.sadd(docUsersKey(schema, documentId), userId); + } + + async removeAuthorizedDocument( + socketId: string, + schema: string, + documentId: string, + userId: string, + ): Promise { + await this.store.srem( + socketKey(socketId), + subscriptionRecord(schema, documentId, userId), + ); + await this.store.srem(userDocSocketsKey(schema, documentId, userId), socketId); + const remaining = await this.store.scard( + userDocSocketsKey(schema, documentId, userId), + ); + if (remaining === 0) { + await this.store.srem(docUsersKey(schema, documentId), userId); + await this.store.del(userDocSocketsKey(schema, documentId, userId)); + } + } + + async listUsers(schema: string, documentId: string): Promise { + return this.store.smembers(docUsersKey(schema, documentId)); + } + + async removeUser(schema: string, documentId: string, userId: string): Promise { + await this.store.srem(docUsersKey(schema, documentId), userId); + await this.store.del(userDocSocketsKey(schema, documentId, userId)); + } + + async disconnect(socketId: string): Promise { + const records = await this.store.smembers(socketKey(socketId)); + for (const record of records) { + try { + const parsed = JSON.parse(record) as { + schema: string; + documentId: string; + userId: string; + }; + await this.removeAuthorizedDocument( + socketId, + parsed.schema, + parsed.documentId, + parsed.userId, + ); + } catch { + // ignore malformed records + } + } + await this.store.del(socketKey(socketId)); + } +} diff --git a/modules/database/src/realtime/topology.ts b/modules/database/src/realtime/topology.ts new file mode 100644 index 000000000..4d62fa3fa --- /dev/null +++ b/modules/database/src/realtime/topology.ts @@ -0,0 +1,50 @@ +const UNUSABLE_RESUME_TOKEN_CODES = new Set([ + 280, // ChangeStreamHistoryLost + 286, // ChangeStreamFatalError +]); + +export type TopologyResult = { + supported: boolean; + message?: string; +}; + +export function topologyFromHello( + hello: + | { + setName?: string; + msg?: string; + } + | null + | undefined, +): TopologyResult { + if (!hello) { + return { supported: false, message: 'Unable to determine MongoDB topology' }; + } + if (hello.msg === 'isdbgrid' || Boolean(hello.setName)) { + return { supported: true }; + } + return { + supported: false, + message: 'A replica set or sharded MongoDB deployment is required for live updates', + }; +} + +export function isResumeTokenUnusable(error: unknown): boolean { + const code = extractErrorCode(error); + if (code === 237 || code === 136) { + return false; + } + if (code !== undefined && UNUSABLE_RESUME_TOKEN_CODES.has(code)) { + return true; + } + const message = error instanceof Error ? error.message : String(error ?? ''); + return /ChangeStreamHistoryLost/i.test(message); +} + +function extractErrorCode(error: unknown): number | undefined { + if (!error || typeof error !== 'object') return undefined; + const candidate = error as { code?: unknown; errorCode?: unknown }; + if (typeof candidate.code === 'number') return candidate.code; + if (typeof candidate.errorCode === 'number') return candidate.errorCode; + return undefined; +} diff --git a/modules/database/src/realtime/types.ts b/modules/database/src/realtime/types.ts new file mode 100644 index 000000000..2b6e3e8c7 --- /dev/null +++ b/modules/database/src/realtime/types.ts @@ -0,0 +1,48 @@ +export const DATABASE_CHANGE_EVENT_VERSION = 1 as const; + +export const DATABASE_CHANGE_OPERATIONS = [ + 'insert', + 'update', + 'replace', + 'delete', +] as const; + +export type DatabaseChangeOperation = (typeof DATABASE_CHANGE_OPERATIONS)[number]; + +export type DatabaseChangeEvent = { + version: typeof DATABASE_CHANGE_EVENT_VERSION; + operation: DatabaseChangeOperation; + schema: string; + documentId: string; + occurredAt: string; +}; + +export type RealtimeStatusCode = + 'unsupported' | 'disabled' | 'idle' | 'starting' | 'live' | 'degraded'; + +export type RealtimeStatus = { + status: RealtimeStatusCode; + engine: string; + activeSchemaCount: number; + lastEventAt?: string; + message?: string; +}; + +export type OptedInSchema = { + name: string; + collectionName: string; + authorizationEnabled: boolean; +}; + +export type SubscribeRequest = { + schema?: unknown; + documentId?: unknown; +}; + +export type ChangeStreamLike = { + on( + event: 'change' | 'error' | 'close' | 'end', + listener: (...args: unknown[]) => void, + ): void; + close(): Promise | void; +}; diff --git a/modules/database/src/realtime/watchPipeline.ts b/modules/database/src/realtime/watchPipeline.ts new file mode 100644 index 000000000..8ca888262 --- /dev/null +++ b/modules/database/src/realtime/watchPipeline.ts @@ -0,0 +1,33 @@ +const DATA_OPERATIONS = ['insert', 'update', 'replace', 'delete'] as const; +const CONTROL_OPERATIONS = ['drop', 'rename', 'invalidate', 'dropDatabase'] as const; + +export const WATCH_RESTART_OPERATIONS = new Set(CONTROL_OPERATIONS); + +export type WatchPipeline = Record[]; + +export function buildWatchPipeline(collectionNames: string[]): WatchPipeline { + return [ + { + $match: { + $or: [ + { + operationType: { $in: [...DATA_OPERATIONS] }, + 'ns.coll': { $in: collectionNames }, + }, + { operationType: { $in: [...CONTROL_OPERATIONS] } }, + ], + }, + }, + { + $project: { + fullDocument: 0, + updateDescription: 0, + fullDocumentBeforeChange: 0, + }, + }, + ]; +} + +export function optedInCollectionsKey(collectionNames: string[]): string { + return [...collectionNames].sort().join('\0'); +} diff --git a/modules/database/src/routes/index.ts b/modules/database/src/routes/index.ts index 2c09cf237..1d4002a74 100644 --- a/modules/database/src/routes/index.ts +++ b/modules/database/src/routes/index.ts @@ -13,6 +13,7 @@ import { import { DatabaseAdapter } from '../adapters/DatabaseAdapter.js'; import { MongooseSchema } from '../adapters/mongoose-adapter/MongooseSchema.js'; import { SequelizeSchema } from '../adapters/sequelize-adapter/SequelizeSchema.js'; +import type { RealtimeService } from '../realtime/index.js'; export class DatabaseRoutes { private readonly handlers: CmsHandlers; @@ -34,6 +35,7 @@ export class DatabaseRoutes { readonly server: GrpcServer, private readonly database: DatabaseAdapter, private readonly grpcSdk: ConduitGrpcSdk, + private readonly realtimeService?: RealtimeService, ) { this.handlers = new CmsHandlers(grpcSdk, database); this._routingManager = new RoutingManager(this.grpcSdk.router!, server); @@ -55,7 +57,6 @@ export class DatabaseRoutes { } requestRefresh() { - if (this.crudRoutes.length === 0 && this.customRoutes.length === 0) return; this._scheduleTimeout(); } @@ -86,6 +87,7 @@ export class DatabaseRoutes { this.crudRoutes.concat(this.customRoutes).forEach(route => { this._routingManager.route(route.input, route.returnType, route.handler); }); + this.realtimeService?.registerClient(this._routingManager); this._routingManager .registerRoutes() .then(() => { diff --git a/modules/database/src/utils/SchemaConverter.ts b/modules/database/src/utils/SchemaConverter.ts index 2e099ed57..f482f4936 100644 --- a/modules/database/src/utils/SchemaConverter.ts +++ b/modules/database/src/utils/SchemaConverter.ts @@ -38,6 +38,9 @@ export namespace SchemaConverter { authorization?: { enabled?: boolean; }; + realtime?: { + enabled?: boolean; + }; permissions?: { extendable?: boolean; canCreate?: boolean; @@ -130,6 +133,9 @@ export namespace SchemaConverter { defaults.conduit!.authorization?.enabled ?? false, }; + modelOptions.conduit.realtime = { + enabled: opts.realtime?.enabled ?? existing?.realtime?.enabled ?? false, + }; modelOptions.conduit.permissions = { extendable: explicit.permissions?.extendable !== undefined diff --git a/modules/database/src/utils/__tests__/validate-model-options.test.ts b/modules/database/src/utils/__tests__/validate-model-options.test.ts new file mode 100644 index 000000000..a832255e5 --- /dev/null +++ b/modules/database/src/utils/__tests__/validate-model-options.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from '@jest/globals'; +import { validateSchemaInput } from '../utilities.js'; + +const baseConduit = { + cms: { enabled: true }, + authorization: { enabled: false }, + permissions: { + extendable: true, + canCreate: true, + canModify: 'Everything' as const, + canDelete: true, + }, +}; + +describe('validateSchemaInput modelOptions.conduit', () => { + it('allows conduit.realtime used by live document updates', () => { + expect(() => + validateSchemaInput('TestSchema', undefined, { + conduit: { + ...baseConduit, + realtime: { enabled: true }, + }, + }), + ).not.toThrow(); + }); + + it('rejects an unknown conduit field', () => { + expect(() => + validateSchemaInput('TestSchema', undefined, { + conduit: { + ...baseConduit, + notARealField: true, + }, + }), + ).toThrow(/fields allowed inside 'conduit' field/); + }); +}); diff --git a/modules/database/src/utils/utilities.ts b/modules/database/src/utils/utilities.ts index ce6cd7328..f3e98a292 100644 --- a/modules/database/src/utils/utilities.ts +++ b/modules/database/src/utils/utilities.ts @@ -183,10 +183,11 @@ function validateModelOptions(modelOptions: ConduitSchemaOptions) { conduitKey !== 'cms' && conduitKey !== 'permissions' && conduitKey !== 'authorization' && - conduitKey !== 'imported' + conduitKey !== 'imported' && + conduitKey !== 'realtime' ) throw new Error( - "Only 'cms', 'permissions', 'authorization', 'imported', and 'readPreference' fields allowed inside 'conduit' field", + "Only 'cms', 'permissions', 'authorization', 'imported', 'readPreference', and 'realtime' fields allowed inside 'conduit' field", ); if (conduitKey === 'imported') { if (!isBoolean(modelOptions.conduit!.imported)) diff --git a/packages/core/package.json b/packages/core/package.json index 0636f4ce2..764e285ab 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -12,6 +12,7 @@ "start": "node dist/bin/www.js", "start:bundle": "node bundle/index.js", "lint": "eslint src", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/admin/realtime/*.test.js", "prebuild": "npm run generateTypes", "build": "rimraf dist && tsc", "generateTypes": "sh build.sh", diff --git a/packages/core/src/admin/AdminModule.ts b/packages/core/src/admin/AdminModule.ts index 3f18ea069..4f4574571 100644 --- a/packages/core/src/admin/AdminModule.ts +++ b/packages/core/src/admin/AdminModule.ts @@ -14,6 +14,7 @@ import { PatchRouteMiddlewaresRequest, RegisterAdminRouteRequest, RegisterAdminRouteRequest_PathDefinition, + SocketPushRequest, } from '../interfaces/index.js'; import { hashPassword } from './utils/auth.js'; import AdminConfigRawSchema from './config/index.js'; @@ -33,6 +34,7 @@ import { ConduitSocket, grpcToConduitRoute, RouteT, + SocketPush, } from '@conduitplatform/hermes'; import convict from 'convict'; import { NextFunction, Request, Response } from 'express'; @@ -48,6 +50,7 @@ import { getReadinessMiddleware } from '../health/readinessMiddleware.js'; import { ReadinessService } from '../health/ReadinessService.js'; import type { CoreHealthProvider } from '../health/types.js'; import { stripUndeclaredConfigParams } from '../utils/stripUndeclaredConfigParams.js'; +import { adminSocketNamespace } from './realtime/namespace.js'; export default class AdminModule { grpcSdk: ConduitGrpcSdk; @@ -93,6 +96,7 @@ export default class AdminModule { adminRoutes.createApiTokenRoute(), adminRoutes.getApiTokensRoute(), adminRoutes.deleteApiTokenRoute(), + adminRoutes.getRealtimeTicketRoute(), configRoutes.getModulesRoute(), adminRoutes.getStateExportRoute(this), adminRoutes.getStateImportRoute(this), @@ -131,6 +135,7 @@ export default class AdminModule { { registerAdminRoute: this.registerAdminRoute.bind(this), patchRouteMiddlewares: this.patchRouteMiddlewares.bind(this), + socketPush: this.socketPush.bind(this), }, ); } @@ -238,6 +243,27 @@ export default class AdminModule { callback(null, {}); } + async socketPush( + call: GrpcRequest, + callback: GrpcCallback>, + ) { + try { + const moduleName = call.metadata?.get('module-name')?.[0] as string | undefined; + const socketData: SocketPush = { + event: call.request.event, + data: call.request.data ? JSON.parse(call.request.data) : undefined, + receivers: call.request.receivers, + rooms: call.request.rooms, + namespace: adminSocketNamespace(moduleName), + }; + await this._router.socketPush(socketData); + } catch (err) { + ConduitGrpcSdk.Logger.error(err as Error); + return callback({ code: status.INTERNAL, message: 'Well that failed :/' }); + } + callback(null, {}); + } + registerRoute(route: ConduitRoute): void { this._sdkRoutes.push(route); this._router.registerConduitRoute(route); @@ -288,6 +314,16 @@ export default class AdminModule { `New admin route registered: ${r.input.action} ${r.input.path} handler url: ${url}`, ); this._router.registerConduitRoute(r); + } else if (r instanceof ConduitMiddleware) { + ConduitGrpcSdk.Logger.log( + `New admin middleware registered: ${r.input.path} handler url: ${url}`, + ); + this._router.registerRouteMiddleware(r, url); + } else if (r instanceof ConduitSocket) { + ConduitGrpcSdk.Logger.log( + `New admin socket registered: ${r.input.path} handler url: ${url}`, + ); + this._router.registerConduitSocket(r); } }); // @ts-ignore diff --git a/packages/core/src/admin/middleware/Admin.middleware.ts b/packages/core/src/admin/middleware/Admin.middleware.ts index 51c811987..4fb743246 100644 --- a/packages/core/src/admin/middleware/Admin.middleware.ts +++ b/packages/core/src/admin/middleware/Admin.middleware.ts @@ -2,7 +2,7 @@ import { NextFunction, Response } from 'express'; import { isNil } from 'lodash-es'; // Removed ConduitCommons import - now using configManager directly import { isDev } from '../utils/middleware.js'; -import { ConduitRequest } from '@conduitplatform/hermes'; +import { ConduitRequest, isSocketHandshake } from '@conduitplatform/hermes'; import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; export function getAdminMiddleware(configManager: any) { @@ -28,9 +28,14 @@ export function getAdminMiddleware(configManager: any) { ) { return next(); } - // Allow API tokens (cdt_*) to bypass masterkey; Auth.middleware will validate the token + // Allow API tokens (cdt_*) and Socket.IO handshakes with a Bearer token to + // skip masterkey. Auth.middleware still validates the credential. const authHeader = req.headers.authorization; - if (authHeader && authHeader.startsWith('Bearer cdt_')) { + if ( + typeof authHeader === 'string' && + authHeader.startsWith('Bearer ') && + (authHeader.startsWith('Bearer cdt_') || isSocketHandshake(req)) + ) { return next(); } const masterKey = req.headers.masterkey; diff --git a/packages/core/src/admin/middleware/Auth.middleware.ts b/packages/core/src/admin/middleware/Auth.middleware.ts index 4c883244d..9bac68d99 100644 --- a/packages/core/src/admin/middleware/Auth.middleware.ts +++ b/packages/core/src/admin/middleware/Auth.middleware.ts @@ -7,6 +7,7 @@ import { isDev } from '../utils/middleware.js'; import { ConduitRequest } from '@conduitplatform/hermes'; import { gql } from 'graphql-tag'; import { ConfigController } from '@conduitplatform/module-tools'; +import { realtimeTicketForbiddenOnHttp } from '../realtime/ticket.js'; const excludedRestRoutes = ['/ready', '/live', '/login', '/config/modules']; const excludedGqlOperations = [ @@ -113,6 +114,10 @@ async function handleJwtToken( } const { id } = decoded; + if (realtimeTicketForbiddenOnHttp(decoded, req)) { + res.status(401).json({ error: 'Realtime ticket cannot be used for HTTP requests' }); + return; + } if (decoded.twoFaRequired && req.path !== '/verify-twofa') { res.status(401).json({ error: 'Two FA required' }); return; diff --git a/packages/core/src/admin/realtime/namespace.test.ts b/packages/core/src/admin/realtime/namespace.test.ts new file mode 100644 index 000000000..6bfcb3b64 --- /dev/null +++ b/packages/core/src/admin/realtime/namespace.test.ts @@ -0,0 +1,11 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { adminSocketNamespace } from './namespace.js'; + +describe('adminSocketNamespace', () => { + it('mirrors the Router module namespace contract', () => { + assert.equal(adminSocketNamespace('database'), '/database/'); + assert.throws(() => adminSocketNamespace(''), /module-name/); + assert.throws(() => adminSocketNamespace(undefined), /module-name/); + }); +}); diff --git a/packages/core/src/admin/realtime/namespace.ts b/packages/core/src/admin/realtime/namespace.ts new file mode 100644 index 000000000..ee66e9092 --- /dev/null +++ b/packages/core/src/admin/realtime/namespace.ts @@ -0,0 +1,7 @@ +export function adminSocketNamespace(moduleName: string | undefined | null): string { + const name = typeof moduleName === 'string' ? moduleName.trim() : ''; + if (!name) { + throw new Error('module-name metadata is required for Admin socket push'); + } + return `/${name}/`; +} diff --git a/packages/core/src/admin/realtime/ticket-http.test.ts b/packages/core/src/admin/realtime/ticket-http.test.ts new file mode 100644 index 000000000..589975acc --- /dev/null +++ b/packages/core/src/admin/realtime/ticket-http.test.ts @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import jwt from 'jsonwebtoken'; +import { ConfigController } from '@conduitplatform/module-tools'; +import { isSocketHandshake } from '@conduitplatform/hermes'; +import { getAuthMiddleware } from '../middleware/Auth.middleware.js'; +import { + buildRealtimeTicketClaims, + isRealtimeTicket, + realtimeTicketForbiddenOnHttp, +} from './ticket.js'; + +function mockResponse() { + let statusCode = 0; + let body: unknown; + const res = { + status(code: number) { + statusCode = code; + return this; + }, + json(payload: unknown) { + body = payload; + return this; + }, + }; + return { + res, + get statusCode() { + return statusCode; + }, + get body() { + return body; + }, + }; +} + +describe('realtime ticket HTTP guard', () => { + it('does not treat POST /realtime/ticket as a handshake so a ticket cannot mint another', () => { + const ticket = buildRealtimeTicketClaims('admin-1'); + assert.equal(isRealtimeTicket(ticket), true); + assert.equal(isSocketHandshake({ url: '/realtime/ticket' }), false); + assert.equal( + isSocketHandshake({ url: '/realtime/ticket?EIO=4&transport=polling' }), + false, + ); + assert.equal( + realtimeTicketForbiddenOnHttp(ticket, { url: '/realtime/ticket' }), + true, + ); + assert.equal( + realtimeTicketForbiddenOnHttp(ticket, { + url: '/realtime/ticket?EIO=4&transport=polling', + }), + true, + ); + }); + + it('allows a ticket on Engine.IO handshake polling without sid', () => { + const ticket = buildRealtimeTicketClaims('admin-1'); + const handshake = { url: '/realtime/?EIO=4&transport=polling' }; + assert.equal(isSocketHandshake(handshake), true); + assert.equal(realtimeTicketForbiddenOnHttp(ticket, handshake), false); + }); + + it('Auth middleware returns 401 on POST /realtime/ticket with a realtime ticket', async () => { + const secret = 'ticket-http-test-secret'; + ConfigController.getInstance().config = { auth: { tokenSecret: secret } }; + const token = jwt.sign(buildRealtimeTicketClaims('admin-1'), secret, { + algorithm: 'HS256', + expiresIn: 30, + }); + const middleware = getAuthMiddleware({} as never, { + get: async () => ({ env: 'production' }), + }); + const mock = mockResponse(); + let nextCalled = false; + await middleware( + { + path: '/realtime/ticket', + originalUrl: '/realtime/ticket', + url: '/realtime/ticket', + method: 'POST', + headers: { authorization: `Bearer ${token}` }, + conduit: {}, + } as never, + mock.res as never, + () => { + nextCalled = true; + }, + ); + assert.equal(nextCalled, false); + assert.equal(mock.statusCode, 401); + assert.deepEqual(mock.body, { + error: 'Realtime ticket cannot be used for HTTP requests', + }); + }); +}); diff --git a/packages/core/src/admin/realtime/ticket.test.ts b/packages/core/src/admin/realtime/ticket.test.ts new file mode 100644 index 000000000..e0d66fff1 --- /dev/null +++ b/packages/core/src/admin/realtime/ticket.test.ts @@ -0,0 +1,26 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { + ADMIN_REALTIME_AUDIENCE, + buildRealtimeTicketClaims, + isRealtimeTicket, +} from './ticket.js'; + +describe('admin realtime tickets', () => { + it('scopes claims to the realtime audience', () => { + const claims = buildRealtimeTicketClaims('admin-1'); + assert.equal(claims.id, 'admin-1'); + assert.equal(claims.aud, ADMIN_REALTIME_AUDIENCE); + assert.equal(isRealtimeTicket(claims), true); + }); + + it('rejects ordinary admin JWTs and missing payloads', () => { + assert.equal(isRealtimeTicket({ id: 'admin-1' }), false); + assert.equal(isRealtimeTicket({ id: 'admin-1', aud: 'other' }), false); + assert.equal(isRealtimeTicket(null), false); + assert.equal( + isRealtimeTicket({ id: 'admin-1', aud: [ADMIN_REALTIME_AUDIENCE, 'extra'] }), + true, + ); + }); +}); diff --git a/packages/core/src/admin/realtime/ticket.ts b/packages/core/src/admin/realtime/ticket.ts new file mode 100644 index 000000000..e797f5966 --- /dev/null +++ b/packages/core/src/admin/realtime/ticket.ts @@ -0,0 +1,35 @@ +import { isSocketHandshake } from '@conduitplatform/hermes'; + +export const ADMIN_REALTIME_AUDIENCE = 'admin-realtime'; +export const ADMIN_REALTIME_TICKET_TTL_SECONDS = 30; + +export type RealtimeTicketClaims = { + id: string; + aud?: string | string[]; + twoFaRequired?: boolean; +}; + +export function buildRealtimeTicketClaims(adminId: string): RealtimeTicketClaims { + return { + id: adminId, + aud: ADMIN_REALTIME_AUDIENCE, + }; +} + +export function isRealtimeTicket( + decoded: RealtimeTicketClaims | null | undefined, +): boolean { + if (!decoded) return false; + const audience = decoded.aud; + if (Array.isArray(audience)) { + return audience.includes(ADMIN_REALTIME_AUDIENCE); + } + return audience === ADMIN_REALTIME_AUDIENCE; +} + +export function realtimeTicketForbiddenOnHttp( + decoded: RealtimeTicketClaims | null | undefined, + req: { url?: string; originalUrl?: string }, +): boolean { + return isRealtimeTicket(decoded) && !isSocketHandshake(req); +} diff --git a/packages/core/src/admin/routes/RealtimeTicket.route.ts b/packages/core/src/admin/routes/RealtimeTicket.route.ts new file mode 100644 index 000000000..4a1b8e11e --- /dev/null +++ b/packages/core/src/admin/routes/RealtimeTicket.route.ts @@ -0,0 +1,51 @@ +import { isNil } from 'lodash-es'; +import { ConduitRoute } from '@conduitplatform/hermes'; +import { + ConduitError, + ConduitRouteActions, + ConduitRouteParameters, + ConduitRouteReturnDefinition, +} from '@conduitplatform/grpc-sdk'; +import { + ConduitNumber, + ConduitString, + ConfigController, +} from '@conduitplatform/module-tools'; +import { signToken } from '../utils/auth.js'; +import { + ADMIN_REALTIME_TICKET_TTL_SECONDS, + buildRealtimeTicketClaims, +} from '../realtime/ticket.js'; + +export function getRealtimeTicketRoute() { + return new ConduitRoute( + { + path: '/realtime/ticket', + action: ConduitRouteActions.POST, + mcp: false, + description: + 'Issue a short-lived token for Admin Socket.IO handshakes. The token cannot be used for REST or GraphQL.', + }, + new ConduitRouteReturnDefinition('RealtimeTicket', { + token: ConduitString.Required, + expiresIn: ConduitNumber.Required, + }), + async (req: ConduitRouteParameters) => { + const admin = req.context?.admin; + if (isNil(admin) || isNil(admin._id)) { + throw new ConduitError('UNAUTHORIZED', 401, 'Authentication required'); + } + const adminId = admin._id.toString(); + const authConfig = ConfigController.getInstance().config.auth; + const token = signToken( + buildRealtimeTicketClaims(adminId), + authConfig.tokenSecret, + ADMIN_REALTIME_TICKET_TTL_SECONDS, + ); + return { + token, + expiresIn: ADMIN_REALTIME_TICKET_TTL_SECONDS, + }; + }, + ); +} diff --git a/packages/core/src/admin/routes/index.ts b/packages/core/src/admin/routes/index.ts index 9bed86a70..f7c422e2c 100644 --- a/packages/core/src/admin/routes/index.ts +++ b/packages/core/src/admin/routes/index.ts @@ -20,3 +20,4 @@ export * from './ToggleTwoFa.route.js'; export * from './VerifyQrCode.route.js'; export * from './VerifyTwoFa.route.js'; export * from './ApiTokens.route.js'; +export * from './RealtimeTicket.route.js'; diff --git a/packages/core/src/core.proto b/packages/core/src/core.proto index 527d24f35..858dfa406 100644 --- a/packages/core/src/core.proto +++ b/packages/core/src/core.proto @@ -21,6 +21,14 @@ service Config { service Admin { rpc RegisterAdminRoute (RegisterAdminRouteRequest) returns (google.protobuf.Empty); rpc PatchRouteMiddlewares(PatchRouteMiddlewaresRequest) returns (google.protobuf.Empty); + rpc SocketPush (SocketPushRequest) returns (google.protobuf.Empty); +} + +message SocketPushRequest { + string event = 1; + optional string data = 2; + repeated string receivers = 3; + repeated string rooms = 4; } message RegisterAdminRouteRequest { diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index f4c4ef779..eb15cc5a9 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -67,5 +67,5 @@ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ }, "include": ["src/**/*"], - "exclude": ["tests", "node_modules", "dist", "bundle", "tsup.config.ts"] + "exclude": ["tests", "node_modules", "dist", "bundle", "tsup.config.ts", "**/*.test.ts"] } diff --git a/packages/core/tsconfig.test.json b/packages/core/tsconfig.test.json new file mode 100644 index 000000000..8559c14bf --- /dev/null +++ b/packages/core/tsconfig.test.json @@ -0,0 +1,16 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": [ + "src/admin/realtime/ticket.ts", + "src/admin/realtime/namespace.ts", + "src/admin/realtime/*.test.ts" + ], + "exclude": ["node_modules", "dist", "bundle"] +} From 9c4a5a5d6c5649ca1d848c2bfbdbe48eb6f10c1d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 14:37:29 +0000 Subject: [PATCH 4/5] fix(database): fence change-stream leadership and expire recovery Redis keys Lock renew failure bumps a generation so a draining watch cannot emit. Do not open a watch unless the leader lock extends after acquire. Arm TTL on realtime socket/doc keys when a recoverable disconnect never recovers; persist on restore. CMS-read deny fails closed at emit without extra ReBAC round-trips and keeps membership; UNAVAILABLE keeps membership. Cover dropDatabase reopen. Co-authored-by: Konstantinos Kopanidis --- modules/database/README.mdx | 2 +- .../realtime/MongoChangeStreamCoordinator.ts | 77 ++++++++++++--- .../realtime/__tests__/coordinator.test.ts | 95 +++++++++++++++++-- .../src/realtime/__tests__/recovery.test.ts | 62 +++++++++++- .../realtime/__tests__/subscriptions.test.ts | 45 ++++++++- modules/database/src/realtime/authorize.ts | 24 +++-- modules/database/src/realtime/rebacCache.ts | 3 +- modules/database/src/realtime/recovery.ts | 12 +-- modules/database/src/realtime/sockets.ts | 5 +- .../database/src/realtime/subscriptions.ts | 56 +++++++++-- modules/database/src/realtime/types.ts | 3 + 11 files changed, 332 insertions(+), 52 deletions(-) diff --git a/modules/database/README.mdx b/modules/database/README.mdx index b84cded13..10703937b 100644 --- a/modules/database/README.mdx +++ b/modules/database/README.mdx @@ -64,7 +64,7 @@ Events arrive as `change` with `{ version, operation, schema, documentId, occurr The leader also publishes `database:change:${schema}` on the Redis bus. **Do not also relay `database:change:*` on `/events/` if the same client is on `/database/`** — that duplicates notifications. Keep the bus for other modules; just do not dual-subscribe. -Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Client sockets also require CMS `crudOperations.read.enabled`. Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; that token cannot mint another ticket or call REST/GraphQL. Session JWTs and masterkeys must not be sent from browser code. +Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Client sockets also require CMS `crudOperations.read.enabled` (checked again at emit: deny skips client delivery and keeps membership; authorization UNAVAILABLE keeps membership without emitting). Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; that token cannot mint another ticket or call REST/GraphQL. Session JWTs and masterkeys must not be sent from browser code. Admin sockets must be enabled (`admin.transports.sockets`) and the Admin socket port (`ADMIN_SOCKET_PORT`, default 3031) reachable from the UI. diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts index d29938d15..d13e5a911 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -67,6 +67,7 @@ export class MongoChangeStreamCoordinator { private watching = false; private opening = false; private ignoreClose = false; + private lockGeneration = 0; private changeQueue: Promise = Promise.resolve(); private watchedCollectionsKey = ''; private readonly rebacCache = new RealtimeRebacCache(); @@ -160,7 +161,20 @@ export class MongoChangeStreamCoordinator { this.scheduleRetry(); return; } - this.lock = acquired; + try { + this.lock = await acquired.extend(LOCK_TTL_MS); + } catch { + try { + await this.options.grpcSdk.state!.releaseLock(acquired); + } catch { + // lock may already have expired + } + this.lock = null; + this.streamState = 'idle'; + this.scheduleRetry(); + return; + } + this.bumpLockGeneration(); this.startRenewal(); await this.openStream(); } catch (err) { @@ -179,40 +193,59 @@ export class MongoChangeStreamCoordinator { private async renewLock() { if (!this.lock) return; + const generation = this.lockGeneration; try { this.lock = await this.lock.extend(LOCK_TTL_MS); } catch { - this.lock = null; - await this.stopStream('idle'); + if (this.lockGeneration !== generation) return; + await this.fenceLock('idle'); this.scheduleRetry(); } } private async openStream() { - if (this.watching || this.closed || this.opening) return; + if (this.watching || this.closed || this.opening || !this.lock) return; this.opening = true; this.streamState = 'starting'; this.ignoreClose = false; + const generation = this.lockGeneration; try { const resumeAfter = parseResumeToken( await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY), ); - if (this.watching || this.closed) return; + if ( + this.watching || + this.closed || + !this.lock || + generation !== this.lockGeneration + ) { + return; + } const collections = this.collectionNames(); const pipeline = buildWatchPipeline(collections); this.watchedCollectionsKey = optedInCollectionsKey(collections); const stream = this.options.watch({ resumeAfter, pipeline }); + if (generation !== this.lockGeneration || this.closed) { + try { + await stream.close(); + } catch { + // already closed + } + return; + } this.stream = stream; this.watching = true; this.streamState = 'live'; this.retryAttempt = 0; stream.on('change', (change: unknown) => { - this.enqueueChange(change as RawChangeEvent); + this.enqueueChange(change as RawChangeEvent, generation); }); stream.on('error', (err: unknown) => { + if (generation !== this.lockGeneration) return; void this.handleStreamError(err); }); stream.on('close', () => { + if (generation !== this.lockGeneration) return; this.watching = false; if (!this.closed && !this.ignoreClose && this.lock) { this.scheduleRetry(); @@ -226,12 +259,13 @@ export class MongoChangeStreamCoordinator { } } - private enqueueChange(change: RawChangeEvent) { + private enqueueChange(change: RawChangeEvent, generation: number) { this.changeQueue = this.changeQueue.then(async () => { - if (this.closed || !this.watching) return; + if (this.closed || !this.watching || generation !== this.lockGeneration) return; try { - await this.handleChange(change); + await this.handleChange(change, generation); } catch (err) { + if (generation !== this.lockGeneration) return; this.lastError = err instanceof Error ? err.message : String(err); ConduitGrpcSdk.Logger.error(err as Error); this.watching = false; @@ -241,12 +275,13 @@ export class MongoChangeStreamCoordinator { }); } - private async handleChange(change: RawChangeEvent) { + private async handleChange(change: RawChangeEvent, generation: number) { + if (generation !== this.lockGeneration) return; const token = serializeResumeToken(change._id); const schema = this.resolveSchema(change.ns?.coll); const event = schema ? normalizeChangeEvent(change, schema.name) : null; if (!event || !schema) { - if (token) { + if (token && generation === this.lockGeneration) { await this.persistResumeToken(token); } if (change.operationType && WATCH_RESTART_OPERATIONS.has(change.operationType)) { @@ -283,6 +318,9 @@ export class MongoChangeStreamCoordinator { ) { const adminRooms = roomsForPublicChange(schema.name, event.documentId); await this.safePush('admin', adminRooms, payload); + if (!schema.cmsReadEnabled) { + return; + } if (!schema.authorizationEnabled) { await this.safePush('router', adminRooms, payload); return; @@ -381,14 +419,25 @@ export class MongoChangeStreamCoordinator { } private async releaseLeader() { + await this.fenceLock(this.streamState); + } + + private async fenceLock(nextState: RealtimeStatusCode) { + this.bumpLockGeneration(); this.clearRenewTimer(); - if (!this.lock) return; + const lock = this.lock; + this.lock = null; + await this.stopStream(nextState); + if (!lock) return; try { - await this.options.grpcSdk.state!.releaseLock(this.lock); + await this.options.grpcSdk.state!.releaseLock(lock); } catch { // lock may already have expired } - this.lock = null; + } + + private bumpLockGeneration() { + this.lockGeneration += 1; } private clearTimers() { diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index cda5dc0e1..3720d5a41 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -7,6 +7,7 @@ import { roomsForPublicChange } from '../rooms.js'; class MemoryStore { private sets = new Map>(); + readonly ttls = new Map(); async sadd(key: string, ...members: string[]) { const set = this.sets.get(key) ?? new Set(); members.forEach(member => set.add(member)); @@ -26,15 +27,29 @@ class MemoryStore { return this.sets.get(key)?.size ?? 0; } async del(...keys: string[]) { - keys.forEach(key => this.sets.delete(key)); + keys.forEach(key => { + this.sets.delete(key); + this.ttls.delete(key); + }); return keys.length; } + async expire(key: string, seconds: number) { + this.ttls.set(key, seconds); + } + async persist(key: string) { + this.ttls.delete(key); + } } function createCoordinator(overrides?: { allow?: boolean; authorizationAvailable?: boolean; - schemas?: { name: string; collectionName: string; authorizationEnabled: boolean }[]; + schemas?: { + name: string; + collectionName: string; + authorizationEnabled: boolean; + cmsReadEnabled?: boolean; + }[]; getKeyDelayMs?: number; }) { const stream = new EventEmitter() as EventEmitter & { close: () => Promise }; @@ -84,9 +99,14 @@ function createCoordinator(overrides?: { watch, hello: async () => ({ setName: 'rs0' }), getOptedInSchemas: () => - overrides?.schemas ?? [ - { name: 'Order', collectionName: 'orders', authorizationEnabled: false }, - ], + ( + overrides?.schemas ?? [ + { name: 'Order', collectionName: 'orders', authorizationEnabled: false }, + ] + ).map(schema => ({ + cmsReadEnabled: true, + ...schema, + })), subscriptions, enabled: () => true, engine: () => 'MongoDB', @@ -358,7 +378,7 @@ describe('MongoChangeStreamCoordinator', () => { jest.useRealTimers(); }); - it.each(['drop', 'rename', 'invalidate'] as const)( + it.each(['drop', 'rename', 'invalidate', 'dropDatabase'] as const)( 'reopens the watch on %s', async operationType => { jest.useFakeTimers(); @@ -419,4 +439,67 @@ describe('MongoChangeStreamCoordinator', () => { ]); await coordinator.shutdown(); }); + + it('does not open a watch when the lock cannot be extended after acquire', async () => { + const { coordinator, watch, lock } = createCoordinator(); + lock.extend.mockRejectedValueOnce(new Error('extend failed')); + await coordinator.reconcile(); + expect(watch).not.toHaveBeenCalled(); + expect(coordinator.getState()).toBe('idle'); + await coordinator.shutdown(); + }); + + it('ignores draining watch events after lock renew failure', async () => { + jest.useFakeTimers(); + const { coordinator, stream, lock, adminPush, state } = createCoordinator(); + lock.extend.mockResolvedValueOnce(lock).mockRejectedValueOnce(new Error('lost lock')); + await coordinator.reconcile(); + expect(coordinator.getState()).toBe('live'); + await jest.advanceTimersByTimeAsync(5_000); + expect(coordinator.getState()).toBe('idle'); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'stale' }), + ); + await coordinator.waitForIdle(); + expect(adminPush).not.toHaveBeenCalled(); + expect(state.get('realtime:resumeToken')).toBeUndefined(); + await coordinator.shutdown(); + jest.useRealTimers(); + }); + + it('does not emit to clients when CMS read is denied and keeps membership', async () => { + const can = jest.fn(async () => ({ allow: true })); + const { coordinator, stream, routerPush, adminPush, subscriptions, grpcSdk } = + createCoordinator({ + schemas: [ + { + name: 'Order', + collectionName: 'orders', + authorizationEnabled: true, + cmsReadEnabled: false, + }, + ], + }); + grpcSdk.authorization = { can }; + await subscriptions.addAuthorizedDocument( + 'sock-1', + 'Order', + '64b64c4c4c4c4c4c4c4c4c4c', + 'user-1', + ); + await coordinator.reconcile(); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), + ); + await coordinator.waitForIdle(); + expect(adminPush).toHaveBeenCalledTimes(1); + expect(routerPush).not.toHaveBeenCalled(); + expect(can).not.toHaveBeenCalled(); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual([ + 'user-1', + ]); + await coordinator.shutdown(); + }); }); diff --git a/modules/database/src/realtime/__tests__/recovery.test.ts b/modules/database/src/realtime/__tests__/recovery.test.ts index eee45b918..4e5aaee37 100644 --- a/modules/database/src/realtime/__tests__/recovery.test.ts +++ b/modules/database/src/realtime/__tests__/recovery.test.ts @@ -1,12 +1,15 @@ import { describe, expect, it } from '@jest/globals'; -import { canReadDocument } from '../authorize.js'; import { authorizedDocumentRoom } from '../rooms.js'; import { isRecoverableDisconnect, restoreAuthorizedSubscriptions } from '../recovery.js'; import { createSocketHandlers } from '../sockets.js'; -import { RealtimeSubscriptionTracker } from '../subscriptions.js'; +import { + RealtimeSubscriptionTracker, + RECOVERY_REDIS_TTL_SECONDS, +} from '../subscriptions.js'; class MemoryStore { private sets = new Map>(); + readonly ttls = new Map(); async sadd(key: string, ...members: string[]) { const set = this.sets.get(key) ?? new Set(); members.forEach(member => set.add(member)); @@ -22,7 +25,16 @@ class MemoryStore { return this.sets.get(key)?.size ?? 0; } async del(...keys: string[]) { - keys.forEach(key => this.sets.delete(key)); + keys.forEach(key => { + this.sets.delete(key); + this.ttls.delete(key); + }); + } + async expire(key: string, seconds: number) { + this.ttls.set(key, seconds); + } + async persist(key: string) { + this.ttls.delete(key); } } @@ -48,7 +60,6 @@ describe('database socket recovery', () => { isAvailable: () => true, authorization: { can: async () => ({ allow: true }) }, }, - canRead: canReadDocument, }); expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); }); @@ -79,4 +90,47 @@ describe('database socket recovery', () => { } as never); expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); }); + + it('keeps membership when recovered authorization is unavailable', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + const room = authorizedDocumentRoom('Order', 'doc-1', 'user-1'); + await tracker.addAuthorizedDocument('sock-1', 'Order', 'doc-1', 'user-1'); + const { leaveRooms } = await restoreAuthorizedSubscriptions({ + socketId: 'sock-1', + rooms: [room], + contextSubs: [], + subscriptions: tracker, + grpcSdk: { + isAvailable: () => false, + }, + }); + expect(leaveRooms).toEqual([]); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + }); + + it('expires Redis membership when a recoverable disconnect never recovers', async () => { + const store = new MemoryStore(); + const tracker = new RealtimeSubscriptionTracker(store); + const handlers = createSocketHandlers({ + mode: 'client', + grpcSdk: { + isAvailable: () => true, + authorization: { can: async () => ({ allow: true }) }, + } as never, + schemaLookup: { getSchema: () => undefined }, + subscriptions: tracker, + isGloballyEnabled: () => true, + }); + await tracker.addAuthorizedDocument('sock-1', 'Order', 'doc-1', 'user-1'); + expect(store.ttls.size).toBe(0); + await handlers.disconnect({ + request: { socketId: 'sock-1', params: ['transport close'] }, + } as never); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + expect(store.ttls.get('realtime:socket:sock-1')).toBe(RECOVERY_REDIS_TTL_SECONDS); + expect(store.ttls.get('realtime:doc:Order:doc-1')).toBe(RECOVERY_REDIS_TTL_SECONDS); + expect(store.ttls.get('realtime:userdoc:Order:doc-1:user-1')).toBe( + RECOVERY_REDIS_TTL_SECONDS, + ); + }); }); diff --git a/modules/database/src/realtime/__tests__/subscriptions.test.ts b/modules/database/src/realtime/__tests__/subscriptions.test.ts index 45ee7d1c9..13bcc7298 100644 --- a/modules/database/src/realtime/__tests__/subscriptions.test.ts +++ b/modules/database/src/realtime/__tests__/subscriptions.test.ts @@ -1,8 +1,12 @@ import { describe, expect, it } from '@jest/globals'; -import { RealtimeSubscriptionTracker } from '../subscriptions.js'; +import { + RealtimeSubscriptionTracker, + RECOVERY_REDIS_TTL_SECONDS, +} from '../subscriptions.js'; class MemoryStore { private sets = new Map>(); + readonly ttls = new Map(); async sadd(key: string, ...members: string[]) { const set = this.sets.get(key) ?? new Set(); @@ -27,9 +31,20 @@ class MemoryStore { } async del(...keys: string[]) { - keys.forEach(key => this.sets.delete(key)); + keys.forEach(key => { + this.sets.delete(key); + this.ttls.delete(key); + }); return keys.length; } + + async expire(key: string, seconds: number) { + this.ttls.set(key, seconds); + } + + async persist(key: string) { + this.ttls.delete(key); + } } describe('RealtimeSubscriptionTracker', () => { @@ -46,10 +61,36 @@ describe('RealtimeSubscriptionTracker', () => { expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); }); + it('does not expire shared document keys while another socket is still live', async () => { + const store = new MemoryStore(); + const tracker = new RealtimeSubscriptionTracker(store); + await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); + await tracker.addAuthorizedDocument('s2', 'Order', 'doc-1', 'user-1'); + await tracker.armRecoverableTtl('s1'); + expect(store.ttls.get('realtime:socket:s1')).toBe(RECOVERY_REDIS_TTL_SECONDS); + expect(store.ttls.has('realtime:doc:Order:doc-1')).toBe(false); + expect(store.ttls.has('realtime:userdoc:Order:doc-1:user-1')).toBe(false); + }); + it('removes revoked users from the document set', async () => { const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); await tracker.removeUser('Order', 'doc-1', 'user-1'); expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); }); + + it('arms Redis TTL on recoverable disconnect keys and persists on restore', async () => { + const store = new MemoryStore(); + const tracker = new RealtimeSubscriptionTracker(store); + await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); + expect(store.ttls.size).toBe(0); + await tracker.armRecoverableTtl('s1'); + expect(store.ttls.get('realtime:socket:s1')).toBe(RECOVERY_REDIS_TTL_SECONDS); + expect(store.ttls.get('realtime:doc:Order:doc-1')).toBe(RECOVERY_REDIS_TTL_SECONDS); + expect(store.ttls.get('realtime:userdoc:Order:doc-1:user-1')).toBe( + RECOVERY_REDIS_TTL_SECONDS, + ); + await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); + expect(store.ttls.size).toBe(0); + }); }); diff --git a/modules/database/src/realtime/authorize.ts b/modules/database/src/realtime/authorize.ts index cf7ed899c..d2a16e1cf 100644 --- a/modules/database/src/realtime/authorize.ts +++ b/modules/database/src/realtime/authorize.ts @@ -1,6 +1,6 @@ import { status } from '@grpc/grpc-js'; import { GrpcError } from '@conduitplatform/grpc-sdk'; -import type { OptedInSchema, SubscribeRequest } from './types.js'; +import type { OptedInSchema, RebacDecision, SubscribeRequest } from './types.js'; export class RealtimeSubscriptionError extends GrpcError { constructor(code: number, message: string) { @@ -61,14 +61,14 @@ export function optionalDocumentId(value: unknown): string | undefined { return value; } -export async function canReadDocument( +export async function readDocumentDecision( grpcSdk: AuthorizationSdk, schema: string, documentId: string, userId: string, -): Promise { +): Promise { if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { - return false; + return 'unavailable'; } try { const decision = await grpcSdk.authorization.can({ @@ -76,12 +76,21 @@ export async function canReadDocument( actions: ['read'], resource: `${schema}:${documentId}`, }); - return decision.allow === true; + return decision.allow === true ? 'allow' : 'deny'; } catch { - return false; + return 'unavailable'; } } +export async function canReadDocument( + grpcSdk: AuthorizationSdk, + schema: string, + documentId: string, + userId: string, +): Promise { + return (await readDocumentDecision(grpcSdk, schema, documentId, userId)) === 'allow'; +} + export function assertSchemaAvailable( lookup: SchemaLookup, schemaName: string, @@ -123,6 +132,7 @@ export function toOptedInSchema(schema: { modelOptions?: { conduit?: { realtime?: { enabled?: boolean }; + cms?: { crudOperations?: { read?: { enabled?: boolean } } }; authorization?: { enabled?: boolean }; }; }; @@ -132,5 +142,7 @@ export function toOptedInSchema(schema: { name: schema.name, collectionName: schema.collectionName, authorizationEnabled: schema.modelOptions.conduit.authorization?.enabled === true, + cmsReadEnabled: + schema.modelOptions.conduit.cms?.crudOperations?.read?.enabled === true, }; } diff --git a/modules/database/src/realtime/rebacCache.ts b/modules/database/src/realtime/rebacCache.ts index b33a9438c..c57595227 100644 --- a/modules/database/src/realtime/rebacCache.ts +++ b/modules/database/src/realtime/rebacCache.ts @@ -1,7 +1,6 @@ +import type { RebacDecision } from './types.js'; import type { AuthorizationSdk } from './authorize.js'; -export type RebacDecision = 'allow' | 'deny' | 'unavailable'; - const DEFAULT_TTL_MS = 12_000; const DEFAULT_MAX_ENTRIES = 10_000; diff --git a/modules/database/src/realtime/recovery.ts b/modules/database/src/realtime/recovery.ts index 3bc9c3868..7ff79735b 100644 --- a/modules/database/src/realtime/recovery.ts +++ b/modules/database/src/realtime/recovery.ts @@ -1,5 +1,5 @@ import type { Indexable, ParsedSocketRequest } from '@conduitplatform/grpc-sdk'; -import type { AuthorizationSdk } from './authorize.js'; +import { readDocumentDecision, type AuthorizationSdk } from './authorize.js'; import { authorizedDocumentRoom, parseAuthorizedDocumentRoom } from './rooms.js'; import type { RealtimeSubscriptionTracker } from './subscriptions.js'; @@ -93,12 +93,6 @@ export async function restoreAuthorizedSubscriptions(options: { contextSubs: AuthorizedSub[]; subscriptions: RealtimeSubscriptionTracker; grpcSdk: AuthorizationSdk; - canRead: ( - grpcSdk: AuthorizationSdk, - schema: string, - documentId: string, - userId: string, - ) => Promise; }): Promise<{ leaveRooms: string[] }> { const seen = new Set(); const subs: AuthorizedSub[] = []; @@ -119,14 +113,14 @@ export async function restoreAuthorizedSubscriptions(options: { const leaveRooms: string[] = []; for (const sub of subs) { - const allowed = await options.canRead( + const decision = await readDocumentDecision( options.grpcSdk, sub.schema, sub.documentId, sub.userId, ); const room = authorizedDocumentRoom(sub.schema, sub.documentId, sub.userId); - if (!allowed) { + if (decision === 'deny') { await options.subscriptions.removeAuthorizedDocument( options.socketId, sub.schema, diff --git a/modules/database/src/realtime/sockets.ts b/modules/database/src/realtime/sockets.ts index e2a80931c..3a06ab121 100644 --- a/modules/database/src/realtime/sockets.ts +++ b/modules/database/src/realtime/sockets.ts @@ -78,7 +78,9 @@ export function createSocketHandlers(options: RealtimeSocketOptions) { }, disconnect: async (call: ParsedSocketRequest): Promise => { const reason = call.request.params?.[0]; - if (!isRecoverableDisconnect(reason)) { + if (isRecoverableDisconnect(reason)) { + await options.subscriptions.armRecoverableTtl(call.request.socketId); + } else { await options.subscriptions.disconnect(call.request.socketId); } return { event: 'disconnected', data: { ok: true } }; @@ -90,7 +92,6 @@ export function createSocketHandlers(options: RealtimeSocketOptions) { contextSubs: authorizedSubsFromContext(call.request.context as Indexable), subscriptions: options.subscriptions, grpcSdk: options.grpcSdk as unknown as AuthorizationSdk, - canRead: canReadDocument, }); if (leaveRooms.length > 0) { return { event: 'leave-room', rooms: leaveRooms }; diff --git a/modules/database/src/realtime/subscriptions.ts b/modules/database/src/realtime/subscriptions.ts index da7bfb867..bc54e8efd 100644 --- a/modules/database/src/realtime/subscriptions.ts +++ b/modules/database/src/realtime/subscriptions.ts @@ -1,9 +1,13 @@ +export const RECOVERY_REDIS_TTL_SECONDS = 120; + export type SubscriptionStore = { sadd(key: string, ...members: string[]): Promise; srem(key: string, ...members: string[]): Promise; smembers(key: string): Promise; scard(key: string): Promise; del(...keys: string[]): Promise; + expire(key: string, seconds: number): Promise; + persist(key: string): Promise; }; function socketKey(socketId: string): string { @@ -31,12 +35,15 @@ export class RealtimeSubscriptionTracker { documentId: string, userId: string, ): Promise { - await this.store.sadd( - socketKey(socketId), - subscriptionRecord(schema, documentId, userId), - ); - await this.store.sadd(userDocSocketsKey(schema, documentId, userId), socketId); - await this.store.sadd(docUsersKey(schema, documentId), userId); + const socket = socketKey(socketId); + const userDoc = userDocSocketsKey(schema, documentId, userId); + const docUsers = docUsersKey(schema, documentId); + await this.store.sadd(socket, subscriptionRecord(schema, documentId, userId)); + await this.store.sadd(userDoc, socketId); + await this.store.sadd(docUsers, userId); + await this.store.persist(socket); + await this.store.persist(userDoc); + await this.store.persist(docUsers); } async removeAuthorizedDocument( @@ -89,4 +96,41 @@ export class RealtimeSubscriptionTracker { } await this.store.del(socketKey(socketId)); } + + async armRecoverableTtl( + socketId: string, + ttlSeconds: number = RECOVERY_REDIS_TTL_SECONDS, + ): Promise { + const socket = socketKey(socketId); + const records = await this.store.smembers(socket); + await this.store.expire(socket, ttlSeconds); + for (const record of records) { + try { + const parsed = JSON.parse(record) as { + schema: string; + documentId: string; + userId: string; + }; + const userDoc = userDocSocketsKey( + parsed.schema, + parsed.documentId, + parsed.userId, + ); + const others = (await this.store.smembers(userDoc)).filter(id => id !== socketId); + if (others.length > 0) continue; + await this.store.expire(userDoc, ttlSeconds); + const users = await this.store.smembers( + docUsersKey(parsed.schema, parsed.documentId), + ); + if (users.length <= 1) { + await this.store.expire( + docUsersKey(parsed.schema, parsed.documentId), + ttlSeconds, + ); + } + } catch { + // ignore malformed records + } + } + } } diff --git a/modules/database/src/realtime/types.ts b/modules/database/src/realtime/types.ts index 2b6e3e8c7..004523198 100644 --- a/modules/database/src/realtime/types.ts +++ b/modules/database/src/realtime/types.ts @@ -28,10 +28,13 @@ export type RealtimeStatus = { message?: string; }; +export type RebacDecision = 'allow' | 'deny' | 'unavailable'; + export type OptedInSchema = { name: string; collectionName: string; authorizationEnabled: boolean; + cmsReadEnabled: boolean; }; export type SubscribeRequest = { From b9af5e549b9fda7001d41cbe99aa74a7d7db29cb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 14:50:48 +0000 Subject: [PATCH 5/5] fix(database): serialize leader acquire so concurrent reconcile still watches Two overlapping ensureLeader calls both bumped the lock generation while the first openStream was still reading the resume token, so the watch never attached. Hold an acquiring flag so only one opener runs. --- .../realtime/MongoChangeStreamCoordinator.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts index d13e5a911..45644fa78 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -66,6 +66,7 @@ export class MongoChangeStreamCoordinator { private retryAttempt = 0; private watching = false; private opening = false; + private acquiring = false; private ignoreClose = false; private lockGeneration = 0; private changeQueue: Promise = Promise.resolve(); @@ -151,7 +152,15 @@ export class MongoChangeStreamCoordinator { } return; } + if (this.acquiring) return; + this.acquiring = true; try { + if (this.lock) { + if (!this.watching) { + await this.openStream(); + } + return; + } const acquired = await this.options.grpcSdk.state!.tryAcquireLock( LEADER_LOCK, LOCK_TTL_MS, @@ -161,6 +170,17 @@ export class MongoChangeStreamCoordinator { this.scheduleRetry(); return; } + if (this.lock) { + try { + await this.options.grpcSdk.state!.releaseLock(acquired); + } catch { + // lock may already have expired + } + if (!this.watching) { + await this.openStream(); + } + return; + } try { this.lock = await acquired.extend(LOCK_TTL_MS); } catch { @@ -181,6 +201,8 @@ export class MongoChangeStreamCoordinator { this.lastError = err instanceof Error ? err.message : String(err); this.streamState = 'degraded'; this.scheduleRetry(); + } finally { + this.acquiring = false; } }