From 9c7af8cfa1470c28998edabd27be3f976769cc4b Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 13 Sep 2026 02:36:16 +0300 Subject: [PATCH 1/5] fix(router): Event Relays interventions (EventBus, Hermes, relay manager) (#1604) * fix(router): harden event relays for HA and auth lifetime EventBus uses a single Redis message dispatcher with subscriberId maps, SIGTERM/SIGINT shutdown, and subscribe-after-ACK. Hermes installs engine middleware once, skips empty-room namespace broadcasts, demotes hot-path logs, and re-authenticates recovered /events/ subscriptions. EventRelayManager gains coalesced periodic reconcile, compiled templates, inbound caps, TTL-cached emit-time ReBAC, room eviction, backpressure, preview API, and metrics/docs aligned with the interventions plan. * fix(router): address #1604 re-review P1 and P2 follow-ups Recovery re-auth keeps per-user subscription state across disconnect; emit-time ReBAC distinguishes unavailable vs deny; EventBus drops process signal handlers and adds subscribeAck; manager subscribes only after Redis ACK and broadcasts evictRelayIds on refresh; Hermes backpressure uses local sockets without emit acks; cache is bounded; preview caps sample JSON; handshake matcher rejects ticket/sid/non-namespace paths; socket middleware rebind avoids duplicate registration on first sockets enable. * fix(hermes): restore production tsc for engine middleware chain Type the Socket.IO engine middleware runner as Express NextFunction so recursive callbacks match registerGlobalMiddleware. Use Logger.info for socket trace lines (IConduitLogger has no debug). * fix(router): import Express Response for socket global middleware _rebindSocketGlobalMiddlewares passes handlers typed with fetch Response because Response was not imported from express, failing tsc against Hermes registerSocketGlobalMiddleware. * fix(router): quit EventBus on module shutdown signals Stop event relays and call grpcSdk.bus.quit() from Router.shutdown(), registered on SIGTERM/SIGINT with process exit so Redis teardown is not left to SDK signal handlers. Tighten registerGlobalMiddleware typing and trim handshake helper comment (deslop). * fix(router): pass-3 recovery, scoped emit, and subscription hygiene Recovery re-auth keeps membership on Authorization UNAVAILABLE and only leaves on deny; re-check subscriptions for restored er: rooms via room map. Prune user subscriptions on disconnect when no other socket holds them. Scope relay emits to receivers in the target room; use Engine.IO writeBuffer/writable for backpressure. Drop hot-path Hermes socket info logs. * fix(router): pass-4 recovery map, backpressure, emit tests Store relay subs on socket.data; resolve recovered rooms from context and TTL room map without last-writer userId. Prune room map when unused on disconnect. Backpressure uses writeBuffer depth only. Add Hermes/router tests for room-scoped emit and queue metric. * fix(router): clear eventRelaySubs from socket.data on recovery deny * fix(router): clear socket.data subs on recovery fail-closed leave --- libraries/grpc-sdk/package.json | 3 +- .../grpc-sdk/src/utilities/EventBus.test.ts | 121 +++++++++ libraries/grpc-sdk/src/utilities/EventBus.ts | 156 ++++++++--- libraries/grpc-sdk/tsconfig.test.json | 11 + libraries/hermes/package.json | 1 + libraries/hermes/src/Socket/Socket.ts | 256 ++++++++++++------ .../src/Socket/isSocketHandshake.test.ts | 26 ++ .../hermes/src/Socket/isSocketHandshake.ts | 26 ++ .../src/Socket/resolveEngineNamespacePath.ts | 11 + .../hermes/src/Socket/socketPushUtils.test.ts | 69 +++++ .../hermes/src/Socket/socketPushUtils.ts | 36 +++ libraries/hermes/src/index.ts | 23 +- libraries/hermes/src/interfaces/Socket.ts | 11 + libraries/hermes/src/interfaces/SocketPush.ts | 4 + libraries/hermes/tsconfig.json | 3 +- libraries/hermes/tsconfig.test.json | 17 ++ modules/router/README.mdx | 25 +- modules/router/src/Router.ts | 50 +++- modules/router/src/admin/event-relays.ts | 34 ++- modules/router/src/admin/index.ts | 15 + .../src/event-relays/EventRelayManager.ts | 242 ++++++++++++++--- .../src/event-relays/EventRelaySockets.ts | 68 ++++- modules/router/src/event-relays/authorize.ts | 2 - modules/router/src/event-relays/compile.ts | 50 ++++ modules/router/src/event-relays/constants.ts | 6 + .../router/src/event-relays/follow-up.test.ts | 186 +++++++++++++ .../src/event-relays/interventions.test.ts | 23 ++ .../router/src/event-relays/process.test.ts | 3 + modules/router/src/event-relays/process.ts | 24 +- modules/router/src/event-relays/push.ts | 13 +- .../src/event-relays/rebacCache.test.ts | 43 +++ modules/router/src/event-relays/rebacCache.ts | 108 ++++++++ modules/router/src/event-relays/recovery.ts | 85 ++++++ .../src/event-relays/relaySocketData.ts | 65 +++++ modules/router/src/event-relays/rooms.ts | 4 + .../router/src/event-relays/subscriptions.ts | 224 +++++++++++++++ modules/router/src/index.ts | 18 ++ modules/router/src/metrics/index.ts | 35 +++ modules/router/tsconfig.test.json | 3 + 39 files changed, 1912 insertions(+), 188 deletions(-) create mode 100644 libraries/grpc-sdk/src/utilities/EventBus.test.ts create mode 100644 libraries/grpc-sdk/tsconfig.test.json create mode 100644 libraries/hermes/src/Socket/isSocketHandshake.test.ts create mode 100644 libraries/hermes/src/Socket/isSocketHandshake.ts create mode 100644 libraries/hermes/src/Socket/resolveEngineNamespacePath.ts create mode 100644 libraries/hermes/src/Socket/socketPushUtils.test.ts create mode 100644 libraries/hermes/src/Socket/socketPushUtils.ts create mode 100644 libraries/hermes/tsconfig.test.json create mode 100644 modules/router/src/event-relays/compile.ts create mode 100644 modules/router/src/event-relays/follow-up.test.ts create mode 100644 modules/router/src/event-relays/interventions.test.ts create mode 100644 modules/router/src/event-relays/rebacCache.test.ts create mode 100644 modules/router/src/event-relays/rebacCache.ts create mode 100644 modules/router/src/event-relays/recovery.ts create mode 100644 modules/router/src/event-relays/relaySocketData.ts create mode 100644 modules/router/src/event-relays/subscriptions.ts diff --git a/libraries/grpc-sdk/package.json b/libraries/grpc-sdk/package.json index 25809a993..bbf717933 100644 --- a/libraries/grpc-sdk/package.json +++ b/libraries/grpc-sdk/package.json @@ -25,7 +25,8 @@ "prepublish": "npm run build", "prebuild": "npm run protoc", "build": "rimraf dist && tsup", - "protoc": "sh build.sh" + "protoc": "sh build.sh", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utilities/EventBus.test.js" }, "license": "MIT", "dependencies": { diff --git a/libraries/grpc-sdk/src/utilities/EventBus.test.ts b/libraries/grpc-sdk/src/utilities/EventBus.test.ts new file mode 100644 index 000000000..c1999901d --- /dev/null +++ b/libraries/grpc-sdk/src/utilities/EventBus.test.ts @@ -0,0 +1,121 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { EventBus } from './EventBus.js'; + +type Listener = (channel: string, message: string) => void; + +class FakeRedis { + handlers: Record = {}; + subscribed = new Set(); + failNext = new Set(); + + on(event: string, listener: Listener) { + this.handlers[event] = this.handlers[event] ?? []; + this.handlers[event].push(listener); + } + + subscribe(channel: string, cb?: (err?: Error | null) => void) { + if (this.failNext.has(channel)) { + this.failNext.delete(channel); + cb?.(new Error('subscribe failed')); + return; + } + this.subscribed.add(channel); + cb?.(null); + } + + unsubscribe(_channel: string, cb?: () => void) { + cb?.(); + } + + publish(_channel: string, _message: string) {} + + quit() {} + + emitMessage(channel: string, message: string) { + for (const listener of this.handlers.message ?? []) { + listener(channel, message); + } + } +} + +function createBus() { + const sub = new FakeRedis(); + const pub = new FakeRedis(); + const manager = { + getClient: () => sub, + }; + const bus = new EventBus(manager as never); + (bus as unknown as { _clientSubscriber: FakeRedis })._clientSubscriber = sub; + (bus as unknown as { _clientPublisher: FakeRedis })._clientPublisher = pub; + return { bus, sub }; +} + +describe('EventBus', () => { + it('fires once after deactivate/reactivate on the same channel', () => { + const { bus, sub } = createBus(); + let count = 0; + bus.subscribe( + 'database:update:Order', + () => { + count += 1; + }, + 'relay-a', + ); + bus.unsubscribe('relay-a'); + bus.subscribe( + 'database:update:Order', + () => { + count += 1; + }, + 'relay-a', + ); + sub.emitMessage('database:update:Order', '{"ok":true}'); + assert.equal(count, 1); + }); + + it('keeps the second subscriber when the first is removed', () => { + const { bus, sub } = createBus(); + let first = 0; + let second = 0; + bus.subscribe( + 'chan', + () => { + first += 1; + }, + 'one', + ); + bus.subscribe( + 'chan', + () => { + second += 1; + }, + 'two', + ); + bus.unsubscribe('one'); + sub.emitMessage('chan', 'x'); + assert.equal(first, 0); + assert.equal(second, 1); + }); + + it('subscribeAck rejects when Redis subscribe fails', async () => { + const { bus, sub } = createBus(); + sub.failNext.add('chan'); + await assert.rejects( + () => + bus.subscribeAck( + 'chan', + () => {}, + 'relay-a', + ), + /subscribe failed/, + ); + sub.failNext.delete('chan'); + let count = 0; + await bus.subscribeAck('chan', () => { + count += 1; + }, 'relay-a'); + sub.emitMessage('chan', 'x'); + assert.equal(count, 1); + }); +}); diff --git a/libraries/grpc-sdk/src/utilities/EventBus.ts b/libraries/grpc-sdk/src/utilities/EventBus.ts index 8cd8512f1..9388ad483 100644 --- a/libraries/grpc-sdk/src/utilities/EventBus.ts +++ b/libraries/grpc-sdk/src/utilities/EventBus.ts @@ -3,35 +3,58 @@ import { Cluster, Redis } from 'ioredis'; import crypto from 'crypto'; import { getLogger } from './GrpcSdkContext.js'; +type ChannelCallbacks = Map void>; + export class EventBus { private _clientSubscriber: Redis | Cluster; private _clientPublisher: Redis | Cluster; - private _subscribedChannels: { [listener: string]: ((message: string) => void)[] }; - private _subscribers: { [listener: string]: [string, number] }; + /** channelName -> subscriberId -> callback */ + private _channelCallbacks: Map; + /** subscriberId -> channelName */ + private _subscriberChannels: Map; + /** channels with a successful Redis SUBSCRIBE */ + private _redisSubscribedChannels: Set; + private _subscribeInFlight = new Map>(); private _signature: string; + private _anonymousSubscriberSeq = 0; + private _shuttingDown = false; constructor(redisManager: RedisManager) { - this._subscribedChannels = {}; - this._subscribers = {}; + this._channelCallbacks = new Map(); + this._subscriberChannels = new Map(); + this._redisSubscribedChannels = new Set(); this._clientSubscriber = redisManager.getClient({ keyPrefix: 'bus_' }); this._clientPublisher = redisManager.getClient({ keyPrefix: 'bus_' }); this._signature = crypto.randomBytes(20).toString('hex'); this._clientSubscriber.on('ready', () => { getLogger().log('The Bus is in the station...hehe'); }); + this._clientSubscriber.on('message', (channel: string, message: string) => { + this.dispatch(channel, message); + }); process.on('exit', () => { - this._clientSubscriber.quit(); - this._clientPublisher.quit(); + this.quit(); }); } + quit(): void { + if (this._shuttingDown) return; + this._shuttingDown = true; + void this._clientSubscriber.quit(); + void this._clientPublisher.quit(); + } + unsubscribe(subscriberId: string): void { - if (this._subscribers[subscriberId]) { - const [channelName, index] = this._subscribers[subscriberId]; - this._subscribedChannels[channelName].splice(index, 1); - delete this._subscribers[subscriberId]; - if (this._subscribedChannels[channelName].length === 0) { - delete this._subscribedChannels[channelName]; + const channelName = this._subscriberChannels.get(subscriberId); + if (!channelName) { + return; + } + const callbacks = this._channelCallbacks.get(channelName); + callbacks?.delete(subscriberId); + this._subscriberChannels.delete(subscriberId); + if (callbacks && callbacks.size === 0) { + this._channelCallbacks.delete(channelName); + if (this._redisSubscribedChannels.delete(channelName)) { this._clientSubscriber.unsubscribe(channelName, () => {}); } } @@ -42,43 +65,100 @@ export class EventBus { callback: (message: string) => void, subscriberId?: string, ): void { + void this.subscribeAck(channelName, callback, subscriberId).catch(err => { + getLogger().error( + `EventBus subscribe failed for ${channelName}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }); + } + + async subscribeAck( + channelName: string, + callback: (message: string) => void, + subscriberId?: string, + ): Promise { + if (this._shuttingDown) { + return; + } + const id = + subscriberId ?? + `anon:${channelName}:${++this._anonymousSubscriberSeq}:${crypto.randomBytes(4).toString('hex')}`; if (subscriberId) { - // if subscriberId is provided, and it is already subscribed, unsubscribe it first this.unsubscribe(subscriberId); } - if (this._subscribedChannels[channelName]) { - this._subscribedChannels[channelName].push(callback); - if (subscriberId) { - this._subscribers[subscriberId] = [ - channelName, - this._subscribedChannels[channelName].length - 1, - ]; - } + + let callbacks = this._channelCallbacks.get(channelName); + if (!callbacks) { + callbacks = new Map(); + this._channelCallbacks.set(channelName, callbacks); + } + callbacks.set(id, callback); + this._subscriberChannels.set(id, channelName); + + if (this._redisSubscribedChannels.has(channelName)) { return; } - this._subscribedChannels[channelName] = [callback]; - this._clientSubscriber.subscribe(channelName, () => {}); - const self = this; - this._clientSubscriber.on('message', (channel: string, message: string) => { - if (channel !== channelName) return; - // if the message supports the signature - if (message.indexOf('CND_Signature') !== -1) { - // if the message does not contain this module's signature - if (message.indexOf(self._signature) === -1) { - self._subscribedChannels[channelName].forEach(fn => { - fn(message.split('CND_Signature:')[0]); - }); - } - } else { - self._subscribedChannels[channelName].forEach(fn => { - fn(message); + + let inFlight = this._subscribeInFlight.get(channelName); + if (!inFlight) { + inFlight = new Promise((resolve, reject) => { + this._clientSubscriber.subscribe(channelName, err => { + if (err) { + reject(err); + return; + } + this._redisSubscribedChannels.add(channelName); + resolve(); }); + }).finally(() => { + this._subscribeInFlight.delete(channelName); + }); + this._subscribeInFlight.set(channelName, inFlight); + } + + try { + await inFlight; + } catch (err) { + if (!this._redisSubscribedChannels.has(channelName)) { + this.removeChannelCallbacks(channelName); } - }); + this.unsubscribe(id); + throw err; + } } publish(channelName: string, message: string) { message = message + `CND_Signature:${this._signature}`; this._clientPublisher.publish(channelName, message); } + + private removeChannelCallbacks(channelName: string): void { + const callbacks = this._channelCallbacks.get(channelName); + if (!callbacks) { + return; + } + for (const subId of callbacks.keys()) { + this._subscriberChannels.delete(subId); + } + this._channelCallbacks.delete(channelName); + } + + private dispatch(channel: string, message: string): void { + const callbacks = this._channelCallbacks.get(channel); + if (!callbacks || callbacks.size === 0) { + return; + } + let payload = message; + if (message.indexOf('CND_Signature') !== -1) { + if (message.indexOf(this._signature) !== -1) { + return; + } + payload = message.split('CND_Signature:')[0]; + } + for (const fn of callbacks.values()) { + fn(payload); + } + } } diff --git a/libraries/grpc-sdk/tsconfig.test.json b/libraries/grpc-sdk/tsconfig.test.json new file mode 100644 index 000000000..121161cd7 --- /dev/null +++ b/libraries/grpc-sdk/tsconfig.test.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": ["src/utilities/EventBus.ts", "src/utilities/EventBus.test.ts"] +} diff --git a/libraries/hermes/package.json b/libraries/hermes/package.json index 62f32ab5a..3beffb157 100644 --- a/libraries/hermes/package.json +++ b/libraries/hermes/package.json @@ -11,6 +11,7 @@ "scripts": { "prepublish": "npm run build", "build": "rimraf dist && tsc", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/Socket/*.test.js", "publish": "npm publish", "postbuild": "copyfiles -u 1 src/*.proto src/**/*.json ./dist/" }, diff --git a/libraries/hermes/src/Socket/Socket.ts b/libraries/hermes/src/Socket/Socket.ts index 74ced4e2b..7084fa857 100644 --- a/libraries/hermes/src/Socket/Socket.ts +++ b/libraries/hermes/src/Socket/Socket.ts @@ -16,6 +16,11 @@ import { import ObjectHash from 'object-hash'; import { ConduitError, ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import { buildSocketMiddlewareParams } from './buildSocketMiddlewareParams.js'; +import { resolveEngineNamespacePath } from './resolveEngineNamespacePath.js'; +import { + filterRemoteSocketsByUserAndRooms, + isEngineSocketBackpressured, +} from './socketPushUtils.js'; export class SocketController extends ConduitRouter { private readonly httpServer: httpServer; @@ -78,6 +83,22 @@ export class SocketController extends ConduitRouter { `Socket connection error, context: ${err?.context ?? 'N/A'}`, ); }); + + this.io.engine.use((req: any, res: any, next: NextFunction) => { + req.path = resolveEngineNamespacePath(req); + let index = 0; + const run: NextFunction = err => { + if (err) { + return next(err); + } + const middleware = this.globalMiddlewares[index++]; + if (!middleware) { + return next(); + } + middleware(req, res, run); + }; + run(); + }); } registerGlobalMiddleware( @@ -107,12 +128,6 @@ export class SocketController extends ConduitRouter { this._registeredNamespaces.set(namespace, conduitSocket); const self = this; - this.globalMiddlewares.forEach(middleware => { - self.io.engine.use((req: any, res: any, next: any) => { - req.path = namespace; - middleware(req, res, next); - }); - }); this.io.of(namespace).use((socket, next) => { const context = buildSocketMiddlewareParams(socket); self @@ -129,13 +144,21 @@ export class SocketController extends ConduitRouter { this.io.of(namespace).on('connect', socket => { if (socket.recovered) { - ConduitGrpcSdk.Logger.info( - `Socket recovered: ${socket.id} to namespace: ${namespace}`, - ); + const recovered = conduitSocket.executeRecovered({ + event: 'recovered', + socketId: socket.id, + context: socket.data, + recoveredRooms: [...socket.rooms].filter(room => room.startsWith('er:')), + }); + if (recovered) { + recovered + .then(res => this.handleResponse(res, socket, namespace)) + .catch(e => { + ConduitGrpcSdk.Logger.error(e); + socket.emit('conduit_error', e); + }); + } } else { - ConduitGrpcSdk.Logger.info( - `Socket connected: ${socket.id} to namespace: ${namespace}`, - ); conduitSocket .executeRequest({ event: 'connect', @@ -150,7 +173,6 @@ export class SocketController extends ConduitRouter { } socket.onAny((event, ...args) => { - ConduitGrpcSdk.Logger.info(`Socket event: ${event} from socket: ${socket.id}`); conduitSocket .executeRequest({ event, @@ -166,9 +188,6 @@ export class SocketController extends ConduitRouter { }); socket.on('disconnect', () => { - ConduitGrpcSdk.Logger.info( - `Socket disconnected: ${socket.id} from namespace: ${namespace}`, - ); conduitSocket .executeRequest({ event: 'disconnect', @@ -184,82 +203,162 @@ export class SocketController extends ConduitRouter { }); } - async handleSocketPush(push: SocketPush) { + async handleSocketPush(push: SocketPush): Promise { const localOnly = push.localOnly === true; if (push.event === 'join-room') { - if (push.rooms.length === 0) return; + if (push.rooms.length === 0) return false; const filteredSockets = await this.findAndFilterSockets( push.receivers, push.namespace, localOnly, ); for (const socket of filteredSockets) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} joining rooms: ${push.rooms.join(', ')} in namespace: ${ - push.namespace - }`, - ); socket.join(push.rooms); } + return true; } else if (push.event === 'leave-room') { if (push.rooms && push.rooms.length !== 0) { - const filteredSockets = await this.findAndFilterSockets( - push.receivers, - push.namespace, - localOnly, - ); + const filteredSockets = await this.socketsForRoomPush(push, localOnly); for (const socket of filteredSockets) { for (const room of push.rooms) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} leaving room: ${room} in namespace: ${push.namespace}`, - ); socket.leave(room); } } } + return true; } else if (isInstanceOfEventResponse(push)) { if ( (isNil(push.receivers) || push.receivers.length === 0) && push.rooms.length === 0 ) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${push.event} to all sockets in namespace: ${push.namespace}`, - ); + return false; + } + if (push.receivers.length !== 0) { const nsp = this.io.of(push.namespace); - if (localOnly) { - nsp.local.emit(push.event, push.data); - } else { - nsp.emit(push.event, push.data); + const filteredSockets = await this.findAndFilterSockets( + push.receivers, + push.namespace, + localOnly, + push.rooms, + ); + if (push.skipEmptyRooms && filteredSockets.length === 0) { + return false; } - } else { - if (push.rooms.length !== 0) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${push.event} to rooms: ${push.rooms.join( - ', ', - )} in namespace: ${push.namespace}`, - ); - const target = this.io.of(push.namespace).to(push.rooms); - if (localOnly) { - target.local.emit(push.event, push.data); - } else { - target.emit(push.event, push.data); + for (const remote of filteredSockets) { + const local = localOnly ? nsp.sockets.get(remote.id) : undefined; + if ( + push.boundedEmit && + localOnly && + local && + this.isLocalSocketBackpressured(local) + ) { + ConduitGrpcSdk.Metrics?.increment('event_relays_emit_dropped_total'); + local.disconnect(true); + continue; } + remote.emit(push.event, push.data); } - if (push.receivers.length !== 0) { - const filteredSockets = await this.findAndFilterSockets( - push.receivers, - push.namespace, - localOnly, - ); - for (const socket of filteredSockets) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${push.event} to socket: ${socket.id} in namespace: ${push.namespace}`, - ); - socket.emit(push.event, push.data); + return true; + } + if (push.rooms.length !== 0) { + const emitted = await this.emitEventToRooms(push, localOnly); + if (!emitted) { + return false; + } + } + return true; + } + return false; + } + + async getLocalRoomUserIds(namespace: string, room: string): Promise { + const sockets = await this.io.of(namespace).in(room).local.fetchSockets(); + const userIds = new Set(); + for (const socket of sockets) { + const userId = socket.data?.user?._id; + if (typeof userId === 'string' && userId.length > 0) { + userIds.add(userId); + } + } + return [...userIds]; + } + + async getLocalRoomsWithPrefix(namespace: string, prefix: string): Promise { + const adapter = this.io.of(namespace).adapter as { rooms?: Map }; + const rooms = adapter.rooms; + if (!rooms) { + return []; + } + return [...rooms.keys()].filter(room => room.startsWith(prefix)); + } + + private async emitEventToRooms(push: SocketPush, localOnly: boolean): Promise { + const nsp = this.io.of(push.namespace); + const localSockets = localOnly ? this.localSocketsInRooms(nsp, push.rooms) : []; + if (push.skipEmptyRooms || push.boundedEmit) { + if (localOnly && localSockets.length === 0) { + return false; + } + if (push.boundedEmit && localOnly) { + for (const socket of localSockets) { + if (this.isLocalSocketBackpressured(socket)) { + ConduitGrpcSdk.Metrics?.increment('event_relays_emit_dropped_total'); + socket.disconnect(true); } } } } + + const target = nsp.to(push.rooms); + if (localOnly) { + target.local.emit(push.event, push.data); + } else { + target.emit(push.event, push.data); + } + return true; + } + + private localSocketsInRooms( + nsp: ReturnType, + rooms: string[], + ): Socket[] { + const sockets: Socket[] = []; + for (const socket of nsp.sockets.values()) { + if (rooms.some(room => socket.rooms.has(room))) { + sockets.push(socket); + } + } + return sockets; + } + + private isLocalSocketBackpressured(socket: Socket): boolean { + return isEngineSocketBackpressured( + socket.conn as unknown as { writeBuffer?: unknown[] }, + ); + } + + private async socketsForRoomPush( + push: SocketPush, + localOnly: boolean, + ): Promise[]> { + if (push.receivers.length > 0) { + return this.findAndFilterSockets(push.receivers, push.namespace, localOnly); + } + const nsp = this.io.of(push.namespace); + const seen = new Set(); + const sockets: RemoteSocket[] = []; + for (const room of push.rooms) { + const inRoom = localOnly + ? await nsp.in(room).local.fetchSockets() + : await nsp.in(room).fetchSockets(); + for (const socket of inRoom) { + if (!seen.has(socket.id)) { + seen.add(socket.id); + sockets.push(socket); + } + } + } + return sockets; } private async handleResponse( @@ -269,19 +368,11 @@ export class SocketController extends ConduitRouter { ) { if (res.event === 'join-room') { if (res.rooms && res.rooms.length !== 0) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} joining rooms: ${res.rooms.join( - ', ', - )} in namespace: ${namespace}`, - ); socket.join(res.rooms); } } else if (res.event === 'leave-room') { if (res.rooms && res.rooms.length !== 0) { for (const room of res.rooms) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} leaving room: ${room} in namespace: ${namespace}`, - ); socket.leave(room); } } @@ -290,17 +381,9 @@ export class SocketController extends ConduitRouter { (!res.receivers || res.receivers.length === 0) && (!res.rooms || res.rooms.length === 0) ) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${res.event} to all sockets in namespace: ${namespace}`, - ); socket.emit(res.event, JSON.parse(res.data)); } else { if (res.rooms && res.rooms.length !== 0) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${res.event} to rooms: ${res.rooms.join( - ', ', - )} in namespace: ${namespace}`, - ); this.io.of(namespace).to(res.rooms).emit(res.event, JSON.parse(res.data)); } if (res.receivers && res.receivers.length !== 0) { @@ -309,9 +392,6 @@ export class SocketController extends ConduitRouter { namespace, ); for (const socket of filteredSockets) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${res.event} to socket: ${socket.id} in namespace: ${namespace}`, - ); socket.emit(res.event, JSON.parse(res.data)); } } @@ -323,15 +403,19 @@ export class SocketController extends ConduitRouter { userIds: string[], namespace: string, localOnly: boolean = false, + rooms: string[] = [], ): Promise[]> { const nsp = this.io.of(namespace); const sockets = localOnly ? await nsp.local.fetchSockets() : await nsp.fetchSockets(); - const userIdSet = new Set(userIds); - return sockets.filter(socket => { - if (socket.data && socket.data.user) { - return userIdSet.has(socket.data.user._id); - } - }); + return filterRemoteSocketsByUserAndRooms( + sockets.map(socket => ({ + id: socket.id, + data: socket.data, + rooms: socket.rooms, + })), + userIds, + rooms, + ).map(filtered => sockets.find(s => s.id === filtered.id)!); } protected _refreshRouter(): void { diff --git a/libraries/hermes/src/Socket/isSocketHandshake.test.ts b/libraries/hermes/src/Socket/isSocketHandshake.test.ts new file mode 100644 index 000000000..267de330f --- /dev/null +++ b/libraries/hermes/src/Socket/isSocketHandshake.test.ts @@ -0,0 +1,26 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { isSocketHandshake } from './isSocketHandshake.js'; + +describe('isSocketHandshake', () => { + it('matches Engine.IO handshake query params only', () => { + assert.equal( + isSocketHandshake({ url: '/events/?EIO=4&transport=polling&t=abc' }), + true, + ); + assert.equal(isSocketHandshake({ url: '/realtime/?EIO=4&transport=websocket' }), true); + }); + + it('rejects HTTP routes that merely mention realtime or established sessions', () => { + assert.equal( + isSocketHandshake({ url: '/realtime/ticket?EIO=4&transport=polling' }), + false, + ); + assert.equal(isSocketHandshake({ originalUrl: '/realtime' }), false); + assert.equal(isSocketHandshake({ url: '/graphql?EIO=4&transport=polling' }), false); + assert.equal( + isSocketHandshake({ url: '/events/?EIO=4&transport=polling&sid=abc' }), + false, + ); + }); +}); diff --git a/libraries/hermes/src/Socket/isSocketHandshake.ts b/libraries/hermes/src/Socket/isSocketHandshake.ts new file mode 100644 index 000000000..aa216d2ac --- /dev/null +++ b/libraries/hermes/src/Socket/isSocketHandshake.ts @@ -0,0 +1,26 @@ +export function isSocketHandshake(req: { url?: string; originalUrl?: string }): boolean { + const raw = req.url ?? req.originalUrl ?? ''; + if (!raw) { + return false; + } + const queryIndex = raw.indexOf('?'); + const pathname = queryIndex === -1 ? raw : raw.slice(0, queryIndex); + if (pathname.includes('ticket')) { + return false; + } + const query = queryIndex === -1 ? '' : raw.slice(queryIndex + 1); + if (!query) { + return false; + } + const params = new URLSearchParams(query); + if (!params.has('EIO') || !params.has('transport')) { + return false; + } + if (params.has('sid')) { + return false; + } + if (pathname !== '/' && !pathname.endsWith('/')) { + return false; + } + return true; +} diff --git a/libraries/hermes/src/Socket/resolveEngineNamespacePath.ts b/libraries/hermes/src/Socket/resolveEngineNamespacePath.ts new file mode 100644 index 000000000..e36aad12b --- /dev/null +++ b/libraries/hermes/src/Socket/resolveEngineNamespacePath.ts @@ -0,0 +1,11 @@ +/** + * Derive the Socket.IO namespace path from an Engine.IO handshake request URL. + */ +export function resolveEngineNamespacePath(req: { url?: string }): string { + const url = req.url ?? '/'; + const pathname = url.split('?')[0] ?? '/'; + if (pathname === '/' || pathname === '') { + return '/'; + } + return pathname.endsWith('/') ? pathname : `${pathname}/`; +} diff --git a/libraries/hermes/src/Socket/socketPushUtils.test.ts b/libraries/hermes/src/Socket/socketPushUtils.test.ts new file mode 100644 index 000000000..7f76e2904 --- /dev/null +++ b/libraries/hermes/src/Socket/socketPushUtils.test.ts @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { + filterRemoteSocketsByUserAndRooms, + isEngineSocketBackpressured, + WRITE_BUFFER_PACKET_HIGH_WATER, +} from './socketPushUtils.js'; + +describe('filterRemoteSocketsByUserAndRooms', () => { + const roomA = 'er:relay-1:aaa'; + const roomB = 'er:relay-1:bbb'; + + it('emits only to sockets in the target room for the same user', () => { + const sockets = [ + { + id: 'tab-a', + data: { user: { _id: 'user-1' } }, + rooms: new Set([roomA]), + }, + { + id: 'tab-b', + data: { user: { _id: 'user-1' } }, + rooms: new Set([roomB]), + }, + ]; + + const filtered = filterRemoteSocketsByUserAndRooms(sockets, ['user-1'], [roomA]); + assert.deepEqual( + filtered.map(s => s.id), + ['tab-a'], + ); + }); +}); + +describe('isEngineSocketBackpressured', () => { + it('uses writeBuffer queue depth only', () => { + assert.equal(isEngineSocketBackpressured(undefined), false); + assert.equal( + isEngineSocketBackpressured({ writeBuffer: new Array(WRITE_BUFFER_PACKET_HIGH_WATER) }), + false, + ); + assert.equal( + isEngineSocketBackpressured({ + writeBuffer: new Array(WRITE_BUFFER_PACKET_HIGH_WATER + 1), + }), + true, + ); + assert.equal( + isEngineSocketBackpressured({ + writeBuffer: [], + }), + false, + ); + }); +}); + +describe('SocketController relay emit', () => { + it('does not reference writableLength or client emit ack callbacks', () => { + const socketSource = readFileSync( + path.join(path.dirname(fileURLToPath(import.meta.url)), '../../src/Socket/Socket.ts'), + 'utf8', + ); + assert.doesNotMatch(socketSource, /writableLength/); + assert.doesNotMatch(socketSource, /\.emit\([^)]*,\s*[^,)]+,\s*\(\)\s*=>/); + }); +}); diff --git a/libraries/hermes/src/Socket/socketPushUtils.ts b/libraries/hermes/src/Socket/socketPushUtils.ts new file mode 100644 index 000000000..6b3e14feb --- /dev/null +++ b/libraries/hermes/src/Socket/socketPushUtils.ts @@ -0,0 +1,36 @@ +export const WRITE_BUFFER_PACKET_HIGH_WATER = 64; + +type SocketLike = { + id: string; + data?: { user?: { _id?: string } }; + rooms: Set; +}; + +export function filterRemoteSocketsByUserAndRooms( + sockets: SocketLike[], + userIds: string[], + rooms: string[], +): SocketLike[] { + const userIdSet = new Set(userIds); + const roomSet = rooms.length > 0 ? new Set(rooms) : null; + return sockets.filter(socket => { + if (!socket.data?.user?._id) { + return false; + } + if (!userIdSet.has(socket.data.user._id)) { + return false; + } + if (roomSet) { + return [...roomSet].some(room => socket.rooms.has(room)); + } + return true; + }); +} + +export function isEngineSocketBackpressured( + conn: { writeBuffer?: unknown[] } | undefined, + highWater = WRITE_BUFFER_PACKET_HIGH_WATER, +): boolean { + const pending = conn?.writeBuffer?.length ?? 0; + return pending > highWater; +} diff --git a/libraries/hermes/src/index.ts b/libraries/hermes/src/index.ts index ab17086b2..ac1a9960b 100644 --- a/libraries/hermes/src/index.ts +++ b/libraries/hermes/src/index.ts @@ -235,6 +235,12 @@ export class ConduitRoutingController { } } + registerSocketGlobalMiddleware( + middleware: (req: ConduitRequest, res: Response, next: NextFunction) => void, + ) { + this._socketRouter?.registerGlobalMiddleware(middleware); + } + registerRouteMiddleware(middleware: ConduitMiddleware, moduleUrl: string) { this._restRouter?.registerMiddleware(middleware, moduleUrl); this._graphQLRouter?.registerMiddleware(middleware, moduleUrl); @@ -322,7 +328,20 @@ export class ConduitRoutingController { } async socketPush(data: SocketPush) { - await this._socketRouter?.handleSocketPush(data); + return (await this._socketRouter?.handleSocketPush(data)) ?? false; + } + + getLocalRoomUserIds(namespace: string, room: string): Promise { + return ( + this._socketRouter?.getLocalRoomUserIds(namespace, room) ?? Promise.resolve([]) + ); + } + + getLocalRoomsWithPrefix(namespace: string, prefix: string): Promise { + return ( + this._socketRouter?.getLocalRoomsWithPrefix(namespace, prefix) ?? + Promise.resolve([]) + ); } /** True if any enabled transport would change this route (new or definition changed). */ @@ -451,6 +470,8 @@ export class ConduitRoutingController { } } +export { isSocketHandshake } from './Socket/isSocketHandshake.js'; +export { resolveEngineNamespacePath } from './Socket/resolveEngineNamespacePath.js'; export * from './interfaces/index.js'; export * from './types/index.js'; export * from './classes/index.js'; diff --git a/libraries/hermes/src/interfaces/Socket.ts b/libraries/hermes/src/interfaces/Socket.ts index 204707624..7c7851094 100644 --- a/libraries/hermes/src/interfaces/Socket.ts +++ b/libraries/hermes/src/interfaces/Socket.ts @@ -10,6 +10,7 @@ export interface ConduitSocketParameters { socketId: string; params?: UntypedArray; context?: Indexable; + recoveredRooms?: string[]; } export type ConduitSocketParamTypes = (TYPE | ConduitSocketParamTypes)[]; @@ -19,6 +20,7 @@ export interface ConduitSocketOptions { name?: string; description?: string; middlewares?: string[]; + onRecovered?: ConduitSocketEventHandler; } export type EventResponse = { @@ -85,6 +87,15 @@ export class ConduitSocket { } return Promise.reject('no such event registered'); } + + executeRecovered( + request: ConduitSocketParameters, + ): ConduitSocketHandlerResponse | null { + if (!this._input.onRecovered) { + return null; + } + return this._input.onRecovered(request); + } } export interface EventsProtoDescription { diff --git a/libraries/hermes/src/interfaces/SocketPush.ts b/libraries/hermes/src/interfaces/SocketPush.ts index 8bbabbc85..7d0f572d3 100644 --- a/libraries/hermes/src/interfaces/SocketPush.ts +++ b/libraries/hermes/src/interfaces/SocketPush.ts @@ -5,4 +5,8 @@ export interface SocketPush { rooms: string[]; namespace: string; localOnly?: boolean; + /** Skip emit when the local room has no connected sockets. */ + skipEmptyRooms?: boolean; + /** Drop or disconnect slow clients instead of blocking the caller. */ + boundedEmit?: boolean; } diff --git a/libraries/hermes/tsconfig.json b/libraries/hermes/tsconfig.json index cd32b1ca8..225b927d7 100644 --- a/libraries/hermes/tsconfig.json +++ b/libraries/hermes/tsconfig.json @@ -65,5 +65,6 @@ /* Advanced Options */ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ - } + }, + "exclude": ["src/**/*.test.ts"] } diff --git a/libraries/hermes/tsconfig.test.json b/libraries/hermes/tsconfig.test.json new file mode 100644 index 000000000..8aecbdee9 --- /dev/null +++ b/libraries/hermes/tsconfig.test.json @@ -0,0 +1,17 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": [ + "src/Socket/isSocketHandshake.ts", + "src/Socket/isSocketHandshake.test.ts", + "src/Socket/socketPushUtils.ts", + "src/Socket/socketPushUtils.test.ts" + ], + "exclude": [] +} diff --git a/modules/router/README.mdx b/modules/router/README.mdx index c896f2cee..a0dad3165 100644 --- a/modules/router/README.mdx +++ b/modules/router/README.mdx @@ -92,12 +92,12 @@ import { io } from 'socket.io-client'; const socket = io(`${SOCKET_BASE_URL}/events/`, { path: '/realtime', - extraHeaders: { - authorization: `Bearer ${accessToken}`, - }, + auth: { token: accessToken }, }); -socket.emit('subscribe', relayId, resourceId); +socket.on('connect', () => { + socket.emit('subscribe', relayId, resourceId); +}); socket.on('order-updated', payload => { // payload is the rendered JSON template }); @@ -108,6 +108,20 @@ socket.emit('unsubscribe', relayId, resourceId); - The outbound event name is the relay `socketEvent`. - Placeholders in `messageTemplate` use `{{payload.path}}` against the bus JSON payload. +**Subscribe-only.** Clients cannot publish on `/events/`; modules publish to the Redis bus and relays forward matching messages to subscribed rooms. + +**Authentication and Authorization** are required (`authMiddleware` on subscribe). Authorization is re-checked on a short TTL during delivery; revoked access triggers `leave-room`. + +**Rooms** are server-assigned: `er::`. Conduit has no tenant model and **team ids are not part of room names** — isolation is ReBAC on `User` × `permission` × `resourceType:resourceId`. + +**HA / `localOnly`:** every Router replica subscribes to the bus and emits with `localOnly`, so clients spread across replicas still receive one copy without sticky sessions for delivery. Engine.IO **polling** transport still benefits from sticky sessions or WebSocket-only clients. + +**EventBus self-publish:** the bus ignores this process’s own publishes (signature filter). A relay on a channel the Router publishes will not deliver to local clients for those self-originated messages (typical for module-originated events). + +**Dual delivery warning:** Database `/database/` `change` events and a relay on overlapping bus channels (for example `database:change:*` or CRUD `database:update:*`) can duplicate notifications. CRUD bus payloads include **full documents** — prefer change-stream style payloads when both paths are enabled. + +Use preview to validate templates: `POST /router/event-relays/preview` with `messageTemplate` and `samplePayload` (same renderer as runtime; sample JSON is capped at 256 KiB like inbound bus payloads). + ### Admin API | Method | Path | Description | @@ -115,5 +129,6 @@ socket.emit('unsubscribe', relayId, resourceId); | `GET` | `/router/event-relays` | Paginated list (`skip`, `limit`, `search`) | | `GET` | `/router/event-relays/:id` | Single relay | | `POST` | `/router/event-relays` | Create | -| `PATCH` | `/router/event-relays/:id` | Update | +| `POST` | `/router/event-relays/preview` | Render template against sample JSON | +| `PATCH` | `/router/event-relays/:id` | Update (deactivating evicts rooms) | | `DELETE` | `/router/event-relays/:id` | Delete | diff --git a/modules/router/src/Router.ts b/modules/router/src/Router.ts index 0a3867ed6..78369f6e7 100644 --- a/modules/router/src/Router.ts +++ b/modules/router/src/Router.ts @@ -1,4 +1,4 @@ -import { NextFunction } from 'express'; +import { NextFunction, Response } from 'express'; import { status } from '@grpc/grpc-js'; import { ConduitGrpcSdk, @@ -66,6 +66,11 @@ export default class ConduitDefaultRouter extends ManagedModule { private adminRouter: AdminHandlers; private readonly _routes: string[]; private readonly _globalMiddlewares: string[]; + private readonly _socketGlobalMiddlewareHandlers: (( + req: ConduitRequest, + res: Response, + next: NextFunction, + ) => void)[]; private _grpcRoutes: { [field: string]: RouteT[]; } = {}; @@ -76,12 +81,15 @@ export default class ConduitDefaultRouter extends ManagedModule { private _haInitialized = false; private eventRelayManager: EventRelayManager; private eventsSocket?: ConduitSocket; + private socketsPreviouslyStopped = false; + private securityMiddlewareInitialized = false; constructor(peerManifestRoot?: string) { super('router', peerManifestRoot); this.updateHealth(HealthCheckStatus.UNKNOWN, true); this._routes = []; this._globalMiddlewares = []; + this._socketGlobalMiddlewareHandlers = []; } async onServerStart() { @@ -112,6 +120,12 @@ export default class ConduitDefaultRouter extends ManagedModule { this.eventRelayManager = new EventRelayManager( this.grpcSdk, createEventRelayPusher(data => this._internalRouter.socketPush(data)), + { + getLocalRoomUserIds: (room: string) => + this._internalRouter.getLocalRoomUserIds('/events/', room), + getLocalRoomsWithPrefix: (prefix: string) => + this._internalRouter.getLocalRoomsWithPrefix('/events/', prefix), + }, ); this.adminRouter = new AdminHandlers( this.grpcServer, @@ -133,6 +147,10 @@ export default class ConduitDefaultRouter extends ManagedModule { async onConfig() { const config = ConfigController.getInstance().config; + const shouldRebindSocketGlobals = + config.transports.sockets && + this.socketsPreviouslyStopped && + this.securityMiddlewareInitialized; let atLeastOne = false; if (config.transports.graphql) { this._internalRouter.initGraphQL(); @@ -148,16 +166,24 @@ export default class ConduitDefaultRouter extends ManagedModule { } if (config.transports.sockets) { this._internalRouter.initSockets(); - this.registerEventsNamespace(); await this.eventRelayManager.start(); atLeastOne = true; } else { await this.eventRelayManager?.stop(); this._internalRouter.stopSockets(); + this.socketsPreviouslyStopped = true; } - if (atLeastOne) { + if (atLeastOne && !this.securityMiddlewareInitialized) { this._security.setupMiddlewares(); + this.securityMiddlewareInitialized = true; + } + if (config.transports.sockets) { + if (shouldRebindSocketGlobals) { + this.rebindSocketGlobalMiddlewares(); + } + this.socketsPreviouslyStopped = false; + this.registerEventsNamespace(); } if (!this._sdkRoutes.some(r => r.path === '/ready')) { this.registerRoute(adminRoutes.getReadyRoute()); @@ -335,13 +361,29 @@ export default class ConduitDefaultRouter extends ManagedModule { registerGlobalMiddleware( name: string, - middleware: any, + middleware: (req: ConduitRequest, res: Response, next: NextFunction) => void, socketMiddleware: boolean = false, ) { this._globalMiddlewares.push(name); + if (socketMiddleware) { + this._socketGlobalMiddlewareHandlers.push(middleware); + } this._internalRouter.registerMiddleware(middleware, socketMiddleware); } + private rebindSocketGlobalMiddlewares() { + for (const middleware of this._socketGlobalMiddlewareHandlers) { + this._internalRouter.registerSocketGlobalMiddleware(middleware); + } + } + + async shutdown(): Promise { + if (this.eventRelayManager) { + await this.eventRelayManager.stop(); + } + this.grpcSdk.bus?.quit(); + } + getRegisteredRoutes() { return this._routes; } diff --git a/modules/router/src/admin/event-relays.ts b/modules/router/src/admin/event-relays.ts index 2e56fbd15..442b3efd7 100644 --- a/modules/router/src/admin/event-relays.ts +++ b/modules/router/src/admin/event-relays.ts @@ -10,7 +10,9 @@ import { EventRelay } from '../models/index.js'; import { EventRelayManager } from '../event-relays/EventRelayManager.js'; import { EventRelayInput, validateEventRelayInput } from '../event-relays/validation.js'; import { EventRelayValidationError } from '../event-relays/validationError.js'; +import { renderMessageTemplate } from '../event-relays/template.js'; import { buildSearchQuery, parsePagination } from '../event-relays/search.js'; +import { assertJsonPayloadSize } from '../event-relays/process.js'; export class EventRelayAdmin { constructor(private readonly manager: EventRelayManager) {} @@ -52,6 +54,23 @@ export class EventRelayAdmin { return relay; } + async previewEventRelay(call: ParsedRouterRequest): Promise { + const { messageTemplate, samplePayload } = call.request.params as { + messageTemplate: unknown; + samplePayload: unknown; + }; + try { + assertJsonPayloadSize(samplePayload); + const rendered = renderMessageTemplate(messageTemplate, samplePayload); + return { rendered }; + } catch (err) { + if (err instanceof EventRelayValidationError) { + throw new GrpcError(status.INVALID_ARGUMENT, err.message); + } + throw err; + } + } + async patchEventRelay(call: ParsedRouterRequest): Promise { const existing = await EventRelay.getInstance().findOne({ _id: call.request.params.id, @@ -89,7 +108,18 @@ export class EventRelayAdmin { ...input, messageTemplate: input.messageTemplate as EventRelay['messageTemplate'], }); - await this.manager.notifyChanged(); + const evictRelayIds: string[] = []; + if (!input.active) { + evictRelayIds.push(existing._id); + } else if ( + input.permission !== existing.permission || + input.resourceType !== existing.resourceType + ) { + evictRelayIds.push(existing._id); + } + await this.manager.notifyChanged({ + evictRelayIds: evictRelayIds.length ? evictRelayIds : undefined, + }); return updated!; } @@ -101,7 +131,7 @@ export class EventRelayAdmin { throw new GrpcError(status.NOT_FOUND, 'Event relay not found'); } await EventRelay.getInstance().deleteOne({ _id: existing._id }); - await this.manager.notifyChanged(); + await this.manager.notifyChanged({ evictRelayIds: [existing._id] }); return { message: 'Event relay deleted' }; } } diff --git a/modules/router/src/admin/index.ts b/modules/router/src/admin/index.ts index 0c3231378..e943c25fa 100644 --- a/modules/router/src/admin/index.ts +++ b/modules/router/src/admin/index.ts @@ -242,6 +242,21 @@ export class AdminHandlers { }), this.eventRelayAdmin.deleteEventRelay.bind(this.eventRelayAdmin), ); + this.routingManager.route( + { + path: '/event-relays/preview', + action: ConduitRouteActions.POST, + description: `Renders a relay message template against sample bus JSON (same engine as runtime).`, + bodyParams: { + messageTemplate: ConduitJson.Required, + samplePayload: ConduitJson.Required, + }, + }, + new ConduitRouteReturnDefinition('PreviewEventRelay', { + rendered: ConduitJson.Required, + }), + this.eventRelayAdmin.previewEventRelay.bind(this.eventRelayAdmin), + ); this.routingManager.registerRoutes(); } } diff --git a/modules/router/src/event-relays/EventRelayManager.ts b/modules/router/src/event-relays/EventRelayManager.ts index 80cb809b7..905b0180e 100644 --- a/modules/router/src/event-relays/EventRelayManager.ts +++ b/modules/router/src/event-relays/EventRelayManager.ts @@ -3,70 +3,170 @@ import { EventRelay } from '../models/index.js'; import { EVENT_RELAY_REFRESH_CHANNEL, EVENT_RELAY_SUBSCRIBER_PREFIX, + RECONCILE_INTERVAL_MS, + RELAY_REBAC_TTL_MS, } from './constants.js'; import { groupRelaysByChannel, planChannelSubscriptions } from './channels.js'; -import { buildRelayEmissions, parseBusPayload } from './process.js'; +import { parseBusPayload } from './process.js'; import { EventRelayPusher } from './push.js'; +import { compileRelay, CompiledRelay } from './compile.js'; +import { checkRebacBatch, RelayRebacCache } from './rebacCache.js'; +import { eventRelayRoomPrefix } from './rooms.js'; +import { removeSubscriptionsForRelay } from './subscriptions.js'; export type { EventRelayPusher } from './push.js'; export { createEventRelayPusher } from './push.js'; +export type EventRelaySocketAccess = { + getLocalRoomUserIds: (room: string) => Promise; + getLocalRoomsWithPrefix: (prefix: string) => Promise; +}; + +type RefreshPayload = { + evictRelayIds?: string[]; +}; + export class EventRelayManager { - private readonly relaysByChannel = new Map(); + private readonly relaysById = new Map(); + private readonly relaysByChannel = new Map(); private readonly subscribedChannels = new Set(); + private readonly rebacCache = new RelayRebacCache(RELAY_REBAC_TTL_MS); private started = false; + private reconcilePending = false; + private reconciling = false; + private reconcileTimer?: NodeJS.Timeout; constructor( private readonly grpcSdk: ConduitGrpcSdk, private readonly push: EventRelayPusher, + private readonly sockets: EventRelaySocketAccess, ) {} async start(): Promise { if (!this.started) { this.grpcSdk.bus?.subscribe( EVENT_RELAY_REFRESH_CHANNEL, - () => { - this.reconcile().catch(err => { - ConduitGrpcSdk.Logger.error(err as Error); - }); + message => { + void this.onRefreshMessage(message); }, 'router-event-relays-refresh', ); + this.reconcileTimer = setInterval(() => { + void this.reconcile(); + }, RECONCILE_INTERVAL_MS); this.started = true; } await this.reconcile(); } async stop(): Promise { + if (this.reconcileTimer) { + clearInterval(this.reconcileTimer); + this.reconcileTimer = undefined; + } for (const channel of [...this.subscribedChannels]) { this.grpcSdk.bus?.unsubscribe(`${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`); this.subscribedChannels.delete(channel); } this.relaysByChannel.clear(); + this.relaysById.clear(); + this.rebacCache.clear(); if (this.started) { this.grpcSdk.bus?.unsubscribe('router-event-relays-refresh'); this.started = false; } + ConduitGrpcSdk.Metrics?.set('event_relays_active_total', 0); + ConduitGrpcSdk.Metrics?.set('event_relays_subscribed_channels_total', 0); } - async notifyChanged(): Promise { + async notifyChanged(options?: { evictRelayIds?: string[] }): Promise { + if (options?.evictRelayIds?.length) { + for (const relayId of options.evictRelayIds) { + await this.evictRelayRooms(relayId); + } + } if (this.started) { await this.reconcile(); } - this.grpcSdk.bus?.publish(EVENT_RELAY_REFRESH_CHANNEL, ''); + const payload: RefreshPayload = { + evictRelayIds: options?.evictRelayIds ?? [], + }; + this.grpcSdk.bus?.publish(EVENT_RELAY_REFRESH_CHANNEL, JSON.stringify(payload)); + } + + async evictRelayRooms(relayId: string): Promise { + const prefix = eventRelayRoomPrefix(relayId); + const rooms = await this.sockets.getLocalRoomsWithPrefix(prefix); + if (rooms.length === 0) { + removeSubscriptionsForRelay(relayId); + return; + } + await this.push('leave-room', undefined, rooms); + removeSubscriptionsForRelay(relayId); + } + + getActiveRelay(id: string): EventRelay | undefined { + return this.relaysById.get(id); + } + + private async onRefreshMessage(raw: string): Promise { + let payload: RefreshPayload = {}; + if (raw.trim()) { + try { + payload = JSON.parse(raw) as RefreshPayload; + } catch { + payload = {}; + } + } + if (payload.evictRelayIds?.length) { + for (const relayId of payload.evictRelayIds) { + await this.evictRelayRooms(relayId); + } + } + await this.reconcile(); } async reconcile(): Promise { + this.reconcilePending = true; + if (this.reconciling) { + return; + } + this.reconciling = true; + try { + while (this.reconcilePending) { + this.reconcilePending = false; + await this.runReconcile(); + } + } finally { + this.reconciling = false; + } + } + + private async runReconcile(): Promise { + const previousIds = new Set(this.relaysById.keys()); const relays = await EventRelay.getInstance().findMany({ active: true }); - const next = groupRelaysByChannel(relays); + const nextByChannel = groupRelaysByChannel(relays); const { toSubscribe, toUnsubscribe } = planChannelSubscriptions( this.subscribedChannels, - next.keys(), + nextByChannel.keys(), ); + this.relaysById.clear(); this.relaysByChannel.clear(); - for (const [channel, list] of next) { - this.relaysByChannel.set(channel, list); + for (const relay of relays) { + this.relaysById.set(relay._id, relay); + } + for (const [channel, channelRelays] of nextByChannel) { + this.relaysByChannel.set( + channel, + channelRelays.map(relay => compileRelay(relay)), + ); + } + + for (const id of previousIds) { + if (!this.relaysById.has(id)) { + await this.evictRelayRooms(id); + } } for (const channel of toUnsubscribe) { @@ -75,21 +175,27 @@ export class EventRelayManager { } for (const channel of toSubscribe) { - this.grpcSdk.bus?.subscribe( - channel, - message => this.onBusMessage(channel, message), - `${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`, - ); - this.subscribedChannels.add(channel); + try { + await this.grpcSdk.bus?.subscribeAck( + channel, + message => this.onBusMessage(channel, message), + `${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`, + ); + this.subscribedChannels.add(channel); + } catch (err) { + ConduitGrpcSdk.Logger.error( + `Event relay failed to subscribe to bus channel ${channel}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } } - } - getActiveRelay(id: string): EventRelay | undefined { - for (const relays of this.relaysByChannel.values()) { - const match = relays.find(relay => relay._id === id); - if (match) return match; - } - return undefined; + ConduitGrpcSdk.Metrics?.set('event_relays_active_total', this.relaysById.size); + ConduitGrpcSdk.Metrics?.set( + 'event_relays_subscribed_channels_total', + this.subscribedChannels.size, + ); } private onBusMessage(channel: string, message: string): void { @@ -103,6 +209,13 @@ export class EventRelayManager { payload = parseBusPayload(message); } catch (err) { ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); + if ( + err instanceof Error && + err.message.includes('exceeds') && + err.message.includes('bytes') + ) { + ConduitGrpcSdk.Metrics?.increment('event_relays_inbound_dropped_total'); + } ConduitGrpcSdk.Logger.error( `Event relay failed to parse payload for ${channel}: ${ err instanceof Error ? err.message : String(err) @@ -111,27 +224,74 @@ export class EventRelayManager { return; } - const { emissions, failures } = buildRelayEmissions(relays, payload); - for (const failure of failures) { + for (const relay of relays) { + void this.emitCompiledRelay(relay, payload, channel); + } + } + + private async emitCompiledRelay( + relay: CompiledRelay, + payload: unknown, + channel: string, + ): Promise { + let room: string; + let data: unknown; + let resourceId: string; + try { + ({ room, data, resourceId } = relay.buildEmission(payload)); + } catch (err) { ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); ConduitGrpcSdk.Logger.warn( - `Event relay ${failure.relayId} skipped on ${failure.busEvent}: ${failure.reason}`, + `Event relay ${relay.relayId} skipped on ${relay.busEvent}: ${ + err instanceof Error ? err.message : String(err) + }`, ); + return; } - for (const emission of emissions) { - this.push(emission.socketEvent, emission.data, [emission.room]).then( - () => { - ConduitGrpcSdk.Metrics?.increment('event_relays_emitted_total'); - }, - err => { - ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); - ConduitGrpcSdk.Logger.error( - `Event relay ${emission.relayId} emit failed on ${channel}: ${ - err instanceof Error ? err.message : String(err) - }`, - ); - }, + const userIds = await this.sockets.getLocalRoomUserIds(room); + if (userIds.length === 0) { + ConduitGrpcSdk.Metrics?.increment('event_relays_empty_room_total'); + return; + } + + const decisions = await checkRebacBatch( + this.rebacCache, + this.grpcSdk, + userIds, + relay.permission, + relay.resourceType, + resourceId, + ); + + const allowedUsers: string[] = []; + for (const userId of userIds) { + const decision = decisions.get(userId) ?? 'unavailable'; + if (decision === 'allow') { + allowedUsers.push(userId); + } else if (decision === 'deny') { + await this.push('leave-room', undefined, [room], [userId]); + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + } + } + + if (allowedUsers.length === 0) { + return; + } + + try { + const emitted = await this.push(relay.socketEvent, data, [room], allowedUsers); + if (emitted) { + ConduitGrpcSdk.Metrics?.increment('event_relays_emitted_total'); + } else { + ConduitGrpcSdk.Metrics?.increment('event_relays_empty_room_total'); + } + } catch (err) { + ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); + ConduitGrpcSdk.Logger.error( + `Event relay ${relay.relayId} emit failed on ${channel}: ${ + err instanceof Error ? err.message : String(err) + }`, ); } } diff --git a/modules/router/src/event-relays/EventRelaySockets.ts b/modules/router/src/event-relays/EventRelaySockets.ts index 32e5fec2a..d9fe6f88c 100644 --- a/modules/router/src/event-relays/EventRelaySockets.ts +++ b/modules/router/src/event-relays/EventRelaySockets.ts @@ -1,11 +1,27 @@ import { status } from '@grpc/grpc-js'; import { ConduitGrpcSdk, GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; import { ConduitSocket, ConduitSocketEvent } from '@conduitplatform/hermes'; -import { EVENTS_NAMESPACE } from './constants.js'; +import { + EVENTS_NAMESPACE, + MAX_SUBSCRIBE_PER_MINUTE, +} from './constants.js'; import { EventRelayManager } from './EventRelayManager.js'; import { eventRelayRoom } from './rooms.js'; import { validateResourceId } from './validation.js'; import { authorizeRelaySubscription, toSubscriptionError } from './authorize.js'; +import { reauthorizeRecoveredSubscriptions } from './recovery.js'; +import { + releaseSocketSubscriptions, + removeSubscription, + trackSubscription, + _clearEventRelaySubscriptionStateForTests, +} from './subscriptions.js'; +import { + persistRelaySubscriptionOnContext, + removeRelaySubscriptionFromContext, +} from './relaySocketData.js'; + +const subscribeTimestamps = new Map(); export function createEventsSocket( grpcSdk: ConduitGrpcSdk, @@ -20,16 +36,40 @@ export function createEventsSocket( events.set('disconnect', { name: 'disconnect', - handler: async () => ({ event: 'leave-room', rooms: [] }), + handler: async request => { + const userId = request.context?.user?._id as string | undefined; + releaseSocketSubscriptions(request.socketId, userId); + subscribeTimestamps.delete(request.socketId); + return { event: 'leave-room', rooms: [] }; + }, }); events.set('subscribe', { name: 'subscribe', params: [TYPE.String, TYPE.String], handler: async request => { + assertSubscribeRateLimit(request.socketId); const userId = request.context?.user?._id as string | undefined; + if (!userId) { + throw new GrpcError(status.UNAUTHENTICATED, 'Authentication required'); + } const [relayId, resourceId] = request.params ?? []; + const validatedResourceId = validateResourceId(resourceId); const room = await authorizeOrThrow(grpcSdk, manager, userId, relayId, resourceId); + try { + trackSubscription( + request.socketId, + userId, + String(relayId), + validatedResourceId, + ); + persistRelaySubscriptionOnContext(request.context, String(relayId), validatedResourceId); + } catch (err) { + throw new GrpcError( + status.RESOURCE_EXHAUSTED, + err instanceof Error ? err.message : 'Subscription limit exceeded', + ); + } return { event: 'join-room', rooms: [room] }; }, }); @@ -38,12 +78,15 @@ export function createEventsSocket( name: 'unsubscribe', params: [TYPE.String, TYPE.String], handler: async request => { + const userId = request.context?.user?._id as string | undefined; const [relayId, resourceId] = request.params ?? []; if (typeof relayId !== 'string' || relayId.trim() === '') { throw new GrpcError(status.INVALID_ARGUMENT, 'Relay ID is required'); } try { const validatedResourceId = validateResourceId(resourceId); + removeSubscription(request.socketId, userId, relayId, validatedResourceId); + removeRelaySubscriptionFromContext(request.context, relayId, validatedResourceId); return { event: 'leave-room', rooms: [eventRelayRoom(relayId, validatedResourceId)], @@ -60,11 +103,26 @@ export function createEventsSocket( name: 'eventRelays', description: 'Declarative bus-to-socket event relays', middlewares: ['authMiddleware'], + onRecovered: async request => + reauthorizeRecoveredSubscriptions(grpcSdk, manager, request), }, events, ); } +function assertSubscribeRateLimit(socketId: string): void { + const now = Date.now(); + const windowStart = now - 60_000; + const timestamps = (subscribeTimestamps.get(socketId) ?? []).filter( + t => t >= windowStart, + ); + if (timestamps.length >= MAX_SUBSCRIBE_PER_MINUTE) { + throw new GrpcError(status.RESOURCE_EXHAUSTED, 'Subscribe rate limit exceeded'); + } + timestamps.push(now); + subscribeTimestamps.set(socketId, timestamps); +} + async function authorizeOrThrow( grpcSdk: ConduitGrpcSdk, manager: EventRelayManager, @@ -94,3 +152,9 @@ function toGrpcError(err: unknown): GrpcError { } export { authorizeRelaySubscription } from './authorize.js'; + +/** @internal test helper */ +export function _clearEventRelaySocketStateForTests(): void { + _clearEventRelaySubscriptionStateForTests(); + subscribeTimestamps.clear(); +} diff --git a/modules/router/src/event-relays/authorize.ts b/modules/router/src/event-relays/authorize.ts index 852cf5595..8de7cc005 100644 --- a/modules/router/src/event-relays/authorize.ts +++ b/modules/router/src/event-relays/authorize.ts @@ -62,7 +62,6 @@ export async function authorizeRelaySubscription( } if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { - onDenied?.(); throw new RelaySubscriptionError(status.UNAVAILABLE, 'Authorization is unavailable'); } @@ -75,7 +74,6 @@ export async function authorizeRelaySubscription( }); allowed = decision.allow; } catch { - onDenied?.(); throw new RelaySubscriptionError(status.UNAVAILABLE, 'Authorization check failed'); } diff --git a/modules/router/src/event-relays/compile.ts b/modules/router/src/event-relays/compile.ts new file mode 100644 index 000000000..bfe636592 --- /dev/null +++ b/modules/router/src/event-relays/compile.ts @@ -0,0 +1,50 @@ +import { requireOwnPath } from './path.js'; +import { eventRelayRoom } from './rooms.js'; +import { renderMessageTemplate } from './template.js'; +import { validateResourceId } from './validation.js'; + +export type RelayCompileInput = { + _id: string; + busEvent: string; + socketEvent: string; + resourceIdPath: string; + messageTemplate: unknown; + permission: string; + resourceType: string; +}; + +export type CompiledRelay = { + relayId: string; + busEvent: string; + socketEvent: string; + permission: string; + resourceType: string; + buildEmission: (payload: unknown) => { + room: string; + data: unknown; + resourceId: string; + }; +}; + +export function compileRelay(relay: RelayCompileInput): CompiledRelay { + const resourceIdPath = relay.resourceIdPath; + const template = relay.messageTemplate; + return { + relayId: relay._id, + busEvent: relay.busEvent, + socketEvent: relay.socketEvent, + permission: relay.permission, + resourceType: relay.resourceType, + buildEmission: (payload: unknown) => { + const resourceId = validateResourceId( + requireOwnPath(payload, resourceIdPath, 'Resource ID path'), + ); + const data = renderMessageTemplate(template, payload); + return { + room: eventRelayRoom(relay._id, resourceId), + data, + resourceId, + }; + }, + }; +} diff --git a/modules/router/src/event-relays/constants.ts b/modules/router/src/event-relays/constants.ts index e283da0dd..ce80f6194 100644 --- a/modules/router/src/event-relays/constants.ts +++ b/modules/router/src/event-relays/constants.ts @@ -15,6 +15,12 @@ export const MAX_RESOURCE_ID_LENGTH = 128; export const MAX_RESOURCE_ID_PATH_LENGTH = 128; export const MAX_PERMISSION_LENGTH = 64; +export const MAX_INBOUND_BUS_BYTES = 256 * 1024; +export const RECONCILE_INTERVAL_MS = 30_000; +export const MAX_SUBSCRIBE_PER_MINUTE = 30; +export const MAX_ROOMS_PER_SOCKET = 32; +export const RELAY_REBAC_TTL_MS = 12_000; + export const FORBIDDEN_PATH_SEGMENTS = new Set(['__proto__', 'constructor', 'prototype']); export const RESERVED_SOCKET_EVENTS = new Set([ diff --git a/modules/router/src/event-relays/follow-up.test.ts b/modules/router/src/event-relays/follow-up.test.ts new file mode 100644 index 000000000..fea6d438e --- /dev/null +++ b/modules/router/src/event-relays/follow-up.test.ts @@ -0,0 +1,186 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { + releaseSocketSubscriptions, + removeSubscriptionsForRelay, + subscriptionsForRecoveredRooms, + subscriptionsForUser, + trackSubscription, + _clearEventRelaySubscriptionStateForTests, +} from './subscriptions.js'; +import { eventRelayRoom } from './rooms.js'; +import { reauthorizeRecoveredSubscriptions } from './recovery.js'; +import { RelayLookup } from './authorize.js'; + +describe('event relay subscription tracking', () => { + it('clears user map entries when the only socket disconnects', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-a', 'user-1'); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); + + it('keeps user map entries when another socket still holds the subscription', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-1', 'relay-1', 'order-1'); + trackSubscription('socket-b', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-a', 'user-1'); + assert.deepEqual(subscriptionsForUser('user-1'), [ + { relayId: 'relay-1', resourceId: 'order-1' }, + ]); + }); + + it('clears relay subscriptions on eviction', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-1', 'relay-1', 'order-1'); + removeSubscriptionsForRelay('relay-1'); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); +}); + +describe('recovery re-authorization', () => { + const room = eventRelayRoom('relay-1', 'order-1'); + const relay = { + _id: 'relay-1', + permission: 'read', + resourceType: 'Order', + }; + + it('keeps restored rooms when Authorization is unavailable', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('old-socket', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('old-socket', 'user-1'); + + const manager = { + getActiveRelay: (id: string) => (id === 'relay-1' ? relay : undefined), + } as unknown as RelayLookup; + + const grpcSdk = { + isAvailable: () => false, + authorization: null, + }; + + const result = await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'new-socket', + context: { + user: { _id: 'user-1' }, + eventRelaySubs: [{ relayId: 'relay-1', resourceId: 'order-1' }], + }, + recoveredRooms: [room], + }); + + assert.deepEqual(result, { event: 'join-room', rooms: [] }); + assert.deepEqual(subscriptionsForUser('user-1'), [ + { relayId: 'relay-1', resourceId: 'order-1' }, + ]); + }); + + it('leaves restored rooms on permission deny', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('old-socket', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('old-socket', 'user-1'); + + const manager = { + getActiveRelay: (id: string) => (id === 'relay-1' ? relay : undefined), + } as unknown as RelayLookup; + + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async () => ({ allow: false }), + }, + }; + + const result = await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'new-socket', + context: { + user: { _id: 'user-1' }, + eventRelaySubs: [{ relayId: 'relay-1', resourceId: 'order-1' }], + }, + recoveredRooms: [room], + }); + + assert.deepEqual(result, { event: 'leave-room', rooms: [room] }); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); + + it('does not re-auth rooms outside the recovered socket session', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('old-socket', 'user-1', 'relay-1', 'order-1'); + trackSubscription('old-socket', 'user-1', 'relay-2', 'order-2'); + releaseSocketSubscriptions('old-socket', 'user-1'); + + const manager = { + getActiveRelay: (id: string) => + id === 'relay-2' + ? { _id: 'relay-2', permission: 'read', resourceType: 'Order' } + : undefined, + } as unknown as RelayLookup; + + let canCalls = 0; + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async () => { + canCalls++; + return { allow: false }; + }, + }, + }; + + const otherRoom = eventRelayRoom('relay-2', 'order-2'); + await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'new-socket', + context: { + user: { _id: 'user-1' }, + eventRelaySubs: [{ relayId: 'relay-2', resourceId: 'order-2' }], + }, + recoveredRooms: [otherRoom], + }); + + assert.equal(canCalls, 1); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); + + it('re-auths the first user after a second user subscribed to the same room', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-a', 'relay-1', 'order-1'); + trackSubscription('socket-b', 'user-b', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-b', 'user-b'); + + let checkedSubject = ''; + const manager = { + getActiveRelay: (id: string) => (id === 'relay-1' ? relay : undefined), + } as unknown as RelayLookup; + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async (request: { subject: string }) => { + checkedSubject = request.subject; + return { allow: true }; + }, + }, + }; + + await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'socket-a-new', + context: { user: { _id: 'user-a' } }, + recoveredRooms: [room], + }); + + assert.equal(checkedSubject, 'User:user-a'); + }); + + it('prefers socket.data subscriptions when the room map was pruned on disconnect', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-a', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-a', 'user-a'); + assert.deepEqual(subscriptionsForRecoveredRooms([room]), []); + assert.deepEqual( + subscriptionsForRecoveredRooms([room], [ + { relayId: 'relay-1', resourceId: 'order-1' }, + ]), + [{ relayId: 'relay-1', resourceId: 'order-1' }], + ); + }); +}); diff --git a/modules/router/src/event-relays/interventions.test.ts b/modules/router/src/event-relays/interventions.test.ts new file mode 100644 index 000000000..f9e37f644 --- /dev/null +++ b/modules/router/src/event-relays/interventions.test.ts @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { MAX_INBOUND_BUS_BYTES } from './constants.js'; +import { parseBusPayload } from './process.js'; +import { renderMessageTemplate } from './template.js'; + +describe('parseBusPayload inbound cap', () => { + it('drops payloads larger than 256KiB before JSON.parse', () => { + const oversized = JSON.stringify({ blob: 'x'.repeat(MAX_INBOUND_BUS_BYTES) }); + assert.throws(() => parseBusPayload(oversized), /exceeds/); + }); +}); + +describe('preview renderer parity', () => { + it('renders templates the same as runtime processing', () => { + const payload = { documentId: 'doc-1', nested: { value: 2 } }; + const template = { id: '{{payload.documentId}}', n: '{{payload.nested.value}}' }; + assert.deepEqual(renderMessageTemplate(template, payload), { + id: 'doc-1', + n: 2, + }); + }); +}); diff --git a/modules/router/src/event-relays/process.test.ts b/modules/router/src/event-relays/process.test.ts index cf3c1c10b..c811bcec5 100644 --- a/modules/router/src/event-relays/process.test.ts +++ b/modules/router/src/event-relays/process.test.ts @@ -69,6 +69,7 @@ describe('createEventRelayPusher', () => { const calls: unknown[] = []; const push = createEventRelayPusher(async data => { calls.push(data); + return true; }); await push('order-updated', { id: '1' }, ['room-1']); assert.deepEqual(calls, [ @@ -79,6 +80,8 @@ describe('createEventRelayPusher', () => { rooms: ['room-1'], namespace: EVENTS_NAMESPACE, localOnly: true, + skipEmptyRooms: true, + boundedEmit: true, }, ]); }); diff --git a/modules/router/src/event-relays/process.ts b/modules/router/src/event-relays/process.ts index 858c9f6d0..ebe6ec1ec 100644 --- a/modules/router/src/event-relays/process.ts +++ b/modules/router/src/event-relays/process.ts @@ -3,6 +3,7 @@ import { eventRelayRoom } from './rooms.js'; import { renderMessageTemplate } from './template.js'; import { validateResourceId } from './validation.js'; import { EventRelayValidationError } from './validationError.js'; +import { MAX_INBOUND_BUS_BYTES } from './constants.js'; export type RelayProcessInput = { _id: string; @@ -31,10 +32,16 @@ export type ProcessResult = { failures: RelayFailure[]; }; -export function parseBusPayload(rawMessage: string): unknown { +export function parseBusPayload( + rawMessage: string, + maxBytes: number = MAX_INBOUND_BUS_BYTES, +): unknown { if (typeof rawMessage !== 'string' || rawMessage.trim() === '') { throw new EventRelayValidationError('Bus payload is empty'); } + if (Buffer.byteLength(rawMessage, 'utf8') > maxBytes) { + throw new EventRelayValidationError(`Bus payload exceeds ${maxBytes} bytes`); + } try { return JSON.parse(rawMessage); } catch { @@ -42,6 +49,21 @@ export function parseBusPayload(rawMessage: string): unknown { } } +export function assertJsonPayloadSize( + payload: unknown, + maxBytes: number = MAX_INBOUND_BUS_BYTES, +): void { + let serialized: string; + try { + serialized = JSON.stringify(payload); + } catch { + throw new EventRelayValidationError('Sample payload must be valid JSON'); + } + if (Buffer.byteLength(serialized, 'utf8') > maxBytes) { + throw new EventRelayValidationError(`Sample payload exceeds ${maxBytes} bytes`); + } +} + export function buildRelayEmissions( relays: RelayProcessInput[], payload: unknown, diff --git a/modules/router/src/event-relays/push.ts b/modules/router/src/event-relays/push.ts index 79d9911b2..e4bed5062 100644 --- a/modules/router/src/event-relays/push.ts +++ b/modules/router/src/event-relays/push.ts @@ -4,7 +4,8 @@ export type EventRelayPusher = ( event: string, data: unknown, rooms: string[], -) => Promise; + receivers?: string[], +) => Promise; export type SocketPushFn = (data: { event: string; @@ -13,16 +14,20 @@ export type SocketPushFn = (data: { rooms: string[]; namespace: string; localOnly?: boolean; -}) => Promise; + skipEmptyRooms?: boolean; + boundedEmit?: boolean; +}) => Promise; export function createEventRelayPusher(socketPush: SocketPushFn): EventRelayPusher { - return (event, data, rooms) => + return async (event, data, rooms, receivers = []) => socketPush({ event, data, - receivers: [], + receivers, rooms, namespace: EVENTS_NAMESPACE, localOnly: true, + skipEmptyRooms: event !== 'leave-room', + boundedEmit: event !== 'leave-room', }); } diff --git a/modules/router/src/event-relays/rebacCache.test.ts b/modules/router/src/event-relays/rebacCache.test.ts new file mode 100644 index 000000000..df20365d5 --- /dev/null +++ b/modules/router/src/event-relays/rebacCache.test.ts @@ -0,0 +1,43 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { RelayRebacCache } from './rebacCache.js'; + +describe('RelayRebacCache', () => { + it('denies after TTL when authorization revokes access', async () => { + let allow = true; + const cache = new RelayRebacCache(10); + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async () => ({ allow }), + }, + }; + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'allow', + ); + allow = false; + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'allow', + ); + await new Promise(resolve => setTimeout(resolve, 15)); + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'deny', + ); + }); + + it('returns unavailable without caching when Authorization is down', async () => { + const cache = new RelayRebacCache(10_000); + const grpcSdk = { isAvailable: () => false, authorization: null }; + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'unavailable', + ); + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'unavailable', + ); + }); +}); diff --git a/modules/router/src/event-relays/rebacCache.ts b/modules/router/src/event-relays/rebacCache.ts new file mode 100644 index 000000000..a2d458300 --- /dev/null +++ b/modules/router/src/event-relays/rebacCache.ts @@ -0,0 +1,108 @@ +type RebacSdk = { + isAvailable: (module: string) => boolean; + authorization?: { + can: (request: { + subject: string; + actions: string[]; + resource: string; + }) => Promise<{ allow: boolean }>; + } | null; +}; + +export type RebacDecision = 'allow' | 'deny' | 'unavailable'; + +const DEFAULT_TTL_MS = 12_000; +const DEFAULT_MAX_ENTRIES = 10_000; + +type CacheEntry = { + decision: 'allow' | 'deny'; + expiresAt: number; +}; + +export class RelayRebacCache { + private readonly entries = new Map(); + constructor( + private readonly ttlMs: number = DEFAULT_TTL_MS, + private readonly maxEntries: number = DEFAULT_MAX_ENTRIES, + ) {} + + async check( + grpcSdk: RebacSdk, + userId: string, + permission: string, + resourceType: string, + resourceId: string, + ): Promise { + const resource = `${resourceType}:${resourceId}`; + const key = `${userId}:${permission}:${resource}`; + const now = Date.now(); + this.sweepExpired(now); + const cached = this.entries.get(key); + if (cached && cached.expiresAt > now) { + return cached.decision; + } + if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { + return 'unavailable'; + } + try { + const decision = await grpcSdk.authorization.can({ + subject: `User:${userId}`, + actions: [permission], + resource, + }); + this.set(key, decision.allow ? 'allow' : 'deny', now); + return decision.allow ? 'allow' : 'deny'; + } catch { + return 'unavailable'; + } + } + + clear(): void { + this.entries.clear(); + } + + private set(key: string, decision: 'allow' | 'deny', now: number): void { + if (this.entries.size >= this.maxEntries && !this.entries.has(key)) { + const firstKey = this.entries.keys().next().value; + if (firstKey) { + this.entries.delete(firstKey); + } + } + this.entries.set(key, { decision, expiresAt: now + this.ttlMs }); + } + + private sweepExpired(now: number): void { + for (const [key, entry] of this.entries) { + if (entry.expiresAt <= now) { + this.entries.delete(key); + } + } + } +} + +export async function checkRebacBatch( + cache: RelayRebacCache, + grpcSdk: RebacSdk, + userIds: string[], + permission: string, + resourceType: string, + resourceId: string, + concurrency = 8, +): Promise> { + const results = new Map(); + let index = 0; + async function worker(): Promise { + while (index < userIds.length) { + const userId = userIds[index++]; + results.set( + userId, + await cache.check(grpcSdk, userId, permission, resourceType, resourceId), + ); + } + } + const workers = Array.from({ length: Math.min(concurrency, userIds.length) }, () => + worker(), + ); + await Promise.all(workers); + return results; +} diff --git a/modules/router/src/event-relays/recovery.ts b/modules/router/src/event-relays/recovery.ts new file mode 100644 index 000000000..ceb6c4217 --- /dev/null +++ b/modules/router/src/event-relays/recovery.ts @@ -0,0 +1,85 @@ +import { status } from '@grpc/grpc-js'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { + authorizeRelaySubscription, + RelaySubscriptionError, + RelayLookup, + toSubscriptionError, +} from './authorize.js'; +import { eventRelayRoom } from './rooms.js'; +import { + removeSubscription, + subscriptionsForRecoveredRooms, + trackSubscription, +} from './subscriptions.js'; +import { relaySubscriptionsFromContext, persistRelaySubscriptionOnContext, removeRelaySubscriptionFromContext } from './relaySocketData.js'; + +type RelayAuthorizationSdk = Parameters[0]; + +export async function reauthorizeRecoveredSubscriptions( + grpcSdk: RelayAuthorizationSdk, + manager: RelayLookup, + request: { + socketId: string; + context?: { user?: { _id?: string } } & Record; + recoveredRooms?: string[]; + }, +): Promise< + { event: 'join-room'; rooms: string[] } | { event: 'leave-room'; rooms: string[] } +> { + const userId = request.context?.user?._id as string | undefined; + if (!userId) { + return { event: 'leave-room', rooms: [] }; + } + + const recoveredRooms = request.recoveredRooms ?? []; + const subs = subscriptionsForRecoveredRooms( + recoveredRooms, + relaySubscriptionsFromContext(request.context), + ); + const leaveRooms: string[] = []; + + for (const sub of subs) { + const room = eventRelayRoom(sub.relayId, sub.resourceId); + try { + await authorizeRelaySubscription( + grpcSdk, + manager, + userId, + sub.relayId, + sub.resourceId, + () => { + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + }, + ); + trackSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + persistRelaySubscriptionOnContext(request.context, sub.relayId, sub.resourceId); + } catch (err) { + const mapped = + err instanceof RelaySubscriptionError ? err : toSubscriptionError(err); + if (mapped.code === status.UNAVAILABLE) { + trackSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + persistRelaySubscriptionOnContext(request.context, sub.relayId, sub.resourceId); + continue; + } + if ( + mapped.code === status.PERMISSION_DENIED || + mapped.code === status.NOT_FOUND + ) { + removeSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + removeRelaySubscriptionFromContext(request.context, sub.relayId, sub.resourceId); + leaveRooms.push(room); + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + continue; + } + removeSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + removeRelaySubscriptionFromContext(request.context, sub.relayId, sub.resourceId); + leaveRooms.push(room); + } + } + + if (leaveRooms.length > 0) { + return { event: 'leave-room', rooms: leaveRooms }; + } + return { event: 'join-room', rooms: [] }; +} diff --git a/modules/router/src/event-relays/relaySocketData.ts b/modules/router/src/event-relays/relaySocketData.ts new file mode 100644 index 000000000..63d0eca9c --- /dev/null +++ b/modules/router/src/event-relays/relaySocketData.ts @@ -0,0 +1,65 @@ +import { Indexable } from '@conduitplatform/grpc-sdk'; +import { RelaySubscription } from './subscriptions.js'; + +const CONTEXT_KEY = 'eventRelaySubs'; + +export function relaySubscriptionsFromContext( + context: Indexable | undefined, +): RelaySubscription[] { + if (!context) { + return []; + } + const raw = context[CONTEXT_KEY]; + if (!Array.isArray(raw)) { + return []; + } + const subs: RelaySubscription[] = []; + for (const item of raw) { + if ( + item && + typeof item === 'object' && + typeof (item as RelaySubscription).relayId === 'string' && + typeof (item as RelaySubscription).resourceId === 'string' + ) { + subs.push({ + relayId: (item as RelaySubscription).relayId, + resourceId: (item as RelaySubscription).resourceId, + }); + } + } + return subs; +} + +export function persistRelaySubscriptionOnContext( + context: Indexable | undefined, + relayId: string, + resourceId: string, +): void { + if (!context) { + return; + } + const subs = relaySubscriptionsFromContext(context); + const withoutDup = subs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + withoutDup.push({ relayId, resourceId }); + context[CONTEXT_KEY] = withoutDup; +} + +export function removeRelaySubscriptionFromContext( + context: Indexable | undefined, + relayId: string, + resourceId: string, +): void { + if (!context) { + return; + } + const next = relaySubscriptionsFromContext(context).filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + if (next.length === 0) { + delete context[CONTEXT_KEY]; + } else { + context[CONTEXT_KEY] = next; + } +} diff --git a/modules/router/src/event-relays/rooms.ts b/modules/router/src/event-relays/rooms.ts index fd5dd54ec..0bf26bf74 100644 --- a/modules/router/src/event-relays/rooms.ts +++ b/modules/router/src/event-relays/rooms.ts @@ -4,3 +4,7 @@ export function eventRelayRoom(relayId: string, resourceId: string): string { const digest = createHash('sha256').update(resourceId).digest('hex'); return `er:${relayId}:${digest}`; } + +export function eventRelayRoomPrefix(relayId: string): string { + return `er:${relayId}:`; +} diff --git a/modules/router/src/event-relays/subscriptions.ts b/modules/router/src/event-relays/subscriptions.ts new file mode 100644 index 000000000..a7a083f56 --- /dev/null +++ b/modules/router/src/event-relays/subscriptions.ts @@ -0,0 +1,224 @@ +import { eventRelayRoom } from './rooms.js'; +import { MAX_ROOMS_PER_SOCKET } from './constants.js'; + +export type RelaySubscription = { + relayId: string; + resourceId: string; +}; + +const RECOVERY_ROOM_MAP_TTL_MS = 120_000; + +const subscriptionsBySocket = new Map(); +const subscriptionsByUser = new Map(); +const subscriptionsByRoom = new Map(); + +function pruneExpiredRoomEntries(now = Date.now()): void { + for (const [room, entry] of subscriptionsByRoom) { + if (entry.expiresAt <= now) { + subscriptionsByRoom.delete(room); + } + } +} + +function touchRoomEntry(relayId: string, resourceId: string): void { + const room = eventRelayRoom(relayId, resourceId); + subscriptionsByRoom.set(room, { + relayId, + resourceId, + expiresAt: Date.now() + RECOVERY_ROOM_MAP_TTL_MS, + }); +} + +function pruneRoomEntryIfUnused(relayId: string, resourceId: string): void { + if (!isSubscriptionTrackedOnAnySocket(relayId, resourceId)) { + subscriptionsByRoom.delete(eventRelayRoom(relayId, resourceId)); + } +} + +export function trackSubscription( + socketId: string, + userId: string, + relayId: string, + resourceId: string, +): void { + const entry = { relayId, resourceId }; + const socketSubs = subscriptionsBySocket.get(socketId) ?? []; + const withoutDup = socketSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + withoutDup.push(entry); + if (withoutDup.length > MAX_ROOMS_PER_SOCKET) { + throw new Error(`Cannot subscribe to more than ${MAX_ROOMS_PER_SOCKET} relay rooms`); + } + subscriptionsBySocket.set(socketId, withoutDup); + + const userSubs = subscriptionsByUser.get(userId) ?? []; + const userWithoutDup = userSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + userWithoutDup.push(entry); + subscriptionsByUser.set(userId, userWithoutDup); + touchRoomEntry(relayId, resourceId); +} + +export function removeSubscription( + socketId: string, + userId: string | undefined, + relayId: string, + resourceId: string, +): void { + const socketSubs = subscriptionsBySocket.get(socketId); + if (socketSubs) { + const next = socketSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + if (next.length === 0) { + subscriptionsBySocket.delete(socketId); + } else { + subscriptionsBySocket.set(socketId, next); + } + } + if (userId && !isSubscriptionTrackedOnOtherSocket(relayId, resourceId, socketId)) { + const userSubs = subscriptionsByUser.get(userId); + if (userSubs) { + const next = userSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + if (next.length === 0) { + subscriptionsByUser.delete(userId); + } else { + subscriptionsByUser.set(userId, next); + } + } + pruneRoomEntryIfUnused(relayId, resourceId); + } +} + +function isSubscriptionTrackedOnOtherSocket( + relayId: string, + resourceId: string, + exceptSocketId: string, +): boolean { + for (const [socketId, subs] of subscriptionsBySocket) { + if (socketId === exceptSocketId) { + continue; + } + if (subs.some(sub => sub.relayId === relayId && sub.resourceId === resourceId)) { + return true; + } + } + return false; +} + +function isSubscriptionTrackedOnAnySocket(relayId: string, resourceId: string): boolean { + for (const subs of subscriptionsBySocket.values()) { + if (subs.some(sub => sub.relayId === relayId && sub.resourceId === resourceId)) { + return true; + } + } + return false; +} + +export function releaseSocketSubscriptions(socketId: string, userId?: string): void { + const subs = subscriptionsBySocket.get(socketId) ?? []; + subscriptionsBySocket.delete(socketId); + if (!userId) { + return; + } + for (const sub of subs) { + if (!isSubscriptionTrackedOnOtherSocket(sub.relayId, sub.resourceId, socketId)) { + const userSubs = subscriptionsByUser.get(userId); + if (!userSubs) { + continue; + } + const next = userSubs.filter( + entry => + !(entry.relayId === sub.relayId && entry.resourceId === sub.resourceId), + ); + if (next.length === 0) { + subscriptionsByUser.delete(userId); + } else { + subscriptionsByUser.set(userId, next); + } + } + pruneRoomEntryIfUnused(sub.relayId, sub.resourceId); + } +} + +export function subscriptionsForRecoveredRooms( + recoveredRooms: string[], + contextSubs: RelaySubscription[] = [], +): RelaySubscription[] { + pruneExpiredRoomEntries(); + const seen = new Set(); + const result: RelaySubscription[] = []; + const recoveredSet = new Set(recoveredRooms.filter(room => room.startsWith('er:'))); + + for (const sub of contextSubs) { + const room = eventRelayRoom(sub.relayId, sub.resourceId); + if (recoveredSet.has(room) && !seen.has(room)) { + seen.add(room); + result.push(sub); + } + } + + for (const room of recoveredSet) { + if (seen.has(room)) { + continue; + } + const entry = subscriptionsByRoom.get(room); + if (entry) { + seen.add(room); + result.push({ relayId: entry.relayId, resourceId: entry.resourceId }); + } + } + + return result; +} + +export function subscriptionsForUser(userId: string): RelaySubscription[] { + return [...(subscriptionsByUser.get(userId) ?? [])]; +} + +export function removeSubscriptionsForRelay(relayId: string): void { + for (const [socketId, subs] of subscriptionsBySocket) { + const next = subs.filter(sub => sub.relayId !== relayId); + if (next.length === 0) { + subscriptionsBySocket.delete(socketId); + } else { + subscriptionsBySocket.set(socketId, next); + } + } + for (const [userId, subs] of subscriptionsByUser) { + const next = subs.filter(sub => sub.relayId !== relayId); + if (next.length === 0) { + subscriptionsByUser.delete(userId); + } else { + subscriptionsByUser.set(userId, next); + } + } + for (const [room, entry] of subscriptionsByRoom) { + if (entry.relayId === relayId) { + subscriptionsByRoom.delete(room); + } + } +} + +export function leaveRoomsForRelay(relayId: string): string[] { + const rooms = new Set(); + for (const subs of subscriptionsByUser.values()) { + for (const sub of subs) { + if (sub.relayId === relayId) { + rooms.add(eventRelayRoom(relayId, sub.resourceId)); + } + } + } + return [...rooms]; +} + +/** @internal test helper */ +export function _clearEventRelaySubscriptionStateForTests(): void { + subscriptionsBySocket.clear(); + subscriptionsByUser.clear(); + subscriptionsByRoom.clear(); +} diff --git a/modules/router/src/index.ts b/modules/router/src/index.ts index 20a0c779e..fd1d069ed 100644 --- a/modules/router/src/index.ts +++ b/modules/router/src/index.ts @@ -1,7 +1,25 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import ConduitDefaultRouter from './Router.js'; const peerManifestRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const router = new ConduitDefaultRouter(peerManifestRoot); + +function registerShutdownSignals(): void { + const shutdown = (signal: NodeJS.Signals) => { + void router + .shutdown() + .catch(err => { + ConduitGrpcSdk.Logger.error(err as Error); + }) + .finally(() => { + process.exit(signal === 'SIGINT' ? 130 : 0); + }); + }; + process.once('SIGTERM', () => shutdown('SIGTERM')); + process.once('SIGINT', () => shutdown('SIGINT')); +} + +registerShutdownSignals(); router.start(); diff --git a/modules/router/src/metrics/index.ts b/modules/router/src/metrics/index.ts index d3461c107..2ed45b20f 100644 --- a/modules/router/src/metrics/index.ts +++ b/modules/router/src/metrics/index.ts @@ -38,4 +38,39 @@ export default { help: 'Tracks denied or unavailable event-relay socket subscriptions', }, }, + eventRelaysActive: { + type: MetricType.Gauge, + config: { + name: 'event_relays_active_total', + help: 'Active event relays on this router replica', + }, + }, + eventRelaysSubscribedChannels: { + type: MetricType.Gauge, + config: { + name: 'event_relays_subscribed_channels_total', + help: 'Bus channels this router replica subscribes to for event relays', + }, + }, + eventRelaysEmptyRoom: { + type: MetricType.Counter, + config: { + name: 'event_relays_empty_room_total', + help: 'Skipped emits because no local sockets were in the relay room', + }, + }, + eventRelaysInboundDropped: { + type: MetricType.Counter, + config: { + name: 'event_relays_inbound_dropped_total', + help: 'Inbound bus payloads dropped for exceeding the size cap', + }, + }, + eventRelaysEmitDropped: { + type: MetricType.Counter, + config: { + name: 'event_relays_emit_dropped_total', + help: 'Socket emits dropped or disconnected due to backpressure', + }, + }, }; diff --git a/modules/router/tsconfig.test.json b/modules/router/tsconfig.test.json index 601e5ccee..229f7f155 100644 --- a/modules/router/tsconfig.test.json +++ b/modules/router/tsconfig.test.json @@ -19,6 +19,9 @@ "src/event-relays/search.ts", "src/event-relays/channels.ts", "src/event-relays/authorize.ts", + "src/event-relays/compile.ts", + "src/event-relays/rebacCache.ts", + "src/event-relays/subscriptions.ts", "src/event-relays/*.test.ts" ] } From 9e6356efccae71fd154e2ea411730a4c892bccf5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 11:48:02 +0000 Subject: [PATCH 2/5] fix(router): address CodeFactor findings on event relay validation Split validateEventRelayInput into per-field parsers to reduce complexity. Silence unused-parameter lint in EventBus test FakeRedis stub. --- .../grpc-sdk/src/utilities/EventBus.test.ts | 25 ++-- modules/router/src/event-relays/validation.ts | 127 +++++++++++------- 2 files changed, 94 insertions(+), 58 deletions(-) diff --git a/libraries/grpc-sdk/src/utilities/EventBus.test.ts b/libraries/grpc-sdk/src/utilities/EventBus.test.ts index c1999901d..4d069c244 100644 --- a/libraries/grpc-sdk/src/utilities/EventBus.test.ts +++ b/libraries/grpc-sdk/src/utilities/EventBus.test.ts @@ -24,11 +24,15 @@ class FakeRedis { cb?.(null); } - unsubscribe(_channel: string, cb?: () => void) { + unsubscribe(channel: string, cb?: () => void) { + void channel; cb?.(); } - publish(_channel: string, _message: string) {} + publish(channel: string, message: string) { + void channel; + void message; + } quit() {} @@ -102,19 +106,18 @@ describe('EventBus', () => { const { bus, sub } = createBus(); sub.failNext.add('chan'); await assert.rejects( - () => - bus.subscribeAck( - 'chan', - () => {}, - 'relay-a', - ), + () => bus.subscribeAck('chan', () => {}, 'relay-a'), /subscribe failed/, ); sub.failNext.delete('chan'); let count = 0; - await bus.subscribeAck('chan', () => { - count += 1; - }, 'relay-a'); + await bus.subscribeAck( + 'chan', + () => { + count += 1; + }, + 'relay-a', + ); sub.emitMessage('chan', 'x'); assert.equal(count, 1); }); diff --git a/modules/router/src/event-relays/validation.ts b/modules/router/src/event-relays/validation.ts index d6c6c5eb6..c59976335 100644 --- a/modules/router/src/event-relays/validation.ts +++ b/modules/router/src/event-relays/validation.ts @@ -33,24 +33,75 @@ const PERMISSION_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,63}$/; const RESOURCE_ID_PATTERN = /^[^\s:]{1,128}$/; export function validateEventRelayInput(input: EventRelayInput): EventRelayInput { - const name = requireTrimmed(input.name, 'Name'); + const name = parseName(input.name); + const notes = parseNotes(input.notes); + const busEvent = parseBusEvent(input.busEvent); + const socketEvent = parseSocketEvent(input.socketEvent); + const resourceType = parseResourceType(input.resourceType); + const resourceIdPath = parseResourceIdPath(input.resourceIdPath); + const permission = parsePermission(input.permission); + const messageTemplate = parseMessageTemplate(input.messageTemplate); + + return { + name, + notes, + active: input.active !== false, + busEvent, + socketEvent, + resourceType, + resourceIdPath, + permission, + messageTemplate, + }; +} + +export function validateResourceId(resourceId: unknown): string { + if (typeof resourceId !== 'string' && typeof resourceId !== 'number') { + throw new EventRelayValidationError('Resource ID must be a string'); + } + const value = String(resourceId).trim(); + if ( + !value || + value.length > MAX_RESOURCE_ID_LENGTH || + !RESOURCE_ID_PATTERN.test(value) + ) { + throw new EventRelayValidationError('Resource ID is invalid'); + } + return value; +} + +function requireTrimmed(value: unknown, label: string): string { + if (typeof value !== 'string' || value.trim() === '') { + throw new EventRelayValidationError(`${label} is required`); + } + return value.trim(); +} + +function parseName(raw: unknown): string { + const name = requireTrimmed(raw, 'Name'); if (name.length > MAX_NAME_LENGTH || !NAME_PATTERN.test(name)) { throw new EventRelayValidationError( 'Name must be 1-64 characters and start with a letter or number', ); } + return name; +} - const notes = - input.notes === undefined || input.notes === '' - ? undefined - : requireTrimmed(input.notes, 'Notes'); - if (notes && notes.length > MAX_DESCRIPTION_LENGTH) { +function parseNotes(raw: unknown): string | undefined { + if (raw === undefined || raw === '') { + return undefined; + } + const notes = requireTrimmed(raw, 'Notes'); + if (notes.length > MAX_DESCRIPTION_LENGTH) { throw new EventRelayValidationError( `Notes must be at most ${MAX_DESCRIPTION_LENGTH} characters`, ); } + return notes; +} - const busEvent = requireTrimmed(input.busEvent, 'Bus event'); +function parseBusEvent(raw: unknown): string { + const busEvent = requireTrimmed(raw, 'Bus event'); if ( busEvent.length > MAX_BUS_EVENT_LENGTH || busEvent.includes('*') || @@ -60,8 +111,11 @@ export function validateEventRelayInput(input: EventRelayInput): EventRelayInput 'Bus event must be an exact channel name with no wildcards', ); } + return busEvent; +} - const socketEvent = requireTrimmed(input.socketEvent, 'Socket event'); +function parseSocketEvent(raw: unknown): string { + const socketEvent = requireTrimmed(raw, 'Socket event'); if ( socketEvent.length > MAX_SOCKET_EVENT_LENGTH || !SOCKET_EVENT_PATTERN.test(socketEvent) || @@ -69,62 +123,41 @@ export function validateEventRelayInput(input: EventRelayInput): EventRelayInput ) { throw new EventRelayValidationError('Socket event must be a non-reserved event name'); } + return socketEvent; +} - const resourceType = requireTrimmed(input.resourceType, 'Resource type'); +function parseResourceType(raw: unknown): string { + const resourceType = requireTrimmed(raw, 'Resource type'); if ( resourceType.length > MAX_RESOURCE_TYPE_LENGTH || !RESOURCE_TYPE_PATTERN.test(resourceType) ) { throw new EventRelayValidationError('Resource type is invalid'); } + return resourceType; +} - const resourceIdPath = requireTrimmed(input.resourceIdPath, 'Resource ID path'); +function parseResourceIdPath(raw: unknown): string { + const resourceIdPath = requireTrimmed(raw, 'Resource ID path'); if (resourceIdPath.length > MAX_RESOURCE_ID_PATH_LENGTH) { throw new EventRelayValidationError('Resource ID path is too long'); } parseDotPath(resourceIdPath, 'Resource ID path'); + return resourceIdPath; +} - const permission = requireTrimmed(input.permission, 'Permission'); +function parsePermission(raw: unknown): string { + const permission = requireTrimmed(raw, 'Permission'); if (permission.length > MAX_PERMISSION_LENGTH || !PERMISSION_PATTERN.test(permission)) { throw new EventRelayValidationError('Permission is invalid'); } - - if (input.messageTemplate === undefined) { - throw new EventRelayValidationError('Message template is required'); - } - assertTemplateSize(input.messageTemplate); - - return { - name, - notes, - active: input.active !== false, - busEvent, - socketEvent, - resourceType, - resourceIdPath, - permission, - messageTemplate: input.messageTemplate, - }; -} - -export function validateResourceId(resourceId: unknown): string { - if (typeof resourceId !== 'string' && typeof resourceId !== 'number') { - throw new EventRelayValidationError('Resource ID must be a string'); - } - const value = String(resourceId).trim(); - if ( - !value || - value.length > MAX_RESOURCE_ID_LENGTH || - !RESOURCE_ID_PATTERN.test(value) - ) { - throw new EventRelayValidationError('Resource ID is invalid'); - } - return value; + return permission; } -function requireTrimmed(value: unknown, label: string): string { - if (typeof value !== 'string' || value.trim() === '') { - throw new EventRelayValidationError(`${label} is required`); +function parseMessageTemplate(raw: unknown): unknown { + if (raw === undefined) { + throw new EventRelayValidationError('Message template is required'); } - return value.trim(); + assertTemplateSize(raw); + return raw; } From 5f938ba9721e3e2e06f6fe3c2fb2a90d93b3d268 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 12:34:07 +0000 Subject: [PATCH 3/5] fix(database): Mongo live-update interventions (watch, token, tickets, recovery) Rebase #1601 onto current Event Relays Hermes and close the stack-review P1s: watch $match/$project, serialized handleChange, persist resume token after successful emit, keep token on CursorKilled 237, hermes isSocketHandshake, restore /database/ Redis on recover, and shutdown the watch plus leader lock. Co-authored-by: Konstantinos Kopanidis --- docker/docker-compose.standalone.yml | 2 + docker/docker-compose.yml | 2 + libraries/hermes/src/Socket/Socket.ts | 9 +- .../src/Socket/isSocketHandshake.test.ts | 1 + libraries/hermes/src/interfaces/Socket.ts | 9 +- modules/database/README.mdx | 10 +- modules/database/src/Database.ts | 4 + modules/database/src/index.ts | 18 ++ .../realtime/MongoChangeStreamCoordinator.ts | 130 ++++++++++---- .../database/src/realtime/RealtimeService.ts | 12 +- .../change-stream.integration.test.ts | 3 +- .../realtime/__tests__/coordinator.test.ts | 162 +++++++++++++++--- .../src/realtime/__tests__/normalize.test.ts | 5 +- .../src/realtime/__tests__/recovery.test.ts | 85 +++++++++ .../src/realtime/__tests__/rooms.test.ts | 6 + .../src/realtime/__tests__/topology.test.ts | 5 +- .../realtime/__tests__/watchPipeline.test.ts | 28 +++ modules/database/src/realtime/normalize.ts | 10 +- modules/database/src/realtime/rebacCache.ts | 96 +++++++++++ modules/database/src/realtime/recovery.ts | 147 ++++++++++++++++ modules/database/src/realtime/rooms.ts | 22 +++ modules/database/src/realtime/sockets.ts | 43 ++++- modules/database/src/realtime/topology.ts | 11 +- modules/database/src/realtime/types.ts | 1 - .../database/src/realtime/watchPipeline.ts | 33 ++++ .../src/admin/middleware/Admin.middleware.ts | 3 +- .../src/admin/middleware/Auth.middleware.ts | 3 +- .../core/src/admin/realtime/handshake.test.ts | 12 -- packages/core/src/admin/realtime/handshake.ts | 14 -- .../src/admin/realtime/ticket-http.test.ts | 24 +++ packages/core/tsconfig.test.json | 1 - 31 files changed, 796 insertions(+), 115 deletions(-) create mode 100644 modules/database/src/realtime/__tests__/recovery.test.ts create mode 100644 modules/database/src/realtime/__tests__/watchPipeline.test.ts create mode 100644 modules/database/src/realtime/rebacCache.ts create mode 100644 modules/database/src/realtime/recovery.ts create mode 100644 modules/database/src/realtime/watchPipeline.ts delete mode 100644 packages/core/src/admin/realtime/handshake.test.ts delete mode 100644 packages/core/src/admin/realtime/handshake.ts create mode 100644 packages/core/src/admin/realtime/ticket-http.test.ts diff --git a/docker/docker-compose.standalone.yml b/docker/docker-compose.standalone.yml index 1c695c107..bd77d5c47 100644 --- a/docker/docker-compose.standalone.yml +++ b/docker/docker-compose.standalone.yml @@ -78,6 +78,8 @@ services: MONGO_INITDB_DATABASE: 'conduit' MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' + # Existing volumes created before rs0 will not elect a replica set. + # Remove the mongo volume (or start with an empty data dir) if hello stays standalone. entrypoint: - bash - -c diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7131604e3..7567df345 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -282,6 +282,8 @@ services: MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' profiles: ['mongodb'] + # Existing volumes created before rs0 will not elect a replica set. + # Remove the mongo volume (or start with an empty data dir) if hello stays standalone. entrypoint: - bash - -c diff --git a/libraries/hermes/src/Socket/Socket.ts b/libraries/hermes/src/Socket/Socket.ts index 55ef763d3..f1c0b47d1 100644 --- a/libraries/hermes/src/Socket/Socket.ts +++ b/libraries/hermes/src/Socket/Socket.ts @@ -152,11 +152,15 @@ export class SocketController extends ConduitRouter { this.io.of(namespace).on('connect', socket => { if (socket.recovered) { + const recoveredRooms = [...socket.rooms].filter( + room => room.startsWith('er:') || room.startsWith('database:'), + ); const recovered = conduitSocket.executeRecovered({ event: 'recovered', socketId: socket.id, context: socket.data, - recoveredRooms: [...socket.rooms].filter(room => room.startsWith('er:')), + params: recoveredRooms, + recoveredRooms, }); if (recovered) { recovered @@ -195,12 +199,13 @@ export class SocketController extends ConduitRouter { }); }); - socket.on('disconnect', () => { + socket.on('disconnect', (reason: string) => { conduitSocket .executeRequest({ event: 'disconnect', socketId: socket.id, context: socket.data, + params: [reason], }) .then(res => this.handleResponse(res, socket, namespace)) .catch(e => { diff --git a/libraries/hermes/src/Socket/isSocketHandshake.test.ts b/libraries/hermes/src/Socket/isSocketHandshake.test.ts index 267de330f..a686bdb76 100644 --- a/libraries/hermes/src/Socket/isSocketHandshake.test.ts +++ b/libraries/hermes/src/Socket/isSocketHandshake.test.ts @@ -16,6 +16,7 @@ describe('isSocketHandshake', () => { isSocketHandshake({ url: '/realtime/ticket?EIO=4&transport=polling' }), false, ); + assert.equal(isSocketHandshake({ url: '/realtime/ticket' }), false); assert.equal(isSocketHandshake({ originalUrl: '/realtime' }), false); assert.equal(isSocketHandshake({ url: '/graphql?EIO=4&transport=polling' }), false); assert.equal( diff --git a/libraries/hermes/src/interfaces/Socket.ts b/libraries/hermes/src/interfaces/Socket.ts index 7c7851094..2b9787257 100644 --- a/libraries/hermes/src/interfaces/Socket.ts +++ b/libraries/hermes/src/interfaces/Socket.ts @@ -91,10 +91,13 @@ export class ConduitSocket { executeRecovered( request: ConduitSocketParameters, ): ConduitSocketHandlerResponse | null { - if (!this._input.onRecovered) { - return null; + if (this._input.onRecovered) { + return this._input.onRecovered(request); } - return this._input.onRecovered(request); + if (this._events.has('recovered')) { + return this._events.get('recovered')!.handler(request); + } + return null; } } diff --git a/modules/database/README.mdx b/modules/database/README.mdx index 61bc195b0..b84cded13 100644 --- a/modules/database/README.mdx +++ b/modules/database/README.mdx @@ -47,7 +47,9 @@ since the latter need to go through parsers that are otherwise unnecessary for M When using MongoDB with a replica set (e.g., MongoDB Atlas), the database module supports configuring read preference, write concern, and read concern through the admin panel at `PATCH /config/database`. -Live document updates also require a replica set or sharded cluster. Local Compose files initialize a single-node replica set so change streams can be exercised. +Live document updates require a replica set or sharded cluster (Atlas is fine). Helm’s bundled Mongo chart is a **standalone** Deployment (`replicas: 1`, no `--replSet`), so live updates stay `idle` there until you point `DB_CONN_URI` at Atlas or an operator-managed replica set. + +Local Compose files initialize a single-node `rs0`. **Existing Compose Mongo volumes will not become a replica set cleanly** — drop the volume or start from an empty data dir if `hello` still reports standalone. ### Live updates @@ -58,9 +60,11 @@ subscribe({ schema: 'Order', documentId?: string }) unsubscribe({ schema: 'Order', documentId?: string }) ``` -Events arrive as `change` with `{ version, operation, schema, documentId, occurredAt, resumeToken }` and contain no document fields. Consumers should refetch through their authorized REST or custom-endpoint path. +Events arrive as `change` with `{ version, operation, schema, documentId, occurredAt }` and contain no document fields and no resume token. This is **live-tail, not backfill**. After a full reconnect, subscribe again and refetch over authorized REST. + +The leader also publishes `database:change:${schema}` on the Redis bus. **Do not also relay `database:change:*` on `/events/` if the same client is on `/database/`** — that duplicates notifications. Keep the bus for other modules; just do not dual-subscribe. -Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; session JWTs and masterkeys must not be sent from browser code. +Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Client sockets also require CMS `crudOperations.read.enabled`. Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; that token cannot mint another ticket or call REST/GraphQL. Session JWTs and masterkeys must not be sent from browser code. Admin sockets must be enabled (`admin.transports.sockets`) and the Admin socket port (`ADMIN_SOCKET_PORT`, default 3031) reachable from the UI. diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index e0208f256..e2d36e26f 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -1053,4 +1053,8 @@ export default class DatabaseModule extends ManagedModule { ); } } + + async shutdown(): Promise { + await this.realtimeService?.shutdown(); + } } diff --git a/modules/database/src/index.ts b/modules/database/src/index.ts index 3e9466196..12261992a 100644 --- a/modules/database/src/index.ts +++ b/modules/database/src/index.ts @@ -1,5 +1,6 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import DatabaseModule from './Database.js'; const dbType = process.env.DB_TYPE ?? 'mongodb'; @@ -7,4 +8,21 @@ const dbUri = process.env.DB_CONN_URI ?? 'mongodb://localhost:27017'; const peerManifestRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const database = new DatabaseModule(dbType, dbUri, peerManifestRoot); + +function registerShutdownSignals(): void { + const shutdown = (signal: NodeJS.Signals) => { + void database + .shutdown() + .catch(err => { + ConduitGrpcSdk.Logger.error(err as Error); + }) + .finally(() => { + process.exit(signal === 'SIGINT' ? 130 : 0); + }); + }; + process.once('SIGTERM', () => shutdown('SIGTERM')); + process.once('SIGINT', () => shutdown('SIGINT')); +} + +registerShutdownSignals(); database.start(); diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts index b41cc2f89..339750065 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -2,6 +2,7 @@ import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import { normalizeChangeEvent, parseResumeToken, + serializeResumeToken, type RawChangeEvent, } from './normalize.js'; import { authorizedDocumentRoom, roomsForPublicChange } from './rooms.js'; @@ -17,7 +18,14 @@ import type { RealtimeStatusCode, } from './types.js'; import type { RealtimeSubscriptionTracker } from './subscriptions.js'; -import { canReadDocument, type AuthorizationSdk } from './authorize.js'; +import { type AuthorizationSdk } from './authorize.js'; +import { checkRebacBatch, RealtimeRebacCache } from './rebacCache.js'; +import { + buildWatchPipeline, + optedInCollectionsKey, + WATCH_RESTART_OPERATIONS, + type WatchPipeline, +} from './watchPipeline.js'; const LEADER_LOCK = 'realtime:change-stream:leader'; const RESUME_TOKEN_KEY = 'realtime:resumeToken'; @@ -30,7 +38,10 @@ type LeaderLock = NonNullable< Awaited['tryAcquireLock']>> >; -export type WatchFactory = (options: { resumeAfter?: unknown }) => ChangeStreamLike; +export type WatchFactory = (options: { + resumeAfter?: unknown; + pipeline: WatchPipeline; +}) => ChangeStreamLike; export type CoordinatorOptions = { grpcSdk: ConduitGrpcSdk; @@ -56,6 +67,9 @@ export class MongoChangeStreamCoordinator { private watching = false; private opening = false; private ignoreClose = false; + private changeQueue: Promise = Promise.resolve(); + private watchedCollectionsKey = ''; + private readonly rebacCache = new RealtimeRebacCache(); constructor(private readonly options: CoordinatorOptions) {} @@ -75,6 +89,10 @@ export class MongoChangeStreamCoordinator { return this.topology; } + async waitForIdle(): Promise { + await this.changeQueue; + } + async reconcile(): Promise { if (this.closed) return; const engine = this.options.engine(); @@ -90,8 +108,6 @@ export class MongoChangeStreamCoordinator { await this.releaseLeader(); this.streamState = 'idle'; this.lastError = this.topology.message; - // Hello can fail during startup before Mongo is ready. Keep retrying - // that case; a confirmed standalone topology will not recover. if ( !this.topology.message || this.topology.message.includes('Unable to determine') @@ -100,20 +116,31 @@ export class MongoChangeStreamCoordinator { } return; } - if (this.options.getOptedInSchemas().length === 0) { + const collections = this.collectionNames(); + if (collections.length === 0) { await this.stopStream('idle'); await this.releaseLeader(); this.streamState = 'idle'; return; } + const nextKey = optedInCollectionsKey(collections); + if (this.watching && nextKey !== this.watchedCollectionsKey) { + await this.stopStream('starting'); + } await this.ensureLeader(); } async shutdown(): Promise { this.closed = true; this.clearTimers(); + await this.changeQueue; await this.stopStream('idle'); await this.releaseLeader(); + this.rebacCache.clear(); + } + + private collectionNames(): string[] { + return this.options.getOptedInSchemas().map(schema => schema.collectionName); } private async ensureLeader(): Promise { @@ -129,9 +156,6 @@ export class MongoChangeStreamCoordinator { LOCK_TTL_MS, ); if (!acquired) { - // Another instance holds the lock, or a crashed holder has not - // expired yet. Without a retry, a standalone process stays idle - // forever after a restart races the previous TTL. this.streamState = 'idle'; this.scheduleRetry(); return; @@ -174,13 +198,16 @@ export class MongoChangeStreamCoordinator { await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY), ); if (this.watching || this.closed) return; - const stream = this.options.watch({ resumeAfter }); + const collections = this.collectionNames(); + const pipeline = buildWatchPipeline(collections); + this.watchedCollectionsKey = optedInCollectionsKey(collections); + const stream = this.options.watch({ resumeAfter, pipeline }); this.stream = stream; this.watching = true; this.streamState = 'live'; this.retryAttempt = 0; stream.on('change', (change: unknown) => { - void this.handleChange(change as RawChangeEvent); + this.enqueueChange(change as RawChangeEvent); }); stream.on('error', (err: unknown) => { void this.handleStreamError(err); @@ -199,26 +226,55 @@ export class MongoChangeStreamCoordinator { } } + private enqueueChange(change: RawChangeEvent) { + this.changeQueue = this.changeQueue + .then(() => this.handleChange(change)) + .catch(err => { + this.lastError = err instanceof Error ? err.message : String(err); + ConduitGrpcSdk.Logger.error(err as Error); + }); + } + private async handleChange(change: RawChangeEvent) { + const token = serializeResumeToken(change._id); const schema = this.resolveSchema(change.ns?.coll); - if (!schema) return; - const event = normalizeChangeEvent(change, schema.name); - if (!event) return; + const event = schema ? normalizeChangeEvent(change, schema.name) : null; + if (!event || !schema) { + if (token) { + await this.persistResumeToken(token); + } + if (change.operationType && WATCH_RESTART_OPERATIONS.has(change.operationType)) { + await this.stopStream('starting'); + this.scheduleRetry(); + } + return; + } this.lastEventAt = event.occurredAt; this.lastError = undefined; - await this.options.grpcSdk.state!.setKey(RESUME_TOKEN_KEY, event.resumeToken); - this.options.grpcSdk.bus?.publish( - `database:change:${schema.name}`, - JSON.stringify(event), - ); + await this.emitChange(schema, event); + if (token) { + await this.persistResumeToken(token); + } + } + + private async persistResumeToken(token: string) { + await this.options.grpcSdk.state!.setKey(RESUME_TOKEN_KEY, token); + } + + private async emitChange(schema: OptedInSchema, event: DatabaseChangeEvent) { + const payload = JSON.stringify(event); + this.options.grpcSdk.bus?.publish(`database:change:${schema.name}`, payload); ConduitGrpcSdk.Metrics?.increment('database_realtime_events_total', 1, { operation: event.operation, }); - await this.pushEvent(schema, event); + await this.pushEvent(schema, event, payload); } - private async pushEvent(schema: OptedInSchema, event: DatabaseChangeEvent) { - const payload = JSON.stringify(event); + private async pushEvent( + schema: OptedInSchema, + event: DatabaseChangeEvent, + payload: string, + ) { const adminRooms = roomsForPublicChange(schema.name, event.documentId); await this.safePush('admin', adminRooms, payload); if (!schema.authorizationEnabled) { @@ -229,15 +285,16 @@ export class MongoChangeStreamCoordinator { schema.name, event.documentId, ); + const decisions = await checkRebacBatch( + this.rebacCache, + this.options.grpcSdk as unknown as AuthorizationSdk, + userIds, + schema.name, + event.documentId, + ); const allowedRooms: string[] = []; for (const userId of userIds) { - const allowed = await canReadDocument( - this.options.grpcSdk as unknown as AuthorizationSdk, - schema.name, - event.documentId, - userId, - ); - if (!allowed) { + if (decisions.get(userId) !== 'allow') { await this.options.subscriptions.removeUser( schema.name, event.documentId, @@ -260,16 +317,12 @@ export class MongoChangeStreamCoordinator { const client = target === 'admin' ? this.options.grpcSdk.admin : this.options.grpcSdk.router; if (!client?.socketPush) return; - try { - await client.socketPush({ - event: 'change', - data, - rooms, - receivers: [], - }); - } catch (err) { - ConduitGrpcSdk.Logger.error(err as Error); - } + await client.socketPush({ + event: 'change', + data, + rooms, + receivers: [], + }); } private resolveSchema(collectionName?: string): OptedInSchema | undefined { @@ -308,6 +361,7 @@ export class MongoChangeStreamCoordinator { this.watching = false; this.streamState = nextState; this.ignoreClose = true; + this.watchedCollectionsKey = ''; if (stream) { try { await stream.close(); diff --git a/modules/database/src/realtime/RealtimeService.ts b/modules/database/src/realtime/RealtimeService.ts index e17e320ea..96d21f485 100644 --- a/modules/database/src/realtime/RealtimeService.ts +++ b/modules/database/src/realtime/RealtimeService.ts @@ -19,6 +19,7 @@ import { registerDatabaseRealtimeSocket } from './sockets.js'; import { buildRealtimeStatus } from './status.js'; import { RealtimeSubscriptionTracker } from './subscriptions.js'; import type { ChangeStreamLike, OptedInSchema, RealtimeStatus } from './types.js'; +import type { WatchPipeline } from './watchPipeline.js'; export class RealtimeService { private readonly subscriptions: RealtimeSubscriptionTracker; @@ -40,7 +41,7 @@ export class RealtimeService { if (adapter instanceof MongooseAdapter) { this.coordinator = new MongoChangeStreamCoordinator({ grpcSdk, - watch: options => this.openWatch(adapter, options.resumeAfter), + watch: options => this.openWatch(adapter, options), hello: () => this.hello(adapter), getOptedInSchemas: () => this.getOptedInSchemas(), subscriptions: this.subscriptions, @@ -128,14 +129,17 @@ export class RealtimeService { return schemas; } - private openWatch(adapter: MongooseAdapter, resumeAfter?: unknown): ChangeStreamLike { + private openWatch( + adapter: MongooseAdapter, + options: { resumeAfter?: unknown; pipeline: WatchPipeline }, + ): ChangeStreamLike { const db = adapter.mongoose.connection.db; if (!db) { throw new Error('MongoDB connection is not ready'); } return db.watch( - [], - resumeAfter ? { resumeAfter: resumeAfter as never } : {}, + options.pipeline, + options.resumeAfter ? { resumeAfter: options.resumeAfter as never } : {}, ) as unknown as ChangeStreamLike; } diff --git a/modules/database/src/realtime/__tests__/change-stream.integration.test.ts b/modules/database/src/realtime/__tests__/change-stream.integration.test.ts index dd74753fc..b4e313310 100644 --- a/modules/database/src/realtime/__tests__/change-stream.integration.test.ts +++ b/modules/database/src/realtime/__tests__/change-stream.integration.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from '@jest/globals'; import { MongoClient, ObjectId } from 'mongodb'; import { normalizeChangeEvent, type RawChangeEvent } from '../normalize.js'; +import { buildWatchPipeline } from '../watchPipeline.js'; const replicaSetUri = process.env.DB_CONN_URI; const integrationEnabled = Boolean(replicaSetUri?.includes('replicaSet')); @@ -14,7 +15,7 @@ describeIntegration('MongoDB change stream contract', () => { const db = client.db(dbName); try { const collection = db.collection('orders'); - const stream = db.watch([]); + const stream = db.watch(buildWatchPipeline(['orders'])); const change = await new Promise((resolve, reject) => { const timer = setTimeout( () => reject(new Error('timed out waiting for change')), diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index efddfca70..b5aa0ff15 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -1,6 +1,6 @@ import { EventEmitter } from 'node:events'; import { describe, expect, it, jest } from '@jest/globals'; -import { ObjectId } from 'bson'; +import { EJSON, ObjectId } from 'bson'; import { MongoChangeStreamCoordinator } from '../MongoChangeStreamCoordinator.js'; import { RealtimeSubscriptionTracker } from '../subscriptions.js'; import { roomsForPublicChange } from '../rooms.js'; @@ -96,26 +96,28 @@ function createCoordinator(overrides?: { }; } +function insertChange(collection: string, id: string, token: unknown) { + return { + operationType: 'insert', + ns: { coll: collection }, + documentKey: { _id: new ObjectId(id) }, + _id: token, + }; +} + describe('MongoChangeStreamCoordinator', () => { it('emits one normalized event to public rooms and ignores other collections', async () => { - const { coordinator, stream, routerPush, adminPush, publish } = createCoordinator(); + const { coordinator, stream, routerPush, adminPush, publish, state } = + createCoordinator(); await coordinator.reconcile(); const resume = { _data: 'token' }; stream.emit('change', { - operationType: 'insert', - ns: { coll: 'orders' }, - documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, + ...insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', resume), fullDocument: { secret: 'nope' }, - _id: resume, wallTime: new Date('2026-01-02T00:00:00.000Z'), }); - stream.emit('change', { - operationType: 'insert', - ns: { coll: 'other' }, - documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4d') }, - _id: resume, - }); - await new Promise(resolve => setImmediate(resolve)); + stream.emit('change', insertChange('other', '64b64c4c4c4c4c4c4c4c4c4d', resume)); + await coordinator.waitForIdle(); expect(publish).toHaveBeenCalledTimes(1); expect(publish.mock.calls[0][0]).toBe('database:change:Order'); const payload = JSON.parse(publish.mock.calls[0][1] as string); @@ -125,6 +127,7 @@ describe('MongoChangeStreamCoordinator', () => { documentId: '64b64c4c4c4c4c4c4c4c4c4c', }); expect(payload).not.toHaveProperty('fullDocument'); + expect(payload).not.toHaveProperty('resumeToken'); expect(payload).not.toHaveProperty('secret'); const expectedRooms = roomsForPublicChange('Order', '64b64c4c4c4c4c4c4c4c4c4c'); expect(routerPush).toHaveBeenCalledWith( @@ -133,6 +136,108 @@ describe('MongoChangeStreamCoordinator', () => { expect(adminPush).toHaveBeenCalledWith( expect.objectContaining({ event: 'change', rooms: expectedRooms }), ); + expect(JSON.parse((adminPush.mock.calls[0][0] as { data: string }).data)).not.toHaveProperty( + 'resumeToken', + ); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(resume)); + await coordinator.shutdown(); + }); + + it('advances the resume token for filtered events', async () => { + const { coordinator, stream, publish, state } = createCoordinator(); + await coordinator.reconcile(); + const skip = { _data: 'skip-token' }; + stream.emit('change', insertChange('other', '64b64c4c4c4c4c4c4c4c4c4d', skip)); + await coordinator.waitForIdle(); + expect(publish).not.toHaveBeenCalled(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(skip)); + await coordinator.shutdown(); + }); + + it('serializes overlapping handlers and persists after emit', async () => { + const { coordinator, stream, state, adminPush } = createCoordinator(); + let release!: () => void; + const gate = new Promise(resolve => { + release = resolve; + }); + let first = true; + adminPush.mockImplementation(async () => { + if (first) { + first = false; + await gate; + } + }); + await coordinator.reconcile(); + const tokenA = { _data: 'token-a' }; + const tokenB = { _data: 'token-b' }; + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA), + ); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', tokenB), + ); + await Promise.resolve(); + await new Promise(resolve => setImmediate(resolve)); + expect(state.get('realtime:resumeToken')).toBeUndefined(); + expect(adminPush).toHaveBeenCalledTimes(1); + release(); + await coordinator.waitForIdle(); + expect(adminPush).toHaveBeenCalledTimes(2); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenB)); + await coordinator.shutdown(); + }); + + it('watches opted-in collections with $match and $project', async () => { + const { coordinator, watch } = createCoordinator(); + await coordinator.reconcile(); + expect(watch).toHaveBeenCalledTimes(1); + const pipeline = (watch.mock.calls[0][0] as { pipeline: Record[] }) + .pipeline; + expect(pipeline[0]).toEqual( + expect.objectContaining({ + $match: expect.objectContaining({ + $or: expect.arrayContaining([ + expect.objectContaining({ + 'ns.coll': { $in: ['orders'] }, + }), + ]), + }), + }), + ); + expect(pipeline[1]).toEqual({ + $project: { + fullDocument: 0, + updateDescription: 0, + fullDocumentBeforeChange: 0, + }, + }); + await coordinator.shutdown(); + }); + + it('reopens the watch when the opt-in set changes', async () => { + const schemas = [ + { name: 'Order', collectionName: 'orders', authorizationEnabled: false }, + ]; + const { coordinator, watch } = createCoordinator({ schemas }); + await coordinator.reconcile(); + expect(watch).toHaveBeenCalledTimes(1); + schemas.push({ + name: 'Item', + collectionName: 'items', + authorizationEnabled: false, + }); + await coordinator.reconcile(); + expect(watch).toHaveBeenCalledTimes(2); + const pipeline = (watch.mock.calls[1][0] as { pipeline: Record[] }) + .pipeline; + const match = pipeline[0] as { + $match: { $or: Array<{ 'ns.coll'?: { $in: string[] } }> }; + }; + expect(match.$match.$or[0]['ns.coll']?.$in).toEqual( + expect.arrayContaining(['orders', 'items']), + ); await coordinator.shutdown(); }); @@ -148,13 +253,11 @@ describe('MongoChangeStreamCoordinator', () => { 'user-1', ); await coordinator.reconcile(); - stream.emit('change', { - operationType: 'update', - ns: { coll: 'orders' }, - documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, - _id: { _data: 'token' }, - }); - await new Promise(resolve => setImmediate(resolve)); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), + ); + await coordinator.waitForIdle(); expect(routerPush).not.toHaveBeenCalled(); expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c')).toEqual([]); await coordinator.shutdown(); @@ -179,7 +282,7 @@ describe('MongoChangeStreamCoordinator', () => { await coordinator.shutdown(); }); - it('clears an unusable resume token and retries', async () => { + it('clears an unusable resume token on 280 and retries', async () => { const { coordinator, stream, grpcSdk } = createCoordinator(); await coordinator.reconcile(); stream.emit('error', { code: 280, message: 'ChangeStreamHistoryLost' }); @@ -187,4 +290,21 @@ describe('MongoChangeStreamCoordinator', () => { expect(grpcSdk.state.clearKey).toHaveBeenCalled(); await coordinator.shutdown(); }); + + it('keeps the resume token on CursorKilled 237', async () => { + const { coordinator, stream, grpcSdk, state } = createCoordinator(); + await coordinator.reconcile(); + const token = { _data: 'keep-me' }; + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', token), + ); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); + stream.emit('error', { code: 237, message: 'CursorKilled' }); + await new Promise(resolve => setImmediate(resolve)); + expect(grpcSdk.state.clearKey).not.toHaveBeenCalled(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); + await coordinator.shutdown(); + }); }); diff --git a/modules/database/src/realtime/__tests__/normalize.test.ts b/modules/database/src/realtime/__tests__/normalize.test.ts index db2278e29..249d86ab9 100644 --- a/modules/database/src/realtime/__tests__/normalize.test.ts +++ b/modules/database/src/realtime/__tests__/normalize.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from '@jest/globals'; import { ObjectId } from 'bson'; -import { normalizeChangeEvent, parseResumeToken } from '../normalize.js'; +import { normalizeChangeEvent, parseResumeToken, serializeResumeToken } from '../normalize.js'; describe('normalizeChangeEvent', () => { it('normalizes insert/update/replace/delete into metadata-only events', () => { @@ -22,7 +22,8 @@ describe('normalizeChangeEvent', () => { documentId: '64b64c4c4c4c4c4c4c4c4c4c', occurredAt: '2026-01-01T00:00:00.000Z', }); - expect(parseResumeToken(event!.resumeToken)).toEqual(resume); + expect(parseResumeToken(serializeResumeToken(resume))).toEqual(resume); + expect(event).not.toHaveProperty('resumeToken'); expect(JSON.parse(JSON.stringify(event))).not.toHaveProperty('fullDocument'); }); diff --git a/modules/database/src/realtime/__tests__/recovery.test.ts b/modules/database/src/realtime/__tests__/recovery.test.ts new file mode 100644 index 000000000..d38749dfa --- /dev/null +++ b/modules/database/src/realtime/__tests__/recovery.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from '@jest/globals'; +import { canReadDocument } from '../authorize.js'; +import { authorizedDocumentRoom } from '../rooms.js'; +import { + isRecoverableDisconnect, + restoreAuthorizedSubscriptions, +} from '../recovery.js'; +import { createSocketHandlers } from '../sockets.js'; +import { RealtimeSubscriptionTracker } from '../subscriptions.js'; + +class MemoryStore { + private sets = new Map>(); + async sadd(key: string, ...members: string[]) { + const set = this.sets.get(key) ?? new Set(); + members.forEach(member => set.add(member)); + this.sets.set(key, set); + } + async srem(key: string, ...members: string[]) { + members.forEach(member => this.sets.get(key)?.delete(member)); + } + async smembers(key: string) { + return [...(this.sets.get(key) ?? new Set())]; + } + async scard(key: string) { + return this.sets.get(key)?.size ?? 0; + } + async del(...keys: string[]) { + keys.forEach(key => this.sets.delete(key)); + } +} + +describe('database socket recovery', () => { + it('does not treat transport close as a wipe', () => { + expect(isRecoverableDisconnect('transport close')).toBe(true); + expect(isRecoverableDisconnect('client namespace disconnect')).toBe(false); + }); + + it('restores authorized Redis membership after a recoverable disconnect', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + const room = authorizedDocumentRoom('Order', 'doc-1', 'user-1'); + await tracker.addAuthorizedDocument('sock-1', 'Order', 'doc-1', 'user-1'); + await tracker.disconnect('sock-1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); + + await restoreAuthorizedSubscriptions({ + socketId: 'sock-1', + rooms: [room], + contextSubs: [], + subscriptions: tracker, + grpcSdk: { + isAvailable: () => true, + authorization: { can: async () => ({ allow: true }) }, + }, + canRead: canReadDocument, + }); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + }); + + it('keeps recovered /database/ sockets in the authorized list', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + await tracker.addAuthorizedDocument('sock-1', 'Order', 'doc-1', 'user-1'); + const handlers = createSocketHandlers({ + mode: 'client', + grpcSdk: { + isAvailable: () => true, + authorization: { can: async () => ({ allow: true }) }, + } as never, + schemaLookup: { getSchema: () => undefined }, + subscriptions: tracker, + isGloballyEnabled: () => true, + }); + await handlers.disconnect({ + request: { socketId: 'sock-1', params: ['transport close'] }, + } as never); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + await handlers.recovered({ + request: { + socketId: 'sock-1', + params: [authorizedDocumentRoom('Order', 'doc-1', 'user-1')], + context: {}, + }, + } as never); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + }); +}); diff --git a/modules/database/src/realtime/__tests__/rooms.test.ts b/modules/database/src/realtime/__tests__/rooms.test.ts index dbdd57e1b..765439786 100644 --- a/modules/database/src/realtime/__tests__/rooms.test.ts +++ b/modules/database/src/realtime/__tests__/rooms.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from '@jest/globals'; import { authorizedDocumentRoom, documentRoom, + parseAuthorizedDocumentRoom, roomsForPublicChange, schemaRoom, } from '../rooms.js'; @@ -21,6 +22,11 @@ describe('realtime rooms', () => { expect(authorizedDocumentRoom('Order', 'abc', 'u1')).not.toBe( authorizedDocumentRoom('Order', 'abc', 'u2'), ); + expect(parseAuthorizedDocumentRoom(authorizedDocumentRoom('Order', 'abc', 'user-1'))).toEqual({ + schema: 'Order', + documentId: 'abc', + userId: 'user-1', + }); }); it('encodes reserved characters so rooms cannot collide', () => { diff --git a/modules/database/src/realtime/__tests__/topology.test.ts b/modules/database/src/realtime/__tests__/topology.test.ts index abf8e8861..79afac87c 100644 --- a/modules/database/src/realtime/__tests__/topology.test.ts +++ b/modules/database/src/realtime/__tests__/topology.test.ts @@ -10,7 +10,10 @@ describe('topology helpers', () => { it('detects unusable resume tokens', () => { expect(isResumeTokenUnusable({ code: 280 })).toBe(true); - expect(isResumeTokenUnusable(new Error('cannot resume'))).toBe(true); + expect(isResumeTokenUnusable({ code: 237 })).toBe(false); + expect(isResumeTokenUnusable({ code: 136 })).toBe(false); + expect(isResumeTokenUnusable(new Error('ChangeStreamHistoryLost'))).toBe(true); + expect(isResumeTokenUnusable(new Error('cannot resume'))).toBe(false); expect(isResumeTokenUnusable(new Error('socket hang up'))).toBe(false); }); }); diff --git a/modules/database/src/realtime/__tests__/watchPipeline.test.ts b/modules/database/src/realtime/__tests__/watchPipeline.test.ts new file mode 100644 index 000000000..7c31a64f5 --- /dev/null +++ b/modules/database/src/realtime/__tests__/watchPipeline.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from '@jest/globals'; +import { buildWatchPipeline } from '../watchPipeline.js'; + +describe('buildWatchPipeline', () => { + it('matches opted-in collections and projects out fullDocument', () => { + const pipeline = buildWatchPipeline(['orders', 'items']); + expect(pipeline[0]).toEqual({ + $match: { + $or: [ + { + operationType: { $in: ['insert', 'update', 'replace', 'delete'] }, + 'ns.coll': { $in: ['orders', 'items'] }, + }, + { + operationType: { $in: ['drop', 'rename', 'invalidate', 'dropDatabase'] }, + }, + ], + }, + }); + expect(pipeline[1]).toEqual({ + $project: { + fullDocument: 0, + updateDescription: 0, + fullDocumentBeforeChange: 0, + }, + }); + }); +}); diff --git a/modules/database/src/realtime/normalize.ts b/modules/database/src/realtime/normalize.ts index bdd876f16..b2f4b8ff3 100644 --- a/modules/database/src/realtime/normalize.ts +++ b/modules/database/src/realtime/normalize.ts @@ -42,10 +42,18 @@ export function normalizeChangeEvent( ? change.wallTime : occurredAt ).toISOString(), - resumeToken: EJSON.stringify(change._id), }; } +export function serializeResumeToken(id: unknown): string | null { + if (id === undefined || id === null) return null; + try { + return EJSON.stringify(id); + } catch { + return null; + } +} + export function parseResumeToken(token: string | null | undefined): unknown | undefined { if (!token) return undefined; try { diff --git a/modules/database/src/realtime/rebacCache.ts b/modules/database/src/realtime/rebacCache.ts new file mode 100644 index 000000000..b33a9438c --- /dev/null +++ b/modules/database/src/realtime/rebacCache.ts @@ -0,0 +1,96 @@ +import type { AuthorizationSdk } from './authorize.js'; + +export type RebacDecision = 'allow' | 'deny' | 'unavailable'; + +const DEFAULT_TTL_MS = 12_000; +const DEFAULT_MAX_ENTRIES = 10_000; + +type CacheEntry = { + decision: 'allow' | 'deny'; + expiresAt: number; +}; + +export class RealtimeRebacCache { + private readonly entries = new Map(); + + constructor( + private readonly ttlMs: number = DEFAULT_TTL_MS, + private readonly maxEntries: number = DEFAULT_MAX_ENTRIES, + ) {} + + async check( + grpcSdk: AuthorizationSdk, + userId: string, + schema: string, + documentId: string, + ): Promise { + const resource = `${schema}:${documentId}`; + const key = `${userId}:read:${resource}`; + const now = Date.now(); + this.sweepExpired(now); + const cached = this.entries.get(key); + if (cached && cached.expiresAt > now) { + return cached.decision; + } + if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { + return 'unavailable'; + } + try { + const decision = await grpcSdk.authorization.can({ + subject: `User:${userId}`, + actions: ['read'], + resource, + }); + const value: 'allow' | 'deny' = decision.allow ? 'allow' : 'deny'; + this.set(key, value, now); + return value; + } catch { + return 'unavailable'; + } + } + + clear(): void { + this.entries.clear(); + } + + private set(key: string, decision: 'allow' | 'deny', now: number): void { + if (this.entries.size >= this.maxEntries && !this.entries.has(key)) { + const firstKey = this.entries.keys().next().value; + if (firstKey) { + this.entries.delete(firstKey); + } + } + this.entries.set(key, { decision, expiresAt: now + this.ttlMs }); + } + + private sweepExpired(now: number): void { + for (const [key, entry] of this.entries) { + if (entry.expiresAt <= now) { + this.entries.delete(key); + } + } + } +} + +export async function checkRebacBatch( + cache: RealtimeRebacCache, + grpcSdk: AuthorizationSdk, + userIds: string[], + schema: string, + documentId: string, + concurrency = 8, +): Promise> { + const results = new Map(); + let index = 0; + async function worker(): Promise { + while (index < userIds.length) { + const userId = userIds[index++]; + results.set(userId, await cache.check(grpcSdk, userId, schema, documentId)); + } + } + const workers = Array.from({ length: Math.min(concurrency, userIds.length) }, () => + worker(), + ); + await Promise.all(workers); + return results; +} diff --git a/modules/database/src/realtime/recovery.ts b/modules/database/src/realtime/recovery.ts new file mode 100644 index 000000000..3bc9c3868 --- /dev/null +++ b/modules/database/src/realtime/recovery.ts @@ -0,0 +1,147 @@ +import type { Indexable, ParsedSocketRequest } from '@conduitplatform/grpc-sdk'; +import type { AuthorizationSdk } from './authorize.js'; +import { authorizedDocumentRoom, parseAuthorizedDocumentRoom } from './rooms.js'; +import type { RealtimeSubscriptionTracker } from './subscriptions.js'; + +const RECOVERABLE_DISCONNECT = new Set([ + 'ping timeout', + 'transport close', + 'transport error', +]); + +const CONTEXT_KEY = 'databaseSubs'; + +export type AuthorizedSub = { + schema: string; + documentId: string; + userId: string; +}; + +export function isRecoverableDisconnect(reason: unknown): boolean { + return typeof reason === 'string' && RECOVERABLE_DISCONNECT.has(reason); +} + +export function authorizedSubsFromContext( + context: Indexable | undefined, +): AuthorizedSub[] { + if (!context) return []; + const raw = context[CONTEXT_KEY]; + if (!Array.isArray(raw)) return []; + const subs: AuthorizedSub[] = []; + for (const item of raw) { + if ( + item && + typeof item === 'object' && + typeof (item as AuthorizedSub).schema === 'string' && + typeof (item as AuthorizedSub).documentId === 'string' && + typeof (item as AuthorizedSub).userId === 'string' + ) { + subs.push({ + schema: (item as AuthorizedSub).schema, + documentId: (item as AuthorizedSub).documentId, + userId: (item as AuthorizedSub).userId, + }); + } + } + return subs; +} + +export function persistAuthorizedSubOnContext( + context: Indexable | undefined, + schema: string, + documentId: string, + userId: string, +): void { + if (!context) return; + const next = authorizedSubsFromContext(context).filter( + sub => + !(sub.schema === schema && sub.documentId === documentId && sub.userId === userId), + ); + next.push({ schema, documentId, userId }); + context[CONTEXT_KEY] = next; +} + +export function removeAuthorizedSubFromContext( + context: Indexable | undefined, + schema: string, + documentId: string, + userId: string, +): void { + if (!context) return; + const next = authorizedSubsFromContext(context).filter( + sub => + !(sub.schema === schema && sub.documentId === documentId && sub.userId === userId), + ); + if (next.length === 0) { + delete context[CONTEXT_KEY]; + } else { + context[CONTEXT_KEY] = next; + } +} + +export function recoveredRoomsFromRequest(call: ParsedSocketRequest): string[] { + const params = call.request.params ?? []; + if (params.every(item => typeof item === 'string')) { + return params as string[]; + } + return []; +} + +export async function restoreAuthorizedSubscriptions(options: { + socketId: string; + rooms: string[]; + contextSubs: AuthorizedSub[]; + subscriptions: RealtimeSubscriptionTracker; + grpcSdk: AuthorizationSdk; + canRead: ( + grpcSdk: AuthorizationSdk, + schema: string, + documentId: string, + userId: string, + ) => Promise; +}): Promise<{ leaveRooms: string[] }> { + const seen = new Set(); + const subs: AuthorizedSub[] = []; + for (const room of options.rooms) { + const parsed = parseAuthorizedDocumentRoom(room); + if (!parsed) continue; + const key = `${parsed.schema}:${parsed.documentId}:${parsed.userId}`; + if (seen.has(key)) continue; + seen.add(key); + subs.push(parsed); + } + for (const sub of options.contextSubs) { + const key = `${sub.schema}:${sub.documentId}:${sub.userId}`; + if (seen.has(key)) continue; + seen.add(key); + subs.push(sub); + } + + const leaveRooms: string[] = []; + for (const sub of subs) { + const allowed = await options.canRead( + options.grpcSdk, + sub.schema, + sub.documentId, + sub.userId, + ); + const room = authorizedDocumentRoom(sub.schema, sub.documentId, sub.userId); + if (!allowed) { + await options.subscriptions.removeAuthorizedDocument( + options.socketId, + sub.schema, + sub.documentId, + sub.userId, + ); + leaveRooms.push(room); + continue; + } + await options.subscriptions.addAuthorizedDocument( + options.socketId, + sub.schema, + sub.documentId, + sub.userId, + ); + } + return { leaveRooms }; +} diff --git a/modules/database/src/realtime/rooms.ts b/modules/database/src/realtime/rooms.ts index 45a69a7cd..5792482f9 100644 --- a/modules/database/src/realtime/rooms.ts +++ b/modules/database/src/realtime/rooms.ts @@ -23,3 +23,25 @@ export function authorizedDocumentRoom( export function roomsForPublicChange(schema: string, documentId: string): string[] { return [schemaRoom(schema), documentRoom(schema, documentId)]; } + +export function parseAuthorizedDocumentRoom(room: string): { + schema: string; + documentId: string; + userId: string; +} | null { + const prefix = `${ROOM_PREFIX}:doc:`; + if (!room.startsWith(prefix)) return null; + const rest = room.slice(prefix.length); + const userMarker = ':user:'; + const userIndex = rest.lastIndexOf(userMarker); + if (userIndex === -1) return null; + const userId = decodeURIComponent(rest.slice(userIndex + userMarker.length)); + const schemaDoc = rest.slice(0, userIndex); + const lastColon = schemaDoc.lastIndexOf(':'); + if (lastColon === -1) return null; + return { + schema: decodeURIComponent(schemaDoc.slice(0, lastColon)), + documentId: decodeURIComponent(schemaDoc.slice(lastColon + 1)), + userId, + }; +} diff --git a/modules/database/src/realtime/sockets.ts b/modules/database/src/realtime/sockets.ts index 4be4e1018..e2a80931c 100644 --- a/modules/database/src/realtime/sockets.ts +++ b/modules/database/src/realtime/sockets.ts @@ -1,6 +1,7 @@ import { ConduitGrpcSdk, ConduitRouteReturnDefinition, + type Indexable, ParsedSocketRequest, TYPE, UnparsedSocketResponse, @@ -19,6 +20,14 @@ import { } from './authorize.js'; import { authorizedDocumentRoom, documentRoom, schemaRoom } from './rooms.js'; import type { RealtimeSubscriptionTracker } from './subscriptions.js'; +import { + authorizedSubsFromContext, + isRecoverableDisconnect, + persistAuthorizedSubOnContext, + recoveredRoomsFromRequest, + removeAuthorizedSubFromContext, + restoreAuthorizedSubscriptions, +} from './recovery.js'; type SocketMode = 'client' | 'admin'; @@ -43,6 +52,7 @@ export function registerDatabaseRealtimeSocket( { connect: { handler: handlers.connect }, disconnect: { handler: handlers.disconnect }, + recovered: { handler: handlers.recovered }, subscribe: { params: [TYPE.JSON], handler: handlers.subscribe, @@ -61,15 +71,32 @@ export function registerDatabaseRealtimeSocket( ); } -function createSocketHandlers(options: RealtimeSocketOptions) { +export function createSocketHandlers(options: RealtimeSocketOptions) { return { connect: async (): Promise => { return { event: 'connected', data: { ok: true } }; }, disconnect: async (call: ParsedSocketRequest): Promise => { - await options.subscriptions.disconnect(call.request.socketId); + const reason = call.request.params?.[0]; + if (!isRecoverableDisconnect(reason)) { + await options.subscriptions.disconnect(call.request.socketId); + } return { event: 'disconnected', data: { ok: true } }; }, + recovered: async (call: ParsedSocketRequest): Promise => { + const { leaveRooms } = await restoreAuthorizedSubscriptions({ + socketId: call.request.socketId, + rooms: recoveredRoomsFromRequest(call), + contextSubs: authorizedSubsFromContext(call.request.context as Indexable), + subscriptions: options.subscriptions, + grpcSdk: options.grpcSdk as unknown as AuthorizationSdk, + canRead: canReadDocument, + }); + if (leaveRooms.length > 0) { + return { event: 'leave-room', rooms: leaveRooms }; + } + return { event: 'join-room', rooms: [] }; + }, subscribe: async (call: ParsedSocketRequest): Promise => { const rooms = await resolveSubscription(call, options, 'join'); return { event: 'join-room', rooms }; @@ -140,6 +167,12 @@ async function resolveSubscription( documentId, userId, ); + persistAuthorizedSubOnContext( + call.request.context as Indexable, + schemaName, + documentId, + userId, + ); } else { await options.subscriptions.removeAuthorizedDocument( call.request.socketId, @@ -147,6 +180,12 @@ async function resolveSubscription( documentId, userId, ); + removeAuthorizedSubFromContext( + call.request.context as Indexable, + schemaName, + documentId, + userId, + ); } return [authorizedDocumentRoom(schemaName, documentId, userId)]; } diff --git a/modules/database/src/realtime/topology.ts b/modules/database/src/realtime/topology.ts index 341d49438..4d62fa3fa 100644 --- a/modules/database/src/realtime/topology.ts +++ b/modules/database/src/realtime/topology.ts @@ -1,6 +1,4 @@ -const CHANGE_STREAM_ERROR_CODES = new Set([ - 136, // CappedPositionLost - 237, // CursorKilled +const UNUSABLE_RESUME_TOKEN_CODES = new Set([ 280, // ChangeStreamHistoryLost 286, // ChangeStreamFatalError ]); @@ -33,11 +31,14 @@ export function topologyFromHello( export function isResumeTokenUnusable(error: unknown): boolean { const code = extractErrorCode(error); - if (code !== undefined && CHANGE_STREAM_ERROR_CODES.has(code)) { + if (code === 237 || code === 136) { + return false; + } + if (code !== undefined && UNUSABLE_RESUME_TOKEN_CODES.has(code)) { return true; } const message = error instanceof Error ? error.message : String(error ?? ''); - return /resume token|ChangeStreamHistoryLost|cannot resume/i.test(message); + return /ChangeStreamHistoryLost/i.test(message); } function extractErrorCode(error: unknown): number | undefined { diff --git a/modules/database/src/realtime/types.ts b/modules/database/src/realtime/types.ts index 9901c2d86..2b6e3e8c7 100644 --- a/modules/database/src/realtime/types.ts +++ b/modules/database/src/realtime/types.ts @@ -15,7 +15,6 @@ export type DatabaseChangeEvent = { schema: string; documentId: string; occurredAt: string; - resumeToken: string; }; export type RealtimeStatusCode = diff --git a/modules/database/src/realtime/watchPipeline.ts b/modules/database/src/realtime/watchPipeline.ts new file mode 100644 index 000000000..8ca888262 --- /dev/null +++ b/modules/database/src/realtime/watchPipeline.ts @@ -0,0 +1,33 @@ +const DATA_OPERATIONS = ['insert', 'update', 'replace', 'delete'] as const; +const CONTROL_OPERATIONS = ['drop', 'rename', 'invalidate', 'dropDatabase'] as const; + +export const WATCH_RESTART_OPERATIONS = new Set(CONTROL_OPERATIONS); + +export type WatchPipeline = Record[]; + +export function buildWatchPipeline(collectionNames: string[]): WatchPipeline { + return [ + { + $match: { + $or: [ + { + operationType: { $in: [...DATA_OPERATIONS] }, + 'ns.coll': { $in: collectionNames }, + }, + { operationType: { $in: [...CONTROL_OPERATIONS] } }, + ], + }, + }, + { + $project: { + fullDocument: 0, + updateDescription: 0, + fullDocumentBeforeChange: 0, + }, + }, + ]; +} + +export function optedInCollectionsKey(collectionNames: string[]): string { + return [...collectionNames].sort().join('\0'); +} diff --git a/packages/core/src/admin/middleware/Admin.middleware.ts b/packages/core/src/admin/middleware/Admin.middleware.ts index 716dae763..4fb743246 100644 --- a/packages/core/src/admin/middleware/Admin.middleware.ts +++ b/packages/core/src/admin/middleware/Admin.middleware.ts @@ -2,9 +2,8 @@ import { NextFunction, Response } from 'express'; import { isNil } from 'lodash-es'; // Removed ConduitCommons import - now using configManager directly import { isDev } from '../utils/middleware.js'; -import { ConduitRequest } from '@conduitplatform/hermes'; +import { ConduitRequest, isSocketHandshake } from '@conduitplatform/hermes'; import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; -import { isSocketHandshake } from '../realtime/handshake.js'; export function getAdminMiddleware(configManager: any) { return async function adminMiddleware( diff --git a/packages/core/src/admin/middleware/Auth.middleware.ts b/packages/core/src/admin/middleware/Auth.middleware.ts index 410ea0fd6..57e5b1600 100644 --- a/packages/core/src/admin/middleware/Auth.middleware.ts +++ b/packages/core/src/admin/middleware/Auth.middleware.ts @@ -4,10 +4,9 @@ import { ConduitGrpcSdk, type FindManyOptions } from '@conduitplatform/grpc-sdk' import { Admin, AdminApiToken } from '../../models/index.js'; import { verifyToken, comparePasswords } from '../utils/auth.js'; import { isDev } from '../utils/middleware.js'; -import { ConduitRequest } from '@conduitplatform/hermes'; +import { ConduitRequest, isSocketHandshake } from '@conduitplatform/hermes'; import { gql } from 'graphql-tag'; import { ConfigController } from '@conduitplatform/module-tools'; -import { isSocketHandshake } from '../realtime/handshake.js'; import { isRealtimeTicket } from '../realtime/ticket.js'; const excludedRestRoutes = ['/ready', '/live', '/login', '/config/modules']; diff --git a/packages/core/src/admin/realtime/handshake.test.ts b/packages/core/src/admin/realtime/handshake.test.ts deleted file mode 100644 index 59d17547f..000000000 --- a/packages/core/src/admin/realtime/handshake.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; -import { isSocketHandshake } from './handshake.js'; - -describe('isSocketHandshake', () => { - it('detects Socket.IO engine requests', () => { - assert.equal(isSocketHandshake({ url: '/realtime/?EIO=4&transport=polling' }), true); - assert.equal(isSocketHandshake({ originalUrl: '/realtime' }), true); - assert.equal(isSocketHandshake({ path: '/login' }), false); - assert.equal(isSocketHandshake({ url: '/graphql' }), false); - }); -}); diff --git a/packages/core/src/admin/realtime/handshake.ts b/packages/core/src/admin/realtime/handshake.ts deleted file mode 100644 index a4bf06fe1..000000000 --- a/packages/core/src/admin/realtime/handshake.ts +++ /dev/null @@ -1,14 +0,0 @@ -export function isSocketHandshake(req: { - url?: string; - originalUrl?: string; - path?: string; -}): boolean { - const parts = [req.url, req.originalUrl, req.path].filter((value): value is string => - Boolean(value), - ); - return parts.some(part => part.includes('EIO=') || isRealtimePath(part)); -} - -function isRealtimePath(value: string): boolean { - return /(?:^|[/?])realtime(?:[/?]|$)/.test(value); -} diff --git a/packages/core/src/admin/realtime/ticket-http.test.ts b/packages/core/src/admin/realtime/ticket-http.test.ts new file mode 100644 index 000000000..9b896da19 --- /dev/null +++ b/packages/core/src/admin/realtime/ticket-http.test.ts @@ -0,0 +1,24 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { isSocketHandshake } from '@conduitplatform/hermes'; +import { buildRealtimeTicketClaims, isRealtimeTicket } from './ticket.js'; + +describe('realtime ticket HTTP guard', () => { + it('does not treat POST /realtime/ticket as a handshake so a ticket cannot mint another', () => { + const ticket = buildRealtimeTicketClaims('admin-1'); + assert.equal(isRealtimeTicket(ticket), true); + assert.equal(isSocketHandshake({ url: '/realtime/ticket' }), false); + assert.equal( + isSocketHandshake({ url: '/realtime/ticket?EIO=4&transport=polling' }), + false, + ); + assert.equal(isRealtimeTicket(ticket) && !isSocketHandshake({ url: '/realtime/ticket' }), true); + }); + + it('allows a ticket on Engine.IO handshake polling without sid', () => { + const ticket = buildRealtimeTicketClaims('admin-1'); + const handshake = { url: '/realtime/?EIO=4&transport=polling' }; + assert.equal(isSocketHandshake(handshake), true); + assert.equal(isRealtimeTicket(ticket) && isSocketHandshake(handshake), true); + }); +}); diff --git a/packages/core/tsconfig.test.json b/packages/core/tsconfig.test.json index b0548057f..8559c14bf 100644 --- a/packages/core/tsconfig.test.json +++ b/packages/core/tsconfig.test.json @@ -9,7 +9,6 @@ }, "include": [ "src/admin/realtime/ticket.ts", - "src/admin/realtime/handshake.ts", "src/admin/realtime/namespace.ts", "src/admin/realtime/*.test.ts" ], From 1f395ef43c0d478382d5106cc90c3e346e1f9e24 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 12:38:10 +0000 Subject: [PATCH 4/5] style: prettier on Mongo intervention tests --- .../realtime/__tests__/coordinator.test.ts | 21 ++++++------------- .../src/realtime/__tests__/normalize.test.ts | 6 +++++- .../src/realtime/__tests__/recovery.test.ts | 5 +---- .../src/realtime/__tests__/rooms.test.ts | 4 +++- .../src/admin/realtime/ticket-http.test.ts | 5 ++++- 5 files changed, 19 insertions(+), 22 deletions(-) diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index b5aa0ff15..4f71baf5c 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -136,9 +136,9 @@ describe('MongoChangeStreamCoordinator', () => { expect(adminPush).toHaveBeenCalledWith( expect.objectContaining({ event: 'change', rooms: expectedRooms }), ); - expect(JSON.parse((adminPush.mock.calls[0][0] as { data: string }).data)).not.toHaveProperty( - 'resumeToken', - ); + expect( + JSON.parse((adminPush.mock.calls[0][0] as { data: string }).data), + ).not.toHaveProperty('resumeToken'); expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(resume)); await coordinator.shutdown(); }); @@ -170,14 +170,8 @@ describe('MongoChangeStreamCoordinator', () => { await coordinator.reconcile(); const tokenA = { _data: 'token-a' }; const tokenB = { _data: 'token-b' }; - stream.emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA), - ); - stream.emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', tokenB), - ); + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', tokenB)); await Promise.resolve(); await new Promise(resolve => setImmediate(resolve)); expect(state.get('realtime:resumeToken')).toBeUndefined(); @@ -295,10 +289,7 @@ describe('MongoChangeStreamCoordinator', () => { const { coordinator, stream, grpcSdk, state } = createCoordinator(); await coordinator.reconcile(); const token = { _data: 'keep-me' }; - stream.emit( - 'change', - insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', token), - ); + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', token)); await coordinator.waitForIdle(); expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); stream.emit('error', { code: 237, message: 'CursorKilled' }); diff --git a/modules/database/src/realtime/__tests__/normalize.test.ts b/modules/database/src/realtime/__tests__/normalize.test.ts index 249d86ab9..f7f933d33 100644 --- a/modules/database/src/realtime/__tests__/normalize.test.ts +++ b/modules/database/src/realtime/__tests__/normalize.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from '@jest/globals'; import { ObjectId } from 'bson'; -import { normalizeChangeEvent, parseResumeToken, serializeResumeToken } from '../normalize.js'; +import { + normalizeChangeEvent, + parseResumeToken, + serializeResumeToken, +} from '../normalize.js'; describe('normalizeChangeEvent', () => { it('normalizes insert/update/replace/delete into metadata-only events', () => { diff --git a/modules/database/src/realtime/__tests__/recovery.test.ts b/modules/database/src/realtime/__tests__/recovery.test.ts index d38749dfa..eee45b918 100644 --- a/modules/database/src/realtime/__tests__/recovery.test.ts +++ b/modules/database/src/realtime/__tests__/recovery.test.ts @@ -1,10 +1,7 @@ import { describe, expect, it } from '@jest/globals'; import { canReadDocument } from '../authorize.js'; import { authorizedDocumentRoom } from '../rooms.js'; -import { - isRecoverableDisconnect, - restoreAuthorizedSubscriptions, -} from '../recovery.js'; +import { isRecoverableDisconnect, restoreAuthorizedSubscriptions } from '../recovery.js'; import { createSocketHandlers } from '../sockets.js'; import { RealtimeSubscriptionTracker } from '../subscriptions.js'; diff --git a/modules/database/src/realtime/__tests__/rooms.test.ts b/modules/database/src/realtime/__tests__/rooms.test.ts index 765439786..96de67f73 100644 --- a/modules/database/src/realtime/__tests__/rooms.test.ts +++ b/modules/database/src/realtime/__tests__/rooms.test.ts @@ -22,7 +22,9 @@ describe('realtime rooms', () => { expect(authorizedDocumentRoom('Order', 'abc', 'u1')).not.toBe( authorizedDocumentRoom('Order', 'abc', 'u2'), ); - expect(parseAuthorizedDocumentRoom(authorizedDocumentRoom('Order', 'abc', 'user-1'))).toEqual({ + expect( + parseAuthorizedDocumentRoom(authorizedDocumentRoom('Order', 'abc', 'user-1')), + ).toEqual({ schema: 'Order', documentId: 'abc', userId: 'user-1', diff --git a/packages/core/src/admin/realtime/ticket-http.test.ts b/packages/core/src/admin/realtime/ticket-http.test.ts index 9b896da19..fa00a00ac 100644 --- a/packages/core/src/admin/realtime/ticket-http.test.ts +++ b/packages/core/src/admin/realtime/ticket-http.test.ts @@ -12,7 +12,10 @@ describe('realtime ticket HTTP guard', () => { isSocketHandshake({ url: '/realtime/ticket?EIO=4&transport=polling' }), false, ); - assert.equal(isRealtimeTicket(ticket) && !isSocketHandshake({ url: '/realtime/ticket' }), true); + assert.equal( + isRealtimeTicket(ticket) && !isSocketHandshake({ url: '/realtime/ticket' }), + true, + ); }); it('allows a ticket on Engine.IO handshake polling without sid', () => { From ecdd5bf53c0342ea2b87b26335371e240929f946 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 13:03:51 +0000 Subject: [PATCH 5/5] fix(database): stop watch on emit failure; keep ReBAC membership Failed socketPush/bus no longer advances the resume token. ReBAC unavailable no longer removeUser. Auth middleware 401s a realtime ticket on POST /realtime/ticket. Drop/rename/invalidate reopen the watch. --- .../realtime/MongoChangeStreamCoordinator.ts | 23 ++- .../realtime/__tests__/coordinator.test.ts | 135 +++++++++++++++++- .../src/admin/middleware/Auth.middleware.ts | 6 +- .../src/admin/realtime/ticket-http.test.ts | 76 +++++++++- packages/core/src/admin/realtime/ticket.ts | 9 ++ 5 files changed, 229 insertions(+), 20 deletions(-) diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts index 339750065..d29938d15 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -227,12 +227,18 @@ export class MongoChangeStreamCoordinator { } private enqueueChange(change: RawChangeEvent) { - this.changeQueue = this.changeQueue - .then(() => this.handleChange(change)) - .catch(err => { + this.changeQueue = this.changeQueue.then(async () => { + if (this.closed || !this.watching) return; + try { + await this.handleChange(change); + } catch (err) { this.lastError = err instanceof Error ? err.message : String(err); ConduitGrpcSdk.Logger.error(err as Error); - }); + this.watching = false; + await this.stopStream('degraded'); + this.scheduleRetry(); + } + }); } private async handleChange(change: RawChangeEvent) { @@ -294,15 +300,18 @@ export class MongoChangeStreamCoordinator { ); const allowedRooms: string[] = []; for (const userId of userIds) { - if (decisions.get(userId) !== 'allow') { + const decision = decisions.get(userId) ?? 'unavailable'; + if (decision === 'allow') { + allowedRooms.push(authorizedDocumentRoom(schema.name, event.documentId, userId)); + continue; + } + if (decision === 'deny') { await this.options.subscriptions.removeUser( schema.name, event.documentId, userId, ); - continue; } - allowedRooms.push(authorizedDocumentRoom(schema.name, event.documentId, userId)); } if (allowedRooms.length > 0) { await this.safePush('router', allowedRooms, payload); diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index 4f71baf5c..cda5dc0e1 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events'; -import { describe, expect, it, jest } from '@jest/globals'; +import { afterEach, describe, expect, it, jest } from '@jest/globals'; import { EJSON, ObjectId } from 'bson'; import { MongoChangeStreamCoordinator } from '../MongoChangeStreamCoordinator.js'; import { RealtimeSubscriptionTracker } from '../subscriptions.js'; @@ -11,9 +11,13 @@ class MemoryStore { const set = this.sets.get(key) ?? new Set(); members.forEach(member => set.add(member)); this.sets.set(key, set); + return members.length; } async srem(key: string, ...members: string[]) { - members.forEach(member => this.sets.get(key)?.delete(member)); + const set = this.sets.get(key); + if (!set) return 0; + members.forEach(member => set.delete(member)); + return members.length; } async smembers(key: string) { return [...(this.sets.get(key) ?? new Set())]; @@ -23,11 +27,13 @@ class MemoryStore { } async del(...keys: string[]) { keys.forEach(key => this.sets.delete(key)); + return keys.length; } } function createCoordinator(overrides?: { allow?: boolean; + authorizationAvailable?: boolean; schemas?: { name: string; collectionName: string; authorizationEnabled: boolean }[]; getKeyDelayMs?: number; }) { @@ -65,10 +71,13 @@ function createCoordinator(overrides?: { bus: { publish }, router: { socketPush: routerPush }, admin: { socketPush: adminPush }, - isAvailable: () => true, - authorization: { - can: async () => ({ allow: overrides?.allow !== false }), - }, + isAvailable: () => overrides?.authorizationAvailable !== false, + authorization: + overrides?.authorizationAvailable === false + ? null + : { + can: async () => ({ allow: overrides?.allow !== false }), + }, }; const coordinator = new MongoChangeStreamCoordinator({ grpcSdk: grpcSdk as never, @@ -106,6 +115,10 @@ function insertChange(collection: string, id: string, token: unknown) { } describe('MongoChangeStreamCoordinator', () => { + afterEach(() => { + jest.useRealTimers(); + }); + it('emits one normalized event to public rooms and ignores other collections', async () => { const { coordinator, stream, routerPush, adminPush, publish, state } = createCoordinator(); @@ -253,7 +266,9 @@ describe('MongoChangeStreamCoordinator', () => { ); await coordinator.waitForIdle(); expect(routerPush).not.toHaveBeenCalled(); - expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c')).toEqual([]); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual( + [], + ); await coordinator.shutdown(); }); @@ -298,4 +313,110 @@ describe('MongoChangeStreamCoordinator', () => { expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); await coordinator.shutdown(); }); + + it('does not persist a later token when emit fails and reopens from the last good token', async () => { + jest.useFakeTimers(); + const streams: Array Promise }> = []; + const { coordinator, adminPush, state, watch } = createCoordinator(); + watch.mockImplementation(() => { + const next = new EventEmitter() as EventEmitter & { close: () => Promise }; + next.close = async () => { + next.emit('close'); + }; + streams.push(next); + return next as never; + }); + adminPush + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error('push failed')) + .mockResolvedValue(undefined); + await coordinator.reconcile(); + const tokenGood = { _data: 'token-good' }; + const tokenA = { _data: 'token-a' }; + const tokenB = { _data: 'token-b' }; + streams[0].emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4b', tokenGood), + ); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenGood)); + streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); + streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', tokenB)); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenGood)); + expect(adminPush).toHaveBeenCalledTimes(2); + await jest.advanceTimersByTimeAsync(1_000); + expect(watch).toHaveBeenCalledTimes(2); + expect((watch.mock.calls[1][0] as { resumeAfter?: unknown }).resumeAfter).toEqual( + tokenGood, + ); + streams[1].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenA)); + expect(adminPush).toHaveBeenCalledTimes(3); + await coordinator.shutdown(); + jest.useRealTimers(); + }); + + it.each(['drop', 'rename', 'invalidate'] as const)( + 'reopens the watch on %s', + async operationType => { + jest.useFakeTimers(); + const streams: Array Promise }> = []; + const { coordinator, watch, state } = createCoordinator(); + watch.mockImplementation(() => { + const next = new EventEmitter() as EventEmitter & { close: () => Promise }; + next.close = async () => { + next.emit('close'); + }; + streams.push(next); + return next as never; + }); + await coordinator.reconcile(); + const token = { _data: `${operationType}-token` }; + streams[0].emit('change', { + operationType, + ns: { coll: 'orders' }, + _id: token, + }); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); + await jest.advanceTimersByTimeAsync(1_000); + expect(watch).toHaveBeenCalledTimes(2); + expect((watch.mock.calls[1][0] as { resumeAfter?: unknown }).resumeAfter).toEqual( + token, + ); + await coordinator.shutdown(); + jest.useRealTimers(); + }, + ); + + it('does not remove users when authorization is unavailable', async () => { + const { coordinator, stream, routerPush, subscriptions } = createCoordinator({ + authorizationAvailable: false, + schemas: [{ name: 'Order', collectionName: 'orders', authorizationEnabled: true }], + }); + const removeUser = jest.spyOn(subscriptions, 'removeUser'); + await subscriptions.addAuthorizedDocument( + 'sock-1', + 'Order', + '64b64c4c4c4c4c4c4c4c4c4c', + 'user-1', + ); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual([ + 'user-1', + ]); + await coordinator.reconcile(); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), + ); + await coordinator.waitForIdle(); + expect(routerPush).not.toHaveBeenCalled(); + expect(removeUser).not.toHaveBeenCalled(); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual([ + 'user-1', + ]); + await coordinator.shutdown(); + }); }); diff --git a/packages/core/src/admin/middleware/Auth.middleware.ts b/packages/core/src/admin/middleware/Auth.middleware.ts index 57e5b1600..9bac68d99 100644 --- a/packages/core/src/admin/middleware/Auth.middleware.ts +++ b/packages/core/src/admin/middleware/Auth.middleware.ts @@ -4,10 +4,10 @@ import { ConduitGrpcSdk, type FindManyOptions } from '@conduitplatform/grpc-sdk' import { Admin, AdminApiToken } from '../../models/index.js'; import { verifyToken, comparePasswords } from '../utils/auth.js'; import { isDev } from '../utils/middleware.js'; -import { ConduitRequest, isSocketHandshake } from '@conduitplatform/hermes'; +import { ConduitRequest } from '@conduitplatform/hermes'; import { gql } from 'graphql-tag'; import { ConfigController } from '@conduitplatform/module-tools'; -import { isRealtimeTicket } from '../realtime/ticket.js'; +import { realtimeTicketForbiddenOnHttp } from '../realtime/ticket.js'; const excludedRestRoutes = ['/ready', '/live', '/login', '/config/modules']; const excludedGqlOperations = [ @@ -114,7 +114,7 @@ async function handleJwtToken( } const { id } = decoded; - if (isRealtimeTicket(decoded) && !isSocketHandshake(req)) { + if (realtimeTicketForbiddenOnHttp(decoded, req)) { res.status(401).json({ error: 'Realtime ticket cannot be used for HTTP requests' }); return; } diff --git a/packages/core/src/admin/realtime/ticket-http.test.ts b/packages/core/src/admin/realtime/ticket-http.test.ts index fa00a00ac..589975acc 100644 --- a/packages/core/src/admin/realtime/ticket-http.test.ts +++ b/packages/core/src/admin/realtime/ticket-http.test.ts @@ -1,7 +1,38 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; +import jwt from 'jsonwebtoken'; +import { ConfigController } from '@conduitplatform/module-tools'; import { isSocketHandshake } from '@conduitplatform/hermes'; -import { buildRealtimeTicketClaims, isRealtimeTicket } from './ticket.js'; +import { getAuthMiddleware } from '../middleware/Auth.middleware.js'; +import { + buildRealtimeTicketClaims, + isRealtimeTicket, + realtimeTicketForbiddenOnHttp, +} from './ticket.js'; + +function mockResponse() { + let statusCode = 0; + let body: unknown; + const res = { + status(code: number) { + statusCode = code; + return this; + }, + json(payload: unknown) { + body = payload; + return this; + }, + }; + return { + res, + get statusCode() { + return statusCode; + }, + get body() { + return body; + }, + }; +} describe('realtime ticket HTTP guard', () => { it('does not treat POST /realtime/ticket as a handshake so a ticket cannot mint another', () => { @@ -13,7 +44,13 @@ describe('realtime ticket HTTP guard', () => { false, ); assert.equal( - isRealtimeTicket(ticket) && !isSocketHandshake({ url: '/realtime/ticket' }), + realtimeTicketForbiddenOnHttp(ticket, { url: '/realtime/ticket' }), + true, + ); + assert.equal( + realtimeTicketForbiddenOnHttp(ticket, { + url: '/realtime/ticket?EIO=4&transport=polling', + }), true, ); }); @@ -22,6 +59,39 @@ describe('realtime ticket HTTP guard', () => { const ticket = buildRealtimeTicketClaims('admin-1'); const handshake = { url: '/realtime/?EIO=4&transport=polling' }; assert.equal(isSocketHandshake(handshake), true); - assert.equal(isRealtimeTicket(ticket) && isSocketHandshake(handshake), true); + assert.equal(realtimeTicketForbiddenOnHttp(ticket, handshake), false); + }); + + it('Auth middleware returns 401 on POST /realtime/ticket with a realtime ticket', async () => { + const secret = 'ticket-http-test-secret'; + ConfigController.getInstance().config = { auth: { tokenSecret: secret } }; + const token = jwt.sign(buildRealtimeTicketClaims('admin-1'), secret, { + algorithm: 'HS256', + expiresIn: 30, + }); + const middleware = getAuthMiddleware({} as never, { + get: async () => ({ env: 'production' }), + }); + const mock = mockResponse(); + let nextCalled = false; + await middleware( + { + path: '/realtime/ticket', + originalUrl: '/realtime/ticket', + url: '/realtime/ticket', + method: 'POST', + headers: { authorization: `Bearer ${token}` }, + conduit: {}, + } as never, + mock.res as never, + () => { + nextCalled = true; + }, + ); + assert.equal(nextCalled, false); + assert.equal(mock.statusCode, 401); + assert.deepEqual(mock.body, { + error: 'Realtime ticket cannot be used for HTTP requests', + }); }); }); diff --git a/packages/core/src/admin/realtime/ticket.ts b/packages/core/src/admin/realtime/ticket.ts index 154630c0c..e797f5966 100644 --- a/packages/core/src/admin/realtime/ticket.ts +++ b/packages/core/src/admin/realtime/ticket.ts @@ -1,3 +1,5 @@ +import { isSocketHandshake } from '@conduitplatform/hermes'; + export const ADMIN_REALTIME_AUDIENCE = 'admin-realtime'; export const ADMIN_REALTIME_TICKET_TTL_SECONDS = 30; @@ -24,3 +26,10 @@ export function isRealtimeTicket( } return audience === ADMIN_REALTIME_AUDIENCE; } + +export function realtimeTicketForbiddenOnHttp( + decoded: RealtimeTicketClaims | null | undefined, + req: { url?: string; originalUrl?: string }, +): boolean { + return isRealtimeTicket(decoded) && !isSocketHandshake(req); +}