diff --git a/docker/docker-compose.standalone.yml b/docker/docker-compose.standalone.yml index 1c695c107..bd77d5c47 100644 --- a/docker/docker-compose.standalone.yml +++ b/docker/docker-compose.standalone.yml @@ -78,6 +78,8 @@ services: MONGO_INITDB_DATABASE: 'conduit' MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' + # Existing volumes created before rs0 will not elect a replica set. + # Remove the mongo volume (or start with an empty data dir) if hello stays standalone. entrypoint: - bash - -c diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7131604e3..7567df345 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -282,6 +282,8 @@ services: MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' profiles: ['mongodb'] + # Existing volumes created before rs0 will not elect a replica set. + # Remove the mongo volume (or start with an empty data dir) if hello stays standalone. entrypoint: - bash - -c diff --git a/libraries/grpc-sdk/package.json b/libraries/grpc-sdk/package.json index 25809a993..bbf717933 100644 --- a/libraries/grpc-sdk/package.json +++ b/libraries/grpc-sdk/package.json @@ -25,7 +25,8 @@ "prepublish": "npm run build", "prebuild": "npm run protoc", "build": "rimraf dist && tsup", - "protoc": "sh build.sh" + "protoc": "sh build.sh", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utilities/EventBus.test.js" }, "license": "MIT", "dependencies": { diff --git a/libraries/grpc-sdk/src/utilities/EventBus.test.ts b/libraries/grpc-sdk/src/utilities/EventBus.test.ts new file mode 100644 index 000000000..4d069c244 --- /dev/null +++ b/libraries/grpc-sdk/src/utilities/EventBus.test.ts @@ -0,0 +1,124 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { EventBus } from './EventBus.js'; + +type Listener = (channel: string, message: string) => void; + +class FakeRedis { + handlers: Record = {}; + subscribed = new Set(); + failNext = new Set(); + + on(event: string, listener: Listener) { + this.handlers[event] = this.handlers[event] ?? []; + this.handlers[event].push(listener); + } + + subscribe(channel: string, cb?: (err?: Error | null) => void) { + if (this.failNext.has(channel)) { + this.failNext.delete(channel); + cb?.(new Error('subscribe failed')); + return; + } + this.subscribed.add(channel); + cb?.(null); + } + + unsubscribe(channel: string, cb?: () => void) { + void channel; + cb?.(); + } + + publish(channel: string, message: string) { + void channel; + void message; + } + + quit() {} + + emitMessage(channel: string, message: string) { + for (const listener of this.handlers.message ?? []) { + listener(channel, message); + } + } +} + +function createBus() { + const sub = new FakeRedis(); + const pub = new FakeRedis(); + const manager = { + getClient: () => sub, + }; + const bus = new EventBus(manager as never); + (bus as unknown as { _clientSubscriber: FakeRedis })._clientSubscriber = sub; + (bus as unknown as { _clientPublisher: FakeRedis })._clientPublisher = pub; + return { bus, sub }; +} + +describe('EventBus', () => { + it('fires once after deactivate/reactivate on the same channel', () => { + const { bus, sub } = createBus(); + let count = 0; + bus.subscribe( + 'database:update:Order', + () => { + count += 1; + }, + 'relay-a', + ); + bus.unsubscribe('relay-a'); + bus.subscribe( + 'database:update:Order', + () => { + count += 1; + }, + 'relay-a', + ); + sub.emitMessage('database:update:Order', '{"ok":true}'); + assert.equal(count, 1); + }); + + it('keeps the second subscriber when the first is removed', () => { + const { bus, sub } = createBus(); + let first = 0; + let second = 0; + bus.subscribe( + 'chan', + () => { + first += 1; + }, + 'one', + ); + bus.subscribe( + 'chan', + () => { + second += 1; + }, + 'two', + ); + bus.unsubscribe('one'); + sub.emitMessage('chan', 'x'); + assert.equal(first, 0); + assert.equal(second, 1); + }); + + it('subscribeAck rejects when Redis subscribe fails', async () => { + const { bus, sub } = createBus(); + sub.failNext.add('chan'); + await assert.rejects( + () => bus.subscribeAck('chan', () => {}, 'relay-a'), + /subscribe failed/, + ); + sub.failNext.delete('chan'); + let count = 0; + await bus.subscribeAck( + 'chan', + () => { + count += 1; + }, + 'relay-a', + ); + sub.emitMessage('chan', 'x'); + assert.equal(count, 1); + }); +}); diff --git a/libraries/grpc-sdk/src/utilities/EventBus.ts b/libraries/grpc-sdk/src/utilities/EventBus.ts index 8cd8512f1..9388ad483 100644 --- a/libraries/grpc-sdk/src/utilities/EventBus.ts +++ b/libraries/grpc-sdk/src/utilities/EventBus.ts @@ -3,35 +3,58 @@ import { Cluster, Redis } from 'ioredis'; import crypto from 'crypto'; import { getLogger } from './GrpcSdkContext.js'; +type ChannelCallbacks = Map void>; + export class EventBus { private _clientSubscriber: Redis | Cluster; private _clientPublisher: Redis | Cluster; - private _subscribedChannels: { [listener: string]: ((message: string) => void)[] }; - private _subscribers: { [listener: string]: [string, number] }; + /** channelName -> subscriberId -> callback */ + private _channelCallbacks: Map; + /** subscriberId -> channelName */ + private _subscriberChannels: Map; + /** channels with a successful Redis SUBSCRIBE */ + private _redisSubscribedChannels: Set; + private _subscribeInFlight = new Map>(); private _signature: string; + private _anonymousSubscriberSeq = 0; + private _shuttingDown = false; constructor(redisManager: RedisManager) { - this._subscribedChannels = {}; - this._subscribers = {}; + this._channelCallbacks = new Map(); + this._subscriberChannels = new Map(); + this._redisSubscribedChannels = new Set(); this._clientSubscriber = redisManager.getClient({ keyPrefix: 'bus_' }); this._clientPublisher = redisManager.getClient({ keyPrefix: 'bus_' }); this._signature = crypto.randomBytes(20).toString('hex'); this._clientSubscriber.on('ready', () => { getLogger().log('The Bus is in the station...hehe'); }); + this._clientSubscriber.on('message', (channel: string, message: string) => { + this.dispatch(channel, message); + }); process.on('exit', () => { - this._clientSubscriber.quit(); - this._clientPublisher.quit(); + this.quit(); }); } + quit(): void { + if (this._shuttingDown) return; + this._shuttingDown = true; + void this._clientSubscriber.quit(); + void this._clientPublisher.quit(); + } + unsubscribe(subscriberId: string): void { - if (this._subscribers[subscriberId]) { - const [channelName, index] = this._subscribers[subscriberId]; - this._subscribedChannels[channelName].splice(index, 1); - delete this._subscribers[subscriberId]; - if (this._subscribedChannels[channelName].length === 0) { - delete this._subscribedChannels[channelName]; + const channelName = this._subscriberChannels.get(subscriberId); + if (!channelName) { + return; + } + const callbacks = this._channelCallbacks.get(channelName); + callbacks?.delete(subscriberId); + this._subscriberChannels.delete(subscriberId); + if (callbacks && callbacks.size === 0) { + this._channelCallbacks.delete(channelName); + if (this._redisSubscribedChannels.delete(channelName)) { this._clientSubscriber.unsubscribe(channelName, () => {}); } } @@ -42,43 +65,100 @@ export class EventBus { callback: (message: string) => void, subscriberId?: string, ): void { + void this.subscribeAck(channelName, callback, subscriberId).catch(err => { + getLogger().error( + `EventBus subscribe failed for ${channelName}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }); + } + + async subscribeAck( + channelName: string, + callback: (message: string) => void, + subscriberId?: string, + ): Promise { + if (this._shuttingDown) { + return; + } + const id = + subscriberId ?? + `anon:${channelName}:${++this._anonymousSubscriberSeq}:${crypto.randomBytes(4).toString('hex')}`; if (subscriberId) { - // if subscriberId is provided, and it is already subscribed, unsubscribe it first this.unsubscribe(subscriberId); } - if (this._subscribedChannels[channelName]) { - this._subscribedChannels[channelName].push(callback); - if (subscriberId) { - this._subscribers[subscriberId] = [ - channelName, - this._subscribedChannels[channelName].length - 1, - ]; - } + + let callbacks = this._channelCallbacks.get(channelName); + if (!callbacks) { + callbacks = new Map(); + this._channelCallbacks.set(channelName, callbacks); + } + callbacks.set(id, callback); + this._subscriberChannels.set(id, channelName); + + if (this._redisSubscribedChannels.has(channelName)) { return; } - this._subscribedChannels[channelName] = [callback]; - this._clientSubscriber.subscribe(channelName, () => {}); - const self = this; - this._clientSubscriber.on('message', (channel: string, message: string) => { - if (channel !== channelName) return; - // if the message supports the signature - if (message.indexOf('CND_Signature') !== -1) { - // if the message does not contain this module's signature - if (message.indexOf(self._signature) === -1) { - self._subscribedChannels[channelName].forEach(fn => { - fn(message.split('CND_Signature:')[0]); - }); - } - } else { - self._subscribedChannels[channelName].forEach(fn => { - fn(message); + + let inFlight = this._subscribeInFlight.get(channelName); + if (!inFlight) { + inFlight = new Promise((resolve, reject) => { + this._clientSubscriber.subscribe(channelName, err => { + if (err) { + reject(err); + return; + } + this._redisSubscribedChannels.add(channelName); + resolve(); }); + }).finally(() => { + this._subscribeInFlight.delete(channelName); + }); + this._subscribeInFlight.set(channelName, inFlight); + } + + try { + await inFlight; + } catch (err) { + if (!this._redisSubscribedChannels.has(channelName)) { + this.removeChannelCallbacks(channelName); } - }); + this.unsubscribe(id); + throw err; + } } publish(channelName: string, message: string) { message = message + `CND_Signature:${this._signature}`; this._clientPublisher.publish(channelName, message); } + + private removeChannelCallbacks(channelName: string): void { + const callbacks = this._channelCallbacks.get(channelName); + if (!callbacks) { + return; + } + for (const subId of callbacks.keys()) { + this._subscriberChannels.delete(subId); + } + this._channelCallbacks.delete(channelName); + } + + private dispatch(channel: string, message: string): void { + const callbacks = this._channelCallbacks.get(channel); + if (!callbacks || callbacks.size === 0) { + return; + } + let payload = message; + if (message.indexOf('CND_Signature') !== -1) { + if (message.indexOf(this._signature) !== -1) { + return; + } + payload = message.split('CND_Signature:')[0]; + } + for (const fn of callbacks.values()) { + fn(payload); + } + } } diff --git a/libraries/grpc-sdk/tsconfig.test.json b/libraries/grpc-sdk/tsconfig.test.json new file mode 100644 index 000000000..121161cd7 --- /dev/null +++ b/libraries/grpc-sdk/tsconfig.test.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": ["src/utilities/EventBus.ts", "src/utilities/EventBus.test.ts"] +} diff --git a/libraries/hermes/package.json b/libraries/hermes/package.json index bfe74ff95..3beffb157 100644 --- a/libraries/hermes/package.json +++ b/libraries/hermes/package.json @@ -11,7 +11,7 @@ "scripts": { "prepublish": "npm run build", "build": "rimraf dist && tsc", - "test": "tsc -p tsconfig.test.json && node --test dist-test/Socket/applySocketGlobalMiddlewares.test.js", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/Socket/*.test.js", "publish": "npm publish", "postbuild": "copyfiles -u 1 src/*.proto src/**/*.json ./dist/" }, diff --git a/libraries/hermes/src/Socket/Socket.ts b/libraries/hermes/src/Socket/Socket.ts index 937ead5b9..f1c0b47d1 100644 --- a/libraries/hermes/src/Socket/Socket.ts +++ b/libraries/hermes/src/Socket/Socket.ts @@ -15,8 +15,12 @@ import { } from '../interfaces/index.js'; import ObjectHash from 'object-hash'; import { ConduitError, ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; -import { applySocketGlobalMiddlewares } from './applySocketGlobalMiddlewares.js'; import { buildSocketMiddlewareParams } from './buildSocketMiddlewareParams.js'; +import { resolveEngineNamespacePath } from './resolveEngineNamespacePath.js'; +import { + filterRemoteSocketsByUserAndRooms, + isEngineSocketBackpressured, +} from './socketPushUtils.js'; export class SocketController extends ConduitRouter { private readonly httpServer: httpServer; @@ -87,6 +91,22 @@ export class SocketController extends ConduitRouter { `Socket connection error, context: ${err?.context ?? 'N/A'}`, ); }); + + this.io.engine.use((req: any, res: any, next: NextFunction) => { + req.path = resolveEngineNamespacePath(req); + let index = 0; + const run: NextFunction = err => { + if (err) { + return next(err); + } + const middleware = this.globalMiddlewares[index++]; + if (!middleware) { + return next(); + } + middleware(req, res, run); + }; + run(); + }); } registerGlobalMiddleware( @@ -117,17 +137,13 @@ export class SocketController extends ConduitRouter { const self = this; this.io.of(namespace).use((socket, next) => { - applySocketGlobalMiddlewares(socket, self.globalMiddlewares) - .then(() => { - const context = buildSocketMiddlewareParams(socket); - Object.assign(context.context, socket.data); - return self - .checkMiddlewares(context, conduitSocket.input.middlewares) - .then(r => { - Object.assign(context.context, r); - socket.data = context.context; - next(); - }); + const context = buildSocketMiddlewareParams(socket); + self + .checkMiddlewares(context, conduitSocket.input.middlewares) + .then(r => { + Object.assign(context.context, r); + socket.data = context.context; + next(); }) .catch((err: Error | ConduitError) => { next(err); @@ -136,13 +152,25 @@ export class SocketController extends ConduitRouter { this.io.of(namespace).on('connect', socket => { if (socket.recovered) { - ConduitGrpcSdk.Logger.info( - `Socket recovered: ${socket.id} to namespace: ${namespace}`, + const recoveredRooms = [...socket.rooms].filter( + room => room.startsWith('er:') || room.startsWith('database:'), ); + const recovered = conduitSocket.executeRecovered({ + event: 'recovered', + socketId: socket.id, + context: socket.data, + params: recoveredRooms, + recoveredRooms, + }); + if (recovered) { + recovered + .then(res => this.handleResponse(res, socket, namespace)) + .catch(e => { + ConduitGrpcSdk.Logger.error(e); + socket.emit('conduit_error', e); + }); + } } else { - ConduitGrpcSdk.Logger.info( - `Socket connected: ${socket.id} to namespace: ${namespace}`, - ); conduitSocket .executeRequest({ event: 'connect', @@ -157,7 +185,6 @@ export class SocketController extends ConduitRouter { } socket.onAny((event, ...args) => { - ConduitGrpcSdk.Logger.info(`Socket event: ${event} from socket: ${socket.id}`); conduitSocket .executeRequest({ event, @@ -172,15 +199,13 @@ export class SocketController extends ConduitRouter { }); }); - socket.on('disconnect', () => { - ConduitGrpcSdk.Logger.info( - `Socket disconnected: ${socket.id} from namespace: ${namespace}`, - ); + socket.on('disconnect', (reason: string) => { conduitSocket .executeRequest({ event: 'disconnect', socketId: socket.id, context: socket.data, + params: [reason], }) .then(res => this.handleResponse(res, socket, namespace)) .catch(e => { @@ -191,82 +216,162 @@ export class SocketController extends ConduitRouter { }); } - async handleSocketPush(push: SocketPush) { + async handleSocketPush(push: SocketPush): Promise { const localOnly = push.localOnly === true; if (push.event === 'join-room') { - if (push.rooms.length === 0) return; + if (push.rooms.length === 0) return false; const filteredSockets = await this.findAndFilterSockets( push.receivers, push.namespace, localOnly, ); for (const socket of filteredSockets) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} joining rooms: ${push.rooms.join(', ')} in namespace: ${ - push.namespace - }`, - ); socket.join(push.rooms); } + return true; } else if (push.event === 'leave-room') { if (push.rooms && push.rooms.length !== 0) { - const filteredSockets = await this.findAndFilterSockets( - push.receivers, - push.namespace, - localOnly, - ); + const filteredSockets = await this.socketsForRoomPush(push, localOnly); for (const socket of filteredSockets) { for (const room of push.rooms) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} leaving room: ${room} in namespace: ${push.namespace}`, - ); socket.leave(room); } } } + return true; } else if (isInstanceOfEventResponse(push)) { if ( (isNil(push.receivers) || push.receivers.length === 0) && push.rooms.length === 0 ) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${push.event} to all sockets in namespace: ${push.namespace}`, - ); + return false; + } + if (push.receivers.length !== 0) { const nsp = this.io.of(push.namespace); - if (localOnly) { - nsp.local.emit(push.event, push.data); - } else { - nsp.emit(push.event, push.data); + const filteredSockets = await this.findAndFilterSockets( + push.receivers, + push.namespace, + localOnly, + push.rooms, + ); + if (push.skipEmptyRooms && filteredSockets.length === 0) { + return false; } - } else { - if (push.rooms.length !== 0) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${push.event} to rooms: ${push.rooms.join( - ', ', - )} in namespace: ${push.namespace}`, - ); - const target = this.io.of(push.namespace).to(push.rooms); - if (localOnly) { - target.local.emit(push.event, push.data); - } else { - target.emit(push.event, push.data); + for (const remote of filteredSockets) { + const local = localOnly ? nsp.sockets.get(remote.id) : undefined; + if ( + push.boundedEmit && + localOnly && + local && + this.isLocalSocketBackpressured(local) + ) { + ConduitGrpcSdk.Metrics?.increment('event_relays_emit_dropped_total'); + local.disconnect(true); + continue; } + remote.emit(push.event, push.data); } - if (push.receivers.length !== 0) { - const filteredSockets = await this.findAndFilterSockets( - push.receivers, - push.namespace, - localOnly, - ); - for (const socket of filteredSockets) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${push.event} to socket: ${socket.id} in namespace: ${push.namespace}`, - ); - socket.emit(push.event, push.data); + return true; + } + if (push.rooms.length !== 0) { + const emitted = await this.emitEventToRooms(push, localOnly); + if (!emitted) { + return false; + } + } + return true; + } + return false; + } + + async getLocalRoomUserIds(namespace: string, room: string): Promise { + const sockets = await this.io.of(namespace).in(room).local.fetchSockets(); + const userIds = new Set(); + for (const socket of sockets) { + const userId = socket.data?.user?._id; + if (typeof userId === 'string' && userId.length > 0) { + userIds.add(userId); + } + } + return [...userIds]; + } + + async getLocalRoomsWithPrefix(namespace: string, prefix: string): Promise { + const adapter = this.io.of(namespace).adapter as { rooms?: Map }; + const rooms = adapter.rooms; + if (!rooms) { + return []; + } + return [...rooms.keys()].filter(room => room.startsWith(prefix)); + } + + private async emitEventToRooms(push: SocketPush, localOnly: boolean): Promise { + const nsp = this.io.of(push.namespace); + const localSockets = localOnly ? this.localSocketsInRooms(nsp, push.rooms) : []; + if (push.skipEmptyRooms || push.boundedEmit) { + if (localOnly && localSockets.length === 0) { + return false; + } + if (push.boundedEmit && localOnly) { + for (const socket of localSockets) { + if (this.isLocalSocketBackpressured(socket)) { + ConduitGrpcSdk.Metrics?.increment('event_relays_emit_dropped_total'); + socket.disconnect(true); } } } } + + const target = nsp.to(push.rooms); + if (localOnly) { + target.local.emit(push.event, push.data); + } else { + target.emit(push.event, push.data); + } + return true; + } + + private localSocketsInRooms( + nsp: ReturnType, + rooms: string[], + ): Socket[] { + const sockets: Socket[] = []; + for (const socket of nsp.sockets.values()) { + if (rooms.some(room => socket.rooms.has(room))) { + sockets.push(socket); + } + } + return sockets; + } + + private isLocalSocketBackpressured(socket: Socket): boolean { + return isEngineSocketBackpressured( + socket.conn as unknown as { writeBuffer?: unknown[] }, + ); + } + + private async socketsForRoomPush( + push: SocketPush, + localOnly: boolean, + ): Promise[]> { + if (push.receivers.length > 0) { + return this.findAndFilterSockets(push.receivers, push.namespace, localOnly); + } + const nsp = this.io.of(push.namespace); + const seen = new Set(); + const sockets: RemoteSocket[] = []; + for (const room of push.rooms) { + const inRoom = localOnly + ? await nsp.in(room).local.fetchSockets() + : await nsp.in(room).fetchSockets(); + for (const socket of inRoom) { + if (!seen.has(socket.id)) { + seen.add(socket.id); + sockets.push(socket); + } + } + } + return sockets; } private async handleResponse( @@ -276,19 +381,11 @@ export class SocketController extends ConduitRouter { ) { if (res.event === 'join-room') { if (res.rooms && res.rooms.length !== 0) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} joining rooms: ${res.rooms.join( - ', ', - )} in namespace: ${namespace}`, - ); socket.join(res.rooms); } } else if (res.event === 'leave-room') { if (res.rooms && res.rooms.length !== 0) { for (const room of res.rooms) { - ConduitGrpcSdk.Logger.info( - `Socket ${socket.id} leaving room: ${room} in namespace: ${namespace}`, - ); socket.leave(room); } } @@ -297,17 +394,9 @@ export class SocketController extends ConduitRouter { (!res.receivers || res.receivers.length === 0) && (!res.rooms || res.rooms.length === 0) ) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${res.event} to all sockets in namespace: ${namespace}`, - ); socket.emit(res.event, JSON.parse(res.data)); } else { if (res.rooms && res.rooms.length !== 0) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${res.event} to rooms: ${res.rooms.join( - ', ', - )} in namespace: ${namespace}`, - ); this.io.of(namespace).to(res.rooms).emit(res.event, JSON.parse(res.data)); } if (res.receivers && res.receivers.length !== 0) { @@ -316,9 +405,6 @@ export class SocketController extends ConduitRouter { namespace, ); for (const socket of filteredSockets) { - ConduitGrpcSdk.Logger.info( - `Emitting event: ${res.event} to socket: ${socket.id} in namespace: ${namespace}`, - ); socket.emit(res.event, JSON.parse(res.data)); } } @@ -330,15 +416,19 @@ export class SocketController extends ConduitRouter { userIds: string[], namespace: string, localOnly: boolean = false, + rooms: string[] = [], ): Promise[]> { const nsp = this.io.of(namespace); const sockets = localOnly ? await nsp.local.fetchSockets() : await nsp.fetchSockets(); - const userIdSet = new Set(userIds); - return sockets.filter(socket => { - if (socket.data && socket.data.user) { - return userIdSet.has(socket.data.user._id); - } - }); + return filterRemoteSocketsByUserAndRooms( + sockets.map(socket => ({ + id: socket.id, + data: socket.data, + rooms: socket.rooms, + })), + userIds, + rooms, + ).map(filtered => sockets.find(s => s.id === filtered.id)!); } protected _refreshRouter(): void { diff --git a/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts b/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts deleted file mode 100644 index 3e2834c0e..000000000 --- a/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; -import type { NextFunction, Request, Response } from 'express'; -import { applySocketGlobalMiddlewares } from './applySocketGlobalMiddlewares.js'; - -function fakeSocket(auth?: Record) { - return { - request: { - headers: {}, - url: '/realtime/?EIO=4&transport=polling', - }, - handshake: { auth: auth ?? {} }, - data: {}, - }; -} - -describe('applySocketGlobalMiddlewares', () => { - it('injects handshake.auth.token and copies conduit onto socket.data', async () => { - const socket = fakeSocket({ token: 'ticket-token' }); - await applySocketGlobalMiddlewares(socket as never, [ - (req: Request, _res: Response, next: NextFunction) => { - assert.equal(req.headers.authorization, 'Bearer ticket-token'); - (req as Request & { conduit: { admin: string } }).conduit = { - admin: 'ok', - }; - next(); - }, - ]); - assert.equal((socket.data as { admin?: string }).admin, 'ok'); - }); - - it('rejects when Express middleware sends 401 json', async () => { - const socket = fakeSocket(); - await assert.rejects( - () => - applySocketGlobalMiddlewares(socket as never, [ - (_req: Request, res: Response) => { - res.status(401).json({ error: 'No token provided' }); - }, - ]), - /No token provided/, - ); - }); -}); diff --git a/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts b/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts deleted file mode 100644 index c3f5bb8a8..000000000 --- a/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { NextFunction, Request, Response } from 'express'; -import type { Socket } from 'socket.io'; -import { buildSocketMiddlewareParams } from './buildSocketMiddlewareParams.js'; - -type ExpressMiddleware = (req: Request, res: Response, next: NextFunction) => void; - -type ConduitRequest = Request & { conduit?: Record }; - -export async function applySocketGlobalMiddlewares( - socket: Socket, - middlewares: ExpressMiddleware[], -): Promise { - const params = buildSocketMiddlewareParams(socket); - const req = Object.assign(socket.request, { - headers: params.headers, - path: '/realtime', - url: socket.request.url ?? '/realtime', - originalUrl: socket.request.url ?? '/realtime', - conduit: { ...params.context }, - }) as ConduitRequest; - - await new Promise((resolve, reject) => { - let settled = false; - const res = { - statusCode: 200, - status(this: { statusCode: number }, code: number) { - this.statusCode = code; - return this; - }, - json(this: unknown, body: { error?: string }) { - if (settled) return this; - settled = true; - reject(new Error(body?.error || 'Unauthorized')); - return this; - }, - setHeader() { - return this; - }, - removeHeader() {}, - } as unknown as Response; - - let index = 0; - const next = (err?: unknown) => { - if (settled) return; - if (err) { - settled = true; - reject(err instanceof Error ? err : new Error(String(err))); - return; - } - const middleware = middlewares[index++]; - if (!middleware) { - settled = true; - socket.data = { ...socket.data, ...(req.conduit ?? {}) }; - resolve(); - return; - } - try { - middleware(req, res, next); - } catch (error) { - settled = true; - reject(error as Error); - } - }; - next(); - }); -} diff --git a/libraries/hermes/src/Socket/isSocketHandshake.test.ts b/libraries/hermes/src/Socket/isSocketHandshake.test.ts new file mode 100644 index 000000000..a686bdb76 --- /dev/null +++ b/libraries/hermes/src/Socket/isSocketHandshake.test.ts @@ -0,0 +1,27 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { isSocketHandshake } from './isSocketHandshake.js'; + +describe('isSocketHandshake', () => { + it('matches Engine.IO handshake query params only', () => { + assert.equal( + isSocketHandshake({ url: '/events/?EIO=4&transport=polling&t=abc' }), + true, + ); + assert.equal(isSocketHandshake({ url: '/realtime/?EIO=4&transport=websocket' }), true); + }); + + it('rejects HTTP routes that merely mention realtime or established sessions', () => { + assert.equal( + isSocketHandshake({ url: '/realtime/ticket?EIO=4&transport=polling' }), + false, + ); + assert.equal(isSocketHandshake({ url: '/realtime/ticket' }), false); + assert.equal(isSocketHandshake({ originalUrl: '/realtime' }), false); + assert.equal(isSocketHandshake({ url: '/graphql?EIO=4&transport=polling' }), false); + assert.equal( + isSocketHandshake({ url: '/events/?EIO=4&transport=polling&sid=abc' }), + false, + ); + }); +}); diff --git a/libraries/hermes/src/Socket/isSocketHandshake.ts b/libraries/hermes/src/Socket/isSocketHandshake.ts new file mode 100644 index 000000000..aa216d2ac --- /dev/null +++ b/libraries/hermes/src/Socket/isSocketHandshake.ts @@ -0,0 +1,26 @@ +export function isSocketHandshake(req: { url?: string; originalUrl?: string }): boolean { + const raw = req.url ?? req.originalUrl ?? ''; + if (!raw) { + return false; + } + const queryIndex = raw.indexOf('?'); + const pathname = queryIndex === -1 ? raw : raw.slice(0, queryIndex); + if (pathname.includes('ticket')) { + return false; + } + const query = queryIndex === -1 ? '' : raw.slice(queryIndex + 1); + if (!query) { + return false; + } + const params = new URLSearchParams(query); + if (!params.has('EIO') || !params.has('transport')) { + return false; + } + if (params.has('sid')) { + return false; + } + if (pathname !== '/' && !pathname.endsWith('/')) { + return false; + } + return true; +} diff --git a/libraries/hermes/src/Socket/resolveEngineNamespacePath.ts b/libraries/hermes/src/Socket/resolveEngineNamespacePath.ts new file mode 100644 index 000000000..e36aad12b --- /dev/null +++ b/libraries/hermes/src/Socket/resolveEngineNamespacePath.ts @@ -0,0 +1,11 @@ +/** + * Derive the Socket.IO namespace path from an Engine.IO handshake request URL. + */ +export function resolveEngineNamespacePath(req: { url?: string }): string { + const url = req.url ?? '/'; + const pathname = url.split('?')[0] ?? '/'; + if (pathname === '/' || pathname === '') { + return '/'; + } + return pathname.endsWith('/') ? pathname : `${pathname}/`; +} diff --git a/libraries/hermes/src/Socket/socketPushUtils.test.ts b/libraries/hermes/src/Socket/socketPushUtils.test.ts new file mode 100644 index 000000000..7f76e2904 --- /dev/null +++ b/libraries/hermes/src/Socket/socketPushUtils.test.ts @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { + filterRemoteSocketsByUserAndRooms, + isEngineSocketBackpressured, + WRITE_BUFFER_PACKET_HIGH_WATER, +} from './socketPushUtils.js'; + +describe('filterRemoteSocketsByUserAndRooms', () => { + const roomA = 'er:relay-1:aaa'; + const roomB = 'er:relay-1:bbb'; + + it('emits only to sockets in the target room for the same user', () => { + const sockets = [ + { + id: 'tab-a', + data: { user: { _id: 'user-1' } }, + rooms: new Set([roomA]), + }, + { + id: 'tab-b', + data: { user: { _id: 'user-1' } }, + rooms: new Set([roomB]), + }, + ]; + + const filtered = filterRemoteSocketsByUserAndRooms(sockets, ['user-1'], [roomA]); + assert.deepEqual( + filtered.map(s => s.id), + ['tab-a'], + ); + }); +}); + +describe('isEngineSocketBackpressured', () => { + it('uses writeBuffer queue depth only', () => { + assert.equal(isEngineSocketBackpressured(undefined), false); + assert.equal( + isEngineSocketBackpressured({ writeBuffer: new Array(WRITE_BUFFER_PACKET_HIGH_WATER) }), + false, + ); + assert.equal( + isEngineSocketBackpressured({ + writeBuffer: new Array(WRITE_BUFFER_PACKET_HIGH_WATER + 1), + }), + true, + ); + assert.equal( + isEngineSocketBackpressured({ + writeBuffer: [], + }), + false, + ); + }); +}); + +describe('SocketController relay emit', () => { + it('does not reference writableLength or client emit ack callbacks', () => { + const socketSource = readFileSync( + path.join(path.dirname(fileURLToPath(import.meta.url)), '../../src/Socket/Socket.ts'), + 'utf8', + ); + assert.doesNotMatch(socketSource, /writableLength/); + assert.doesNotMatch(socketSource, /\.emit\([^)]*,\s*[^,)]+,\s*\(\)\s*=>/); + }); +}); diff --git a/libraries/hermes/src/Socket/socketPushUtils.ts b/libraries/hermes/src/Socket/socketPushUtils.ts new file mode 100644 index 000000000..6b3e14feb --- /dev/null +++ b/libraries/hermes/src/Socket/socketPushUtils.ts @@ -0,0 +1,36 @@ +export const WRITE_BUFFER_PACKET_HIGH_WATER = 64; + +type SocketLike = { + id: string; + data?: { user?: { _id?: string } }; + rooms: Set; +}; + +export function filterRemoteSocketsByUserAndRooms( + sockets: SocketLike[], + userIds: string[], + rooms: string[], +): SocketLike[] { + const userIdSet = new Set(userIds); + const roomSet = rooms.length > 0 ? new Set(rooms) : null; + return sockets.filter(socket => { + if (!socket.data?.user?._id) { + return false; + } + if (!userIdSet.has(socket.data.user._id)) { + return false; + } + if (roomSet) { + return [...roomSet].some(room => socket.rooms.has(room)); + } + return true; + }); +} + +export function isEngineSocketBackpressured( + conn: { writeBuffer?: unknown[] } | undefined, + highWater = WRITE_BUFFER_PACKET_HIGH_WATER, +): boolean { + const pending = conn?.writeBuffer?.length ?? 0; + return pending > highWater; +} diff --git a/libraries/hermes/src/index.ts b/libraries/hermes/src/index.ts index ac22e28ba..46808add1 100644 --- a/libraries/hermes/src/index.ts +++ b/libraries/hermes/src/index.ts @@ -254,6 +254,12 @@ export class ConduitRoutingController { } } + registerSocketGlobalMiddleware( + middleware: (req: ConduitRequest, res: Response, next: NextFunction) => void, + ) { + this._socketRouter?.registerGlobalMiddleware(middleware); + } + registerRouteMiddleware(middleware: ConduitMiddleware, moduleUrl: string) { this._restRouter?.registerMiddleware(middleware, moduleUrl); this._graphQLRouter?.registerMiddleware(middleware, moduleUrl); @@ -343,7 +349,20 @@ export class ConduitRoutingController { } async socketPush(data: SocketPush) { - await this._socketRouter?.handleSocketPush(data); + return (await this._socketRouter?.handleSocketPush(data)) ?? false; + } + + getLocalRoomUserIds(namespace: string, room: string): Promise { + return ( + this._socketRouter?.getLocalRoomUserIds(namespace, room) ?? Promise.resolve([]) + ); + } + + getLocalRoomsWithPrefix(namespace: string, prefix: string): Promise { + return ( + this._socketRouter?.getLocalRoomsWithPrefix(namespace, prefix) ?? + Promise.resolve([]) + ); } /** True if any enabled transport would change this route (new or definition changed). */ @@ -472,6 +491,8 @@ export class ConduitRoutingController { } } +export { isSocketHandshake } from './Socket/isSocketHandshake.js'; +export { resolveEngineNamespacePath } from './Socket/resolveEngineNamespacePath.js'; export * from './interfaces/index.js'; export * from './types/index.js'; export * from './classes/index.js'; diff --git a/libraries/hermes/src/interfaces/Socket.ts b/libraries/hermes/src/interfaces/Socket.ts index 204707624..2b9787257 100644 --- a/libraries/hermes/src/interfaces/Socket.ts +++ b/libraries/hermes/src/interfaces/Socket.ts @@ -10,6 +10,7 @@ export interface ConduitSocketParameters { socketId: string; params?: UntypedArray; context?: Indexable; + recoveredRooms?: string[]; } export type ConduitSocketParamTypes = (TYPE | ConduitSocketParamTypes)[]; @@ -19,6 +20,7 @@ export interface ConduitSocketOptions { name?: string; description?: string; middlewares?: string[]; + onRecovered?: ConduitSocketEventHandler; } export type EventResponse = { @@ -85,6 +87,18 @@ export class ConduitSocket { } return Promise.reject('no such event registered'); } + + executeRecovered( + request: ConduitSocketParameters, + ): ConduitSocketHandlerResponse | null { + if (this._input.onRecovered) { + return this._input.onRecovered(request); + } + if (this._events.has('recovered')) { + return this._events.get('recovered')!.handler(request); + } + return null; + } } export interface EventsProtoDescription { diff --git a/libraries/hermes/src/interfaces/SocketPush.ts b/libraries/hermes/src/interfaces/SocketPush.ts index 8bbabbc85..7d0f572d3 100644 --- a/libraries/hermes/src/interfaces/SocketPush.ts +++ b/libraries/hermes/src/interfaces/SocketPush.ts @@ -5,4 +5,8 @@ export interface SocketPush { rooms: string[]; namespace: string; localOnly?: boolean; + /** Skip emit when the local room has no connected sockets. */ + skipEmptyRooms?: boolean; + /** Drop or disconnect slow clients instead of blocking the caller. */ + boundedEmit?: boolean; } diff --git a/libraries/hermes/tsconfig.json b/libraries/hermes/tsconfig.json index 0420a284b..225b927d7 100644 --- a/libraries/hermes/tsconfig.json +++ b/libraries/hermes/tsconfig.json @@ -66,5 +66,5 @@ /* Advanced Options */ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ }, - "exclude": ["node_modules", "dist", "**/*.test.ts"] + "exclude": ["src/**/*.test.ts"] } diff --git a/libraries/hermes/tsconfig.test.json b/libraries/hermes/tsconfig.test.json index 8c2d1a100..8aecbdee9 100644 --- a/libraries/hermes/tsconfig.test.json +++ b/libraries/hermes/tsconfig.test.json @@ -8,9 +8,10 @@ "types": ["node"] }, "include": [ - "src/Socket/applySocketGlobalMiddlewares.ts", - "src/Socket/applySocketGlobalMiddlewares.test.ts", - "src/Socket/buildSocketMiddlewareParams.ts" + "src/Socket/isSocketHandshake.ts", + "src/Socket/isSocketHandshake.test.ts", + "src/Socket/socketPushUtils.ts", + "src/Socket/socketPushUtils.test.ts" ], - "exclude": ["node_modules", "dist"] + "exclude": [] } diff --git a/modules/database/README.mdx b/modules/database/README.mdx index 61bc195b0..b84cded13 100644 --- a/modules/database/README.mdx +++ b/modules/database/README.mdx @@ -47,7 +47,9 @@ since the latter need to go through parsers that are otherwise unnecessary for M When using MongoDB with a replica set (e.g., MongoDB Atlas), the database module supports configuring read preference, write concern, and read concern through the admin panel at `PATCH /config/database`. -Live document updates also require a replica set or sharded cluster. Local Compose files initialize a single-node replica set so change streams can be exercised. +Live document updates require a replica set or sharded cluster (Atlas is fine). Helm’s bundled Mongo chart is a **standalone** Deployment (`replicas: 1`, no `--replSet`), so live updates stay `idle` there until you point `DB_CONN_URI` at Atlas or an operator-managed replica set. + +Local Compose files initialize a single-node `rs0`. **Existing Compose Mongo volumes will not become a replica set cleanly** — drop the volume or start from an empty data dir if `hello` still reports standalone. ### Live updates @@ -58,9 +60,11 @@ subscribe({ schema: 'Order', documentId?: string }) unsubscribe({ schema: 'Order', documentId?: string }) ``` -Events arrive as `change` with `{ version, operation, schema, documentId, occurredAt, resumeToken }` and contain no document fields. Consumers should refetch through their authorized REST or custom-endpoint path. +Events arrive as `change` with `{ version, operation, schema, documentId, occurredAt }` and contain no document fields and no resume token. This is **live-tail, not backfill**. After a full reconnect, subscribe again and refetch over authorized REST. + +The leader also publishes `database:change:${schema}` on the Redis bus. **Do not also relay `database:change:*` on `/events/` if the same client is on `/database/`** — that duplicates notifications. Keep the bus for other modules; just do not dual-subscribe. -Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; session JWTs and masterkeys must not be sent from browser code. +Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Client sockets also require CMS `crudOperations.read.enabled`. Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; that token cannot mint another ticket or call REST/GraphQL. Session JWTs and masterkeys must not be sent from browser code. Admin sockets must be enabled (`admin.transports.sockets`) and the Admin socket port (`ADMIN_SOCKET_PORT`, default 3031) reachable from the UI. diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index e0208f256..e2d36e26f 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -1053,4 +1053,8 @@ export default class DatabaseModule extends ManagedModule { ); } } + + async shutdown(): Promise { + await this.realtimeService?.shutdown(); + } } diff --git a/modules/database/src/index.ts b/modules/database/src/index.ts index 3e9466196..12261992a 100644 --- a/modules/database/src/index.ts +++ b/modules/database/src/index.ts @@ -1,5 +1,6 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import DatabaseModule from './Database.js'; const dbType = process.env.DB_TYPE ?? 'mongodb'; @@ -7,4 +8,21 @@ const dbUri = process.env.DB_CONN_URI ?? 'mongodb://localhost:27017'; const peerManifestRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const database = new DatabaseModule(dbType, dbUri, peerManifestRoot); + +function registerShutdownSignals(): void { + const shutdown = (signal: NodeJS.Signals) => { + void database + .shutdown() + .catch(err => { + ConduitGrpcSdk.Logger.error(err as Error); + }) + .finally(() => { + process.exit(signal === 'SIGINT' ? 130 : 0); + }); + }; + process.once('SIGTERM', () => shutdown('SIGTERM')); + process.once('SIGINT', () => shutdown('SIGINT')); +} + +registerShutdownSignals(); database.start(); diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts index b41cc2f89..d29938d15 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -2,6 +2,7 @@ import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import { normalizeChangeEvent, parseResumeToken, + serializeResumeToken, type RawChangeEvent, } from './normalize.js'; import { authorizedDocumentRoom, roomsForPublicChange } from './rooms.js'; @@ -17,7 +18,14 @@ import type { RealtimeStatusCode, } from './types.js'; import type { RealtimeSubscriptionTracker } from './subscriptions.js'; -import { canReadDocument, type AuthorizationSdk } from './authorize.js'; +import { type AuthorizationSdk } from './authorize.js'; +import { checkRebacBatch, RealtimeRebacCache } from './rebacCache.js'; +import { + buildWatchPipeline, + optedInCollectionsKey, + WATCH_RESTART_OPERATIONS, + type WatchPipeline, +} from './watchPipeline.js'; const LEADER_LOCK = 'realtime:change-stream:leader'; const RESUME_TOKEN_KEY = 'realtime:resumeToken'; @@ -30,7 +38,10 @@ type LeaderLock = NonNullable< Awaited['tryAcquireLock']>> >; -export type WatchFactory = (options: { resumeAfter?: unknown }) => ChangeStreamLike; +export type WatchFactory = (options: { + resumeAfter?: unknown; + pipeline: WatchPipeline; +}) => ChangeStreamLike; export type CoordinatorOptions = { grpcSdk: ConduitGrpcSdk; @@ -56,6 +67,9 @@ export class MongoChangeStreamCoordinator { private watching = false; private opening = false; private ignoreClose = false; + private changeQueue: Promise = Promise.resolve(); + private watchedCollectionsKey = ''; + private readonly rebacCache = new RealtimeRebacCache(); constructor(private readonly options: CoordinatorOptions) {} @@ -75,6 +89,10 @@ export class MongoChangeStreamCoordinator { return this.topology; } + async waitForIdle(): Promise { + await this.changeQueue; + } + async reconcile(): Promise { if (this.closed) return; const engine = this.options.engine(); @@ -90,8 +108,6 @@ export class MongoChangeStreamCoordinator { await this.releaseLeader(); this.streamState = 'idle'; this.lastError = this.topology.message; - // Hello can fail during startup before Mongo is ready. Keep retrying - // that case; a confirmed standalone topology will not recover. if ( !this.topology.message || this.topology.message.includes('Unable to determine') @@ -100,20 +116,31 @@ export class MongoChangeStreamCoordinator { } return; } - if (this.options.getOptedInSchemas().length === 0) { + const collections = this.collectionNames(); + if (collections.length === 0) { await this.stopStream('idle'); await this.releaseLeader(); this.streamState = 'idle'; return; } + const nextKey = optedInCollectionsKey(collections); + if (this.watching && nextKey !== this.watchedCollectionsKey) { + await this.stopStream('starting'); + } await this.ensureLeader(); } async shutdown(): Promise { this.closed = true; this.clearTimers(); + await this.changeQueue; await this.stopStream('idle'); await this.releaseLeader(); + this.rebacCache.clear(); + } + + private collectionNames(): string[] { + return this.options.getOptedInSchemas().map(schema => schema.collectionName); } private async ensureLeader(): Promise { @@ -129,9 +156,6 @@ export class MongoChangeStreamCoordinator { LOCK_TTL_MS, ); if (!acquired) { - // Another instance holds the lock, or a crashed holder has not - // expired yet. Without a retry, a standalone process stays idle - // forever after a restart races the previous TTL. this.streamState = 'idle'; this.scheduleRetry(); return; @@ -174,13 +198,16 @@ export class MongoChangeStreamCoordinator { await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY), ); if (this.watching || this.closed) return; - const stream = this.options.watch({ resumeAfter }); + const collections = this.collectionNames(); + const pipeline = buildWatchPipeline(collections); + this.watchedCollectionsKey = optedInCollectionsKey(collections); + const stream = this.options.watch({ resumeAfter, pipeline }); this.stream = stream; this.watching = true; this.streamState = 'live'; this.retryAttempt = 0; stream.on('change', (change: unknown) => { - void this.handleChange(change as RawChangeEvent); + this.enqueueChange(change as RawChangeEvent); }); stream.on('error', (err: unknown) => { void this.handleStreamError(err); @@ -199,26 +226,61 @@ export class MongoChangeStreamCoordinator { } } + private enqueueChange(change: RawChangeEvent) { + this.changeQueue = this.changeQueue.then(async () => { + if (this.closed || !this.watching) return; + try { + await this.handleChange(change); + } catch (err) { + this.lastError = err instanceof Error ? err.message : String(err); + ConduitGrpcSdk.Logger.error(err as Error); + this.watching = false; + await this.stopStream('degraded'); + this.scheduleRetry(); + } + }); + } + private async handleChange(change: RawChangeEvent) { + const token = serializeResumeToken(change._id); const schema = this.resolveSchema(change.ns?.coll); - if (!schema) return; - const event = normalizeChangeEvent(change, schema.name); - if (!event) return; + const event = schema ? normalizeChangeEvent(change, schema.name) : null; + if (!event || !schema) { + if (token) { + await this.persistResumeToken(token); + } + if (change.operationType && WATCH_RESTART_OPERATIONS.has(change.operationType)) { + await this.stopStream('starting'); + this.scheduleRetry(); + } + return; + } this.lastEventAt = event.occurredAt; this.lastError = undefined; - await this.options.grpcSdk.state!.setKey(RESUME_TOKEN_KEY, event.resumeToken); - this.options.grpcSdk.bus?.publish( - `database:change:${schema.name}`, - JSON.stringify(event), - ); + await this.emitChange(schema, event); + if (token) { + await this.persistResumeToken(token); + } + } + + private async persistResumeToken(token: string) { + await this.options.grpcSdk.state!.setKey(RESUME_TOKEN_KEY, token); + } + + private async emitChange(schema: OptedInSchema, event: DatabaseChangeEvent) { + const payload = JSON.stringify(event); + this.options.grpcSdk.bus?.publish(`database:change:${schema.name}`, payload); ConduitGrpcSdk.Metrics?.increment('database_realtime_events_total', 1, { operation: event.operation, }); - await this.pushEvent(schema, event); + await this.pushEvent(schema, event, payload); } - private async pushEvent(schema: OptedInSchema, event: DatabaseChangeEvent) { - const payload = JSON.stringify(event); + private async pushEvent( + schema: OptedInSchema, + event: DatabaseChangeEvent, + payload: string, + ) { const adminRooms = roomsForPublicChange(schema.name, event.documentId); await this.safePush('admin', adminRooms, payload); if (!schema.authorizationEnabled) { @@ -229,23 +291,27 @@ export class MongoChangeStreamCoordinator { schema.name, event.documentId, ); + const decisions = await checkRebacBatch( + this.rebacCache, + this.options.grpcSdk as unknown as AuthorizationSdk, + userIds, + schema.name, + event.documentId, + ); const allowedRooms: string[] = []; for (const userId of userIds) { - const allowed = await canReadDocument( - this.options.grpcSdk as unknown as AuthorizationSdk, - schema.name, - event.documentId, - userId, - ); - if (!allowed) { + const decision = decisions.get(userId) ?? 'unavailable'; + if (decision === 'allow') { + allowedRooms.push(authorizedDocumentRoom(schema.name, event.documentId, userId)); + continue; + } + if (decision === 'deny') { await this.options.subscriptions.removeUser( schema.name, event.documentId, userId, ); - continue; } - allowedRooms.push(authorizedDocumentRoom(schema.name, event.documentId, userId)); } if (allowedRooms.length > 0) { await this.safePush('router', allowedRooms, payload); @@ -260,16 +326,12 @@ export class MongoChangeStreamCoordinator { const client = target === 'admin' ? this.options.grpcSdk.admin : this.options.grpcSdk.router; if (!client?.socketPush) return; - try { - await client.socketPush({ - event: 'change', - data, - rooms, - receivers: [], - }); - } catch (err) { - ConduitGrpcSdk.Logger.error(err as Error); - } + await client.socketPush({ + event: 'change', + data, + rooms, + receivers: [], + }); } private resolveSchema(collectionName?: string): OptedInSchema | undefined { @@ -308,6 +370,7 @@ export class MongoChangeStreamCoordinator { this.watching = false; this.streamState = nextState; this.ignoreClose = true; + this.watchedCollectionsKey = ''; if (stream) { try { await stream.close(); diff --git a/modules/database/src/realtime/RealtimeService.ts b/modules/database/src/realtime/RealtimeService.ts index e17e320ea..96d21f485 100644 --- a/modules/database/src/realtime/RealtimeService.ts +++ b/modules/database/src/realtime/RealtimeService.ts @@ -19,6 +19,7 @@ import { registerDatabaseRealtimeSocket } from './sockets.js'; import { buildRealtimeStatus } from './status.js'; import { RealtimeSubscriptionTracker } from './subscriptions.js'; import type { ChangeStreamLike, OptedInSchema, RealtimeStatus } from './types.js'; +import type { WatchPipeline } from './watchPipeline.js'; export class RealtimeService { private readonly subscriptions: RealtimeSubscriptionTracker; @@ -40,7 +41,7 @@ export class RealtimeService { if (adapter instanceof MongooseAdapter) { this.coordinator = new MongoChangeStreamCoordinator({ grpcSdk, - watch: options => this.openWatch(adapter, options.resumeAfter), + watch: options => this.openWatch(adapter, options), hello: () => this.hello(adapter), getOptedInSchemas: () => this.getOptedInSchemas(), subscriptions: this.subscriptions, @@ -128,14 +129,17 @@ export class RealtimeService { return schemas; } - private openWatch(adapter: MongooseAdapter, resumeAfter?: unknown): ChangeStreamLike { + private openWatch( + adapter: MongooseAdapter, + options: { resumeAfter?: unknown; pipeline: WatchPipeline }, + ): ChangeStreamLike { const db = adapter.mongoose.connection.db; if (!db) { throw new Error('MongoDB connection is not ready'); } return db.watch( - [], - resumeAfter ? { resumeAfter: resumeAfter as never } : {}, + options.pipeline, + options.resumeAfter ? { resumeAfter: options.resumeAfter as never } : {}, ) as unknown as ChangeStreamLike; } diff --git a/modules/database/src/realtime/__tests__/change-stream.integration.test.ts b/modules/database/src/realtime/__tests__/change-stream.integration.test.ts index dd74753fc..b4e313310 100644 --- a/modules/database/src/realtime/__tests__/change-stream.integration.test.ts +++ b/modules/database/src/realtime/__tests__/change-stream.integration.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from '@jest/globals'; import { MongoClient, ObjectId } from 'mongodb'; import { normalizeChangeEvent, type RawChangeEvent } from '../normalize.js'; +import { buildWatchPipeline } from '../watchPipeline.js'; const replicaSetUri = process.env.DB_CONN_URI; const integrationEnabled = Boolean(replicaSetUri?.includes('replicaSet')); @@ -14,7 +15,7 @@ describeIntegration('MongoDB change stream contract', () => { const db = client.db(dbName); try { const collection = db.collection('orders'); - const stream = db.watch([]); + const stream = db.watch(buildWatchPipeline(['orders'])); const change = await new Promise((resolve, reject) => { const timer = setTimeout( () => reject(new Error('timed out waiting for change')), diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index efddfca70..cda5dc0e1 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -1,6 +1,6 @@ import { EventEmitter } from 'node:events'; -import { describe, expect, it, jest } from '@jest/globals'; -import { ObjectId } from 'bson'; +import { afterEach, describe, expect, it, jest } from '@jest/globals'; +import { EJSON, ObjectId } from 'bson'; import { MongoChangeStreamCoordinator } from '../MongoChangeStreamCoordinator.js'; import { RealtimeSubscriptionTracker } from '../subscriptions.js'; import { roomsForPublicChange } from '../rooms.js'; @@ -11,9 +11,13 @@ class MemoryStore { const set = this.sets.get(key) ?? new Set(); members.forEach(member => set.add(member)); this.sets.set(key, set); + return members.length; } async srem(key: string, ...members: string[]) { - members.forEach(member => this.sets.get(key)?.delete(member)); + const set = this.sets.get(key); + if (!set) return 0; + members.forEach(member => set.delete(member)); + return members.length; } async smembers(key: string) { return [...(this.sets.get(key) ?? new Set())]; @@ -23,11 +27,13 @@ class MemoryStore { } async del(...keys: string[]) { keys.forEach(key => this.sets.delete(key)); + return keys.length; } } function createCoordinator(overrides?: { allow?: boolean; + authorizationAvailable?: boolean; schemas?: { name: string; collectionName: string; authorizationEnabled: boolean }[]; getKeyDelayMs?: number; }) { @@ -65,10 +71,13 @@ function createCoordinator(overrides?: { bus: { publish }, router: { socketPush: routerPush }, admin: { socketPush: adminPush }, - isAvailable: () => true, - authorization: { - can: async () => ({ allow: overrides?.allow !== false }), - }, + isAvailable: () => overrides?.authorizationAvailable !== false, + authorization: + overrides?.authorizationAvailable === false + ? null + : { + can: async () => ({ allow: overrides?.allow !== false }), + }, }; const coordinator = new MongoChangeStreamCoordinator({ grpcSdk: grpcSdk as never, @@ -96,26 +105,32 @@ function createCoordinator(overrides?: { }; } +function insertChange(collection: string, id: string, token: unknown) { + return { + operationType: 'insert', + ns: { coll: collection }, + documentKey: { _id: new ObjectId(id) }, + _id: token, + }; +} + describe('MongoChangeStreamCoordinator', () => { + afterEach(() => { + jest.useRealTimers(); + }); + it('emits one normalized event to public rooms and ignores other collections', async () => { - const { coordinator, stream, routerPush, adminPush, publish } = createCoordinator(); + const { coordinator, stream, routerPush, adminPush, publish, state } = + createCoordinator(); await coordinator.reconcile(); const resume = { _data: 'token' }; stream.emit('change', { - operationType: 'insert', - ns: { coll: 'orders' }, - documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, + ...insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', resume), fullDocument: { secret: 'nope' }, - _id: resume, wallTime: new Date('2026-01-02T00:00:00.000Z'), }); - stream.emit('change', { - operationType: 'insert', - ns: { coll: 'other' }, - documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4d') }, - _id: resume, - }); - await new Promise(resolve => setImmediate(resolve)); + stream.emit('change', insertChange('other', '64b64c4c4c4c4c4c4c4c4c4d', resume)); + await coordinator.waitForIdle(); expect(publish).toHaveBeenCalledTimes(1); expect(publish.mock.calls[0][0]).toBe('database:change:Order'); const payload = JSON.parse(publish.mock.calls[0][1] as string); @@ -125,6 +140,7 @@ describe('MongoChangeStreamCoordinator', () => { documentId: '64b64c4c4c4c4c4c4c4c4c4c', }); expect(payload).not.toHaveProperty('fullDocument'); + expect(payload).not.toHaveProperty('resumeToken'); expect(payload).not.toHaveProperty('secret'); const expectedRooms = roomsForPublicChange('Order', '64b64c4c4c4c4c4c4c4c4c4c'); expect(routerPush).toHaveBeenCalledWith( @@ -133,6 +149,102 @@ describe('MongoChangeStreamCoordinator', () => { expect(adminPush).toHaveBeenCalledWith( expect.objectContaining({ event: 'change', rooms: expectedRooms }), ); + expect( + JSON.parse((adminPush.mock.calls[0][0] as { data: string }).data), + ).not.toHaveProperty('resumeToken'); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(resume)); + await coordinator.shutdown(); + }); + + it('advances the resume token for filtered events', async () => { + const { coordinator, stream, publish, state } = createCoordinator(); + await coordinator.reconcile(); + const skip = { _data: 'skip-token' }; + stream.emit('change', insertChange('other', '64b64c4c4c4c4c4c4c4c4c4d', skip)); + await coordinator.waitForIdle(); + expect(publish).not.toHaveBeenCalled(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(skip)); + await coordinator.shutdown(); + }); + + it('serializes overlapping handlers and persists after emit', async () => { + const { coordinator, stream, state, adminPush } = createCoordinator(); + let release!: () => void; + const gate = new Promise(resolve => { + release = resolve; + }); + let first = true; + adminPush.mockImplementation(async () => { + if (first) { + first = false; + await gate; + } + }); + await coordinator.reconcile(); + const tokenA = { _data: 'token-a' }; + const tokenB = { _data: 'token-b' }; + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', tokenB)); + await Promise.resolve(); + await new Promise(resolve => setImmediate(resolve)); + expect(state.get('realtime:resumeToken')).toBeUndefined(); + expect(adminPush).toHaveBeenCalledTimes(1); + release(); + await coordinator.waitForIdle(); + expect(adminPush).toHaveBeenCalledTimes(2); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenB)); + await coordinator.shutdown(); + }); + + it('watches opted-in collections with $match and $project', async () => { + const { coordinator, watch } = createCoordinator(); + await coordinator.reconcile(); + expect(watch).toHaveBeenCalledTimes(1); + const pipeline = (watch.mock.calls[0][0] as { pipeline: Record[] }) + .pipeline; + expect(pipeline[0]).toEqual( + expect.objectContaining({ + $match: expect.objectContaining({ + $or: expect.arrayContaining([ + expect.objectContaining({ + 'ns.coll': { $in: ['orders'] }, + }), + ]), + }), + }), + ); + expect(pipeline[1]).toEqual({ + $project: { + fullDocument: 0, + updateDescription: 0, + fullDocumentBeforeChange: 0, + }, + }); + await coordinator.shutdown(); + }); + + it('reopens the watch when the opt-in set changes', async () => { + const schemas = [ + { name: 'Order', collectionName: 'orders', authorizationEnabled: false }, + ]; + const { coordinator, watch } = createCoordinator({ schemas }); + await coordinator.reconcile(); + expect(watch).toHaveBeenCalledTimes(1); + schemas.push({ + name: 'Item', + collectionName: 'items', + authorizationEnabled: false, + }); + await coordinator.reconcile(); + expect(watch).toHaveBeenCalledTimes(2); + const pipeline = (watch.mock.calls[1][0] as { pipeline: Record[] }) + .pipeline; + const match = pipeline[0] as { + $match: { $or: Array<{ 'ns.coll'?: { $in: string[] } }> }; + }; + expect(match.$match.$or[0]['ns.coll']?.$in).toEqual( + expect.arrayContaining(['orders', 'items']), + ); await coordinator.shutdown(); }); @@ -148,15 +260,15 @@ describe('MongoChangeStreamCoordinator', () => { 'user-1', ); await coordinator.reconcile(); - stream.emit('change', { - operationType: 'update', - ns: { coll: 'orders' }, - documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, - _id: { _data: 'token' }, - }); - await new Promise(resolve => setImmediate(resolve)); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), + ); + await coordinator.waitForIdle(); expect(routerPush).not.toHaveBeenCalled(); - expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c')).toEqual([]); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual( + [], + ); await coordinator.shutdown(); }); @@ -179,7 +291,7 @@ describe('MongoChangeStreamCoordinator', () => { await coordinator.shutdown(); }); - it('clears an unusable resume token and retries', async () => { + it('clears an unusable resume token on 280 and retries', async () => { const { coordinator, stream, grpcSdk } = createCoordinator(); await coordinator.reconcile(); stream.emit('error', { code: 280, message: 'ChangeStreamHistoryLost' }); @@ -187,4 +299,124 @@ describe('MongoChangeStreamCoordinator', () => { expect(grpcSdk.state.clearKey).toHaveBeenCalled(); await coordinator.shutdown(); }); + + it('keeps the resume token on CursorKilled 237', async () => { + const { coordinator, stream, grpcSdk, state } = createCoordinator(); + await coordinator.reconcile(); + const token = { _data: 'keep-me' }; + stream.emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', token)); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); + stream.emit('error', { code: 237, message: 'CursorKilled' }); + await new Promise(resolve => setImmediate(resolve)); + expect(grpcSdk.state.clearKey).not.toHaveBeenCalled(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); + await coordinator.shutdown(); + }); + + it('does not persist a later token when emit fails and reopens from the last good token', async () => { + jest.useFakeTimers(); + const streams: Array Promise }> = []; + const { coordinator, adminPush, state, watch } = createCoordinator(); + watch.mockImplementation(() => { + const next = new EventEmitter() as EventEmitter & { close: () => Promise }; + next.close = async () => { + next.emit('close'); + }; + streams.push(next); + return next as never; + }); + adminPush + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error('push failed')) + .mockResolvedValue(undefined); + await coordinator.reconcile(); + const tokenGood = { _data: 'token-good' }; + const tokenA = { _data: 'token-a' }; + const tokenB = { _data: 'token-b' }; + streams[0].emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4b', tokenGood), + ); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenGood)); + streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); + streams[0].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4d', tokenB)); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenGood)); + expect(adminPush).toHaveBeenCalledTimes(2); + await jest.advanceTimersByTimeAsync(1_000); + expect(watch).toHaveBeenCalledTimes(2); + expect((watch.mock.calls[1][0] as { resumeAfter?: unknown }).resumeAfter).toEqual( + tokenGood, + ); + streams[1].emit('change', insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', tokenA)); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(tokenA)); + expect(adminPush).toHaveBeenCalledTimes(3); + await coordinator.shutdown(); + jest.useRealTimers(); + }); + + it.each(['drop', 'rename', 'invalidate'] as const)( + 'reopens the watch on %s', + async operationType => { + jest.useFakeTimers(); + const streams: Array Promise }> = []; + const { coordinator, watch, state } = createCoordinator(); + watch.mockImplementation(() => { + const next = new EventEmitter() as EventEmitter & { close: () => Promise }; + next.close = async () => { + next.emit('close'); + }; + streams.push(next); + return next as never; + }); + await coordinator.reconcile(); + const token = { _data: `${operationType}-token` }; + streams[0].emit('change', { + operationType, + ns: { coll: 'orders' }, + _id: token, + }); + await coordinator.waitForIdle(); + expect(state.get('realtime:resumeToken')).toBe(EJSON.stringify(token)); + await jest.advanceTimersByTimeAsync(1_000); + expect(watch).toHaveBeenCalledTimes(2); + expect((watch.mock.calls[1][0] as { resumeAfter?: unknown }).resumeAfter).toEqual( + token, + ); + await coordinator.shutdown(); + jest.useRealTimers(); + }, + ); + + it('does not remove users when authorization is unavailable', async () => { + const { coordinator, stream, routerPush, subscriptions } = createCoordinator({ + authorizationAvailable: false, + schemas: [{ name: 'Order', collectionName: 'orders', authorizationEnabled: true }], + }); + const removeUser = jest.spyOn(subscriptions, 'removeUser'); + await subscriptions.addAuthorizedDocument( + 'sock-1', + 'Order', + '64b64c4c4c4c4c4c4c4c4c4c', + 'user-1', + ); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual([ + 'user-1', + ]); + await coordinator.reconcile(); + stream.emit( + 'change', + insertChange('orders', '64b64c4c4c4c4c4c4c4c4c4c', { _data: 'token' }), + ); + await coordinator.waitForIdle(); + expect(routerPush).not.toHaveBeenCalled(); + expect(removeUser).not.toHaveBeenCalled(); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c4c')).toEqual([ + 'user-1', + ]); + await coordinator.shutdown(); + }); }); diff --git a/modules/database/src/realtime/__tests__/normalize.test.ts b/modules/database/src/realtime/__tests__/normalize.test.ts index db2278e29..f7f933d33 100644 --- a/modules/database/src/realtime/__tests__/normalize.test.ts +++ b/modules/database/src/realtime/__tests__/normalize.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from '@jest/globals'; import { ObjectId } from 'bson'; -import { normalizeChangeEvent, parseResumeToken } from '../normalize.js'; +import { + normalizeChangeEvent, + parseResumeToken, + serializeResumeToken, +} from '../normalize.js'; describe('normalizeChangeEvent', () => { it('normalizes insert/update/replace/delete into metadata-only events', () => { @@ -22,7 +26,8 @@ describe('normalizeChangeEvent', () => { documentId: '64b64c4c4c4c4c4c4c4c4c4c', occurredAt: '2026-01-01T00:00:00.000Z', }); - expect(parseResumeToken(event!.resumeToken)).toEqual(resume); + expect(parseResumeToken(serializeResumeToken(resume))).toEqual(resume); + expect(event).not.toHaveProperty('resumeToken'); expect(JSON.parse(JSON.stringify(event))).not.toHaveProperty('fullDocument'); }); diff --git a/modules/database/src/realtime/__tests__/recovery.test.ts b/modules/database/src/realtime/__tests__/recovery.test.ts new file mode 100644 index 000000000..eee45b918 --- /dev/null +++ b/modules/database/src/realtime/__tests__/recovery.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from '@jest/globals'; +import { canReadDocument } from '../authorize.js'; +import { authorizedDocumentRoom } from '../rooms.js'; +import { isRecoverableDisconnect, restoreAuthorizedSubscriptions } from '../recovery.js'; +import { createSocketHandlers } from '../sockets.js'; +import { RealtimeSubscriptionTracker } from '../subscriptions.js'; + +class MemoryStore { + private sets = new Map>(); + async sadd(key: string, ...members: string[]) { + const set = this.sets.get(key) ?? new Set(); + members.forEach(member => set.add(member)); + this.sets.set(key, set); + } + async srem(key: string, ...members: string[]) { + members.forEach(member => this.sets.get(key)?.delete(member)); + } + async smembers(key: string) { + return [...(this.sets.get(key) ?? new Set())]; + } + async scard(key: string) { + return this.sets.get(key)?.size ?? 0; + } + async del(...keys: string[]) { + keys.forEach(key => this.sets.delete(key)); + } +} + +describe('database socket recovery', () => { + it('does not treat transport close as a wipe', () => { + expect(isRecoverableDisconnect('transport close')).toBe(true); + expect(isRecoverableDisconnect('client namespace disconnect')).toBe(false); + }); + + it('restores authorized Redis membership after a recoverable disconnect', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + const room = authorizedDocumentRoom('Order', 'doc-1', 'user-1'); + await tracker.addAuthorizedDocument('sock-1', 'Order', 'doc-1', 'user-1'); + await tracker.disconnect('sock-1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); + + await restoreAuthorizedSubscriptions({ + socketId: 'sock-1', + rooms: [room], + contextSubs: [], + subscriptions: tracker, + grpcSdk: { + isAvailable: () => true, + authorization: { can: async () => ({ allow: true }) }, + }, + canRead: canReadDocument, + }); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + }); + + it('keeps recovered /database/ sockets in the authorized list', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + await tracker.addAuthorizedDocument('sock-1', 'Order', 'doc-1', 'user-1'); + const handlers = createSocketHandlers({ + mode: 'client', + grpcSdk: { + isAvailable: () => true, + authorization: { can: async () => ({ allow: true }) }, + } as never, + schemaLookup: { getSchema: () => undefined }, + subscriptions: tracker, + isGloballyEnabled: () => true, + }); + await handlers.disconnect({ + request: { socketId: 'sock-1', params: ['transport close'] }, + } as never); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + await handlers.recovered({ + request: { + socketId: 'sock-1', + params: [authorizedDocumentRoom('Order', 'doc-1', 'user-1')], + context: {}, + }, + } as never); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + }); +}); diff --git a/modules/database/src/realtime/__tests__/rooms.test.ts b/modules/database/src/realtime/__tests__/rooms.test.ts index dbdd57e1b..96de67f73 100644 --- a/modules/database/src/realtime/__tests__/rooms.test.ts +++ b/modules/database/src/realtime/__tests__/rooms.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from '@jest/globals'; import { authorizedDocumentRoom, documentRoom, + parseAuthorizedDocumentRoom, roomsForPublicChange, schemaRoom, } from '../rooms.js'; @@ -21,6 +22,13 @@ describe('realtime rooms', () => { expect(authorizedDocumentRoom('Order', 'abc', 'u1')).not.toBe( authorizedDocumentRoom('Order', 'abc', 'u2'), ); + expect( + parseAuthorizedDocumentRoom(authorizedDocumentRoom('Order', 'abc', 'user-1')), + ).toEqual({ + schema: 'Order', + documentId: 'abc', + userId: 'user-1', + }); }); it('encodes reserved characters so rooms cannot collide', () => { diff --git a/modules/database/src/realtime/__tests__/topology.test.ts b/modules/database/src/realtime/__tests__/topology.test.ts index abf8e8861..79afac87c 100644 --- a/modules/database/src/realtime/__tests__/topology.test.ts +++ b/modules/database/src/realtime/__tests__/topology.test.ts @@ -10,7 +10,10 @@ describe('topology helpers', () => { it('detects unusable resume tokens', () => { expect(isResumeTokenUnusable({ code: 280 })).toBe(true); - expect(isResumeTokenUnusable(new Error('cannot resume'))).toBe(true); + expect(isResumeTokenUnusable({ code: 237 })).toBe(false); + expect(isResumeTokenUnusable({ code: 136 })).toBe(false); + expect(isResumeTokenUnusable(new Error('ChangeStreamHistoryLost'))).toBe(true); + expect(isResumeTokenUnusable(new Error('cannot resume'))).toBe(false); expect(isResumeTokenUnusable(new Error('socket hang up'))).toBe(false); }); }); diff --git a/modules/database/src/realtime/__tests__/watchPipeline.test.ts b/modules/database/src/realtime/__tests__/watchPipeline.test.ts new file mode 100644 index 000000000..7c31a64f5 --- /dev/null +++ b/modules/database/src/realtime/__tests__/watchPipeline.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from '@jest/globals'; +import { buildWatchPipeline } from '../watchPipeline.js'; + +describe('buildWatchPipeline', () => { + it('matches opted-in collections and projects out fullDocument', () => { + const pipeline = buildWatchPipeline(['orders', 'items']); + expect(pipeline[0]).toEqual({ + $match: { + $or: [ + { + operationType: { $in: ['insert', 'update', 'replace', 'delete'] }, + 'ns.coll': { $in: ['orders', 'items'] }, + }, + { + operationType: { $in: ['drop', 'rename', 'invalidate', 'dropDatabase'] }, + }, + ], + }, + }); + expect(pipeline[1]).toEqual({ + $project: { + fullDocument: 0, + updateDescription: 0, + fullDocumentBeforeChange: 0, + }, + }); + }); +}); diff --git a/modules/database/src/realtime/normalize.ts b/modules/database/src/realtime/normalize.ts index bdd876f16..b2f4b8ff3 100644 --- a/modules/database/src/realtime/normalize.ts +++ b/modules/database/src/realtime/normalize.ts @@ -42,10 +42,18 @@ export function normalizeChangeEvent( ? change.wallTime : occurredAt ).toISOString(), - resumeToken: EJSON.stringify(change._id), }; } +export function serializeResumeToken(id: unknown): string | null { + if (id === undefined || id === null) return null; + try { + return EJSON.stringify(id); + } catch { + return null; + } +} + export function parseResumeToken(token: string | null | undefined): unknown | undefined { if (!token) return undefined; try { diff --git a/modules/database/src/realtime/rebacCache.ts b/modules/database/src/realtime/rebacCache.ts new file mode 100644 index 000000000..b33a9438c --- /dev/null +++ b/modules/database/src/realtime/rebacCache.ts @@ -0,0 +1,96 @@ +import type { AuthorizationSdk } from './authorize.js'; + +export type RebacDecision = 'allow' | 'deny' | 'unavailable'; + +const DEFAULT_TTL_MS = 12_000; +const DEFAULT_MAX_ENTRIES = 10_000; + +type CacheEntry = { + decision: 'allow' | 'deny'; + expiresAt: number; +}; + +export class RealtimeRebacCache { + private readonly entries = new Map(); + + constructor( + private readonly ttlMs: number = DEFAULT_TTL_MS, + private readonly maxEntries: number = DEFAULT_MAX_ENTRIES, + ) {} + + async check( + grpcSdk: AuthorizationSdk, + userId: string, + schema: string, + documentId: string, + ): Promise { + const resource = `${schema}:${documentId}`; + const key = `${userId}:read:${resource}`; + const now = Date.now(); + this.sweepExpired(now); + const cached = this.entries.get(key); + if (cached && cached.expiresAt > now) { + return cached.decision; + } + if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { + return 'unavailable'; + } + try { + const decision = await grpcSdk.authorization.can({ + subject: `User:${userId}`, + actions: ['read'], + resource, + }); + const value: 'allow' | 'deny' = decision.allow ? 'allow' : 'deny'; + this.set(key, value, now); + return value; + } catch { + return 'unavailable'; + } + } + + clear(): void { + this.entries.clear(); + } + + private set(key: string, decision: 'allow' | 'deny', now: number): void { + if (this.entries.size >= this.maxEntries && !this.entries.has(key)) { + const firstKey = this.entries.keys().next().value; + if (firstKey) { + this.entries.delete(firstKey); + } + } + this.entries.set(key, { decision, expiresAt: now + this.ttlMs }); + } + + private sweepExpired(now: number): void { + for (const [key, entry] of this.entries) { + if (entry.expiresAt <= now) { + this.entries.delete(key); + } + } + } +} + +export async function checkRebacBatch( + cache: RealtimeRebacCache, + grpcSdk: AuthorizationSdk, + userIds: string[], + schema: string, + documentId: string, + concurrency = 8, +): Promise> { + const results = new Map(); + let index = 0; + async function worker(): Promise { + while (index < userIds.length) { + const userId = userIds[index++]; + results.set(userId, await cache.check(grpcSdk, userId, schema, documentId)); + } + } + const workers = Array.from({ length: Math.min(concurrency, userIds.length) }, () => + worker(), + ); + await Promise.all(workers); + return results; +} diff --git a/modules/database/src/realtime/recovery.ts b/modules/database/src/realtime/recovery.ts new file mode 100644 index 000000000..3bc9c3868 --- /dev/null +++ b/modules/database/src/realtime/recovery.ts @@ -0,0 +1,147 @@ +import type { Indexable, ParsedSocketRequest } from '@conduitplatform/grpc-sdk'; +import type { AuthorizationSdk } from './authorize.js'; +import { authorizedDocumentRoom, parseAuthorizedDocumentRoom } from './rooms.js'; +import type { RealtimeSubscriptionTracker } from './subscriptions.js'; + +const RECOVERABLE_DISCONNECT = new Set([ + 'ping timeout', + 'transport close', + 'transport error', +]); + +const CONTEXT_KEY = 'databaseSubs'; + +export type AuthorizedSub = { + schema: string; + documentId: string; + userId: string; +}; + +export function isRecoverableDisconnect(reason: unknown): boolean { + return typeof reason === 'string' && RECOVERABLE_DISCONNECT.has(reason); +} + +export function authorizedSubsFromContext( + context: Indexable | undefined, +): AuthorizedSub[] { + if (!context) return []; + const raw = context[CONTEXT_KEY]; + if (!Array.isArray(raw)) return []; + const subs: AuthorizedSub[] = []; + for (const item of raw) { + if ( + item && + typeof item === 'object' && + typeof (item as AuthorizedSub).schema === 'string' && + typeof (item as AuthorizedSub).documentId === 'string' && + typeof (item as AuthorizedSub).userId === 'string' + ) { + subs.push({ + schema: (item as AuthorizedSub).schema, + documentId: (item as AuthorizedSub).documentId, + userId: (item as AuthorizedSub).userId, + }); + } + } + return subs; +} + +export function persistAuthorizedSubOnContext( + context: Indexable | undefined, + schema: string, + documentId: string, + userId: string, +): void { + if (!context) return; + const next = authorizedSubsFromContext(context).filter( + sub => + !(sub.schema === schema && sub.documentId === documentId && sub.userId === userId), + ); + next.push({ schema, documentId, userId }); + context[CONTEXT_KEY] = next; +} + +export function removeAuthorizedSubFromContext( + context: Indexable | undefined, + schema: string, + documentId: string, + userId: string, +): void { + if (!context) return; + const next = authorizedSubsFromContext(context).filter( + sub => + !(sub.schema === schema && sub.documentId === documentId && sub.userId === userId), + ); + if (next.length === 0) { + delete context[CONTEXT_KEY]; + } else { + context[CONTEXT_KEY] = next; + } +} + +export function recoveredRoomsFromRequest(call: ParsedSocketRequest): string[] { + const params = call.request.params ?? []; + if (params.every(item => typeof item === 'string')) { + return params as string[]; + } + return []; +} + +export async function restoreAuthorizedSubscriptions(options: { + socketId: string; + rooms: string[]; + contextSubs: AuthorizedSub[]; + subscriptions: RealtimeSubscriptionTracker; + grpcSdk: AuthorizationSdk; + canRead: ( + grpcSdk: AuthorizationSdk, + schema: string, + documentId: string, + userId: string, + ) => Promise; +}): Promise<{ leaveRooms: string[] }> { + const seen = new Set(); + const subs: AuthorizedSub[] = []; + for (const room of options.rooms) { + const parsed = parseAuthorizedDocumentRoom(room); + if (!parsed) continue; + const key = `${parsed.schema}:${parsed.documentId}:${parsed.userId}`; + if (seen.has(key)) continue; + seen.add(key); + subs.push(parsed); + } + for (const sub of options.contextSubs) { + const key = `${sub.schema}:${sub.documentId}:${sub.userId}`; + if (seen.has(key)) continue; + seen.add(key); + subs.push(sub); + } + + const leaveRooms: string[] = []; + for (const sub of subs) { + const allowed = await options.canRead( + options.grpcSdk, + sub.schema, + sub.documentId, + sub.userId, + ); + const room = authorizedDocumentRoom(sub.schema, sub.documentId, sub.userId); + if (!allowed) { + await options.subscriptions.removeAuthorizedDocument( + options.socketId, + sub.schema, + sub.documentId, + sub.userId, + ); + leaveRooms.push(room); + continue; + } + await options.subscriptions.addAuthorizedDocument( + options.socketId, + sub.schema, + sub.documentId, + sub.userId, + ); + } + return { leaveRooms }; +} diff --git a/modules/database/src/realtime/rooms.ts b/modules/database/src/realtime/rooms.ts index 45a69a7cd..5792482f9 100644 --- a/modules/database/src/realtime/rooms.ts +++ b/modules/database/src/realtime/rooms.ts @@ -23,3 +23,25 @@ export function authorizedDocumentRoom( export function roomsForPublicChange(schema: string, documentId: string): string[] { return [schemaRoom(schema), documentRoom(schema, documentId)]; } + +export function parseAuthorizedDocumentRoom(room: string): { + schema: string; + documentId: string; + userId: string; +} | null { + const prefix = `${ROOM_PREFIX}:doc:`; + if (!room.startsWith(prefix)) return null; + const rest = room.slice(prefix.length); + const userMarker = ':user:'; + const userIndex = rest.lastIndexOf(userMarker); + if (userIndex === -1) return null; + const userId = decodeURIComponent(rest.slice(userIndex + userMarker.length)); + const schemaDoc = rest.slice(0, userIndex); + const lastColon = schemaDoc.lastIndexOf(':'); + if (lastColon === -1) return null; + return { + schema: decodeURIComponent(schemaDoc.slice(0, lastColon)), + documentId: decodeURIComponent(schemaDoc.slice(lastColon + 1)), + userId, + }; +} diff --git a/modules/database/src/realtime/sockets.ts b/modules/database/src/realtime/sockets.ts index 4be4e1018..e2a80931c 100644 --- a/modules/database/src/realtime/sockets.ts +++ b/modules/database/src/realtime/sockets.ts @@ -1,6 +1,7 @@ import { ConduitGrpcSdk, ConduitRouteReturnDefinition, + type Indexable, ParsedSocketRequest, TYPE, UnparsedSocketResponse, @@ -19,6 +20,14 @@ import { } from './authorize.js'; import { authorizedDocumentRoom, documentRoom, schemaRoom } from './rooms.js'; import type { RealtimeSubscriptionTracker } from './subscriptions.js'; +import { + authorizedSubsFromContext, + isRecoverableDisconnect, + persistAuthorizedSubOnContext, + recoveredRoomsFromRequest, + removeAuthorizedSubFromContext, + restoreAuthorizedSubscriptions, +} from './recovery.js'; type SocketMode = 'client' | 'admin'; @@ -43,6 +52,7 @@ export function registerDatabaseRealtimeSocket( { connect: { handler: handlers.connect }, disconnect: { handler: handlers.disconnect }, + recovered: { handler: handlers.recovered }, subscribe: { params: [TYPE.JSON], handler: handlers.subscribe, @@ -61,15 +71,32 @@ export function registerDatabaseRealtimeSocket( ); } -function createSocketHandlers(options: RealtimeSocketOptions) { +export function createSocketHandlers(options: RealtimeSocketOptions) { return { connect: async (): Promise => { return { event: 'connected', data: { ok: true } }; }, disconnect: async (call: ParsedSocketRequest): Promise => { - await options.subscriptions.disconnect(call.request.socketId); + const reason = call.request.params?.[0]; + if (!isRecoverableDisconnect(reason)) { + await options.subscriptions.disconnect(call.request.socketId); + } return { event: 'disconnected', data: { ok: true } }; }, + recovered: async (call: ParsedSocketRequest): Promise => { + const { leaveRooms } = await restoreAuthorizedSubscriptions({ + socketId: call.request.socketId, + rooms: recoveredRoomsFromRequest(call), + contextSubs: authorizedSubsFromContext(call.request.context as Indexable), + subscriptions: options.subscriptions, + grpcSdk: options.grpcSdk as unknown as AuthorizationSdk, + canRead: canReadDocument, + }); + if (leaveRooms.length > 0) { + return { event: 'leave-room', rooms: leaveRooms }; + } + return { event: 'join-room', rooms: [] }; + }, subscribe: async (call: ParsedSocketRequest): Promise => { const rooms = await resolveSubscription(call, options, 'join'); return { event: 'join-room', rooms }; @@ -140,6 +167,12 @@ async function resolveSubscription( documentId, userId, ); + persistAuthorizedSubOnContext( + call.request.context as Indexable, + schemaName, + documentId, + userId, + ); } else { await options.subscriptions.removeAuthorizedDocument( call.request.socketId, @@ -147,6 +180,12 @@ async function resolveSubscription( documentId, userId, ); + removeAuthorizedSubFromContext( + call.request.context as Indexable, + schemaName, + documentId, + userId, + ); } return [authorizedDocumentRoom(schemaName, documentId, userId)]; } diff --git a/modules/database/src/realtime/topology.ts b/modules/database/src/realtime/topology.ts index 341d49438..4d62fa3fa 100644 --- a/modules/database/src/realtime/topology.ts +++ b/modules/database/src/realtime/topology.ts @@ -1,6 +1,4 @@ -const CHANGE_STREAM_ERROR_CODES = new Set([ - 136, // CappedPositionLost - 237, // CursorKilled +const UNUSABLE_RESUME_TOKEN_CODES = new Set([ 280, // ChangeStreamHistoryLost 286, // ChangeStreamFatalError ]); @@ -33,11 +31,14 @@ export function topologyFromHello( export function isResumeTokenUnusable(error: unknown): boolean { const code = extractErrorCode(error); - if (code !== undefined && CHANGE_STREAM_ERROR_CODES.has(code)) { + if (code === 237 || code === 136) { + return false; + } + if (code !== undefined && UNUSABLE_RESUME_TOKEN_CODES.has(code)) { return true; } const message = error instanceof Error ? error.message : String(error ?? ''); - return /resume token|ChangeStreamHistoryLost|cannot resume/i.test(message); + return /ChangeStreamHistoryLost/i.test(message); } function extractErrorCode(error: unknown): number | undefined { diff --git a/modules/database/src/realtime/types.ts b/modules/database/src/realtime/types.ts index 9901c2d86..2b6e3e8c7 100644 --- a/modules/database/src/realtime/types.ts +++ b/modules/database/src/realtime/types.ts @@ -15,7 +15,6 @@ export type DatabaseChangeEvent = { schema: string; documentId: string; occurredAt: string; - resumeToken: string; }; export type RealtimeStatusCode = diff --git a/modules/database/src/realtime/watchPipeline.ts b/modules/database/src/realtime/watchPipeline.ts new file mode 100644 index 000000000..8ca888262 --- /dev/null +++ b/modules/database/src/realtime/watchPipeline.ts @@ -0,0 +1,33 @@ +const DATA_OPERATIONS = ['insert', 'update', 'replace', 'delete'] as const; +const CONTROL_OPERATIONS = ['drop', 'rename', 'invalidate', 'dropDatabase'] as const; + +export const WATCH_RESTART_OPERATIONS = new Set(CONTROL_OPERATIONS); + +export type WatchPipeline = Record[]; + +export function buildWatchPipeline(collectionNames: string[]): WatchPipeline { + return [ + { + $match: { + $or: [ + { + operationType: { $in: [...DATA_OPERATIONS] }, + 'ns.coll': { $in: collectionNames }, + }, + { operationType: { $in: [...CONTROL_OPERATIONS] } }, + ], + }, + }, + { + $project: { + fullDocument: 0, + updateDescription: 0, + fullDocumentBeforeChange: 0, + }, + }, + ]; +} + +export function optedInCollectionsKey(collectionNames: string[]): string { + return [...collectionNames].sort().join('\0'); +} diff --git a/modules/router/README.mdx b/modules/router/README.mdx index c896f2cee..a0dad3165 100644 --- a/modules/router/README.mdx +++ b/modules/router/README.mdx @@ -92,12 +92,12 @@ import { io } from 'socket.io-client'; const socket = io(`${SOCKET_BASE_URL}/events/`, { path: '/realtime', - extraHeaders: { - authorization: `Bearer ${accessToken}`, - }, + auth: { token: accessToken }, }); -socket.emit('subscribe', relayId, resourceId); +socket.on('connect', () => { + socket.emit('subscribe', relayId, resourceId); +}); socket.on('order-updated', payload => { // payload is the rendered JSON template }); @@ -108,6 +108,20 @@ socket.emit('unsubscribe', relayId, resourceId); - The outbound event name is the relay `socketEvent`. - Placeholders in `messageTemplate` use `{{payload.path}}` against the bus JSON payload. +**Subscribe-only.** Clients cannot publish on `/events/`; modules publish to the Redis bus and relays forward matching messages to subscribed rooms. + +**Authentication and Authorization** are required (`authMiddleware` on subscribe). Authorization is re-checked on a short TTL during delivery; revoked access triggers `leave-room`. + +**Rooms** are server-assigned: `er::`. Conduit has no tenant model and **team ids are not part of room names** — isolation is ReBAC on `User` × `permission` × `resourceType:resourceId`. + +**HA / `localOnly`:** every Router replica subscribes to the bus and emits with `localOnly`, so clients spread across replicas still receive one copy without sticky sessions for delivery. Engine.IO **polling** transport still benefits from sticky sessions or WebSocket-only clients. + +**EventBus self-publish:** the bus ignores this process’s own publishes (signature filter). A relay on a channel the Router publishes will not deliver to local clients for those self-originated messages (typical for module-originated events). + +**Dual delivery warning:** Database `/database/` `change` events and a relay on overlapping bus channels (for example `database:change:*` or CRUD `database:update:*`) can duplicate notifications. CRUD bus payloads include **full documents** — prefer change-stream style payloads when both paths are enabled. + +Use preview to validate templates: `POST /router/event-relays/preview` with `messageTemplate` and `samplePayload` (same renderer as runtime; sample JSON is capped at 256 KiB like inbound bus payloads). + ### Admin API | Method | Path | Description | @@ -115,5 +129,6 @@ socket.emit('unsubscribe', relayId, resourceId); | `GET` | `/router/event-relays` | Paginated list (`skip`, `limit`, `search`) | | `GET` | `/router/event-relays/:id` | Single relay | | `POST` | `/router/event-relays` | Create | -| `PATCH` | `/router/event-relays/:id` | Update | +| `POST` | `/router/event-relays/preview` | Render template against sample JSON | +| `PATCH` | `/router/event-relays/:id` | Update (deactivating evicts rooms) | | `DELETE` | `/router/event-relays/:id` | Delete | diff --git a/modules/router/src/Router.ts b/modules/router/src/Router.ts index 0a3867ed6..78369f6e7 100644 --- a/modules/router/src/Router.ts +++ b/modules/router/src/Router.ts @@ -1,4 +1,4 @@ -import { NextFunction } from 'express'; +import { NextFunction, Response } from 'express'; import { status } from '@grpc/grpc-js'; import { ConduitGrpcSdk, @@ -66,6 +66,11 @@ export default class ConduitDefaultRouter extends ManagedModule { private adminRouter: AdminHandlers; private readonly _routes: string[]; private readonly _globalMiddlewares: string[]; + private readonly _socketGlobalMiddlewareHandlers: (( + req: ConduitRequest, + res: Response, + next: NextFunction, + ) => void)[]; private _grpcRoutes: { [field: string]: RouteT[]; } = {}; @@ -76,12 +81,15 @@ export default class ConduitDefaultRouter extends ManagedModule { private _haInitialized = false; private eventRelayManager: EventRelayManager; private eventsSocket?: ConduitSocket; + private socketsPreviouslyStopped = false; + private securityMiddlewareInitialized = false; constructor(peerManifestRoot?: string) { super('router', peerManifestRoot); this.updateHealth(HealthCheckStatus.UNKNOWN, true); this._routes = []; this._globalMiddlewares = []; + this._socketGlobalMiddlewareHandlers = []; } async onServerStart() { @@ -112,6 +120,12 @@ export default class ConduitDefaultRouter extends ManagedModule { this.eventRelayManager = new EventRelayManager( this.grpcSdk, createEventRelayPusher(data => this._internalRouter.socketPush(data)), + { + getLocalRoomUserIds: (room: string) => + this._internalRouter.getLocalRoomUserIds('/events/', room), + getLocalRoomsWithPrefix: (prefix: string) => + this._internalRouter.getLocalRoomsWithPrefix('/events/', prefix), + }, ); this.adminRouter = new AdminHandlers( this.grpcServer, @@ -133,6 +147,10 @@ export default class ConduitDefaultRouter extends ManagedModule { async onConfig() { const config = ConfigController.getInstance().config; + const shouldRebindSocketGlobals = + config.transports.sockets && + this.socketsPreviouslyStopped && + this.securityMiddlewareInitialized; let atLeastOne = false; if (config.transports.graphql) { this._internalRouter.initGraphQL(); @@ -148,16 +166,24 @@ export default class ConduitDefaultRouter extends ManagedModule { } if (config.transports.sockets) { this._internalRouter.initSockets(); - this.registerEventsNamespace(); await this.eventRelayManager.start(); atLeastOne = true; } else { await this.eventRelayManager?.stop(); this._internalRouter.stopSockets(); + this.socketsPreviouslyStopped = true; } - if (atLeastOne) { + if (atLeastOne && !this.securityMiddlewareInitialized) { this._security.setupMiddlewares(); + this.securityMiddlewareInitialized = true; + } + if (config.transports.sockets) { + if (shouldRebindSocketGlobals) { + this.rebindSocketGlobalMiddlewares(); + } + this.socketsPreviouslyStopped = false; + this.registerEventsNamespace(); } if (!this._sdkRoutes.some(r => r.path === '/ready')) { this.registerRoute(adminRoutes.getReadyRoute()); @@ -335,13 +361,29 @@ export default class ConduitDefaultRouter extends ManagedModule { registerGlobalMiddleware( name: string, - middleware: any, + middleware: (req: ConduitRequest, res: Response, next: NextFunction) => void, socketMiddleware: boolean = false, ) { this._globalMiddlewares.push(name); + if (socketMiddleware) { + this._socketGlobalMiddlewareHandlers.push(middleware); + } this._internalRouter.registerMiddleware(middleware, socketMiddleware); } + private rebindSocketGlobalMiddlewares() { + for (const middleware of this._socketGlobalMiddlewareHandlers) { + this._internalRouter.registerSocketGlobalMiddleware(middleware); + } + } + + async shutdown(): Promise { + if (this.eventRelayManager) { + await this.eventRelayManager.stop(); + } + this.grpcSdk.bus?.quit(); + } + getRegisteredRoutes() { return this._routes; } diff --git a/modules/router/src/admin/event-relays.ts b/modules/router/src/admin/event-relays.ts index 2e56fbd15..442b3efd7 100644 --- a/modules/router/src/admin/event-relays.ts +++ b/modules/router/src/admin/event-relays.ts @@ -10,7 +10,9 @@ import { EventRelay } from '../models/index.js'; import { EventRelayManager } from '../event-relays/EventRelayManager.js'; import { EventRelayInput, validateEventRelayInput } from '../event-relays/validation.js'; import { EventRelayValidationError } from '../event-relays/validationError.js'; +import { renderMessageTemplate } from '../event-relays/template.js'; import { buildSearchQuery, parsePagination } from '../event-relays/search.js'; +import { assertJsonPayloadSize } from '../event-relays/process.js'; export class EventRelayAdmin { constructor(private readonly manager: EventRelayManager) {} @@ -52,6 +54,23 @@ export class EventRelayAdmin { return relay; } + async previewEventRelay(call: ParsedRouterRequest): Promise { + const { messageTemplate, samplePayload } = call.request.params as { + messageTemplate: unknown; + samplePayload: unknown; + }; + try { + assertJsonPayloadSize(samplePayload); + const rendered = renderMessageTemplate(messageTemplate, samplePayload); + return { rendered }; + } catch (err) { + if (err instanceof EventRelayValidationError) { + throw new GrpcError(status.INVALID_ARGUMENT, err.message); + } + throw err; + } + } + async patchEventRelay(call: ParsedRouterRequest): Promise { const existing = await EventRelay.getInstance().findOne({ _id: call.request.params.id, @@ -89,7 +108,18 @@ export class EventRelayAdmin { ...input, messageTemplate: input.messageTemplate as EventRelay['messageTemplate'], }); - await this.manager.notifyChanged(); + const evictRelayIds: string[] = []; + if (!input.active) { + evictRelayIds.push(existing._id); + } else if ( + input.permission !== existing.permission || + input.resourceType !== existing.resourceType + ) { + evictRelayIds.push(existing._id); + } + await this.manager.notifyChanged({ + evictRelayIds: evictRelayIds.length ? evictRelayIds : undefined, + }); return updated!; } @@ -101,7 +131,7 @@ export class EventRelayAdmin { throw new GrpcError(status.NOT_FOUND, 'Event relay not found'); } await EventRelay.getInstance().deleteOne({ _id: existing._id }); - await this.manager.notifyChanged(); + await this.manager.notifyChanged({ evictRelayIds: [existing._id] }); return { message: 'Event relay deleted' }; } } diff --git a/modules/router/src/admin/index.ts b/modules/router/src/admin/index.ts index 0c3231378..e943c25fa 100644 --- a/modules/router/src/admin/index.ts +++ b/modules/router/src/admin/index.ts @@ -242,6 +242,21 @@ export class AdminHandlers { }), this.eventRelayAdmin.deleteEventRelay.bind(this.eventRelayAdmin), ); + this.routingManager.route( + { + path: '/event-relays/preview', + action: ConduitRouteActions.POST, + description: `Renders a relay message template against sample bus JSON (same engine as runtime).`, + bodyParams: { + messageTemplate: ConduitJson.Required, + samplePayload: ConduitJson.Required, + }, + }, + new ConduitRouteReturnDefinition('PreviewEventRelay', { + rendered: ConduitJson.Required, + }), + this.eventRelayAdmin.previewEventRelay.bind(this.eventRelayAdmin), + ); this.routingManager.registerRoutes(); } } diff --git a/modules/router/src/event-relays/EventRelayManager.ts b/modules/router/src/event-relays/EventRelayManager.ts index 80cb809b7..905b0180e 100644 --- a/modules/router/src/event-relays/EventRelayManager.ts +++ b/modules/router/src/event-relays/EventRelayManager.ts @@ -3,70 +3,170 @@ import { EventRelay } from '../models/index.js'; import { EVENT_RELAY_REFRESH_CHANNEL, EVENT_RELAY_SUBSCRIBER_PREFIX, + RECONCILE_INTERVAL_MS, + RELAY_REBAC_TTL_MS, } from './constants.js'; import { groupRelaysByChannel, planChannelSubscriptions } from './channels.js'; -import { buildRelayEmissions, parseBusPayload } from './process.js'; +import { parseBusPayload } from './process.js'; import { EventRelayPusher } from './push.js'; +import { compileRelay, CompiledRelay } from './compile.js'; +import { checkRebacBatch, RelayRebacCache } from './rebacCache.js'; +import { eventRelayRoomPrefix } from './rooms.js'; +import { removeSubscriptionsForRelay } from './subscriptions.js'; export type { EventRelayPusher } from './push.js'; export { createEventRelayPusher } from './push.js'; +export type EventRelaySocketAccess = { + getLocalRoomUserIds: (room: string) => Promise; + getLocalRoomsWithPrefix: (prefix: string) => Promise; +}; + +type RefreshPayload = { + evictRelayIds?: string[]; +}; + export class EventRelayManager { - private readonly relaysByChannel = new Map(); + private readonly relaysById = new Map(); + private readonly relaysByChannel = new Map(); private readonly subscribedChannels = new Set(); + private readonly rebacCache = new RelayRebacCache(RELAY_REBAC_TTL_MS); private started = false; + private reconcilePending = false; + private reconciling = false; + private reconcileTimer?: NodeJS.Timeout; constructor( private readonly grpcSdk: ConduitGrpcSdk, private readonly push: EventRelayPusher, + private readonly sockets: EventRelaySocketAccess, ) {} async start(): Promise { if (!this.started) { this.grpcSdk.bus?.subscribe( EVENT_RELAY_REFRESH_CHANNEL, - () => { - this.reconcile().catch(err => { - ConduitGrpcSdk.Logger.error(err as Error); - }); + message => { + void this.onRefreshMessage(message); }, 'router-event-relays-refresh', ); + this.reconcileTimer = setInterval(() => { + void this.reconcile(); + }, RECONCILE_INTERVAL_MS); this.started = true; } await this.reconcile(); } async stop(): Promise { + if (this.reconcileTimer) { + clearInterval(this.reconcileTimer); + this.reconcileTimer = undefined; + } for (const channel of [...this.subscribedChannels]) { this.grpcSdk.bus?.unsubscribe(`${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`); this.subscribedChannels.delete(channel); } this.relaysByChannel.clear(); + this.relaysById.clear(); + this.rebacCache.clear(); if (this.started) { this.grpcSdk.bus?.unsubscribe('router-event-relays-refresh'); this.started = false; } + ConduitGrpcSdk.Metrics?.set('event_relays_active_total', 0); + ConduitGrpcSdk.Metrics?.set('event_relays_subscribed_channels_total', 0); } - async notifyChanged(): Promise { + async notifyChanged(options?: { evictRelayIds?: string[] }): Promise { + if (options?.evictRelayIds?.length) { + for (const relayId of options.evictRelayIds) { + await this.evictRelayRooms(relayId); + } + } if (this.started) { await this.reconcile(); } - this.grpcSdk.bus?.publish(EVENT_RELAY_REFRESH_CHANNEL, ''); + const payload: RefreshPayload = { + evictRelayIds: options?.evictRelayIds ?? [], + }; + this.grpcSdk.bus?.publish(EVENT_RELAY_REFRESH_CHANNEL, JSON.stringify(payload)); + } + + async evictRelayRooms(relayId: string): Promise { + const prefix = eventRelayRoomPrefix(relayId); + const rooms = await this.sockets.getLocalRoomsWithPrefix(prefix); + if (rooms.length === 0) { + removeSubscriptionsForRelay(relayId); + return; + } + await this.push('leave-room', undefined, rooms); + removeSubscriptionsForRelay(relayId); + } + + getActiveRelay(id: string): EventRelay | undefined { + return this.relaysById.get(id); + } + + private async onRefreshMessage(raw: string): Promise { + let payload: RefreshPayload = {}; + if (raw.trim()) { + try { + payload = JSON.parse(raw) as RefreshPayload; + } catch { + payload = {}; + } + } + if (payload.evictRelayIds?.length) { + for (const relayId of payload.evictRelayIds) { + await this.evictRelayRooms(relayId); + } + } + await this.reconcile(); } async reconcile(): Promise { + this.reconcilePending = true; + if (this.reconciling) { + return; + } + this.reconciling = true; + try { + while (this.reconcilePending) { + this.reconcilePending = false; + await this.runReconcile(); + } + } finally { + this.reconciling = false; + } + } + + private async runReconcile(): Promise { + const previousIds = new Set(this.relaysById.keys()); const relays = await EventRelay.getInstance().findMany({ active: true }); - const next = groupRelaysByChannel(relays); + const nextByChannel = groupRelaysByChannel(relays); const { toSubscribe, toUnsubscribe } = planChannelSubscriptions( this.subscribedChannels, - next.keys(), + nextByChannel.keys(), ); + this.relaysById.clear(); this.relaysByChannel.clear(); - for (const [channel, list] of next) { - this.relaysByChannel.set(channel, list); + for (const relay of relays) { + this.relaysById.set(relay._id, relay); + } + for (const [channel, channelRelays] of nextByChannel) { + this.relaysByChannel.set( + channel, + channelRelays.map(relay => compileRelay(relay)), + ); + } + + for (const id of previousIds) { + if (!this.relaysById.has(id)) { + await this.evictRelayRooms(id); + } } for (const channel of toUnsubscribe) { @@ -75,21 +175,27 @@ export class EventRelayManager { } for (const channel of toSubscribe) { - this.grpcSdk.bus?.subscribe( - channel, - message => this.onBusMessage(channel, message), - `${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`, - ); - this.subscribedChannels.add(channel); + try { + await this.grpcSdk.bus?.subscribeAck( + channel, + message => this.onBusMessage(channel, message), + `${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`, + ); + this.subscribedChannels.add(channel); + } catch (err) { + ConduitGrpcSdk.Logger.error( + `Event relay failed to subscribe to bus channel ${channel}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } } - } - getActiveRelay(id: string): EventRelay | undefined { - for (const relays of this.relaysByChannel.values()) { - const match = relays.find(relay => relay._id === id); - if (match) return match; - } - return undefined; + ConduitGrpcSdk.Metrics?.set('event_relays_active_total', this.relaysById.size); + ConduitGrpcSdk.Metrics?.set( + 'event_relays_subscribed_channels_total', + this.subscribedChannels.size, + ); } private onBusMessage(channel: string, message: string): void { @@ -103,6 +209,13 @@ export class EventRelayManager { payload = parseBusPayload(message); } catch (err) { ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); + if ( + err instanceof Error && + err.message.includes('exceeds') && + err.message.includes('bytes') + ) { + ConduitGrpcSdk.Metrics?.increment('event_relays_inbound_dropped_total'); + } ConduitGrpcSdk.Logger.error( `Event relay failed to parse payload for ${channel}: ${ err instanceof Error ? err.message : String(err) @@ -111,27 +224,74 @@ export class EventRelayManager { return; } - const { emissions, failures } = buildRelayEmissions(relays, payload); - for (const failure of failures) { + for (const relay of relays) { + void this.emitCompiledRelay(relay, payload, channel); + } + } + + private async emitCompiledRelay( + relay: CompiledRelay, + payload: unknown, + channel: string, + ): Promise { + let room: string; + let data: unknown; + let resourceId: string; + try { + ({ room, data, resourceId } = relay.buildEmission(payload)); + } catch (err) { ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); ConduitGrpcSdk.Logger.warn( - `Event relay ${failure.relayId} skipped on ${failure.busEvent}: ${failure.reason}`, + `Event relay ${relay.relayId} skipped on ${relay.busEvent}: ${ + err instanceof Error ? err.message : String(err) + }`, ); + return; } - for (const emission of emissions) { - this.push(emission.socketEvent, emission.data, [emission.room]).then( - () => { - ConduitGrpcSdk.Metrics?.increment('event_relays_emitted_total'); - }, - err => { - ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); - ConduitGrpcSdk.Logger.error( - `Event relay ${emission.relayId} emit failed on ${channel}: ${ - err instanceof Error ? err.message : String(err) - }`, - ); - }, + const userIds = await this.sockets.getLocalRoomUserIds(room); + if (userIds.length === 0) { + ConduitGrpcSdk.Metrics?.increment('event_relays_empty_room_total'); + return; + } + + const decisions = await checkRebacBatch( + this.rebacCache, + this.grpcSdk, + userIds, + relay.permission, + relay.resourceType, + resourceId, + ); + + const allowedUsers: string[] = []; + for (const userId of userIds) { + const decision = decisions.get(userId) ?? 'unavailable'; + if (decision === 'allow') { + allowedUsers.push(userId); + } else if (decision === 'deny') { + await this.push('leave-room', undefined, [room], [userId]); + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + } + } + + if (allowedUsers.length === 0) { + return; + } + + try { + const emitted = await this.push(relay.socketEvent, data, [room], allowedUsers); + if (emitted) { + ConduitGrpcSdk.Metrics?.increment('event_relays_emitted_total'); + } else { + ConduitGrpcSdk.Metrics?.increment('event_relays_empty_room_total'); + } + } catch (err) { + ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); + ConduitGrpcSdk.Logger.error( + `Event relay ${relay.relayId} emit failed on ${channel}: ${ + err instanceof Error ? err.message : String(err) + }`, ); } } diff --git a/modules/router/src/event-relays/EventRelaySockets.ts b/modules/router/src/event-relays/EventRelaySockets.ts index 32e5fec2a..d9fe6f88c 100644 --- a/modules/router/src/event-relays/EventRelaySockets.ts +++ b/modules/router/src/event-relays/EventRelaySockets.ts @@ -1,11 +1,27 @@ import { status } from '@grpc/grpc-js'; import { ConduitGrpcSdk, GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; import { ConduitSocket, ConduitSocketEvent } from '@conduitplatform/hermes'; -import { EVENTS_NAMESPACE } from './constants.js'; +import { + EVENTS_NAMESPACE, + MAX_SUBSCRIBE_PER_MINUTE, +} from './constants.js'; import { EventRelayManager } from './EventRelayManager.js'; import { eventRelayRoom } from './rooms.js'; import { validateResourceId } from './validation.js'; import { authorizeRelaySubscription, toSubscriptionError } from './authorize.js'; +import { reauthorizeRecoveredSubscriptions } from './recovery.js'; +import { + releaseSocketSubscriptions, + removeSubscription, + trackSubscription, + _clearEventRelaySubscriptionStateForTests, +} from './subscriptions.js'; +import { + persistRelaySubscriptionOnContext, + removeRelaySubscriptionFromContext, +} from './relaySocketData.js'; + +const subscribeTimestamps = new Map(); export function createEventsSocket( grpcSdk: ConduitGrpcSdk, @@ -20,16 +36,40 @@ export function createEventsSocket( events.set('disconnect', { name: 'disconnect', - handler: async () => ({ event: 'leave-room', rooms: [] }), + handler: async request => { + const userId = request.context?.user?._id as string | undefined; + releaseSocketSubscriptions(request.socketId, userId); + subscribeTimestamps.delete(request.socketId); + return { event: 'leave-room', rooms: [] }; + }, }); events.set('subscribe', { name: 'subscribe', params: [TYPE.String, TYPE.String], handler: async request => { + assertSubscribeRateLimit(request.socketId); const userId = request.context?.user?._id as string | undefined; + if (!userId) { + throw new GrpcError(status.UNAUTHENTICATED, 'Authentication required'); + } const [relayId, resourceId] = request.params ?? []; + const validatedResourceId = validateResourceId(resourceId); const room = await authorizeOrThrow(grpcSdk, manager, userId, relayId, resourceId); + try { + trackSubscription( + request.socketId, + userId, + String(relayId), + validatedResourceId, + ); + persistRelaySubscriptionOnContext(request.context, String(relayId), validatedResourceId); + } catch (err) { + throw new GrpcError( + status.RESOURCE_EXHAUSTED, + err instanceof Error ? err.message : 'Subscription limit exceeded', + ); + } return { event: 'join-room', rooms: [room] }; }, }); @@ -38,12 +78,15 @@ export function createEventsSocket( name: 'unsubscribe', params: [TYPE.String, TYPE.String], handler: async request => { + const userId = request.context?.user?._id as string | undefined; const [relayId, resourceId] = request.params ?? []; if (typeof relayId !== 'string' || relayId.trim() === '') { throw new GrpcError(status.INVALID_ARGUMENT, 'Relay ID is required'); } try { const validatedResourceId = validateResourceId(resourceId); + removeSubscription(request.socketId, userId, relayId, validatedResourceId); + removeRelaySubscriptionFromContext(request.context, relayId, validatedResourceId); return { event: 'leave-room', rooms: [eventRelayRoom(relayId, validatedResourceId)], @@ -60,11 +103,26 @@ export function createEventsSocket( name: 'eventRelays', description: 'Declarative bus-to-socket event relays', middlewares: ['authMiddleware'], + onRecovered: async request => + reauthorizeRecoveredSubscriptions(grpcSdk, manager, request), }, events, ); } +function assertSubscribeRateLimit(socketId: string): void { + const now = Date.now(); + const windowStart = now - 60_000; + const timestamps = (subscribeTimestamps.get(socketId) ?? []).filter( + t => t >= windowStart, + ); + if (timestamps.length >= MAX_SUBSCRIBE_PER_MINUTE) { + throw new GrpcError(status.RESOURCE_EXHAUSTED, 'Subscribe rate limit exceeded'); + } + timestamps.push(now); + subscribeTimestamps.set(socketId, timestamps); +} + async function authorizeOrThrow( grpcSdk: ConduitGrpcSdk, manager: EventRelayManager, @@ -94,3 +152,9 @@ function toGrpcError(err: unknown): GrpcError { } export { authorizeRelaySubscription } from './authorize.js'; + +/** @internal test helper */ +export function _clearEventRelaySocketStateForTests(): void { + _clearEventRelaySubscriptionStateForTests(); + subscribeTimestamps.clear(); +} diff --git a/modules/router/src/event-relays/authorize.ts b/modules/router/src/event-relays/authorize.ts index 852cf5595..8de7cc005 100644 --- a/modules/router/src/event-relays/authorize.ts +++ b/modules/router/src/event-relays/authorize.ts @@ -62,7 +62,6 @@ export async function authorizeRelaySubscription( } if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { - onDenied?.(); throw new RelaySubscriptionError(status.UNAVAILABLE, 'Authorization is unavailable'); } @@ -75,7 +74,6 @@ export async function authorizeRelaySubscription( }); allowed = decision.allow; } catch { - onDenied?.(); throw new RelaySubscriptionError(status.UNAVAILABLE, 'Authorization check failed'); } diff --git a/modules/router/src/event-relays/compile.ts b/modules/router/src/event-relays/compile.ts new file mode 100644 index 000000000..bfe636592 --- /dev/null +++ b/modules/router/src/event-relays/compile.ts @@ -0,0 +1,50 @@ +import { requireOwnPath } from './path.js'; +import { eventRelayRoom } from './rooms.js'; +import { renderMessageTemplate } from './template.js'; +import { validateResourceId } from './validation.js'; + +export type RelayCompileInput = { + _id: string; + busEvent: string; + socketEvent: string; + resourceIdPath: string; + messageTemplate: unknown; + permission: string; + resourceType: string; +}; + +export type CompiledRelay = { + relayId: string; + busEvent: string; + socketEvent: string; + permission: string; + resourceType: string; + buildEmission: (payload: unknown) => { + room: string; + data: unknown; + resourceId: string; + }; +}; + +export function compileRelay(relay: RelayCompileInput): CompiledRelay { + const resourceIdPath = relay.resourceIdPath; + const template = relay.messageTemplate; + return { + relayId: relay._id, + busEvent: relay.busEvent, + socketEvent: relay.socketEvent, + permission: relay.permission, + resourceType: relay.resourceType, + buildEmission: (payload: unknown) => { + const resourceId = validateResourceId( + requireOwnPath(payload, resourceIdPath, 'Resource ID path'), + ); + const data = renderMessageTemplate(template, payload); + return { + room: eventRelayRoom(relay._id, resourceId), + data, + resourceId, + }; + }, + }; +} diff --git a/modules/router/src/event-relays/constants.ts b/modules/router/src/event-relays/constants.ts index e283da0dd..ce80f6194 100644 --- a/modules/router/src/event-relays/constants.ts +++ b/modules/router/src/event-relays/constants.ts @@ -15,6 +15,12 @@ export const MAX_RESOURCE_ID_LENGTH = 128; export const MAX_RESOURCE_ID_PATH_LENGTH = 128; export const MAX_PERMISSION_LENGTH = 64; +export const MAX_INBOUND_BUS_BYTES = 256 * 1024; +export const RECONCILE_INTERVAL_MS = 30_000; +export const MAX_SUBSCRIBE_PER_MINUTE = 30; +export const MAX_ROOMS_PER_SOCKET = 32; +export const RELAY_REBAC_TTL_MS = 12_000; + export const FORBIDDEN_PATH_SEGMENTS = new Set(['__proto__', 'constructor', 'prototype']); export const RESERVED_SOCKET_EVENTS = new Set([ diff --git a/modules/router/src/event-relays/follow-up.test.ts b/modules/router/src/event-relays/follow-up.test.ts new file mode 100644 index 000000000..fea6d438e --- /dev/null +++ b/modules/router/src/event-relays/follow-up.test.ts @@ -0,0 +1,186 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { + releaseSocketSubscriptions, + removeSubscriptionsForRelay, + subscriptionsForRecoveredRooms, + subscriptionsForUser, + trackSubscription, + _clearEventRelaySubscriptionStateForTests, +} from './subscriptions.js'; +import { eventRelayRoom } from './rooms.js'; +import { reauthorizeRecoveredSubscriptions } from './recovery.js'; +import { RelayLookup } from './authorize.js'; + +describe('event relay subscription tracking', () => { + it('clears user map entries when the only socket disconnects', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-a', 'user-1'); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); + + it('keeps user map entries when another socket still holds the subscription', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-1', 'relay-1', 'order-1'); + trackSubscription('socket-b', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-a', 'user-1'); + assert.deepEqual(subscriptionsForUser('user-1'), [ + { relayId: 'relay-1', resourceId: 'order-1' }, + ]); + }); + + it('clears relay subscriptions on eviction', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-1', 'relay-1', 'order-1'); + removeSubscriptionsForRelay('relay-1'); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); +}); + +describe('recovery re-authorization', () => { + const room = eventRelayRoom('relay-1', 'order-1'); + const relay = { + _id: 'relay-1', + permission: 'read', + resourceType: 'Order', + }; + + it('keeps restored rooms when Authorization is unavailable', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('old-socket', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('old-socket', 'user-1'); + + const manager = { + getActiveRelay: (id: string) => (id === 'relay-1' ? relay : undefined), + } as unknown as RelayLookup; + + const grpcSdk = { + isAvailable: () => false, + authorization: null, + }; + + const result = await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'new-socket', + context: { + user: { _id: 'user-1' }, + eventRelaySubs: [{ relayId: 'relay-1', resourceId: 'order-1' }], + }, + recoveredRooms: [room], + }); + + assert.deepEqual(result, { event: 'join-room', rooms: [] }); + assert.deepEqual(subscriptionsForUser('user-1'), [ + { relayId: 'relay-1', resourceId: 'order-1' }, + ]); + }); + + it('leaves restored rooms on permission deny', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('old-socket', 'user-1', 'relay-1', 'order-1'); + releaseSocketSubscriptions('old-socket', 'user-1'); + + const manager = { + getActiveRelay: (id: string) => (id === 'relay-1' ? relay : undefined), + } as unknown as RelayLookup; + + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async () => ({ allow: false }), + }, + }; + + const result = await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'new-socket', + context: { + user: { _id: 'user-1' }, + eventRelaySubs: [{ relayId: 'relay-1', resourceId: 'order-1' }], + }, + recoveredRooms: [room], + }); + + assert.deepEqual(result, { event: 'leave-room', rooms: [room] }); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); + + it('does not re-auth rooms outside the recovered socket session', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('old-socket', 'user-1', 'relay-1', 'order-1'); + trackSubscription('old-socket', 'user-1', 'relay-2', 'order-2'); + releaseSocketSubscriptions('old-socket', 'user-1'); + + const manager = { + getActiveRelay: (id: string) => + id === 'relay-2' + ? { _id: 'relay-2', permission: 'read', resourceType: 'Order' } + : undefined, + } as unknown as RelayLookup; + + let canCalls = 0; + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async () => { + canCalls++; + return { allow: false }; + }, + }, + }; + + const otherRoom = eventRelayRoom('relay-2', 'order-2'); + await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'new-socket', + context: { + user: { _id: 'user-1' }, + eventRelaySubs: [{ relayId: 'relay-2', resourceId: 'order-2' }], + }, + recoveredRooms: [otherRoom], + }); + + assert.equal(canCalls, 1); + assert.deepEqual(subscriptionsForUser('user-1'), []); + }); + + it('re-auths the first user after a second user subscribed to the same room', async () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-a', 'relay-1', 'order-1'); + trackSubscription('socket-b', 'user-b', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-b', 'user-b'); + + let checkedSubject = ''; + const manager = { + getActiveRelay: (id: string) => (id === 'relay-1' ? relay : undefined), + } as unknown as RelayLookup; + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async (request: { subject: string }) => { + checkedSubject = request.subject; + return { allow: true }; + }, + }, + }; + + await reauthorizeRecoveredSubscriptions(grpcSdk, manager, { + socketId: 'socket-a-new', + context: { user: { _id: 'user-a' } }, + recoveredRooms: [room], + }); + + assert.equal(checkedSubject, 'User:user-a'); + }); + + it('prefers socket.data subscriptions when the room map was pruned on disconnect', () => { + _clearEventRelaySubscriptionStateForTests(); + trackSubscription('socket-a', 'user-a', 'relay-1', 'order-1'); + releaseSocketSubscriptions('socket-a', 'user-a'); + assert.deepEqual(subscriptionsForRecoveredRooms([room]), []); + assert.deepEqual( + subscriptionsForRecoveredRooms([room], [ + { relayId: 'relay-1', resourceId: 'order-1' }, + ]), + [{ relayId: 'relay-1', resourceId: 'order-1' }], + ); + }); +}); diff --git a/modules/router/src/event-relays/interventions.test.ts b/modules/router/src/event-relays/interventions.test.ts new file mode 100644 index 000000000..f9e37f644 --- /dev/null +++ b/modules/router/src/event-relays/interventions.test.ts @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { MAX_INBOUND_BUS_BYTES } from './constants.js'; +import { parseBusPayload } from './process.js'; +import { renderMessageTemplate } from './template.js'; + +describe('parseBusPayload inbound cap', () => { + it('drops payloads larger than 256KiB before JSON.parse', () => { + const oversized = JSON.stringify({ blob: 'x'.repeat(MAX_INBOUND_BUS_BYTES) }); + assert.throws(() => parseBusPayload(oversized), /exceeds/); + }); +}); + +describe('preview renderer parity', () => { + it('renders templates the same as runtime processing', () => { + const payload = { documentId: 'doc-1', nested: { value: 2 } }; + const template = { id: '{{payload.documentId}}', n: '{{payload.nested.value}}' }; + assert.deepEqual(renderMessageTemplate(template, payload), { + id: 'doc-1', + n: 2, + }); + }); +}); diff --git a/modules/router/src/event-relays/process.test.ts b/modules/router/src/event-relays/process.test.ts index cf3c1c10b..c811bcec5 100644 --- a/modules/router/src/event-relays/process.test.ts +++ b/modules/router/src/event-relays/process.test.ts @@ -69,6 +69,7 @@ describe('createEventRelayPusher', () => { const calls: unknown[] = []; const push = createEventRelayPusher(async data => { calls.push(data); + return true; }); await push('order-updated', { id: '1' }, ['room-1']); assert.deepEqual(calls, [ @@ -79,6 +80,8 @@ describe('createEventRelayPusher', () => { rooms: ['room-1'], namespace: EVENTS_NAMESPACE, localOnly: true, + skipEmptyRooms: true, + boundedEmit: true, }, ]); }); diff --git a/modules/router/src/event-relays/process.ts b/modules/router/src/event-relays/process.ts index 858c9f6d0..ebe6ec1ec 100644 --- a/modules/router/src/event-relays/process.ts +++ b/modules/router/src/event-relays/process.ts @@ -3,6 +3,7 @@ import { eventRelayRoom } from './rooms.js'; import { renderMessageTemplate } from './template.js'; import { validateResourceId } from './validation.js'; import { EventRelayValidationError } from './validationError.js'; +import { MAX_INBOUND_BUS_BYTES } from './constants.js'; export type RelayProcessInput = { _id: string; @@ -31,10 +32,16 @@ export type ProcessResult = { failures: RelayFailure[]; }; -export function parseBusPayload(rawMessage: string): unknown { +export function parseBusPayload( + rawMessage: string, + maxBytes: number = MAX_INBOUND_BUS_BYTES, +): unknown { if (typeof rawMessage !== 'string' || rawMessage.trim() === '') { throw new EventRelayValidationError('Bus payload is empty'); } + if (Buffer.byteLength(rawMessage, 'utf8') > maxBytes) { + throw new EventRelayValidationError(`Bus payload exceeds ${maxBytes} bytes`); + } try { return JSON.parse(rawMessage); } catch { @@ -42,6 +49,21 @@ export function parseBusPayload(rawMessage: string): unknown { } } +export function assertJsonPayloadSize( + payload: unknown, + maxBytes: number = MAX_INBOUND_BUS_BYTES, +): void { + let serialized: string; + try { + serialized = JSON.stringify(payload); + } catch { + throw new EventRelayValidationError('Sample payload must be valid JSON'); + } + if (Buffer.byteLength(serialized, 'utf8') > maxBytes) { + throw new EventRelayValidationError(`Sample payload exceeds ${maxBytes} bytes`); + } +} + export function buildRelayEmissions( relays: RelayProcessInput[], payload: unknown, diff --git a/modules/router/src/event-relays/push.ts b/modules/router/src/event-relays/push.ts index 79d9911b2..e4bed5062 100644 --- a/modules/router/src/event-relays/push.ts +++ b/modules/router/src/event-relays/push.ts @@ -4,7 +4,8 @@ export type EventRelayPusher = ( event: string, data: unknown, rooms: string[], -) => Promise; + receivers?: string[], +) => Promise; export type SocketPushFn = (data: { event: string; @@ -13,16 +14,20 @@ export type SocketPushFn = (data: { rooms: string[]; namespace: string; localOnly?: boolean; -}) => Promise; + skipEmptyRooms?: boolean; + boundedEmit?: boolean; +}) => Promise; export function createEventRelayPusher(socketPush: SocketPushFn): EventRelayPusher { - return (event, data, rooms) => + return async (event, data, rooms, receivers = []) => socketPush({ event, data, - receivers: [], + receivers, rooms, namespace: EVENTS_NAMESPACE, localOnly: true, + skipEmptyRooms: event !== 'leave-room', + boundedEmit: event !== 'leave-room', }); } diff --git a/modules/router/src/event-relays/rebacCache.test.ts b/modules/router/src/event-relays/rebacCache.test.ts new file mode 100644 index 000000000..df20365d5 --- /dev/null +++ b/modules/router/src/event-relays/rebacCache.test.ts @@ -0,0 +1,43 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { RelayRebacCache } from './rebacCache.js'; + +describe('RelayRebacCache', () => { + it('denies after TTL when authorization revokes access', async () => { + let allow = true; + const cache = new RelayRebacCache(10); + const grpcSdk = { + isAvailable: () => true, + authorization: { + can: async () => ({ allow }), + }, + }; + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'allow', + ); + allow = false; + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'allow', + ); + await new Promise(resolve => setTimeout(resolve, 15)); + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'deny', + ); + }); + + it('returns unavailable without caching when Authorization is down', async () => { + const cache = new RelayRebacCache(10_000); + const grpcSdk = { isAvailable: () => false, authorization: null }; + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'unavailable', + ); + assert.equal( + await cache.check(grpcSdk as never, 'user-1', 'read', 'Order', 'order-1'), + 'unavailable', + ); + }); +}); diff --git a/modules/router/src/event-relays/rebacCache.ts b/modules/router/src/event-relays/rebacCache.ts new file mode 100644 index 000000000..a2d458300 --- /dev/null +++ b/modules/router/src/event-relays/rebacCache.ts @@ -0,0 +1,108 @@ +type RebacSdk = { + isAvailable: (module: string) => boolean; + authorization?: { + can: (request: { + subject: string; + actions: string[]; + resource: string; + }) => Promise<{ allow: boolean }>; + } | null; +}; + +export type RebacDecision = 'allow' | 'deny' | 'unavailable'; + +const DEFAULT_TTL_MS = 12_000; +const DEFAULT_MAX_ENTRIES = 10_000; + +type CacheEntry = { + decision: 'allow' | 'deny'; + expiresAt: number; +}; + +export class RelayRebacCache { + private readonly entries = new Map(); + constructor( + private readonly ttlMs: number = DEFAULT_TTL_MS, + private readonly maxEntries: number = DEFAULT_MAX_ENTRIES, + ) {} + + async check( + grpcSdk: RebacSdk, + userId: string, + permission: string, + resourceType: string, + resourceId: string, + ): Promise { + const resource = `${resourceType}:${resourceId}`; + const key = `${userId}:${permission}:${resource}`; + const now = Date.now(); + this.sweepExpired(now); + const cached = this.entries.get(key); + if (cached && cached.expiresAt > now) { + return cached.decision; + } + if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { + return 'unavailable'; + } + try { + const decision = await grpcSdk.authorization.can({ + subject: `User:${userId}`, + actions: [permission], + resource, + }); + this.set(key, decision.allow ? 'allow' : 'deny', now); + return decision.allow ? 'allow' : 'deny'; + } catch { + return 'unavailable'; + } + } + + clear(): void { + this.entries.clear(); + } + + private set(key: string, decision: 'allow' | 'deny', now: number): void { + if (this.entries.size >= this.maxEntries && !this.entries.has(key)) { + const firstKey = this.entries.keys().next().value; + if (firstKey) { + this.entries.delete(firstKey); + } + } + this.entries.set(key, { decision, expiresAt: now + this.ttlMs }); + } + + private sweepExpired(now: number): void { + for (const [key, entry] of this.entries) { + if (entry.expiresAt <= now) { + this.entries.delete(key); + } + } + } +} + +export async function checkRebacBatch( + cache: RelayRebacCache, + grpcSdk: RebacSdk, + userIds: string[], + permission: string, + resourceType: string, + resourceId: string, + concurrency = 8, +): Promise> { + const results = new Map(); + let index = 0; + async function worker(): Promise { + while (index < userIds.length) { + const userId = userIds[index++]; + results.set( + userId, + await cache.check(grpcSdk, userId, permission, resourceType, resourceId), + ); + } + } + const workers = Array.from({ length: Math.min(concurrency, userIds.length) }, () => + worker(), + ); + await Promise.all(workers); + return results; +} diff --git a/modules/router/src/event-relays/recovery.ts b/modules/router/src/event-relays/recovery.ts new file mode 100644 index 000000000..ceb6c4217 --- /dev/null +++ b/modules/router/src/event-relays/recovery.ts @@ -0,0 +1,85 @@ +import { status } from '@grpc/grpc-js'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { + authorizeRelaySubscription, + RelaySubscriptionError, + RelayLookup, + toSubscriptionError, +} from './authorize.js'; +import { eventRelayRoom } from './rooms.js'; +import { + removeSubscription, + subscriptionsForRecoveredRooms, + trackSubscription, +} from './subscriptions.js'; +import { relaySubscriptionsFromContext, persistRelaySubscriptionOnContext, removeRelaySubscriptionFromContext } from './relaySocketData.js'; + +type RelayAuthorizationSdk = Parameters[0]; + +export async function reauthorizeRecoveredSubscriptions( + grpcSdk: RelayAuthorizationSdk, + manager: RelayLookup, + request: { + socketId: string; + context?: { user?: { _id?: string } } & Record; + recoveredRooms?: string[]; + }, +): Promise< + { event: 'join-room'; rooms: string[] } | { event: 'leave-room'; rooms: string[] } +> { + const userId = request.context?.user?._id as string | undefined; + if (!userId) { + return { event: 'leave-room', rooms: [] }; + } + + const recoveredRooms = request.recoveredRooms ?? []; + const subs = subscriptionsForRecoveredRooms( + recoveredRooms, + relaySubscriptionsFromContext(request.context), + ); + const leaveRooms: string[] = []; + + for (const sub of subs) { + const room = eventRelayRoom(sub.relayId, sub.resourceId); + try { + await authorizeRelaySubscription( + grpcSdk, + manager, + userId, + sub.relayId, + sub.resourceId, + () => { + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + }, + ); + trackSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + persistRelaySubscriptionOnContext(request.context, sub.relayId, sub.resourceId); + } catch (err) { + const mapped = + err instanceof RelaySubscriptionError ? err : toSubscriptionError(err); + if (mapped.code === status.UNAVAILABLE) { + trackSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + persistRelaySubscriptionOnContext(request.context, sub.relayId, sub.resourceId); + continue; + } + if ( + mapped.code === status.PERMISSION_DENIED || + mapped.code === status.NOT_FOUND + ) { + removeSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + removeRelaySubscriptionFromContext(request.context, sub.relayId, sub.resourceId); + leaveRooms.push(room); + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + continue; + } + removeSubscription(request.socketId, userId, sub.relayId, sub.resourceId); + removeRelaySubscriptionFromContext(request.context, sub.relayId, sub.resourceId); + leaveRooms.push(room); + } + } + + if (leaveRooms.length > 0) { + return { event: 'leave-room', rooms: leaveRooms }; + } + return { event: 'join-room', rooms: [] }; +} diff --git a/modules/router/src/event-relays/relaySocketData.ts b/modules/router/src/event-relays/relaySocketData.ts new file mode 100644 index 000000000..63d0eca9c --- /dev/null +++ b/modules/router/src/event-relays/relaySocketData.ts @@ -0,0 +1,65 @@ +import { Indexable } from '@conduitplatform/grpc-sdk'; +import { RelaySubscription } from './subscriptions.js'; + +const CONTEXT_KEY = 'eventRelaySubs'; + +export function relaySubscriptionsFromContext( + context: Indexable | undefined, +): RelaySubscription[] { + if (!context) { + return []; + } + const raw = context[CONTEXT_KEY]; + if (!Array.isArray(raw)) { + return []; + } + const subs: RelaySubscription[] = []; + for (const item of raw) { + if ( + item && + typeof item === 'object' && + typeof (item as RelaySubscription).relayId === 'string' && + typeof (item as RelaySubscription).resourceId === 'string' + ) { + subs.push({ + relayId: (item as RelaySubscription).relayId, + resourceId: (item as RelaySubscription).resourceId, + }); + } + } + return subs; +} + +export function persistRelaySubscriptionOnContext( + context: Indexable | undefined, + relayId: string, + resourceId: string, +): void { + if (!context) { + return; + } + const subs = relaySubscriptionsFromContext(context); + const withoutDup = subs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + withoutDup.push({ relayId, resourceId }); + context[CONTEXT_KEY] = withoutDup; +} + +export function removeRelaySubscriptionFromContext( + context: Indexable | undefined, + relayId: string, + resourceId: string, +): void { + if (!context) { + return; + } + const next = relaySubscriptionsFromContext(context).filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + if (next.length === 0) { + delete context[CONTEXT_KEY]; + } else { + context[CONTEXT_KEY] = next; + } +} diff --git a/modules/router/src/event-relays/rooms.ts b/modules/router/src/event-relays/rooms.ts index fd5dd54ec..0bf26bf74 100644 --- a/modules/router/src/event-relays/rooms.ts +++ b/modules/router/src/event-relays/rooms.ts @@ -4,3 +4,7 @@ export function eventRelayRoom(relayId: string, resourceId: string): string { const digest = createHash('sha256').update(resourceId).digest('hex'); return `er:${relayId}:${digest}`; } + +export function eventRelayRoomPrefix(relayId: string): string { + return `er:${relayId}:`; +} diff --git a/modules/router/src/event-relays/subscriptions.ts b/modules/router/src/event-relays/subscriptions.ts new file mode 100644 index 000000000..a7a083f56 --- /dev/null +++ b/modules/router/src/event-relays/subscriptions.ts @@ -0,0 +1,224 @@ +import { eventRelayRoom } from './rooms.js'; +import { MAX_ROOMS_PER_SOCKET } from './constants.js'; + +export type RelaySubscription = { + relayId: string; + resourceId: string; +}; + +const RECOVERY_ROOM_MAP_TTL_MS = 120_000; + +const subscriptionsBySocket = new Map(); +const subscriptionsByUser = new Map(); +const subscriptionsByRoom = new Map(); + +function pruneExpiredRoomEntries(now = Date.now()): void { + for (const [room, entry] of subscriptionsByRoom) { + if (entry.expiresAt <= now) { + subscriptionsByRoom.delete(room); + } + } +} + +function touchRoomEntry(relayId: string, resourceId: string): void { + const room = eventRelayRoom(relayId, resourceId); + subscriptionsByRoom.set(room, { + relayId, + resourceId, + expiresAt: Date.now() + RECOVERY_ROOM_MAP_TTL_MS, + }); +} + +function pruneRoomEntryIfUnused(relayId: string, resourceId: string): void { + if (!isSubscriptionTrackedOnAnySocket(relayId, resourceId)) { + subscriptionsByRoom.delete(eventRelayRoom(relayId, resourceId)); + } +} + +export function trackSubscription( + socketId: string, + userId: string, + relayId: string, + resourceId: string, +): void { + const entry = { relayId, resourceId }; + const socketSubs = subscriptionsBySocket.get(socketId) ?? []; + const withoutDup = socketSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + withoutDup.push(entry); + if (withoutDup.length > MAX_ROOMS_PER_SOCKET) { + throw new Error(`Cannot subscribe to more than ${MAX_ROOMS_PER_SOCKET} relay rooms`); + } + subscriptionsBySocket.set(socketId, withoutDup); + + const userSubs = subscriptionsByUser.get(userId) ?? []; + const userWithoutDup = userSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + userWithoutDup.push(entry); + subscriptionsByUser.set(userId, userWithoutDup); + touchRoomEntry(relayId, resourceId); +} + +export function removeSubscription( + socketId: string, + userId: string | undefined, + relayId: string, + resourceId: string, +): void { + const socketSubs = subscriptionsBySocket.get(socketId); + if (socketSubs) { + const next = socketSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + if (next.length === 0) { + subscriptionsBySocket.delete(socketId); + } else { + subscriptionsBySocket.set(socketId, next); + } + } + if (userId && !isSubscriptionTrackedOnOtherSocket(relayId, resourceId, socketId)) { + const userSubs = subscriptionsByUser.get(userId); + if (userSubs) { + const next = userSubs.filter( + sub => !(sub.relayId === relayId && sub.resourceId === resourceId), + ); + if (next.length === 0) { + subscriptionsByUser.delete(userId); + } else { + subscriptionsByUser.set(userId, next); + } + } + pruneRoomEntryIfUnused(relayId, resourceId); + } +} + +function isSubscriptionTrackedOnOtherSocket( + relayId: string, + resourceId: string, + exceptSocketId: string, +): boolean { + for (const [socketId, subs] of subscriptionsBySocket) { + if (socketId === exceptSocketId) { + continue; + } + if (subs.some(sub => sub.relayId === relayId && sub.resourceId === resourceId)) { + return true; + } + } + return false; +} + +function isSubscriptionTrackedOnAnySocket(relayId: string, resourceId: string): boolean { + for (const subs of subscriptionsBySocket.values()) { + if (subs.some(sub => sub.relayId === relayId && sub.resourceId === resourceId)) { + return true; + } + } + return false; +} + +export function releaseSocketSubscriptions(socketId: string, userId?: string): void { + const subs = subscriptionsBySocket.get(socketId) ?? []; + subscriptionsBySocket.delete(socketId); + if (!userId) { + return; + } + for (const sub of subs) { + if (!isSubscriptionTrackedOnOtherSocket(sub.relayId, sub.resourceId, socketId)) { + const userSubs = subscriptionsByUser.get(userId); + if (!userSubs) { + continue; + } + const next = userSubs.filter( + entry => + !(entry.relayId === sub.relayId && entry.resourceId === sub.resourceId), + ); + if (next.length === 0) { + subscriptionsByUser.delete(userId); + } else { + subscriptionsByUser.set(userId, next); + } + } + pruneRoomEntryIfUnused(sub.relayId, sub.resourceId); + } +} + +export function subscriptionsForRecoveredRooms( + recoveredRooms: string[], + contextSubs: RelaySubscription[] = [], +): RelaySubscription[] { + pruneExpiredRoomEntries(); + const seen = new Set(); + const result: RelaySubscription[] = []; + const recoveredSet = new Set(recoveredRooms.filter(room => room.startsWith('er:'))); + + for (const sub of contextSubs) { + const room = eventRelayRoom(sub.relayId, sub.resourceId); + if (recoveredSet.has(room) && !seen.has(room)) { + seen.add(room); + result.push(sub); + } + } + + for (const room of recoveredSet) { + if (seen.has(room)) { + continue; + } + const entry = subscriptionsByRoom.get(room); + if (entry) { + seen.add(room); + result.push({ relayId: entry.relayId, resourceId: entry.resourceId }); + } + } + + return result; +} + +export function subscriptionsForUser(userId: string): RelaySubscription[] { + return [...(subscriptionsByUser.get(userId) ?? [])]; +} + +export function removeSubscriptionsForRelay(relayId: string): void { + for (const [socketId, subs] of subscriptionsBySocket) { + const next = subs.filter(sub => sub.relayId !== relayId); + if (next.length === 0) { + subscriptionsBySocket.delete(socketId); + } else { + subscriptionsBySocket.set(socketId, next); + } + } + for (const [userId, subs] of subscriptionsByUser) { + const next = subs.filter(sub => sub.relayId !== relayId); + if (next.length === 0) { + subscriptionsByUser.delete(userId); + } else { + subscriptionsByUser.set(userId, next); + } + } + for (const [room, entry] of subscriptionsByRoom) { + if (entry.relayId === relayId) { + subscriptionsByRoom.delete(room); + } + } +} + +export function leaveRoomsForRelay(relayId: string): string[] { + const rooms = new Set(); + for (const subs of subscriptionsByUser.values()) { + for (const sub of subs) { + if (sub.relayId === relayId) { + rooms.add(eventRelayRoom(relayId, sub.resourceId)); + } + } + } + return [...rooms]; +} + +/** @internal test helper */ +export function _clearEventRelaySubscriptionStateForTests(): void { + subscriptionsBySocket.clear(); + subscriptionsByUser.clear(); + subscriptionsByRoom.clear(); +} diff --git a/modules/router/src/event-relays/validation.ts b/modules/router/src/event-relays/validation.ts index d6c6c5eb6..c59976335 100644 --- a/modules/router/src/event-relays/validation.ts +++ b/modules/router/src/event-relays/validation.ts @@ -33,24 +33,75 @@ const PERMISSION_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,63}$/; const RESOURCE_ID_PATTERN = /^[^\s:]{1,128}$/; export function validateEventRelayInput(input: EventRelayInput): EventRelayInput { - const name = requireTrimmed(input.name, 'Name'); + const name = parseName(input.name); + const notes = parseNotes(input.notes); + const busEvent = parseBusEvent(input.busEvent); + const socketEvent = parseSocketEvent(input.socketEvent); + const resourceType = parseResourceType(input.resourceType); + const resourceIdPath = parseResourceIdPath(input.resourceIdPath); + const permission = parsePermission(input.permission); + const messageTemplate = parseMessageTemplate(input.messageTemplate); + + return { + name, + notes, + active: input.active !== false, + busEvent, + socketEvent, + resourceType, + resourceIdPath, + permission, + messageTemplate, + }; +} + +export function validateResourceId(resourceId: unknown): string { + if (typeof resourceId !== 'string' && typeof resourceId !== 'number') { + throw new EventRelayValidationError('Resource ID must be a string'); + } + const value = String(resourceId).trim(); + if ( + !value || + value.length > MAX_RESOURCE_ID_LENGTH || + !RESOURCE_ID_PATTERN.test(value) + ) { + throw new EventRelayValidationError('Resource ID is invalid'); + } + return value; +} + +function requireTrimmed(value: unknown, label: string): string { + if (typeof value !== 'string' || value.trim() === '') { + throw new EventRelayValidationError(`${label} is required`); + } + return value.trim(); +} + +function parseName(raw: unknown): string { + const name = requireTrimmed(raw, 'Name'); if (name.length > MAX_NAME_LENGTH || !NAME_PATTERN.test(name)) { throw new EventRelayValidationError( 'Name must be 1-64 characters and start with a letter or number', ); } + return name; +} - const notes = - input.notes === undefined || input.notes === '' - ? undefined - : requireTrimmed(input.notes, 'Notes'); - if (notes && notes.length > MAX_DESCRIPTION_LENGTH) { +function parseNotes(raw: unknown): string | undefined { + if (raw === undefined || raw === '') { + return undefined; + } + const notes = requireTrimmed(raw, 'Notes'); + if (notes.length > MAX_DESCRIPTION_LENGTH) { throw new EventRelayValidationError( `Notes must be at most ${MAX_DESCRIPTION_LENGTH} characters`, ); } + return notes; +} - const busEvent = requireTrimmed(input.busEvent, 'Bus event'); +function parseBusEvent(raw: unknown): string { + const busEvent = requireTrimmed(raw, 'Bus event'); if ( busEvent.length > MAX_BUS_EVENT_LENGTH || busEvent.includes('*') || @@ -60,8 +111,11 @@ export function validateEventRelayInput(input: EventRelayInput): EventRelayInput 'Bus event must be an exact channel name with no wildcards', ); } + return busEvent; +} - const socketEvent = requireTrimmed(input.socketEvent, 'Socket event'); +function parseSocketEvent(raw: unknown): string { + const socketEvent = requireTrimmed(raw, 'Socket event'); if ( socketEvent.length > MAX_SOCKET_EVENT_LENGTH || !SOCKET_EVENT_PATTERN.test(socketEvent) || @@ -69,62 +123,41 @@ export function validateEventRelayInput(input: EventRelayInput): EventRelayInput ) { throw new EventRelayValidationError('Socket event must be a non-reserved event name'); } + return socketEvent; +} - const resourceType = requireTrimmed(input.resourceType, 'Resource type'); +function parseResourceType(raw: unknown): string { + const resourceType = requireTrimmed(raw, 'Resource type'); if ( resourceType.length > MAX_RESOURCE_TYPE_LENGTH || !RESOURCE_TYPE_PATTERN.test(resourceType) ) { throw new EventRelayValidationError('Resource type is invalid'); } + return resourceType; +} - const resourceIdPath = requireTrimmed(input.resourceIdPath, 'Resource ID path'); +function parseResourceIdPath(raw: unknown): string { + const resourceIdPath = requireTrimmed(raw, 'Resource ID path'); if (resourceIdPath.length > MAX_RESOURCE_ID_PATH_LENGTH) { throw new EventRelayValidationError('Resource ID path is too long'); } parseDotPath(resourceIdPath, 'Resource ID path'); + return resourceIdPath; +} - const permission = requireTrimmed(input.permission, 'Permission'); +function parsePermission(raw: unknown): string { + const permission = requireTrimmed(raw, 'Permission'); if (permission.length > MAX_PERMISSION_LENGTH || !PERMISSION_PATTERN.test(permission)) { throw new EventRelayValidationError('Permission is invalid'); } - - if (input.messageTemplate === undefined) { - throw new EventRelayValidationError('Message template is required'); - } - assertTemplateSize(input.messageTemplate); - - return { - name, - notes, - active: input.active !== false, - busEvent, - socketEvent, - resourceType, - resourceIdPath, - permission, - messageTemplate: input.messageTemplate, - }; -} - -export function validateResourceId(resourceId: unknown): string { - if (typeof resourceId !== 'string' && typeof resourceId !== 'number') { - throw new EventRelayValidationError('Resource ID must be a string'); - } - const value = String(resourceId).trim(); - if ( - !value || - value.length > MAX_RESOURCE_ID_LENGTH || - !RESOURCE_ID_PATTERN.test(value) - ) { - throw new EventRelayValidationError('Resource ID is invalid'); - } - return value; + return permission; } -function requireTrimmed(value: unknown, label: string): string { - if (typeof value !== 'string' || value.trim() === '') { - throw new EventRelayValidationError(`${label} is required`); +function parseMessageTemplate(raw: unknown): unknown { + if (raw === undefined) { + throw new EventRelayValidationError('Message template is required'); } - return value.trim(); + assertTemplateSize(raw); + return raw; } diff --git a/modules/router/src/index.ts b/modules/router/src/index.ts index 20a0c779e..fd1d069ed 100644 --- a/modules/router/src/index.ts +++ b/modules/router/src/index.ts @@ -1,7 +1,25 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import ConduitDefaultRouter from './Router.js'; const peerManifestRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const router = new ConduitDefaultRouter(peerManifestRoot); + +function registerShutdownSignals(): void { + const shutdown = (signal: NodeJS.Signals) => { + void router + .shutdown() + .catch(err => { + ConduitGrpcSdk.Logger.error(err as Error); + }) + .finally(() => { + process.exit(signal === 'SIGINT' ? 130 : 0); + }); + }; + process.once('SIGTERM', () => shutdown('SIGTERM')); + process.once('SIGINT', () => shutdown('SIGINT')); +} + +registerShutdownSignals(); router.start(); diff --git a/modules/router/src/metrics/index.ts b/modules/router/src/metrics/index.ts index d3461c107..2ed45b20f 100644 --- a/modules/router/src/metrics/index.ts +++ b/modules/router/src/metrics/index.ts @@ -38,4 +38,39 @@ export default { help: 'Tracks denied or unavailable event-relay socket subscriptions', }, }, + eventRelaysActive: { + type: MetricType.Gauge, + config: { + name: 'event_relays_active_total', + help: 'Active event relays on this router replica', + }, + }, + eventRelaysSubscribedChannels: { + type: MetricType.Gauge, + config: { + name: 'event_relays_subscribed_channels_total', + help: 'Bus channels this router replica subscribes to for event relays', + }, + }, + eventRelaysEmptyRoom: { + type: MetricType.Counter, + config: { + name: 'event_relays_empty_room_total', + help: 'Skipped emits because no local sockets were in the relay room', + }, + }, + eventRelaysInboundDropped: { + type: MetricType.Counter, + config: { + name: 'event_relays_inbound_dropped_total', + help: 'Inbound bus payloads dropped for exceeding the size cap', + }, + }, + eventRelaysEmitDropped: { + type: MetricType.Counter, + config: { + name: 'event_relays_emit_dropped_total', + help: 'Socket emits dropped or disconnected due to backpressure', + }, + }, }; diff --git a/modules/router/tsconfig.test.json b/modules/router/tsconfig.test.json index 601e5ccee..229f7f155 100644 --- a/modules/router/tsconfig.test.json +++ b/modules/router/tsconfig.test.json @@ -19,6 +19,9 @@ "src/event-relays/search.ts", "src/event-relays/channels.ts", "src/event-relays/authorize.ts", + "src/event-relays/compile.ts", + "src/event-relays/rebacCache.ts", + "src/event-relays/subscriptions.ts", "src/event-relays/*.test.ts" ] } diff --git a/packages/core/src/admin/middleware/Admin.middleware.ts b/packages/core/src/admin/middleware/Admin.middleware.ts index 716dae763..4fb743246 100644 --- a/packages/core/src/admin/middleware/Admin.middleware.ts +++ b/packages/core/src/admin/middleware/Admin.middleware.ts @@ -2,9 +2,8 @@ import { NextFunction, Response } from 'express'; import { isNil } from 'lodash-es'; // Removed ConduitCommons import - now using configManager directly import { isDev } from '../utils/middleware.js'; -import { ConduitRequest } from '@conduitplatform/hermes'; +import { ConduitRequest, isSocketHandshake } from '@conduitplatform/hermes'; import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; -import { isSocketHandshake } from '../realtime/handshake.js'; export function getAdminMiddleware(configManager: any) { return async function adminMiddleware( diff --git a/packages/core/src/admin/middleware/Auth.middleware.ts b/packages/core/src/admin/middleware/Auth.middleware.ts index 410ea0fd6..9bac68d99 100644 --- a/packages/core/src/admin/middleware/Auth.middleware.ts +++ b/packages/core/src/admin/middleware/Auth.middleware.ts @@ -7,8 +7,7 @@ import { isDev } from '../utils/middleware.js'; import { ConduitRequest } from '@conduitplatform/hermes'; import { gql } from 'graphql-tag'; import { ConfigController } from '@conduitplatform/module-tools'; -import { isSocketHandshake } from '../realtime/handshake.js'; -import { isRealtimeTicket } from '../realtime/ticket.js'; +import { realtimeTicketForbiddenOnHttp } from '../realtime/ticket.js'; const excludedRestRoutes = ['/ready', '/live', '/login', '/config/modules']; const excludedGqlOperations = [ @@ -115,7 +114,7 @@ async function handleJwtToken( } const { id } = decoded; - if (isRealtimeTicket(decoded) && !isSocketHandshake(req)) { + if (realtimeTicketForbiddenOnHttp(decoded, req)) { res.status(401).json({ error: 'Realtime ticket cannot be used for HTTP requests' }); return; } diff --git a/packages/core/src/admin/realtime/handshake.test.ts b/packages/core/src/admin/realtime/handshake.test.ts deleted file mode 100644 index 59d17547f..000000000 --- a/packages/core/src/admin/realtime/handshake.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; -import { isSocketHandshake } from './handshake.js'; - -describe('isSocketHandshake', () => { - it('detects Socket.IO engine requests', () => { - assert.equal(isSocketHandshake({ url: '/realtime/?EIO=4&transport=polling' }), true); - assert.equal(isSocketHandshake({ originalUrl: '/realtime' }), true); - assert.equal(isSocketHandshake({ path: '/login' }), false); - assert.equal(isSocketHandshake({ url: '/graphql' }), false); - }); -}); diff --git a/packages/core/src/admin/realtime/handshake.ts b/packages/core/src/admin/realtime/handshake.ts deleted file mode 100644 index a4bf06fe1..000000000 --- a/packages/core/src/admin/realtime/handshake.ts +++ /dev/null @@ -1,14 +0,0 @@ -export function isSocketHandshake(req: { - url?: string; - originalUrl?: string; - path?: string; -}): boolean { - const parts = [req.url, req.originalUrl, req.path].filter((value): value is string => - Boolean(value), - ); - return parts.some(part => part.includes('EIO=') || isRealtimePath(part)); -} - -function isRealtimePath(value: string): boolean { - return /(?:^|[/?])realtime(?:[/?]|$)/.test(value); -} diff --git a/packages/core/src/admin/realtime/ticket-http.test.ts b/packages/core/src/admin/realtime/ticket-http.test.ts new file mode 100644 index 000000000..589975acc --- /dev/null +++ b/packages/core/src/admin/realtime/ticket-http.test.ts @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import jwt from 'jsonwebtoken'; +import { ConfigController } from '@conduitplatform/module-tools'; +import { isSocketHandshake } from '@conduitplatform/hermes'; +import { getAuthMiddleware } from '../middleware/Auth.middleware.js'; +import { + buildRealtimeTicketClaims, + isRealtimeTicket, + realtimeTicketForbiddenOnHttp, +} from './ticket.js'; + +function mockResponse() { + let statusCode = 0; + let body: unknown; + const res = { + status(code: number) { + statusCode = code; + return this; + }, + json(payload: unknown) { + body = payload; + return this; + }, + }; + return { + res, + get statusCode() { + return statusCode; + }, + get body() { + return body; + }, + }; +} + +describe('realtime ticket HTTP guard', () => { + it('does not treat POST /realtime/ticket as a handshake so a ticket cannot mint another', () => { + const ticket = buildRealtimeTicketClaims('admin-1'); + assert.equal(isRealtimeTicket(ticket), true); + assert.equal(isSocketHandshake({ url: '/realtime/ticket' }), false); + assert.equal( + isSocketHandshake({ url: '/realtime/ticket?EIO=4&transport=polling' }), + false, + ); + assert.equal( + realtimeTicketForbiddenOnHttp(ticket, { url: '/realtime/ticket' }), + true, + ); + assert.equal( + realtimeTicketForbiddenOnHttp(ticket, { + url: '/realtime/ticket?EIO=4&transport=polling', + }), + true, + ); + }); + + it('allows a ticket on Engine.IO handshake polling without sid', () => { + const ticket = buildRealtimeTicketClaims('admin-1'); + const handshake = { url: '/realtime/?EIO=4&transport=polling' }; + assert.equal(isSocketHandshake(handshake), true); + assert.equal(realtimeTicketForbiddenOnHttp(ticket, handshake), false); + }); + + it('Auth middleware returns 401 on POST /realtime/ticket with a realtime ticket', async () => { + const secret = 'ticket-http-test-secret'; + ConfigController.getInstance().config = { auth: { tokenSecret: secret } }; + const token = jwt.sign(buildRealtimeTicketClaims('admin-1'), secret, { + algorithm: 'HS256', + expiresIn: 30, + }); + const middleware = getAuthMiddleware({} as never, { + get: async () => ({ env: 'production' }), + }); + const mock = mockResponse(); + let nextCalled = false; + await middleware( + { + path: '/realtime/ticket', + originalUrl: '/realtime/ticket', + url: '/realtime/ticket', + method: 'POST', + headers: { authorization: `Bearer ${token}` }, + conduit: {}, + } as never, + mock.res as never, + () => { + nextCalled = true; + }, + ); + assert.equal(nextCalled, false); + assert.equal(mock.statusCode, 401); + assert.deepEqual(mock.body, { + error: 'Realtime ticket cannot be used for HTTP requests', + }); + }); +}); diff --git a/packages/core/src/admin/realtime/ticket.ts b/packages/core/src/admin/realtime/ticket.ts index 154630c0c..e797f5966 100644 --- a/packages/core/src/admin/realtime/ticket.ts +++ b/packages/core/src/admin/realtime/ticket.ts @@ -1,3 +1,5 @@ +import { isSocketHandshake } from '@conduitplatform/hermes'; + export const ADMIN_REALTIME_AUDIENCE = 'admin-realtime'; export const ADMIN_REALTIME_TICKET_TTL_SECONDS = 30; @@ -24,3 +26,10 @@ export function isRealtimeTicket( } return audience === ADMIN_REALTIME_AUDIENCE; } + +export function realtimeTicketForbiddenOnHttp( + decoded: RealtimeTicketClaims | null | undefined, + req: { url?: string; originalUrl?: string }, +): boolean { + return isRealtimeTicket(decoded) && !isSocketHandshake(req); +} diff --git a/packages/core/tsconfig.test.json b/packages/core/tsconfig.test.json index b0548057f..8559c14bf 100644 --- a/packages/core/tsconfig.test.json +++ b/packages/core/tsconfig.test.json @@ -9,7 +9,6 @@ }, "include": [ "src/admin/realtime/ticket.ts", - "src/admin/realtime/handshake.ts", "src/admin/realtime/namespace.ts", "src/admin/realtime/*.test.ts" ],