From ec7e08d198a3f0499649f607c539cd40b08b3d0c Mon Sep 17 00:00:00 2001 From: John Chantzigoulas Date: Fri, 4 Sep 2026 14:43:54 +0300 Subject: [PATCH 1/6] feat(router): add ReBAC-protected event-to-socket relays Forward exact Redis bus events as templated Socket.io messages on /events/, with resource-scoped rooms and local-only HA emission. --- libraries/hermes/src/Socket/Socket.ts | 25 +++- libraries/hermes/src/interfaces/Socket.ts | 9 +- libraries/hermes/src/interfaces/SocketPush.ts | 1 + modules/router/README.mdx | 40 +++++ modules/router/package.json | 1 + modules/router/src/Router.ts | 28 +++- modules/router/src/admin/event-relays.ts | 128 ++++++++++++++++ modules/router/src/admin/index.ts | 96 +++++++++++- .../src/event-relays/EventRelayManager.ts | 138 ++++++++++++++++++ .../event-relays/EventRelaySockets.test.ts | 91 ++++++++++++ .../src/event-relays/EventRelaySockets.ts | 96 ++++++++++++ modules/router/src/event-relays/authorize.ts | 101 +++++++++++++ .../router/src/event-relays/channels.test.ts | 32 ++++ modules/router/src/event-relays/channels.ts | 23 +++ modules/router/src/event-relays/constants.ts | 32 ++++ modules/router/src/event-relays/index.ts | 25 ++++ modules/router/src/event-relays/path.test.ts | 39 +++++ modules/router/src/event-relays/path.ts | 49 +++++++ .../router/src/event-relays/process.test.ts | 85 +++++++++++ modules/router/src/event-relays/process.ts | 75 ++++++++++ modules/router/src/event-relays/push.ts | 28 ++++ modules/router/src/event-relays/rooms.test.ts | 29 ++++ modules/router/src/event-relays/rooms.ts | 6 + .../router/src/event-relays/search.test.ts | 34 +++++ modules/router/src/event-relays/search.ts | 26 ++++ .../router/src/event-relays/template.test.ts | 39 +++++ modules/router/src/event-relays/template.ts | 136 +++++++++++++++++ .../src/event-relays/validation.test.ts | 69 +++++++++ modules/router/src/event-relays/validation.ts | 130 +++++++++++++++++ .../src/event-relays/validationError.ts | 6 + modules/router/src/metrics/index.ts | 21 +++ .../router/src/models/EventRelay.schema.ts | 94 ++++++++++++ modules/router/src/models/index.ts | 1 + modules/router/tsconfig.test.json | 24 +++ 34 files changed, 1750 insertions(+), 7 deletions(-) create mode 100644 modules/router/src/admin/event-relays.ts create mode 100644 modules/router/src/event-relays/EventRelayManager.ts create mode 100644 modules/router/src/event-relays/EventRelaySockets.test.ts create mode 100644 modules/router/src/event-relays/EventRelaySockets.ts create mode 100644 modules/router/src/event-relays/authorize.ts create mode 100644 modules/router/src/event-relays/channels.test.ts create mode 100644 modules/router/src/event-relays/channels.ts create mode 100644 modules/router/src/event-relays/constants.ts create mode 100644 modules/router/src/event-relays/index.ts create mode 100644 modules/router/src/event-relays/path.test.ts create mode 100644 modules/router/src/event-relays/path.ts create mode 100644 modules/router/src/event-relays/process.test.ts create mode 100644 modules/router/src/event-relays/process.ts create mode 100644 modules/router/src/event-relays/push.ts create mode 100644 modules/router/src/event-relays/rooms.test.ts create mode 100644 modules/router/src/event-relays/rooms.ts create mode 100644 modules/router/src/event-relays/search.test.ts create mode 100644 modules/router/src/event-relays/search.ts create mode 100644 modules/router/src/event-relays/template.test.ts create mode 100644 modules/router/src/event-relays/template.ts create mode 100644 modules/router/src/event-relays/validation.test.ts create mode 100644 modules/router/src/event-relays/validation.ts create mode 100644 modules/router/src/event-relays/validationError.ts create mode 100644 modules/router/src/models/EventRelay.schema.ts create mode 100644 modules/router/tsconfig.test.json diff --git a/libraries/hermes/src/Socket/Socket.ts b/libraries/hermes/src/Socket/Socket.ts index 7d9b0b09f..74ced4e2b 100644 --- a/libraries/hermes/src/Socket/Socket.ts +++ b/libraries/hermes/src/Socket/Socket.ts @@ -10,6 +10,7 @@ import { EventResponse, isInstanceOfEventResponse, JoinRoomResponse, + LeaveRoomResponse, SocketPush, } from '../interfaces/index.js'; import ObjectHash from 'object-hash'; @@ -184,11 +185,13 @@ export class SocketController extends ConduitRouter { } async handleSocketPush(push: SocketPush) { + const localOnly = push.localOnly === true; if (push.event === 'join-room') { if (push.rooms.length === 0) return; const filteredSockets = await this.findAndFilterSockets( push.receivers, push.namespace, + localOnly, ); for (const socket of filteredSockets) { ConduitGrpcSdk.Logger.info( @@ -203,6 +206,7 @@ export class SocketController extends ConduitRouter { const filteredSockets = await this.findAndFilterSockets( push.receivers, push.namespace, + localOnly, ); for (const socket of filteredSockets) { for (const room of push.rooms) { @@ -221,7 +225,12 @@ export class SocketController extends ConduitRouter { ConduitGrpcSdk.Logger.info( `Emitting event: ${push.event} to all sockets in namespace: ${push.namespace}`, ); - this.io.of(push.namespace).emit(push.event, push.data); + const nsp = this.io.of(push.namespace); + if (localOnly) { + nsp.local.emit(push.event, push.data); + } else { + nsp.emit(push.event, push.data); + } } else { if (push.rooms.length !== 0) { ConduitGrpcSdk.Logger.info( @@ -229,12 +238,18 @@ export class SocketController extends ConduitRouter { ', ', )} in namespace: ${push.namespace}`, ); - this.io.of(push.namespace).to(push.rooms).emit(push.event, push.data); + const target = this.io.of(push.namespace).to(push.rooms); + if (localOnly) { + target.local.emit(push.event, push.data); + } else { + target.emit(push.event, push.data); + } } if (push.receivers.length !== 0) { const filteredSockets = await this.findAndFilterSockets( push.receivers, push.namespace, + localOnly, ); for (const socket of filteredSockets) { ConduitGrpcSdk.Logger.info( @@ -248,7 +263,7 @@ export class SocketController extends ConduitRouter { } private async handleResponse( - res: EventResponse | JoinRoomResponse, + res: EventResponse | JoinRoomResponse | LeaveRoomResponse, socket: Socket, namespace: string, ) { @@ -307,8 +322,10 @@ export class SocketController extends ConduitRouter { async findAndFilterSockets( userIds: string[], namespace: string, + localOnly: boolean = false, ): Promise[]> { - const sockets = await this.io.of(namespace).fetchSockets(); + const nsp = this.io.of(namespace); + const sockets = localOnly ? await nsp.local.fetchSockets() : await nsp.fetchSockets(); const userIdSet = new Set(userIds); return sockets.filter(socket => { if (socket.data && socket.data.user) { diff --git a/libraries/hermes/src/interfaces/Socket.ts b/libraries/hermes/src/interfaces/Socket.ts index a477c5c47..204707624 100644 --- a/libraries/hermes/src/interfaces/Socket.ts +++ b/libraries/hermes/src/interfaces/Socket.ts @@ -40,7 +40,14 @@ export type JoinRoomResponse = { rooms: string[]; }; -export type ConduitSocketHandlerResponse = Promise; +export type LeaveRoomResponse = { + event: 'leave-room'; + rooms: string[]; +}; + +export type ConduitSocketHandlerResponse = Promise< + EventResponse | JoinRoomResponse | LeaveRoomResponse +>; export type ConduitSocketEventHandler = ( request: ConduitSocketParameters, diff --git a/libraries/hermes/src/interfaces/SocketPush.ts b/libraries/hermes/src/interfaces/SocketPush.ts index 2a2ca5e10..8bbabbc85 100644 --- a/libraries/hermes/src/interfaces/SocketPush.ts +++ b/libraries/hermes/src/interfaces/SocketPush.ts @@ -4,4 +4,5 @@ export interface SocketPush { receivers: string[]; rooms: string[]; namespace: string; + localOnly?: boolean; } diff --git a/modules/router/README.mdx b/modules/router/README.mdx index b1dff474d..c896f2cee 100644 --- a/modules/router/README.mdx +++ b/modules/router/README.mdx @@ -77,3 +77,43 @@ If you sent any other string that those provided conduit will not utilize cachin The number that you specify also sets the expiry time of conduit's cache. All caching takes place, AFTER middleware execution - cacheControl?: string; + +## Event Relays + +Event Relays map an **exact** Redis bus channel to a Socket.io event on the dedicated `/events/` namespace. +They are configured through the Admin API (`/router/event-relays`) and are **ephemeral**: Redis pub/sub is not replayed, and missed events are not stored. + +Each relay stores a ReBAC check (`permission` on `resourceType:resourceId`). Clients never choose a raw room name. Optional `notes` describe the relay for operators. + +### Client contract + +```javascript +import { io } from 'socket.io-client'; + +const socket = io(`${SOCKET_BASE_URL}/events/`, { + path: '/realtime', + extraHeaders: { + authorization: `Bearer ${accessToken}`, + }, +}); + +socket.emit('subscribe', relayId, resourceId); +socket.on('order-updated', payload => { + // payload is the rendered JSON template +}); +socket.emit('unsubscribe', relayId, resourceId); +``` + +- Subscribe is allowed only when `User:` has the relay `permission` on `:`. +- The outbound event name is the relay `socketEvent`. +- Placeholders in `messageTemplate` use `{{payload.path}}` against the bus JSON payload. + +### Admin API + +| Method | Path | Description | +|:-------|:-----|:------------| +| `GET` | `/router/event-relays` | Paginated list (`skip`, `limit`, `search`) | +| `GET` | `/router/event-relays/:id` | Single relay | +| `POST` | `/router/event-relays` | Create | +| `PATCH` | `/router/event-relays/:id` | Update | +| `DELETE` | `/router/event-relays/:id` | Delete | diff --git a/modules/router/package.json b/modules/router/package.json index 72670f268..c8d9c12cf 100644 --- a/modules/router/package.json +++ b/modules/router/package.json @@ -23,6 +23,7 @@ "prebuild:bundle": "pnpm --filter @conduitplatform/service-bundle run build", "build:bundle": "rimraf bundle && node ../../libraries/service-bundle/dist/cli.js generate-manifest && tsup && node ../../libraries/service-bundle/dist/cli.js copy-assets && node ../../libraries/service-bundle/dist/cli.js generate-lockfile", "generateTypes": "sh build.sh", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/event-relays/*.test.js", "build:docker": "docker build -t ghcr.io/conduitplatform/router:latest -f ./Dockerfile ../../ && docker push ghcr.io/conduitplatform/router:latest" }, "license": "ISC", diff --git a/modules/router/src/Router.ts b/modules/router/src/Router.ts index 80f9aa56e..0a3867ed6 100644 --- a/modules/router/src/Router.ts +++ b/modules/router/src/Router.ts @@ -40,6 +40,11 @@ import * as adminRoutes from './admin/routes/index.js'; import metricsSchema from './metrics/index.js'; import { ConfigController, ManagedModule } from '@conduitplatform/module-tools'; import { fileURLToPath } from 'node:url'; +import { + createEventRelayPusher, + createEventsSocket, + EventRelayManager, +} from './event-relays/index.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -69,6 +74,8 @@ export default class ConduitDefaultRouter extends ManagedModule { private hasAppliedMiddleware: string[] = []; private _refreshTimeout: NodeJS.Timeout | null = null; private _haInitialized = false; + private eventRelayManager: EventRelayManager; + private eventsSocket?: ConduitSocket; constructor(peerManifestRoot?: string) { super('router', peerManifestRoot); @@ -102,7 +109,16 @@ export default class ConduitDefaultRouter extends ManagedModule { } async onRegister() { - this.adminRouter = new AdminHandlers(this.grpcServer, this.grpcSdk, this); + this.eventRelayManager = new EventRelayManager( + this.grpcSdk, + createEventRelayPusher(data => this._internalRouter.socketPush(data)), + ); + this.adminRouter = new AdminHandlers( + this.grpcServer, + this.grpcSdk, + this, + this.eventRelayManager, + ); this._security = new SecurityModule(this.grpcSdk, this); } @@ -132,8 +148,11 @@ export default class ConduitDefaultRouter extends ManagedModule { } if (config.transports.sockets) { this._internalRouter.initSockets(); + this.registerEventsNamespace(); + await this.eventRelayManager.start(); atLeastOne = true; } else { + await this.eventRelayManager?.stop(); this._internalRouter.stopSockets(); } @@ -336,6 +355,13 @@ export default class ConduitDefaultRouter extends ManagedModule { this._internalRouter.registerConduitRoute(route); } + private registerEventsNamespace() { + if (!this.eventsSocket) { + this.eventsSocket = createEventsSocket(this.grpcSdk, this.eventRelayManager); + } + this._internalRouter.registerConduitSocket(this.eventsSocket); + } + protected registerSchemas(): Promise { const promises = Object.values(models).map(model => { const modelInstance = model.getInstance(this.grpcSdk.database!); diff --git a/modules/router/src/admin/event-relays.ts b/modules/router/src/admin/event-relays.ts new file mode 100644 index 000000000..2e56fbd15 --- /dev/null +++ b/modules/router/src/admin/event-relays.ts @@ -0,0 +1,128 @@ +import { status } from '@grpc/grpc-js'; +import { + GrpcError, + ParsedRouterRequest, + Query, + UnparsedRouterResponse, +} from '@conduitplatform/grpc-sdk'; +import { isNil } from 'lodash-es'; +import { EventRelay } from '../models/index.js'; +import { EventRelayManager } from '../event-relays/EventRelayManager.js'; +import { EventRelayInput, validateEventRelayInput } from '../event-relays/validation.js'; +import { EventRelayValidationError } from '../event-relays/validationError.js'; +import { buildSearchQuery, parsePagination } from '../event-relays/search.js'; + +export class EventRelayAdmin { + constructor(private readonly manager: EventRelayManager) {} + + async listEventRelays(call: ParsedRouterRequest): Promise { + const { skip, limit } = parsePagination( + call.request.params.skip, + call.request.params.limit, + ); + const { search } = call.request.params; + const query = buildSearchQuery(search) as Query; + const relays = await EventRelay.getInstance().findMany(query, { + skip, + limit, + sort: { updatedAt: -1 }, + }); + const count = await EventRelay.getInstance().countDocuments(query); + return { relays, count }; + } + + async getEventRelay(call: ParsedRouterRequest): Promise { + const relay = await EventRelay.getInstance().findOne({ + _id: call.request.params.id, + }); + if (isNil(relay)) { + throw new GrpcError(status.NOT_FOUND, 'Event relay not found'); + } + return relay; + } + + async createEventRelay(call: ParsedRouterRequest): Promise { + const input = parseInput(call.request.params as EventRelayInput); + await assertUniqueName(input.name); + const relay = await EventRelay.getInstance().create({ + ...input, + messageTemplate: input.messageTemplate as EventRelay['messageTemplate'], + }); + await this.manager.notifyChanged(); + return relay; + } + + async patchEventRelay(call: ParsedRouterRequest): Promise { + const existing = await EventRelay.getInstance().findOne({ + _id: call.request.params.id, + }); + if (isNil(existing)) { + throw new GrpcError(status.NOT_FOUND, 'Event relay not found'); + } + + const merged: EventRelayInput = { + name: call.request.params.name ?? existing.name, + notes: + call.request.params.notes === undefined + ? existing.notes + : call.request.params.notes, + active: + call.request.params.active === undefined + ? existing.active + : call.request.params.active, + busEvent: call.request.params.busEvent ?? existing.busEvent, + socketEvent: call.request.params.socketEvent ?? existing.socketEvent, + resourceType: call.request.params.resourceType ?? existing.resourceType, + resourceIdPath: call.request.params.resourceIdPath ?? existing.resourceIdPath, + permission: call.request.params.permission ?? existing.permission, + messageTemplate: + call.request.params.messageTemplate === undefined + ? existing.messageTemplate + : call.request.params.messageTemplate, + }; + const input = parseInput(merged); + if (input.name !== existing.name) { + await assertUniqueName(input.name, existing._id); + } + + const updated = await EventRelay.getInstance().findByIdAndUpdate(existing._id, { + ...input, + messageTemplate: input.messageTemplate as EventRelay['messageTemplate'], + }); + await this.manager.notifyChanged(); + return updated!; + } + + async deleteEventRelay(call: ParsedRouterRequest): Promise { + const existing = await EventRelay.getInstance().findOne({ + _id: call.request.params.id, + }); + if (isNil(existing)) { + throw new GrpcError(status.NOT_FOUND, 'Event relay not found'); + } + await EventRelay.getInstance().deleteOne({ _id: existing._id }); + await this.manager.notifyChanged(); + return { message: 'Event relay deleted' }; + } +} + +function parseInput(params: EventRelayInput): EventRelayInput { + try { + return validateEventRelayInput(params); + } catch (err) { + if (err instanceof EventRelayValidationError) { + throw new GrpcError(status.INVALID_ARGUMENT, err.message); + } + throw err; + } +} + +async function assertUniqueName(name: string, excludeId?: string): Promise { + const existing = await EventRelay.getInstance().findOne({ name }); + if (existing && existing._id !== excludeId) { + throw new GrpcError( + status.ALREADY_EXISTS, + `An event relay named '${name}' already exists`, + ); + } +} diff --git a/modules/router/src/admin/index.ts b/modules/router/src/admin/index.ts index 72065b6f9..0c3231378 100644 --- a/modules/router/src/admin/index.ts +++ b/modules/router/src/admin/index.ts @@ -6,27 +6,34 @@ import { } from '@conduitplatform/grpc-sdk'; import { ConduitBoolean, + ConduitJson, + ConduitNumber, ConduitString, GrpcServer, RoutingManager, } from '@conduitplatform/module-tools'; import { RouterAdmin } from './router.js'; import { SecurityAdmin } from './security.js'; +import { EventRelayAdmin } from './event-relays.js'; import ConduitDefaultRouter from '../Router.js'; -import { Client } from '../models/index.js'; +import { Client, EventRelay } from '../models/index.js'; +import { EventRelayManager } from '../event-relays/EventRelayManager.js'; export class AdminHandlers { private readonly routerAdmin: RouterAdmin; private readonly securityAdmin: SecurityAdmin; + private readonly eventRelayAdmin: EventRelayAdmin; private readonly routingManager: RoutingManager; constructor( private readonly server: GrpcServer, private readonly grpcSdk: ConduitGrpcSdk, private readonly router: ConduitDefaultRouter, + eventRelayManager: EventRelayManager, ) { this.routerAdmin = new RouterAdmin(this.grpcSdk, router); this.securityAdmin = new SecurityAdmin(this.grpcSdk); + this.eventRelayAdmin = new EventRelayAdmin(eventRelayManager); this.routingManager = new RoutingManager(this.grpcSdk.admin, this.server); this.registerAdminRoutes(); } @@ -148,6 +155,93 @@ export class AdminHandlers { new ConduitRouteReturnDefinition('UpdateSecurityClient', Client.name), this.securityAdmin.updateSecurityClient.bind(this.securityAdmin), ); + + this.routingManager.route( + { + path: '/event-relays', + action: ConduitRouteActions.GET, + description: `Returns event-to-socket relays. Delivery is ephemeral Redis pub/sub with no replay.`, + queryParams: { + skip: ConduitNumber.Optional, + limit: ConduitNumber.Optional, + search: ConduitString.Optional, + }, + }, + new ConduitRouteReturnDefinition('GetEventRelays', { + relays: [EventRelay.name], + count: ConduitNumber.Required, + }), + this.eventRelayAdmin.listEventRelays.bind(this.eventRelayAdmin), + ); + this.routingManager.route( + { + path: '/event-relays/:id', + action: ConduitRouteActions.GET, + description: `Returns a single event-to-socket relay.`, + urlParams: { + id: ConduitString.Required, + }, + }, + new ConduitRouteReturnDefinition(EventRelay.name), + this.eventRelayAdmin.getEventRelay.bind(this.eventRelayAdmin), + ); + this.routingManager.route( + { + path: '/event-relays', + action: ConduitRouteActions.POST, + description: `Creates an event-to-socket relay. Clients subscribe on /events/ after a ReBAC check.`, + bodyParams: { + name: ConduitString.Required, + notes: ConduitString.Optional, + active: ConduitBoolean.Optional, + busEvent: ConduitString.Required, + socketEvent: ConduitString.Required, + resourceType: ConduitString.Required, + resourceIdPath: ConduitString.Required, + permission: ConduitString.Required, + messageTemplate: ConduitJson.Required, + }, + }, + new ConduitRouteReturnDefinition('CreateEventRelay', EventRelay.name), + this.eventRelayAdmin.createEventRelay.bind(this.eventRelayAdmin), + ); + this.routingManager.route( + { + path: '/event-relays/:id', + action: ConduitRouteActions.PATCH, + description: `Updates an event-to-socket relay.`, + urlParams: { + id: ConduitString.Required, + }, + bodyParams: { + name: ConduitString.Optional, + notes: ConduitString.Optional, + active: ConduitBoolean.Optional, + busEvent: ConduitString.Optional, + socketEvent: ConduitString.Optional, + resourceType: ConduitString.Optional, + resourceIdPath: ConduitString.Optional, + permission: ConduitString.Optional, + messageTemplate: ConduitJson.Optional, + }, + }, + new ConduitRouteReturnDefinition('PatchEventRelay', EventRelay.name), + this.eventRelayAdmin.patchEventRelay.bind(this.eventRelayAdmin), + ); + this.routingManager.route( + { + path: '/event-relays/:id', + action: ConduitRouteActions.DELETE, + description: `Deletes an event-to-socket relay.`, + urlParams: { + id: ConduitString.Required, + }, + }, + new ConduitRouteReturnDefinition('DeleteEventRelay', { + message: ConduitString.Required, + }), + this.eventRelayAdmin.deleteEventRelay.bind(this.eventRelayAdmin), + ); this.routingManager.registerRoutes(); } } diff --git a/modules/router/src/event-relays/EventRelayManager.ts b/modules/router/src/event-relays/EventRelayManager.ts new file mode 100644 index 000000000..bddae2127 --- /dev/null +++ b/modules/router/src/event-relays/EventRelayManager.ts @@ -0,0 +1,138 @@ +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { EventRelay } from '../models/index.js'; +import { + EVENT_RELAY_REFRESH_CHANNEL, + EVENT_RELAY_SUBSCRIBER_PREFIX, +} from './constants.js'; +import { groupRelaysByChannel, planChannelSubscriptions } from './channels.js'; +import { buildRelayEmissions, parseBusPayload } from './process.js'; +import { EventRelayPusher } from './push.js'; + +export type { EventRelayPusher } from './push.js'; +export { createEventRelayPusher } from './push.js'; + +export class EventRelayManager { + private readonly relaysByChannel = new Map(); + private readonly subscribedChannels = new Set(); + private started = false; + + constructor( + private readonly grpcSdk: ConduitGrpcSdk, + private readonly push: EventRelayPusher, + ) {} + + async start(): Promise { + if (!this.started) { + this.grpcSdk.bus?.subscribe( + EVENT_RELAY_REFRESH_CHANNEL, + () => { + this.reconcile().catch(err => { + ConduitGrpcSdk.Logger.error(err as Error); + }); + }, + 'router-event-relays-refresh', + ); + this.started = true; + } + await this.reconcile(); + } + + async stop(): Promise { + for (const channel of [...this.subscribedChannels]) { + this.grpcSdk.bus?.unsubscribe(`${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`); + this.subscribedChannels.delete(channel); + } + this.relaysByChannel.clear(); + if (this.started) { + this.grpcSdk.bus?.unsubscribe('router-event-relays-refresh'); + this.started = false; + } + } + + async notifyChanged(): Promise { + if (this.started) { + await this.reconcile(); + } + this.grpcSdk.bus?.publish(EVENT_RELAY_REFRESH_CHANNEL, ''); + } + + async reconcile(): Promise { + const relays = await EventRelay.getInstance().findMany({ active: true }); + const next = groupRelaysByChannel(relays); + const { toSubscribe, toUnsubscribe } = planChannelSubscriptions( + this.subscribedChannels, + next.keys(), + ); + + this.relaysByChannel.clear(); + for (const [channel, list] of next) { + this.relaysByChannel.set(channel, list); + } + + for (const channel of toUnsubscribe) { + this.grpcSdk.bus?.unsubscribe(`${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`); + this.subscribedChannels.delete(channel); + } + + for (const channel of toSubscribe) { + this.grpcSdk.bus?.subscribe( + channel, + message => this.onBusMessage(channel, message), + `${EVENT_RELAY_SUBSCRIBER_PREFIX}${channel}`, + ); + this.subscribedChannels.add(channel); + } + } + + getActiveRelay(id: string): EventRelay | undefined { + for (const relays of this.relaysByChannel.values()) { + const match = relays.find(relay => relay._id === id); + if (match) return match; + } + return undefined; + } + + private onBusMessage(channel: string, message: string): void { + const relays = this.relaysByChannel.get(channel); + if (!relays || relays.length === 0) { + return; + } + + let payload: unknown; + try { + payload = parseBusPayload(message); + } catch (err) { + ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); + ConduitGrpcSdk.Logger.error( + `Event relay failed to parse payload for ${channel}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return; + } + + const { emissions, failures } = buildRelayEmissions(relays, payload); + for (const failure of failures) { + ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); + ConduitGrpcSdk.Logger.warn( + `Event relay ${failure.relayId} skipped on ${failure.busEvent}: ${failure.reason}`, + ); + } + + for (const emission of emissions) { + this.push(emission.socketEvent, emission.data, [emission.room]).then( + () => { + ConduitGrpcSdk.Metrics?.increment('event_relays_emitted_total'); + }, + err => { + ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); + ConduitGrpcSdk.Logger.error( + `Event relay ${emission.relayId} emit failed on ${channel}: ${ + (err as Error).message + }`, + ); + }, + ); + } + } +} diff --git a/modules/router/src/event-relays/EventRelaySockets.test.ts b/modules/router/src/event-relays/EventRelaySockets.test.ts new file mode 100644 index 000000000..19f455b7a --- /dev/null +++ b/modules/router/src/event-relays/EventRelaySockets.test.ts @@ -0,0 +1,91 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { status } from '@grpc/grpc-js'; +import { authorizeRelaySubscription } from './authorize.js'; +import { eventRelayRoom } from './rooms.js'; +import type { RelayLookup } from './authorize.js'; + +function mockSdk(options: { + available?: boolean; + allow?: boolean; + throwOnCan?: boolean; +}) { + return { + isAvailable: () => options.available !== false, + authorization: + options.available === false + ? null + : { + can: async () => { + if (options.throwOnCan) { + throw new Error('authz down'); + } + return { allow: options.allow === true }; + }, + }, + }; +} + +function mockManager(relay: unknown): RelayLookup { + return { + getActiveRelay: () => relay as ReturnType, + }; +} + +const relay = { + _id: 'relay-1', + permission: 'read', + resourceType: 'Order', +}; + +describe('authorizeRelaySubscription', () => { + it('returns the deterministic room when ReBAC allows', async () => { + const room = await authorizeRelaySubscription( + mockSdk({ allow: true }), + mockManager(relay), + 'user-1', + 'relay-1', + 'order-1', + ); + assert.equal(room, eventRelayRoom('relay-1', 'order-1')); + }); + + it('denies when ReBAC returns false', async () => { + await assert.rejects( + () => + authorizeRelaySubscription( + mockSdk({ allow: false }), + mockManager(relay), + 'user-1', + 'relay-1', + 'order-1', + ), + (err: any) => err.code === status.PERMISSION_DENIED, + ); + }); + + it('fails closed when authorization is unavailable', async () => { + await assert.rejects( + () => + authorizeRelaySubscription( + mockSdk({ available: false }), + mockManager(relay), + 'user-1', + 'relay-1', + 'order-1', + ), + (err: any) => err.code === status.UNAVAILABLE, + ); + }); + + it('unsubscribe room matches subscribe room for the same ids', async () => { + const subscribed = await authorizeRelaySubscription( + mockSdk({ allow: true }), + mockManager(relay), + 'user-1', + 'relay-1', + 'order-1', + ); + assert.equal(subscribed, eventRelayRoom('relay-1', 'order-1')); + }); +}); diff --git a/modules/router/src/event-relays/EventRelaySockets.ts b/modules/router/src/event-relays/EventRelaySockets.ts new file mode 100644 index 000000000..32e5fec2a --- /dev/null +++ b/modules/router/src/event-relays/EventRelaySockets.ts @@ -0,0 +1,96 @@ +import { status } from '@grpc/grpc-js'; +import { ConduitGrpcSdk, GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; +import { ConduitSocket, ConduitSocketEvent } from '@conduitplatform/hermes'; +import { EVENTS_NAMESPACE } from './constants.js'; +import { EventRelayManager } from './EventRelayManager.js'; +import { eventRelayRoom } from './rooms.js'; +import { validateResourceId } from './validation.js'; +import { authorizeRelaySubscription, toSubscriptionError } from './authorize.js'; + +export function createEventsSocket( + grpcSdk: ConduitGrpcSdk, + manager: EventRelayManager, +): ConduitSocket { + const events = new Map(); + + events.set('connect', { + name: 'connect', + handler: async () => ({ event: 'join-room', rooms: [] }), + }); + + events.set('disconnect', { + name: 'disconnect', + handler: async () => ({ event: 'leave-room', rooms: [] }), + }); + + events.set('subscribe', { + name: 'subscribe', + params: [TYPE.String, TYPE.String], + handler: async request => { + const userId = request.context?.user?._id as string | undefined; + const [relayId, resourceId] = request.params ?? []; + const room = await authorizeOrThrow(grpcSdk, manager, userId, relayId, resourceId); + return { event: 'join-room', rooms: [room] }; + }, + }); + + events.set('unsubscribe', { + name: 'unsubscribe', + params: [TYPE.String, TYPE.String], + handler: async request => { + const [relayId, resourceId] = request.params ?? []; + if (typeof relayId !== 'string' || relayId.trim() === '') { + throw new GrpcError(status.INVALID_ARGUMENT, 'Relay ID is required'); + } + try { + const validatedResourceId = validateResourceId(resourceId); + return { + event: 'leave-room', + rooms: [eventRelayRoom(relayId, validatedResourceId)], + }; + } catch (err) { + throw toGrpcError(err); + } + }, + }); + + return new ConduitSocket( + { + path: EVENTS_NAMESPACE, + name: 'eventRelays', + description: 'Declarative bus-to-socket event relays', + middlewares: ['authMiddleware'], + }, + events, + ); +} + +async function authorizeOrThrow( + grpcSdk: ConduitGrpcSdk, + manager: EventRelayManager, + userId: string | undefined, + relayId: unknown, + resourceId: unknown, +): Promise { + try { + return await authorizeRelaySubscription( + grpcSdk, + manager, + userId, + relayId, + resourceId, + () => { + ConduitGrpcSdk.Metrics?.increment('event_relay_subscriptions_denied_total'); + }, + ); + } catch (err) { + throw toGrpcError(err); + } +} + +function toGrpcError(err: unknown): GrpcError { + const mapped = toSubscriptionError(err); + return new GrpcError(mapped.code, mapped.message); +} + +export { authorizeRelaySubscription } from './authorize.js'; diff --git a/modules/router/src/event-relays/authorize.ts b/modules/router/src/event-relays/authorize.ts new file mode 100644 index 000000000..852cf5595 --- /dev/null +++ b/modules/router/src/event-relays/authorize.ts @@ -0,0 +1,101 @@ +import { status } from '@grpc/grpc-js'; +import { eventRelayRoom } from './rooms.js'; +import { validateResourceId } from './validation.js'; +import { EventRelayValidationError } from './validationError.js'; + +export class RelaySubscriptionError extends Error { + constructor( + readonly code: number, + message: string, + ) { + super(message); + this.name = 'RelaySubscriptionError'; + } +} + +export type RelaySubscriptionTarget = { + _id: string; + permission: string; + resourceType: string; +}; + +export type RelayAuthorizationSdk = { + isAvailable: (module: string) => boolean; + authorization?: { + can: (request: { + subject: string; + actions: string[]; + resource: string; + }) => Promise<{ allow: boolean }>; + } | null; +}; + +export type RelayLookup = { + getActiveRelay(id: string): RelaySubscriptionTarget | undefined; +}; + +export async function authorizeRelaySubscription( + grpcSdk: RelayAuthorizationSdk, + manager: RelayLookup, + userId: string | undefined, + relayId: unknown, + resourceId: unknown, + onDenied?: () => void, +): Promise { + if (!userId) { + throw new RelaySubscriptionError(status.UNAUTHENTICATED, 'Authentication required'); + } + if (typeof relayId !== 'string' || relayId.trim() === '') { + throw new RelaySubscriptionError(status.INVALID_ARGUMENT, 'Relay ID is required'); + } + + let validatedResourceId: string; + try { + validatedResourceId = validateResourceId(resourceId); + } catch (err) { + throw toSubscriptionError(err); + } + + const relay = manager.getActiveRelay(relayId); + if (!relay) { + throw new RelaySubscriptionError(status.NOT_FOUND, 'Event relay not found'); + } + + if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { + onDenied?.(); + throw new RelaySubscriptionError(status.UNAVAILABLE, 'Authorization is unavailable'); + } + + let allowed = false; + try { + const decision = await grpcSdk.authorization.can({ + subject: `User:${userId}`, + actions: [relay.permission], + resource: `${relay.resourceType}:${validatedResourceId}`, + }); + allowed = decision.allow; + } catch { + onDenied?.(); + throw new RelaySubscriptionError(status.UNAVAILABLE, 'Authorization check failed'); + } + + if (!allowed) { + onDenied?.(); + throw new RelaySubscriptionError(status.PERMISSION_DENIED, 'Permission denied'); + } + + return eventRelayRoom(relay._id, validatedResourceId); +} + +export function toSubscriptionError(err: unknown): RelaySubscriptionError { + if (err instanceof EventRelayValidationError) { + return new RelaySubscriptionError(status.INVALID_ARGUMENT, err.message); + } + if (err instanceof RelaySubscriptionError) { + return err; + } + return new RelaySubscriptionError( + status.INTERNAL, + err instanceof Error ? err.message : 'Unexpected error', + ); +} diff --git a/modules/router/src/event-relays/channels.test.ts b/modules/router/src/event-relays/channels.test.ts new file mode 100644 index 000000000..9870d0bd9 --- /dev/null +++ b/modules/router/src/event-relays/channels.test.ts @@ -0,0 +1,32 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { groupRelaysByChannel, planChannelSubscriptions } from './channels.js'; + +describe('groupRelaysByChannel', () => { + it('groups multiple relays onto the same exact channel', () => { + const grouped = groupRelaysByChannel([ + { _id: 'a', busEvent: 'orders.paid' }, + { _id: 'b', busEvent: 'orders.paid' }, + { _id: 'c', busEvent: 'orders.shipped' }, + ]); + assert.equal(grouped.get('orders.paid')?.length, 2); + assert.equal(grouped.get('orders.shipped')?.length, 1); + }); +}); + +describe('planChannelSubscriptions', () => { + it('subscribes new channels and unsubscribes removed ones', () => { + const plan = planChannelSubscriptions( + ['orders.paid', 'stale.channel'], + ['orders.paid', 'orders.shipped'], + ); + assert.deepEqual(plan.toSubscribe, ['orders.shipped']); + assert.deepEqual(plan.toUnsubscribe, ['stale.channel']); + }); + + it('is a no-op when the channel set is unchanged', () => { + const plan = planChannelSubscriptions(['orders.paid'], ['orders.paid']); + assert.deepEqual(plan.toSubscribe, []); + assert.deepEqual(plan.toUnsubscribe, []); + }); +}); diff --git a/modules/router/src/event-relays/channels.ts b/modules/router/src/event-relays/channels.ts new file mode 100644 index 000000000..c6ae3d692 --- /dev/null +++ b/modules/router/src/event-relays/channels.ts @@ -0,0 +1,23 @@ +export function groupRelaysByChannel( + relays: T[], +): Map { + const next = new Map(); + for (const relay of relays) { + const list = next.get(relay.busEvent) ?? []; + list.push(relay); + next.set(relay.busEvent, list); + } + return next; +} + +export function planChannelSubscriptions( + currentlySubscribed: Iterable, + nextChannels: Iterable, +): { toSubscribe: string[]; toUnsubscribe: string[] } { + const current = new Set(currentlySubscribed); + const next = new Set(nextChannels); + return { + toUnsubscribe: [...current].filter(channel => !next.has(channel)), + toSubscribe: [...next].filter(channel => !current.has(channel)), + }; +} diff --git a/modules/router/src/event-relays/constants.ts b/modules/router/src/event-relays/constants.ts new file mode 100644 index 000000000..e283da0dd --- /dev/null +++ b/modules/router/src/event-relays/constants.ts @@ -0,0 +1,32 @@ +export const EVENTS_NAMESPACE = '/events/'; +export const EVENT_RELAY_REFRESH_CHANNEL = 'router:event-relays:refresh'; +export const EVENT_RELAY_SUBSCRIBER_PREFIX = 'event-relay:'; + +export const MAX_TEMPLATE_BYTES = 16 * 1024; +export const MAX_OUTPUT_BYTES = 64 * 1024; +export const MAX_TEMPLATE_DEPTH = 10; +export const MAX_PATH_SEGMENTS = 8; +export const MAX_NAME_LENGTH = 64; +export const MAX_DESCRIPTION_LENGTH = 256; +export const MAX_BUS_EVENT_LENGTH = 128; +export const MAX_SOCKET_EVENT_LENGTH = 64; +export const MAX_RESOURCE_TYPE_LENGTH = 64; +export const MAX_RESOURCE_ID_LENGTH = 128; +export const MAX_RESOURCE_ID_PATH_LENGTH = 128; +export const MAX_PERMISSION_LENGTH = 64; + +export const FORBIDDEN_PATH_SEGMENTS = new Set(['__proto__', 'constructor', 'prototype']); + +export const RESERVED_SOCKET_EVENTS = new Set([ + 'connect', + 'disconnect', + 'connect_error', + 'error', + 'join-room', + 'leave-room', + 'conduit_error', + 'subscribe', + 'unsubscribe', + 'ping', + 'pong', +]); diff --git a/modules/router/src/event-relays/index.ts b/modules/router/src/event-relays/index.ts new file mode 100644 index 000000000..caa9a4091 --- /dev/null +++ b/modules/router/src/event-relays/index.ts @@ -0,0 +1,25 @@ +export { EventRelayValidationError } from './validationError.js'; +export { + EVENTS_NAMESPACE, + EVENT_RELAY_REFRESH_CHANNEL, + RESERVED_SOCKET_EVENTS, +} from './constants.js'; +export { lookupOwnPath, parseDotPath, requireOwnPath } from './path.js'; +export { renderMessageTemplate, assertTemplateSize } from './template.js'; +export { + validateEventRelayInput, + validateResourceId, + type EventRelayInput, +} from './validation.js'; +export { eventRelayRoom } from './rooms.js'; +export { + parseBusPayload, + buildRelayEmissions, + type RelayEmission, + type ProcessResult, +} from './process.js'; +export { EventRelayManager } from './EventRelayManager.js'; +export { createEventRelayPusher } from './push.js'; +export { createEventsSocket, authorizeRelaySubscription } from './EventRelaySockets.js'; +export { buildSearchQuery, parsePagination } from './search.js'; +export { groupRelaysByChannel, planChannelSubscriptions } from './channels.js'; diff --git a/modules/router/src/event-relays/path.test.ts b/modules/router/src/event-relays/path.test.ts new file mode 100644 index 000000000..ecd1b0f2b --- /dev/null +++ b/modules/router/src/event-relays/path.test.ts @@ -0,0 +1,39 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { lookupOwnPath, parseDotPath, requireOwnPath } from './path.js'; + +describe('parseDotPath', () => { + it('accepts safe own-property paths', () => { + assert.deepEqual(parseDotPath('_id', 'Path'), ['_id']); + assert.deepEqual(parseDotPath('document.ownerId', 'Path'), ['document', 'ownerId']); + }); + + it('rejects prototype pollution segments', () => { + assert.throws(() => parseDotPath('__proto__.polluted', 'Path'), { + name: 'EventRelayValidationError', + }); + assert.throws(() => parseDotPath('constructor.prototype', 'Path'), { + name: 'EventRelayValidationError', + }); + }); +}); + +describe('lookupOwnPath', () => { + it('reads own properties only', () => { + const payload = { document: { _id: 'abc' } }; + assert.equal(lookupOwnPath(payload, 'document._id'), 'abc'); + }); + + it('does not follow inherited properties', () => { + const payload = Object.create({ leaked: 'nope' }); + payload.own = 'yes'; + assert.equal(lookupOwnPath(payload, 'own'), 'yes'); + assert.equal(lookupOwnPath(payload, 'leaked'), undefined); + }); + + it('requireOwnPath fails closed on missing fields', () => { + assert.throws(() => requireOwnPath({ a: 1 }, 'b', 'Resource ID path'), { + name: 'EventRelayValidationError', + }); + }); +}); diff --git a/modules/router/src/event-relays/path.ts b/modules/router/src/event-relays/path.ts new file mode 100644 index 000000000..e4ba89d7c --- /dev/null +++ b/modules/router/src/event-relays/path.ts @@ -0,0 +1,49 @@ +import { FORBIDDEN_PATH_SEGMENTS, MAX_PATH_SEGMENTS } from './constants.js'; +import { EventRelayValidationError } from './validationError.js'; + +const PATH_SEGMENT = /^[A-Za-z_][A-Za-z0-9_]*$/; + +export function parseDotPath(path: string, label: string): string[] { + const trimmed = path.trim(); + if (!trimmed) { + throw new EventRelayValidationError(`${label} is required`); + } + const segments = trimmed.split('.'); + if (segments.length > MAX_PATH_SEGMENTS) { + throw new EventRelayValidationError(`${label} exceeds ${MAX_PATH_SEGMENTS} segments`); + } + for (const segment of segments) { + if (FORBIDDEN_PATH_SEGMENTS.has(segment) || !PATH_SEGMENT.test(segment)) { + throw new EventRelayValidationError(`${label} contains an invalid path segment`); + } + } + return segments; +} + +export function lookupOwnPath(source: unknown, path: string): unknown { + const segments = parseDotPath(path, 'Path'); + let current: unknown = source; + for (const segment of segments) { + if (current === null || typeof current !== 'object') { + return undefined; + } + if (FORBIDDEN_PATH_SEGMENTS.has(segment)) { + return undefined; + } + if (!Object.prototype.hasOwnProperty.call(current, segment)) { + return undefined; + } + current = (current as Record)[segment]; + } + return current; +} + +export function requireOwnPath(source: unknown, path: string, label: string): unknown { + const value = lookupOwnPath(source, path); + if (value === undefined) { + throw new EventRelayValidationError( + `${label} '${path}' was not found on the payload`, + ); + } + return value; +} diff --git a/modules/router/src/event-relays/process.test.ts b/modules/router/src/event-relays/process.test.ts new file mode 100644 index 000000000..cf3c1c10b --- /dev/null +++ b/modules/router/src/event-relays/process.test.ts @@ -0,0 +1,85 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { buildRelayEmissions, parseBusPayload } from './process.js'; +import { createEventRelayPusher } from './push.js'; +import { eventRelayRoom } from './rooms.js'; +import { EVENTS_NAMESPACE } from './constants.js'; + +const relayA = { + _id: 'relay-a', + busEvent: 'database:update:Order', + socketEvent: 'order-updated', + resourceIdPath: '_id', + messageTemplate: { id: '{{payload._id}}', status: '{{payload.status}}' }, +}; + +const relayB = { + _id: 'relay-b', + busEvent: 'database:update:Order', + socketEvent: 'order-paid', + resourceIdPath: '_id', + messageTemplate: { paid: '{{payload.status}}' }, +}; + +describe('parseBusPayload', () => { + it('parses JSON once', () => { + assert.deepEqual(parseBusPayload('{"_id":"1"}'), { _id: '1' }); + }); + + it('fails closed on malformed JSON', () => { + assert.throws(() => parseBusPayload('{'), { name: 'EventRelayValidationError' }); + assert.throws(() => parseBusPayload(''), { name: 'EventRelayValidationError' }); + }); +}); + +describe('buildRelayEmissions', () => { + it('emits to every mapping on the same channel', () => { + const result = buildRelayEmissions([relayA, relayB], { + _id: 'order-1', + status: 'paid', + }); + assert.equal(result.failures.length, 0); + assert.equal(result.emissions.length, 2); + assert.deepEqual( + result.emissions.map(item => item.socketEvent), + ['order-updated', 'order-paid'], + ); + assert.equal(result.emissions[0].room, eventRelayRoom('relay-a', 'order-1')); + }); + + it('records per-relay failures without dropping siblings', () => { + const result = buildRelayEmissions( + [ + relayA, + { + ...relayB, + resourceIdPath: 'missing', + }, + ], + { _id: 'order-1', status: 'paid' }, + ); + assert.equal(result.emissions.length, 1); + assert.equal(result.failures.length, 1); + assert.equal(result.failures[0].relayId, 'relay-b'); + }); +}); + +describe('createEventRelayPusher', () => { + it('pushes locally to /events/ rooms so HA instances do not duplicate', async () => { + const calls: unknown[] = []; + const push = createEventRelayPusher(async data => { + calls.push(data); + }); + await push('order-updated', { id: '1' }, ['room-1']); + assert.deepEqual(calls, [ + { + event: 'order-updated', + data: { id: '1' }, + receivers: [], + rooms: ['room-1'], + namespace: EVENTS_NAMESPACE, + localOnly: true, + }, + ]); + }); +}); diff --git a/modules/router/src/event-relays/process.ts b/modules/router/src/event-relays/process.ts new file mode 100644 index 000000000..858c9f6d0 --- /dev/null +++ b/modules/router/src/event-relays/process.ts @@ -0,0 +1,75 @@ +import { requireOwnPath } from './path.js'; +import { eventRelayRoom } from './rooms.js'; +import { renderMessageTemplate } from './template.js'; +import { validateResourceId } from './validation.js'; +import { EventRelayValidationError } from './validationError.js'; + +export type RelayProcessInput = { + _id: string; + busEvent: string; + socketEvent: string; + resourceIdPath: string; + messageTemplate: unknown; +}; + +export type RelayEmission = { + relayId: string; + busEvent: string; + socketEvent: string; + room: string; + data: unknown; +}; + +export type RelayFailure = { + relayId: string; + busEvent: string; + reason: string; +}; + +export type ProcessResult = { + emissions: RelayEmission[]; + failures: RelayFailure[]; +}; + +export function parseBusPayload(rawMessage: string): unknown { + if (typeof rawMessage !== 'string' || rawMessage.trim() === '') { + throw new EventRelayValidationError('Bus payload is empty'); + } + try { + return JSON.parse(rawMessage); + } catch { + throw new EventRelayValidationError('Bus payload is not valid JSON'); + } +} + +export function buildRelayEmissions( + relays: RelayProcessInput[], + payload: unknown, +): ProcessResult { + const emissions: RelayEmission[] = []; + const failures: RelayFailure[] = []; + + for (const relay of relays) { + try { + const resourceId = validateResourceId( + requireOwnPath(payload, relay.resourceIdPath, 'Resource ID path'), + ); + const data = renderMessageTemplate(relay.messageTemplate, payload); + emissions.push({ + relayId: relay._id, + busEvent: relay.busEvent, + socketEvent: relay.socketEvent, + room: eventRelayRoom(relay._id, resourceId), + data, + }); + } catch (err) { + failures.push({ + relayId: relay._id, + busEvent: relay.busEvent, + reason: err instanceof Error ? err.message : String(err), + }); + } + } + + return { emissions, failures }; +} diff --git a/modules/router/src/event-relays/push.ts b/modules/router/src/event-relays/push.ts new file mode 100644 index 000000000..79d9911b2 --- /dev/null +++ b/modules/router/src/event-relays/push.ts @@ -0,0 +1,28 @@ +import { EVENTS_NAMESPACE } from './constants.js'; + +export type EventRelayPusher = ( + event: string, + data: unknown, + rooms: string[], +) => Promise; + +export type SocketPushFn = (data: { + event: string; + data?: unknown; + receivers: string[]; + rooms: string[]; + namespace: string; + localOnly?: boolean; +}) => Promise; + +export function createEventRelayPusher(socketPush: SocketPushFn): EventRelayPusher { + return (event, data, rooms) => + socketPush({ + event, + data, + receivers: [], + rooms, + namespace: EVENTS_NAMESPACE, + localOnly: true, + }); +} diff --git a/modules/router/src/event-relays/rooms.test.ts b/modules/router/src/event-relays/rooms.test.ts new file mode 100644 index 000000000..e73b19a77 --- /dev/null +++ b/modules/router/src/event-relays/rooms.test.ts @@ -0,0 +1,29 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { eventRelayRoom } from './rooms.js'; + +describe('eventRelayRoom', () => { + it('is deterministic for the same relay and resource', () => { + assert.equal( + eventRelayRoom('relay-1', 'resource-a'), + eventRelayRoom('relay-1', 'resource-a'), + ); + }); + + it('changes when the resource or relay changes', () => { + assert.notEqual( + eventRelayRoom('relay-1', 'resource-a'), + eventRelayRoom('relay-1', 'resource-b'), + ); + assert.notEqual( + eventRelayRoom('relay-1', 'resource-a'), + eventRelayRoom('relay-2', 'resource-a'), + ); + }); + + it('does not embed the raw resource id', () => { + const room = eventRelayRoom('relay-1', 'secret-resource'); + assert.equal(room.includes('secret-resource'), false); + assert.match(room, /^er:relay-1:[a-f0-9]{64}$/); + }); +}); diff --git a/modules/router/src/event-relays/rooms.ts b/modules/router/src/event-relays/rooms.ts new file mode 100644 index 000000000..fd5dd54ec --- /dev/null +++ b/modules/router/src/event-relays/rooms.ts @@ -0,0 +1,6 @@ +import { createHash } from 'node:crypto'; + +export function eventRelayRoom(relayId: string, resourceId: string): string { + const digest = createHash('sha256').update(resourceId).digest('hex'); + return `er:${relayId}:${digest}`; +} diff --git a/modules/router/src/event-relays/search.test.ts b/modules/router/src/event-relays/search.test.ts new file mode 100644 index 000000000..308ea2890 --- /dev/null +++ b/modules/router/src/event-relays/search.test.ts @@ -0,0 +1,34 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { buildSearchQuery, parsePagination } from './search.js'; + +describe('parsePagination', () => { + it('defaults skip and limit for the admin list', () => { + assert.deepEqual(parsePagination(), { skip: 0, limit: 25 }); + assert.deepEqual(parsePagination(10, 5), { skip: 10, limit: 5 }); + }); +}); + +describe('buildSearchQuery', () => { + it('returns an empty query without search', () => { + assert.deepEqual(buildSearchQuery(), {}); + assert.deepEqual(buildSearchQuery(''), {}); + }); + + it('looks up ObjectIds exactly', () => { + assert.deepEqual(buildSearchQuery('507f1f77bcf86cd799439011'), { + _id: '507f1f77bcf86cd799439011', + }); + }); + + it('escapes regex metacharacters in name/channel search', () => { + const query = buildSearchQuery('order.paid*'); + assert.deepEqual(query, { + $or: [ + { name: { $regex: '.*order\\.paid\\*.*', $options: 'i' } }, + { busEvent: { $regex: '.*order\\.paid\\*.*', $options: 'i' } }, + { socketEvent: { $regex: '.*order\\.paid\\*.*', $options: 'i' } }, + ], + }); + }); +}); diff --git a/modules/router/src/event-relays/search.ts b/modules/router/src/event-relays/search.ts new file mode 100644 index 000000000..c336aa363 --- /dev/null +++ b/modules/router/src/event-relays/search.ts @@ -0,0 +1,26 @@ +export function parsePagination( + skip?: number, + limit?: number, +): { skip: number; limit: number } { + return { + skip: skip ?? 0, + limit: limit ?? 25, + }; +} + +export function buildSearchQuery(search?: string | null): Record { + if (search == null || search === '') { + return {}; + } + if (/^[a-fA-F\d]{24}$/.test(search)) { + return { _id: search }; + } + const identifier = search.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return { + $or: [ + { name: { $regex: `.*${identifier}.*`, $options: 'i' } }, + { busEvent: { $regex: `.*${identifier}.*`, $options: 'i' } }, + { socketEvent: { $regex: `.*${identifier}.*`, $options: 'i' } }, + ], + }; +} diff --git a/modules/router/src/event-relays/template.test.ts b/modules/router/src/event-relays/template.test.ts new file mode 100644 index 000000000..7959a08bd --- /dev/null +++ b/modules/router/src/event-relays/template.test.ts @@ -0,0 +1,39 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { renderMessageTemplate } from './template.js'; + +describe('renderMessageTemplate', () => { + it('substitutes exact placeholders with the raw JSON value', () => { + const rendered = renderMessageTemplate( + { id: '{{payload._id}}', count: '{{payload.count}}' }, + { _id: 'abc', count: 3 }, + ); + assert.deepEqual(rendered, { id: 'abc', count: 3 }); + }); + + it('interpolates placeholders inside strings', () => { + const rendered = renderMessageTemplate( + { label: 'Order {{payload.status}}' }, + { status: 'paid' }, + ); + assert.deepEqual(rendered, { label: 'Order paid' }); + }); + + it('fails closed when a placeholder is missing', () => { + assert.throws( + () => renderMessageTemplate({ id: '{{payload.missing}}' }, { _id: 'abc' }), + { name: 'EventRelayValidationError' }, + ); + }); + + it('rejects prototype paths in placeholders', () => { + assert.throws( + () => + renderMessageTemplate( + { hack: '{{payload.__proto__.polluted}}' }, + { __proto__: { polluted: true } }, + ), + { name: 'EventRelayValidationError' }, + ); + }); +}); diff --git a/modules/router/src/event-relays/template.ts b/modules/router/src/event-relays/template.ts new file mode 100644 index 000000000..f0f70dcc9 --- /dev/null +++ b/modules/router/src/event-relays/template.ts @@ -0,0 +1,136 @@ +import { MAX_OUTPUT_BYTES, MAX_TEMPLATE_BYTES, MAX_TEMPLATE_DEPTH } from './constants.js'; +import { lookupOwnPath, parseDotPath } from './path.js'; +import { EventRelayValidationError } from './validationError.js'; + +const PLACEHOLDER = /\{\{\s*payload\.([A-Za-z_][A-Za-z0-9_.]*)\s*\}\}/g; +const EXACT_PLACEHOLDER = /^\{\{\s*payload\.([A-Za-z_][A-Za-z0-9_.]*)\s*\}\}$/; + +export function assertTemplateSize(template: unknown): void { + let serialized: string; + try { + serialized = JSON.stringify(template); + } catch { + throw new EventRelayValidationError('Message template must be valid JSON'); + } + if (serialized === undefined) { + throw new EventRelayValidationError('Message template must be valid JSON'); + } + if (Buffer.byteLength(serialized, 'utf8') > MAX_TEMPLATE_BYTES) { + throw new EventRelayValidationError( + `Message template exceeds ${MAX_TEMPLATE_BYTES} bytes`, + ); + } + walkTemplate(template, 0, (value, depth) => { + if (depth > MAX_TEMPLATE_DEPTH) { + throw new EventRelayValidationError( + `Message template exceeds ${MAX_TEMPLATE_DEPTH} nested levels`, + ); + } + if (typeof value === 'string') { + validatePlaceholders(value); + } + }); +} + +export function renderMessageTemplate(template: unknown, payload: unknown): unknown { + assertTemplateSize(template); + const rendered = renderValue(template, payload, 0); + const serialized = JSON.stringify(rendered); + if (Buffer.byteLength(serialized, 'utf8') > MAX_OUTPUT_BYTES) { + throw new EventRelayValidationError( + `Rendered message exceeds ${MAX_OUTPUT_BYTES} bytes`, + ); + } + return rendered; +} + +function renderValue(value: unknown, payload: unknown, depth: number): unknown { + if (depth > MAX_TEMPLATE_DEPTH) { + throw new EventRelayValidationError( + `Message template exceeds ${MAX_TEMPLATE_DEPTH} nested levels`, + ); + } + if (typeof value === 'string') { + return interpolateString(value, payload); + } + if (Array.isArray(value)) { + return value.map(item => renderValue(item, payload, depth + 1)); + } + if (value !== null && typeof value === 'object') { + const output: Record = {}; + for (const key of Object.keys(value as Record)) { + output[key] = renderValue( + (value as Record)[key], + payload, + depth + 1, + ); + } + return output; + } + return value; +} + +function interpolateString(value: string, payload: unknown): unknown { + const exact = value.trim().match(EXACT_PLACEHOLDER); + if (exact) { + parseDotPath(exact[1], 'Placeholder path'); + const resolved = lookupOwnPath(payload, exact[1]); + if (resolved === undefined) { + throw new EventRelayValidationError( + `Placeholder payload.${exact[1]} was not found on the payload`, + ); + } + return resolved; + } + + return value.replace(PLACEHOLDER, (_match, path: string) => { + parseDotPath(path, 'Placeholder path'); + const resolved = lookupOwnPath(payload, path); + if (resolved === undefined) { + throw new EventRelayValidationError( + `Placeholder payload.${path} was not found on the payload`, + ); + } + return stringifyPlaceholder(resolved); + }); +} + +function stringifyPlaceholder(value: unknown): string { + if (value === null || typeof value === 'string' || typeof value === 'number') { + return String(value); + } + if (typeof value === 'boolean') { + return value ? 'true' : 'false'; + } + return JSON.stringify(value); +} + +function validatePlaceholders(value: string): void { + const exact = value.trim().match(EXACT_PLACEHOLDER); + if (exact) { + parseDotPath(exact[1], 'Placeholder path'); + return; + } + for (const match of value.matchAll(PLACEHOLDER)) { + parseDotPath(match[1], 'Placeholder path'); + } +} + +function walkTemplate( + value: unknown, + depth: number, + visit: (value: unknown, depth: number) => void, +): void { + visit(value, depth); + if (Array.isArray(value)) { + for (const item of value) { + walkTemplate(item, depth + 1, visit); + } + return; + } + if (value !== null && typeof value === 'object') { + for (const nested of Object.values(value as Record)) { + walkTemplate(nested, depth + 1, visit); + } + } +} diff --git a/modules/router/src/event-relays/validation.test.ts b/modules/router/src/event-relays/validation.test.ts new file mode 100644 index 000000000..d8337a5f1 --- /dev/null +++ b/modules/router/src/event-relays/validation.test.ts @@ -0,0 +1,69 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { validateEventRelayInput, validateResourceId } from './validation.js'; + +const validInput = { + name: 'Order paid', + busEvent: 'database:update:Order', + socketEvent: 'order-updated', + resourceType: 'Order', + resourceIdPath: '_id', + permission: 'read', + messageTemplate: { id: '{{payload._id}}' }, +}; + +describe('validateEventRelayInput', () => { + it('accepts a complete relay definition', () => { + const parsed = validateEventRelayInput(validInput); + assert.equal(parsed.active, true); + assert.equal(parsed.busEvent, 'database:update:Order'); + }); + + it('rejects wildcard bus channels', () => { + assert.throws( + () => validateEventRelayInput({ ...validInput, busEvent: 'database:update:*' }), + { name: 'EventRelayValidationError' }, + ); + }); + + it('rejects reserved socket events', () => { + assert.throws( + () => validateEventRelayInput({ ...validInput, socketEvent: 'subscribe' }), + { name: 'EventRelayValidationError' }, + ); + assert.throws( + () => validateEventRelayInput({ ...validInput, socketEvent: 'join-room' }), + { name: 'EventRelayValidationError' }, + ); + }); + + it('rejects empty required fields', () => { + assert.throws(() => validateEventRelayInput({ ...validInput, name: '' }), { + name: 'EventRelayValidationError', + }); + assert.throws( + () => validateEventRelayInput({ ...validInput, messageTemplate: undefined }), + { name: 'EventRelayValidationError' }, + ); + }); +}); + +describe('validateResourceId', () => { + it('accepts object ids and uuids', () => { + assert.equal( + validateResourceId('507f1f77bcf86cd799439011'), + '507f1f77bcf86cd799439011', + ); + assert.equal( + validateResourceId('3b2c1a90-1111-2222-3333-444444444444'), + '3b2c1a90-1111-2222-3333-444444444444', + ); + }); + + it('rejects empty or colon-containing identifiers', () => { + assert.throws(() => validateResourceId(''), { name: 'EventRelayValidationError' }); + assert.throws(() => validateResourceId('Team:123'), { + name: 'EventRelayValidationError', + }); + }); +}); diff --git a/modules/router/src/event-relays/validation.ts b/modules/router/src/event-relays/validation.ts new file mode 100644 index 000000000..d6c6c5eb6 --- /dev/null +++ b/modules/router/src/event-relays/validation.ts @@ -0,0 +1,130 @@ +import { + MAX_BUS_EVENT_LENGTH, + MAX_DESCRIPTION_LENGTH, + MAX_NAME_LENGTH, + MAX_PERMISSION_LENGTH, + MAX_RESOURCE_ID_LENGTH, + MAX_RESOURCE_ID_PATH_LENGTH, + MAX_RESOURCE_TYPE_LENGTH, + MAX_SOCKET_EVENT_LENGTH, + RESERVED_SOCKET_EVENTS, +} from './constants.js'; +import { parseDotPath } from './path.js'; +import { assertTemplateSize } from './template.js'; +import { EventRelayValidationError } from './validationError.js'; + +export type EventRelayInput = { + name: string; + notes?: string; + active?: boolean; + busEvent: string; + socketEvent: string; + resourceType: string; + resourceIdPath: string; + permission: string; + messageTemplate: unknown; +}; + +const NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9 _.-]{0,63}$/; +const BUS_EVENT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/; +const SOCKET_EVENT_PATTERN = /^[A-Za-z][A-Za-z0-9_:-]{0,63}$/; +const RESOURCE_TYPE_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,63}$/; +const PERMISSION_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,63}$/; +const RESOURCE_ID_PATTERN = /^[^\s:]{1,128}$/; + +export function validateEventRelayInput(input: EventRelayInput): EventRelayInput { + const name = requireTrimmed(input.name, 'Name'); + if (name.length > MAX_NAME_LENGTH || !NAME_PATTERN.test(name)) { + throw new EventRelayValidationError( + 'Name must be 1-64 characters and start with a letter or number', + ); + } + + const notes = + input.notes === undefined || input.notes === '' + ? undefined + : requireTrimmed(input.notes, 'Notes'); + if (notes && notes.length > MAX_DESCRIPTION_LENGTH) { + throw new EventRelayValidationError( + `Notes must be at most ${MAX_DESCRIPTION_LENGTH} characters`, + ); + } + + const busEvent = requireTrimmed(input.busEvent, 'Bus event'); + if ( + busEvent.length > MAX_BUS_EVENT_LENGTH || + busEvent.includes('*') || + !BUS_EVENT_PATTERN.test(busEvent) + ) { + throw new EventRelayValidationError( + 'Bus event must be an exact channel name with no wildcards', + ); + } + + const socketEvent = requireTrimmed(input.socketEvent, 'Socket event'); + if ( + socketEvent.length > MAX_SOCKET_EVENT_LENGTH || + !SOCKET_EVENT_PATTERN.test(socketEvent) || + RESERVED_SOCKET_EVENTS.has(socketEvent) + ) { + throw new EventRelayValidationError('Socket event must be a non-reserved event name'); + } + + const resourceType = requireTrimmed(input.resourceType, 'Resource type'); + if ( + resourceType.length > MAX_RESOURCE_TYPE_LENGTH || + !RESOURCE_TYPE_PATTERN.test(resourceType) + ) { + throw new EventRelayValidationError('Resource type is invalid'); + } + + const resourceIdPath = requireTrimmed(input.resourceIdPath, 'Resource ID path'); + if (resourceIdPath.length > MAX_RESOURCE_ID_PATH_LENGTH) { + throw new EventRelayValidationError('Resource ID path is too long'); + } + parseDotPath(resourceIdPath, 'Resource ID path'); + + const permission = requireTrimmed(input.permission, 'Permission'); + if (permission.length > MAX_PERMISSION_LENGTH || !PERMISSION_PATTERN.test(permission)) { + throw new EventRelayValidationError('Permission is invalid'); + } + + if (input.messageTemplate === undefined) { + throw new EventRelayValidationError('Message template is required'); + } + assertTemplateSize(input.messageTemplate); + + return { + name, + notes, + active: input.active !== false, + busEvent, + socketEvent, + resourceType, + resourceIdPath, + permission, + messageTemplate: input.messageTemplate, + }; +} + +export function validateResourceId(resourceId: unknown): string { + if (typeof resourceId !== 'string' && typeof resourceId !== 'number') { + throw new EventRelayValidationError('Resource ID must be a string'); + } + const value = String(resourceId).trim(); + if ( + !value || + value.length > MAX_RESOURCE_ID_LENGTH || + !RESOURCE_ID_PATTERN.test(value) + ) { + throw new EventRelayValidationError('Resource ID is invalid'); + } + return value; +} + +function requireTrimmed(value: unknown, label: string): string { + if (typeof value !== 'string' || value.trim() === '') { + throw new EventRelayValidationError(`${label} is required`); + } + return value.trim(); +} diff --git a/modules/router/src/event-relays/validationError.ts b/modules/router/src/event-relays/validationError.ts new file mode 100644 index 000000000..90151b042 --- /dev/null +++ b/modules/router/src/event-relays/validationError.ts @@ -0,0 +1,6 @@ +export class EventRelayValidationError extends Error { + constructor(message: string) { + super(message); + this.name = 'EventRelayValidationError'; + } +} diff --git a/modules/router/src/metrics/index.ts b/modules/router/src/metrics/index.ts index 9dd6d9789..d3461c107 100644 --- a/modules/router/src/metrics/index.ts +++ b/modules/router/src/metrics/index.ts @@ -17,4 +17,25 @@ export default { labelNames: ['platform'], }, }, + eventRelaysEmitted: { + type: MetricType.Counter, + config: { + name: 'event_relays_emitted_total', + help: 'Tracks successfully emitted event-to-socket relays', + }, + }, + eventRelaysFailed: { + type: MetricType.Counter, + config: { + name: 'event_relays_failed_total', + help: 'Tracks event-to-socket relay parse, render, or emit failures', + }, + }, + eventRelaySubscriptionsDenied: { + type: MetricType.Counter, + config: { + name: 'event_relay_subscriptions_denied_total', + help: 'Tracks denied or unavailable event-relay socket subscriptions', + }, + }, }; diff --git a/modules/router/src/models/EventRelay.schema.ts b/modules/router/src/models/EventRelay.schema.ts new file mode 100644 index 000000000..e0c07b1d9 --- /dev/null +++ b/modules/router/src/models/EventRelay.schema.ts @@ -0,0 +1,94 @@ +import { + ConduitModel, + DatabaseProvider, + Indexable, + TYPE, +} from '@conduitplatform/grpc-sdk'; +import { ConduitActiveSchema } from '@conduitplatform/module-tools'; + +const schema: ConduitModel = { + _id: TYPE.ObjectId, + name: { + type: TYPE.String, + unique: true, + required: true, + }, + notes: { + type: TYPE.String, + required: false, + }, + active: { + type: TYPE.Boolean, + required: true, + default: true, + }, + busEvent: { + type: TYPE.String, + required: true, + }, + socketEvent: { + type: TYPE.String, + required: true, + }, + resourceType: { + type: TYPE.String, + required: true, + }, + resourceIdPath: { + type: TYPE.String, + required: true, + }, + permission: { + type: TYPE.String, + required: true, + }, + messageTemplate: { + type: TYPE.JSON, + required: true, + }, + createdAt: TYPE.Date, + updatedAt: TYPE.Date, +}; + +const modelOptions = { + timestamps: true, + conduit: { + permissions: { + extendable: true, + canCreate: false, + canModify: 'ExtensionOnly', + canDelete: false, + }, + }, +} as const; + +const collectionName = undefined; + +export class EventRelay extends ConduitActiveSchema { + private static _instance: EventRelay; + _id!: string; + declare name: string; + notes?: string; + active!: boolean; + busEvent!: string; + socketEvent!: string; + resourceType!: string; + resourceIdPath!: string; + permission!: string; + messageTemplate!: Indexable; + createdAt!: Date; + updatedAt!: Date; + + private constructor(database: DatabaseProvider) { + super(database, EventRelay.name, schema, modelOptions, collectionName); + } + + static getInstance(database?: DatabaseProvider) { + if (EventRelay._instance) return EventRelay._instance; + if (!database) { + throw new Error('No database instance provided!'); + } + EventRelay._instance = new EventRelay(database); + return EventRelay._instance; + } +} diff --git a/modules/router/src/models/index.ts b/modules/router/src/models/index.ts index 400d89cdd..ae11db58e 100644 --- a/modules/router/src/models/index.ts +++ b/modules/router/src/models/index.ts @@ -1,2 +1,3 @@ export * from './Client.schema.js'; export * from './AppMiddleware.schema.js'; +export * from './EventRelay.schema.js'; diff --git a/modules/router/tsconfig.test.json b/modules/router/tsconfig.test.json new file mode 100644 index 000000000..601e5ccee --- /dev/null +++ b/modules/router/tsconfig.test.json @@ -0,0 +1,24 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": [ + "src/event-relays/constants.ts", + "src/event-relays/validationError.ts", + "src/event-relays/path.ts", + "src/event-relays/template.ts", + "src/event-relays/validation.ts", + "src/event-relays/rooms.ts", + "src/event-relays/process.ts", + "src/event-relays/push.ts", + "src/event-relays/search.ts", + "src/event-relays/channels.ts", + "src/event-relays/authorize.ts", + "src/event-relays/*.test.ts" + ] +} From 1609f4d4a571e165c559b92343fb40b21d740c5c Mon Sep 17 00:00:00 2001 From: John Chantzigoulas Date: Wed, 9 Sep 2026 11:09:40 +0300 Subject: [PATCH 2/6] feat(database): stream live document updates over admin sockets MongoDB change streams require a replica set, so local Compose now initializes a single-node rs0. Schemas can opt in via realtime.enabled. --- docker/.env | 2 +- docker/docker-compose.standalone.yml | 25 +- docker/docker-compose.yml | 30 +- docker/mongo/init-replica.sh | 20 ++ docker/mongo/keyfile | 1 + libraries/grpc-sdk/src/interfaces/Model.ts | 7 +- libraries/grpc-sdk/src/modules/admin/index.ts | 7 +- modules/database/README.mdx | 18 + modules/database/src/Database.ts | 7 + modules/database/src/admin/index.ts | 3 + modules/database/src/admin/schema.admin.ts | 2 + modules/database/src/config/index.ts | 7 + .../database/src/interfaces/ConduitOptions.ts | 3 + modules/database/src/metrics/index.ts | 15 + .../realtime/MongoChangeStreamCoordinator.ts | 329 ++++++++++++++++++ .../database/src/realtime/RealtimeService.ts | 160 +++++++++ .../src/realtime/__tests__/authorize.test.ts | 82 +++++ .../change-stream.integration.test.ts | 47 +++ .../realtime/__tests__/coordinator.test.ts | 166 +++++++++ .../src/realtime/__tests__/normalize.test.ts | 38 ++ .../src/realtime/__tests__/rooms.test.ts | 30 ++ .../src/realtime/__tests__/status.test.ts | 53 +++ .../realtime/__tests__/subscriptions.test.ts | 55 +++ .../src/realtime/__tests__/topology.test.ts | 16 + modules/database/src/realtime/authorize.ts | 147 ++++++++ modules/database/src/realtime/index.ts | 11 + modules/database/src/realtime/normalize.ts | 70 ++++ modules/database/src/realtime/rooms.ts | 25 ++ modules/database/src/realtime/sockets.ts | 166 +++++++++ modules/database/src/realtime/status.ts | 74 ++++ .../database/src/realtime/subscriptions.ts | 92 +++++ modules/database/src/realtime/topology.ts | 49 +++ modules/database/src/realtime/types.ts | 49 +++ modules/database/src/routes/index.ts | 4 +- modules/database/src/utils/SchemaConverter.ts | 6 + packages/core/package.json | 1 + packages/core/src/admin/AdminModule.ts | 36 ++ .../src/admin/middleware/Admin.middleware.ts | 10 +- .../src/admin/middleware/Auth.middleware.ts | 6 + .../core/src/admin/realtime/handshake.test.ts | 12 + packages/core/src/admin/realtime/handshake.ts | 14 + .../core/src/admin/realtime/namespace.test.ts | 11 + packages/core/src/admin/realtime/namespace.ts | 7 + .../core/src/admin/realtime/ticket.test.ts | 26 ++ packages/core/src/admin/realtime/ticket.ts | 26 ++ .../src/admin/routes/RealtimeTicket.route.ts | 51 +++ packages/core/src/admin/routes/index.ts | 1 + packages/core/src/core.proto | 8 + packages/core/tsconfig.json | 2 +- packages/core/tsconfig.test.json | 17 + 50 files changed, 2033 insertions(+), 11 deletions(-) create mode 100644 docker/mongo/init-replica.sh create mode 100644 docker/mongo/keyfile create mode 100644 modules/database/src/realtime/MongoChangeStreamCoordinator.ts create mode 100644 modules/database/src/realtime/RealtimeService.ts create mode 100644 modules/database/src/realtime/__tests__/authorize.test.ts create mode 100644 modules/database/src/realtime/__tests__/change-stream.integration.test.ts create mode 100644 modules/database/src/realtime/__tests__/coordinator.test.ts create mode 100644 modules/database/src/realtime/__tests__/normalize.test.ts create mode 100644 modules/database/src/realtime/__tests__/rooms.test.ts create mode 100644 modules/database/src/realtime/__tests__/status.test.ts create mode 100644 modules/database/src/realtime/__tests__/subscriptions.test.ts create mode 100644 modules/database/src/realtime/__tests__/topology.test.ts create mode 100644 modules/database/src/realtime/authorize.ts create mode 100644 modules/database/src/realtime/index.ts create mode 100644 modules/database/src/realtime/normalize.ts create mode 100644 modules/database/src/realtime/rooms.ts create mode 100644 modules/database/src/realtime/sockets.ts create mode 100644 modules/database/src/realtime/status.ts create mode 100644 modules/database/src/realtime/subscriptions.ts create mode 100644 modules/database/src/realtime/topology.ts create mode 100644 modules/database/src/realtime/types.ts create mode 100644 packages/core/src/admin/realtime/handshake.test.ts create mode 100644 packages/core/src/admin/realtime/handshake.ts create mode 100644 packages/core/src/admin/realtime/namespace.test.ts create mode 100644 packages/core/src/admin/realtime/namespace.ts create mode 100644 packages/core/src/admin/realtime/ticket.test.ts create mode 100644 packages/core/src/admin/realtime/ticket.ts create mode 100644 packages/core/src/admin/routes/RealtimeTicket.route.ts create mode 100644 packages/core/tsconfig.test.json diff --git a/docker/.env b/docker/.env index 66dad89bd..b71166ea3 100644 --- a/docker/.env +++ b/docker/.env @@ -26,7 +26,7 @@ DB_TYPE="mongodb" DB_USER="conduit" DB_PASS="pass" DB_PORT="27017" -DB_CONN_URI="mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin" # profile: mongodb +DB_CONN_URI="mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin&replicaSet=rs0" # profile: mongodb #DB_CONN_URI="postgres://conduit:pass@conduit-postgres:5432/conduit" # profile: postgres # Security diff --git a/docker/docker-compose.standalone.yml b/docker/docker-compose.standalone.yml index 8cd0b0e9b..e1e5225be 100644 --- a/docker/docker-compose.standalone.yml +++ b/docker/docker-compose.standalone.yml @@ -38,7 +38,7 @@ services: ADMIN_SOCKET_PORT: '${ADMIN_SOCKET_PORT:-3031}' __DEFAULT_HOST_URL: '${ADMIN_DEFAULT_HOST_URL:-http://localhost:3030}' GRPC_KEY: '${GRPC_KEY}' - DB_CONN_URI: '${DB_CONN_URI:-mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin}' + DB_CONN_URI: '${DB_CONN_URI:-mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin&replicaSet=rs0}' networks: default: aliases: @@ -74,12 +74,35 @@ services: MONGO_INITDB_DATABASE: 'conduit' MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' + entrypoint: + - bash + - -c + - | + cp /mongo-keyfile /tmp/keyfile + chmod 400 /tmp/keyfile + chown mongodb:mongodb /tmp/keyfile + exec docker-entrypoint.sh mongod --replSet rs0 --bind_ip_all --keyFile /tmp/keyfile networks: default: aliases: - conduit-mongo volumes: - mongo:/data/db + - ./mongo/keyfile:/mongo-keyfile:ro + + mongo-init-replica: + container_name: 'conduit-mongo-init' + image: 'docker.io/library/mongo:4.4.15' + restart: on-failure + depends_on: + - mongodb + environment: + MONGO_HOST: 'conduit-mongo' + MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' + MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' + volumes: + - ./mongo/init-replica.sh:/init-replica.sh:ro + command: ['bash', '/init-replica.sh'] # Persistent Volumes volumes: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 83b3e9b55..751ea8bc6 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -85,7 +85,7 @@ services: LOKI_URL: 'http://conduit-loki:3100' GRPC_KEY: '${GRPC_KEY}' DB_TYPE: '${DB_TYPE:-mongodb}' - DB_CONN_URI: '${DB_CONN_URI:-mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin}' + DB_CONN_URI: '${DB_CONN_URI:-mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin&replicaSet=rs0}' networks: default: aliases: @@ -271,12 +271,40 @@ services: MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' profiles: ['mongodb'] + entrypoint: + - bash + - -c + - | + cp /mongo-keyfile /tmp/keyfile + chmod 400 /tmp/keyfile + chown mongodb:mongodb /tmp/keyfile + exec docker-entrypoint.sh mongod --replSet rs0 --bind_ip_all --keyFile /tmp/keyfile networks: default: aliases: - conduit-mongo volumes: - mongo:/data/db + - ./mongo/keyfile:/mongo-keyfile:ro + + mongo-init-replica: + container_name: 'conduit-mongo-init' + image: 'docker.io/library/mongo:4.4.15' + restart: on-failure + profiles: ['mongodb'] + depends_on: + - mongodb + environment: + MONGO_HOST: 'conduit-mongo' + MONGO_INITDB_ROOT_USERNAME: '${DB_USER:-conduit}' + MONGO_INITDB_ROOT_PASSWORD: '${DB_PASS:-pass}' + volumes: + - ./mongo/init-replica.sh:/init-replica.sh:ro + command: ['bash', '/init-replica.sh'] + networks: + default: + aliases: + - conduit-mongo-init postgres: container_name: 'conduit-postgres' diff --git a/docker/mongo/init-replica.sh b/docker/mongo/init-replica.sh new file mode 100644 index 000000000..e4bb536d2 --- /dev/null +++ b/docker/mongo/init-replica.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail +HOST="${MONGO_HOST:-conduit-mongo}" +USER="${MONGO_INITDB_ROOT_USERNAME:-conduit}" +PASS="${MONGO_INITDB_ROOT_PASSWORD:-pass}" + +until mongo --host "$HOST" -u "$USER" -p "$PASS" --authenticationDatabase admin --quiet --eval 'db.adminCommand({ ping: 1 })' >/dev/null 2>&1; do + sleep 2 +done + +mongo --host "$HOST" -u "$USER" -p "$PASS" --authenticationDatabase admin --quiet --eval ' + try { + rs.status(); + } catch (err) { + rs.initiate({ + _id: "rs0", + members: [{ _id: 0, host: "'"$HOST"':27017" }] + }); + } +' diff --git a/docker/mongo/keyfile b/docker/mongo/keyfile new file mode 100644 index 000000000..8d7ffaf78 --- /dev/null +++ b/docker/mongo/keyfile @@ -0,0 +1 @@ +conduit-dev-mongo-replica-keyfile-do-not-use-in-production-replace-me-before-any-real-deployment-0123456789abcdefghijklmnopqrstuvwxyz diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 6b7319a9b..b13fd687b 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -71,9 +71,7 @@ export interface ConduitArrayValidation { } export type ConduitValidationRules = - | ConduitStringValidation - | ConduitNumberValidation - | ConduitArrayValidation; + ConduitStringValidation | ConduitNumberValidation | ConduitArrayValidation; type BaseConduitModelField = { type?: TYPE | TYPE[] | ConduitModel | ArrayConduitModel[]; @@ -190,6 +188,9 @@ export interface ConduitSchemaOptions { authorization?: { enabled: boolean; }; + realtime?: { + enabled: boolean; + }; /** Mongoose read preference for this schema (ignored by SQL); per-query wins. */ readPreference?: string; }; diff --git a/libraries/grpc-sdk/src/modules/admin/index.ts b/libraries/grpc-sdk/src/modules/admin/index.ts index 889ca0ead..57a6ac7bd 100644 --- a/libraries/grpc-sdk/src/modules/admin/index.ts +++ b/libraries/grpc-sdk/src/modules/admin/index.ts @@ -3,7 +3,8 @@ import { AdminDefinition, RegisterAdminRouteRequest, RegisterAdminRouteRequest_PathDefinition, -} from '../../protoUtils/index.js'; + SocketPushRequest, +} from '../../protoUtils/core.js'; import { ConduitRouteActions } from '../../interfaces/index.js'; export class Admin extends ConduitModule { @@ -28,6 +29,10 @@ export class Admin extends ConduitModule { return this.client!.registerAdminRoute(request); } + socketPush(data: SocketPushRequest) { + return this.client!.socketPush(data); + } + patchRouteMiddlewares( path: string, action: ConduitRouteActions, diff --git a/modules/database/README.mdx b/modules/database/README.mdx index c340b86bd..61bc195b0 100644 --- a/modules/database/README.mdx +++ b/modules/database/README.mdx @@ -47,6 +47,23 @@ since the latter need to go through parsers that are otherwise unnecessary for M When using MongoDB with a replica set (e.g., MongoDB Atlas), the database module supports configuring read preference, write concern, and read concern through the admin panel at `PATCH /config/database`. +Live document updates also require a replica set or sharded cluster. Local Compose files initialize a single-node replica set so change streams can be exercised. + +### Live updates + +Enable `realtime.enabled` in database module config, then opt a schema in with `modelOptions.conduit.realtime.enabled`. Clients connect to the `/database/` Socket.IO namespace (path `/realtime`) and emit: + +``` +subscribe({ schema: 'Order', documentId?: string }) +unsubscribe({ schema: 'Order', documentId?: string }) +``` + +Events arrive as `change` with `{ version, operation, schema, documentId, occurredAt, resumeToken }` and contain no document fields. Consumers should refetch through their authorized REST or custom-endpoint path. + +Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; session JWTs and masterkeys must not be sent from browser code. + +Admin sockets must be enabled (`admin.transports.sockets`) and the Admin socket port (`ADMIN_SOCKET_PORT`, default 3031) reachable from the UI. + ### Configuration Options | Setting | Values | Default | Description | @@ -54,6 +71,7 @@ When using MongoDB with a replica set (e.g., MongoDB Atlas), the database module | `readPreference` | `primary`, `primaryPreferred`, `secondary`, `secondaryPreferred`, `nearest` | `primary` | Controls which replica set members receive read queries | | `writeConcern` | `1`, `majority` | `1` | How many members must acknowledge a write | | `readConcern` | `local`, `available`, `majority`, `linearizable`, `snapshot` | `local` | Consistency level for read operations | +| `realtime.enabled` | `true`, `false` | `false` | Enable MongoDB change-stream live updates for opted-in schemas | ### Recommended Production Settings diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index 57286cb1e..e0208f256 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -64,6 +64,7 @@ import { QueueController } from './controllers/queue.controller.js'; import AppConfigSchema, { Config } from './config/index.js'; import { Empty } from './protoTypes/google/protobuf/empty.js'; import { fileURLToPath } from 'node:url'; +import { RealtimeService } from './realtime/index.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -107,6 +108,7 @@ export default class DatabaseModule extends ManagedModule { private customEndpointController?: CustomEndpointController; private _authorizationDefinitionsRegistered = false; private _databaseRouterWatchDispose: (() => void) | null = null; + private realtimeService?: RealtimeService; constructor(dbType: string, dbUri: string, peerManifestRoot?: string) { super('database', peerManifestRoot); @@ -164,6 +166,7 @@ export default class DatabaseModule extends ManagedModule { readConcern: config.readConcern ?? 'local', }); } + void this.realtimeService?.reconcile(); if (!config.viewCleanup.enabled) { try { await QueueController.getInstance().drainViewCleanupQueue(); @@ -1015,13 +1018,16 @@ export default class DatabaseModule extends ManagedModule { this.grpcSdk, this._activeAdapter, ); + this.realtimeService = new RealtimeService(this.grpcSdk, this._activeAdapter); this.adminRouter = new AdminHandlers( this.grpcServer, this.grpcSdk, this._activeAdapter, this.schemaController, this.customEndpointController, + this.realtimeService, ); + void this.realtimeService.reconcile(); this._databaseRouterWatchDispose?.(); this._databaseRouterWatchDispose = this.grpcSdk.watchPeer( 'router', @@ -1031,6 +1037,7 @@ export default class DatabaseModule extends ManagedModule { this.grpcServer, this._activeAdapter, this.grpcSdk, + this.realtimeService, ); this.schemaController?.setRouter(this.userRouter); this.customEndpointController?.setRouter(this.userRouter); diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index a4c1be91f..9742f31eb 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -23,6 +23,7 @@ import { SchemaController } from '../controllers/cms/schema.controller.js'; import { CustomEndpointController } from '../controllers/customEndpoints/customEndpoint.controller.js'; import { CustomEndpoints, DeclaredSchema, PendingSchemas } from '../models/index.js'; import { ConduitOptions, SchemaFieldsRequired } from '../interfaces/index.js'; +import type { RealtimeService } from '../realtime/index.js'; export class AdminHandlers { private readonly schemaAdmin: SchemaAdmin; @@ -36,6 +37,7 @@ export class AdminHandlers { private readonly _activeAdapter: DatabaseAdapter, private readonly schemaController: SchemaController, private readonly customEndpointController: CustomEndpointController, + private readonly realtimeService: RealtimeService, ) { this.schemaAdmin = new SchemaAdmin( this.grpcSdk, @@ -686,6 +688,7 @@ export class AdminHandlers { new ConduitRouteReturnDefinition('getDatabaseType', 'String'), this.schemaAdmin.getDatabaseType.bind(this.schemaAdmin), ); + this.realtimeService.registerAdmin(this.routingManager); this.routingManager.registerRoutes(); } } diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index 9c149d6db..cc8309664 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -250,6 +250,7 @@ export class SchemaAdmin { cms: call.request.params.conduitOptions?.cms, permissions: call.request.params.conduitOptions?.permissions, authorization: call.request.params.conduitOptions?.authorization, + realtime: call.request.params.conduitOptions?.realtime, readPreference: call.request.params.conduitOptions?.readPreference, timestamps: call.request.params.timestamps, }); @@ -343,6 +344,7 @@ export class SchemaAdmin { cms: conduitOptions?.cms, authorization: conduitOptions?.authorization, permissions: conduitOptions?.permissions, + realtime: conduitOptions?.realtime, readPreference: conduitOptions?.readPreference, existingModelOptions: requestedSchema.modelOptions, }); diff --git a/modules/database/src/config/index.ts b/modules/database/src/config/index.ts index 0f5a4fcc8..1ddda80d7 100644 --- a/modules/database/src/config/index.ts +++ b/modules/database/src/config/index.ts @@ -47,6 +47,13 @@ const AppConfigSchema = { default: 0, }, }, + realtime: { + enabled: { + doc: 'Enable MongoDB change-stream live updates for opted-in schemas', + format: 'Boolean', + default: false, + }, + }, }; const config = convict(AppConfigSchema); diff --git a/modules/database/src/interfaces/ConduitOptions.ts b/modules/database/src/interfaces/ConduitOptions.ts index 82ee970fb..f5f1b3b02 100644 --- a/modules/database/src/interfaces/ConduitOptions.ts +++ b/modules/database/src/interfaces/ConduitOptions.ts @@ -30,5 +30,8 @@ export const ConduitOptions = { authorization: { enabled: ConduitBoolean.Optional, }, + realtime: { + enabled: ConduitBoolean.Optional, + }, readPreference: ConduitString.Optional, }; diff --git a/modules/database/src/metrics/index.ts b/modules/database/src/metrics/index.ts index e00ad3a33..3d3ef0256 100644 --- a/modules/database/src/metrics/index.ts +++ b/modules/database/src/metrics/index.ts @@ -31,4 +31,19 @@ export default { help: 'Tracks the total number of custom endpoints', }, }, + realtimeEvents: { + type: MetricType.Counter, + config: { + name: 'database_realtime_events_total', + help: 'Tracks normalized database change-stream events', + labelNames: ['operation'], + }, + }, + realtimeStreamErrors: { + type: MetricType.Counter, + config: { + name: 'database_realtime_stream_errors_total', + help: 'Tracks change-stream errors', + }, + }, }; diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts new file mode 100644 index 000000000..32ff41dc7 --- /dev/null +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -0,0 +1,329 @@ +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { + normalizeChangeEvent, + parseResumeToken, + type RawChangeEvent, +} from './normalize.js'; +import { authorizedDocumentRoom, roomsForPublicChange } from './rooms.js'; +import { + isResumeTokenUnusable, + topologyFromHello, + type TopologyResult, +} from './topology.js'; +import type { + ChangeStreamLike, + DatabaseChangeEvent, + OptedInSchema, + RealtimeStatusCode, +} from './types.js'; +import type { RealtimeSubscriptionTracker } from './subscriptions.js'; +import { canReadDocument, type AuthorizationSdk } from './authorize.js'; + +const LEADER_LOCK = 'realtime:change-stream:leader'; +const RESUME_TOKEN_KEY = 'realtime:resumeToken'; +const LOCK_TTL_MS = 15_000; +const LOCK_RENEW_MS = 5_000; +const RETRY_BASE_MS = 1_000; +const RETRY_MAX_MS = 30_000; + +type LeaderLock = NonNullable< + Awaited['tryAcquireLock']>> +>; + +export type WatchFactory = (options: { resumeAfter?: unknown }) => ChangeStreamLike; + +export type CoordinatorOptions = { + grpcSdk: ConduitGrpcSdk; + watch: WatchFactory; + hello: () => Promise<{ setName?: string; msg?: string } | null>; + getOptedInSchemas: () => OptedInSchema[]; + subscriptions: RealtimeSubscriptionTracker; + enabled: () => boolean; + engine: () => string; + socketsEnabled?: () => Promise; +}; + +export class MongoChangeStreamCoordinator { + private lock: LeaderLock | null = null; + private stream: ChangeStreamLike | null = null; + private renewTimer: NodeJS.Timeout | null = null; + private retryTimer: NodeJS.Timeout | null = null; + private closed = false; + private streamState: RealtimeStatusCode = 'idle'; + private lastEventAt?: string; + private lastError?: string; + private topology: TopologyResult = { supported: false }; + private retryAttempt = 0; + private watching = false; + private ignoreClose = false; + + constructor(private readonly options: CoordinatorOptions) {} + + getState(): RealtimeStatusCode { + return this.streamState; + } + + getLastEventAt(): string | undefined { + return this.lastEventAt; + } + + getLastError(): string | undefined { + return this.lastError; + } + + getTopology(): TopologyResult { + return this.topology; + } + + async reconcile(): Promise { + if (this.closed) return; + const engine = this.options.engine(); + if (engine !== 'MongoDB' || !this.options.enabled()) { + await this.stopStream('idle'); + await this.releaseLeader(); + this.streamState = engine !== 'MongoDB' ? 'unsupported' : 'disabled'; + return; + } + this.topology = topologyFromHello(await this.options.hello().catch(() => null)); + if (!this.topology.supported) { + await this.stopStream('idle'); + await this.releaseLeader(); + this.streamState = 'idle'; + this.lastError = this.topology.message; + return; + } + if (this.options.getOptedInSchemas().length === 0) { + await this.stopStream('idle'); + await this.releaseLeader(); + this.streamState = 'idle'; + return; + } + await this.ensureLeader(); + } + + async shutdown(): Promise { + this.closed = true; + this.clearTimers(); + await this.stopStream('idle'); + await this.releaseLeader(); + } + + private async ensureLeader(): Promise { + if (this.lock) { + if (!this.watching) { + await this.openStream(); + } + return; + } + try { + const acquired = await this.options.grpcSdk.state!.tryAcquireLock( + LEADER_LOCK, + LOCK_TTL_MS, + ); + if (!acquired) { + this.streamState = 'idle'; + return; + } + this.lock = acquired; + this.startRenewal(); + await this.openStream(); + } catch (err) { + this.lastError = err instanceof Error ? err.message : String(err); + this.streamState = 'degraded'; + this.scheduleRetry(); + } + } + + private startRenewal() { + this.clearRenewTimer(); + this.renewTimer = setInterval(() => { + void this.renewLock(); + }, LOCK_RENEW_MS); + } + + private async renewLock() { + if (!this.lock) return; + try { + this.lock = await this.lock.extend(LOCK_TTL_MS); + } catch { + this.lock = null; + await this.stopStream('idle'); + this.scheduleRetry(); + } + } + + private async openStream() { + if (this.watching || this.closed) return; + this.streamState = 'starting'; + this.ignoreClose = false; + const resumeAfter = parseResumeToken( + (await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY)) ?? undefined, + ); + try { + const stream = this.options.watch({ resumeAfter: resumeAfter ?? undefined }); + this.stream = stream; + this.watching = true; + this.streamState = 'live'; + this.retryAttempt = 0; + stream.on('change', (change: unknown) => { + void this.handleChange(change as RawChangeEvent); + }); + stream.on('error', (err: unknown) => { + void this.handleStreamError(err); + }); + stream.on('close', () => { + this.watching = false; + if (!this.closed && !this.ignoreClose && this.lock) { + this.scheduleRetry(); + } + }); + } catch (err) { + this.watching = false; + await this.handleStreamError(err); + } + } + + private async handleChange(change: RawChangeEvent) { + const schema = this.resolveSchema(change.ns?.coll); + if (!schema) return; + const event = normalizeChangeEvent(change, schema.name); + if (!event) return; + this.lastEventAt = event.occurredAt; + this.lastError = undefined; + await this.options.grpcSdk.state!.setKey(RESUME_TOKEN_KEY, event.resumeToken); + this.options.grpcSdk.bus?.publish( + `database:change:${schema.name}`, + JSON.stringify(event), + ); + ConduitGrpcSdk.Metrics?.increment('database_realtime_events_total', 1, { + operation: event.operation, + }); + await this.pushEvent(schema, event); + } + + private async pushEvent(schema: OptedInSchema, event: DatabaseChangeEvent) { + const payload = JSON.stringify(event); + const adminRooms = roomsForPublicChange(schema.name, event.documentId); + await this.safePush('admin', adminRooms, payload); + if (!schema.authorizationEnabled) { + await this.safePush('router', adminRooms, payload); + return; + } + const userIds = await this.options.subscriptions.listUsers( + schema.name, + event.documentId, + ); + const allowedRooms: string[] = []; + for (const userId of userIds) { + const allowed = await canReadDocument( + this.options.grpcSdk as unknown as AuthorizationSdk, + schema.name, + event.documentId, + userId, + ); + if (!allowed) { + await this.options.subscriptions.removeUser( + schema.name, + event.documentId, + userId, + ); + continue; + } + allowedRooms.push(authorizedDocumentRoom(schema.name, event.documentId, userId)); + } + if (allowedRooms.length > 0) { + await this.safePush('router', allowedRooms, payload); + } + } + + private async safePush( + target: 'admin' | 'router', + rooms: string[], + data: string, + ): Promise { + const client = + target === 'admin' ? this.options.grpcSdk.admin : this.options.grpcSdk.router; + if (!client?.socketPush) return; + try { + await client.socketPush({ + event: 'change', + data, + rooms, + receivers: [], + }); + } catch (err) { + ConduitGrpcSdk.Logger.error(err as Error); + } + } + + private resolveSchema(collectionName?: string): OptedInSchema | undefined { + if (!collectionName) return undefined; + return this.options + .getOptedInSchemas() + .find(schema => schema.collectionName === collectionName); + } + + private async handleStreamError(err: unknown) { + this.watching = false; + this.lastError = err instanceof Error ? err.message : String(err); + this.streamState = 'degraded'; + ConduitGrpcSdk.Metrics?.increment('database_realtime_stream_errors_total'); + ConduitGrpcSdk.Logger.error(err as Error); + if (isResumeTokenUnusable(err)) { + await this.options.grpcSdk.state!.clearKey(RESUME_TOKEN_KEY); + } + await this.stopStream('degraded'); + this.scheduleRetry(); + } + + private scheduleRetry() { + if (this.closed || this.retryTimer) return; + const delay = Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** this.retryAttempt); + this.retryAttempt += 1; + this.retryTimer = setTimeout(() => { + this.retryTimer = null; + void this.reconcile(); + }, delay); + } + + private async stopStream(nextState: RealtimeStatusCode) { + const stream = this.stream; + this.stream = null; + this.watching = false; + this.streamState = nextState; + this.ignoreClose = true; + if (stream) { + try { + await stream.close(); + } catch { + // already closed + } + } + } + + private async releaseLeader() { + this.clearRenewTimer(); + if (!this.lock) return; + try { + await this.options.grpcSdk.state!.releaseLock(this.lock); + } catch { + // lock may already have expired + } + this.lock = null; + } + + private clearTimers() { + this.clearRenewTimer(); + if (this.retryTimer) { + clearTimeout(this.retryTimer); + this.retryTimer = null; + } + } + + private clearRenewTimer() { + if (this.renewTimer) { + clearInterval(this.renewTimer); + this.renewTimer = null; + } + } +} diff --git a/modules/database/src/realtime/RealtimeService.ts b/modules/database/src/realtime/RealtimeService.ts new file mode 100644 index 000000000..db6e77d21 --- /dev/null +++ b/modules/database/src/realtime/RealtimeService.ts @@ -0,0 +1,160 @@ +import { + ConduitGrpcSdk, + ConduitRouteActions, + ConduitRouteReturnDefinition, +} from '@conduitplatform/grpc-sdk'; +import { + ConduitNumber, + ConduitString, + ConfigController, + RoutingManager, +} from '@conduitplatform/module-tools'; +import { DatabaseAdapter } from '../adapters/DatabaseAdapter.js'; +import { MongooseAdapter } from '../adapters/mongoose-adapter/index.js'; +import { MongooseSchema } from '../adapters/mongoose-adapter/MongooseSchema.js'; +import { SequelizeSchema } from '../adapters/sequelize-adapter/SequelizeSchema.js'; +import { toOptedInSchema } from './authorize.js'; +import { MongoChangeStreamCoordinator } from './MongoChangeStreamCoordinator.js'; +import { registerDatabaseRealtimeSocket } from './sockets.js'; +import { buildRealtimeStatus } from './status.js'; +import { RealtimeSubscriptionTracker } from './subscriptions.js'; +import type { OptedInSchema, RealtimeStatus } from './types.js'; +import type { ChangeStreamLike } from './types.js'; + +export class RealtimeService { + private readonly subscriptions: RealtimeSubscriptionTracker; + private coordinator?: MongoChangeStreamCoordinator; + + constructor( + private readonly grpcSdk: ConduitGrpcSdk, + private readonly adapter: DatabaseAdapter, + ) { + this.subscriptions = new RealtimeSubscriptionTracker( + grpcSdk.redisManager.getClient(), + ); + this.grpcSdk.bus?.subscribe('database:create:schema', () => { + void this.reconcile(); + }); + this.grpcSdk.bus?.subscribe('database:delete:schema', () => { + void this.reconcile(); + }); + if (adapter instanceof MongooseAdapter) { + this.coordinator = new MongoChangeStreamCoordinator({ + grpcSdk, + watch: options => this.openWatch(adapter, options.resumeAfter), + hello: () => this.hello(adapter), + getOptedInSchemas: () => this.getOptedInSchemas(), + subscriptions: this.subscriptions, + enabled: () => this.isGloballyEnabled(), + engine: () => adapter.getDatabaseType(), + socketsEnabled: () => this.areAdminSocketsEnabled(), + }); + } + } + + registerAdmin(routingManager: RoutingManager) { + routingManager.route( + { + path: '/realtime/status', + action: ConduitRouteActions.GET, + description: `Returns live-update capability and runtime status for the database module.`, + }, + new ConduitRouteReturnDefinition('DatabaseRealtimeStatus', { + status: ConduitString.Required, + engine: ConduitString.Required, + activeSchemaCount: ConduitNumber.Required, + lastEventAt: ConduitString.Optional, + message: ConduitString.Optional, + }), + async () => this.getStatus(), + ); + registerDatabaseRealtimeSocket(routingManager, { + mode: 'admin', + grpcSdk: this.grpcSdk, + schemaLookup: this.adapter, + subscriptions: this.subscriptions, + isGloballyEnabled: () => this.isGloballyEnabled(), + }); + } + + registerClient(routingManager: RoutingManager) { + registerDatabaseRealtimeSocket(routingManager, { + mode: 'client', + grpcSdk: this.grpcSdk, + schemaLookup: this.adapter, + subscriptions: this.subscriptions, + isGloballyEnabled: () => this.isGloballyEnabled(), + }); + } + + async reconcile(): Promise { + if (!this.coordinator) return; + await this.coordinator.reconcile(); + } + + async shutdown(): Promise { + await this.coordinator?.shutdown(); + } + + async getStatus(): Promise { + const engine = this.adapter.getDatabaseType(); + const optedIn = this.getOptedInSchemas(); + return buildRealtimeStatus({ + engine, + enabled: this.isGloballyEnabled(), + topologySupported: this.coordinator?.getTopology().supported ?? false, + topologyMessage: this.coordinator?.getTopology().message, + activeSchemaCount: optedIn.length, + streamState: + this.coordinator?.getState() ?? (engine === 'MongoDB' ? 'idle' : 'unsupported'), + lastEventAt: this.coordinator?.getLastEventAt(), + lastError: this.coordinator?.getLastError(), + socketsEnabled: await this.areAdminSocketsEnabled(), + }); + } + + private isGloballyEnabled(): boolean { + return ConfigController.getInstance().config?.realtime?.enabled === true; + } + + private getOptedInSchemas(): OptedInSchema[] { + const schemas: OptedInSchema[] = []; + for (const schema of this.adapter.registeredSchemas.values()) { + const optedIn = toOptedInSchema({ + name: schema.name, + collectionName: schema.collectionName, + modelOptions: schema.modelOptions, + }); + if (optedIn) schemas.push(optedIn); + } + return schemas; + } + + private openWatch(adapter: MongooseAdapter, resumeAfter?: unknown): ChangeStreamLike { + const db = adapter.mongoose.connection.db; + if (!db) { + throw new Error('MongoDB connection is not ready'); + } + return db.watch( + [], + resumeAfter ? { resumeAfter: resumeAfter as never } : {}, + ) as unknown as ChangeStreamLike; + } + + private async hello( + adapter: MongooseAdapter, + ): Promise<{ setName?: string; msg?: string } | null> { + const db = adapter.mongoose.connection.db; + if (!db) return null; + return db.admin().command({ hello: 1 }); + } + + private async areAdminSocketsEnabled(): Promise { + try { + const adminConfig = await this.grpcSdk.config.get('admin'); + return adminConfig?.transports?.sockets === true; + } catch { + return true; + } + } +} diff --git a/modules/database/src/realtime/__tests__/authorize.test.ts b/modules/database/src/realtime/__tests__/authorize.test.ts new file mode 100644 index 000000000..c441863eb --- /dev/null +++ b/modules/database/src/realtime/__tests__/authorize.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from '@jest/globals'; +import { status } from '@grpc/grpc-js'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { + assertSchemaAvailable, + optionalDocumentId, + parseSubscribeRequest, + requireSchemaName, +} from '../authorize.js'; + +describe('realtime authorization helpers', () => { + it('parses subscribe payloads and required schema names', () => { + expect(parseSubscribeRequest([{ schema: 'Order', documentId: '1' }])).toEqual({ + schema: 'Order', + documentId: '1', + }); + expect(requireSchemaName('Order')).toBe('Order'); + expect(optionalDocumentId(undefined)).toBeUndefined(); + try { + requireSchemaName(''); + throw new Error('expected throw'); + } catch (err: any) { + expect(err.code).toBe(status.INVALID_ARGUMENT); + } + }); + + it('rejects missing schemas, disabled realtime, and CMS-read for clients', () => { + const lookup = { + getSchema: (name: string) => { + if (name === 'Missing') return undefined; + if (name === 'Off') { + return { name, modelOptions: { conduit: { realtime: { enabled: false } } } }; + } + return { + name, + modelOptions: { + conduit: { + realtime: { enabled: true }, + cms: { crudOperations: { read: { enabled: false } } }, + authorization: { enabled: true }, + }, + }, + }; + }, + }; + try { + assertSchemaAvailable(lookup, 'Missing', true); + throw new Error('expected throw'); + } catch (err: any) { + expect(err.code).toBe(status.NOT_FOUND); + } + try { + assertSchemaAvailable(lookup, 'Off', false); + throw new Error('expected throw'); + } catch (err: any) { + expect(err.code).toBe(status.FAILED_PRECONDITION); + } + try { + assertSchemaAvailable(lookup, 'Order', true); + throw new Error('expected throw'); + } catch (err: any) { + expect(err.code).toBe(status.PERMISSION_DENIED); + } + const admin = assertSchemaAvailable(lookup, 'Order', false); + expect(admin.authorizationEnabled).toBe(true); + }); + + it('maps adapter NOT_FOUND throws to a subscription error', () => { + const lookup = { + getSchema: () => { + throw new GrpcError(status.NOT_FOUND, 'Schema Missing not defined yet'); + }, + }; + try { + assertSchemaAvailable(lookup, 'Missing', false); + throw new Error('expected throw'); + } catch (err: any) { + expect(err.code).toBe(status.NOT_FOUND); + expect(err.message).toBe('Schema does not exist'); + } + }); +}); diff --git a/modules/database/src/realtime/__tests__/change-stream.integration.test.ts b/modules/database/src/realtime/__tests__/change-stream.integration.test.ts new file mode 100644 index 000000000..dd74753fc --- /dev/null +++ b/modules/database/src/realtime/__tests__/change-stream.integration.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from '@jest/globals'; +import { MongoClient, ObjectId } from 'mongodb'; +import { normalizeChangeEvent, type RawChangeEvent } from '../normalize.js'; + +const replicaSetUri = process.env.DB_CONN_URI; +const integrationEnabled = Boolean(replicaSetUri?.includes('replicaSet')); +const describeIntegration = integrationEnabled ? describe : describe.skip; + +describeIntegration('MongoDB change stream contract', () => { + it('normalizes insert events from a replica-set watch without document fields', async () => { + const client = new MongoClient(replicaSetUri!); + await client.connect(); + const dbName = `conduit_realtime_${Date.now()}`; + const db = client.db(dbName); + try { + const collection = db.collection('orders'); + const stream = db.watch([]); + const change = await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error('timed out waiting for change')), + 10_000, + ); + stream.on('change', event => { + clearTimeout(timer); + resolve(event as RawChangeEvent); + }); + stream.on('error', err => { + clearTimeout(timer); + reject(err); + }); + void collection.insertOne({ _id: new ObjectId(), secret: 'do-not-leak' }); + }); + await stream.close(); + const event = normalizeChangeEvent(change, 'Order'); + expect(event).toMatchObject({ + version: 1, + operation: 'insert', + schema: 'Order', + }); + expect(event).not.toHaveProperty('fullDocument'); + expect(JSON.stringify(event)).not.toContain('do-not-leak'); + } finally { + await db.dropDatabase().catch(() => undefined); + await client.close(); + } + }); +}); diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts new file mode 100644 index 000000000..5ec3a4af3 --- /dev/null +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -0,0 +1,166 @@ +import { EventEmitter } from 'node:events'; +import { describe, expect, it, jest } from '@jest/globals'; +import { ObjectId } from 'bson'; +import { MongoChangeStreamCoordinator } from '../MongoChangeStreamCoordinator.js'; +import { RealtimeSubscriptionTracker } from '../subscriptions.js'; +import { authorizedDocumentRoom, roomsForPublicChange } from '../rooms.js'; + +class MemoryStore { + private sets = new Map>(); + async sadd(key: string, ...members: string[]) { + const set = this.sets.get(key) ?? new Set(); + members.forEach(member => set.add(member)); + this.sets.set(key, set); + } + async srem(key: string, ...members: string[]) { + members.forEach(member => this.sets.get(key)?.delete(member)); + } + async smembers(key: string) { + return [...(this.sets.get(key) ?? new Set())]; + } + async scard(key: string) { + return this.sets.get(key)?.size ?? 0; + } + async del(...keys: string[]) { + keys.forEach(key => this.sets.delete(key)); + } +} + +function createCoordinator(overrides?: { + allow?: boolean; + schemas?: { name: string; collectionName: string; authorizationEnabled: boolean }[]; +}) { + const stream = new EventEmitter() as EventEmitter & { close: () => Promise }; + stream.close = async () => { + stream.emit('close'); + }; + const state = new Map(); + const lock = { + extend: jest.fn(async () => lock), + release: jest.fn(async () => undefined), + }; + const routerPush = jest.fn(async () => undefined); + const adminPush = jest.fn(async () => undefined); + const publish = jest.fn(); + const subscriptions = new RealtimeSubscriptionTracker(new MemoryStore()); + const grpcSdk = { + state: { + tryAcquireLock: jest.fn(async () => lock), + releaseLock: jest.fn(async () => undefined), + getKey: jest.fn(async (key: string) => state.get(key) ?? null), + setKey: jest.fn(async (key: string, value: string) => { + state.set(key, value); + }), + clearKey: jest.fn(async (key: string) => { + state.delete(key); + }), + }, + bus: { publish }, + router: { socketPush: routerPush }, + admin: { socketPush: adminPush }, + isAvailable: () => true, + authorization: { + can: async () => ({ allow: overrides?.allow !== false }), + }, + }; + const coordinator = new MongoChangeStreamCoordinator({ + grpcSdk: grpcSdk as never, + watch: () => stream as never, + hello: async () => ({ setName: 'rs0' }), + getOptedInSchemas: () => + overrides?.schemas ?? [ + { name: 'Order', collectionName: 'orders', authorizationEnabled: false }, + ], + subscriptions, + enabled: () => true, + engine: () => 'MongoDB', + }); + return { + coordinator, + stream, + routerPush, + adminPush, + publish, + subscriptions, + grpcSdk, + state, + lock, + }; +} + +describe('MongoChangeStreamCoordinator', () => { + it('emits one normalized event to public rooms and ignores other collections', async () => { + const { coordinator, stream, routerPush, adminPush, publish } = createCoordinator(); + await coordinator.reconcile(); + const resume = { _data: 'token' }; + stream.emit('change', { + operationType: 'insert', + ns: { coll: 'orders' }, + documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, + fullDocument: { secret: 'nope' }, + _id: resume, + wallTime: new Date('2026-01-02T00:00:00.000Z'), + }); + stream.emit('change', { + operationType: 'insert', + ns: { coll: 'other' }, + documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4d') }, + _id: resume, + }); + await new Promise(resolve => setImmediate(resolve)); + expect(publish).toHaveBeenCalledTimes(1); + expect(publish.mock.calls[0][0]).toBe('database:change:Order'); + const payload = JSON.parse(publish.mock.calls[0][1] as string); + expect(payload).toMatchObject({ + operation: 'insert', + schema: 'Order', + documentId: '64b64c4c4c4c4c4c4c4c4c4c', + }); + expect(payload).not.toHaveProperty('fullDocument'); + expect(payload).not.toHaveProperty('secret'); + const expectedRooms = roomsForPublicChange('Order', '64b64c4c4c4c4c4c4c4c4c4c'); + expect(routerPush).toHaveBeenCalledWith( + expect.objectContaining({ event: 'change', rooms: expectedRooms }), + ); + expect(adminPush).toHaveBeenCalledWith( + expect.objectContaining({ event: 'change', rooms: expectedRooms }), + ); + await coordinator.shutdown(); + }); + + it('re-checks ReBAC before emission and drops revoked users', async () => { + const { coordinator, stream, routerPush, subscriptions } = createCoordinator({ + allow: false, + schemas: [{ name: 'Order', collectionName: 'orders', authorizationEnabled: true }], + }); + await subscriptions.addAuthorizedDocument( + 'sock-1', + 'Order', + '64b64c4c4c4c4c4c4c4c4c4c', + 'user-1', + ); + await coordinator.reconcile(); + stream.emit('change', { + operationType: 'update', + ns: { coll: 'orders' }, + documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, + _id: { _data: 'token' }, + }); + await new Promise(resolve => setImmediate(resolve)); + expect(routerPush).not.toHaveBeenCalled(); + expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c')).toEqual([]); + expect(authorizedDocumentRoom('Order', '64b64c4c4c4c4c4c4c4c4c', 'user-1')).toContain( + 'user-1', + ); + await coordinator.shutdown(); + }); + + it('clears an unusable resume token and retries', async () => { + const { coordinator, stream, grpcSdk } = createCoordinator(); + await coordinator.reconcile(); + stream.emit('error', { code: 280, message: 'ChangeStreamHistoryLost' }); + await new Promise(resolve => setImmediate(resolve)); + expect(grpcSdk.state.clearKey).toHaveBeenCalled(); + await coordinator.shutdown(); + }); +}); diff --git a/modules/database/src/realtime/__tests__/normalize.test.ts b/modules/database/src/realtime/__tests__/normalize.test.ts new file mode 100644 index 000000000..d51eaa549 --- /dev/null +++ b/modules/database/src/realtime/__tests__/normalize.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from '@jest/globals'; +import { ObjectId } from 'bson'; +import { normalizeChangeEvent, parseResumeToken } from '../normalize.js'; + +describe('normalizeChangeEvent', () => { + it('normalizes insert/update/replace/delete into metadata-only events', () => { + const resume = { _data: 'token-1' }; + const event = normalizeChangeEvent( + { + operationType: 'insert', + documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, + wallTime: new Date('2026-01-01T00:00:00.000Z'), + _id: resume, + }, + 'Order', + ); + expect(event).toMatchObject({ + version: 1, + operation: 'insert', + schema: 'Order', + documentId: '64b64c4c4c4c4c4c4c4c4c4c', + occurredAt: '2026-01-01T00:00:00.000Z', + }); + expect(event?.resumeToken).toEqual( + parseResumeToken(event!.resumeToken) ? event!.resumeToken : event?.resumeToken, + ); + expect(JSON.parse(JSON.stringify(event))).not.toHaveProperty('fullDocument'); + }); + + it('ignores drop/invalidate and missing document ids', () => { + expect( + normalizeChangeEvent({ operationType: 'drop', _id: { _data: 'x' } }, 'Order'), + ).toBeNull(); + expect( + normalizeChangeEvent({ operationType: 'insert', _id: { _data: 'x' } }, 'Order'), + ).toBeNull(); + }); +}); diff --git a/modules/database/src/realtime/__tests__/rooms.test.ts b/modules/database/src/realtime/__tests__/rooms.test.ts new file mode 100644 index 000000000..dbdd57e1b --- /dev/null +++ b/modules/database/src/realtime/__tests__/rooms.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from '@jest/globals'; +import { + authorizedDocumentRoom, + documentRoom, + roomsForPublicChange, + schemaRoom, +} from '../rooms.js'; + +describe('realtime rooms', () => { + it('isolates schema, document, and per-user rooms', () => { + expect(schemaRoom('Order')).toBe('database:schema:Order'); + expect(documentRoom('Order', 'abc')).toBe('database:doc:Order:abc'); + expect(authorizedDocumentRoom('Order', 'abc', 'user-1')).toBe( + 'database:doc:Order:abc:user:user-1', + ); + expect(roomsForPublicChange('Order', 'abc')).toEqual([ + 'database:schema:Order', + 'database:doc:Order:abc', + ]); + expect(schemaRoom('Order')).not.toBe(schemaRoom('Orders')); + expect(authorizedDocumentRoom('Order', 'abc', 'u1')).not.toBe( + authorizedDocumentRoom('Order', 'abc', 'u2'), + ); + }); + + it('encodes reserved characters so rooms cannot collide', () => { + expect(schemaRoom('a/b')).toBe('database:schema:a%2Fb'); + expect(documentRoom('Order', 'id:1')).toBe('database:doc:Order:id%3A1'); + }); +}); diff --git a/modules/database/src/realtime/__tests__/status.test.ts b/modules/database/src/realtime/__tests__/status.test.ts new file mode 100644 index 000000000..a94790cfb --- /dev/null +++ b/modules/database/src/realtime/__tests__/status.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from '@jest/globals'; +import { buildRealtimeStatus } from '../status.js'; + +describe('buildRealtimeStatus', () => { + it('reports unsupported, disabled, idle, live, and degraded states', () => { + expect( + buildRealtimeStatus({ + engine: 'PostgreSQL', + enabled: true, + topologySupported: true, + activeSchemaCount: 1, + streamState: 'live', + }).status, + ).toBe('unsupported'); + expect( + buildRealtimeStatus({ + engine: 'MongoDB', + enabled: false, + topologySupported: true, + activeSchemaCount: 1, + streamState: 'live', + }).status, + ).toBe('disabled'); + expect( + buildRealtimeStatus({ + engine: 'MongoDB', + enabled: true, + topologySupported: false, + activeSchemaCount: 1, + streamState: 'idle', + }).message, + ).toMatch(/replica set/i); + expect( + buildRealtimeStatus({ + engine: 'MongoDB', + enabled: true, + topologySupported: true, + activeSchemaCount: 2, + streamState: 'live', + }).status, + ).toBe('live'); + expect( + buildRealtimeStatus({ + engine: 'MongoDB', + enabled: true, + topologySupported: true, + activeSchemaCount: 1, + streamState: 'degraded', + lastError: 'boom', + }), + ).toMatchObject({ status: 'degraded', message: 'boom' }); + }); +}); diff --git a/modules/database/src/realtime/__tests__/subscriptions.test.ts b/modules/database/src/realtime/__tests__/subscriptions.test.ts new file mode 100644 index 000000000..45ee7d1c9 --- /dev/null +++ b/modules/database/src/realtime/__tests__/subscriptions.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from '@jest/globals'; +import { RealtimeSubscriptionTracker } from '../subscriptions.js'; + +class MemoryStore { + private sets = new Map>(); + + async sadd(key: string, ...members: string[]) { + const set = this.sets.get(key) ?? new Set(); + members.forEach(member => set.add(member)); + this.sets.set(key, set); + return members.length; + } + + async srem(key: string, ...members: string[]) { + const set = this.sets.get(key); + if (!set) return 0; + members.forEach(member => set.delete(member)); + return members.length; + } + + async smembers(key: string) { + return [...(this.sets.get(key) ?? new Set())]; + } + + async scard(key: string) { + return this.sets.get(key)?.size ?? 0; + } + + async del(...keys: string[]) { + keys.forEach(key => this.sets.delete(key)); + return keys.length; + } +} + +describe('RealtimeSubscriptionTracker', () => { + it('tracks users per document and cleans up on disconnect', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); + await tracker.addAuthorizedDocument('s2', 'Order', 'doc-1', 'user-1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + + await tracker.disconnect('s1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual(['user-1']); + + await tracker.disconnect('s2'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); + }); + + it('removes revoked users from the document set', async () => { + const tracker = new RealtimeSubscriptionTracker(new MemoryStore()); + await tracker.addAuthorizedDocument('s1', 'Order', 'doc-1', 'user-1'); + await tracker.removeUser('Order', 'doc-1', 'user-1'); + expect(await tracker.listUsers('Order', 'doc-1')).toEqual([]); + }); +}); diff --git a/modules/database/src/realtime/__tests__/topology.test.ts b/modules/database/src/realtime/__tests__/topology.test.ts new file mode 100644 index 000000000..abf8e8861 --- /dev/null +++ b/modules/database/src/realtime/__tests__/topology.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from '@jest/globals'; +import { isResumeTokenUnusable, topologyFromHello } from '../topology.js'; + +describe('topology helpers', () => { + it('accepts replica sets and mongos, rejects standalone', () => { + expect(topologyFromHello({ setName: 'rs0' })).toEqual({ supported: true }); + expect(topologyFromHello({ msg: 'isdbgrid' })).toEqual({ supported: true }); + expect(topologyFromHello({}).supported).toBe(false); + }); + + it('detects unusable resume tokens', () => { + expect(isResumeTokenUnusable({ code: 280 })).toBe(true); + expect(isResumeTokenUnusable(new Error('cannot resume'))).toBe(true); + expect(isResumeTokenUnusable(new Error('socket hang up'))).toBe(false); + }); +}); diff --git a/modules/database/src/realtime/authorize.ts b/modules/database/src/realtime/authorize.ts new file mode 100644 index 000000000..330992f13 --- /dev/null +++ b/modules/database/src/realtime/authorize.ts @@ -0,0 +1,147 @@ +import { status } from '@grpc/grpc-js'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import type { OptedInSchema, SubscribeRequest } from './types.js'; + +export class RealtimeSubscriptionError extends GrpcError { + constructor(code: number, message: string) { + super(code, message); + this.name = 'RealtimeSubscriptionError'; + } +} + +export type AuthorizationSdk = { + isAvailable: (module: string) => boolean; + authorization?: { + can: (request: { + subject: string; + actions: string[]; + resource: string; + }) => Promise<{ allow: boolean }>; + } | null; +}; + +export type SchemaLookup = { + getSchema(name: string): + | { + name: string; + modelOptions?: { + conduit?: { + realtime?: { enabled?: boolean }; + cms?: { crudOperations?: { read?: { enabled?: boolean } } }; + authorization?: { enabled?: boolean }; + }; + }; + } + | undefined; +}; + +export type ResolvedSubscription = { + schema: string; + documentId?: string; + authorizationEnabled: boolean; + rooms: string[]; +}; + +export function parseSubscribeRequest(params: unknown[]): SubscribeRequest { + const raw = params[0]; + if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + return raw as SubscribeRequest; + } + return {}; +} + +export function requireSchemaName(value: unknown): string { + if (typeof value !== 'string' || value.trim() === '') { + throw new RealtimeSubscriptionError(status.INVALID_ARGUMENT, 'schema is required'); + } + return value.trim(); +} + +export function optionalDocumentId(value: unknown): string | undefined { + if (value === undefined || value === null || value === '') return undefined; + if (typeof value !== 'string') { + throw new RealtimeSubscriptionError( + status.INVALID_ARGUMENT, + 'documentId must be a string', + ); + } + return value; +} + +export async function canReadDocument( + grpcSdk: AuthorizationSdk, + schema: string, + documentId: string, + userId: string, +): Promise { + if (!grpcSdk.authorization || !grpcSdk.isAvailable('authorization')) { + return false; + } + try { + const decision = await grpcSdk.authorization.can({ + subject: `User:${userId}`, + actions: ['read'], + resource: `${schema}:${documentId}`, + }); + return decision.allow === true; + } catch { + return false; + } +} + +export function assertSchemaAvailable( + lookup: SchemaLookup, + schemaName: string, + requireCmsRead: boolean, +): { + authorizationEnabled: boolean; + realtimeEnabled: boolean; +} { + let schema: ReturnType; + try { + schema = lookup.getSchema(schemaName); + } catch (err) { + if (err instanceof GrpcError && err.code === status.NOT_FOUND) { + throw new RealtimeSubscriptionError(status.NOT_FOUND, 'Schema does not exist'); + } + throw err; + } + if (!schema) { + throw new RealtimeSubscriptionError(status.NOT_FOUND, 'Schema does not exist'); + } + const conduit = schema.modelOptions?.conduit; + if (!conduit?.realtime?.enabled) { + throw new RealtimeSubscriptionError( + status.FAILED_PRECONDITION, + 'Live updates are not enabled for this schema', + ); + } + if (requireCmsRead && conduit.cms?.crudOperations?.read?.enabled !== true) { + throw new RealtimeSubscriptionError( + status.PERMISSION_DENIED, + 'CMS read is not enabled for this schema', + ); + } + return { + authorizationEnabled: conduit.authorization?.enabled === true, + realtimeEnabled: true, + }; +} + +export function toOptedInSchema(schema: { + name: string; + collectionName: string; + modelOptions?: { + conduit?: { + realtime?: { enabled?: boolean }; + authorization?: { enabled?: boolean }; + }; + }; +}): OptedInSchema | null { + if (!schema.modelOptions?.conduit?.realtime?.enabled) return null; + return { + name: schema.name, + collectionName: schema.collectionName, + authorizationEnabled: schema.modelOptions.conduit.authorization?.enabled === true, + }; +} diff --git a/modules/database/src/realtime/index.ts b/modules/database/src/realtime/index.ts new file mode 100644 index 000000000..655c04bba --- /dev/null +++ b/modules/database/src/realtime/index.ts @@ -0,0 +1,11 @@ +export { RealtimeService } from './RealtimeService.js'; +export { buildRealtimeStatus } from './status.js'; +export { normalizeChangeEvent } from './normalize.js'; +export { + schemaRoom, + documentRoom, + authorizedDocumentRoom, + roomsForPublicChange, +} from './rooms.js'; +export { RealtimeSubscriptionTracker } from './subscriptions.js'; +export type { RealtimeStatus, DatabaseChangeEvent } from './types.js'; diff --git a/modules/database/src/realtime/normalize.ts b/modules/database/src/realtime/normalize.ts new file mode 100644 index 000000000..bdd876f16 --- /dev/null +++ b/modules/database/src/realtime/normalize.ts @@ -0,0 +1,70 @@ +import { EJSON } from 'bson'; +import { + DATABASE_CHANGE_EVENT_VERSION, + DATABASE_CHANGE_OPERATIONS, + type DatabaseChangeEvent, + type DatabaseChangeOperation, +} from './types.js'; + +const OPERATION_SET = new Set(DATABASE_CHANGE_OPERATIONS); + +export type RawChangeEvent = { + operationType?: string; + ns?: { coll?: string }; + documentKey?: { _id?: unknown }; + wallTime?: Date; + clusterTime?: { toString?: () => string }; + _id?: unknown; +}; + +export function normalizeChangeEvent( + change: RawChangeEvent, + schemaName: string, + occurredAt: Date = new Date(), +): DatabaseChangeEvent | null { + const operation = change.operationType; + if (!operation || !OPERATION_SET.has(operation)) { + return null; + } + const documentId = extractDocumentId(change.documentKey?._id); + if (!documentId) { + return null; + } + if (!change._id) { + return null; + } + return { + version: DATABASE_CHANGE_EVENT_VERSION, + operation: operation as DatabaseChangeOperation, + schema: schemaName, + documentId, + occurredAt: (change.wallTime instanceof Date + ? change.wallTime + : occurredAt + ).toISOString(), + resumeToken: EJSON.stringify(change._id), + }; +} + +export function parseResumeToken(token: string | null | undefined): unknown | undefined { + if (!token) return undefined; + try { + return EJSON.parse(token); + } catch { + return undefined; + } +} + +function extractDocumentId(id: unknown): string | null { + if (id === undefined || id === null) return null; + if (typeof id === 'string' || typeof id === 'number') return String(id); + if (typeof id === 'object' && id !== null && 'toHexString' in id) { + const hex = (id as { toHexString: () => string }).toHexString(); + return typeof hex === 'string' && hex.length > 0 ? hex : null; + } + if (typeof id === 'object' && id !== null && 'toString' in id) { + const value = String(id); + return value && value !== '[object Object]' ? value : null; + } + return null; +} diff --git a/modules/database/src/realtime/rooms.ts b/modules/database/src/realtime/rooms.ts new file mode 100644 index 000000000..45a69a7cd --- /dev/null +++ b/modules/database/src/realtime/rooms.ts @@ -0,0 +1,25 @@ +const ROOM_PREFIX = 'database'; + +function encodeSegment(value: string): string { + return encodeURIComponent(value); +} + +export function schemaRoom(schema: string): string { + return `${ROOM_PREFIX}:schema:${encodeSegment(schema)}`; +} + +export function documentRoom(schema: string, documentId: string): string { + return `${ROOM_PREFIX}:doc:${encodeSegment(schema)}:${encodeSegment(documentId)}`; +} + +export function authorizedDocumentRoom( + schema: string, + documentId: string, + userId: string, +): string { + return `${ROOM_PREFIX}:doc:${encodeSegment(schema)}:${encodeSegment(documentId)}:user:${encodeSegment(userId)}`; +} + +export function roomsForPublicChange(schema: string, documentId: string): string[] { + return [schemaRoom(schema), documentRoom(schema, documentId)]; +} diff --git a/modules/database/src/realtime/sockets.ts b/modules/database/src/realtime/sockets.ts new file mode 100644 index 000000000..5cc97957e --- /dev/null +++ b/modules/database/src/realtime/sockets.ts @@ -0,0 +1,166 @@ +import { + ConduitGrpcSdk, + ConduitRouteReturnDefinition, + ParsedSocketRequest, + TYPE, + UnparsedSocketResponse, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { RoutingManager } from '@conduitplatform/module-tools'; +import { + assertSchemaAvailable, + canReadDocument, + optionalDocumentId, + parseSubscribeRequest, + RealtimeSubscriptionError, + requireSchemaName, + type AuthorizationSdk, + type SchemaLookup, +} from './authorize.js'; +import { authorizedDocumentRoom, documentRoom, schemaRoom } from './rooms.js'; +import type { RealtimeSubscriptionTracker } from './subscriptions.js'; + +type SocketMode = 'client' | 'admin'; + +export function registerDatabaseRealtimeSocket( + routingManager: RoutingManager, + options: { + mode: SocketMode; + grpcSdk: ConduitGrpcSdk; + schemaLookup: SchemaLookup; + subscriptions: RealtimeSubscriptionTracker; + isGloballyEnabled: () => boolean; + }, +) { + const handlers = createSocketHandlers(options); + routingManager.socket( + { + path: '/', + middlewares: options.mode === 'client' ? ['authMiddleware'] : undefined, + }, + { + connect: { handler: handlers.connect }, + disconnect: { handler: handlers.disconnect }, + subscribe: { + params: [TYPE.JSON], + handler: handlers.subscribe, + returnType: new ConduitRouteReturnDefinition('DatabaseRealtimeSubscribe', { + rooms: [TYPE.String], + }), + }, + unsubscribe: { + params: [TYPE.JSON], + handler: handlers.unsubscribe, + returnType: new ConduitRouteReturnDefinition('DatabaseRealtimeUnsubscribe', { + rooms: [TYPE.String], + }), + }, + }, + ); +} + +function createSocketHandlers(options: { + mode: SocketMode; + grpcSdk: ConduitGrpcSdk; + schemaLookup: SchemaLookup; + subscriptions: RealtimeSubscriptionTracker; + isGloballyEnabled: () => boolean; +}) { + return { + connect: async (): Promise => { + return { event: 'connected', data: { ok: true } }; + }, + disconnect: async (call: ParsedSocketRequest): Promise => { + await options.subscriptions.disconnect(call.request.socketId); + return { event: 'disconnected', data: { ok: true } }; + }, + subscribe: async (call: ParsedSocketRequest): Promise => { + const rooms = await resolveSubscription(call, options, 'join'); + return { event: 'join-room', rooms }; + }, + unsubscribe: async (call: ParsedSocketRequest): Promise => { + const rooms = await resolveSubscription(call, options, 'leave'); + return { event: 'leave-room', rooms }; + }, + }; +} + +async function resolveSubscription( + call: ParsedSocketRequest, + options: { + mode: SocketMode; + grpcSdk: ConduitGrpcSdk; + schemaLookup: SchemaLookup; + subscriptions: RealtimeSubscriptionTracker; + isGloballyEnabled: () => boolean; + }, + action: 'join' | 'leave', +): Promise { + if (!options.isGloballyEnabled()) { + throw new RealtimeSubscriptionError( + status.FAILED_PRECONDITION, + 'Live updates are disabled', + ); + } + const payload = parseSubscribeRequest(call.request.params ?? []); + const schemaName = requireSchemaName(payload.schema); + const documentId = optionalDocumentId(payload.documentId); + const meta = assertSchemaAvailable( + options.schemaLookup, + schemaName, + options.mode === 'client', + ); + + if (options.mode === 'admin') { + if (documentId) return [documentRoom(schemaName, documentId)]; + return [schemaRoom(schemaName)]; + } + + const userId = call.request.context?.user?._id; + if (!userId) { + throw new RealtimeSubscriptionError( + status.UNAUTHENTICATED, + 'Authentication required', + ); + } + + if (meta.authorizationEnabled) { + if (!documentId) { + throw new RealtimeSubscriptionError( + status.PERMISSION_DENIED, + 'Document ID is required for authorized schemas', + ); + } + const allowed = await canReadDocument( + options.grpcSdk as unknown as AuthorizationSdk, + schemaName, + documentId, + userId, + ); + if (!allowed) { + throw new RealtimeSubscriptionError( + status.PERMISSION_DENIED, + 'Read permission denied', + ); + } + if (action === 'join') { + await options.subscriptions.addAuthorizedDocument( + call.request.socketId, + schemaName, + documentId, + userId, + ); + } else { + await options.subscriptions.removeAuthorizedDocument( + call.request.socketId, + schemaName, + documentId, + userId, + ); + } + return [authorizedDocumentRoom(schemaName, documentId, userId)]; + } + + if (documentId) return [documentRoom(schemaName, documentId)]; + return [schemaRoom(schemaName)]; +} diff --git a/modules/database/src/realtime/status.ts b/modules/database/src/realtime/status.ts new file mode 100644 index 000000000..b1da21b03 --- /dev/null +++ b/modules/database/src/realtime/status.ts @@ -0,0 +1,74 @@ +import type { RealtimeStatus, RealtimeStatusCode } from './types.js'; + +export type RealtimeStatusInput = { + engine: string; + enabled: boolean; + topologySupported: boolean; + topologyMessage?: string; + activeSchemaCount: number; + streamState: RealtimeStatusCode; + lastEventAt?: string; + lastError?: string; + socketsEnabled?: boolean; +}; + +export function buildRealtimeStatus(input: RealtimeStatusInput): RealtimeStatus { + if (input.engine !== 'MongoDB') { + return { + status: 'unsupported', + engine: input.engine, + activeSchemaCount: 0, + message: 'Live updates require MongoDB', + }; + } + if (!input.enabled) { + return { + status: 'disabled', + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + }; + } + if (!input.topologySupported) { + return { + status: 'idle', + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + message: + input.topologyMessage ?? + 'A replica set or sharded MongoDB deployment is required for live updates', + }; + } + if (input.socketsEnabled === false) { + return { + status: 'idle', + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + message: + 'Enable Admin socket transport to consume live updates in the control panel', + lastEventAt: input.lastEventAt, + }; + } + if (input.activeSchemaCount === 0) { + return { + status: 'idle', + engine: input.engine, + activeSchemaCount: 0, + message: 'No schemas have live updates enabled', + }; + } + if (input.streamState === 'degraded') { + return { + status: 'degraded', + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + lastEventAt: input.lastEventAt, + message: input.lastError ?? 'Change stream is retrying after an error', + }; + } + return { + status: input.streamState, + engine: input.engine, + activeSchemaCount: input.activeSchemaCount, + lastEventAt: input.lastEventAt, + }; +} diff --git a/modules/database/src/realtime/subscriptions.ts b/modules/database/src/realtime/subscriptions.ts new file mode 100644 index 000000000..da7bfb867 --- /dev/null +++ b/modules/database/src/realtime/subscriptions.ts @@ -0,0 +1,92 @@ +export type SubscriptionStore = { + sadd(key: string, ...members: string[]): Promise; + srem(key: string, ...members: string[]): Promise; + smembers(key: string): Promise; + scard(key: string): Promise; + del(...keys: string[]): Promise; +}; + +function socketKey(socketId: string): string { + return `realtime:socket:${socketId}`; +} + +function docUsersKey(schema: string, documentId: string): string { + return `realtime:doc:${schema}:${documentId}`; +} + +function userDocSocketsKey(schema: string, documentId: string, userId: string): string { + return `realtime:userdoc:${schema}:${documentId}:${userId}`; +} + +function subscriptionRecord(schema: string, documentId: string, userId: string): string { + return JSON.stringify({ schema, documentId, userId }); +} + +export class RealtimeSubscriptionTracker { + constructor(private readonly store: SubscriptionStore) {} + + async addAuthorizedDocument( + socketId: string, + schema: string, + documentId: string, + userId: string, + ): Promise { + await this.store.sadd( + socketKey(socketId), + subscriptionRecord(schema, documentId, userId), + ); + await this.store.sadd(userDocSocketsKey(schema, documentId, userId), socketId); + await this.store.sadd(docUsersKey(schema, documentId), userId); + } + + async removeAuthorizedDocument( + socketId: string, + schema: string, + documentId: string, + userId: string, + ): Promise { + await this.store.srem( + socketKey(socketId), + subscriptionRecord(schema, documentId, userId), + ); + await this.store.srem(userDocSocketsKey(schema, documentId, userId), socketId); + const remaining = await this.store.scard( + userDocSocketsKey(schema, documentId, userId), + ); + if (remaining === 0) { + await this.store.srem(docUsersKey(schema, documentId), userId); + await this.store.del(userDocSocketsKey(schema, documentId, userId)); + } + } + + async listUsers(schema: string, documentId: string): Promise { + return this.store.smembers(docUsersKey(schema, documentId)); + } + + async removeUser(schema: string, documentId: string, userId: string): Promise { + await this.store.srem(docUsersKey(schema, documentId), userId); + await this.store.del(userDocSocketsKey(schema, documentId, userId)); + } + + async disconnect(socketId: string): Promise { + const records = await this.store.smembers(socketKey(socketId)); + for (const record of records) { + try { + const parsed = JSON.parse(record) as { + schema: string; + documentId: string; + userId: string; + }; + await this.removeAuthorizedDocument( + socketId, + parsed.schema, + parsed.documentId, + parsed.userId, + ); + } catch { + // ignore malformed records + } + } + await this.store.del(socketKey(socketId)); + } +} diff --git a/modules/database/src/realtime/topology.ts b/modules/database/src/realtime/topology.ts new file mode 100644 index 000000000..341d49438 --- /dev/null +++ b/modules/database/src/realtime/topology.ts @@ -0,0 +1,49 @@ +const CHANGE_STREAM_ERROR_CODES = new Set([ + 136, // CappedPositionLost + 237, // CursorKilled + 280, // ChangeStreamHistoryLost + 286, // ChangeStreamFatalError +]); + +export type TopologyResult = { + supported: boolean; + message?: string; +}; + +export function topologyFromHello( + hello: + | { + setName?: string; + msg?: string; + } + | null + | undefined, +): TopologyResult { + if (!hello) { + return { supported: false, message: 'Unable to determine MongoDB topology' }; + } + if (hello.msg === 'isdbgrid' || Boolean(hello.setName)) { + return { supported: true }; + } + return { + supported: false, + message: 'A replica set or sharded MongoDB deployment is required for live updates', + }; +} + +export function isResumeTokenUnusable(error: unknown): boolean { + const code = extractErrorCode(error); + if (code !== undefined && CHANGE_STREAM_ERROR_CODES.has(code)) { + return true; + } + const message = error instanceof Error ? error.message : String(error ?? ''); + return /resume token|ChangeStreamHistoryLost|cannot resume/i.test(message); +} + +function extractErrorCode(error: unknown): number | undefined { + if (!error || typeof error !== 'object') return undefined; + const candidate = error as { code?: unknown; errorCode?: unknown }; + if (typeof candidate.code === 'number') return candidate.code; + if (typeof candidate.errorCode === 'number') return candidate.errorCode; + return undefined; +} diff --git a/modules/database/src/realtime/types.ts b/modules/database/src/realtime/types.ts new file mode 100644 index 000000000..9901c2d86 --- /dev/null +++ b/modules/database/src/realtime/types.ts @@ -0,0 +1,49 @@ +export const DATABASE_CHANGE_EVENT_VERSION = 1 as const; + +export const DATABASE_CHANGE_OPERATIONS = [ + 'insert', + 'update', + 'replace', + 'delete', +] as const; + +export type DatabaseChangeOperation = (typeof DATABASE_CHANGE_OPERATIONS)[number]; + +export type DatabaseChangeEvent = { + version: typeof DATABASE_CHANGE_EVENT_VERSION; + operation: DatabaseChangeOperation; + schema: string; + documentId: string; + occurredAt: string; + resumeToken: string; +}; + +export type RealtimeStatusCode = + 'unsupported' | 'disabled' | 'idle' | 'starting' | 'live' | 'degraded'; + +export type RealtimeStatus = { + status: RealtimeStatusCode; + engine: string; + activeSchemaCount: number; + lastEventAt?: string; + message?: string; +}; + +export type OptedInSchema = { + name: string; + collectionName: string; + authorizationEnabled: boolean; +}; + +export type SubscribeRequest = { + schema?: unknown; + documentId?: unknown; +}; + +export type ChangeStreamLike = { + on( + event: 'change' | 'error' | 'close' | 'end', + listener: (...args: unknown[]) => void, + ): void; + close(): Promise | void; +}; diff --git a/modules/database/src/routes/index.ts b/modules/database/src/routes/index.ts index 2c09cf237..1d4002a74 100644 --- a/modules/database/src/routes/index.ts +++ b/modules/database/src/routes/index.ts @@ -13,6 +13,7 @@ import { import { DatabaseAdapter } from '../adapters/DatabaseAdapter.js'; import { MongooseSchema } from '../adapters/mongoose-adapter/MongooseSchema.js'; import { SequelizeSchema } from '../adapters/sequelize-adapter/SequelizeSchema.js'; +import type { RealtimeService } from '../realtime/index.js'; export class DatabaseRoutes { private readonly handlers: CmsHandlers; @@ -34,6 +35,7 @@ export class DatabaseRoutes { readonly server: GrpcServer, private readonly database: DatabaseAdapter, private readonly grpcSdk: ConduitGrpcSdk, + private readonly realtimeService?: RealtimeService, ) { this.handlers = new CmsHandlers(grpcSdk, database); this._routingManager = new RoutingManager(this.grpcSdk.router!, server); @@ -55,7 +57,6 @@ export class DatabaseRoutes { } requestRefresh() { - if (this.crudRoutes.length === 0 && this.customRoutes.length === 0) return; this._scheduleTimeout(); } @@ -86,6 +87,7 @@ export class DatabaseRoutes { this.crudRoutes.concat(this.customRoutes).forEach(route => { this._routingManager.route(route.input, route.returnType, route.handler); }); + this.realtimeService?.registerClient(this._routingManager); this._routingManager .registerRoutes() .then(() => { diff --git a/modules/database/src/utils/SchemaConverter.ts b/modules/database/src/utils/SchemaConverter.ts index 2e099ed57..f482f4936 100644 --- a/modules/database/src/utils/SchemaConverter.ts +++ b/modules/database/src/utils/SchemaConverter.ts @@ -38,6 +38,9 @@ export namespace SchemaConverter { authorization?: { enabled?: boolean; }; + realtime?: { + enabled?: boolean; + }; permissions?: { extendable?: boolean; canCreate?: boolean; @@ -130,6 +133,9 @@ export namespace SchemaConverter { defaults.conduit!.authorization?.enabled ?? false, }; + modelOptions.conduit.realtime = { + enabled: opts.realtime?.enabled ?? existing?.realtime?.enabled ?? false, + }; modelOptions.conduit.permissions = { extendable: explicit.permissions?.extendable !== undefined diff --git a/packages/core/package.json b/packages/core/package.json index 0636f4ce2..764e285ab 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -12,6 +12,7 @@ "start": "node dist/bin/www.js", "start:bundle": "node bundle/index.js", "lint": "eslint src", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/admin/realtime/*.test.js", "prebuild": "npm run generateTypes", "build": "rimraf dist && tsc", "generateTypes": "sh build.sh", diff --git a/packages/core/src/admin/AdminModule.ts b/packages/core/src/admin/AdminModule.ts index 3f18ea069..4f4574571 100644 --- a/packages/core/src/admin/AdminModule.ts +++ b/packages/core/src/admin/AdminModule.ts @@ -14,6 +14,7 @@ import { PatchRouteMiddlewaresRequest, RegisterAdminRouteRequest, RegisterAdminRouteRequest_PathDefinition, + SocketPushRequest, } from '../interfaces/index.js'; import { hashPassword } from './utils/auth.js'; import AdminConfigRawSchema from './config/index.js'; @@ -33,6 +34,7 @@ import { ConduitSocket, grpcToConduitRoute, RouteT, + SocketPush, } from '@conduitplatform/hermes'; import convict from 'convict'; import { NextFunction, Request, Response } from 'express'; @@ -48,6 +50,7 @@ import { getReadinessMiddleware } from '../health/readinessMiddleware.js'; import { ReadinessService } from '../health/ReadinessService.js'; import type { CoreHealthProvider } from '../health/types.js'; import { stripUndeclaredConfigParams } from '../utils/stripUndeclaredConfigParams.js'; +import { adminSocketNamespace } from './realtime/namespace.js'; export default class AdminModule { grpcSdk: ConduitGrpcSdk; @@ -93,6 +96,7 @@ export default class AdminModule { adminRoutes.createApiTokenRoute(), adminRoutes.getApiTokensRoute(), adminRoutes.deleteApiTokenRoute(), + adminRoutes.getRealtimeTicketRoute(), configRoutes.getModulesRoute(), adminRoutes.getStateExportRoute(this), adminRoutes.getStateImportRoute(this), @@ -131,6 +135,7 @@ export default class AdminModule { { registerAdminRoute: this.registerAdminRoute.bind(this), patchRouteMiddlewares: this.patchRouteMiddlewares.bind(this), + socketPush: this.socketPush.bind(this), }, ); } @@ -238,6 +243,27 @@ export default class AdminModule { callback(null, {}); } + async socketPush( + call: GrpcRequest, + callback: GrpcCallback>, + ) { + try { + const moduleName = call.metadata?.get('module-name')?.[0] as string | undefined; + const socketData: SocketPush = { + event: call.request.event, + data: call.request.data ? JSON.parse(call.request.data) : undefined, + receivers: call.request.receivers, + rooms: call.request.rooms, + namespace: adminSocketNamespace(moduleName), + }; + await this._router.socketPush(socketData); + } catch (err) { + ConduitGrpcSdk.Logger.error(err as Error); + return callback({ code: status.INTERNAL, message: 'Well that failed :/' }); + } + callback(null, {}); + } + registerRoute(route: ConduitRoute): void { this._sdkRoutes.push(route); this._router.registerConduitRoute(route); @@ -288,6 +314,16 @@ export default class AdminModule { `New admin route registered: ${r.input.action} ${r.input.path} handler url: ${url}`, ); this._router.registerConduitRoute(r); + } else if (r instanceof ConduitMiddleware) { + ConduitGrpcSdk.Logger.log( + `New admin middleware registered: ${r.input.path} handler url: ${url}`, + ); + this._router.registerRouteMiddleware(r, url); + } else if (r instanceof ConduitSocket) { + ConduitGrpcSdk.Logger.log( + `New admin socket registered: ${r.input.path} handler url: ${url}`, + ); + this._router.registerConduitSocket(r); } }); // @ts-ignore diff --git a/packages/core/src/admin/middleware/Admin.middleware.ts b/packages/core/src/admin/middleware/Admin.middleware.ts index 51c811987..716dae763 100644 --- a/packages/core/src/admin/middleware/Admin.middleware.ts +++ b/packages/core/src/admin/middleware/Admin.middleware.ts @@ -4,6 +4,7 @@ import { isNil } from 'lodash-es'; import { isDev } from '../utils/middleware.js'; import { ConduitRequest } from '@conduitplatform/hermes'; import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { isSocketHandshake } from '../realtime/handshake.js'; export function getAdminMiddleware(configManager: any) { return async function adminMiddleware( @@ -28,9 +29,14 @@ export function getAdminMiddleware(configManager: any) { ) { return next(); } - // Allow API tokens (cdt_*) to bypass masterkey; Auth.middleware will validate the token + // Allow API tokens (cdt_*) and Socket.IO handshakes with a Bearer token to + // skip masterkey. Auth.middleware still validates the credential. const authHeader = req.headers.authorization; - if (authHeader && authHeader.startsWith('Bearer cdt_')) { + if ( + typeof authHeader === 'string' && + authHeader.startsWith('Bearer ') && + (authHeader.startsWith('Bearer cdt_') || isSocketHandshake(req)) + ) { return next(); } const masterKey = req.headers.masterkey; diff --git a/packages/core/src/admin/middleware/Auth.middleware.ts b/packages/core/src/admin/middleware/Auth.middleware.ts index 4c883244d..410ea0fd6 100644 --- a/packages/core/src/admin/middleware/Auth.middleware.ts +++ b/packages/core/src/admin/middleware/Auth.middleware.ts @@ -7,6 +7,8 @@ import { isDev } from '../utils/middleware.js'; import { ConduitRequest } from '@conduitplatform/hermes'; import { gql } from 'graphql-tag'; import { ConfigController } from '@conduitplatform/module-tools'; +import { isSocketHandshake } from '../realtime/handshake.js'; +import { isRealtimeTicket } from '../realtime/ticket.js'; const excludedRestRoutes = ['/ready', '/live', '/login', '/config/modules']; const excludedGqlOperations = [ @@ -113,6 +115,10 @@ async function handleJwtToken( } const { id } = decoded; + if (isRealtimeTicket(decoded) && !isSocketHandshake(req)) { + res.status(401).json({ error: 'Realtime ticket cannot be used for HTTP requests' }); + return; + } if (decoded.twoFaRequired && req.path !== '/verify-twofa') { res.status(401).json({ error: 'Two FA required' }); return; diff --git a/packages/core/src/admin/realtime/handshake.test.ts b/packages/core/src/admin/realtime/handshake.test.ts new file mode 100644 index 000000000..59d17547f --- /dev/null +++ b/packages/core/src/admin/realtime/handshake.test.ts @@ -0,0 +1,12 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { isSocketHandshake } from './handshake.js'; + +describe('isSocketHandshake', () => { + it('detects Socket.IO engine requests', () => { + assert.equal(isSocketHandshake({ url: '/realtime/?EIO=4&transport=polling' }), true); + assert.equal(isSocketHandshake({ originalUrl: '/realtime' }), true); + assert.equal(isSocketHandshake({ path: '/login' }), false); + assert.equal(isSocketHandshake({ url: '/graphql' }), false); + }); +}); diff --git a/packages/core/src/admin/realtime/handshake.ts b/packages/core/src/admin/realtime/handshake.ts new file mode 100644 index 000000000..a4bf06fe1 --- /dev/null +++ b/packages/core/src/admin/realtime/handshake.ts @@ -0,0 +1,14 @@ +export function isSocketHandshake(req: { + url?: string; + originalUrl?: string; + path?: string; +}): boolean { + const parts = [req.url, req.originalUrl, req.path].filter((value): value is string => + Boolean(value), + ); + return parts.some(part => part.includes('EIO=') || isRealtimePath(part)); +} + +function isRealtimePath(value: string): boolean { + return /(?:^|[/?])realtime(?:[/?]|$)/.test(value); +} diff --git a/packages/core/src/admin/realtime/namespace.test.ts b/packages/core/src/admin/realtime/namespace.test.ts new file mode 100644 index 000000000..6bfcb3b64 --- /dev/null +++ b/packages/core/src/admin/realtime/namespace.test.ts @@ -0,0 +1,11 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { adminSocketNamespace } from './namespace.js'; + +describe('adminSocketNamespace', () => { + it('mirrors the Router module namespace contract', () => { + assert.equal(adminSocketNamespace('database'), '/database/'); + assert.throws(() => adminSocketNamespace(''), /module-name/); + assert.throws(() => adminSocketNamespace(undefined), /module-name/); + }); +}); diff --git a/packages/core/src/admin/realtime/namespace.ts b/packages/core/src/admin/realtime/namespace.ts new file mode 100644 index 000000000..ee66e9092 --- /dev/null +++ b/packages/core/src/admin/realtime/namespace.ts @@ -0,0 +1,7 @@ +export function adminSocketNamespace(moduleName: string | undefined | null): string { + const name = typeof moduleName === 'string' ? moduleName.trim() : ''; + if (!name) { + throw new Error('module-name metadata is required for Admin socket push'); + } + return `/${name}/`; +} diff --git a/packages/core/src/admin/realtime/ticket.test.ts b/packages/core/src/admin/realtime/ticket.test.ts new file mode 100644 index 000000000..e0d66fff1 --- /dev/null +++ b/packages/core/src/admin/realtime/ticket.test.ts @@ -0,0 +1,26 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { + ADMIN_REALTIME_AUDIENCE, + buildRealtimeTicketClaims, + isRealtimeTicket, +} from './ticket.js'; + +describe('admin realtime tickets', () => { + it('scopes claims to the realtime audience', () => { + const claims = buildRealtimeTicketClaims('admin-1'); + assert.equal(claims.id, 'admin-1'); + assert.equal(claims.aud, ADMIN_REALTIME_AUDIENCE); + assert.equal(isRealtimeTicket(claims), true); + }); + + it('rejects ordinary admin JWTs and missing payloads', () => { + assert.equal(isRealtimeTicket({ id: 'admin-1' }), false); + assert.equal(isRealtimeTicket({ id: 'admin-1', aud: 'other' }), false); + assert.equal(isRealtimeTicket(null), false); + assert.equal( + isRealtimeTicket({ id: 'admin-1', aud: [ADMIN_REALTIME_AUDIENCE, 'extra'] }), + true, + ); + }); +}); diff --git a/packages/core/src/admin/realtime/ticket.ts b/packages/core/src/admin/realtime/ticket.ts new file mode 100644 index 000000000..154630c0c --- /dev/null +++ b/packages/core/src/admin/realtime/ticket.ts @@ -0,0 +1,26 @@ +export const ADMIN_REALTIME_AUDIENCE = 'admin-realtime'; +export const ADMIN_REALTIME_TICKET_TTL_SECONDS = 30; + +export type RealtimeTicketClaims = { + id: string; + aud?: string | string[]; + twoFaRequired?: boolean; +}; + +export function buildRealtimeTicketClaims(adminId: string): RealtimeTicketClaims { + return { + id: adminId, + aud: ADMIN_REALTIME_AUDIENCE, + }; +} + +export function isRealtimeTicket( + decoded: RealtimeTicketClaims | null | undefined, +): boolean { + if (!decoded) return false; + const audience = decoded.aud; + if (Array.isArray(audience)) { + return audience.includes(ADMIN_REALTIME_AUDIENCE); + } + return audience === ADMIN_REALTIME_AUDIENCE; +} diff --git a/packages/core/src/admin/routes/RealtimeTicket.route.ts b/packages/core/src/admin/routes/RealtimeTicket.route.ts new file mode 100644 index 000000000..4a1b8e11e --- /dev/null +++ b/packages/core/src/admin/routes/RealtimeTicket.route.ts @@ -0,0 +1,51 @@ +import { isNil } from 'lodash-es'; +import { ConduitRoute } from '@conduitplatform/hermes'; +import { + ConduitError, + ConduitRouteActions, + ConduitRouteParameters, + ConduitRouteReturnDefinition, +} from '@conduitplatform/grpc-sdk'; +import { + ConduitNumber, + ConduitString, + ConfigController, +} from '@conduitplatform/module-tools'; +import { signToken } from '../utils/auth.js'; +import { + ADMIN_REALTIME_TICKET_TTL_SECONDS, + buildRealtimeTicketClaims, +} from '../realtime/ticket.js'; + +export function getRealtimeTicketRoute() { + return new ConduitRoute( + { + path: '/realtime/ticket', + action: ConduitRouteActions.POST, + mcp: false, + description: + 'Issue a short-lived token for Admin Socket.IO handshakes. The token cannot be used for REST or GraphQL.', + }, + new ConduitRouteReturnDefinition('RealtimeTicket', { + token: ConduitString.Required, + expiresIn: ConduitNumber.Required, + }), + async (req: ConduitRouteParameters) => { + const admin = req.context?.admin; + if (isNil(admin) || isNil(admin._id)) { + throw new ConduitError('UNAUTHORIZED', 401, 'Authentication required'); + } + const adminId = admin._id.toString(); + const authConfig = ConfigController.getInstance().config.auth; + const token = signToken( + buildRealtimeTicketClaims(adminId), + authConfig.tokenSecret, + ADMIN_REALTIME_TICKET_TTL_SECONDS, + ); + return { + token, + expiresIn: ADMIN_REALTIME_TICKET_TTL_SECONDS, + }; + }, + ); +} diff --git a/packages/core/src/admin/routes/index.ts b/packages/core/src/admin/routes/index.ts index 9bed86a70..f7c422e2c 100644 --- a/packages/core/src/admin/routes/index.ts +++ b/packages/core/src/admin/routes/index.ts @@ -20,3 +20,4 @@ export * from './ToggleTwoFa.route.js'; export * from './VerifyQrCode.route.js'; export * from './VerifyTwoFa.route.js'; export * from './ApiTokens.route.js'; +export * from './RealtimeTicket.route.js'; diff --git a/packages/core/src/core.proto b/packages/core/src/core.proto index 527d24f35..858dfa406 100644 --- a/packages/core/src/core.proto +++ b/packages/core/src/core.proto @@ -21,6 +21,14 @@ service Config { service Admin { rpc RegisterAdminRoute (RegisterAdminRouteRequest) returns (google.protobuf.Empty); rpc PatchRouteMiddlewares(PatchRouteMiddlewaresRequest) returns (google.protobuf.Empty); + rpc SocketPush (SocketPushRequest) returns (google.protobuf.Empty); +} + +message SocketPushRequest { + string event = 1; + optional string data = 2; + repeated string receivers = 3; + repeated string rooms = 4; } message RegisterAdminRouteRequest { diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index f4c4ef779..eb15cc5a9 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -67,5 +67,5 @@ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ }, "include": ["src/**/*"], - "exclude": ["tests", "node_modules", "dist", "bundle", "tsup.config.ts"] + "exclude": ["tests", "node_modules", "dist", "bundle", "tsup.config.ts", "**/*.test.ts"] } diff --git a/packages/core/tsconfig.test.json b/packages/core/tsconfig.test.json new file mode 100644 index 000000000..b0548057f --- /dev/null +++ b/packages/core/tsconfig.test.json @@ -0,0 +1,17 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": [ + "src/admin/realtime/ticket.ts", + "src/admin/realtime/handshake.ts", + "src/admin/realtime/namespace.ts", + "src/admin/realtime/*.test.ts" + ], + "exclude": ["node_modules", "dist", "bundle"] +} From ea51aabf34552038e5ad757e702af856246f2415 Mon Sep 17 00:00:00 2001 From: John Chantzigoulas Date: Wed, 9 Sep 2026 12:04:48 +0300 Subject: [PATCH 3/6] fix(docker): initiate local Mongo replica set before Conduit starts Mongo 4.4 keyfiles reject hyphens, and rs.status() returns ok:0 instead of throwing, so Compose never elected a primary for replicaSet URIs. --- docker/docker-compose.standalone.yml | 8 +++++-- docker/docker-compose.yml | 19 +++++++++++++---- docker/mongo/init-replica.sh | 32 ++++++++++++++++++++-------- docker/mongo/keyfile | 2 +- 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/docker/docker-compose.standalone.yml b/docker/docker-compose.standalone.yml index e1e5225be..1c695c107 100644 --- a/docker/docker-compose.standalone.yml +++ b/docker/docker-compose.standalone.yml @@ -15,8 +15,12 @@ services: image: 'docker.io/conduitplatform/conduit-standalone:${IMAGE_TAG}' restart: unless-stopped depends_on: - - redis - - mongodb + redis: + condition: service_started + mongodb: + condition: service_started + mongo-init-replica: + condition: service_completed_successfully ports: - '${CORE_GRPC_PORT:-55152}:55152' - '${DB_GRPC_PORT:-55160}:55160' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 751ea8bc6..7131604e3 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -71,10 +71,21 @@ services: image: 'docker.io/conduitplatform/database:${IMAGE_TAG}' restart: unless-stopped depends_on: - - core - - ${DB_TYPE:-mongodb} - - prometheus - - loki + core: + condition: service_started + prometheus: + condition: service_started + loki: + condition: service_started + mongodb: + condition: service_started + required: false + postgres: + condition: service_started + required: false + mongo-init-replica: + condition: service_completed_successfully + required: false ports: - '${DB_GRPC_PORT:-55160}:${DB_GRPC_PORT:-55160}' environment: diff --git a/docker/mongo/init-replica.sh b/docker/mongo/init-replica.sh index e4bb536d2..f94f5dbf4 100644 --- a/docker/mongo/init-replica.sh +++ b/docker/mongo/init-replica.sh @@ -4,17 +4,31 @@ HOST="${MONGO_HOST:-conduit-mongo}" USER="${MONGO_INITDB_ROOT_USERNAME:-conduit}" PASS="${MONGO_INITDB_ROOT_PASSWORD:-pass}" -until mongo --host "$HOST" -u "$USER" -p "$PASS" --authenticationDatabase admin --quiet --eval 'db.adminCommand({ ping: 1 })' >/dev/null 2>&1; do +mongo_eval() { + mongo --host "$HOST" -u "$USER" -p "$PASS" --authenticationDatabase admin --quiet --eval "$1" +} + +until mongo_eval 'db.adminCommand({ ping: 1 })' >/dev/null 2>&1; do sleep 2 done -mongo --host "$HOST" -u "$USER" -p "$PASS" --authenticationDatabase admin --quiet --eval ' - try { - rs.status(); - } catch (err) { - rs.initiate({ - _id: "rs0", - members: [{ _id: 0, host: "'"$HOST"':27017" }] - }); +# rs.status() returns { ok: 0 } before initiate; it does not throw. +mongo_eval ' + var status = rs.status(); + if (status.ok === 1) { + quit(0); + } + var result = rs.initiate({ + _id: "rs0", + members: [{ _id: 0, host: "'"$HOST"':27017" }] + }); + if (result.ok !== 1) { + printjson(result); + quit(1); } ' + +# Mongoose with replicaSet=rs0 only selects a PRIMARY (myState === 1). +until mongo_eval 'var s = rs.status(); if (s.ok === 1 && s.myState === 1) { quit(0); } quit(1);' >/dev/null 2>&1; do + sleep 1 +done diff --git a/docker/mongo/keyfile b/docker/mongo/keyfile index 8d7ffaf78..056a007b0 100644 --- a/docker/mongo/keyfile +++ b/docker/mongo/keyfile @@ -1 +1 @@ -conduit-dev-mongo-replica-keyfile-do-not-use-in-production-replace-me-before-any-real-deployment-0123456789abcdefghijklmnopqrstuvwxyz +ConduitLocalDevMongoReplicaSetKeyFileDoNotUseInProductionReplaceBeforeAnyRealDeployment0123456789abcdefghijklmnopqrstuvwxyz From cb1a6a98557a363a39bce546a9c65a78bba93d48 Mon Sep 17 00:00:00 2001 From: John Chantzigoulas Date: Wed, 9 Sep 2026 15:07:33 +0300 Subject: [PATCH 4/6] fix: isolate socket streams and persist realtime schema options Stop duplicate admin/router deliveries and apply handshake auth on connect. Retry change-stream leadership after lock races. --- libraries/hermes/package.json | 1 + libraries/hermes/src/Socket/Socket.ts | 33 ++++++---- .../applySocketGlobalMiddlewares.test.ts | 44 +++++++++++++ .../Socket/applySocketGlobalMiddlewares.ts | 66 +++++++++++++++++++ libraries/hermes/src/index.ts | 21 ++++++ libraries/hermes/tsconfig.json | 3 +- libraries/hermes/tsconfig.test.json | 16 +++++ .../__tests__/schema.admin.put-patch.test.ts | 15 +++++ .../realtime/MongoChangeStreamCoordinator.ts | 25 +++++-- .../realtime/__tests__/coordinator.test.ts | 31 ++++++++- .../__tests__/validate-model-options.test.ts | 37 +++++++++++ modules/database/src/utils/utilities.ts | 5 +- 12 files changed, 275 insertions(+), 22 deletions(-) create mode 100644 libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts create mode 100644 libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts create mode 100644 libraries/hermes/tsconfig.test.json create mode 100644 modules/database/src/utils/__tests__/validate-model-options.test.ts diff --git a/libraries/hermes/package.json b/libraries/hermes/package.json index 62f32ab5a..bfe74ff95 100644 --- a/libraries/hermes/package.json +++ b/libraries/hermes/package.json @@ -11,6 +11,7 @@ "scripts": { "prepublish": "npm run build", "build": "rimraf dist && tsc", + "test": "tsc -p tsconfig.test.json && node --test dist-test/Socket/applySocketGlobalMiddlewares.test.js", "publish": "npm publish", "postbuild": "copyfiles -u 1 src/*.proto src/**/*.json ./dist/" }, diff --git a/libraries/hermes/src/Socket/Socket.ts b/libraries/hermes/src/Socket/Socket.ts index 74ced4e2b..937ead5b9 100644 --- a/libraries/hermes/src/Socket/Socket.ts +++ b/libraries/hermes/src/Socket/Socket.ts @@ -15,6 +15,7 @@ import { } from '../interfaces/index.js'; import ObjectHash from 'object-hash'; import { ConduitError, ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { applySocketGlobalMiddlewares } from './applySocketGlobalMiddlewares.js'; import { buildSocketMiddlewareParams } from './buildSocketMiddlewareParams.js'; export class SocketController extends ConduitRouter { @@ -57,9 +58,17 @@ export class SocketController extends ConduitRouter { }; this.io = new IOServer(this.httpServer, this.options); this.redisClient = grpcSdk.redisManager.getClient(); + // Admin (e.g. :3031) and Router (e.g. :3001) are separate Socket.IO + // servers that share Redis. The adapter defaults would put both on the + // same stream, so a database change pushed to admin *and* router is + // delivered twice to every socket in those rooms. + const adapterKey = `socket.io:${this.port}`; this.io.adapter( createAdapter(this.redisClient, { onlyPlaintext: true, + streamName: adapterKey, + channelPrefix: adapterKey, + sessionKeyPrefix: `sio:session:${this.port}:`, }), ); this.httpServer.listen(this.port); @@ -107,20 +116,18 @@ export class SocketController extends ConduitRouter { this._registeredNamespaces.set(namespace, conduitSocket); const self = this; - this.globalMiddlewares.forEach(middleware => { - self.io.engine.use((req: any, res: any, next: any) => { - req.path = namespace; - middleware(req, res, next); - }); - }); this.io.of(namespace).use((socket, next) => { - const context = buildSocketMiddlewareParams(socket); - self - .checkMiddlewares(context, conduitSocket.input.middlewares) - .then(r => { - Object.assign(context.context, r); - socket.data = context.context; - next(); + applySocketGlobalMiddlewares(socket, self.globalMiddlewares) + .then(() => { + const context = buildSocketMiddlewareParams(socket); + Object.assign(context.context, socket.data); + return self + .checkMiddlewares(context, conduitSocket.input.middlewares) + .then(r => { + Object.assign(context.context, r); + socket.data = context.context; + next(); + }); }) .catch((err: Error | ConduitError) => { next(err); diff --git a/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts b/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts new file mode 100644 index 000000000..3e2834c0e --- /dev/null +++ b/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.test.ts @@ -0,0 +1,44 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import type { NextFunction, Request, Response } from 'express'; +import { applySocketGlobalMiddlewares } from './applySocketGlobalMiddlewares.js'; + +function fakeSocket(auth?: Record) { + return { + request: { + headers: {}, + url: '/realtime/?EIO=4&transport=polling', + }, + handshake: { auth: auth ?? {} }, + data: {}, + }; +} + +describe('applySocketGlobalMiddlewares', () => { + it('injects handshake.auth.token and copies conduit onto socket.data', async () => { + const socket = fakeSocket({ token: 'ticket-token' }); + await applySocketGlobalMiddlewares(socket as never, [ + (req: Request, _res: Response, next: NextFunction) => { + assert.equal(req.headers.authorization, 'Bearer ticket-token'); + (req as Request & { conduit: { admin: string } }).conduit = { + admin: 'ok', + }; + next(); + }, + ]); + assert.equal((socket.data as { admin?: string }).admin, 'ok'); + }); + + it('rejects when Express middleware sends 401 json', async () => { + const socket = fakeSocket(); + await assert.rejects( + () => + applySocketGlobalMiddlewares(socket as never, [ + (_req: Request, res: Response) => { + res.status(401).json({ error: 'No token provided' }); + }, + ]), + /No token provided/, + ); + }); +}); diff --git a/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts b/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts new file mode 100644 index 000000000..c3f5bb8a8 --- /dev/null +++ b/libraries/hermes/src/Socket/applySocketGlobalMiddlewares.ts @@ -0,0 +1,66 @@ +import type { NextFunction, Request, Response } from 'express'; +import type { Socket } from 'socket.io'; +import { buildSocketMiddlewareParams } from './buildSocketMiddlewareParams.js'; + +type ExpressMiddleware = (req: Request, res: Response, next: NextFunction) => void; + +type ConduitRequest = Request & { conduit?: Record }; + +export async function applySocketGlobalMiddlewares( + socket: Socket, + middlewares: ExpressMiddleware[], +): Promise { + const params = buildSocketMiddlewareParams(socket); + const req = Object.assign(socket.request, { + headers: params.headers, + path: '/realtime', + url: socket.request.url ?? '/realtime', + originalUrl: socket.request.url ?? '/realtime', + conduit: { ...params.context }, + }) as ConduitRequest; + + await new Promise((resolve, reject) => { + let settled = false; + const res = { + statusCode: 200, + status(this: { statusCode: number }, code: number) { + this.statusCode = code; + return this; + }, + json(this: unknown, body: { error?: string }) { + if (settled) return this; + settled = true; + reject(new Error(body?.error || 'Unauthorized')); + return this; + }, + setHeader() { + return this; + }, + removeHeader() {}, + } as unknown as Response; + + let index = 0; + const next = (err?: unknown) => { + if (settled) return; + if (err) { + settled = true; + reject(err instanceof Error ? err : new Error(String(err))); + return; + } + const middleware = middlewares[index++]; + if (!middleware) { + settled = true; + socket.data = { ...socket.data, ...(req.conduit ?? {}) }; + resolve(); + return; + } + try { + middleware(req, res, next); + } catch (error) { + settled = true; + reject(error as Error); + } + }; + next(); + }); +} diff --git a/libraries/hermes/src/index.ts b/libraries/hermes/src/index.ts index ab17086b2..ac22e28ba 100644 --- a/libraries/hermes/src/index.ts +++ b/libraries/hermes/src/index.ts @@ -47,6 +47,15 @@ export class ConduitRoutingController { private _cleanupTimeout: NodeJS.Timeout | null = null; /** Routes registered before MCP starts; replayed in initMCP. */ private readonly _conduitRoutesByKey: Map = new Map(); + /** Sockets registered before Socket.IO starts; replayed in initSockets. */ + private readonly _conduitSocketsByPath: Map = new Map(); + private readonly _socketMiddlewares: Array< + (req: ConduitRequest, res: Response, next: NextFunction) => void + > = []; + private readonly _socketRouteMiddlewares: Array<{ + middleware: ConduitMiddleware; + moduleUrl: string; + }> = []; private readonly routeTrie: RouteTrie = new RouteTrie(); readonly expressApp: Express = express(); readonly server = http.createServer(this.expressApp); @@ -143,6 +152,15 @@ export class ConduitRoutingController { this.expressApp, this.metrics, ); + for (const middleware of this._socketMiddlewares) { + this._socketRouter.registerGlobalMiddleware(middleware); + } + for (const { middleware, moduleUrl } of this._socketRouteMiddlewares) { + this._socketRouter.registerMiddleware(middleware, moduleUrl); + } + for (const socket of this._conduitSocketsByPath.values()) { + this._socketRouter.registerConduitSocket(socket); + } } initMCP(config?: { @@ -231,6 +249,7 @@ export class ConduitRoutingController { ) { this._middlewareRouter.use(middleware); if (socketMiddleware) { + this._socketMiddlewares.push(middleware); this._socketRouter?.registerGlobalMiddleware(middleware); } } @@ -238,6 +257,7 @@ export class ConduitRoutingController { registerRouteMiddleware(middleware: ConduitMiddleware, moduleUrl: string) { this._restRouter?.registerMiddleware(middleware, moduleUrl); this._graphQLRouter?.registerMiddleware(middleware, moduleUrl); + this._socketRouteMiddlewares.push({ middleware, moduleUrl }); this._socketRouter?.registerMiddleware(middleware, moduleUrl); this._mcpRouter?.registerMiddleware(middleware, moduleUrl); } @@ -286,6 +306,7 @@ export class ConduitRoutingController { } registerConduitSocket(socket: ConduitSocket) { + this._conduitSocketsByPath.set(socket.input.path, socket); this._socketRouter?.registerConduitSocket(socket); } diff --git a/libraries/hermes/tsconfig.json b/libraries/hermes/tsconfig.json index cd32b1ca8..0420a284b 100644 --- a/libraries/hermes/tsconfig.json +++ b/libraries/hermes/tsconfig.json @@ -65,5 +65,6 @@ /* Advanced Options */ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ - } + }, + "exclude": ["node_modules", "dist", "**/*.test.ts"] } diff --git a/libraries/hermes/tsconfig.test.json b/libraries/hermes/tsconfig.test.json new file mode 100644 index 000000000..8c2d1a100 --- /dev/null +++ b/libraries/hermes/tsconfig.test.json @@ -0,0 +1,16 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": [ + "src/Socket/applySocketGlobalMiddlewares.ts", + "src/Socket/applySocketGlobalMiddlewares.test.ts", + "src/Socket/buildSocketMiddlewareParams.ts" + ], + "exclude": ["node_modules", "dist"] +} diff --git a/modules/database/src/admin/__tests__/schema.admin.put-patch.test.ts b/modules/database/src/admin/__tests__/schema.admin.put-patch.test.ts index 89a7ae8ea..464bf4462 100644 --- a/modules/database/src/admin/__tests__/schema.admin.put-patch.test.ts +++ b/modules/database/src/admin/__tests__/schema.admin.put-patch.test.ts @@ -185,6 +185,21 @@ describe('SchemaAdmin PUT/PATCH split', () => { warnSpy.mockRestore(); }); + it('persists realtime conduitOptions used by live document updates', async () => { + const requestedSchema = makeRequestedSchema(); + const { admin, createSchema } = setup(requestedSchema); + + await admin.patchSchema( + makeCall({ + id: 'schema-1', + conduitOptions: { realtime: { enabled: true } }, + }), + ); + + const [writtenSchema] = createSchema.mock.calls[0]; + expect(writtenSchema.modelOptions.conduit.realtime.enabled).toBe(true); + }); + it('wipes existing documents when enabling authorization via conduitOptions', async () => { const requestedSchema = makeRequestedSchema(); const { admin, deleteMany } = setup(requestedSchema); diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts index 32ff41dc7..7b82797dd 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/MongoChangeStreamCoordinator.ts @@ -55,6 +55,7 @@ export class MongoChangeStreamCoordinator { private topology: TopologyResult = { supported: false }; private retryAttempt = 0; private watching = false; + private opening = false; private ignoreClose = false; constructor(private readonly options: CoordinatorOptions) {} @@ -90,6 +91,14 @@ export class MongoChangeStreamCoordinator { await this.releaseLeader(); this.streamState = 'idle'; this.lastError = this.topology.message; + // Hello can fail during startup before Mongo is ready. Keep retrying + // that case; a confirmed standalone topology will not recover. + if ( + !this.topology.message || + this.topology.message.includes('Unable to determine') + ) { + this.scheduleRetry(); + } return; } if (this.options.getOptedInSchemas().length === 0) { @@ -121,7 +130,11 @@ export class MongoChangeStreamCoordinator { LOCK_TTL_MS, ); if (!acquired) { + // Another instance holds the lock, or a crashed holder has not + // expired yet. Without a retry, a standalone process stays idle + // forever after a restart races the previous TTL. this.streamState = 'idle'; + this.scheduleRetry(); return; } this.lock = acquired; @@ -153,13 +166,15 @@ export class MongoChangeStreamCoordinator { } private async openStream() { - if (this.watching || this.closed) return; + if (this.watching || this.closed || this.opening) return; + this.opening = true; this.streamState = 'starting'; this.ignoreClose = false; - const resumeAfter = parseResumeToken( - (await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY)) ?? undefined, - ); try { + const resumeAfter = parseResumeToken( + (await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY)) ?? undefined, + ); + if (this.watching || this.closed) return; const stream = this.options.watch({ resumeAfter: resumeAfter ?? undefined }); this.stream = stream; this.watching = true; @@ -180,6 +195,8 @@ export class MongoChangeStreamCoordinator { } catch (err) { this.watching = false; await this.handleStreamError(err); + } finally { + this.opening = false; } } diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index 5ec3a4af3..a523919b7 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -29,6 +29,7 @@ class MemoryStore { function createCoordinator(overrides?: { allow?: boolean; schemas?: { name: string; collectionName: string; authorizationEnabled: boolean }[]; + getKeyDelayMs?: number; }) { const stream = new EventEmitter() as EventEmitter & { close: () => Promise }; stream.close = async () => { @@ -42,12 +43,18 @@ function createCoordinator(overrides?: { const routerPush = jest.fn(async () => undefined); const adminPush = jest.fn(async () => undefined); const publish = jest.fn(); + const watch = jest.fn(() => stream as never); const subscriptions = new RealtimeSubscriptionTracker(new MemoryStore()); const grpcSdk = { state: { tryAcquireLock: jest.fn(async () => lock), releaseLock: jest.fn(async () => undefined), - getKey: jest.fn(async (key: string) => state.get(key) ?? null), + getKey: jest.fn(async (key: string) => { + if (overrides?.getKeyDelayMs) { + await new Promise(resolve => setTimeout(resolve, overrides.getKeyDelayMs)); + } + return state.get(key) ?? null; + }), setKey: jest.fn(async (key: string, value: string) => { state.set(key, value); }), @@ -65,7 +72,7 @@ function createCoordinator(overrides?: { }; const coordinator = new MongoChangeStreamCoordinator({ grpcSdk: grpcSdk as never, - watch: () => stream as never, + watch, hello: async () => ({ setName: 'rs0' }), getOptedInSchemas: () => overrides?.schemas ?? [ @@ -78,6 +85,7 @@ function createCoordinator(overrides?: { return { coordinator, stream, + watch, routerPush, adminPush, publish, @@ -155,6 +163,25 @@ describe('MongoChangeStreamCoordinator', () => { await coordinator.shutdown(); }); + it('retries later when the leader lock is held by another instance', async () => { + jest.useFakeTimers(); + const { coordinator, grpcSdk, lock } = createCoordinator(); + grpcSdk.state.tryAcquireLock.mockResolvedValueOnce(null).mockResolvedValue(lock); + await coordinator.reconcile(); + expect(coordinator.getState()).toBe('idle'); + await jest.advanceTimersByTimeAsync(1_000); + expect(coordinator.getState()).toBe('live'); + await coordinator.shutdown(); + jest.useRealTimers(); + }); + + it('opens a single watch when reconcile runs concurrently', async () => { + const { coordinator, watch } = createCoordinator({ getKeyDelayMs: 40 }); + await Promise.all([coordinator.reconcile(), coordinator.reconcile()]); + expect(watch).toHaveBeenCalledTimes(1); + await coordinator.shutdown(); + }); + it('clears an unusable resume token and retries', async () => { const { coordinator, stream, grpcSdk } = createCoordinator(); await coordinator.reconcile(); diff --git a/modules/database/src/utils/__tests__/validate-model-options.test.ts b/modules/database/src/utils/__tests__/validate-model-options.test.ts new file mode 100644 index 000000000..a832255e5 --- /dev/null +++ b/modules/database/src/utils/__tests__/validate-model-options.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from '@jest/globals'; +import { validateSchemaInput } from '../utilities.js'; + +const baseConduit = { + cms: { enabled: true }, + authorization: { enabled: false }, + permissions: { + extendable: true, + canCreate: true, + canModify: 'Everything' as const, + canDelete: true, + }, +}; + +describe('validateSchemaInput modelOptions.conduit', () => { + it('allows conduit.realtime used by live document updates', () => { + expect(() => + validateSchemaInput('TestSchema', undefined, { + conduit: { + ...baseConduit, + realtime: { enabled: true }, + }, + }), + ).not.toThrow(); + }); + + it('rejects an unknown conduit field', () => { + expect(() => + validateSchemaInput('TestSchema', undefined, { + conduit: { + ...baseConduit, + notARealField: true, + }, + }), + ).toThrow(/fields allowed inside 'conduit' field/); + }); +}); diff --git a/modules/database/src/utils/utilities.ts b/modules/database/src/utils/utilities.ts index ce6cd7328..f3e98a292 100644 --- a/modules/database/src/utils/utilities.ts +++ b/modules/database/src/utils/utilities.ts @@ -183,10 +183,11 @@ function validateModelOptions(modelOptions: ConduitSchemaOptions) { conduitKey !== 'cms' && conduitKey !== 'permissions' && conduitKey !== 'authorization' && - conduitKey !== 'imported' + conduitKey !== 'imported' && + conduitKey !== 'realtime' ) throw new Error( - "Only 'cms', 'permissions', 'authorization', 'imported', and 'readPreference' fields allowed inside 'conduit' field", + "Only 'cms', 'permissions', 'authorization', 'imported', 'readPreference', and 'realtime' fields allowed inside 'conduit' field", ); if (conduitKey === 'imported') { if (!isBoolean(modelOptions.conduit!.imported)) From 2c637c873d5e5de14ad9398fa10c8d65506f126b Mon Sep 17 00:00:00 2001 From: John Chantzigoulas Date: Fri, 11 Sep 2026 13:37:42 +0300 Subject: [PATCH 5/6] feat(database): add SQL live updates via triggers and change-log Extend realtime beyond MongoDB so PostgreSQL, MySQL, MariaDB, and SQLite emit the same metadata-only change events using table triggers and a durable internal log. --- modules/database/README.mdx | 60 ++++---- modules/database/package.json | 3 +- .../src/adapters/sequelize-adapter/index.ts | 1 + modules/database/src/config/index.ts | 2 +- ...rdinator.ts => ChangeStreamCoordinator.ts} | 60 +++++--- .../database/src/realtime/RealtimeService.ts | 36 ++++- .../realtime/__tests__/coordinator.test.ts | 33 +++- .../realtime/__tests__/sql-builders.test.ts | 90 +++++++++++ .../sql-change-stream.integration.test.ts | 82 ++++++++++ .../src/realtime/__tests__/status.test.ts | 20 ++- modules/database/src/realtime/index.ts | 4 + .../src/realtime/sql/SqlChangeStream.ts | 142 +++++++++++++++++ .../src/realtime/sql/SqlRealtimeSupport.ts | 30 ++++ .../database/src/realtime/sql/changelog.ts | 118 ++++++++++++++ .../database/src/realtime/sql/constants.ts | 23 +++ modules/database/src/realtime/sql/ddl.ts | 79 ++++++++++ .../database/src/realtime/sql/identifiers.ts | 63 ++++++++ modules/database/src/realtime/sql/index.ts | 13 ++ modules/database/src/realtime/sql/mapEvent.ts | 21 +++ modules/database/src/realtime/sql/resume.ts | 41 +++++ .../database/src/realtime/sql/triggerSql.ts | 145 ++++++++++++++++++ modules/database/src/realtime/sql/triggers.ts | 106 +++++++++++++ modules/database/src/realtime/status.ts | 16 +- pnpm-lock.yaml | 21 ++- 24 files changed, 1140 insertions(+), 69 deletions(-) rename modules/database/src/realtime/{MongoChangeStreamCoordinator.ts => ChangeStreamCoordinator.ts} (86%) create mode 100644 modules/database/src/realtime/__tests__/sql-builders.test.ts create mode 100644 modules/database/src/realtime/__tests__/sql-change-stream.integration.test.ts create mode 100644 modules/database/src/realtime/sql/SqlChangeStream.ts create mode 100644 modules/database/src/realtime/sql/SqlRealtimeSupport.ts create mode 100644 modules/database/src/realtime/sql/changelog.ts create mode 100644 modules/database/src/realtime/sql/constants.ts create mode 100644 modules/database/src/realtime/sql/ddl.ts create mode 100644 modules/database/src/realtime/sql/identifiers.ts create mode 100644 modules/database/src/realtime/sql/index.ts create mode 100644 modules/database/src/realtime/sql/mapEvent.ts create mode 100644 modules/database/src/realtime/sql/resume.ts create mode 100644 modules/database/src/realtime/sql/triggerSql.ts create mode 100644 modules/database/src/realtime/sql/triggers.ts diff --git a/modules/database/README.mdx b/modules/database/README.mdx index 61bc195b0..8b3d1aa45 100644 --- a/modules/database/README.mdx +++ b/modules/database/README.mdx @@ -34,20 +34,20 @@ since the latter need to go through parsers that are otherwise unnecessary for M ## Environment Variables 📃 -| Variable | Description | Required | Example | Default | -|:--------------------:|:-----------------------------------------------------| :------: | :----------------: | :------: | -| `CONDUIT_SERVER` | Conduit Core's address and port | True | `0.0.0.0:55152` | - | -| `SERVICE_URL` | This should be where this service listens on. If behind a LB it should point to the LB's IP/DNS | False | `0.0.0.0:55190` | -| `GRPC_PORT` | The port number the gRPC server will listen to | False | `55190` | -| `GRPC_KEY` | Specifying a secret enables gRPC signed request protection (**use across modules**) | False | `someRandomSecret` | - | -| `DB_CONN_URI` | DB Connection URI | False | `postgres://conduit:pass@localhost:5432/conduit` | `mongodb://localhost:27017` | -| `DB_TYPE` | DB Engine Type | False | `postgres` | `mongodb` | +| Variable | Description | Required | Example | Default | +| :--------------: | :---------------------------------------------------------------------------------------------- | :------: | :----------------------------------------------: | :-------------------------: | +| `CONDUIT_SERVER` | Conduit Core's address and port | True | `0.0.0.0:55152` | - | +| `SERVICE_URL` | This should be where this service listens on. If behind a LB it should point to the LB's IP/DNS | False | `0.0.0.0:55190` | +| `GRPC_PORT` | The port number the gRPC server will listen to | False | `55190` | +| `GRPC_KEY` | Specifying a secret enables gRPC signed request protection (**use across modules**) | False | `someRandomSecret` | - | +| `DB_CONN_URI` | DB Connection URI | False | `postgres://conduit:pass@localhost:5432/conduit` | `mongodb://localhost:27017` | +| `DB_TYPE` | DB Engine Type | False | `postgres` | `mongodb` | ## Replica Set Configuration When using MongoDB with a replica set (e.g., MongoDB Atlas), the database module supports configuring read preference, write concern, and read concern through the admin panel at `PATCH /config/database`. -Live document updates also require a replica set or sharded cluster. Local Compose files initialize a single-node replica set so change streams can be exercised. +Live document updates on MongoDB also require a replica set or sharded cluster. Local Compose files initialize a single-node replica set so change streams can be exercised. SQL engines do not need a replica set. ### Live updates @@ -60,22 +60,25 @@ unsubscribe({ schema: 'Order', documentId?: string }) Events arrive as `change` with `{ version, operation, schema, documentId, occurredAt, resumeToken }` and contain no document fields. Consumers should refetch through their authorized REST or custom-endpoint path. +MongoDB uses native change streams. PostgreSQL, MySQL, MariaDB, and SQLite capture the same metadata through table triggers and an internal `_cnd_DatabaseChange` log (not a CMS schema). PostgreSQL wakes the listener with `NOTIFY`; the other SQL dialects poll. The database role needs permission to `CREATE TRIGGER` (and on PostgreSQL, to create a function). SQL live updates do not require a replica set. + Client subscribers must authenticate. Schemas with document-level authorization reject schema-wide subscriptions and require a document ID plus a `read` check. Admin consumers use `POST /realtime/ticket` for a 30-second handshake token; session JWTs and masterkeys must not be sent from browser code. Admin sockets must be enabled (`admin.transports.sockets`) and the Admin socket port (`ADMIN_SOCKET_PORT`, default 3031) reachable from the UI. ### Configuration Options -| Setting | Values | Default | Description | -| :---------------: | :------------------------------------------------------------------------ | :-------: | :------------------------------------------------------- | -| `readPreference` | `primary`, `primaryPreferred`, `secondary`, `secondaryPreferred`, `nearest` | `primary` | Controls which replica set members receive read queries | -| `writeConcern` | `1`, `majority` | `1` | How many members must acknowledge a write | -| `readConcern` | `local`, `available`, `majority`, `linearizable`, `snapshot` | `local` | Consistency level for read operations | -| `realtime.enabled` | `true`, `false` | `false` | Enable MongoDB change-stream live updates for opted-in schemas | +| Setting | Values | Default | Description | +| :----------------: | :-------------------------------------------------------------------------- | :-------: | :------------------------------------------------------ | +| `readPreference` | `primary`, `primaryPreferred`, `secondary`, `secondaryPreferred`, `nearest` | `primary` | Controls which replica set members receive read queries | +| `writeConcern` | `1`, `majority` | `1` | How many members must acknowledge a write | +| `readConcern` | `local`, `available`, `majority`, `linearizable`, `snapshot` | `local` | Consistency level for read operations | +| `realtime.enabled` | `true`, `false` | `false` | Enable live updates for opted-in schemas | ### Recommended Production Settings For MongoDB Atlas deployments with read replicas: + - **readPreference**: `secondaryPreferred` — distributes reads across replicas, falls back to primary - **writeConcern**: `majority` — ensures writes survive replica set elections - **readConcern**: `local` — suitable for most workloads @@ -87,7 +90,7 @@ Modules can override the configured readPreference on individual queries when th ```typescript const doc = await MySchema.getInstance().findOne( { _id: someId }, - { readPreference: 'primary' } + { readPreference: 'primary' }, ); ``` @@ -100,19 +103,19 @@ Standalone deployments (without replicas) work identically regardless of these s | Operator | Description | | :--------: | :----------------------------------------------------------------------------------------------------------------------- | -| `in` | Matches any of the values specified in an array | +| `in` | Matches any of the values specified in an array | | `contains` | Checks if a value is contained in an array or not. | -| `nin` | Selects the documents where the value of a field is not equal any value in the specified array. | -| `eq` | Matches documents where the value of a field equals the specified value. | -| `ne` | Selects the documents where the value of the field is not equal to the specified value | -| `lt` | Selects the documents where the value of the field is less than (i.e. <) the specified value. | -| `gt` | Selects the documents where the value of the field is greater than (i.e. >) the specified value. | -| `lte` | Selects the documents where the value of the field is less or equal than (i.e. <=) the specified value. | -| `gte` | Selects the documents where the value of the field is greater or equal than (i.e. >=) the specified value. | -| `or` | Performs OR operation on an array of two or more expressions and selects the documents that satisfy at least one | -| `and` | Performs AND operation on an array of two or more expressions and selects the documents that satisfy at least one | -| `not` | Performs NOT operation on an array of two or more expressions and selects the documents that do not match the expression | -| `regex` | Select the documents where the value of the field matches the regex. | +| `nin` | Selects the documents where the value of a field is not equal any value in the specified array. | +| `eq` | Matches documents where the value of a field equals the specified value. | +| `ne` | Selects the documents where the value of the field is not equal to the specified value | +| `lt` | Selects the documents where the value of the field is less than (i.e. <) the specified value. | +| `gt` | Selects the documents where the value of the field is greater than (i.e. >) the specified value. | +| `lte` | Selects the documents where the value of the field is less or equal than (i.e. <=) the specified value. | +| `gte` | Selects the documents where the value of the field is greater or equal than (i.e. >=) the specified value. | +| `or` | Performs OR operation on an array of two or more expressions and selects the documents that satisfy at least one | +| `and` | Performs AND operation on an array of two or more expressions and selects the documents that satisfy at least one | +| `not` | Performs NOT operation on an array of two or more expressions and selects the documents that do not match the expression | +| `regex` | Select the documents where the value of the field matches the regex. | These operators have been tested thoroughly in varying levels of complexity. @@ -137,6 +140,7 @@ add its id in place of the data. ## Sequelize Caveats Currently, the Sequelize implementation has the following limitations: + - No index creation - Update queries only update provided fields without the option to replace entire rows, unless all columns are provided. - Like operations do not work diff --git a/modules/database/package.json b/modules/database/package.json index 656bee595..c5a30675a 100644 --- a/modules/database/package.json +++ b/modules/database/package.json @@ -72,9 +72,10 @@ "@jest/globals": "^30.5.1", "@types/convict": "^6.1.6", "@types/jest": "^30.0.0", - "@types/object-hash": "^3.0.6", "@types/lodash-es": "^4.17.12", "@types/node": "24.13.3", + "@types/object-hash": "^3.0.6", + "@types/pg": "^8.23.1", "copyfiles": "^2.4.1", "jest": "^30.5.1", "rimraf": "^6.1.3", diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 30df75904..e4a172219 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -140,6 +140,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter this.models['_DeclaredSchema'].originalSchema.collectionName; for (const table of tableNames) { if (table === declaredSchemaTableName) continue; + if (table.startsWith('_cnd_')) continue; const tableInDeclaredSchemas = declaredSchemas.some( (declaredSchema: ConduitSchema) => { if (declaredSchema.collectionName && declaredSchema.collectionName !== '') { diff --git a/modules/database/src/config/index.ts b/modules/database/src/config/index.ts index 1ddda80d7..9fda0bb6b 100644 --- a/modules/database/src/config/index.ts +++ b/modules/database/src/config/index.ts @@ -49,7 +49,7 @@ const AppConfigSchema = { }, realtime: { enabled: { - doc: 'Enable MongoDB change-stream live updates for opted-in schemas', + doc: 'Enable live updates for opted-in schemas', format: 'Boolean', default: false, }, diff --git a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts b/modules/database/src/realtime/ChangeStreamCoordinator.ts similarity index 86% rename from modules/database/src/realtime/MongoChangeStreamCoordinator.ts rename to modules/database/src/realtime/ChangeStreamCoordinator.ts index 7b82797dd..00b6659ac 100644 --- a/modules/database/src/realtime/MongoChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/ChangeStreamCoordinator.ts @@ -1,15 +1,11 @@ import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import { normalizeChangeEvent, - parseResumeToken, + parseResumeToken as parseMongoResumeToken, type RawChangeEvent, } from './normalize.js'; import { authorizedDocumentRoom, roomsForPublicChange } from './rooms.js'; -import { - isResumeTokenUnusable, - topologyFromHello, - type TopologyResult, -} from './topology.js'; +import { isResumeTokenUnusable, type TopologyResult } from './topology.js'; import type { ChangeStreamLike, DatabaseChangeEvent, @@ -35,15 +31,17 @@ export type WatchFactory = (options: { resumeAfter?: unknown }) => ChangeStreamL export type CoordinatorOptions = { grpcSdk: ConduitGrpcSdk; watch: WatchFactory; - hello: () => Promise<{ setName?: string; msg?: string } | null>; + checkTopology: () => Promise; getOptedInSchemas: () => OptedInSchema[]; subscriptions: RealtimeSubscriptionTracker; enabled: () => boolean; - engine: () => string; socketsEnabled?: () => Promise; + parseResumeToken?: (token: string | null | undefined) => unknown | undefined; + prepare?: () => Promise; + onResumePersisted?: (resumeToken: string) => Promise; }; -export class MongoChangeStreamCoordinator { +export class ChangeStreamCoordinator { private lock: LeaderLock | null = null; private stream: ChangeStreamLike | null = null; private renewTimer: NodeJS.Timeout | null = null; @@ -78,21 +76,22 @@ export class MongoChangeStreamCoordinator { async reconcile(): Promise { if (this.closed) return; - const engine = this.options.engine(); - if (engine !== 'MongoDB' || !this.options.enabled()) { + if (!this.options.enabled()) { + await this.safePrepare(); await this.stopStream('idle'); await this.releaseLeader(); - this.streamState = engine !== 'MongoDB' ? 'unsupported' : 'disabled'; + this.streamState = 'disabled'; return; } - this.topology = topologyFromHello(await this.options.hello().catch(() => null)); + this.topology = await this.options.checkTopology().catch(() => ({ + supported: false, + message: 'Unable to determine database topology', + })); if (!this.topology.supported) { await this.stopStream('idle'); await this.releaseLeader(); this.streamState = 'idle'; this.lastError = this.topology.message; - // Hello can fail during startup before Mongo is ready. Keep retrying - // that case; a confirmed standalone topology will not recover. if ( !this.topology.message || this.topology.message.includes('Unable to determine') @@ -101,6 +100,15 @@ export class MongoChangeStreamCoordinator { } return; } + try { + await this.options.prepare?.(); + } catch (err) { + this.lastError = err instanceof Error ? err.message : String(err); + this.streamState = 'degraded'; + ConduitGrpcSdk.Logger.error(err as Error); + this.scheduleRetry(); + return; + } if (this.options.getOptedInSchemas().length === 0) { await this.stopStream('idle'); await this.releaseLeader(); @@ -117,6 +125,14 @@ export class MongoChangeStreamCoordinator { await this.releaseLeader(); } + private async safePrepare(): Promise { + try { + await this.options.prepare?.(); + } catch (err) { + ConduitGrpcSdk.Logger.error(err as Error); + } + } + private async ensureLeader(): Promise { if (this.lock) { if (!this.watching) { @@ -130,9 +146,6 @@ export class MongoChangeStreamCoordinator { LOCK_TTL_MS, ); if (!acquired) { - // Another instance holds the lock, or a crashed holder has not - // expired yet. Without a retry, a standalone process stays idle - // forever after a restart races the previous TTL. this.streamState = 'idle'; this.scheduleRetry(); return; @@ -171,7 +184,8 @@ export class MongoChangeStreamCoordinator { this.streamState = 'starting'; this.ignoreClose = false; try { - const resumeAfter = parseResumeToken( + const parseToken = this.options.parseResumeToken ?? parseMongoResumeToken; + const resumeAfter = parseToken( (await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY)) ?? undefined, ); if (this.watching || this.closed) return; @@ -208,6 +222,11 @@ export class MongoChangeStreamCoordinator { this.lastEventAt = event.occurredAt; this.lastError = undefined; await this.options.grpcSdk.state!.setKey(RESUME_TOKEN_KEY, event.resumeToken); + try { + await this.options.onResumePersisted?.(event.resumeToken); + } catch (err) { + ConduitGrpcSdk.Logger.error(err as Error); + } this.options.grpcSdk.bus?.publish( `database:change:${schema.name}`, JSON.stringify(event), @@ -344,3 +363,6 @@ export class MongoChangeStreamCoordinator { } } } + +/** @deprecated Use ChangeStreamCoordinator */ +export { ChangeStreamCoordinator as MongoChangeStreamCoordinator }; diff --git a/modules/database/src/realtime/RealtimeService.ts b/modules/database/src/realtime/RealtimeService.ts index db6e77d21..b85c3aa76 100644 --- a/modules/database/src/realtime/RealtimeService.ts +++ b/modules/database/src/realtime/RealtimeService.ts @@ -11,19 +11,23 @@ import { } from '@conduitplatform/module-tools'; import { DatabaseAdapter } from '../adapters/DatabaseAdapter.js'; import { MongooseAdapter } from '../adapters/mongoose-adapter/index.js'; +import { SequelizeAdapter } from '../adapters/sequelize-adapter/index.js'; import { MongooseSchema } from '../adapters/mongoose-adapter/MongooseSchema.js'; import { SequelizeSchema } from '../adapters/sequelize-adapter/SequelizeSchema.js'; import { toOptedInSchema } from './authorize.js'; -import { MongoChangeStreamCoordinator } from './MongoChangeStreamCoordinator.js'; +import { ChangeStreamCoordinator } from './ChangeStreamCoordinator.js'; import { registerDatabaseRealtimeSocket } from './sockets.js'; import { buildRealtimeStatus } from './status.js'; import { RealtimeSubscriptionTracker } from './subscriptions.js'; import type { OptedInSchema, RealtimeStatus } from './types.js'; import type { ChangeStreamLike } from './types.js'; +import { topologyFromHello } from './topology.js'; +import { SqlRealtimeSupport } from './sql/SqlRealtimeSupport.js'; export class RealtimeService { private readonly subscriptions: RealtimeSubscriptionTracker; - private coordinator?: MongoChangeStreamCoordinator; + private coordinator?: ChangeStreamCoordinator; + private sqlSupport?: SqlRealtimeSupport; constructor( private readonly grpcSdk: ConduitGrpcSdk, @@ -39,16 +43,31 @@ export class RealtimeService { void this.reconcile(); }); if (adapter instanceof MongooseAdapter) { - this.coordinator = new MongoChangeStreamCoordinator({ + this.coordinator = new ChangeStreamCoordinator({ grpcSdk, watch: options => this.openWatch(adapter, options.resumeAfter), - hello: () => this.hello(adapter), + checkTopology: () => this.checkMongoTopology(adapter), getOptedInSchemas: () => this.getOptedInSchemas(), subscriptions: this.subscriptions, enabled: () => this.isGloballyEnabled(), - engine: () => adapter.getDatabaseType(), socketsEnabled: () => this.areAdminSocketsEnabled(), }); + } else if (adapter instanceof SequelizeAdapter) { + this.sqlSupport = new SqlRealtimeSupport(adapter); + this.coordinator = new ChangeStreamCoordinator({ + grpcSdk, + watch: options => this.sqlSupport!.openWatch(options.resumeAfter), + checkTopology: async () => ({ supported: true }), + getOptedInSchemas: () => this.getOptedInSchemas(), + subscriptions: this.subscriptions, + enabled: () => this.isGloballyEnabled(), + socketsEnabled: () => this.areAdminSocketsEnabled(), + prepare: () => + this.sqlSupport!.prepare( + this.isGloballyEnabled() ? this.getOptedInSchemas() : [], + ), + onResumePersisted: token => this.sqlSupport!.trimThrough(token), + }); } } @@ -105,8 +124,7 @@ export class RealtimeService { topologySupported: this.coordinator?.getTopology().supported ?? false, topologyMessage: this.coordinator?.getTopology().message, activeSchemaCount: optedIn.length, - streamState: - this.coordinator?.getState() ?? (engine === 'MongoDB' ? 'idle' : 'unsupported'), + streamState: this.coordinator?.getState() ?? 'unsupported', lastEventAt: this.coordinator?.getLastEventAt(), lastError: this.coordinator?.getLastError(), socketsEnabled: await this.areAdminSocketsEnabled(), @@ -141,6 +159,10 @@ export class RealtimeService { ) as unknown as ChangeStreamLike; } + private async checkMongoTopology(adapter: MongooseAdapter) { + return topologyFromHello(await this.hello(adapter).catch(() => null)); + } + private async hello( adapter: MongooseAdapter, ): Promise<{ setName?: string; msg?: string } | null> { diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index a523919b7..995401e7d 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -1,7 +1,7 @@ import { EventEmitter } from 'node:events'; import { describe, expect, it, jest } from '@jest/globals'; import { ObjectId } from 'bson'; -import { MongoChangeStreamCoordinator } from '../MongoChangeStreamCoordinator.js'; +import { ChangeStreamCoordinator } from '../ChangeStreamCoordinator.js'; import { RealtimeSubscriptionTracker } from '../subscriptions.js'; import { authorizedDocumentRoom, roomsForPublicChange } from '../rooms.js'; @@ -70,17 +70,16 @@ function createCoordinator(overrides?: { can: async () => ({ allow: overrides?.allow !== false }), }, }; - const coordinator = new MongoChangeStreamCoordinator({ + const coordinator = new ChangeStreamCoordinator({ grpcSdk: grpcSdk as never, watch, - hello: async () => ({ setName: 'rs0' }), + checkTopology: async () => ({ supported: true }), getOptedInSchemas: () => overrides?.schemas ?? [ { name: 'Order', collectionName: 'orders', authorizationEnabled: false }, ], subscriptions, enabled: () => true, - engine: () => 'MongoDB', }); return { coordinator, @@ -96,7 +95,7 @@ function createCoordinator(overrides?: { }; } -describe('MongoChangeStreamCoordinator', () => { +describe('ChangeStreamCoordinator', () => { it('emits one normalized event to public rooms and ignores other collections', async () => { const { coordinator, stream, routerPush, adminPush, publish } = createCoordinator(); await coordinator.reconcile(); @@ -190,4 +189,28 @@ describe('MongoChangeStreamCoordinator', () => { expect(grpcSdk.state.clearKey).toHaveBeenCalled(); await coordinator.shutdown(); }); + + it('fans out SQL-shaped log events without document fields', async () => { + const { coordinator, stream, publish } = createCoordinator(); + await coordinator.reconcile(); + stream.emit('change', { + operationType: 'update', + ns: { coll: 'orders' }, + documentKey: { _id: 'order-1' }, + _id: '1842', + wallTime: new Date('2026-03-01T00:00:00.000Z'), + fullDocument: { secret: 'nope' }, + }); + await new Promise(resolve => setImmediate(resolve)); + expect(publish).toHaveBeenCalledTimes(1); + const payload = JSON.parse(publish.mock.calls[0][1] as string); + expect(payload).toMatchObject({ + operation: 'update', + schema: 'Order', + documentId: 'order-1', + }); + expect(payload).not.toHaveProperty('fullDocument'); + expect(JSON.stringify(payload)).not.toContain('nope'); + await coordinator.shutdown(); + }); }); diff --git a/modules/database/src/realtime/__tests__/sql-builders.test.ts b/modules/database/src/realtime/__tests__/sql-builders.test.ts new file mode 100644 index 000000000..08a9dd47d --- /dev/null +++ b/modules/database/src/realtime/__tests__/sql-builders.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from '@jest/globals'; +import { EJSON } from 'bson'; +import { parseSqlResumeId, sqlCursorFromResumeAfter } from '../sql/resume.js'; +import { createCaptureFunctionSql, createChangeLogTableSql } from '../sql/ddl.js'; +import { desiredTriggers } from '../sql/triggerSql.js'; +import { CHANGE_LOG_TABLE, CHANGE_LOG_FUNCTION } from '../sql/constants.js'; +import { + fitIdentifier, + quoteIdent, + rowTriggerName, + triggerBaseName, +} from '../sql/identifiers.js'; +import { toRawChangeEvent } from '../sql/mapEvent.js'; +import { normalizeChangeEvent } from '../normalize.js'; + +describe('SQL realtime SQL builders', () => { + it('quotes identifiers per dialect', () => { + expect(quoteIdent('postgres', 'orders')).toBe('"orders"'); + expect(quoteIdent('mysql', 'orders')).toBe('`orders`'); + expect(quoteIdent('sqlite', 'weird"name')).toBe('"weird""name"'); + }); + + it('fits trigger names into dialect identifier limits', () => { + const long = 'c'.repeat(80); + expect(fitIdentifier(long, 64).length).toBeLessThanOrEqual(64); + expect(triggerBaseName(long, 'postgres').length).toBeLessThanOrEqual(63); + expect(rowTriggerName(long, 'i', 'mysql').length).toBeLessThanOrEqual(64); + }); + + it('builds a postgres capture function and per-table trigger', () => { + const fn = createCaptureFunctionSql(); + expect(fn).toContain(CHANGE_LOG_FUNCTION); + expect(fn).toContain(CHANGE_LOG_TABLE); + expect(fn).toContain('pg_notify'); + const triggers = desiredTriggers('postgres', 'orders'); + expect(triggers).toHaveLength(1); + expect(triggers[0].sql).toMatch(/AFTER INSERT OR UPDATE OR DELETE/); + expect(triggers[0].sql).toContain('EXECUTE PROCEDURE'); + expect(triggers[0].dropSql).toMatch(/DROP TRIGGER IF EXISTS/); + }); + + it('builds three row triggers for mysql and sqlite', () => { + expect(desiredTriggers('mysql', 'orders')).toHaveLength(3); + expect(desiredTriggers('mariadb', 'cnd_User')).toHaveLength(3); + const sqlite = desiredTriggers('sqlite', 'orders'); + expect(sqlite.map(t => t.triggerName)).toEqual([ + rowTriggerName('orders', 'i', 'sqlite'), + rowTriggerName('orders', 'u', 'sqlite'), + rowTriggerName('orders', 'd', 'sqlite'), + ]); + expect(sqlite[0].sql).toContain(CHANGE_LOG_TABLE); + expect(sqlite[0].sql).not.toContain('secret'); + }); + + it('builds dialect-specific change-log tables', () => { + expect(createChangeLogTableSql('postgres')).toMatch(/BIGSERIAL/); + expect(createChangeLogTableSql('mysql')).toMatch(/AUTO_INCREMENT/); + expect(createChangeLogTableSql('sqlite')).toMatch(/AUTOINCREMENT/); + }); +}); + +describe('SQL resume tokens', () => { + it('accepts decimal ids and ignores leftover Mongo tokens', () => { + expect(parseSqlResumeId('1842')).toBe('1842'); + expect(parseSqlResumeId(EJSON.stringify('1842'))).toBe('1842'); + expect(parseSqlResumeId(EJSON.stringify(1842))).toBe('1842'); + expect(parseSqlResumeId(EJSON.stringify({ _data: 'mongo' }))).toBeUndefined(); + expect(sqlCursorFromResumeAfter(1842)).toBe('1842'); + expect(sqlCursorFromResumeAfter({ _data: 'mongo' })).toBeUndefined(); + }); +}); + +describe('SQL change-log event mapping', () => { + it('maps log rows to metadata-only change events', () => { + const raw = toRawChangeEvent({ + id: '1842', + collection_name: 'orders', + document_id: 'order-1', + operation: 'insert', + occurred_at: '2026-03-01T00:00:00.000Z', + }); + const event = normalizeChangeEvent(raw, 'Order'); + expect(event).toMatchObject({ + operation: 'insert', + schema: 'Order', + documentId: 'order-1', + }); + expect(JSON.stringify(raw)).not.toContain('secret'); + }); +}); diff --git a/modules/database/src/realtime/__tests__/sql-change-stream.integration.test.ts b/modules/database/src/realtime/__tests__/sql-change-stream.integration.test.ts new file mode 100644 index 000000000..706bb15d8 --- /dev/null +++ b/modules/database/src/realtime/__tests__/sql-change-stream.integration.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from '@jest/globals'; +import sqlite3 from 'sqlite3'; +import { normalizeChangeEvent } from '../normalize.js'; +import { CHANGE_LOG_TABLE } from '../sql/constants.js'; +import { createChangeLogTableSql } from '../sql/ddl.js'; +import { toRawChangeEvent } from '../sql/mapEvent.js'; +import { desiredTriggers } from '../sql/triggerSql.js'; + +function run(db: sqlite3.Database, sql: string): Promise { + return new Promise((resolve, reject) => { + db.exec(sql, err => { + if (err) reject(err); + else resolve(); + }); + }); +} + +function all(db: sqlite3.Database, sql: string): Promise { + return new Promise((resolve, reject) => { + db.all(sql, (err, rows) => { + if (err) reject(err); + else resolve(rows as T[]); + }); + }); +} + +describe('SQLite change-log contract', () => { + it('captures insert/update/delete including raw SQL without document fields', async () => { + const db = new sqlite3.Database(':memory:'); + try { + await run(db, `CREATE TABLE "orders" (_id TEXT PRIMARY KEY, secret TEXT)`); + await run(db, createChangeLogTableSql('sqlite')); + for (const trigger of desiredTriggers('sqlite', 'orders')) { + await run(db, trigger.sql); + } + await run( + db, + `INSERT INTO "orders" (_id, secret) VALUES ('order-1', 'do-not-leak')`, + ); + await run(db, `UPDATE "orders" SET secret = 'still-secret' WHERE _id = 'order-1'`); + await run(db, `DELETE FROM "orders" WHERE _id = 'order-1'`); + const rows = await all<{ + id: number; + collection_name: string; + document_id: string; + operation: string; + occurred_at: string; + }>( + db, + `SELECT id, collection_name, document_id, operation, occurred_at FROM "${CHANGE_LOG_TABLE}" ORDER BY id ASC`, + ); + expect(rows.map(row => row.operation)).toEqual(['insert', 'update', 'delete']); + const events = rows.map(row => + normalizeChangeEvent( + toRawChangeEvent({ + id: String(row.id), + collection_name: row.collection_name, + document_id: row.document_id, + operation: row.operation, + occurred_at: row.occurred_at, + }), + 'Order', + ), + ); + expect(events.map(event => event?.operation)).toEqual([ + 'insert', + 'update', + 'delete', + ]); + for (const event of events) { + expect(event).toMatchObject({ schema: 'Order', documentId: 'order-1' }); + expect(event).not.toHaveProperty('fullDocument'); + expect(JSON.stringify(event)).not.toContain('do-not-leak'); + expect(JSON.stringify(event)).not.toContain('still-secret'); + } + } finally { + await new Promise(resolve => { + db.close(() => resolve()); + }); + } + }); +}); diff --git a/modules/database/src/realtime/__tests__/status.test.ts b/modules/database/src/realtime/__tests__/status.test.ts index a94790cfb..542793178 100644 --- a/modules/database/src/realtime/__tests__/status.test.ts +++ b/modules/database/src/realtime/__tests__/status.test.ts @@ -5,13 +5,31 @@ describe('buildRealtimeStatus', () => { it('reports unsupported, disabled, idle, live, and degraded states', () => { expect( buildRealtimeStatus({ - engine: 'PostgreSQL', + engine: 'oracle', enabled: true, topologySupported: true, activeSchemaCount: 1, streamState: 'live', }).status, ).toBe('unsupported'); + expect( + buildRealtimeStatus({ + engine: 'PostgreSQL', + enabled: true, + topologySupported: true, + activeSchemaCount: 1, + streamState: 'live', + }).status, + ).toBe('live'); + expect( + buildRealtimeStatus({ + engine: 'mysql', + enabled: false, + topologySupported: true, + activeSchemaCount: 1, + streamState: 'live', + }).status, + ).toBe('disabled'); expect( buildRealtimeStatus({ engine: 'MongoDB', diff --git a/modules/database/src/realtime/index.ts b/modules/database/src/realtime/index.ts index 655c04bba..e3de1442d 100644 --- a/modules/database/src/realtime/index.ts +++ b/modules/database/src/realtime/index.ts @@ -1,4 +1,8 @@ export { RealtimeService } from './RealtimeService.js'; +export { + ChangeStreamCoordinator, + MongoChangeStreamCoordinator, +} from './ChangeStreamCoordinator.js'; export { buildRealtimeStatus } from './status.js'; export { normalizeChangeEvent } from './normalize.js'; export { diff --git a/modules/database/src/realtime/sql/SqlChangeStream.ts b/modules/database/src/realtime/sql/SqlChangeStream.ts new file mode 100644 index 000000000..9ad6eccdb --- /dev/null +++ b/modules/database/src/realtime/sql/SqlChangeStream.ts @@ -0,0 +1,142 @@ +import { EventEmitter } from 'node:events'; +import pg from 'pg'; +import type { Sequelize } from 'sequelize'; +import type { ChangeStreamLike } from '../types.js'; +import { + CHANGE_LOG_BATCH_SIZE, + NOTIFY_CHANNEL, + POSTGRES_FALLBACK_POLL_MS, + SQL_POLL_INTERVAL_MS, + assertSqlDialect, + type SqlDialect, +} from './constants.js'; +import { fetchChangeLogBatch, maxChangeLogId } from './changelog.js'; +import { toRawChangeEvent } from './mapEvent.js'; +import { sqlCursorFromResumeAfter } from './resume.js'; + +export type SqlChangeStreamOptions = { + sequelize: Sequelize; + connectionUri: string; + resumeAfter?: unknown; +}; + +export class SqlChangeStream implements ChangeStreamLike { + private readonly emitter = new EventEmitter(); + private readonly sequelize: Sequelize; + private readonly connectionUri: string; + private readonly dialect: SqlDialect; + private cursor: string | undefined; + private listenClient: pg.Client | null = null; + private pollTimer: NodeJS.Timeout | null = null; + private closed = false; + private draining = false; + private started = false; + + constructor(options: SqlChangeStreamOptions) { + this.sequelize = options.sequelize; + this.connectionUri = options.connectionUri; + this.dialect = assertSqlDialect(options.sequelize.getDialect()); + this.cursor = sqlCursorFromResumeAfter(options.resumeAfter); + queueMicrotask(() => { + if (!this.closed) { + void this.start(); + } + }); + } + + on( + event: 'change' | 'error' | 'close' | 'end', + listener: (...args: unknown[]) => void, + ): void { + this.emitter.on(event, listener); + } + + async close(): Promise { + if (this.closed) return; + this.closed = true; + if (this.pollTimer) { + clearInterval(this.pollTimer); + this.pollTimer = null; + } + const client = this.listenClient; + this.listenClient = null; + if (client) { + try { + await client.query(`UNLISTEN ${NOTIFY_CHANNEL}`); + } catch { + // ignore + } + try { + await client.end(); + } catch { + // ignore + } + } + this.emitter.emit('close'); + } + + private async start(): Promise { + if (this.closed || this.started) return; + this.started = true; + try { + if (this.cursor === undefined) { + this.cursor = await maxChangeLogId(this.sequelize); + } + if (this.dialect === 'postgres') { + await this.startPostgresListen(); + this.pollTimer = setInterval(() => { + void this.drain(); + }, POSTGRES_FALLBACK_POLL_MS); + } else { + this.pollTimer = setInterval(() => { + void this.drain(); + }, SQL_POLL_INTERVAL_MS); + } + await this.drain(); + } catch (err) { + this.emitError(err); + } + } + + private async startPostgresListen(): Promise { + const client = new pg.Client({ connectionString: this.connectionUri }); + this.listenClient = client; + client.on('notification', () => { + void this.drain(); + }); + client.on('error', (err: Error) => { + this.emitError(err); + }); + await client.connect(); + await client.query(`LISTEN ${NOTIFY_CHANNEL}`); + } + + private async drain(): Promise { + if (this.draining || this.closed) return; + this.draining = true; + try { + while (!this.closed) { + const rows = await fetchChangeLogBatch( + this.sequelize, + this.cursor ?? '0', + CHANGE_LOG_BATCH_SIZE, + ); + if (rows.length === 0) break; + for (const row of rows) { + if (this.closed) return; + this.emitter.emit('change', toRawChangeEvent(row)); + this.cursor = row.id; + } + } + } catch (err) { + this.emitError(err); + } finally { + this.draining = false; + } + } + + private emitError(err: unknown): void { + if (this.closed) return; + this.emitter.emit('error', err); + } +} diff --git a/modules/database/src/realtime/sql/SqlRealtimeSupport.ts b/modules/database/src/realtime/sql/SqlRealtimeSupport.ts new file mode 100644 index 000000000..2163a1242 --- /dev/null +++ b/modules/database/src/realtime/sql/SqlRealtimeSupport.ts @@ -0,0 +1,30 @@ +import type { SequelizeAdapter } from '../../adapters/sequelize-adapter/index.js'; +import type { OptedInSchema } from '../types.js'; +import type { ChangeStreamLike } from '../types.js'; +import { ensureChangeLog, trimChangeLog } from './changelog.js'; +import { parseSqlResumeId } from './resume.js'; +import { SqlChangeStream } from './SqlChangeStream.js'; +import { syncTriggers } from './triggers.js'; + +export class SqlRealtimeSupport { + constructor(private readonly adapter: SequelizeAdapter) {} + + async prepare(schemas: OptedInSchema[]): Promise { + await ensureChangeLog(this.adapter.sequelize); + await syncTriggers(this.adapter.sequelize, schemas); + } + + openWatch(resumeAfter?: unknown): ChangeStreamLike { + return new SqlChangeStream({ + sequelize: this.adapter.sequelize, + connectionUri: this.adapter.connectionUri, + resumeAfter, + }); + } + + async trimThrough(resumeToken: string): Promise { + const id = parseSqlResumeId(resumeToken); + if (!id) return; + await trimChangeLog(this.adapter.sequelize, id); + } +} diff --git a/modules/database/src/realtime/sql/changelog.ts b/modules/database/src/realtime/sql/changelog.ts new file mode 100644 index 000000000..88382ce26 --- /dev/null +++ b/modules/database/src/realtime/sql/changelog.ts @@ -0,0 +1,118 @@ +import { QueryTypes, Sequelize } from 'sequelize'; +import { + CHANGE_LOG_BATCH_SIZE, + CHANGE_LOG_TABLE, + type SqlDialect, + assertSqlDialect, +} from './constants.js'; +import { quoteIdent } from './identifiers.js'; +import { createCaptureFunctionSql, createChangeLogTableSql } from './ddl.js'; +import type { ChangeLogRow } from './mapEvent.js'; + +export { createChangeLogTableSql, createCaptureFunctionSql } from './ddl.js'; +export type { ChangeLogRow } from './mapEvent.js'; + +export async function ensureChangeLog(sequelize: Sequelize): Promise { + const dialect = assertSqlDialect(sequelize.getDialect()); + await sequelize.query(createChangeLogTableSql(dialect)); + if (dialect === 'postgres') { + await sequelize.query(createCaptureFunctionSql()); + } +} + +export async function fetchChangeLogBatch( + sequelize: Sequelize, + resumeId: string, + limit: number = CHANGE_LOG_BATCH_SIZE, +): Promise { + const dialect = assertSqlDialect(sequelize.getDialect()); + const table = quoteIdent(dialect, CHANGE_LOG_TABLE); + const rows = await sequelize.query( + `SELECT id, collection_name, document_id, operation, occurred_at + FROM ${table} + WHERE id > :resumeId + ORDER BY id ASC + LIMIT :limit`, + { + type: QueryTypes.SELECT, + replacements: { resumeId, limit }, + }, + ); + return (rows as Record[]).map(row => ({ + id: String(row.id), + collection_name: String(row.collection_name), + document_id: String(row.document_id), + operation: String(row.operation), + occurred_at: + row.occurred_at instanceof Date + ? row.occurred_at + : typeof row.occurred_at === 'string' || typeof row.occurred_at === 'number' + ? row.occurred_at + : new Date().toISOString(), + })); +} + +export async function maxChangeLogId(sequelize: Sequelize): Promise { + const dialect = assertSqlDialect(sequelize.getDialect()); + const table = quoteIdent(dialect, CHANGE_LOG_TABLE); + const rows = await sequelize.query(`SELECT MAX(id) AS max_id FROM ${table}`, { + type: QueryTypes.SELECT, + }); + const maxId = (rows[0] as { max_id?: unknown } | undefined)?.max_id; + if (maxId === undefined || maxId === null) { + return '0'; + } + return String(maxId); +} + +export async function trimChangeLog( + sequelize: Sequelize, + throughId: string, +): Promise { + if (!/^\d+$/.test(throughId)) { + return; + } + const dialect = assertSqlDialect(sequelize.getDialect()); + const table = quoteIdent(dialect, CHANGE_LOG_TABLE); + for (let i = 0; i < 50; i++) { + const sql = trimSql(dialect, table); + const [, metadata] = await sequelize.query(sql, { + replacements: { id: throughId, limit: CHANGE_LOG_BATCH_SIZE }, + }); + const affected = affectedRows(metadata); + if (affected === 0) { + return; + } + } +} + +function trimSql(dialect: SqlDialect, table: string): string { + switch (dialect) { + case 'mysql': + case 'mariadb': + return `DELETE FROM ${table} WHERE id <= :id ORDER BY id ASC LIMIT :limit`; + case 'postgres': + case 'sqlite': + return `DELETE FROM ${table} WHERE id IN ( + SELECT id FROM ${table} WHERE id <= :id ORDER BY id ASC LIMIT :limit + )`; + default: { + const _exhaustive: never = dialect; + return _exhaustive; + } + } +} + +function affectedRows(metadata: unknown): number { + if (!metadata || typeof metadata !== 'object') { + return 0; + } + const record = metadata as { rowCount?: unknown; affectedRows?: unknown }; + if (typeof record.rowCount === 'number') { + return record.rowCount; + } + if (typeof record.affectedRows === 'number') { + return record.affectedRows; + } + return 0; +} diff --git a/modules/database/src/realtime/sql/constants.ts b/modules/database/src/realtime/sql/constants.ts new file mode 100644 index 000000000..8073df1b1 --- /dev/null +++ b/modules/database/src/realtime/sql/constants.ts @@ -0,0 +1,23 @@ +export const CHANGE_LOG_TABLE = '_cnd_DatabaseChange'; +export const CHANGE_LOG_FUNCTION = 'conduit_realtime_capture'; +export const NOTIFY_CHANNEL = 'conduit_realtime'; +export const TRIGGER_NAME_PREFIX = 'cnd_rt_'; +export const PK_COLUMN = '_id'; + +export const SQL_POLL_INTERVAL_MS = 250; +export const POSTGRES_FALLBACK_POLL_MS = 2_000; +export const CHANGE_LOG_BATCH_SIZE = 200; + +export const SQL_DIALECTS = ['postgres', 'mysql', 'mariadb', 'sqlite'] as const; +export type SqlDialect = (typeof SQL_DIALECTS)[number]; + +export function isSqlDialect(dialect: string): dialect is SqlDialect { + return (SQL_DIALECTS as readonly string[]).includes(dialect); +} + +export function assertSqlDialect(dialect: string): SqlDialect { + if (isSqlDialect(dialect)) { + return dialect; + } + throw new Error(`Unsupported SQL dialect for live updates: ${dialect}`); +} diff --git a/modules/database/src/realtime/sql/ddl.ts b/modules/database/src/realtime/sql/ddl.ts new file mode 100644 index 000000000..9dc03e239 --- /dev/null +++ b/modules/database/src/realtime/sql/ddl.ts @@ -0,0 +1,79 @@ +import { + CHANGE_LOG_FUNCTION, + CHANGE_LOG_TABLE, + NOTIFY_CHANNEL, + PK_COLUMN, + type SqlDialect, +} from './constants.js'; +import { quoteIdent } from './identifiers.js'; + +export function createChangeLogTableSql(dialect: SqlDialect): string { + const table = quoteIdent(dialect, CHANGE_LOG_TABLE); + switch (dialect) { + case 'postgres': + return `CREATE TABLE IF NOT EXISTS ${table} ( + id BIGSERIAL PRIMARY KEY, + collection_name TEXT NOT NULL, + document_id TEXT NOT NULL, + operation TEXT NOT NULL, + occurred_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + )`; + case 'mysql': + case 'mariadb': + return `CREATE TABLE IF NOT EXISTS ${table} ( + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + collection_name VARCHAR(255) NOT NULL, + document_id VARCHAR(255) NOT NULL, + operation VARCHAR(16) NOT NULL, + occurred_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + )`; + case 'sqlite': + return `CREATE TABLE IF NOT EXISTS ${table} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + collection_name TEXT NOT NULL, + document_id TEXT NOT NULL, + operation TEXT NOT NULL, + occurred_at TEXT NOT NULL DEFAULT (datetime('now')) + )`; + default: { + const _exhaustive: never = dialect; + return _exhaustive; + } + } +} + +export function createCaptureFunctionSql(): string { + const table = quoteIdent('postgres', CHANGE_LOG_TABLE); + const pk = quoteIdent('postgres', PK_COLUMN); + const channel = NOTIFY_CHANNEL.replace(/'/g, "''"); + return `CREATE OR REPLACE FUNCTION ${CHANGE_LOG_FUNCTION}() RETURNS trigger AS $$ +DECLARE + doc_id text; + op text; +BEGIN + IF TG_OP = 'DELETE' THEN + doc_id := OLD.${pk}::text; + op := 'delete'; + ELSIF TG_OP = 'INSERT' THEN + doc_id := NEW.${pk}::text; + op := 'insert'; + ELSE + doc_id := NEW.${pk}::text; + op := 'update'; + END IF; + IF doc_id IS NULL THEN + IF TG_OP = 'DELETE' THEN + RETURN OLD; + END IF; + RETURN NEW; + END IF; + INSERT INTO ${table} (collection_name, document_id, operation, occurred_at) + VALUES (TG_TABLE_NAME, doc_id, op, NOW()); + PERFORM pg_notify('${channel}', ''); + IF TG_OP = 'DELETE' THEN + RETURN OLD; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql`; +} diff --git a/modules/database/src/realtime/sql/identifiers.ts b/modules/database/src/realtime/sql/identifiers.ts new file mode 100644 index 000000000..49c035348 --- /dev/null +++ b/modules/database/src/realtime/sql/identifiers.ts @@ -0,0 +1,63 @@ +import { createHash } from 'node:crypto'; +import type { SqlDialect } from './constants.js'; +import { TRIGGER_NAME_PREFIX } from './constants.js'; + +const MYSQL_IDENT_LIMIT = 64; +const POSTGRES_IDENT_LIMIT = 63; +const SQLITE_IDENT_LIMIT = 128; + +export type QuoteIdent = (name: string) => string; + +export function quoteIdent(dialect: SqlDialect, name: string): string { + if (dialect === 'mysql' || dialect === 'mariadb') { + return `\`${name.replace(/`/g, '``')}\``; + } + return `"${name.replace(/"/g, '""')}"`; +} + +export function sqlStringLiteral(value: string): string { + return `'${value.replace(/'/g, "''")}'`; +} + +export function identifierLimit(dialect: SqlDialect): number { + switch (dialect) { + case 'postgres': + return POSTGRES_IDENT_LIMIT; + case 'mysql': + case 'mariadb': + return MYSQL_IDENT_LIMIT; + case 'sqlite': + return SQLITE_IDENT_LIMIT; + default: { + const _exhaustive: never = dialect; + return _exhaustive; + } + } +} + +export function triggerBaseName(collectionName: string, dialect: SqlDialect): string { + return fitIdentifier( + `${TRIGGER_NAME_PREFIX}${collectionName}`, + identifierLimit(dialect), + ); +} + +export function rowTriggerName( + collectionName: string, + operation: 'i' | 'u' | 'd', + dialect: SqlDialect, +): string { + return fitIdentifier( + `${TRIGGER_NAME_PREFIX}${operation}_${collectionName}`, + identifierLimit(dialect), + ); +} + +export function fitIdentifier(raw: string, maxLength: number): string { + if (raw.length <= maxLength) { + return raw; + } + const hash = createHash('sha1').update(raw).digest('hex').slice(0, 8); + const keep = Math.max(0, maxLength - hash.length - 1); + return `${raw.slice(0, keep)}_${hash}`; +} diff --git a/modules/database/src/realtime/sql/index.ts b/modules/database/src/realtime/sql/index.ts new file mode 100644 index 000000000..d0f7fd4e5 --- /dev/null +++ b/modules/database/src/realtime/sql/index.ts @@ -0,0 +1,13 @@ +export { CHANGE_LOG_TABLE, SQL_DIALECTS } from './constants.js'; +export { SqlChangeStream } from './SqlChangeStream.js'; +export { SqlRealtimeSupport } from './SqlRealtimeSupport.js'; +export { parseSqlResumeId, sqlCursorFromResumeAfter } from './resume.js'; +export { desiredTriggers, syncTriggers } from './triggers.js'; +export { createChangeLogTableSql, createCaptureFunctionSql } from './ddl.js'; +export { + quoteIdent, + triggerBaseName, + rowTriggerName, + fitIdentifier, +} from './identifiers.js'; +export { toRawChangeEvent } from './mapEvent.js'; diff --git a/modules/database/src/realtime/sql/mapEvent.ts b/modules/database/src/realtime/sql/mapEvent.ts new file mode 100644 index 000000000..0a569beed --- /dev/null +++ b/modules/database/src/realtime/sql/mapEvent.ts @@ -0,0 +1,21 @@ +import type { RawChangeEvent } from '../normalize.js'; + +export type ChangeLogRow = { + id: string; + collection_name: string; + document_id: string; + operation: string; + occurred_at: Date | string | number; +}; + +export function toRawChangeEvent(row: ChangeLogRow): RawChangeEvent { + const occurredAt = + row.occurred_at instanceof Date ? row.occurred_at : new Date(row.occurred_at); + return { + operationType: row.operation, + ns: { coll: row.collection_name }, + documentKey: { _id: row.document_id }, + wallTime: Number.isNaN(occurredAt.getTime()) ? new Date() : occurredAt, + _id: row.id, + }; +} diff --git a/modules/database/src/realtime/sql/resume.ts b/modules/database/src/realtime/sql/resume.ts new file mode 100644 index 000000000..eaa4f3efa --- /dev/null +++ b/modules/database/src/realtime/sql/resume.ts @@ -0,0 +1,41 @@ +import { EJSON } from 'bson'; +import { parseResumeToken } from '../normalize.js'; + +export function parseSqlResumeId(token: string | null | undefined): string | undefined { + if (!token) return undefined; + if (/^\d+$/.test(token)) { + return token; + } + try { + const parsed = parseResumeToken(token) ?? EJSON.parse(token); + if (typeof parsed === 'number' && Number.isInteger(parsed) && parsed >= 0) { + return String(parsed); + } + if (typeof parsed === 'bigint' && parsed >= 0n) { + return parsed.toString(); + } + if (typeof parsed === 'string' && /^\d+$/.test(parsed)) { + return parsed; + } + } catch { + return undefined; + } + return undefined; +} + +export function sqlCursorFromResumeAfter(resumeAfter: unknown): string | undefined { + if ( + typeof resumeAfter === 'number' && + Number.isInteger(resumeAfter) && + resumeAfter >= 0 + ) { + return String(resumeAfter); + } + if (typeof resumeAfter === 'bigint' && resumeAfter >= 0n) { + return resumeAfter.toString(); + } + if (typeof resumeAfter === 'string' && /^\d+$/.test(resumeAfter)) { + return resumeAfter; + } + return undefined; +} diff --git a/modules/database/src/realtime/sql/triggerSql.ts b/modules/database/src/realtime/sql/triggerSql.ts new file mode 100644 index 000000000..809928007 --- /dev/null +++ b/modules/database/src/realtime/sql/triggerSql.ts @@ -0,0 +1,145 @@ +import { + CHANGE_LOG_FUNCTION, + CHANGE_LOG_TABLE, + PK_COLUMN, + type SqlDialect, +} from './constants.js'; +import { + quoteIdent, + rowTriggerName, + sqlStringLiteral, + triggerBaseName, +} from './identifiers.js'; + +export type DesiredTrigger = { + triggerName: string; + collectionName: string; + sql: string; + dropSql: string; +}; + +export function desiredTriggers( + dialect: SqlDialect, + collectionName: string, +): DesiredTrigger[] { + const table = quoteIdent(dialect, collectionName); + const pk = quoteIdent(dialect, PK_COLUMN); + const logTable = quoteIdent(dialect, CHANGE_LOG_TABLE); + const collectionLiteral = sqlStringLiteral(collectionName); + switch (dialect) { + case 'postgres': { + const triggerName = triggerBaseName(collectionName, dialect); + const quotedTrigger = quoteIdent(dialect, triggerName); + return [ + { + triggerName, + collectionName, + sql: `CREATE TRIGGER ${quotedTrigger} +AFTER INSERT OR UPDATE OR DELETE ON ${table} +FOR EACH ROW EXECUTE PROCEDURE ${CHANGE_LOG_FUNCTION}()`, + dropSql: `DROP TRIGGER IF EXISTS ${quotedTrigger} ON ${table}`, + }, + ]; + } + case 'mysql': + case 'mariadb': + return [ + mysqlRowTrigger(dialect, collectionName, 'i', 'INSERT', 'insert', 'NEW'), + mysqlRowTrigger(dialect, collectionName, 'u', 'UPDATE', 'update', 'NEW'), + mysqlRowTrigger(dialect, collectionName, 'd', 'DELETE', 'delete', 'OLD'), + ]; + case 'sqlite': + return [ + sqliteRowTrigger( + collectionName, + 'i', + 'INSERT', + 'insert', + 'NEW', + table, + pk, + logTable, + collectionLiteral, + ), + sqliteRowTrigger( + collectionName, + 'u', + 'UPDATE', + 'update', + 'NEW', + table, + pk, + logTable, + collectionLiteral, + ), + sqliteRowTrigger( + collectionName, + 'd', + 'DELETE', + 'delete', + 'OLD', + table, + pk, + logTable, + collectionLiteral, + ), + ]; + default: { + const _exhaustive: never = dialect; + return _exhaustive; + } + } +} + +function mysqlRowTrigger( + dialect: SqlDialect, + collectionName: string, + opKey: 'i' | 'u' | 'd', + timing: 'INSERT' | 'UPDATE' | 'DELETE', + operation: 'insert' | 'update' | 'delete', + row: 'NEW' | 'OLD', +): DesiredTrigger { + const triggerName = rowTriggerName(collectionName, opKey, dialect); + const quotedTrigger = quoteIdent(dialect, triggerName); + const table = quoteIdent(dialect, collectionName); + const pk = quoteIdent(dialect, PK_COLUMN); + const logTable = quoteIdent(dialect, CHANGE_LOG_TABLE); + const collectionLiteral = sqlStringLiteral(collectionName); + const opLiteral = sqlStringLiteral(operation); + return { + triggerName, + collectionName, + sql: `CREATE TRIGGER ${quotedTrigger} AFTER ${timing} ON ${table} +FOR EACH ROW BEGIN + INSERT INTO ${logTable} (collection_name, document_id, operation, occurred_at) + VALUES (${collectionLiteral}, CAST(${row}.${pk} AS CHAR), ${opLiteral}, CURRENT_TIMESTAMP); +END`, + dropSql: `DROP TRIGGER IF EXISTS ${quotedTrigger}`, + }; +} + +function sqliteRowTrigger( + collectionName: string, + opKey: 'i' | 'u' | 'd', + timing: 'INSERT' | 'UPDATE' | 'DELETE', + operation: 'insert' | 'update' | 'delete', + row: 'NEW' | 'OLD', + table: string, + pk: string, + logTable: string, + collectionLiteral: string, +): DesiredTrigger { + const triggerName = rowTriggerName(collectionName, opKey, 'sqlite'); + const quotedTrigger = quoteIdent('sqlite', triggerName); + const opLiteral = sqlStringLiteral(operation); + return { + triggerName, + collectionName, + sql: `CREATE TRIGGER ${quotedTrigger} AFTER ${timing} ON ${table} +BEGIN + INSERT INTO ${logTable} (collection_name, document_id, operation, occurred_at) + VALUES (${collectionLiteral}, ${row}.${pk}, ${opLiteral}, datetime('now')); +END`, + dropSql: `DROP TRIGGER IF EXISTS ${quotedTrigger}`, + }; +} diff --git a/modules/database/src/realtime/sql/triggers.ts b/modules/database/src/realtime/sql/triggers.ts new file mode 100644 index 000000000..1bd7f096c --- /dev/null +++ b/modules/database/src/realtime/sql/triggers.ts @@ -0,0 +1,106 @@ +import { QueryTypes, Sequelize } from 'sequelize'; +import type { OptedInSchema } from '../types.js'; +import { + CHANGE_LOG_TABLE, + TRIGGER_NAME_PREFIX, + type SqlDialect, + assertSqlDialect, +} from './constants.js'; +import { quoteIdent } from './identifiers.js'; +import { desiredTriggers, type DesiredTrigger } from './triggerSql.js'; + +export type { DesiredTrigger }; +export { desiredTriggers }; + +export type ExistingTrigger = { + triggerName: string; + tableName: string; +}; + +export async function listExistingTriggers( + sequelize: Sequelize, +): Promise { + const dialect = assertSqlDialect(sequelize.getDialect()); + const prefix = `${TRIGGER_NAME_PREFIX}%`; + switch (dialect) { + case 'postgres': { + const rows = await sequelize.query( + `SELECT trigger_name AS trigger_name, event_object_table AS table_name + FROM information_schema.triggers + WHERE trigger_name LIKE :prefix`, + { type: QueryTypes.SELECT, replacements: { prefix } }, + ); + return uniqueTriggers(rows as { trigger_name: string; table_name: string }[]); + } + case 'mysql': + case 'mariadb': { + const rows = await sequelize.query( + `SELECT trigger_name AS trigger_name, event_object_table AS table_name + FROM information_schema.triggers + WHERE trigger_schema = DATABASE() + AND trigger_name LIKE :prefix`, + { type: QueryTypes.SELECT, replacements: { prefix } }, + ); + return uniqueTriggers(rows as { trigger_name: string; table_name: string }[]); + } + case 'sqlite': { + const rows = await sequelize.query( + `SELECT name AS trigger_name, tbl_name AS table_name + FROM sqlite_master + WHERE type = 'trigger' AND name LIKE :prefix`, + { type: QueryTypes.SELECT, replacements: { prefix } }, + ); + return uniqueTriggers(rows as { trigger_name: string; table_name: string }[]); + } + default: { + const _exhaustive: never = dialect; + return _exhaustive; + } + } +} + +function uniqueTriggers( + rows: { trigger_name: string; table_name: string }[], +): ExistingTrigger[] { + const seen = new Set(); + const result: ExistingTrigger[] = []; + for (const row of rows) { + const triggerName = String(row.trigger_name); + if (seen.has(triggerName)) continue; + seen.add(triggerName); + result.push({ triggerName, tableName: String(row.table_name) }); + } + return result; +} + +export async function syncTriggers( + sequelize: Sequelize, + schemas: OptedInSchema[], +): Promise { + const dialect = assertSqlDialect(sequelize.getDialect()); + const desired = new Map(); + for (const schema of schemas) { + if (schema.collectionName === CHANGE_LOG_TABLE) continue; + for (const trigger of desiredTriggers(dialect, schema.collectionName)) { + desired.set(trigger.triggerName, trigger); + } + } + const existing = await listExistingTriggers(sequelize); + for (const current of existing) { + if (desired.has(current.triggerName)) continue; + const dropSql = dropExistingSql(dialect, current); + await sequelize.query(dropSql); + } + for (const trigger of desired.values()) { + await sequelize.query(trigger.dropSql); + await sequelize.query(trigger.sql); + } +} + +function dropExistingSql(dialect: SqlDialect, trigger: ExistingTrigger): string { + const quotedTrigger = quoteIdent(dialect, trigger.triggerName); + if (dialect === 'postgres') { + return `DROP TRIGGER IF EXISTS ${quotedTrigger} ON ${quoteIdent(dialect, trigger.tableName)}`; + } + return `DROP TRIGGER IF EXISTS ${quotedTrigger}`; +} diff --git a/modules/database/src/realtime/status.ts b/modules/database/src/realtime/status.ts index b1da21b03..436523498 100644 --- a/modules/database/src/realtime/status.ts +++ b/modules/database/src/realtime/status.ts @@ -1,5 +1,13 @@ import type { RealtimeStatus, RealtimeStatusCode } from './types.js'; +const SUPPORTED_REALTIME_ENGINES = new Set([ + 'MongoDB', + 'PostgreSQL', + 'mysql', + 'mariadb', + 'sqlite', +]); + export type RealtimeStatusInput = { engine: string; enabled: boolean; @@ -13,12 +21,12 @@ export type RealtimeStatusInput = { }; export function buildRealtimeStatus(input: RealtimeStatusInput): RealtimeStatus { - if (input.engine !== 'MongoDB') { + if (!SUPPORTED_REALTIME_ENGINES.has(input.engine)) { return { status: 'unsupported', engine: input.engine, activeSchemaCount: 0, - message: 'Live updates require MongoDB', + message: 'Live updates are not supported for this database engine', }; } if (!input.enabled) { @@ -35,7 +43,9 @@ export function buildRealtimeStatus(input: RealtimeStatusInput): RealtimeStatus activeSchemaCount: input.activeSchemaCount, message: input.topologyMessage ?? - 'A replica set or sharded MongoDB deployment is required for live updates', + (input.engine === 'MongoDB' + ? 'A replica set or sharded MongoDB deployment is required for live updates' + : 'Database topology does not support live updates'), }; } if (input.socketsEnabled === false) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6753ee73d..7c698b171 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -427,7 +427,7 @@ importers: version: 6.1.3 ts-jest: specifier: ^29.4.12 - version: 29.4.12(@babel/core@7.29.7)(@jest/transform@30.5.1)(@jest/types@30.5.1)(babel-jest@30.5.1(@babel/core@7.29.7))(jest-util@30.5.1)(jest@30.5.1(@types/node@24.13.3)(ts-node@10.9.2(@types/node@24.13.3)(typescript@6.0.3)))(typescript@6.0.3) + version: 29.4.12(@babel/core@7.29.7)(@jest/transform@30.5.1)(@jest/types@30.5.1)(babel-jest@30.5.1(@babel/core@7.29.7))(esbuild@0.28.1)(jest-util@30.5.1)(jest@30.5.1(@types/node@24.13.3)(ts-node@10.9.2(@types/node@24.13.3)(typescript@6.0.3)))(typescript@6.0.3) typescript: specifier: ~6.0.2 version: 6.0.3 @@ -914,6 +914,9 @@ importers: '@types/object-hash': specifier: ^3.0.6 version: 3.0.6 + '@types/pg': + specifier: ^8.23.1 + version: 8.23.1 copyfiles: specifier: ^2.4.1 version: 2.4.1 @@ -925,7 +928,7 @@ importers: version: 6.1.3 ts-jest: specifier: ^29.4.12 - version: 29.4.12(@babel/core@7.29.7)(@jest/transform@30.5.1)(@jest/types@30.5.1)(babel-jest@30.5.1(@babel/core@7.29.7))(jest-util@30.5.1)(jest@30.5.1(@types/node@24.13.3)(ts-node@10.9.2(@types/node@24.13.3)(typescript@6.0.3)))(typescript@6.0.3) + version: 29.4.12(@babel/core@7.29.7)(@jest/transform@30.5.1)(@jest/types@30.5.1)(babel-jest@30.5.1(@babel/core@7.29.7))(esbuild@0.28.1)(jest-util@30.5.1)(jest@30.5.1(@types/node@24.13.3)(ts-node@10.9.2(@types/node@24.13.3)(typescript@6.0.3)))(typescript@6.0.3) ts-proto: specifier: ^2.12.1 version: 2.12.1 @@ -1283,7 +1286,7 @@ importers: version: 6.1.3 ts-jest: specifier: ^29.4.12 - version: 29.4.12(@babel/core@7.29.7)(@jest/transform@30.5.1)(@jest/types@30.5.1)(babel-jest@30.5.1(@babel/core@7.29.7))(jest-util@30.5.1)(jest@30.5.1(@types/node@24.13.3)(ts-node@10.9.2(@types/node@24.13.3)(typescript@6.0.3)))(typescript@6.0.3) + version: 29.4.12(@babel/core@7.29.7)(@jest/transform@30.5.1)(@jest/types@30.5.1)(babel-jest@30.5.1(@babel/core@7.29.7))(esbuild@0.28.1)(jest-util@30.5.1)(jest@30.5.1(@types/node@24.13.3)(ts-node@10.9.2(@types/node@24.13.3)(typescript@6.0.3)))(typescript@6.0.3) ts-node: specifier: ^10.9.2 version: 10.9.2(@types/node@24.13.3)(typescript@6.0.3) @@ -3384,6 +3387,9 @@ packages: '@types/otp-generator@4.0.2': resolution: {integrity: sha512-9+qqWzuFb332hXPbLgjUyOXlbcaTQkmkmqQjTduvNuOmPV5fW+iLv70JsVEhdUy0DWi4kY34++HDCaWl6N0AYg==} + '@types/pg@8.23.1': + resolution: {integrity: sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==} + '@types/qs@6.15.1': resolution: {integrity: sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==} @@ -11049,6 +11055,12 @@ snapshots: '@types/otp-generator@4.0.2': {} + '@types/pg@8.23.1': + dependencies: + '@types/node': 24.13.3 + pg-protocol: 1.15.0 + pg-types: 2.2.0 + '@types/qs@6.15.1': {} '@types/range-parser@1.2.7': {} @@ -16182,7 +16194,7 @@ snapshots: ts-interface-checker@0.1.13: {} - ts-jest@29.4.12(@babel/core@7.29.7)(@jest/transform@30.5.1)(@jest/types@30.5.1)(babel-jest@30.5.1(@babel/core@7.29.7))(jest-util@30.5.1)(jest@30.5.1(@types/node@24.13.3)(ts-node@10.9.2(@types/node@24.13.3)(typescript@6.0.3)))(typescript@6.0.3): + ts-jest@29.4.12(@babel/core@7.29.7)(@jest/transform@30.5.1)(@jest/types@30.5.1)(babel-jest@30.5.1(@babel/core@7.29.7))(esbuild@0.28.1)(jest-util@30.5.1)(jest@30.5.1(@types/node@24.13.3)(ts-node@10.9.2(@types/node@24.13.3)(typescript@6.0.3)))(typescript@6.0.3): dependencies: bs-logger: 0.2.6 fast-json-stable-stringify: 2.1.0 @@ -16200,6 +16212,7 @@ snapshots: '@jest/transform': 30.5.1 '@jest/types': 30.5.1 babel-jest: 30.5.1(@babel/core@7.29.7) + esbuild: 0.28.1 jest-util: 30.5.1 ts-node@10.9.2(@types/node@24.13.3)(typescript@6.0.3): From 98bbbc0aa76b2fa27120c58eb0dc10ed60721151 Mon Sep 17 00:00:00 2001 From: John Chantzigoulas Date: Fri, 11 Sep 2026 13:59:47 +0300 Subject: [PATCH 6/6] refactor: drop unused realtime aliases and tautological tests --- .../src/realtime/ChangeStreamCoordinator.ts | 8 +-- .../database/src/realtime/RealtimeService.ts | 5 +- .../src/realtime/__tests__/authorize.test.ts | 49 +++++++------------ .../realtime/__tests__/coordinator.test.ts | 5 +- .../src/realtime/__tests__/normalize.test.ts | 5 +- .../realtime/__tests__/sql-builders.test.ts | 4 +- modules/database/src/realtime/authorize.ts | 13 +---- modules/database/src/realtime/index.ts | 5 +- modules/database/src/realtime/sockets.ts | 32 +++++------- .../database/src/realtime/sql/changelog.ts | 17 +++---- .../database/src/realtime/sql/identifiers.ts | 2 - modules/database/src/realtime/sql/resume.ts | 23 ++++----- .../src/event-relays/EventRelayManager.ts | 4 +- .../event-relays/EventRelaySockets.test.ts | 19 ++----- modules/router/src/event-relays/template.ts | 9 ++-- 15 files changed, 65 insertions(+), 135 deletions(-) diff --git a/modules/database/src/realtime/ChangeStreamCoordinator.ts b/modules/database/src/realtime/ChangeStreamCoordinator.ts index 00b6659ac..c6fa87b73 100644 --- a/modules/database/src/realtime/ChangeStreamCoordinator.ts +++ b/modules/database/src/realtime/ChangeStreamCoordinator.ts @@ -35,7 +35,6 @@ export type CoordinatorOptions = { getOptedInSchemas: () => OptedInSchema[]; subscriptions: RealtimeSubscriptionTracker; enabled: () => boolean; - socketsEnabled?: () => Promise; parseResumeToken?: (token: string | null | undefined) => unknown | undefined; prepare?: () => Promise; onResumePersisted?: (resumeToken: string) => Promise; @@ -186,10 +185,10 @@ export class ChangeStreamCoordinator { try { const parseToken = this.options.parseResumeToken ?? parseMongoResumeToken; const resumeAfter = parseToken( - (await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY)) ?? undefined, + await this.options.grpcSdk.state!.getKey(RESUME_TOKEN_KEY), ); if (this.watching || this.closed) return; - const stream = this.options.watch({ resumeAfter: resumeAfter ?? undefined }); + const stream = this.options.watch({ resumeAfter }); this.stream = stream; this.watching = true; this.streamState = 'live'; @@ -363,6 +362,3 @@ export class ChangeStreamCoordinator { } } } - -/** @deprecated Use ChangeStreamCoordinator */ -export { ChangeStreamCoordinator as MongoChangeStreamCoordinator }; diff --git a/modules/database/src/realtime/RealtimeService.ts b/modules/database/src/realtime/RealtimeService.ts index b85c3aa76..0786801ed 100644 --- a/modules/database/src/realtime/RealtimeService.ts +++ b/modules/database/src/realtime/RealtimeService.ts @@ -19,8 +19,7 @@ import { ChangeStreamCoordinator } from './ChangeStreamCoordinator.js'; import { registerDatabaseRealtimeSocket } from './sockets.js'; import { buildRealtimeStatus } from './status.js'; import { RealtimeSubscriptionTracker } from './subscriptions.js'; -import type { OptedInSchema, RealtimeStatus } from './types.js'; -import type { ChangeStreamLike } from './types.js'; +import type { ChangeStreamLike, OptedInSchema, RealtimeStatus } from './types.js'; import { topologyFromHello } from './topology.js'; import { SqlRealtimeSupport } from './sql/SqlRealtimeSupport.js'; @@ -50,7 +49,6 @@ export class RealtimeService { getOptedInSchemas: () => this.getOptedInSchemas(), subscriptions: this.subscriptions, enabled: () => this.isGloballyEnabled(), - socketsEnabled: () => this.areAdminSocketsEnabled(), }); } else if (adapter instanceof SequelizeAdapter) { this.sqlSupport = new SqlRealtimeSupport(adapter); @@ -61,7 +59,6 @@ export class RealtimeService { getOptedInSchemas: () => this.getOptedInSchemas(), subscriptions: this.subscriptions, enabled: () => this.isGloballyEnabled(), - socketsEnabled: () => this.areAdminSocketsEnabled(), prepare: () => this.sqlSupport!.prepare( this.isGloballyEnabled() ? this.getOptedInSchemas() : [], diff --git a/modules/database/src/realtime/__tests__/authorize.test.ts b/modules/database/src/realtime/__tests__/authorize.test.ts index c441863eb..9c5eba15b 100644 --- a/modules/database/src/realtime/__tests__/authorize.test.ts +++ b/modules/database/src/realtime/__tests__/authorize.test.ts @@ -16,12 +16,9 @@ describe('realtime authorization helpers', () => { }); expect(requireSchemaName('Order')).toBe('Order'); expect(optionalDocumentId(undefined)).toBeUndefined(); - try { - requireSchemaName(''); - throw new Error('expected throw'); - } catch (err: any) { - expect(err.code).toBe(status.INVALID_ARGUMENT); - } + expect(() => requireSchemaName('')).toThrow( + expect.objectContaining({ code: status.INVALID_ARGUMENT }), + ); }); it('rejects missing schemas, disabled realtime, and CMS-read for clients', () => { @@ -43,24 +40,15 @@ describe('realtime authorization helpers', () => { }; }, }; - try { - assertSchemaAvailable(lookup, 'Missing', true); - throw new Error('expected throw'); - } catch (err: any) { - expect(err.code).toBe(status.NOT_FOUND); - } - try { - assertSchemaAvailable(lookup, 'Off', false); - throw new Error('expected throw'); - } catch (err: any) { - expect(err.code).toBe(status.FAILED_PRECONDITION); - } - try { - assertSchemaAvailable(lookup, 'Order', true); - throw new Error('expected throw'); - } catch (err: any) { - expect(err.code).toBe(status.PERMISSION_DENIED); - } + expect(() => assertSchemaAvailable(lookup, 'Missing', true)).toThrow( + expect.objectContaining({ code: status.NOT_FOUND }), + ); + expect(() => assertSchemaAvailable(lookup, 'Off', false)).toThrow( + expect.objectContaining({ code: status.FAILED_PRECONDITION }), + ); + expect(() => assertSchemaAvailable(lookup, 'Order', true)).toThrow( + expect.objectContaining({ code: status.PERMISSION_DENIED }), + ); const admin = assertSchemaAvailable(lookup, 'Order', false); expect(admin.authorizationEnabled).toBe(true); }); @@ -71,12 +59,11 @@ describe('realtime authorization helpers', () => { throw new GrpcError(status.NOT_FOUND, 'Schema Missing not defined yet'); }, }; - try { - assertSchemaAvailable(lookup, 'Missing', false); - throw new Error('expected throw'); - } catch (err: any) { - expect(err.code).toBe(status.NOT_FOUND); - expect(err.message).toBe('Schema does not exist'); - } + expect(() => assertSchemaAvailable(lookup, 'Missing', false)).toThrow( + expect.objectContaining({ + code: status.NOT_FOUND, + message: 'Schema does not exist', + }), + ); }); }); diff --git a/modules/database/src/realtime/__tests__/coordinator.test.ts b/modules/database/src/realtime/__tests__/coordinator.test.ts index 995401e7d..55fa5baa9 100644 --- a/modules/database/src/realtime/__tests__/coordinator.test.ts +++ b/modules/database/src/realtime/__tests__/coordinator.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it, jest } from '@jest/globals'; import { ObjectId } from 'bson'; import { ChangeStreamCoordinator } from '../ChangeStreamCoordinator.js'; import { RealtimeSubscriptionTracker } from '../subscriptions.js'; -import { authorizedDocumentRoom, roomsForPublicChange } from '../rooms.js'; +import { roomsForPublicChange } from '../rooms.js'; class MemoryStore { private sets = new Map>(); @@ -156,9 +156,6 @@ describe('ChangeStreamCoordinator', () => { await new Promise(resolve => setImmediate(resolve)); expect(routerPush).not.toHaveBeenCalled(); expect(await subscriptions.listUsers('Order', '64b64c4c4c4c4c4c4c4c4c')).toEqual([]); - expect(authorizedDocumentRoom('Order', '64b64c4c4c4c4c4c4c4c4c', 'user-1')).toContain( - 'user-1', - ); await coordinator.shutdown(); }); diff --git a/modules/database/src/realtime/__tests__/normalize.test.ts b/modules/database/src/realtime/__tests__/normalize.test.ts index d51eaa549..db2278e29 100644 --- a/modules/database/src/realtime/__tests__/normalize.test.ts +++ b/modules/database/src/realtime/__tests__/normalize.test.ts @@ -9,6 +9,7 @@ describe('normalizeChangeEvent', () => { { operationType: 'insert', documentKey: { _id: new ObjectId('64b64c4c4c4c4c4c4c4c4c4c') }, + fullDocument: { secret: 'nope' }, wallTime: new Date('2026-01-01T00:00:00.000Z'), _id: resume, }, @@ -21,9 +22,7 @@ describe('normalizeChangeEvent', () => { documentId: '64b64c4c4c4c4c4c4c4c4c4c', occurredAt: '2026-01-01T00:00:00.000Z', }); - expect(event?.resumeToken).toEqual( - parseResumeToken(event!.resumeToken) ? event!.resumeToken : event?.resumeToken, - ); + expect(parseResumeToken(event!.resumeToken)).toEqual(resume); expect(JSON.parse(JSON.stringify(event))).not.toHaveProperty('fullDocument'); }); diff --git a/modules/database/src/realtime/__tests__/sql-builders.test.ts b/modules/database/src/realtime/__tests__/sql-builders.test.ts index 08a9dd47d..37692e2d1 100644 --- a/modules/database/src/realtime/__tests__/sql-builders.test.ts +++ b/modules/database/src/realtime/__tests__/sql-builders.test.ts @@ -13,7 +13,7 @@ import { import { toRawChangeEvent } from '../sql/mapEvent.js'; import { normalizeChangeEvent } from '../normalize.js'; -describe('SQL realtime SQL builders', () => { +describe('SQL realtime builders', () => { it('quotes identifiers per dialect', () => { expect(quoteIdent('postgres', 'orders')).toBe('"orders"'); expect(quoteIdent('mysql', 'orders')).toBe('`orders`'); @@ -49,7 +49,6 @@ describe('SQL realtime SQL builders', () => { rowTriggerName('orders', 'd', 'sqlite'), ]); expect(sqlite[0].sql).toContain(CHANGE_LOG_TABLE); - expect(sqlite[0].sql).not.toContain('secret'); }); it('builds dialect-specific change-log tables', () => { @@ -85,6 +84,5 @@ describe('SQL change-log event mapping', () => { schema: 'Order', documentId: 'order-1', }); - expect(JSON.stringify(raw)).not.toContain('secret'); }); }); diff --git a/modules/database/src/realtime/authorize.ts b/modules/database/src/realtime/authorize.ts index 330992f13..cf7ed899c 100644 --- a/modules/database/src/realtime/authorize.ts +++ b/modules/database/src/realtime/authorize.ts @@ -35,13 +35,6 @@ export type SchemaLookup = { | undefined; }; -export type ResolvedSubscription = { - schema: string; - documentId?: string; - authorizationEnabled: boolean; - rooms: string[]; -}; - export function parseSubscribeRequest(params: unknown[]): SubscribeRequest { const raw = params[0]; if (raw && typeof raw === 'object' && !Array.isArray(raw)) { @@ -93,10 +86,7 @@ export function assertSchemaAvailable( lookup: SchemaLookup, schemaName: string, requireCmsRead: boolean, -): { - authorizationEnabled: boolean; - realtimeEnabled: boolean; -} { +): { authorizationEnabled: boolean } { let schema: ReturnType; try { schema = lookup.getSchema(schemaName); @@ -124,7 +114,6 @@ export function assertSchemaAvailable( } return { authorizationEnabled: conduit.authorization?.enabled === true, - realtimeEnabled: true, }; } diff --git a/modules/database/src/realtime/index.ts b/modules/database/src/realtime/index.ts index e3de1442d..dab82831a 100644 --- a/modules/database/src/realtime/index.ts +++ b/modules/database/src/realtime/index.ts @@ -1,8 +1,5 @@ export { RealtimeService } from './RealtimeService.js'; -export { - ChangeStreamCoordinator, - MongoChangeStreamCoordinator, -} from './ChangeStreamCoordinator.js'; +export { ChangeStreamCoordinator } from './ChangeStreamCoordinator.js'; export { buildRealtimeStatus } from './status.js'; export { normalizeChangeEvent } from './normalize.js'; export { diff --git a/modules/database/src/realtime/sockets.ts b/modules/database/src/realtime/sockets.ts index 5cc97957e..4be4e1018 100644 --- a/modules/database/src/realtime/sockets.ts +++ b/modules/database/src/realtime/sockets.ts @@ -22,15 +22,17 @@ import type { RealtimeSubscriptionTracker } from './subscriptions.js'; type SocketMode = 'client' | 'admin'; +type RealtimeSocketOptions = { + mode: SocketMode; + grpcSdk: ConduitGrpcSdk; + schemaLookup: SchemaLookup; + subscriptions: RealtimeSubscriptionTracker; + isGloballyEnabled: () => boolean; +}; + export function registerDatabaseRealtimeSocket( routingManager: RoutingManager, - options: { - mode: SocketMode; - grpcSdk: ConduitGrpcSdk; - schemaLookup: SchemaLookup; - subscriptions: RealtimeSubscriptionTracker; - isGloballyEnabled: () => boolean; - }, + options: RealtimeSocketOptions, ) { const handlers = createSocketHandlers(options); routingManager.socket( @@ -59,13 +61,7 @@ export function registerDatabaseRealtimeSocket( ); } -function createSocketHandlers(options: { - mode: SocketMode; - grpcSdk: ConduitGrpcSdk; - schemaLookup: SchemaLookup; - subscriptions: RealtimeSubscriptionTracker; - isGloballyEnabled: () => boolean; -}) { +function createSocketHandlers(options: RealtimeSocketOptions) { return { connect: async (): Promise => { return { event: 'connected', data: { ok: true } }; @@ -87,13 +83,7 @@ function createSocketHandlers(options: { async function resolveSubscription( call: ParsedSocketRequest, - options: { - mode: SocketMode; - grpcSdk: ConduitGrpcSdk; - schemaLookup: SchemaLookup; - subscriptions: RealtimeSubscriptionTracker; - isGloballyEnabled: () => boolean; - }, + options: RealtimeSocketOptions, action: 'join' | 'leave', ): Promise { if (!options.isGloballyEnabled()) { diff --git a/modules/database/src/realtime/sql/changelog.ts b/modules/database/src/realtime/sql/changelog.ts index 88382ce26..1596d4c5b 100644 --- a/modules/database/src/realtime/sql/changelog.ts +++ b/modules/database/src/realtime/sql/changelog.ts @@ -9,9 +9,6 @@ import { quoteIdent } from './identifiers.js'; import { createCaptureFunctionSql, createChangeLogTableSql } from './ddl.js'; import type { ChangeLogRow } from './mapEvent.js'; -export { createChangeLogTableSql, createCaptureFunctionSql } from './ddl.js'; -export type { ChangeLogRow } from './mapEvent.js'; - export async function ensureChangeLog(sequelize: Sequelize): Promise { const dialect = assertSqlDialect(sequelize.getDialect()); await sequelize.query(createChangeLogTableSql(dialect)); @@ -43,12 +40,7 @@ export async function fetchChangeLogBatch( collection_name: String(row.collection_name), document_id: String(row.document_id), operation: String(row.operation), - occurred_at: - row.occurred_at instanceof Date - ? row.occurred_at - : typeof row.occurred_at === 'string' || typeof row.occurred_at === 'number' - ? row.occurred_at - : new Date().toISOString(), + occurred_at: occurredAtValue(row.occurred_at), })); } @@ -103,6 +95,13 @@ function trimSql(dialect: SqlDialect, table: string): string { } } +function occurredAtValue(value: unknown): Date | string | number { + if (value instanceof Date || typeof value === 'string' || typeof value === 'number') { + return value; + } + return new Date().toISOString(); +} + function affectedRows(metadata: unknown): number { if (!metadata || typeof metadata !== 'object') { return 0; diff --git a/modules/database/src/realtime/sql/identifiers.ts b/modules/database/src/realtime/sql/identifiers.ts index 49c035348..5e6a89349 100644 --- a/modules/database/src/realtime/sql/identifiers.ts +++ b/modules/database/src/realtime/sql/identifiers.ts @@ -6,8 +6,6 @@ const MYSQL_IDENT_LIMIT = 64; const POSTGRES_IDENT_LIMIT = 63; const SQLITE_IDENT_LIMIT = 128; -export type QuoteIdent = (name: string) => string; - export function quoteIdent(dialect: SqlDialect, name: string): string { if (dialect === 'mysql' || dialect === 'mariadb') { return `\`${name.replace(/`/g, '``')}\``; diff --git a/modules/database/src/realtime/sql/resume.ts b/modules/database/src/realtime/sql/resume.ts index eaa4f3efa..42c58d52f 100644 --- a/modules/database/src/realtime/sql/resume.ts +++ b/modules/database/src/realtime/sql/resume.ts @@ -1,4 +1,3 @@ -import { EJSON } from 'bson'; import { parseResumeToken } from '../normalize.js'; export function parseSqlResumeId(token: string | null | undefined): string | undefined { @@ -6,19 +5,15 @@ export function parseSqlResumeId(token: string | null | undefined): string | und if (/^\d+$/.test(token)) { return token; } - try { - const parsed = parseResumeToken(token) ?? EJSON.parse(token); - if (typeof parsed === 'number' && Number.isInteger(parsed) && parsed >= 0) { - return String(parsed); - } - if (typeof parsed === 'bigint' && parsed >= 0n) { - return parsed.toString(); - } - if (typeof parsed === 'string' && /^\d+$/.test(parsed)) { - return parsed; - } - } catch { - return undefined; + const parsed = parseResumeToken(token); + if (typeof parsed === 'number' && Number.isInteger(parsed) && parsed >= 0) { + return String(parsed); + } + if (typeof parsed === 'bigint' && parsed >= 0n) { + return parsed.toString(); + } + if (typeof parsed === 'string' && /^\d+$/.test(parsed)) { + return parsed; } return undefined; } diff --git a/modules/router/src/event-relays/EventRelayManager.ts b/modules/router/src/event-relays/EventRelayManager.ts index bddae2127..80cb809b7 100644 --- a/modules/router/src/event-relays/EventRelayManager.ts +++ b/modules/router/src/event-relays/EventRelayManager.ts @@ -94,7 +94,7 @@ export class EventRelayManager { private onBusMessage(channel: string, message: string): void { const relays = this.relaysByChannel.get(channel); - if (!relays || relays.length === 0) { + if (!relays?.length) { return; } @@ -128,7 +128,7 @@ export class EventRelayManager { ConduitGrpcSdk.Metrics?.increment('event_relays_failed_total'); ConduitGrpcSdk.Logger.error( `Event relay ${emission.relayId} emit failed on ${channel}: ${ - (err as Error).message + err instanceof Error ? err.message : String(err) }`, ); }, diff --git a/modules/router/src/event-relays/EventRelaySockets.test.ts b/modules/router/src/event-relays/EventRelaySockets.test.ts index 19f455b7a..c0a633bb1 100644 --- a/modules/router/src/event-relays/EventRelaySockets.test.ts +++ b/modules/router/src/event-relays/EventRelaySockets.test.ts @@ -1,7 +1,7 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { status } from '@grpc/grpc-js'; -import { authorizeRelaySubscription } from './authorize.js'; +import { authorizeRelaySubscription, RelaySubscriptionError } from './authorize.js'; import { eventRelayRoom } from './rooms.js'; import type { RelayLookup } from './authorize.js'; @@ -60,7 +60,8 @@ describe('authorizeRelaySubscription', () => { 'relay-1', 'order-1', ), - (err: any) => err.code === status.PERMISSION_DENIED, + (err: unknown) => + err instanceof RelaySubscriptionError && err.code === status.PERMISSION_DENIED, ); }); @@ -74,18 +75,8 @@ describe('authorizeRelaySubscription', () => { 'relay-1', 'order-1', ), - (err: any) => err.code === status.UNAVAILABLE, + (err: unknown) => + err instanceof RelaySubscriptionError && err.code === status.UNAVAILABLE, ); }); - - it('unsubscribe room matches subscribe room for the same ids', async () => { - const subscribed = await authorizeRelaySubscription( - mockSdk({ allow: true }), - mockManager(relay), - 'user-1', - 'relay-1', - 'order-1', - ); - assert.equal(subscribed, eventRelayRoom('relay-1', 'order-1')); - }); }); diff --git a/modules/router/src/event-relays/template.ts b/modules/router/src/event-relays/template.ts index f0f70dcc9..0fa4926ac 100644 --- a/modules/router/src/event-relays/template.ts +++ b/modules/router/src/event-relays/template.ts @@ -57,13 +57,10 @@ function renderValue(value: unknown, payload: unknown, depth: number): unknown { return value.map(item => renderValue(item, payload, depth + 1)); } if (value !== null && typeof value === 'object') { + const record = value as Record; const output: Record = {}; - for (const key of Object.keys(value as Record)) { - output[key] = renderValue( - (value as Record)[key], - payload, - depth + 1, - ); + for (const key of Object.keys(record)) { + output[key] = renderValue(record[key], payload, depth + 1); } return output; }