From 89e0c4b04dc179c6573b8b6c85f40dac35944416 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Thu, 4 Jun 2026 15:50:41 +0300 Subject: [PATCH 01/29] feat: embedding support + new module --- libraries/grpc-sdk/src/index.ts | 11 + libraries/grpc-sdk/src/interfaces/Model.ts | 77 ++++ .../grpc-sdk/src/modules/database/index.ts | 69 ++++ .../grpc-sdk/src/modules/embeddings/index.ts | 64 ++++ libraries/grpc-sdk/src/modules/index.ts | 1 + libraries/hermes/src/MCP/constants.ts | 1 + modules/database/README.md | 41 +++ modules/database/package.json | 1 + modules/database/src/Database.ts | 136 +++++++ .../database/src/adapters/DatabaseAdapter.ts | 40 +++ .../mongoose-adapter/SchemaConverter.ts | 4 + .../src/adapters/mongoose-adapter/index.ts | 239 ++++++++++++- .../src/adapters/sequelize-adapter/index.ts | 289 +++++++++++++++ .../postgres-adapter/PgSchemaConverter.ts | 5 + .../postgres-adapter/index.ts | 11 + .../sql-adapter/SqlSchemaConverter.ts | 2 + .../sequelize-adapter/utils/sqlTypeMap.ts | 1 + .../utils/validateFieldConstraints.ts | 12 + modules/database/src/admin/index.ts | 85 +++++ modules/database/src/admin/schema.admin.ts | 56 +++ modules/database/src/database.proto | 59 +++ .../src/interfaces/SchemaFieldTypes.ts | 20 +- modules/embeddings/README.md | 41 +++ modules/embeddings/build.sh | 19 + modules/embeddings/package.json | 45 +++ modules/embeddings/src/Embeddings.ts | 336 ++++++++++++++++++ modules/embeddings/src/config/index.ts | 39 ++ .../src/controllers/queue.controller.ts | 65 ++++ modules/embeddings/src/embeddings.proto | 43 +++ modules/embeddings/src/index.ts | 7 + modules/embeddings/src/metrics/index.ts | 18 + .../src/models/EmbeddingConfig.schema.ts | 66 ++++ modules/embeddings/src/models/index.ts | 1 + modules/embeddings/src/providers/index.ts | 48 +++ .../test/embedding-contract.test.mjs | 19 + modules/embeddings/tsconfig.json | 18 + pnpm-lock.yaml | 73 ++++ 37 files changed, 2058 insertions(+), 4 deletions(-) create mode 100644 libraries/grpc-sdk/src/modules/embeddings/index.ts create mode 100644 modules/database/README.md create mode 100644 modules/embeddings/README.md create mode 100644 modules/embeddings/build.sh create mode 100644 modules/embeddings/package.json create mode 100644 modules/embeddings/src/Embeddings.ts create mode 100644 modules/embeddings/src/config/index.ts create mode 100644 modules/embeddings/src/controllers/queue.controller.ts create mode 100644 modules/embeddings/src/embeddings.proto create mode 100644 modules/embeddings/src/index.ts create mode 100644 modules/embeddings/src/metrics/index.ts create mode 100644 modules/embeddings/src/models/EmbeddingConfig.schema.ts create mode 100644 modules/embeddings/src/models/index.ts create mode 100644 modules/embeddings/src/providers/index.ts create mode 100644 modules/embeddings/test/embedding-contract.test.mjs create mode 100644 modules/embeddings/tsconfig.json diff --git a/libraries/grpc-sdk/src/index.ts b/libraries/grpc-sdk/src/index.ts index 7ccac264f..31a709991 100644 --- a/libraries/grpc-sdk/src/index.ts +++ b/libraries/grpc-sdk/src/index.ts @@ -7,6 +7,7 @@ import { Config, Core, DatabaseProvider, + EmbeddingsProvider, Email, PushNotifications, Router, @@ -66,6 +67,7 @@ class ConduitGrpcSdk { private readonly _availableModules: any = { router: Router, database: DatabaseProvider, + embeddings: EmbeddingsProvider, storage: Storage, email: Email, pushNotifications: PushNotifications, @@ -223,6 +225,15 @@ class ConduitGrpcSdk { return this.database; } + get embeddings(): EmbeddingsProvider | null { + if (this._modules['embeddings']) { + return this._modules['embeddings'] as EmbeddingsProvider; + } else { + ConduitGrpcSdk.Logger.warn('Embeddings provider not up yet!'); + return null; + } + } + get storage(): Storage | null { if (this._modules['storage']) { return this._modules['storage'] as Storage; diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 6b7319a9b..697bf152a 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -1,3 +1,5 @@ +import type { Indexable } from './Indexable.js'; + export enum TYPE { String = 'String', Number = 'Number', @@ -6,6 +8,7 @@ export enum TYPE { ObjectId = 'ObjectId', JSON = 'JSON', Relation = 'Relation', + Vector = 'Vector', } export enum SQLDataType { @@ -20,6 +23,19 @@ export enum SQLDataType { TIME = 'TIME', DATETIME = 'DATETIME', TIMESTAMP = 'TIMESTAMP', + VECTOR = 'VECTOR', +} + +export enum VectorSimilarity { + Cosine = 'cosine', + Euclidean = 'euclidean', + DotProduct = 'dotProduct', +} + +export enum VectorIndexMethod { + HNSW = 'hnsw', + IVFFlat = 'ivfflat', + Flat = 'flat', } export enum MongoIndexType { @@ -109,6 +125,14 @@ export type ConduitModelFieldJSON = BasicConduitModelField & { type: TYPE.JSON | TYPE.JSON[]; }; +export type ConduitModelFieldVector = BasicConduitModelField & { + type: TYPE.Vector; + dimensions: number; + similarity?: VectorSimilarity; + provider?: string; + model?: string; +}; + export type ConduitModelFieldEnum = BasicConduitModelField & { type: ExcludeJSONRelation | ExcludeJSONRelation[]; enum: any; @@ -127,6 +151,7 @@ export type allowedTypes = | ConduitModelField | ConduitModelFieldEnum | ConduitModelFieldJSON + | ConduitModelFieldVector | ConduitModelFieldRelation; type embeddableArray = @@ -195,6 +220,7 @@ export interface ConduitSchemaOptions { }; /** Includes readonly/const-asserted index arrays (e.g. `as const` in schema definitions). */ indexes?: ReadonlyArray; + vectorIndexes?: ReadonlyArray; } export interface SchemaFieldIndex { @@ -247,3 +273,54 @@ export interface PostgresIndexOptions { [opt: string]: any; }; } + +export interface VectorIndexDefinition { + name?: string; + field: string; + dimensions: number; + similarity: VectorSimilarity; + method?: VectorIndexMethod; + filterFields?: string[]; + options?: { + numCandidates?: number; + quantization?: 'none' | 'scalar' | 'binary'; + hnsw?: { + maxEdges?: number; + numEdgeCandidates?: number; + m?: number; + efConstruction?: number; + }; + ivfflat?: { + lists?: number; + probes?: number; + }; + storedSource?: boolean | { include?: string[]; exclude?: string[] }; + }; +} + +export interface VectorCapabilities { + supported: boolean; + storage: boolean; + indexing: boolean; + search: boolean; + provider: 'mongodb' | 'postgres' | 'unsupported'; + reason?: string; +} + +export interface VectorSearchInput { + schemaName: string; + field: string; + vector: number[]; + indexName?: string; + filter?: Indexable; + limit?: number; + numCandidates?: number; + select?: string; + userId?: string; + scope?: string; +} + +export interface VectorSearchResult { + document: T; + score: number; +} diff --git a/libraries/grpc-sdk/src/modules/database/index.ts b/libraries/grpc-sdk/src/modules/database/index.ts index 218407713..b37ef8f4d 100644 --- a/libraries/grpc-sdk/src/modules/database/index.ts +++ b/libraries/grpc-sdk/src/modules/database/index.ts @@ -14,6 +14,12 @@ import { Query } from '../../types/db.js'; import type { FindOneOptions, FindManyOptions } from './types.js'; export type { FindOneOptions, FindManyOptions } from './types.js'; import { AuthzOptions, PopulateAuthzOptions } from '../../types/options.js'; +import type { + VectorCapabilities, + VectorIndexDefinition, + VectorSearchInput, + VectorSearchResult, +} from '../../interfaces/Model.js'; export type CountDocumentsOptions = AuthzOptions & { readPreference?: string }; @@ -319,6 +325,69 @@ export class DatabaseProvider extends ConduitModule { + return this.client!.getVectorCapabilities({ schemaName }).then(res => ({ + supported: res.supported, + storage: res.storage, + indexing: res.indexing, + search: res.search, + provider: res.provider as VectorCapabilities['provider'], + reason: res.reason, + })); + } + + createVectorIndex(schemaName: string, index: VectorIndexDefinition): Promise { + return this.client!.createVectorIndex({ + schemaName, + index: { + field: index.field, + dimensions: index.dimensions, + similarity: index.similarity, + name: index.name, + method: index.method, + filterFields: [...(index.filterFields ?? [])], + options: index.options ? JSON.stringify(index.options) : undefined, + }, + }).then(res => JSON.parse(res.result)); + } + + getVectorIndexes(schemaName: string): Promise { + return this.client!.getVectorIndexes({ schemaName }).then(res => + res.indexes.map(index => ({ + field: index.field, + dimensions: index.dimensions, + similarity: index.similarity as VectorIndexDefinition['similarity'], + name: index.name, + method: index.method as VectorIndexDefinition['method'], + filterFields: index.filterFields, + options: index.options ? JSON.parse(index.options) : undefined, + })), + ); + } + + deleteVectorIndex(schemaName: string, indexName: string): Promise { + return this.client!.deleteVectorIndex({ schemaName, indexName }).then(res => + JSON.parse(res.result), + ); + } + + vectorSearch( + request: VectorSearchInput, + ): Promise[]> { + return this.client!.vectorSearch({ + schemaName: request.schemaName, + field: request.field, + vector: request.vector, + indexName: request.indexName, + filter: request.filter ? JSON.stringify(request.filter) : undefined, + limit: request.limit, + numCandidates: request.numCandidates, + select: request.select, + userId: request.userId, + scope: request.scope, + }).then(res => JSON.parse(res.result)); + } + createView( schemaName: string, viewName: string, diff --git a/libraries/grpc-sdk/src/modules/embeddings/index.ts b/libraries/grpc-sdk/src/modules/embeddings/index.ts new file mode 100644 index 000000000..bb2e124f0 --- /dev/null +++ b/libraries/grpc-sdk/src/modules/embeddings/index.ts @@ -0,0 +1,64 @@ +import { ConduitModule } from '../../classes/index.js'; +import { EmbeddingsProviderDefinition } from '../../protoUtils/embeddings.js'; +import type { Indexable, VectorSearchResult } from '../../interfaces/index.js'; + +export interface EmbeddingConfigInput { + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + model: string; + dimensions: number; + similarity?: string; +} + +export interface SemanticSearchInput { + schemaName: string; + text: string; + targetField?: string; + filter?: Indexable; + limit?: number; + userId?: string; + scope?: string; +} + +export class EmbeddingsProvider extends ConduitModule< + typeof EmbeddingsProviderDefinition +> { + constructor( + private readonly moduleName: string, + url: string, + grpcToken?: string, + ) { + super(moduleName, 'embeddings', url, grpcToken); + this.initializeClient(EmbeddingsProviderDefinition); + } + + upsertConfig(config: EmbeddingConfigInput): Promise { + return this.client!.upsertConfig(config).then(res => res.result); + } + + getConfigs(): Promise { + return this.client!.getConfigs({}).then(res => JSON.parse(res.result)); + } + + startBackfill(schemaName: string, batchSize?: number): Promise<{ queued: number }> { + return this.client!.startBackfill({ schemaName, batchSize }).then(res => + JSON.parse(res.result), + ); + } + + semanticSearch( + input: SemanticSearchInput, + ): Promise[]> { + return this.client!.semanticSearch({ + schemaName: input.schemaName, + text: input.text, + targetField: input.targetField, + filter: input.filter ? JSON.stringify(input.filter) : undefined, + limit: input.limit, + userId: input.userId, + scope: input.scope, + }).then(res => JSON.parse(res.result)); + } +} diff --git a/libraries/grpc-sdk/src/modules/index.ts b/libraries/grpc-sdk/src/modules/index.ts index bfd9e6ed9..a31c9b5fd 100644 --- a/libraries/grpc-sdk/src/modules/index.ts +++ b/libraries/grpc-sdk/src/modules/index.ts @@ -2,6 +2,7 @@ export * from './storage/index.js'; export * from './router/index.js'; export * from './email/index.js'; export * from './database/index.js'; +export * from './embeddings/index.js'; export * from './config/index.js'; export * from './core/index.js'; export * from './admin/index.js'; diff --git a/libraries/hermes/src/MCP/constants.ts b/libraries/hermes/src/MCP/constants.ts index 01977b0c5..3a7b8c62b 100644 --- a/libraries/hermes/src/MCP/constants.ts +++ b/libraries/hermes/src/MCP/constants.ts @@ -39,6 +39,7 @@ Module discovery and activation: Common modules: - database: schemas, documents, custom endpoints, indexes +- embeddings: embedding configuration, backfills, semantic search - authentication: users, teams, OAuth services - storage: file storage configuration - authorization: relations, resources, permission checks (RBAC/ReBAC) diff --git a/modules/database/README.md b/modules/database/README.md new file mode 100644 index 000000000..adbff147f --- /dev/null +++ b/modules/database/README.md @@ -0,0 +1,41 @@ +# Database Module + +## Vector Search + +Conduit supports provider-neutral vector storage and search through `TYPE.Vector`, +vector index contracts, and the Database gRPC/admin vector APIs. + +### Schema Field + +```ts +{ + embedding: { + type: TYPE.Vector, + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + select: false, + }, +} +``` + +### Capabilities + +Use `getVectorCapabilities` before creating indexes or running searches. Capability +responses distinguish storage support from index/search support: + +- MongoDB stores vectors as numeric arrays and uses MongoDB Search/Vector Search + indexes when `createSearchIndex` and `listSearchIndexes` are available. +- PostgreSQL uses `pgvector`; the adapter attempts `CREATE EXTENSION IF NOT EXISTS + vector` during startup and reports missing privileges or extension support through + capabilities. +- Other Sequelize dialects report vector search as unsupported. + +### Rollout + +1. Add a `TYPE.Vector` field to the schema or via the Embeddings module schema + extension. +2. Call `getVectorCapabilities` and verify `indexing` and `search` are true. +3. Create a vector index with the field, dimensions, similarity, and optional + filter fields. +4. Backfill embeddings. +5. Run `vectorSearch` with a query vector. diff --git a/modules/database/package.json b/modules/database/package.json index 4a17ae844..d7f771863 100644 --- a/modules/database/package.json +++ b/modules/database/package.json @@ -54,6 +54,7 @@ "object-hash": "^3.0.0", "pg": "^8.22.0", "pg-hstore": "^2.3.4", + "pgvector": "^0.3.0", "sequelize": "^6.37.8", "sequelize-auto": "^0.8.8", "sqlite3": "^6.0.1" diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index 57286cb1e..7f2419404 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -31,6 +31,14 @@ import { Schema as SchemaDto, UpdateManyRequest, UpdateRequest, + DeleteVectorIndexRequest, + VectorCapabilitiesRequest, + VectorCapabilitiesResponse, + VectorIndex, + VectorIndexListRequest, + VectorIndexListResponse, + VectorIndexRequest, + VectorSearchRequest, } from './protoTypes/database.js'; import { CreateSchemaExtensionRequest, @@ -97,6 +105,11 @@ export default class DatabaseModule extends ManagedModule { migrate: this.migrate.bind(this), getDatabaseType: this.getDatabaseType.bind(this), generateId: this.generateId.bind(this), + getVectorCapabilities: this.getVectorCapabilities.bind(this), + createVectorIndex: this.createVectorIndex.bind(this), + getVectorIndexes: this.getVectorIndexes.bind(this), + deleteVectorIndex: this.deleteVectorIndex.bind(this), + vectorSearch: this.vectorSearch.bind(this), }, }; protected metricsSchema = metricsSchema; @@ -943,6 +956,117 @@ export default class DatabaseModule extends ManagedModule { callback(null, { result: exist }); } + async getVectorCapabilities( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this._activeAdapter.getVectorCapabilities( + call.request.schemaName, + ); + callback(null, result); + } catch (err) { + callback({ code: status.INTERNAL, message: (err as Error).message }); + } + } + + async createVectorIndex( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + if (!call.request.index) { + return callback({ + code: status.INVALID_ARGUMENT, + message: 'Vector index definition is required', + }); + } + const moduleName = call.metadata!.get('module-name')![0] as string; + const schemaAdapter = this._activeAdapter.getSchemaModel(call.request.schemaName); + if (!(await canModify(moduleName, schemaAdapter.model))) { + return callback({ + code: status.PERMISSION_DENIED, + message: `Module ${moduleName} is not authorized to create vector indexes for ${call.request.schemaName}!`, + }); + } + const result = await this._activeAdapter.createVectorIndex( + call.request.schemaName, + this.parseVectorIndex(call.request.index), + ); + callback(null, { result: JSON.stringify(result) }); + } catch (err) { + callback({ code: status.INTERNAL, message: (err as Error).message }); + } + } + + async getVectorIndexes( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const indexes = await this._activeAdapter.getVectorIndexes(call.request.schemaName); + callback(null, { + indexes: indexes.map(index => ({ + field: index.field, + dimensions: index.dimensions, + similarity: index.similarity, + name: index.name, + method: index.method, + filterFields: [...(index.filterFields ?? [])], + options: index.options ? JSON.stringify(index.options) : undefined, + })), + }); + } catch (err) { + callback({ code: status.INTERNAL, message: (err as Error).message }); + } + } + + async deleteVectorIndex( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const moduleName = call.metadata!.get('module-name')![0] as string; + const schemaAdapter = this._activeAdapter.getSchemaModel(call.request.schemaName); + if (!(await canModify(moduleName, schemaAdapter.model))) { + return callback({ + code: status.PERMISSION_DENIED, + message: `Module ${moduleName} is not authorized to delete vector indexes for ${call.request.schemaName}!`, + }); + } + const result = await this._activeAdapter.deleteVectorIndex( + call.request.schemaName, + call.request.indexName, + ); + callback(null, { result: JSON.stringify(result) }); + } catch (err) { + callback({ code: status.INTERNAL, message: (err as Error).message }); + } + } + + async vectorSearch( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this._activeAdapter.vectorSearch({ + schemaName: call.request.schemaName, + field: call.request.field, + vector: call.request.vector, + indexName: call.request.indexName, + filter: call.request.filter ? JSON.parse(call.request.filter) : undefined, + limit: call.request.limit, + numCandidates: call.request.numCandidates, + select: call.request.select, + userId: call.request.userId, + scope: call.request.scope, + }); + callback(null, { result: JSON.stringify(result) }); + } catch (err) { + callback({ code: status.INTERNAL, message: (err as Error).message }); + } + } + async migrate(call: GrpcRequest, callback: GrpcResponse) { if (this._activeAdapter.getDatabaseType() !== 'MongoDB') { const schemaName = call.request.schemaName; @@ -977,6 +1101,18 @@ export default class DatabaseModule extends ManagedModule { callback(null, { result }); } + private parseVectorIndex(index: VectorIndex) { + return { + field: index.field, + dimensions: index.dimensions, + similarity: index.similarity as any, + name: index.name, + method: index.method as any, + filterFields: index.filterFields, + options: index.options ? JSON.parse(index.options) : undefined, + }; + } + private registerInstanceSyncEvents() { this.grpcSdk.bus?.subscribe('database:request:schemas', () => { this._activeAdapter.registeredSchemas.forEach(schema => { diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index 7e5f898c9..7e3801ff6 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -8,6 +8,10 @@ import { RawMongoQuery, RawSQLQuery, TYPE, + VectorCapabilities, + VectorIndexDefinition, + VectorSearchInput, + VectorSearchResult, } from '@conduitplatform/grpc-sdk'; import { ConfigController } from '@conduitplatform/module-tools'; import type { Config } from '../config/index.js'; @@ -296,6 +300,42 @@ export abstract class DatabaseAdapter { rawQuery: RawMongoQuery | RawSQLQuery, ): Promise; + getVectorCapabilities(_schemaName?: string): Promise { + return Promise.resolve({ + supported: false, + storage: false, + indexing: false, + search: false, + provider: 'unsupported', + reason: `${this.getDatabaseType()} does not support Conduit vector search`, + }); + } + + createVectorIndex(_schemaName: string, _index: VectorIndexDefinition): Promise { + throw new GrpcError( + status.UNIMPLEMENTED, + `${this.getDatabaseType()} does not support vector indexes`, + ); + } + + getVectorIndexes(_schemaName: string): Promise { + return Promise.resolve([]); + } + + deleteVectorIndex(_schemaName: string, _indexName: string): Promise { + throw new GrpcError( + status.UNIMPLEMENTED, + `${this.getDatabaseType()} does not support vector indexes`, + ); + } + + vectorSearch(_request: VectorSearchInput): Promise { + throw new GrpcError( + status.UNIMPLEMENTED, + `${this.getDatabaseType()} does not support vector search`, + ); + } + abstract syncSchema(name: string): Promise; fixDatabaseSchemaOwnership(schema: ConduitSchema) { diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index 81a995253..35f47913d 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -87,6 +87,10 @@ function convert(value: any, key: any, parentValue: any) { parentValue[key].type = Schema.Types.Mixed; } + if (parentValue[key]?.type === 'Vector') { + parentValue[key].type = [Number]; + } + if (!isNil(parentValue[key]) && parentValue[key] === 'JSON') { parentValue[key] = Schema.Types.Mixed; } diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 7d5362f57..fd9ac3cba 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -9,6 +9,10 @@ import { ModelOptionsIndexes, MongoIndexType, RawMongoQuery, + VectorCapabilities, + VectorIndexDefinition, + VectorSearchInput, + VectorSearchResult, } from '@conduitplatform/grpc-sdk'; import { DatabaseAdapter } from '../DatabaseAdapter.js'; import { validateFieldChanges, validateFieldConstraints } from '../utils/index.js'; @@ -20,7 +24,7 @@ import { ConduitDatabaseSchema, introspectedSchemaCmsOptionsDefaults, } from '../../interfaces/index.js'; -import { isArray, isEqual } from 'lodash-es'; +import { isArray, isEqual, isNil } from 'lodash-es'; import { parseSchema } from 'mongodb-schema'; const VIEW_LOCK_TTL_MS = 60_000; @@ -720,6 +724,155 @@ export class MongooseAdapter extends DatabaseAdapter { return 'Indexes deleted'; } + async getVectorCapabilities(schemaName?: string): Promise { + const modelName = schemaName ?? Object.keys(this.models)[0]; + if (!modelName || !this.models[modelName]) { + return { + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + reason: 'No schema is available to probe MongoDB Vector Search support', + }; + } + + try { + const collection: any = this.mongoose.model(modelName).collection; + if (typeof collection.listSearchIndexes !== 'function') { + return { + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + reason: 'MongoDB driver does not expose search index commands', + }; + } + await collection.listSearchIndexes().toArray(); + return { + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'mongodb', + }; + } catch (err) { + return { + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + reason: (err as Error).message, + }; + } + } + + async createVectorIndex( + schemaName: string, + index: VectorIndexDefinition, + ): Promise { + if (!this.models[schemaName]) + throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + this.validateVectorField(schemaName, index); + const collection: any = this.mongoose.model(schemaName).collection; + if (typeof collection.createSearchIndex !== 'function') { + throw new GrpcError( + status.FAILED_PRECONDITION, + 'MongoDB Vector Search index commands are not available for this deployment', + ); + } + await collection.createSearchIndex({ + name: index.name ?? `${index.field}_vector`, + type: 'vectorSearch', + definition: this.toMongoVectorIndexDefinition(index), + }); + return 'Vector index created!'; + } + + async getVectorIndexes(schemaName: string): Promise { + if (!this.models[schemaName]) + throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const collection: any = this.mongoose.model(schemaName).collection; + if (typeof collection.listSearchIndexes !== 'function') return []; + const indexes = await collection.listSearchIndexes().toArray(); + return indexes + .filter((index: any) => index.type === 'vectorSearch') + .map((index: any) => this.fromMongoVectorIndex(index)); + } + + async deleteVectorIndex(schemaName: string, indexName: string): Promise { + if (!this.models[schemaName]) + throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const collection: any = this.mongoose.model(schemaName).collection; + if (typeof collection.dropSearchIndex !== 'function') { + throw new GrpcError( + status.FAILED_PRECONDITION, + 'MongoDB Vector Search index commands are not available for this deployment', + ); + } + await collection.dropSearchIndex(indexName); + return 'Vector index deleted'; + } + + async vectorSearch(request: VectorSearchInput): Promise { + if (!this.models[request.schemaName]) + throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const model = this.models[request.schemaName]; + const schemaField = + model.originalSchema.compiledFields?.[request.field] ?? + model.originalSchema.fields?.[request.field]; + if (schemaField?.type !== 'Vector') { + throw new GrpcError(status.INVALID_ARGUMENT, 'Requested field is not a vector'); + } + if (request.vector.length !== schemaField.dimensions) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Vector dimensions mismatch: expected ${schemaField.dimensions}`, + ); + } + + const filter = request.filter ?? {}; + const authorizedQuery = await model.getAuthorizedQuery( + 'read', + filter, + true, + request.userId, + request.scope, + ); + if (isNil(authorizedQuery)) return []; + + const vectorStage: any = { + index: request.indexName ?? `${request.field}_vector`, + path: request.field, + queryVector: request.vector, + numCandidates: request.numCandidates ?? Math.max((request.limit ?? 10) * 10, 100), + limit: request.limit ?? 10, + }; + if (Object.keys(authorizedQuery).length > 0) { + vectorStage.filter = authorizedQuery; + } + + const pipeline: any[] = [ + { $vectorSearch: vectorStage }, + { $addFields: { _score: { $meta: 'vectorSearchScore' } } }, + ]; + pipeline.push({ + $project: this.buildVectorProjection(model.originalSchema, request.select), + }); + + const docs = await this.mongoose + .model(request.schemaName) + .collection.aggregate(pipeline) + .toArray(); + return docs.map((doc: any) => { + const score = doc._score ?? 0; + delete doc._score; + return { document: doc, score }; + }); + } + async execRawQuery(schemaName: string, rawQuery: RawMongoQuery) { let collection = this.models[schemaName]?.model.collection; if (!collection) { @@ -804,6 +957,90 @@ export class MongooseAdapter extends DatabaseAdapter { ); } + private validateVectorField(schemaName: string, index: VectorIndexDefinition) { + const schema = this.models[schemaName].originalSchema as any; + const field = schema.compiledFields?.[index.field] ?? schema.fields?.[index.field]; + if (!field || field.type !== 'Vector') { + throw new GrpcError(status.INVALID_ARGUMENT, 'Vector index field is not a vector'); + } + if (field.dimensions !== index.dimensions) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Vector index dimensions mismatch: field ${field.dimensions}, index ${index.dimensions}`, + ); + } + } + + private toMongoVectorIndexDefinition(index: VectorIndexDefinition) { + const vectorField: any = { + type: 'vector', + path: index.field, + numDimensions: index.dimensions, + similarity: index.similarity, + }; + if (index.options?.quantization) + vectorField.quantization = index.options.quantization; + if (index.method) vectorField.indexingMethod = index.method; + if (index.options?.hnsw) { + vectorField.hnswOptions = { + ...(index.options.hnsw.maxEdges && { maxEdges: index.options.hnsw.maxEdges }), + ...(index.options.hnsw.numEdgeCandidates && { + numEdgeCandidates: index.options.hnsw.numEdgeCandidates, + }), + }; + } + return { + fields: [ + vectorField, + ...(index.filterFields ?? []).map((path: string) => ({ type: 'filter', path })), + ], + ...(index.options?.storedSource !== undefined && { + storedSource: index.options.storedSource, + }), + }; + } + + private fromMongoVectorIndex(index: any): VectorIndexDefinition { + const fields = index.latestDefinition?.fields ?? index.definition?.fields ?? []; + const vectorField = fields.find((field: any) => field.type === 'vector') ?? {}; + return { + name: index.name, + field: vectorField.path, + dimensions: vectorField.numDimensions, + similarity: vectorField.similarity, + method: vectorField.indexingMethod, + filterFields: fields + .filter((field: any) => field.type === 'filter') + .map((field: any) => field.path), + }; + } + + private buildVectorProjection(schema: ConduitDatabaseSchema, select?: string) { + const hiddenFields = new Set( + Object.entries(schema.compiledFields ?? schema.fields) + .filter(([, field]: [string, any]) => field?.select === false) + .map(([field]) => field), + ); + const tokens = select?.split(' ').filter(Boolean) ?? []; + const includeTokens = tokens.filter(token => !token.startsWith('-')); + if (includeTokens.length) { + return includeTokens.reduce( + (projection: any, token: string) => { + if (!hiddenFields.has(token)) projection[token] = 1; + return projection; + }, + { _score: 1 }, + ); + } + return [ + ...hiddenFields, + ...tokens.filter(token => token.startsWith('-')).map(token => token.slice(1)), + ].reduce((projection: any, field: string) => { + projection[field] = 0; + return projection; + }, {}); + } + protected async _createSchemaFromAdapter( schema: ConduitDatabaseSchema, saveToDb: boolean = true, diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 30df75904..176271b65 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -10,6 +10,11 @@ import { PostgresIndexType, RawSQLQuery, UntypedArray, + VectorCapabilities, + VectorIndexDefinition, + VectorIndexMethod, + VectorSearchInput, + VectorSearchResult, } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { SequelizeAuto } from 'sequelize-auto'; @@ -422,6 +427,131 @@ export abstract class SequelizeAdapter extends DatabaseAdapter return 'Indexes deleted'; } + async getVectorCapabilities(schemaName?: string): Promise { + if (this.sequelize.getDialect() !== 'postgres') { + return { + supported: false, + storage: false, + indexing: false, + search: false, + provider: 'unsupported', + reason: `${this.sequelize.getDialect()} does not support Conduit vector search`, + }; + } + try { + await this.sequelize.query("SELECT 'vector'::regtype"); + return { + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'postgres', + }; + } catch (err) { + return { + supported: true, + storage: false, + indexing: false, + search: false, + provider: 'postgres', + reason: schemaName + ? `Schema ${schemaName} cannot use pgvector: ${(err as Error).message}` + : (err as Error).message, + }; + } + } + + async createVectorIndex( + schemaName: string, + index: VectorIndexDefinition, + ): Promise { + this.ensurePostgresVectorSupport(schemaName); + this.validateVectorField(schemaName, index); + const tableName = this.getPhysicalTableName(schemaName); + const indexName = this.quoteIdentifier( + index.name ?? `${tableName}_${index.field}_vector`, + ); + const method = index.method === VectorIndexMethod.IVFFlat ? 'ivfflat' : 'hnsw'; + const operator = this.pgVectorOperator(index.similarity); + const withOptions = + method === 'ivfflat' + ? this.renderWithOptions({ lists: index.options?.ivfflat?.lists }) + : this.renderWithOptions({ + m: index.options?.hnsw?.m, + ef_construction: index.options?.hnsw?.efConstruction, + }); + await this.sequelize.query( + `CREATE INDEX IF NOT EXISTS ${indexName} ON ${this.quoteIdentifier( + tableName, + )} USING ${method} (${this.quoteIdentifier(index.field)} ${operator})${withOptions}`, + ); + return 'Vector index created!'; + } + + async getVectorIndexes(schemaName: string): Promise { + this.ensurePostgresVectorSupport(schemaName); + const tableName = this.getPhysicalTableName(schemaName); + const rows = await this.sequelize.query( + `SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = current_schema() AND tablename = ${this.sequelize.escape( + tableName, + )}`, + ); + return (rows[0] as any[]) + .filter(row => /USING (hnsw|ivfflat)/i.test(row.indexdef)) + .map(row => this.fromPostgresVectorIndex(row.indexname, row.indexdef)); + } + + async deleteVectorIndex(schemaName: string, indexName: string): Promise { + this.ensurePostgresVectorSupport(schemaName); + await this.sequelize.query(`DROP INDEX IF EXISTS ${this.quoteIdentifier(indexName)}`); + return 'Vector index deleted'; + } + + async vectorSearch(request: VectorSearchInput): Promise { + this.ensurePostgresVectorSupport(request.schemaName); + const schema = this.models[request.schemaName]; + const field = (schema.originalSchema.compiledFields?.[request.field] ?? + schema.originalSchema.fields?.[request.field]) as any; + if (field?.type !== 'Vector') { + throw new GrpcError(status.INVALID_ARGUMENT, 'Requested field is not a vector'); + } + if (request.vector.length !== field.dimensions) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Vector dimensions mismatch: expected ${field.dimensions}`, + ); + } + const authorizedQuery = await schema.getAuthorizedQuery( + 'read', + request.filter ?? {}, + true, + request.userId, + request.scope, + ); + if (isNil(authorizedQuery)) return []; + const tableName = this.getPhysicalTableName(request.schemaName); + const distance = this.pgVectorDistanceOperator(field.similarity ?? 'cosine'); + const where = this.renderSimpleWhere(authorizedQuery); + const limit = Math.max(1, Math.min(request.limit ?? 10, 1000)); + const vector = `[${request.vector.join(',')}]`; + const selectedColumns = this.buildVectorSelectList( + schema.originalSchema, + request.select, + ); + const rows = await this.sequelize.query( + `SELECT ${selectedColumns}, (${this.quoteIdentifier(request.field)} ${distance} ${this.sequelize.escape( + vector, + )}::vector) AS _score FROM ${this.quoteIdentifier(tableName)}${where} ORDER BY ${this.quoteIdentifier( + request.field, + )} ${distance} ${this.sequelize.escape(vector)}::vector LIMIT ${limit}`, + ); + return (rows[0] as any[]).map(row => { + const score = Number(row._score ?? 0); + delete row._score; + return { document: row, score }; + }); + } + async execRawQuery(schemaName: string, rawQuery: RawSQLQuery) { return await this.sequelize .query(rawQuery.query, rawQuery.options) @@ -464,6 +594,165 @@ export abstract class SequelizeAdapter extends DatabaseAdapter protected abstract hasLegacyCollections(): Promise; + private ensurePostgresVectorSupport(schemaName: string) { + if (this.sequelize.getDialect() !== 'postgres') { + throw new GrpcError( + status.UNIMPLEMENTED, + `${this.sequelize.getDialect()} does not support vector search`, + ); + } + if (!this.models[schemaName]) { + throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + } + } + + private validateVectorField(schemaName: string, index: VectorIndexDefinition) { + const schema = this.models[schemaName].originalSchema; + const field = (schema.compiledFields?.[index.field] ?? + schema.fields?.[index.field]) as any; + if (!field || field.type !== 'Vector') { + throw new GrpcError(status.INVALID_ARGUMENT, 'Vector index field is not a vector'); + } + if (field.dimensions !== index.dimensions) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Vector index dimensions mismatch: field ${field.dimensions}, index ${index.dimensions}`, + ); + } + } + + private getPhysicalTableName(schemaName: string) { + return this.models[schemaName].originalSchema.collectionName || `cnd_${schemaName}`; + } + + private quoteIdentifier(identifier: string) { + return `"${identifier.replace(/"/g, '""')}"`; + } + + private pgVectorOperator(similarity: string) { + if (similarity === 'euclidean') return 'vector_l2_ops'; + if (similarity === 'dotProduct') return 'vector_ip_ops'; + return 'vector_cosine_ops'; + } + + private pgVectorDistanceOperator(similarity: string) { + if (similarity === 'euclidean') return '<->'; + if (similarity === 'dotProduct') return '<#>'; + return '<=>'; + } + + private renderWithOptions(options: Record) { + const entries = Object.entries(options).filter((entry): entry is [string, number] => + Number.isFinite(entry[1]), + ); + if (!entries.length) return ''; + return ` WITH (${entries.map(([key, value]) => `${key} = ${value}`).join(', ')})`; + } + + private renderSimpleWhere(query: Indexable): string { + const clauses: string[] = Reflect.ownKeys(query).flatMap((fieldKey): string[] => { + const value = (query as any)[fieldKey as any]; + const field = String(fieldKey); + if (field === '$and' && Array.isArray(value)) { + return value + .map(item => this.renderSimpleWhere(item as Indexable).replace(/^ WHERE /, '')) + .filter(Boolean); + } + if ( + typeof fieldKey === 'symbol' && + fieldKey.description === 'and' && + Array.isArray(value) + ) { + return value + .map(item => this.renderSimpleWhere(item as Indexable).replace(/^ WHERE /, '')) + .filter(Boolean); + } + if (typeof fieldKey === 'symbol') { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Unsupported vector search filter operator', + ); + } + const inValues = this.extractInValues(value); + if (field === '_id' && inValues) { + const ids = inValues.map(id => this.sequelize.escape(String(id))); + return ids.length + ? [`${this.quoteIdentifier(field)} IN (${ids.join(', ')})`] + : []; + } + if (value === null) { + return [`${this.quoteIdentifier(field)} IS NULL`]; + } + if (typeof value === 'boolean') { + return [`${this.quoteIdentifier(field)} = ${value ? 'TRUE' : 'FALSE'}`]; + } + if (value === null || ['string', 'number', 'boolean'].includes(typeof value)) { + return [ + `${this.quoteIdentifier(field)} = ${this.sequelize.escape(value as string | number)}`, + ]; + } + if (value && typeof value === 'object') { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Unsupported vector search filter shape', + ); + } + return []; + }); + return clauses.length ? ` WHERE ${clauses.join(' AND ')}` : ''; + } + + private extractInValues(value: unknown): unknown[] | null { + if (!value || typeof value !== 'object') return null; + if ('$in' in value) return (value as { $in: unknown[] }).$in; + const inSymbol = Object.getOwnPropertySymbols(value).find( + symbol => symbol.description === 'in' || symbol.toString() === 'Symbol(in)', + ); + return inSymbol ? ((value as any)[inSymbol] as unknown[]) : null; + } + + private buildVectorSelectList(schema: ConduitDatabaseSchema, select?: string) { + const fields = schema.compiledFields ?? schema.fields; + const hiddenFields = new Set( + Object.entries(fields) + .filter(([, field]: [string, any]) => field?.select === false) + .map(([field]) => field), + ); + const availableFields = Object.keys(fields).filter(field => !hiddenFields.has(field)); + const tokens = select?.split(' ').filter(Boolean) ?? []; + const includeTokens = tokens.filter(token => !token.startsWith('-')); + const selected = new Set(includeTokens.length ? includeTokens : availableFields); + tokens + .filter(token => token.startsWith('-')) + .map(token => token.slice(1)) + .forEach(field => selected.delete(field)); + hiddenFields.forEach(field => selected.delete(field)); + if (!selected.size) selected.add('_id'); + return [...selected].map(field => this.quoteIdentifier(field)).join(', '); + } + + private fromPostgresVectorIndex( + name: string, + definition: string, + ): VectorIndexDefinition { + const method = /USING\s+(\w+)/i.exec(definition)?.[1] as + | VectorIndexMethod + | undefined; + const field = /\((?:"([^"]+)"|(\w+))\s+vector_/i.exec(definition); + const operator = /vector_(l2|cosine|ip)_ops/i.exec(definition)?.[1]; + return { + name, + field: field?.[1] ?? field?.[2] ?? '', + dimensions: 0, + similarity: (operator === 'l2' + ? 'euclidean' + : operator === 'ip' + ? 'dotProduct' + : 'cosine') as any, + method: method as VectorIndexMethod | undefined, + }; + } + private checkAndConvertIndexes( schemaName: string, indexes: ModelOptionsIndexes[], diff --git a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts index 8c023fd7a..20dafc8cd 100644 --- a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts @@ -94,6 +94,8 @@ function extractType(type: string, sqlType?: SQLDataType) { } case 'JSON': return DataTypes.JSONB; + case 'Vector': + return (DataTypes as any).VECTOR; case 'Relation': case 'ObjectId': return DataTypes.UUID; @@ -152,6 +154,9 @@ function extractObjectType(objectField: Indexable): res.type = extractArrayType(objectField.type).type; } else { res.type = extractType(objectField.type, objectField.sqlType); + if (objectField.type === 'Vector') { + res.type = res.type(objectField.dimensions); + } } if (objectField.hasOwnProperty('default')) { res.defaultValue = checkDefaultValue(objectField.type, objectField.default); diff --git a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/index.ts index a3aa471f7..9d09b79f0 100644 --- a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/index.ts @@ -1,12 +1,23 @@ import { SequelizeAdapter } from '../index.js'; +import pgvector from 'pgvector/sequelize'; +import { Sequelize } from 'sequelize'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; export class PostgresAdapter extends SequelizeAdapter { constructor(connectionUri: string) { + pgvector.registerTypes(Sequelize); super(connectionUri); } + protected async ensureConnected() { + await super.ensureConnected(); + await this.sequelize.query('CREATE EXTENSION IF NOT EXISTS vector').catch(() => { + // Capability probing reports missing privileges/extension support later. Keeping + // startup alive lets non-vector schemas continue to work on restricted Postgres. + }); + } + protected async hasLegacyCollections() { const res = await this.sequelize .query( diff --git a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts index fc869ca37..7f418d43a 100644 --- a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts @@ -92,6 +92,8 @@ function extractType(type: string, sqlType?: SQLDataType) { } case 'JSON': return DataTypes.JSON; + case 'Vector': + return DataTypes.JSON; case 'Relation': case 'ObjectId': return DataTypes.UUID; diff --git a/modules/database/src/adapters/sequelize-adapter/utils/sqlTypeMap.ts b/modules/database/src/adapters/sequelize-adapter/utils/sqlTypeMap.ts index 9def07866..80179feb7 100644 --- a/modules/database/src/adapters/sequelize-adapter/utils/sqlTypeMap.ts +++ b/modules/database/src/adapters/sequelize-adapter/utils/sqlTypeMap.ts @@ -12,4 +12,5 @@ export const sqlDataTypeMap = new Map([ [SQLDataType.TIME, 'Date'], [SQLDataType.DATETIME, 'Date'], [SQLDataType.TIMESTAMP, 'Date'], + [SQLDataType.VECTOR, 'Vector'], ]); diff --git a/modules/database/src/adapters/utils/validateFieldConstraints.ts b/modules/database/src/adapters/utils/validateFieldConstraints.ts index b04f2593a..6b2b7f71a 100644 --- a/modules/database/src/adapters/utils/validateFieldConstraints.ts +++ b/modules/database/src/adapters/utils/validateFieldConstraints.ts @@ -36,6 +36,18 @@ export function fieldsValidator( ); } + if ((target as ConduitModelField & { type?: string }).type === 'Vector') { + const dimensions = (target as ConduitModelField & { dimensions?: number }) + .dimensions; + if (!Number.isInteger(dimensions) || dimensions! <= 0) { + throw new ConduitError( + 'INVALID_ARGUMENTS', + 400, + `Schema '${schemaName}' vector field '${f}' requires a positive integer 'dimensions' value.`, + ); + } + } + if (target.hasOwnProperty('type') && typeof target.type === 'object') { if (Array.isArray(target.type)) { if ((target.type as unknown[]).length !== 1) { diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index a4c1be91f..5a4605e9f 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -677,6 +677,91 @@ export class AdminHandlers { new ConduitRouteReturnDefinition('deleteIndexes', 'String'), this.schemaAdmin.deleteIndexes.bind(this.schemaAdmin), ); + this.routingManager.route( + { + path: '/vector/capabilities', + action: ConduitRouteActions.GET, + description: `Returns vector storage, index, and search capabilities for the active database.`, + queryParams: { + schemaName: ConduitString.Optional, + }, + }, + new ConduitRouteReturnDefinition('getVectorCapabilities', { + supported: { type: TYPE.Boolean, required: true }, + storage: { type: TYPE.Boolean, required: true }, + indexing: { type: TYPE.Boolean, required: true }, + search: { type: TYPE.Boolean, required: true }, + provider: ConduitString.Required, + reason: ConduitString.Optional, + }), + this.schemaAdmin.getVectorCapabilities.bind(this.schemaAdmin), + ); + this.routingManager.route( + { + path: '/schemas/:id/vector-indexes', + action: ConduitRouteActions.POST, + description: `Creates a vector index for a schema.`, + urlParams: { + id: { type: TYPE.String, required: true }, + }, + bodyParams: { + index: ConduitJson.Required, + }, + } as any, + new ConduitRouteReturnDefinition('createVectorIndex', 'String'), + this.schemaAdmin.createVectorIndex.bind(this.schemaAdmin), + ); + this.routingManager.route( + { + path: '/schemas/:id/vector-indexes', + action: ConduitRouteActions.GET, + description: `Returns vector indexes of a schema.`, + urlParams: { + id: { type: TYPE.String, required: true }, + }, + }, + new ConduitRouteReturnDefinition('getVectorIndexes', { + indexes: [ConduitJson.Required], + }), + this.schemaAdmin.getVectorIndexes.bind(this.schemaAdmin), + ); + this.routingManager.route( + { + path: '/schemas/:id/vector-indexes/:indexName', + action: ConduitRouteActions.DELETE, + description: `Deletes a vector index of a schema.`, + urlParams: { + id: { type: TYPE.String, required: true }, + indexName: ConduitString.Required, + }, + }, + new ConduitRouteReturnDefinition('deleteVectorIndex', 'String'), + this.schemaAdmin.deleteVectorIndex.bind(this.schemaAdmin), + ); + this.routingManager.route( + { + path: '/schemas/:schemaName/vector-search', + action: ConduitRouteActions.POST, + description: `Runs vector search for a schema.`, + urlParams: { + schemaName: ConduitString.Required, + }, + bodyParams: { + field: ConduitString.Required, + vector: [ConduitNumber.Required], + indexName: ConduitString.Optional, + filter: ConduitJson.Optional, + limit: ConduitNumber.Optional, + numCandidates: ConduitNumber.Optional, + select: ConduitString.Optional, + scope: ConduitString.Optional, + }, + } as any, + new ConduitRouteReturnDefinition('vectorSearch', { + results: [ConduitJson.Required], + }), + this.schemaAdmin.vectorSearch.bind(this.schemaAdmin), + ); this.routingManager.route( { path: '/database-type', diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index 9c149d6db..4ea29b4e0 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -684,6 +684,62 @@ export class SchemaAdmin { return this.database.getDatabaseType(); } + async getVectorCapabilities( + call: ParsedRouterRequest, + ): Promise { + return this.database.getVectorCapabilities(call.request.params.schemaName); + } + + async createVectorIndex(call: ParsedRouterRequest): Promise { + const { id, index } = call.request.params; + const requestedSchema = await this.database + .getSchemaModel('_DeclaredSchema') + .model.findOne({ _id: id }); + if (isNil(requestedSchema)) { + throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); + } + return this.database.createVectorIndex(requestedSchema.name, index); + } + + async getVectorIndexes(call: ParsedRouterRequest): Promise { + const id = call.request.params.id; + const requestedSchema = await this.database + .getSchemaModel('_DeclaredSchema') + .model.findOne({ _id: id }); + if (isNil(requestedSchema)) { + throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); + } + return { indexes: await this.database.getVectorIndexes(requestedSchema.name) }; + } + + async deleteVectorIndex(call: ParsedRouterRequest): Promise { + const { id, indexName } = call.request.params; + const requestedSchema = await this.database + .getSchemaModel('_DeclaredSchema') + .model.findOne({ _id: id }); + if (isNil(requestedSchema)) { + throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); + } + return this.database.deleteVectorIndex(requestedSchema.name, indexName); + } + + async vectorSearch(call: ParsedRouterRequest): Promise { + const { schemaName } = call.request.params; + const results = await this.database.vectorSearch({ + schemaName, + field: call.request.params.field, + vector: call.request.params.vector, + indexName: call.request.params.indexName, + filter: call.request.params.filter, + limit: call.request.params.limit, + numCandidates: call.request.params.numCandidates, + select: call.request.params.select, + userId: call.request.context.user?._id, + scope: call.request.params.scope, + }); + return { results }; + } + async createIndexes(call: ParsedRouterRequest): Promise { const { id, indexes } = call.request.params; const requestedSchema = await this.database diff --git a/modules/database/src/database.proto b/modules/database/src/database.proto index 7ab8c2e91..14e95f88d 100644 --- a/modules/database/src/database.proto +++ b/modules/database/src/database.proto @@ -156,6 +156,60 @@ message GetDatabaseTypeResponse { string result = 1; } +message VectorCapabilitiesRequest { + optional string schemaName = 1; +} + +message VectorCapabilitiesResponse { + bool supported = 1; + bool storage = 2; + bool indexing = 3; + bool search = 4; + string provider = 5; + optional string reason = 6; +} + +message VectorIndex { + string field = 1; + int32 dimensions = 2; + string similarity = 3; + optional string name = 4; + optional string method = 5; + repeated string filterFields = 6; + optional string options = 7; +} + +message VectorIndexRequest { + string schemaName = 1; + VectorIndex index = 2; +} + +message VectorIndexListRequest { + string schemaName = 1; +} + +message VectorIndexListResponse { + repeated VectorIndex indexes = 1; +} + +message DeleteVectorIndexRequest { + string schemaName = 1; + string indexName = 2; +} + +message VectorSearchRequest { + string schemaName = 1; + string field = 2; + repeated double vector = 3; + optional string indexName = 4; + optional string filter = 5; + optional int32 limit = 6; + optional int32 numCandidates = 7; + optional string select = 8; + optional string userId = 9; + optional string scope = 10; +} + service DatabaseProvider { rpc CreateSchemaFromAdapter(CreateSchemaRequest) returns (Schema); rpc GetSchema(GetSchemaRequest) returns (Schema); @@ -182,4 +236,9 @@ service DatabaseProvider { rpc createView(CreateViewRequest) returns (google.protobuf.Empty); rpc deleteView(DeleteViewRequest) returns (google.protobuf.Empty); rpc columnExistence(ColumnExistenceRequest) returns (ColumnExistenceResponse); + rpc getVectorCapabilities(VectorCapabilitiesRequest) returns (VectorCapabilitiesResponse); + rpc createVectorIndex(VectorIndexRequest) returns (QueryResponse); + rpc getVectorIndexes(VectorIndexListRequest) returns (VectorIndexListResponse); + rpc deleteVectorIndex(DeleteVectorIndexRequest) returns (QueryResponse); + rpc vectorSearch(VectorSearchRequest) returns (QueryResponse); } diff --git a/modules/database/src/interfaces/SchemaFieldTypes.ts b/modules/database/src/interfaces/SchemaFieldTypes.ts index 7df176006..938ce0088 100644 --- a/modules/database/src/interfaces/SchemaFieldTypes.ts +++ b/modules/database/src/interfaces/SchemaFieldTypes.ts @@ -23,7 +23,7 @@ export const SchemaField = { type: 'String', required: true, description: - 'Field type. One of: String, Number, Boolean, Date, ObjectId, JSON, Relation', + 'Field type. One of: String, Number, Boolean, Date, ObjectId, JSON, Relation, Vector', }, required: ConduitBoolean.Optional, unique: ConduitBoolean.Optional, @@ -35,6 +35,16 @@ export const SchemaField = { required: false, description: 'Required when type is "Relation". The name of the related schema.', }, + dimensions: { + type: 'Number', + required: false, + description: 'Required when type is "Vector". The number of embedding dimensions.', + }, + similarity: { + type: 'String', + required: false, + description: 'Optional for Vector fields. One of: cosine, euclidean, dotProduct.', + }, }; /** @@ -46,23 +56,26 @@ export const SchemaField = { */ export const SchemaFieldsDescription = `Object mapping field names to field definitions. -**Field Types:** String, Number, Boolean, Date, ObjectId, JSON, Relation +**Field Types:** String, Number, Boolean, Date, ObjectId, JSON, Relation, Vector **Definition Formats:** - Shorthand: \`{ fieldName: "String" }\` - Object: \`{ fieldName: { type: "String", required: true } }\` - Array: \`{ fieldName: ["String"] }\` or \`{ fieldName: [{ type: "String" }] }\` - Relation: \`{ fieldName: { type: "Relation", model: "SchemaName" } }\` +- Vector: \`{ fieldName: { type: "Vector", dimensions: 1536, similarity: "cosine", select: false } }\` - Nested: \`{ fieldName: { nestedField: { type: "String" } } }\` **Field Properties:** -- \`type\` (required): String | Number | Boolean | Date | ObjectId | JSON | Relation +- \`type\` (required): String | Number | Boolean | Date | ObjectId | JSON | Relation | Vector - \`required\` (optional): boolean - Whether the field is required - \`unique\` (optional): boolean - Whether values must be unique (requires required: true) - \`select\` (optional): boolean - Whether to include in query results by default - \`default\` (optional): string - Default value for the field - \`description\` (optional): string - Field description - \`model\` (required for Relation): string - Name of the related schema +- \`dimensions\` (required for Vector): number - Embedding vector dimensions +- \`similarity\` (optional for Vector): cosine | euclidean | dotProduct **Example:** \`\`\`json @@ -71,6 +84,7 @@ export const SchemaFieldsDescription = `Object mapping field names to field defi "price": { "type": "Number", "required": true }, "description": "String", "category": { "type": "Relation", "model": "Category" }, + "embedding": { "type": "Vector", "dimensions": 1536, "similarity": "cosine", "select": false }, "tags": ["String"], "metadata": { "key": "String", "value": "String" } } diff --git a/modules/embeddings/README.md b/modules/embeddings/README.md new file mode 100644 index 000000000..cdd112e81 --- /dev/null +++ b/modules/embeddings/README.md @@ -0,0 +1,41 @@ +# Embeddings Module + +The Embeddings module owns text-to-vector generation, embedding configuration, +backfills, and semantic search by text. The Database module remains responsible +for vector storage, index creation, and vector-in/vector-out search. + +## Configuration + +The module is disabled by default. Enable it and configure an OpenAI-compatible +provider: + +```json +{ + "enabled": true, + "defaultProvider": "openai-compatible", + "providers": { + "openai-compatible": { + "endpoint": "https://api.openai.com/v1/embeddings", + "apiKey": "..." + } + }, + "queue": { + "concurrency": 2, + "attempts": 3 + } +} +``` + +## Workflow + +1. Create an embedding config with `schemaName`, `sourceFields`, `targetField`, + `provider`, `model`, and `dimensions`. +2. The module adds a vector schema extension for the target field and a source + hash field used to skip unchanged documents. +3. Start a backfill, or rely on database create/update events to enqueue + incremental embedding jobs. +4. Use `semanticSearch` to generate a query embedding and delegate search to the + Database module. + +Provider output dimensions must match the configured vector dimensions. Mismatches +fail before vectors are written or searched. diff --git a/modules/embeddings/build.sh b/modules/embeddings/build.sh new file mode 100644 index 000000000..41bfc3986 --- /dev/null +++ b/modules/embeddings/build.sh @@ -0,0 +1,19 @@ +rm -rf ./src/protoTypes +mkdir ./src/protoTypes + +cp ./src/*.proto ./src/protoTypes + +cd ./src/protoTypes || exit + +echo "Generating typescript code" +protoc \ + --plugin=protoc-gen-ts_proto=../../node_modules/.bin/protoc-gen-ts_proto \ + --ts_proto_opt=esModuleInterop=true \ + --ts_proto_opt=outputServices=generic-definitions,useExactTypes=false \ + --ts_proto_out=./ \ + --ts_proto_opt=importSuffix=.js \ + --ts_proto_opt=snakeToCamel=false \ + ./*.proto + +echo "Cleaning up folders" +rm -rf ./*.proto diff --git a/modules/embeddings/package.json b/modules/embeddings/package.json new file mode 100644 index 000000000..5145ab715 --- /dev/null +++ b/modules/embeddings/package.json @@ -0,0 +1,45 @@ +{ + "name": "@conduitplatform/embeddings", + "version": "1.0.0", + "description": "", + "private": true, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "type": "module", + "engines": { + "node": ">=24" + }, + "conduit": { + "peers": { + "await": ["database"], + "watch": [{ "module": "database", "edge": "rising" }] + } + }, + "scripts": { + "start": "node dist/index.js", + "prebuild": "npm run generateTypes", + "build": "rimraf dist && tsc", + "postbuild": "copyfiles -u 1 src/**/*.proto src/*.proto ./dist/", + "generateTypes": "sh build.sh" + }, + "dependencies": { + "@bufbuild/protobuf": "^2.10.2", + "@conduitplatform/grpc-sdk": "workspace:*", + "@conduitplatform/module-tools": "workspace:*", + "@grpc/grpc-js": "^1.14.3", + "@grpc/proto-loader": "^0.8.0", + "bullmq": "^5.21.2", + "convict": "^6.2.5", + "ioredis": "^5.10.1", + "lodash-es": "^4.18.1" + }, + "devDependencies": { + "@types/convict": "^6.1.6", + "@types/lodash-es": "^4.17.12", + "@types/node": "24.9.1", + "copyfiles": "^2.4.1", + "rimraf": "^6.1.3", + "ts-proto": "^2.11.6", + "typescript": "~6.0.2" + } +} diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts new file mode 100644 index 000000000..b1afa3759 --- /dev/null +++ b/modules/embeddings/src/Embeddings.ts @@ -0,0 +1,336 @@ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + ConduitGrpcSdk, + DatabaseProvider, + GrpcRequest, + GrpcResponse, + HealthCheckStatus, + TYPE, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { + ConfigController, + ConduitActiveSchema, + ManagedModule, +} from '@conduitplatform/module-tools'; +import AppConfigSchema, { Config } from './config/index.js'; +import * as models from './models/index.js'; +import { EmbeddingConfig } from './models/index.js'; +import { QueueController } from './controllers/queue.controller.js'; +import { getProvider, hashEmbeddingInput } from './providers/index.js'; +import metricsSchema from './metrics/index.js'; +import { + BackfillRequest, + EmbeddingConfigRequest, + EmbeddingConfigResponse, + EmbeddingsQueryResponse, + SemanticSearchRequest, +} from './protoTypes/embeddings.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +export default class EmbeddingsModule extends ManagedModule { + configSchema = AppConfigSchema; + protected metricsSchema = metricsSchema; + service = { + protoPath: path.resolve(__dirname, 'embeddings.proto'), + protoDescription: 'embeddings.EmbeddingsProvider', + functions: { + upsertConfig: this.upsertConfig.bind(this), + getConfigs: this.getConfigs.bind(this), + startBackfill: this.startBackfill.bind(this), + semanticSearch: this.semanticSearch.bind(this), + }, + }; + + private database: DatabaseProvider; + private queueController: QueueController; + private subscribedSchemas = new Set(); + + constructor(peerManifestRoot?: string) { + super('embeddings', peerManifestRoot); + this.updateHealth(HealthCheckStatus.UNKNOWN, true); + } + + async onServerStart() { + await this.awaitPeersFromManifest(); + this.database = this.grpcSdk.database!; + await this.registerSchemas(); + this.queueController = QueueController.getInstance(this.grpcSdk); + await this.configureRuntime(); + this.updateHealth(HealthCheckStatus.SERVING); + } + + async onConfig() { + if (!this.database) return; + await this.configureRuntime(); + } + + async upsertConfig( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const config = this.validateConfigRequest(call.request); + const schema = await this.database.getSchema(config.schemaName); + if (!config.sourceFields.every(field => schema.fields[field])) { + return callback({ + code: 3, + message: 'All source fields must exist on the target schema', + }); + } + await this.database.setSchemaExtension({ + schemaName: config.schemaName, + fields: { + [config.targetField]: { + type: TYPE.Vector, + dimensions: config.dimensions, + similarity: config.similarity, + select: false, + }, + [`${config.targetField}SourceHash`]: { + type: TYPE.String, + required: false, + select: false, + }, + }, + }); + const model = EmbeddingConfig.getInstance(); + const existing = await model.findOne({ + schemaName: config.schemaName, + targetField: config.targetField, + }); + if (existing) { + await model.findByIdAndUpdate(existing._id, config); + } else { + await model.create({ ...config, enabled: true }); + } + this.subscribeToSchema(config.schemaName); + callback(null, { result: 'Embedding config saved' }); + } catch (err) { + callback({ code: 13, message: (err as Error).message }); + } + } + + async getConfigs( + _call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const configs = await EmbeddingConfig.getInstance().findMany({}); + callback(null, { result: JSON.stringify(configs) }); + } catch (err) { + callback({ code: 13, message: (err as Error).message }); + } + } + + async startBackfill( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const configs = await EmbeddingConfig.getInstance().findMany({ + schemaName: call.request.schemaName, + enabled: true, + }); + const batchSize = call.request.batchSize ?? 100; + const docs = await this.database.findMany>( + call.request.schemaName, + {}, + { limit: batchSize, select: '_id' }, + ); + const attempts = this.currentConfig().queue.attempts; + await this.queueController.addBulkEmbeddingJobs( + docs.flatMap(doc => + configs.map(config => ({ + schemaName: config.schemaName, + documentId: String(doc._id), + configId: config._id, + })), + ), + attempts, + ); + callback(null, { + result: JSON.stringify({ queued: docs.length * configs.length }), + }); + } catch (err) { + callback({ code: 13, message: (err as Error).message }); + } + } + + async semanticSearch( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const config = await this.resolveConfig( + call.request.schemaName, + call.request.targetField, + ); + const providerConfig = this.providerConfig(config.provider, config.modelName); + const vector = await getProvider(config.provider).embed( + call.request.text, + providerConfig, + ); + if (vector.length !== config.dimensions) { + throw new Error( + `Embedding provider returned ${vector.length} dimensions; expected ${config.dimensions}`, + ); + } + const results = await this.database.vectorSearch({ + schemaName: call.request.schemaName, + field: config.targetField, + vector, + filter: call.request.filter ? JSON.parse(call.request.filter) : undefined, + limit: call.request.limit, + userId: call.request.userId, + scope: call.request.scope, + }); + callback(null, { result: JSON.stringify(results) }); + } catch (err) { + callback({ code: 13, message: (err as Error).message }); + } + } + + private async configureRuntime() { + const config = this.currentConfig(); + if (!config.enabled) return; + this.queueController ??= QueueController.getInstance(this.grpcSdk); + this.queueController.addWorker( + data => this.processEmbeddingJob(data.schemaName, data.documentId, data.configId), + config.queue.concurrency, + ); + const configs = await EmbeddingConfig.getInstance().findMany({ enabled: true }); + configs.forEach(config => this.subscribeToSchema(config.schemaName)); + } + + private subscribeToSchema(schemaName: string) { + if (this.subscribedSchemas.has(schemaName)) return; + this.subscribedSchemas.add(schemaName); + const idPrefix = `embeddings:${schemaName}`; + this.grpcSdk.bus?.subscribe( + `database:create:${schemaName}`, + message => this.enqueueMutation(schemaName, message), + `${idPrefix}:create`, + ); + this.grpcSdk.bus?.subscribe( + `database:update:${schemaName}`, + message => this.enqueueMutation(schemaName, message), + `${idPrefix}:update`, + ); + this.grpcSdk.bus?.subscribe( + `database:createMany:${schemaName}`, + message => this.enqueueMutation(schemaName, message), + `${idPrefix}:createMany`, + ); + this.grpcSdk.bus?.subscribe( + `database:updateMany:${schemaName}`, + message => this.enqueueMutation(schemaName, message), + `${idPrefix}:updateMany`, + ); + } + + private enqueueMutation(schemaName: string, message: string) { + const attempts = this.currentConfig().queue.attempts; + const payload = JSON.parse(message); + const docs = Array.isArray(payload) ? payload : [payload]; + docs + .filter(doc => doc?._id) + .forEach(doc => { + this.queueController + .addEmbeddingJob({ schemaName, documentId: String(doc._id) }, attempts) + .catch(err => ConduitGrpcSdk.Logger.error(err)); + }); + } + + private async processEmbeddingJob( + schemaName: string, + documentId: string, + configId?: string, + ) { + const configs = configId + ? [await EmbeddingConfig.getInstance().findOne({ _id: configId })] + : await EmbeddingConfig.getInstance().findMany({ schemaName, enabled: true }); + const doc = await this.database.findOne>(schemaName, { + _id: documentId, + }); + if (!doc) return; + for (const config of configs.filter(Boolean) as EmbeddingConfig[]) { + const input = config.sourceFields.map(field => doc[field] ?? '').join('\n'); + const sourceHash = hashEmbeddingInput(input); + if (doc[`${config.targetField}SourceHash`] === sourceHash) continue; + const vector = await getProvider(config.provider).embed( + input, + this.providerConfig(config.provider, config.modelName), + ); + if (vector.length !== config.dimensions) { + throw new Error( + `Embedding provider returned ${vector.length} dimensions; expected ${config.dimensions}`, + ); + } + await this.database.findByIdAndUpdate(schemaName, documentId, { + [config.targetField]: vector, + [`${config.targetField}SourceHash`]: sourceHash, + }); + } + } + + private validateConfigRequest(request: EmbeddingConfigRequest) { + if (!request.schemaName || !request.targetField || !request.sourceFields.length) { + throw new Error('schemaName, targetField, and sourceFields are required'); + } + if (!Number.isInteger(request.dimensions) || request.dimensions <= 0) { + throw new Error('dimensions must be a positive integer'); + } + return { + schemaName: request.schemaName, + sourceFields: request.sourceFields, + targetField: request.targetField, + provider: request.provider || this.currentConfig().defaultProvider, + modelName: request.model, + dimensions: request.dimensions, + similarity: (request.similarity || VectorSimilarity.Cosine) as VectorSimilarity, + }; + } + + private async resolveConfig(schemaName: string, targetField?: string) { + const query: Record = { schemaName, enabled: true }; + if (targetField) query.targetField = targetField; + const config = await EmbeddingConfig.getInstance().findOne(query); + if (!config) throw new Error('No embedding config found for semantic search'); + return config; + } + + private providerConfig(provider: string, model: string) { + const providers = this.currentConfig().providers as Record< + string, + Record + >; + const providerConfig = providers[provider] ?? {}; + return { + ...providerConfig, + model: String(providerConfig.model ?? model), + }; + } + + private currentConfig() { + return ConfigController.getInstance().config as Config; + } + + protected registerSchemas(): Promise { + const promises = Object.values(models).map(model => { + const modelInstance = model.getInstance(this.database); + if ( + Object.keys((modelInstance as ConduitActiveSchema).fields) + .length !== 0 + ) { + return this.database + .createSchemaFromAdapter(modelInstance) + .then(() => this.database.migrate(modelInstance.name)); + } + }); + return Promise.all(promises); + } +} diff --git a/modules/embeddings/src/config/index.ts b/modules/embeddings/src/config/index.ts new file mode 100644 index 000000000..15a00d256 --- /dev/null +++ b/modules/embeddings/src/config/index.ts @@ -0,0 +1,39 @@ +import convict from 'convict'; + +const AppConfigSchema = { + doc: 'Embeddings module configuration', + enabled: { + doc: 'Enable embedding generation workers and event subscriptions', + format: 'Boolean', + default: false, + }, + defaultProvider: { + doc: 'Default embedding provider', + format: 'String', + default: 'openai-compatible', + }, + providers: { + doc: 'Embedding provider configuration keyed by provider name', + format: Object, + default: {}, + }, + queue: { + concurrency: { + doc: 'Embedding generation worker concurrency', + format: 'Number', + default: 2, + }, + attempts: { + doc: 'Embedding generation retry attempts', + format: 'Number', + default: 3, + }, + }, +}; + +const config = convict(AppConfigSchema); +const configProperties = config.getProperties(); +export type Config = typeof configProperties & { + providers: Record>; +}; +export default AppConfigSchema; diff --git a/modules/embeddings/src/controllers/queue.controller.ts b/modules/embeddings/src/controllers/queue.controller.ts new file mode 100644 index 000000000..b6344ac4d --- /dev/null +++ b/modules/embeddings/src/controllers/queue.controller.ts @@ -0,0 +1,65 @@ +import { Queue, Worker } from 'bullmq'; +import { Cluster, Redis } from 'ioredis'; +import { randomUUID } from 'node:crypto'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; + +export interface EmbeddingJobData { + schemaName: string; + documentId: string; + configId?: string; +} + +export class QueueController { + private static _instance: QueueController; + private readonly redisConnection: Redis | Cluster; + private readonly embeddingQueue: Queue; + + constructor(private readonly grpcSdk: ConduitGrpcSdk) { + this.redisConnection = this.grpcSdk.redisManager.getClient(); + this.embeddingQueue = new Queue('embeddings-generation-queue', { + connection: this.redisConnection, + }); + } + + static getInstance(grpcSdk?: ConduitGrpcSdk) { + if (QueueController._instance) return QueueController._instance; + if (!grpcSdk) throw new Error('No grpcSdk instance provided!'); + return (QueueController._instance = new QueueController(grpcSdk)); + } + + addWorker(processor: (data: EmbeddingJobData) => Promise, concurrency: number) { + const worker = new Worker( + 'embeddings-generation-queue', + job => processor(job.data), + { + concurrency, + connection: this.redisConnection, + removeOnComplete: { age: 3600, count: 1000 }, + removeOnFail: { age: 24 * 3600 }, + }, + ); + worker.on('failed', (_job, error) => ConduitGrpcSdk.Logger.error(error)); + worker.on('error', error => ConduitGrpcSdk.Logger.error(error)); + return worker; + } + + async addEmbeddingJob(data: EmbeddingJobData, attempts: number) { + await this.embeddingQueue.add(randomUUID(), data, { + attempts, + backoff: { type: 'exponential', delay: 1000 }, + }); + } + + async addBulkEmbeddingJobs(data: EmbeddingJobData[], attempts: number) { + await this.embeddingQueue.addBulk( + data.map(job => ({ + name: randomUUID(), + data: job, + opts: { + attempts, + backoff: { type: 'exponential', delay: 1000 }, + }, + })), + ); + } +} diff --git a/modules/embeddings/src/embeddings.proto b/modules/embeddings/src/embeddings.proto new file mode 100644 index 000000000..e44a3f2b2 --- /dev/null +++ b/modules/embeddings/src/embeddings.proto @@ -0,0 +1,43 @@ +syntax = 'proto3'; +import "google/protobuf/empty.proto"; +package embeddings; + +message EmbeddingConfigRequest { + string schemaName = 1; + repeated string sourceFields = 2; + string targetField = 3; + string provider = 4; + string model = 5; + int32 dimensions = 6; + optional string similarity = 7; +} + +message EmbeddingConfigResponse { + string result = 1; +} + +message BackfillRequest { + string schemaName = 1; + optional int32 batchSize = 2; +} + +message SemanticSearchRequest { + string schemaName = 1; + string text = 2; + optional string targetField = 3; + optional string filter = 4; + optional int32 limit = 5; + optional string userId = 6; + optional string scope = 7; +} + +message EmbeddingsQueryResponse { + string result = 1; +} + +service EmbeddingsProvider { + rpc upsertConfig(EmbeddingConfigRequest) returns (EmbeddingConfigResponse); + rpc getConfigs(google.protobuf.Empty) returns (EmbeddingsQueryResponse); + rpc startBackfill(BackfillRequest) returns (EmbeddingsQueryResponse); + rpc semanticSearch(SemanticSearchRequest) returns (EmbeddingsQueryResponse); +} diff --git a/modules/embeddings/src/index.ts b/modules/embeddings/src/index.ts new file mode 100644 index 000000000..b6b9fa59b --- /dev/null +++ b/modules/embeddings/src/index.ts @@ -0,0 +1,7 @@ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import EmbeddingsModule from './Embeddings.js'; + +const peerManifestRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); +const embeddings = new EmbeddingsModule(peerManifestRoot); +embeddings.start(); diff --git a/modules/embeddings/src/metrics/index.ts b/modules/embeddings/src/metrics/index.ts new file mode 100644 index 000000000..52cea73ce --- /dev/null +++ b/modules/embeddings/src/metrics/index.ts @@ -0,0 +1,18 @@ +import { MetricType } from '@conduitplatform/grpc-sdk'; + +export default { + generatedEmbeddings: { + type: MetricType.Counter, + config: { + name: 'generated_embeddings_total', + help: 'Tracks the total number of generated embeddings', + }, + }, + failedEmbeddings: { + type: MetricType.Counter, + config: { + name: 'failed_embeddings_total', + help: 'Tracks the total number of failed embedding generation attempts', + }, + }, +}; diff --git a/modules/embeddings/src/models/EmbeddingConfig.schema.ts b/modules/embeddings/src/models/EmbeddingConfig.schema.ts new file mode 100644 index 000000000..a2c26e697 --- /dev/null +++ b/modules/embeddings/src/models/EmbeddingConfig.schema.ts @@ -0,0 +1,66 @@ +import { + ConduitModel, + DatabaseProvider, + TYPE, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { ConduitActiveSchema } from '@conduitplatform/module-tools'; + +const schema: ConduitModel = { + _id: TYPE.ObjectId, + schemaName: { type: TYPE.String, required: true }, + sourceFields: { type: [TYPE.String], required: true }, + targetField: { type: TYPE.String, required: true }, + provider: { type: TYPE.String, required: true }, + modelName: { type: TYPE.String, required: true }, + dimensions: { type: TYPE.Number, required: true }, + similarity: { + type: TYPE.String, + enum: Object.values(VectorSimilarity), + default: VectorSimilarity.Cosine, + }, + enabled: { type: TYPE.Boolean, default: true }, + createdAt: TYPE.Date, + updatedAt: TYPE.Date, +}; + +const modelOptions = { + timestamps: true, + indexes: [{ fields: ['schemaName', 'targetField'], options: { unique: true } }], + conduit: { + permissions: { + extendable: false, + canCreate: false, + canModify: 'Nothing', + canDelete: false, + }, + }, +} as const; + +export class EmbeddingConfig extends ConduitActiveSchema { + private static _instance: EmbeddingConfig; + _id: string; + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + modelName: string; + dimensions: number; + similarity: VectorSimilarity; + enabled: boolean; + createdAt: Date; + updatedAt: Date; + + private constructor(database: DatabaseProvider) { + super(database, EmbeddingConfig.name, schema, modelOptions); + } + + static getInstance(database?: DatabaseProvider) { + if (EmbeddingConfig._instance) return EmbeddingConfig._instance; + if (!database) { + throw new Error('No database instance provided!'); + } + EmbeddingConfig._instance = new EmbeddingConfig(database); + return EmbeddingConfig._instance; + } +} diff --git a/modules/embeddings/src/models/index.ts b/modules/embeddings/src/models/index.ts new file mode 100644 index 000000000..9b3e3b4e4 --- /dev/null +++ b/modules/embeddings/src/models/index.ts @@ -0,0 +1 @@ +export * from './EmbeddingConfig.schema.js'; diff --git a/modules/embeddings/src/providers/index.ts b/modules/embeddings/src/providers/index.ts new file mode 100644 index 000000000..2ecf09f16 --- /dev/null +++ b/modules/embeddings/src/providers/index.ts @@ -0,0 +1,48 @@ +import { createHash } from 'node:crypto'; + +export interface EmbeddingProviderConfig { + endpoint?: string; + apiKey?: string; + model?: string; +} + +export interface EmbeddingProvider { + embed(input: string, config: EmbeddingProviderConfig): Promise; +} + +export class OpenAICompatibleEmbeddingProvider implements EmbeddingProvider { + async embed(input: string, config: EmbeddingProviderConfig): Promise { + if (!config.endpoint) { + throw new Error('Embedding provider endpoint is not configured'); + } + const response = await fetch(config.endpoint, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(config.apiKey ? { authorization: `Bearer ${config.apiKey}` } : {}), + }, + body: JSON.stringify({ + input, + model: config.model, + }), + }); + if (!response.ok) { + throw new Error(`Embedding provider failed with HTTP ${response.status}`); + } + const body = (await response.json()) as { data?: { embedding?: number[] }[] }; + const embedding = body.data?.[0]?.embedding; + if (!embedding?.length) { + throw new Error('Embedding provider response did not include an embedding'); + } + return embedding; + } +} + +export function getProvider(name: string): EmbeddingProvider { + if (name === 'openai-compatible') return new OpenAICompatibleEmbeddingProvider(); + throw new Error(`Unsupported embedding provider: ${name}`); +} + +export function hashEmbeddingInput(input: string) { + return createHash('sha256').update(input).digest('hex'); +} diff --git a/modules/embeddings/test/embedding-contract.test.mjs b/modules/embeddings/test/embedding-contract.test.mjs new file mode 100644 index 000000000..c0f985e26 --- /dev/null +++ b/modules/embeddings/test/embedding-contract.test.mjs @@ -0,0 +1,19 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { test } from 'node:test'; + +const protoSource = readFileSync(new URL('../src/embeddings.proto', import.meta.url), 'utf8'); +const readmeSource = readFileSync(new URL('../README.md', import.meta.url), 'utf8'); + +test('embeddings module exposes configuration, backfill, and semantic search RPCs', () => { + assert.match(protoSource, /rpc upsertConfig/); + assert.match(protoSource, /rpc getConfigs/); + assert.match(protoSource, /rpc startBackfill/); + assert.match(protoSource, /rpc semanticSearch/); +}); + +test('deployment docs describe provider configuration and rollout workflow', () => { + assert.match(readmeSource, /openai-compatible/); + assert.match(readmeSource, /backfill/); + assert.match(readmeSource, /semanticSearch/); +}); diff --git a/modules/embeddings/tsconfig.json b/modules/embeddings/tsconfig.json new file mode 100644 index 000000000..554dca661 --- /dev/null +++ b/modules/embeddings/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "NodeNext", + "resolveJsonModule": true, + "skipLibCheck": true, + "declaration": true, + "sourceMap": true, + "outDir": "./dist", + "rootDir": "./src", + "removeComments": true, + "strict": true, + "strictPropertyInitialization": false, + "moduleResolution": "NodeNext", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ca5262f01..5a6b5425c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -883,6 +883,9 @@ importers: pg-hstore: specifier: ^2.3.4 version: 2.3.4 + pgvector: + specifier: ^0.3.0 + version: 0.3.0 sequelize: specifier: ^6.37.8 version: 6.37.8(mariadb@3.5.4)(mysql2@3.23.1(@types/node@24.13.4))(pg-hstore@2.3.4)(pg@8.22.0)(sqlite3@6.0.1) @@ -936,6 +939,58 @@ importers: specifier: ~6.0.3 version: 6.0.3 + modules/embeddings: + dependencies: + '@bufbuild/protobuf': + specifier: ^2.10.2 + version: 2.12.0 + '@conduitplatform/grpc-sdk': + specifier: workspace:* + version: link:../../libraries/grpc-sdk + '@conduitplatform/module-tools': + specifier: workspace:* + version: link:../../libraries/module-tools + '@grpc/grpc-js': + specifier: ^1.14.3 + version: 1.14.4 + '@grpc/proto-loader': + specifier: ^0.8.0 + version: 0.8.1 + bullmq: + specifier: ^5.21.2 + version: 5.79.0 + convict: + specifier: ^6.2.5 + version: 6.2.5 + ioredis: + specifier: 5.11.1 + version: 5.11.1 + lodash-es: + specifier: ^4.18.1 + version: 4.18.1 + devDependencies: + '@types/convict': + specifier: ^6.1.6 + version: 6.1.6 + '@types/lodash-es': + specifier: ^4.17.12 + version: 4.17.12 + '@types/node': + specifier: 24.9.1 + version: 24.9.1 + copyfiles: + specifier: ^2.4.1 + version: 2.4.1 + rimraf: + specifier: ^6.1.3 + version: 6.1.3 + ts-proto: + specifier: ^2.11.6 + version: 2.12.1 + typescript: + specifier: ~6.0.2 + version: 6.0.3 + modules/functions: dependencies: '@conduitplatform/grpc-sdk': @@ -3378,6 +3433,9 @@ packages: '@types/node@24.13.4': resolution: {integrity: sha512-YJ7EqCstVTzIr0fMr7qul/977en+pQHrfmuKIo6Zr9i75Be21dr3MovcfvGtyvi2HAUrRerWps5sMO9I7WaxDw==} + '@types/node@24.9.1': + resolution: {integrity: sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg==} + '@types/nodemailer-mailgun-transport@1.4.6': resolution: {integrity: sha512-6qhtDo+1ZLtrmmpQN7O9e3NLK5ggnTS2Oca+22SvmhwChNKxDZErecTlF6qTOLnNW/CCcHmDaSmG2MXUeP1w9g==} @@ -7008,6 +7066,10 @@ packages: pgpass@1.0.5: resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + pgvector@0.3.0: + resolution: {integrity: sha512-+t7qcQD2us8fO8YIq/3lA0gUrD+bVO70MG1MhcDcxJz/OlRGGIIHzFq/4x57Vn/LpzX5wFdfOTLQp9QMPd4ljQ==} + engines: {node: '>=22'} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -8077,6 +8139,9 @@ packages: underscore@1.13.8: resolution: {integrity: sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==} + undici-types@7.16.0: + resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + undici-types@7.18.2: resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} @@ -11045,6 +11110,10 @@ snapshots: dependencies: undici-types: 7.18.2 + '@types/node@24.9.1': + dependencies: + undici-types: 7.16.0 + '@types/nodemailer-mailgun-transport@1.4.6': dependencies: '@types/nodemailer': 8.0.1 @@ -15166,6 +15235,8 @@ snapshots: dependencies: split2: 4.2.0 + pgvector@0.3.0: {} + picocolors@1.1.1: {} picomatch@2.3.2: {} @@ -16363,6 +16434,8 @@ snapshots: underscore@1.13.8: {} + undici-types@7.16.0: {} + undici-types@7.18.2: {} undici@6.27.0: From ac54ecaed192fc8aae4f301ee4ebf428e7a5d35c Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 17:11:47 +0300 Subject: [PATCH 02/29] feat(database): enforce strict vector contracts with offline tests Require object-form Vector fields and validate dimensions, similarity, and index methods. Extract testable Mongo/Postgres mappings and capability helpers. --- .../database/src/adapters/DatabaseAdapter.ts | 10 +- .../mongoose-adapter/SchemaConverter.ts | 5 +- .../src/adapters/mongoose-adapter/index.ts | 110 ++------- .../src/adapters/sequelize-adapter/index.ts | 113 ++++------ .../postgres-adapter/PgSchemaConverter.ts | 11 +- .../sql-adapter/SqlSchemaConverter.ts | 7 + .../__tests__/vectorCapabilities.test.ts | 89 ++++++++ .../utils/__tests__/vectorField.test.ts | 166 ++++++++++++++ .../utils/__tests__/vectorMappings.test.ts | 172 ++++++++++++++ modules/database/src/adapters/utils/index.ts | 3 + .../adapters/utils/validateFieldChanges.ts | 2 + .../utils/validateFieldConstraints.ts | 14 +- .../src/adapters/utils/vectorCapabilities.ts | 96 ++++++++ .../src/adapters/utils/vectorField.ts | 209 ++++++++++++++++++ .../src/adapters/utils/vectorMappings.ts | 187 ++++++++++++++++ .../src/interfaces/SchemaFieldTypes.ts | 4 +- modules/embeddings/package.json | 3 +- modules/embeddings/src/Embeddings.ts | 20 +- .../src/utils/validateEmbeddingConfig.test.ts | 58 +++++ .../src/utils/validateEmbeddingConfig.ts | 55 +++++ modules/embeddings/tsconfig.json | 4 +- modules/embeddings/tsconfig.test.json | 15 ++ 22 files changed, 1149 insertions(+), 204 deletions(-) create mode 100644 modules/database/src/adapters/utils/__tests__/vectorCapabilities.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/vectorField.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts create mode 100644 modules/database/src/adapters/utils/vectorCapabilities.ts create mode 100644 modules/database/src/adapters/utils/vectorField.ts create mode 100644 modules/database/src/adapters/utils/vectorMappings.ts create mode 100644 modules/embeddings/src/utils/validateEmbeddingConfig.test.ts create mode 100644 modules/embeddings/src/utils/validateEmbeddingConfig.ts create mode 100644 modules/embeddings/tsconfig.test.json diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index 7e3801ff6..e9328bf76 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -15,6 +15,7 @@ import { } from '@conduitplatform/grpc-sdk'; import { ConfigController } from '@conduitplatform/module-tools'; import type { Config } from '../config/index.js'; +import { unsupportedVectorCapabilities } from './utils/vectorCapabilities.js'; import { _ConduitSchema, ConduitDatabaseSchema, @@ -301,14 +302,7 @@ export abstract class DatabaseAdapter { ): Promise; getVectorCapabilities(_schemaName?: string): Promise { - return Promise.resolve({ - supported: false, - storage: false, - indexing: false, - search: false, - provider: 'unsupported', - reason: `${this.getDatabaseType()} does not support Conduit vector search`, - }); + return Promise.resolve(unsupportedVectorCapabilities(this.getDatabaseType())); } createVectorIndex(_schemaName: string, _index: VectorIndexDefinition): Promise { diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index 35f47913d..dead468dd 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -8,6 +8,7 @@ import { } from '@conduitplatform/grpc-sdk'; import { cloneDeep, isArray, isNil, isObject } from 'lodash-es'; import { checkIfMongoOptions } from './utils.js'; +import { applyMongoVectorField, isVectorSchemaType } from '../utils/vectorMappings.js'; import * as deepdash from 'deepdash-es/standalone'; @@ -87,8 +88,8 @@ function convert(value: any, key: any, parentValue: any) { parentValue[key].type = Schema.Types.Mixed; } - if (parentValue[key]?.type === 'Vector') { - parentValue[key].type = [Number]; + if (isVectorSchemaType(parentValue[key]?.type)) { + parentValue[key] = applyMongoVectorField(parentValue[key]); } if (!isNil(parentValue[key]) && parentValue[key] === 'JSON') { diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index fd9ac3cba..3e59c7023 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -15,7 +15,15 @@ import { VectorSearchResult, } from '@conduitplatform/grpc-sdk'; import { DatabaseAdapter } from '../DatabaseAdapter.js'; -import { validateFieldChanges, validateFieldConstraints } from '../utils/index.js'; +import { + validateFieldChanges, + validateFieldConstraints, + assertVectorIndexContract, + assertVectorIndexMatchesField, + mongoVectorCapabilities, + fromMongoVectorIndex, + toMongoVectorIndexDefinition, +} from '../utils/index.js'; import pluralize from '../../utils/pluralize.js'; import { mongoSchemaConverter } from '../../introspection/mongoose/utils.js'; import { status } from '@grpc/grpc-js'; @@ -727,45 +735,24 @@ export class MongooseAdapter extends DatabaseAdapter { async getVectorCapabilities(schemaName?: string): Promise { const modelName = schemaName ?? Object.keys(this.models)[0]; if (!modelName || !this.models[modelName]) { - return { - supported: true, - storage: true, - indexing: false, - search: false, - provider: 'mongodb', - reason: 'No schema is available to probe MongoDB Vector Search support', - }; + return mongoVectorCapabilities({ hasSchema: false }); } try { const collection: any = this.mongoose.model(modelName).collection; if (typeof collection.listSearchIndexes !== 'function') { - return { - supported: true, - storage: true, - indexing: false, - search: false, - provider: 'mongodb', - reason: 'MongoDB driver does not expose search index commands', - }; + return mongoVectorCapabilities({ + hasSchema: true, + searchIndexCommandsAvailable: false, + }); } await collection.listSearchIndexes().toArray(); - return { - supported: true, - storage: true, - indexing: true, - search: true, - provider: 'mongodb', - }; + return mongoVectorCapabilities({ hasSchema: true }); } catch (err) { - return { - supported: true, - storage: true, - indexing: false, - search: false, - provider: 'mongodb', - reason: (err as Error).message, - }; + return mongoVectorCapabilities({ + hasSchema: true, + probeError: (err as Error).message, + }); } } @@ -776,6 +763,7 @@ export class MongooseAdapter extends DatabaseAdapter { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); this.validateVectorField(schemaName, index); + assertVectorIndexContract('mongodb', index); const collection: any = this.mongoose.model(schemaName).collection; if (typeof collection.createSearchIndex !== 'function') { throw new GrpcError( @@ -786,7 +774,7 @@ export class MongooseAdapter extends DatabaseAdapter { await collection.createSearchIndex({ name: index.name ?? `${index.field}_vector`, type: 'vectorSearch', - definition: this.toMongoVectorIndexDefinition(index), + definition: toMongoVectorIndexDefinition(index), }); return 'Vector index created!'; } @@ -799,7 +787,7 @@ export class MongooseAdapter extends DatabaseAdapter { const indexes = await collection.listSearchIndexes().toArray(); return indexes .filter((index: any) => index.type === 'vectorSearch') - .map((index: any) => this.fromMongoVectorIndex(index)); + .map((index: any) => fromMongoVectorIndex(index)); } async deleteVectorIndex(schemaName: string, indexName: string): Promise { @@ -960,59 +948,7 @@ export class MongooseAdapter extends DatabaseAdapter { private validateVectorField(schemaName: string, index: VectorIndexDefinition) { const schema = this.models[schemaName].originalSchema as any; const field = schema.compiledFields?.[index.field] ?? schema.fields?.[index.field]; - if (!field || field.type !== 'Vector') { - throw new GrpcError(status.INVALID_ARGUMENT, 'Vector index field is not a vector'); - } - if (field.dimensions !== index.dimensions) { - throw new GrpcError( - status.INVALID_ARGUMENT, - `Vector index dimensions mismatch: field ${field.dimensions}, index ${index.dimensions}`, - ); - } - } - - private toMongoVectorIndexDefinition(index: VectorIndexDefinition) { - const vectorField: any = { - type: 'vector', - path: index.field, - numDimensions: index.dimensions, - similarity: index.similarity, - }; - if (index.options?.quantization) - vectorField.quantization = index.options.quantization; - if (index.method) vectorField.indexingMethod = index.method; - if (index.options?.hnsw) { - vectorField.hnswOptions = { - ...(index.options.hnsw.maxEdges && { maxEdges: index.options.hnsw.maxEdges }), - ...(index.options.hnsw.numEdgeCandidates && { - numEdgeCandidates: index.options.hnsw.numEdgeCandidates, - }), - }; - } - return { - fields: [ - vectorField, - ...(index.filterFields ?? []).map((path: string) => ({ type: 'filter', path })), - ], - ...(index.options?.storedSource !== undefined && { - storedSource: index.options.storedSource, - }), - }; - } - - private fromMongoVectorIndex(index: any): VectorIndexDefinition { - const fields = index.latestDefinition?.fields ?? index.definition?.fields ?? []; - const vectorField = fields.find((field: any) => field.type === 'vector') ?? {}; - return { - name: index.name, - field: vectorField.path, - dimensions: vectorField.numDimensions, - similarity: vectorField.similarity, - method: vectorField.indexingMethod, - filterFields: fields - .filter((field: any) => field.type === 'filter') - .map((field: any) => field.path), - }; + assertVectorIndexMatchesField(field, index); } private buildVectorProjection(schema: ConduitDatabaseSchema, select?: string) { diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 176271b65..739498972 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -12,7 +12,6 @@ import { UntypedArray, VectorCapabilities, VectorIndexDefinition, - VectorIndexMethod, VectorSearchInput, VectorSearchResult, } from '@conduitplatform/grpc-sdk'; @@ -34,6 +33,21 @@ import { import { sqlSchemaConverter } from './sql-adapter/SqlSchemaConverter.js'; import { pgSchemaConverter } from './postgres-adapter/PgSchemaConverter.js'; import { isEqual, isNil } from 'lodash-es'; +import { + assertVectorIndexContract, + assertVectorIndexMatchesField, +} from '../utils/vectorField.js'; +import { + postgresVectorCapabilities, + sqlFallbackVectorCapabilities, +} from '../utils/vectorCapabilities.js'; +import { + fromPostgresVectorIndex, + pgVectorDistanceOperator, + pgVectorOperator, + postgresIndexMethodSql, + resolveVectorFieldFromSchema, +} from '../utils/vectorMappings.js'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; @@ -429,35 +443,17 @@ export abstract class SequelizeAdapter extends DatabaseAdapter async getVectorCapabilities(schemaName?: string): Promise { if (this.sequelize.getDialect() !== 'postgres') { - return { - supported: false, - storage: false, - indexing: false, - search: false, - provider: 'unsupported', - reason: `${this.sequelize.getDialect()} does not support Conduit vector search`, - }; + return sqlFallbackVectorCapabilities(this.sequelize.getDialect()); } try { await this.sequelize.query("SELECT 'vector'::regtype"); - return { - supported: true, - storage: true, - indexing: true, - search: true, - provider: 'postgres', - }; + return postgresVectorCapabilities({ pgvectorAvailable: true }); } catch (err) { - return { - supported: true, - storage: false, - indexing: false, - search: false, - provider: 'postgres', - reason: schemaName - ? `Schema ${schemaName} cannot use pgvector: ${(err as Error).message}` - : (err as Error).message, - }; + return postgresVectorCapabilities({ + pgvectorAvailable: false, + error: (err as Error).message, + schemaName, + }); } } @@ -467,12 +463,13 @@ export abstract class SequelizeAdapter extends DatabaseAdapter ): Promise { this.ensurePostgresVectorSupport(schemaName); this.validateVectorField(schemaName, index); + assertVectorIndexContract('postgres', index); const tableName = this.getPhysicalTableName(schemaName); const indexName = this.quoteIdentifier( index.name ?? `${tableName}_${index.field}_vector`, ); - const method = index.method === VectorIndexMethod.IVFFlat ? 'ivfflat' : 'hnsw'; - const operator = this.pgVectorOperator(index.similarity); + const method = postgresIndexMethodSql(index.method); + const operator = pgVectorOperator(index.similarity); const withOptions = method === 'ivfflat' ? this.renderWithOptions({ lists: index.options?.ivfflat?.lists }) @@ -496,9 +493,21 @@ export abstract class SequelizeAdapter extends DatabaseAdapter tableName, )}`, ); + const schema = this.models[schemaName]?.originalSchema; + const schemaFields = (schema?.compiledFields ?? schema?.fields) as + Record | undefined; return (rows[0] as any[]) .filter(row => /USING (hnsw|ivfflat)/i.test(row.indexdef)) - .map(row => this.fromPostgresVectorIndex(row.indexname, row.indexdef)); + .map(row => { + const mapped = fromPostgresVectorIndex(row.indexname, row.indexdef); + const field = resolveVectorFieldFromSchema(schemaFields, mapped.field); + if (!field) return mapped; + return { + ...mapped, + dimensions: field.dimensions, + similarity: field.similarity ?? mapped.similarity, + }; + }); } async deleteVectorIndex(schemaName: string, indexName: string): Promise { @@ -530,7 +539,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter ); if (isNil(authorizedQuery)) return []; const tableName = this.getPhysicalTableName(request.schemaName); - const distance = this.pgVectorDistanceOperator(field.similarity ?? 'cosine'); + const distance = pgVectorDistanceOperator(field.similarity ?? 'cosine'); const where = this.renderSimpleWhere(authorizedQuery); const limit = Math.max(1, Math.min(request.limit ?? 10, 1000)); const vector = `[${request.vector.join(',')}]`; @@ -610,15 +619,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter const schema = this.models[schemaName].originalSchema; const field = (schema.compiledFields?.[index.field] ?? schema.fields?.[index.field]) as any; - if (!field || field.type !== 'Vector') { - throw new GrpcError(status.INVALID_ARGUMENT, 'Vector index field is not a vector'); - } - if (field.dimensions !== index.dimensions) { - throw new GrpcError( - status.INVALID_ARGUMENT, - `Vector index dimensions mismatch: field ${field.dimensions}, index ${index.dimensions}`, - ); - } + assertVectorIndexMatchesField(field, index); } private getPhysicalTableName(schemaName: string) { @@ -629,18 +630,6 @@ export abstract class SequelizeAdapter extends DatabaseAdapter return `"${identifier.replace(/"/g, '""')}"`; } - private pgVectorOperator(similarity: string) { - if (similarity === 'euclidean') return 'vector_l2_ops'; - if (similarity === 'dotProduct') return 'vector_ip_ops'; - return 'vector_cosine_ops'; - } - - private pgVectorDistanceOperator(similarity: string) { - if (similarity === 'euclidean') return '<->'; - if (similarity === 'dotProduct') return '<#>'; - return '<=>'; - } - private renderWithOptions(options: Record) { const entries = Object.entries(options).filter((entry): entry is [string, number] => Number.isFinite(entry[1]), @@ -731,28 +720,6 @@ export abstract class SequelizeAdapter extends DatabaseAdapter return [...selected].map(field => this.quoteIdentifier(field)).join(', '); } - private fromPostgresVectorIndex( - name: string, - definition: string, - ): VectorIndexDefinition { - const method = /USING\s+(\w+)/i.exec(definition)?.[1] as - | VectorIndexMethod - | undefined; - const field = /\((?:"([^"]+)"|(\w+))\s+vector_/i.exec(definition); - const operator = /vector_(l2|cosine|ip)_ops/i.exec(definition)?.[1]; - return { - name, - field: field?.[1] ?? field?.[2] ?? '', - dimensions: 0, - similarity: (operator === 'l2' - ? 'euclidean' - : operator === 'ip' - ? 'dotProduct' - : 'cosine') as any, - method: method as VectorIndexMethod | undefined, - }; - } - private checkAndConvertIndexes( schemaName: string, indexes: ModelOptionsIndexes[], diff --git a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts index 20dafc8cd..a9b7939c3 100644 --- a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts @@ -18,6 +18,10 @@ import { extractRelations, RelationType, } from '../utils/extractors/index.js'; +import { + isVectorSchemaType, + vectorFieldStorageMapping, +} from '../../utils/vectorMappings.js'; /** * This function should take as an input a JSON schema and convert it to the sequelize equivalent @@ -154,8 +158,11 @@ function extractObjectType(objectField: Indexable): res.type = extractArrayType(objectField.type).type; } else { res.type = extractType(objectField.type, objectField.sqlType); - if (objectField.type === 'Vector') { - res.type = res.type(objectField.dimensions); + if (isVectorSchemaType(objectField.type)) { + const mapping = vectorFieldStorageMapping('postgres', { + dimensions: objectField.dimensions, + }); + res.type = res.type(mapping.dimensions); } } if (objectField.hasOwnProperty('default')) { diff --git a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts index 7f418d43a..26106fce8 100644 --- a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts @@ -18,6 +18,10 @@ import { extractRelations, RelationType, } from '../utils/extractors/index.js'; +import { + isVectorSchemaType, + vectorFieldStorageMapping, +} from '../../utils/vectorMappings.js'; /** * This function should take as an input a JSON schema and convert it to the sequelize equivalent @@ -163,6 +167,9 @@ function extractObjectType(objectField: Indexable, field: string) { res.type = extractType(objectField.type, objectField.sqlType); } res.type = extractType(objectField.type, objectField.sqlType); + if (isVectorSchemaType(objectField.type)) { + vectorFieldStorageMapping('sql', { dimensions: objectField.dimensions }); + } if (objectField.hasOwnProperty('default')) { res.defaultValue = checkDefaultValue(objectField.type, objectField.default); } diff --git a/modules/database/src/adapters/utils/__tests__/vectorCapabilities.test.ts b/modules/database/src/adapters/utils/__tests__/vectorCapabilities.test.ts new file mode 100644 index 000000000..13ac55579 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorCapabilities.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from '@jest/globals'; +import { + mongoVectorCapabilities, + postgresVectorCapabilities, + sqlFallbackVectorCapabilities, + unsupportedVectorCapabilities, +} from '../vectorCapabilities.js'; + +describe('vector capability contracts', () => { + it('reports MongoDB storage even when search indexes cannot be probed', () => { + expect(mongoVectorCapabilities({ hasSchema: false })).toMatchObject({ + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + }); + expect( + mongoVectorCapabilities({ + hasSchema: true, + searchIndexCommandsAvailable: false, + }), + ).toMatchObject({ + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + }); + expect(mongoVectorCapabilities({ hasSchema: true })).toEqual({ + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'mongodb', + }); + }); + + it('keeps Postgres as a vector provider while distinguishing missing pgvector', () => { + expect(postgresVectorCapabilities({ pgvectorAvailable: true })).toEqual({ + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'postgres', + }); + expect( + postgresVectorCapabilities({ + pgvectorAvailable: false, + error: 'type "vector" does not exist', + }), + ).toMatchObject({ + supported: true, + storage: false, + indexing: false, + search: false, + provider: 'postgres', + }); + }); + + it('describes JSON storage fallback for non-Postgres SQL without claiming search', () => { + expect(sqlFallbackVectorCapabilities('mysql')).toEqual({ + supported: false, + storage: true, + indexing: false, + search: false, + provider: 'unsupported', + reason: + 'mysql does not support Conduit vector search; Vector fields can be stored as JSON', + }); + expect(sqlFallbackVectorCapabilities('sqlite')).toMatchObject({ + supported: false, + storage: true, + search: false, + provider: 'unsupported', + }); + }); + + it('keeps unknown adapters unsupported with no storage fallback', () => { + expect(unsupportedVectorCapabilities('custom')).toEqual({ + supported: false, + storage: false, + indexing: false, + search: false, + provider: 'unsupported', + reason: 'custom does not support Conduit vector search', + }); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorField.test.ts b/modules/database/src/adapters/utils/__tests__/vectorField.test.ts new file mode 100644 index 000000000..815456293 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorField.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it } from '@jest/globals'; +import { + ConduitError, + GrpcError, + TYPE, + VectorIndexMethod, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertObjectFormVectorField, + assertSafeVectorFieldChange, + assertSupportedVectorIndexMethod, + assertVectorFieldIfPresent, + assertVectorIndexContract, + assertVectorIndexMatchesField, + isVectorShorthand, + parseVectorSimilarity, + SUPPORTED_VECTOR_INDEX_METHODS, +} from '../vectorField.js'; +import { fieldsValidator, validateFieldChanges } from '../index.js'; +import { ConduitDatabaseSchema } from '../../../interfaces/index.js'; + +describe('vector field contracts', () => { + const validField = { + type: TYPE.Vector, + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + }; + + it('rejects shorthand Vector definitions', () => { + expect(isVectorShorthand(TYPE.Vector)).toBe(true); + expect(isVectorShorthand('Vector')).toBe(true); + expect(isVectorShorthand(['Vector'])).toBe(true); + expect(() => assertVectorFieldIfPresent('Docs', 'embedding', 'Vector')).toThrow( + ConduitError, + ); + expect(() => fieldsValidator('Docs', { embedding: 'Vector' }, 'mongodb')).toThrow( + /object form/, + ); + }); + + it('requires a positive integer dimensions value', () => { + expect(() => + assertObjectFormVectorField('Docs', 'embedding', { + type: TYPE.Vector, + dimensions: 0, + }), + ).toThrow(/positive integer/); + expect(() => + assertObjectFormVectorField('Docs', 'embedding', { + type: TYPE.Vector, + dimensions: 1.5, + }), + ).toThrow(/positive integer/); + expect(() => + assertObjectFormVectorField('Docs', 'embedding', { + type: TYPE.Vector, + dimensions: -8, + }), + ).toThrow(/positive integer/); + }); + + it('rejects unsupported similarity values and accepts the enum', () => { + expect(() => + assertObjectFormVectorField('Docs', 'embedding', { + type: TYPE.Vector, + dimensions: 8, + similarity: 'manhattan', + }), + ).toThrow(/unsupported similarity/i); + expect(assertObjectFormVectorField('Docs', 'embedding', validField)).toEqual( + validField, + ); + expect(parseVectorSimilarity(undefined)).toBe(VectorSimilarity.Cosine); + expect(parseVectorSimilarity(VectorSimilarity.DotProduct)).toBe( + VectorSimilarity.DotProduct, + ); + }); + + it('rejects unsafe in-place dimension changes and allows same-dimension updates', () => { + expect(() => + assertSafeVectorFieldChange('embedding', validField, { + ...validField, + dimensions: 768, + }), + ).toThrow(/dimensions/); + expect(() => + assertSafeVectorFieldChange('embedding', validField, { + ...validField, + similarity: VectorSimilarity.Euclidean, + }), + ).not.toThrow(); + + const oldSchema = { + compiledFields: { embedding: validField, title: TYPE.String }, + } as unknown as ConduitDatabaseSchema; + const newSchema = { + compiledFields: { + embedding: { ...validField, dimensions: 3072 }, + title: TYPE.String, + }, + } as unknown as ConduitDatabaseSchema; + expect(() => validateFieldChanges(oldSchema, newSchema)).toThrow(ConduitError); + }); + + it('validates provider-specific index methods', () => { + expect(SUPPORTED_VECTOR_INDEX_METHODS.mongodb).toEqual([ + VectorIndexMethod.HNSW, + VectorIndexMethod.Flat, + ]); + expect(SUPPORTED_VECTOR_INDEX_METHODS.postgres).toEqual([ + VectorIndexMethod.HNSW, + VectorIndexMethod.IVFFlat, + ]); + expect(() => + assertSupportedVectorIndexMethod('mongodb', VectorIndexMethod.IVFFlat), + ).toThrow(GrpcError); + expect(() => + assertSupportedVectorIndexMethod('postgres', VectorIndexMethod.Flat), + ).toThrow(GrpcError); + expect(() => + assertSupportedVectorIndexMethod('postgres', VectorIndexMethod.HNSW), + ).not.toThrow(); + }); + + it('rejects index contracts that do not match the schema field', () => { + expect(() => + assertVectorIndexMatchesField( + { type: TYPE.String }, + { + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + }, + ), + ).toThrow(/not a vector/); + expect(() => + assertVectorIndexMatchesField(validField, { + field: 'embedding', + dimensions: 768, + similarity: VectorSimilarity.Cosine, + }), + ).toThrow(/dimensions mismatch/); + expect(() => + assertVectorIndexContract('mongodb', { + field: 'embedding', + dimensions: 1536, + similarity: 'manhattan' as VectorSimilarity, + method: VectorIndexMethod.HNSW, + }), + ).toThrow(GrpcError); + try { + assertVectorIndexContract('mongodb', { + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.IVFFlat, + }); + throw new Error('expected method rejection'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.INVALID_ARGUMENT); + } + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts b/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts new file mode 100644 index 000000000..0b076edb1 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, it } from '@jest/globals'; +import { + ConduitSchema, + TYPE, + VectorIndexMethod, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { DataTypes } from 'sequelize'; +import 'pgvector/sequelize'; +import { schemaConverter } from '../../mongoose-adapter/SchemaConverter.js'; +import { pgSchemaConverter } from '../../sequelize-adapter/postgres-adapter/PgSchemaConverter.js'; +import { sqlSchemaConverter } from '../../sequelize-adapter/sql-adapter/SqlSchemaConverter.js'; +import { + applyMongoVectorField, + fromMongoVectorIndex, + fromPostgresVectorIndex, + mongoVectorStorageType, + pgVectorDistanceOperator, + pgVectorOperator, + postgresIndexMethodSql, + toMongoVectorIndexDefinition, + vectorFieldStorageMapping, +} from '../vectorMappings.js'; + +describe('vector field and index mappings', () => { + const vectorField = { + type: TYPE.Vector, + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + select: false, + }; + + it('maps provider-neutral storage contracts', () => { + expect(vectorFieldStorageMapping('mongodb', vectorField)).toEqual({ + backend: 'mongodb', + storage: 'numberArray', + dimensions: 1536, + searchSupported: true, + }); + expect(vectorFieldStorageMapping('postgres', vectorField)).toEqual({ + backend: 'postgres', + storage: 'pgvector', + dimensions: 1536, + searchSupported: true, + }); + expect(vectorFieldStorageMapping('sql', vectorField)).toEqual({ + backend: 'sql', + storage: 'json', + dimensions: 1536, + searchSupported: false, + }); + expect(mongoVectorStorageType()).toEqual([Number]); + expect(applyMongoVectorField(vectorField).type).toEqual([Number]); + }); + + it('converts Mongo schema Vector fields to a number array without a live database', () => { + const converted = schemaConverter( + new ConduitSchema('Article', { + title: TYPE.String, + embedding: vectorField, + }), + ); + expect(converted.fields.embedding).toMatchObject({ + type: [Number], + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + select: false, + }); + }); + + it('converts Postgres schema Vector fields to pgvector with dimensions', () => { + const [converted] = pgSchemaConverter( + new ConduitSchema('Article', { + title: { type: TYPE.String }, + embedding: vectorField, + }), + ); + const columnType = converted.fields.embedding.type as { + key?: string; + _dimensions?: number; + toSql?: () => string; + }; + expect(columnType.key).toBe('vector'); + expect(columnType._dimensions).toBe(1536); + expect(columnType.toSql?.()).toBe('VECTOR(1536)'); + }); + + it('converts non-Postgres SQL Vector fields to JSON storage', () => { + const [converted] = sqlSchemaConverter( + new ConduitSchema('Article', { + title: { type: TYPE.String }, + embedding: vectorField, + }), + ); + expect(converted.fields.embedding.type).toBe(DataTypes.JSON); + }); + + it('round-trips Mongo vector index definitions', () => { + const definition = toMongoVectorIndexDefinition({ + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.HNSW, + filterFields: ['_id', 'tenantId'], + options: { quantization: 'scalar', hnsw: { maxEdges: 16 } }, + }); + expect(definition).toEqual({ + fields: [ + { + type: 'vector', + path: 'embedding', + numDimensions: 1536, + similarity: VectorSimilarity.Cosine, + quantization: 'scalar', + indexingMethod: VectorIndexMethod.HNSW, + hnswOptions: { maxEdges: 16 }, + }, + { type: 'filter', path: '_id' }, + { type: 'filter', path: 'tenantId' }, + ], + }); + expect( + fromMongoVectorIndex({ + name: 'embedding_vector', + latestDefinition: definition, + }), + ).toMatchObject({ + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.HNSW, + filterFields: ['_id', 'tenantId'], + }); + }); + + it('maps Postgres similarity operators and parses index definitions', () => { + expect(pgVectorOperator(VectorSimilarity.Cosine)).toBe('vector_cosine_ops'); + expect(pgVectorOperator(VectorSimilarity.Euclidean)).toBe('vector_l2_ops'); + expect(pgVectorOperator(VectorSimilarity.DotProduct)).toBe('vector_ip_ops'); + expect(pgVectorDistanceOperator(VectorSimilarity.Cosine)).toBe('<=>'); + expect(pgVectorDistanceOperator(VectorSimilarity.Euclidean)).toBe('<->'); + expect(pgVectorDistanceOperator(VectorSimilarity.DotProduct)).toBe('<#>'); + expect(postgresIndexMethodSql(VectorIndexMethod.IVFFlat)).toBe('ivfflat'); + expect(postgresIndexMethodSql()).toBe('hnsw'); + + const parsed = fromPostgresVectorIndex( + 'cnd_article_embedding_vector', + 'CREATE INDEX cnd_article_embedding_vector ON cnd_article USING hnsw ("embedding" vector_cosine_ops)', + ); + expect(parsed).toMatchObject({ + name: 'cnd_article_embedding_vector', + field: 'embedding', + dimensions: 0, + similarity: VectorSimilarity.Cosine, + method: 'hnsw', + }); + expect( + fromPostgresVectorIndex( + 'cnd_article_embedding_vector', + 'CREATE INDEX cnd_article_embedding_vector ON cnd_article USING ivfflat (embedding vector_l2_ops)', + { dimensions: 1536, similarity: VectorSimilarity.Euclidean }, + ), + ).toMatchObject({ + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Euclidean, + method: 'ivfflat', + }); + }); +}); diff --git a/modules/database/src/adapters/utils/index.ts b/modules/database/src/adapters/utils/index.ts index ec8998dd4..5e2459dc3 100644 --- a/modules/database/src/adapters/utils/index.ts +++ b/modules/database/src/adapters/utils/index.ts @@ -2,3 +2,6 @@ export * from './validateFieldChanges.js'; export * from './validateFieldConstraints.js'; export * from './database-transform-utils.js'; export * from './extensions.js'; +export * from './vectorField.js'; +export * from './vectorCapabilities.js'; +export * from './vectorMappings.js'; diff --git a/modules/database/src/adapters/utils/validateFieldChanges.ts b/modules/database/src/adapters/utils/validateFieldChanges.ts index 187ad21be..92052979e 100644 --- a/modules/database/src/adapters/utils/validateFieldChanges.ts +++ b/modules/database/src/adapters/utils/validateFieldChanges.ts @@ -2,6 +2,7 @@ import { ConduitError, Indexable } from '@conduitplatform/grpc-sdk'; import { ConduitDatabaseSchema, Fields } from '../../interfaces/index.js'; import { isArray, isEqual, isNil, isString } from 'lodash-es'; import { DataTypes } from 'sequelize'; +import { assertSafeVectorFieldChange } from './vectorField.js'; /* * Validates schema compiled fields for type changes. @@ -29,6 +30,7 @@ function validateSchemaFields(oldSchemaFields: Indexable, newSchemaFields: Index if (isNil(newSchemaFields)) return; const newType = newSchemaFields[key]?.type ?? null; if (!newType) return; + assertSafeVectorFieldChange(key, oldSchemaFields[key], newSchemaFields[key]); if (oldType === DataTypes.JSONB && newType === 'JSON') return; if (isArray(oldType) && isArray(newType)) { if (typeof oldType[0] === 'object') { diff --git a/modules/database/src/adapters/utils/validateFieldConstraints.ts b/modules/database/src/adapters/utils/validateFieldConstraints.ts index 6b2b7f71a..a13fa6d1f 100644 --- a/modules/database/src/adapters/utils/validateFieldConstraints.ts +++ b/modules/database/src/adapters/utils/validateFieldConstraints.ts @@ -1,6 +1,7 @@ import { ConduitError, ConduitModel, ConduitModelField } from '@conduitplatform/grpc-sdk'; import { ConduitDatabaseSchema } from '../../interfaces/index.js'; import { isObject } from 'lodash-es'; +import { assertVectorFieldIfPresent } from './vectorField.js'; /* * Validates schema field constraints. @@ -24,6 +25,7 @@ export function fieldsValidator( `Schema '${schemaName}' violates field '${f}' constraint (field names cannot contain '.').`, ); } + assertVectorFieldIfPresent(schemaName, f, schemaFields[f]); if (typeof schemaFields[f] === 'object') { const target: ConduitModelField = schemaFields[f] as ConduitModelField; const isUnique = !!target.unique; @@ -36,18 +38,6 @@ export function fieldsValidator( ); } - if ((target as ConduitModelField & { type?: string }).type === 'Vector') { - const dimensions = (target as ConduitModelField & { dimensions?: number }) - .dimensions; - if (!Number.isInteger(dimensions) || dimensions! <= 0) { - throw new ConduitError( - 'INVALID_ARGUMENTS', - 400, - `Schema '${schemaName}' vector field '${f}' requires a positive integer 'dimensions' value.`, - ); - } - } - if (target.hasOwnProperty('type') && typeof target.type === 'object') { if (Array.isArray(target.type)) { if ((target.type as unknown[]).length !== 1) { diff --git a/modules/database/src/adapters/utils/vectorCapabilities.ts b/modules/database/src/adapters/utils/vectorCapabilities.ts new file mode 100644 index 000000000..ba612b339 --- /dev/null +++ b/modules/database/src/adapters/utils/vectorCapabilities.ts @@ -0,0 +1,96 @@ +import { VectorCapabilities } from '@conduitplatform/grpc-sdk'; + +export function mongoVectorCapabilities(input: { + hasSchema: boolean; + searchIndexCommandsAvailable?: boolean; + probeError?: string; +}): VectorCapabilities { + if (!input.hasSchema) { + return { + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + reason: 'No schema is available to probe MongoDB Vector Search support', + }; + } + if (input.searchIndexCommandsAvailable === false) { + return { + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + reason: 'MongoDB driver does not expose search index commands', + }; + } + if (input.probeError) { + return { + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + reason: input.probeError, + }; + } + return { + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'mongodb', + }; +} + +export function postgresVectorCapabilities(input: { + pgvectorAvailable: boolean; + error?: string; + schemaName?: string; +}): VectorCapabilities { + if (!input.pgvectorAvailable) { + const detail = input.error ?? 'pgvector is not available'; + return { + supported: true, + storage: false, + indexing: false, + search: false, + provider: 'postgres', + reason: input.schemaName + ? `Schema ${input.schemaName} cannot use pgvector: ${detail}` + : detail, + }; + } + return { + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'postgres', + }; +} + +export function sqlFallbackVectorCapabilities(dialect: string): VectorCapabilities { + return { + supported: false, + storage: true, + indexing: false, + search: false, + provider: 'unsupported', + reason: + `${dialect} does not support Conduit vector search; ` + + 'Vector fields can be stored as JSON', + }; +} + +export function unsupportedVectorCapabilities(databaseType: string): VectorCapabilities { + return { + supported: false, + storage: false, + indexing: false, + search: false, + provider: 'unsupported', + reason: `${databaseType} does not support Conduit vector search`, + }; +} diff --git a/modules/database/src/adapters/utils/vectorField.ts b/modules/database/src/adapters/utils/vectorField.ts new file mode 100644 index 000000000..cf15c7d42 --- /dev/null +++ b/modules/database/src/adapters/utils/vectorField.ts @@ -0,0 +1,209 @@ +import { + ConduitError, + GrpcError, + TYPE, + VectorIndexDefinition, + VectorIndexMethod, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export type VectorIndexProvider = 'mongodb' | 'postgres'; + +export const SUPPORTED_VECTOR_INDEX_METHODS: Record< + VectorIndexProvider, + readonly VectorIndexMethod[] +> = { + mongodb: [VectorIndexMethod.HNSW, VectorIndexMethod.Flat], + postgres: [VectorIndexMethod.HNSW, VectorIndexMethod.IVFFlat], +}; + +export interface ParsedVectorField { + type: TYPE.Vector; + dimensions: number; + similarity?: VectorSimilarity; +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function isVectorTypeName(value: unknown): value is TYPE.Vector { + return value === TYPE.Vector || value === 'Vector'; +} + +export function isVectorShorthand(field: unknown): boolean { + if (isVectorTypeName(field)) return true; + return Array.isArray(field) && field.length === 1 && isVectorTypeName(field[0]); +} + +export function isObjectFormVectorField(field: unknown): field is ParsedVectorField { + return isPlainObject(field) && isVectorTypeName(field.type); +} + +export function assertObjectFormVectorField( + schemaName: string, + fieldName: string, + field: unknown, +): ParsedVectorField { + if (isVectorShorthand(field)) { + throw new ConduitError( + 'INVALID_ARGUMENTS', + 400, + `Schema '${schemaName}' vector field '${fieldName}' must use object form ` + + `'{ type: "Vector", dimensions, similarity }'. Shorthand 'Vector' is not allowed.`, + ); + } + if (!isObjectFormVectorField(field)) { + throw new ConduitError( + 'INVALID_ARGUMENTS', + 400, + `Schema '${schemaName}' field '${fieldName}' is not a Vector field.`, + ); + } + if (!Number.isInteger(field.dimensions) || field.dimensions <= 0) { + throw new ConduitError( + 'INVALID_ARGUMENTS', + 400, + `Schema '${schemaName}' vector field '${fieldName}' requires a positive integer 'dimensions' value.`, + ); + } + if (field.similarity !== undefined && !isSupportedVectorSimilarity(field.similarity)) { + throw new ConduitError( + 'INVALID_ARGUMENTS', + 400, + `Schema '${schemaName}' vector field '${fieldName}' has unsupported similarity ` + + `'${String(field.similarity)}'. Supported values: ${Object.values(VectorSimilarity).join(', ')}.`, + ); + } + return { + type: TYPE.Vector, + dimensions: field.dimensions, + similarity: field.similarity, + }; +} + +export function assertVectorFieldIfPresent( + schemaName: string, + fieldName: string, + field: unknown, +): ParsedVectorField | undefined { + if (isVectorShorthand(field) || isObjectFormVectorField(field)) { + return assertObjectFormVectorField(schemaName, fieldName, field); + } + return undefined; +} + +export function isSupportedVectorSimilarity(value: unknown): value is VectorSimilarity { + return Object.values(VectorSimilarity).includes(value as VectorSimilarity); +} + +export function parseVectorSimilarity( + value: unknown, + fallback: VectorSimilarity = VectorSimilarity.Cosine, +): VectorSimilarity { + if (value === undefined || value === null || value === '') { + return fallback; + } + if (!isSupportedVectorSimilarity(value)) { + throw new ConduitError( + 'INVALID_ARGUMENTS', + 400, + `Unsupported similarity '${String(value)}'. Supported values: ${Object.values(VectorSimilarity).join(', ')}.`, + ); + } + return value; +} + +export function assertSafeVectorFieldChange( + fieldName: string, + oldField: unknown, + newField: unknown, +): void { + if (!isObjectFormVectorField(oldField) && !isVectorShorthand(oldField)) { + return; + } + if (newField == null) return; + const newType = isPlainObject(newField) ? newField.type : newField; + if (!isVectorTypeName(newType) && !isObjectFormVectorField(newField)) { + return; + } + if (isVectorShorthand(newField) || !isObjectFormVectorField(newField)) { + throw ConduitError.forbidden( + `Vector field '${fieldName}' must keep object form '{ type: "Vector", dimensions, similarity }'.`, + ); + } + if (!isObjectFormVectorField(oldField)) return; + if ( + Number.isInteger(oldField.dimensions) && + Number.isInteger(newField.dimensions) && + oldField.dimensions !== newField.dimensions + ) { + throw ConduitError.forbidden( + `Changing vector field '${fieldName}' dimensions from ${oldField.dimensions} to ${newField.dimensions} is not allowed.`, + ); + } +} + +export function assertSupportedVectorIndexMethod( + provider: VectorIndexProvider, + method?: string, +): void { + if (method === undefined || method === '') return; + const supported = SUPPORTED_VECTOR_INDEX_METHODS[provider]; + if (!supported.includes(method as VectorIndexMethod)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Unsupported vector index method '${method}' for ${provider}. ` + + `Supported methods: ${supported.join(', ')}.`, + ); + } +} + +export function assertVectorIndexMatchesField( + field: unknown, + index: VectorIndexDefinition, +): void { + if (!isObjectFormVectorField(field)) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Vector index field is not a vector'); + } + if (field.dimensions !== index.dimensions) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Vector index dimensions mismatch: field ${field.dimensions}, index ${index.dimensions}`, + ); + } + if ( + index.similarity !== undefined && + field.similarity !== undefined && + field.similarity !== index.similarity + ) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Vector index similarity mismatch: field ${field.similarity}, index ${index.similarity}`, + ); + } +} + +export function assertVectorIndexContract( + provider: VectorIndexProvider, + index: VectorIndexDefinition, +): void { + if (!index.field || typeof index.field !== 'string') { + throw new GrpcError(status.INVALID_ARGUMENT, 'Vector index field is required'); + } + if (!Number.isInteger(index.dimensions) || index.dimensions <= 0) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Vector index dimensions must be a positive integer', + ); + } + if (!isSupportedVectorSimilarity(index.similarity)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Unsupported vector index similarity '${String(index.similarity)}'. ` + + `Supported values: ${Object.values(VectorSimilarity).join(', ')}.`, + ); + } + assertSupportedVectorIndexMethod(provider, index.method); +} diff --git a/modules/database/src/adapters/utils/vectorMappings.ts b/modules/database/src/adapters/utils/vectorMappings.ts new file mode 100644 index 000000000..08f6cd19b --- /dev/null +++ b/modules/database/src/adapters/utils/vectorMappings.ts @@ -0,0 +1,187 @@ +import { + TYPE, + VectorIndexDefinition, + VectorIndexMethod, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { isObjectFormVectorField } from './vectorField.js'; + +export type VectorStorageBackend = 'mongodb' | 'postgres' | 'sql'; + +export type VectorFieldStorageMapping = + | { + backend: 'mongodb'; + storage: 'numberArray'; + dimensions: number; + searchSupported: true; + } + | { + backend: 'postgres'; + storage: 'pgvector'; + dimensions: number; + searchSupported: true; + } + | { + backend: 'sql'; + storage: 'json'; + dimensions: number; + searchSupported: false; + }; + +export function mongoVectorStorageType() { + return [Number]; +} + +export function vectorFieldStorageMapping( + backend: VectorStorageBackend, + field: { dimensions: number }, +): VectorFieldStorageMapping { + switch (backend) { + case 'mongodb': + return { + backend: 'mongodb', + storage: 'numberArray', + dimensions: field.dimensions, + searchSupported: true, + }; + case 'postgres': + return { + backend: 'postgres', + storage: 'pgvector', + dimensions: field.dimensions, + searchSupported: true, + }; + case 'sql': + return { + backend: 'sql', + storage: 'json', + dimensions: field.dimensions, + searchSupported: false, + }; + default: { + const exhaustive: never = backend; + throw new Error(`Unsupported vector storage backend: ${String(exhaustive)}`); + } + } +} + +export function applyMongoVectorField>(field: T): T { + return { + ...field, + type: mongoVectorStorageType(), + }; +} + +export function pgVectorOperator(similarity: string) { + if (similarity === VectorSimilarity.Euclidean || similarity === 'euclidean') { + return 'vector_l2_ops'; + } + if (similarity === VectorSimilarity.DotProduct || similarity === 'dotProduct') { + return 'vector_ip_ops'; + } + return 'vector_cosine_ops'; +} + +export function pgVectorDistanceOperator(similarity: string) { + if (similarity === VectorSimilarity.Euclidean || similarity === 'euclidean') { + return '<->'; + } + if (similarity === VectorSimilarity.DotProduct || similarity === 'dotProduct') { + return '<#>'; + } + return '<=>'; +} + +export function toMongoVectorIndexDefinition(index: VectorIndexDefinition) { + const vectorField: Record = { + type: 'vector', + path: index.field, + numDimensions: index.dimensions, + similarity: index.similarity, + }; + if (index.options?.quantization) { + vectorField.quantization = index.options.quantization; + } + if (index.method) { + vectorField.indexingMethod = index.method; + } + if (index.options?.hnsw) { + vectorField.hnswOptions = { + ...(index.options.hnsw.maxEdges && { maxEdges: index.options.hnsw.maxEdges }), + ...(index.options.hnsw.numEdgeCandidates && { + numEdgeCandidates: index.options.hnsw.numEdgeCandidates, + }), + }; + } + return { + fields: [ + vectorField, + ...(index.filterFields ?? []).map((path: string) => ({ type: 'filter', path })), + ], + ...(index.options?.storedSource !== undefined && { + storedSource: index.options.storedSource, + }), + }; +} + +export function fromMongoVectorIndex(index: { + name?: string; + latestDefinition?: { fields?: Array> }; + definition?: { fields?: Array> }; +}): VectorIndexDefinition { + const fields = index.latestDefinition?.fields ?? index.definition?.fields ?? []; + const vectorField = fields.find(field => field.type === 'vector') ?? {}; + return { + name: index.name, + field: vectorField.path, + dimensions: vectorField.numDimensions, + similarity: vectorField.similarity, + method: vectorField.indexingMethod, + filterFields: fields + .filter(field => field.type === 'filter') + .map(field => field.path), + }; +} + +export function fromPostgresVectorIndex( + name: string, + definition: string, + field?: { dimensions?: number; similarity?: VectorSimilarity }, +): VectorIndexDefinition { + const method = /USING\s+(\w+)/i.exec(definition)?.[1]; + const fieldMatch = /\((?:"([^"]+)"|(\w+))\s+vector_/i.exec(definition); + const operator = /vector_(l2|cosine|ip)_ops/i.exec(definition)?.[1]; + const similarity = + field?.similarity ?? + (operator === 'l2' + ? VectorSimilarity.Euclidean + : operator === 'ip' + ? VectorSimilarity.DotProduct + : VectorSimilarity.Cosine); + return { + name, + field: fieldMatch?.[1] ?? fieldMatch?.[2] ?? '', + dimensions: field?.dimensions ?? 0, + similarity, + method: method as VectorIndexMethod | undefined, + }; +} + +export function postgresIndexMethodSql(method?: VectorIndexMethod | string) { + return method === VectorIndexMethod.IVFFlat || method === 'ivfflat' + ? 'ivfflat' + : 'hnsw'; +} + +export function resolveVectorFieldFromSchema( + schemaFields: Record | undefined, + fieldName: string, +) { + const field = schemaFields?.[fieldName]; + if (!isObjectFormVectorField(field)) return undefined; + return field; +} + +export function isVectorSchemaType(type: unknown) { + return type === TYPE.Vector || type === 'Vector'; +} diff --git a/modules/database/src/interfaces/SchemaFieldTypes.ts b/modules/database/src/interfaces/SchemaFieldTypes.ts index 938ce0088..6b961e92b 100644 --- a/modules/database/src/interfaces/SchemaFieldTypes.ts +++ b/modules/database/src/interfaces/SchemaFieldTypes.ts @@ -63,7 +63,7 @@ export const SchemaFieldsDescription = `Object mapping field names to field defi - Object: \`{ fieldName: { type: "String", required: true } }\` - Array: \`{ fieldName: ["String"] }\` or \`{ fieldName: [{ type: "String" }] }\` - Relation: \`{ fieldName: { type: "Relation", model: "SchemaName" } }\` -- Vector: \`{ fieldName: { type: "Vector", dimensions: 1536, similarity: "cosine", select: false } }\` +- Vector: \`{ fieldName: { type: "Vector", dimensions: 1536, similarity: "cosine", select: false } }\` (object form only; shorthand \`"Vector"\` is rejected) - Nested: \`{ fieldName: { nestedField: { type: "String" } } }\` **Field Properties:** @@ -74,7 +74,7 @@ export const SchemaFieldsDescription = `Object mapping field names to field defi - \`default\` (optional): string - Default value for the field - \`description\` (optional): string - Field description - \`model\` (required for Relation): string - Name of the related schema -- \`dimensions\` (required for Vector): number - Embedding vector dimensions +- \`dimensions\` (required for Vector): positive integer - Embedding vector dimensions - \`similarity\` (optional for Vector): cosine | euclidean | dotProduct **Example:** diff --git a/modules/embeddings/package.json b/modules/embeddings/package.json index 5145ab715..f411c166e 100644 --- a/modules/embeddings/package.json +++ b/modules/embeddings/package.json @@ -20,7 +20,8 @@ "prebuild": "npm run generateTypes", "build": "rimraf dist && tsc", "postbuild": "copyfiles -u 1 src/**/*.proto src/*.proto ./dist/", - "generateTypes": "sh build.sh" + "generateTypes": "sh build.sh", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utils/validateEmbeddingConfig.test.js" }, "dependencies": { "@bufbuild/protobuf": "^2.10.2", diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts index b1afa3759..b08af9475 100644 --- a/modules/embeddings/src/Embeddings.ts +++ b/modules/embeddings/src/Embeddings.ts @@ -7,7 +7,6 @@ import { GrpcResponse, HealthCheckStatus, TYPE, - VectorSimilarity, } from '@conduitplatform/grpc-sdk'; import { ConfigController, @@ -19,6 +18,7 @@ import * as models from './models/index.js'; import { EmbeddingConfig } from './models/index.js'; import { QueueController } from './controllers/queue.controller.js'; import { getProvider, hashEmbeddingInput } from './providers/index.js'; +import { validateEmbeddingConfigInput } from './utils/validateEmbeddingConfig.js'; import metricsSchema from './metrics/index.js'; import { BackfillRequest, @@ -278,21 +278,9 @@ export default class EmbeddingsModule extends ManagedModule { } private validateConfigRequest(request: EmbeddingConfigRequest) { - if (!request.schemaName || !request.targetField || !request.sourceFields.length) { - throw new Error('schemaName, targetField, and sourceFields are required'); - } - if (!Number.isInteger(request.dimensions) || request.dimensions <= 0) { - throw new Error('dimensions must be a positive integer'); - } - return { - schemaName: request.schemaName, - sourceFields: request.sourceFields, - targetField: request.targetField, - provider: request.provider || this.currentConfig().defaultProvider, - modelName: request.model, - dimensions: request.dimensions, - similarity: (request.similarity || VectorSimilarity.Cosine) as VectorSimilarity, - }; + return validateEmbeddingConfigInput(request, { + provider: this.currentConfig().defaultProvider, + }); } private async resolveConfig(schemaName: string, targetField?: string) { diff --git a/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts b/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts new file mode 100644 index 000000000..4ae68d177 --- /dev/null +++ b/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts @@ -0,0 +1,58 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { validateEmbeddingConfigInput } from './validateEmbeddingConfig.js'; + +describe('validateEmbeddingConfigInput', () => { + const defaults = { provider: 'openai-compatible' }; + const valid = { + schemaName: 'Article', + sourceFields: ['title', 'body'], + targetField: 'embedding', + dimensions: 1536, + }; + + it('accepts object-form config with a supported similarity enum', () => { + const result = validateEmbeddingConfigInput( + { + ...valid, + similarity: VectorSimilarity.DotProduct, + model: 'text-embedding-3-small', + }, + defaults, + ); + assert.equal(result.similarity, VectorSimilarity.DotProduct); + assert.equal(result.provider, 'openai-compatible'); + assert.equal(result.dimensions, 1536); + }); + + it('defaults omitted similarity to cosine', () => { + const result = validateEmbeddingConfigInput(valid, defaults); + assert.equal(result.similarity, VectorSimilarity.Cosine); + }); + + it('rejects missing identity fields', () => { + assert.throws( + () => validateEmbeddingConfigInput({ ...valid, sourceFields: [] }, defaults), + /required/, + ); + }); + + it('rejects non-positive and non-integer dimensions', () => { + assert.throws( + () => validateEmbeddingConfigInput({ ...valid, dimensions: 0 }, defaults), + /positive integer/, + ); + assert.throws( + () => validateEmbeddingConfigInput({ ...valid, dimensions: 12.3 }, defaults), + /positive integer/, + ); + }); + + it('rejects unsupported similarity values', () => { + assert.throws( + () => validateEmbeddingConfigInput({ ...valid, similarity: 'manhattan' }, defaults), + /Unsupported similarity/, + ); + }); +}); diff --git a/modules/embeddings/src/utils/validateEmbeddingConfig.ts b/modules/embeddings/src/utils/validateEmbeddingConfig.ts new file mode 100644 index 000000000..704d964b9 --- /dev/null +++ b/modules/embeddings/src/utils/validateEmbeddingConfig.ts @@ -0,0 +1,55 @@ +import { VectorSimilarity } from '@conduitplatform/grpc-sdk'; + +export interface EmbeddingConfigInput { + schemaName?: string; + sourceFields?: string[]; + targetField?: string; + provider?: string; + model?: string; + dimensions?: number; + similarity?: string; +} + +export interface ValidatedEmbeddingConfig { + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + modelName: string | undefined; + dimensions: number; + similarity: VectorSimilarity; +} + +const SUPPORTED_SIMILARITY = Object.values(VectorSimilarity); + +export function validateEmbeddingConfigInput( + request: EmbeddingConfigInput, + defaults: { provider: string }, +): ValidatedEmbeddingConfig { + if (!request.schemaName || !request.targetField || !request.sourceFields?.length) { + throw new Error('schemaName, targetField, and sourceFields are required'); + } + const dimensions = request.dimensions; + if ( + typeof dimensions !== 'number' || + !Number.isInteger(dimensions) || + dimensions <= 0 + ) { + throw new Error('dimensions must be a positive integer'); + } + const similarity = request.similarity || VectorSimilarity.Cosine; + if (!SUPPORTED_SIMILARITY.includes(similarity as VectorSimilarity)) { + throw new Error( + `Unsupported similarity '${similarity}'. Supported values: ${SUPPORTED_SIMILARITY.join(', ')}`, + ); + } + return { + schemaName: request.schemaName, + sourceFields: request.sourceFields, + targetField: request.targetField, + provider: request.provider || defaults.provider, + modelName: request.model, + dimensions, + similarity: similarity as VectorSimilarity, + }; +} diff --git a/modules/embeddings/tsconfig.json b/modules/embeddings/tsconfig.json index 554dca661..28f0b0240 100644 --- a/modules/embeddings/tsconfig.json +++ b/modules/embeddings/tsconfig.json @@ -14,5 +14,7 @@ "moduleResolution": "NodeNext", "esModuleInterop": true, "forceConsistentCasingInFileNames": true - } + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "src/**/*.test.ts"] } diff --git a/modules/embeddings/tsconfig.test.json b/modules/embeddings/tsconfig.test.json new file mode 100644 index 000000000..e6e240235 --- /dev/null +++ b/modules/embeddings/tsconfig.test.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": [ + "src/utils/validateEmbeddingConfig.ts", + "src/utils/validateEmbeddingConfig.test.ts" + ], + "exclude": [] +} From 836c232d235150c4d926336180a01749dc8211ed Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 17:36:04 +0300 Subject: [PATCH 03/29] fix(embeddings): prevent self-update loops and make jobs deterministic Load hidden source hashes, suppress embeddings-owned write-back events, and deduplicate queue work so one source change yields one provider call. --- .../grpc-sdk/src/modules/database/index.ts | 20 ++- libraries/grpc-sdk/src/types/options.ts | 5 + modules/database/src/Database.ts | 82 ++++++++- .../utils/__tests__/mutationEvents.test.ts | 55 ++++++ modules/database/src/adapters/utils/index.ts | 1 + .../src/adapters/utils/mutationEvents.ts | 80 +++++++++ modules/database/src/database.proto | 4 + modules/embeddings/package.json | 2 +- modules/embeddings/src/Embeddings.ts | 148 ++++++++++----- .../src/controllers/queue.controller.test.ts | 118 ++++++++++++ .../src/controllers/queue.controller.ts | 169 ++++++++++++++---- .../src/utils/embeddingJobs.test.ts | 28 +++ modules/embeddings/src/utils/embeddingJobs.ts | 28 +++ .../src/utils/mutationEvents.test.ts | 68 +++++++ .../embeddings/src/utils/mutationEvents.ts | 107 +++++++++++ .../src/utils/processEmbedding.test.ts | 85 +++++++++ .../embeddings/src/utils/processEmbedding.ts | 84 +++++++++ modules/embeddings/tsconfig.test.json | 5 +- 18 files changed, 996 insertions(+), 93 deletions(-) create mode 100644 modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts create mode 100644 modules/database/src/adapters/utils/mutationEvents.ts create mode 100644 modules/embeddings/src/controllers/queue.controller.test.ts create mode 100644 modules/embeddings/src/utils/embeddingJobs.test.ts create mode 100644 modules/embeddings/src/utils/embeddingJobs.ts create mode 100644 modules/embeddings/src/utils/mutationEvents.test.ts create mode 100644 modules/embeddings/src/utils/mutationEvents.ts create mode 100644 modules/embeddings/src/utils/processEmbedding.test.ts create mode 100644 modules/embeddings/src/utils/processEmbedding.ts diff --git a/libraries/grpc-sdk/src/modules/database/index.ts b/libraries/grpc-sdk/src/modules/database/index.ts index b37ef8f4d..473a3f460 100644 --- a/libraries/grpc-sdk/src/modules/database/index.ts +++ b/libraries/grpc-sdk/src/modules/database/index.ts @@ -193,8 +193,10 @@ export class DatabaseProvider extends ConduitModule { return JSON.parse(res.result); }); @@ -212,8 +214,10 @@ export class DatabaseProvider extends ConduitModule { return JSON.parse(res.result); }); @@ -231,8 +235,10 @@ export class DatabaseProvider extends ConduitModule { return JSON.parse(res.result); }); @@ -250,8 +256,10 @@ export class DatabaseProvider extends ConduitModule { return JSON.parse(res.result); }); @@ -269,8 +277,10 @@ export class DatabaseProvider extends ConduitModule { return JSON.parse(res.result); }); diff --git a/libraries/grpc-sdk/src/types/options.ts b/libraries/grpc-sdk/src/types/options.ts index 122083286..836dee600 100644 --- a/libraries/grpc-sdk/src/types/options.ts +++ b/libraries/grpc-sdk/src/types/options.ts @@ -5,4 +5,9 @@ export type AuthzOptions = { export type PopulateAuthzOptions = { populate?: string | string[]; + /** + * When true, Database skips publishing the mutation event. + * Existing callers omit this and keep the default publish behavior. + */ + suppressEvent?: boolean; } & AuthzOptions; diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index 7f2419404..fc289b9f1 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -69,6 +69,12 @@ import { type ImportResult, } from '@conduitplatform/module-tools'; import { QueueController } from './controllers/queue.controller.js'; +import { + buildMutationEventChunks, + collectDocumentIds, + mutationEventChannel, + shouldPublishMutationEvent, +} from './adapters/utils/mutationEvents.js'; import AppConfigSchema, { Config } from './config/index.js'; import { Empty } from './protoTypes/google/protobuf/empty.js'; import { fileURLToPath } from 'node:url'; @@ -574,7 +580,10 @@ export default class DatabaseModule extends ManagedModule { }); const docString = JSON.stringify(doc); - this.grpcSdk.bus?.publish(`${this.name}:create:${schemaName}`, docString); + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'create', schemaName), + docString, + ); callback(null, { result: docString }); } catch (err) { @@ -606,7 +615,10 @@ export default class DatabaseModule extends ManagedModule { }); const docsString = JSON.stringify(docs); - this.grpcSdk.bus?.publish(`${this.name}:createMany:${schemaName}`, docsString); + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'createMany', schemaName), + docsString, + ); callback(null, { result: docsString }); } catch (err) { @@ -649,7 +661,12 @@ export default class DatabaseModule extends ManagedModule { ); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:update:${schemaName}`, resultString); + if (shouldPublishMutationEvent(call.request.suppressEvent)) { + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'update', schemaName), + resultString, + ); + } callback(null, { result: resultString }); } catch (err) { @@ -686,7 +703,12 @@ export default class DatabaseModule extends ManagedModule { ); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:update:${schemaName}`, resultString); + if (shouldPublishMutationEvent(call.request.suppressEvent)) { + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'update', schemaName), + resultString, + ); + } callback(null, { result: resultString }); } catch (err) { @@ -723,7 +745,12 @@ export default class DatabaseModule extends ManagedModule { ); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:update:${schemaName}`, resultString); + if (shouldPublishMutationEvent(call.request.suppressEvent)) { + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'update', schemaName), + resultString, + ); + } callback(null, { result: resultString }); } catch (err) { @@ -766,7 +793,12 @@ export default class DatabaseModule extends ManagedModule { ); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:updateMany:${schemaName}`, resultString); + if (shouldPublishMutationEvent(call.request.suppressEvent)) { + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'update', schemaName), + resultString, + ); + } callback(null, { result: resultString }); } catch (err) { @@ -798,6 +830,12 @@ export default class DatabaseModule extends ManagedModule { }); } + const ids = shouldPublishMutationEvent(call.request.suppressEvent) + ? await this.collectMutationIds(schemaAdapter.model, call.request.filterQuery, { + userId: call.request.userId, + scope: call.request.scope, + }) + : []; const result = await schemaAdapter.model.updateMany( call.request.filterQuery, call.request.query, @@ -809,7 +847,14 @@ export default class DatabaseModule extends ManagedModule { ); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:updateMany:${schemaName}`, resultString); + if (shouldPublishMutationEvent(call.request.suppressEvent)) { + for (const payload of buildMutationEventChunks(ids)) { + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'updateMany', schemaName), + payload, + ); + } + } callback(null, { result: resultString }); } catch (err) { @@ -841,7 +886,10 @@ export default class DatabaseModule extends ManagedModule { }); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:delete:${schemaName}`, resultString); + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'delete', schemaName), + resultString, + ); callback(null, { result: resultString }); } catch (err) { @@ -873,7 +921,10 @@ export default class DatabaseModule extends ManagedModule { }); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:delete:${schemaName}`, resultString); + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'delete', schemaName), + resultString, + ); callback(null, { result: resultString }); } catch (err) { @@ -1182,4 +1233,17 @@ export default class DatabaseModule extends ManagedModule { ); } } + + private async collectMutationIds( + model: MongooseSchema | SequelizeSchema, + filterQuery: string, + options: { userId?: string; scope?: string }, + ): Promise { + const docs = await model.findMany(filterQuery, { + select: '_id', + userId: options.userId, + scope: options.scope, + }); + return collectDocumentIds(docs); + } } diff --git a/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts b/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts new file mode 100644 index 000000000..0fb4da174 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from '@jest/globals'; +import { + buildMutationEventChunks, + collectDocumentIds, + mutationEventChannel, + shouldPublishMutationEvent, +} from '../mutationEvents.js'; + +describe('mutation event helpers', () => { + it('publishes events unless suppressEvent is explicitly true', () => { + expect(shouldPublishMutationEvent()).toBe(true); + expect(shouldPublishMutationEvent(false)).toBe(true); + expect(shouldPublishMutationEvent(true)).toBe(false); + }); + + it('maps updateOne onto the update channel instead of updateMany', () => { + expect(mutationEventChannel('database', 'update', 'Article')).toBe( + 'database:update:Article', + ); + expect(mutationEventChannel('database', 'updateMany', 'Article')).toBe( + 'database:updateMany:Article', + ); + }); + + it('collects document ids from create and bulk payloads', () => { + expect(collectDocumentIds({ _id: 'a' })).toEqual(['a']); + expect(collectDocumentIds([{ _id: 'a' }, { _id: 'b' }, { _id: 'a' }])).toEqual([ + 'a', + 'b', + ]); + }); + + it('does not treat a Mongo updateMany result as document ids', () => { + expect( + collectDocumentIds({ + acknowledged: true, + matchedCount: 3, + modifiedCount: 3, + upsertedCount: 0, + upsertedId: null, + }), + ).toEqual([]); + }); + + it('publishes affected ids in bounded chunks without altering caller-supplied ids', () => { + const ids = ['1', '2', '3', '4', '5']; + const chunks = buildMutationEventChunks(ids, 2); + expect(chunks).toEqual([ + JSON.stringify([{ _id: '1' }, { _id: '2' }]), + JSON.stringify([{ _id: '3' }, { _id: '4' }]), + JSON.stringify([{ _id: '5' }]), + ]); + expect(ids).toEqual(['1', '2', '3', '4', '5']); + }); +}); diff --git a/modules/database/src/adapters/utils/index.ts b/modules/database/src/adapters/utils/index.ts index 5e2459dc3..1cad8afa5 100644 --- a/modules/database/src/adapters/utils/index.ts +++ b/modules/database/src/adapters/utils/index.ts @@ -5,3 +5,4 @@ export * from './extensions.js'; export * from './vectorField.js'; export * from './vectorCapabilities.js'; export * from './vectorMappings.js'; +export * from './mutationEvents.js'; diff --git a/modules/database/src/adapters/utils/mutationEvents.ts b/modules/database/src/adapters/utils/mutationEvents.ts new file mode 100644 index 000000000..d947bfeb2 --- /dev/null +++ b/modules/database/src/adapters/utils/mutationEvents.ts @@ -0,0 +1,80 @@ +export const MUTATION_EVENT_ID_CHUNK_SIZE = 500; + +export type MutationOperation = + 'create' | 'createMany' | 'update' | 'updateMany' | 'delete'; + +export function shouldPublishMutationEvent(suppressEvent?: boolean): boolean { + return suppressEvent !== true; +} + +export function mutationEventChannel( + moduleName: string, + operation: MutationOperation, + schemaName: string, +): string { + return `${moduleName}:${operation}:${schemaName}`; +} + +export function chunkItems( + items: T[], + size: number = MUTATION_EVENT_ID_CHUNK_SIZE, +): T[][] { + const chunkSize = size > 0 ? size : MUTATION_EVENT_ID_CHUNK_SIZE; + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += chunkSize) { + chunks.push(items.slice(i, i + chunkSize)); + } + return chunks; +} + +export function collectDocumentIds(docs: unknown): string[] { + if (isMongoBulkWriteResult(docs)) return []; + const values = Array.isArray(docs) ? docs : docs == null ? [] : [docs]; + const ids = new Set(); + for (const value of values) { + const id = extractId(value); + if (id) ids.add(id); + } + return [...ids]; +} + +export function toIdEventPayload(ids: string[]): { _id: string }[] { + return ids.map(_id => ({ _id })); +} + +export function buildMutationEventChunks( + ids: string[], + chunkSize: number = MUTATION_EVENT_ID_CHUNK_SIZE, +): string[] { + return chunkItems( + ids.filter(id => id.length > 0), + chunkSize, + ).map(chunk => JSON.stringify(toIdEventPayload(chunk))); +} + +function extractId(value: unknown): string | undefined { + if (value == null) return undefined; + if (typeof value === 'string' || typeof value === 'number') { + const id = String(value); + return id.length ? id : undefined; + } + if (typeof value !== 'object') return undefined; + const record = value as Record; + if (record._id !== undefined) return extractId(record._id); + if (record.id !== undefined) return extractId(record.id); + return undefined; +} + +function isMongoBulkWriteResult(payload: unknown): boolean { + if (payload == null || typeof payload !== 'object' || Array.isArray(payload)) { + return false; + } + const record = payload as Record; + if (record._id !== undefined) return false; + return ( + typeof record.matchedCount === 'number' || + typeof record.modifiedCount === 'number' || + typeof record.nModified === 'number' || + typeof record.n === 'number' + ); +} diff --git a/modules/database/src/database.proto b/modules/database/src/database.proto index 14e95f88d..690d17244 100644 --- a/modules/database/src/database.proto +++ b/modules/database/src/database.proto @@ -123,6 +123,8 @@ message UpdateRequest { repeated string populate = 4; optional string userId = 5; optional string scope = 6; + // When true, skip bus publication. Existing callers omit this and keep publishing. + optional bool suppressEvent = 7; } message UpdateManyRequest { @@ -132,6 +134,8 @@ message UpdateManyRequest { repeated string populate = 4; optional string userId = 5; optional string scope = 6; + // When true, skip bus publication. Existing callers omit this and keep publishing. + optional bool suppressEvent = 7; } message DropCollectionRequest { diff --git a/modules/embeddings/package.json b/modules/embeddings/package.json index f411c166e..3bfa2b1a7 100644 --- a/modules/embeddings/package.json +++ b/modules/embeddings/package.json @@ -21,7 +21,7 @@ "build": "rimraf dist && tsc", "postbuild": "copyfiles -u 1 src/**/*.proto src/*.proto ./dist/", "generateTypes": "sh build.sh", - "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utils/validateEmbeddingConfig.test.js" + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utils/*.test.js dist-test/controllers/*.test.js" }, "dependencies": { "@bufbuild/protobuf": "^2.10.2", diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts index b08af9475..36b063f0a 100644 --- a/modules/embeddings/src/Embeddings.ts +++ b/modules/embeddings/src/Embeddings.ts @@ -19,6 +19,15 @@ import { EmbeddingConfig } from './models/index.js'; import { QueueController } from './controllers/queue.controller.js'; import { getProvider, hashEmbeddingInput } from './providers/index.js'; import { validateEmbeddingConfigInput } from './utils/validateEmbeddingConfig.js'; +import { + embeddingOwnedFields, + isEmbeddingOwnedMutation, + parseMutationEvent, +} from './utils/mutationEvents.js'; +import { + buildEmbeddingDocumentSelect, + generateEmbeddingsForDocument, +} from './utils/processEmbedding.js'; import metricsSchema from './metrics/index.js'; import { BackfillRequest, @@ -47,7 +56,7 @@ export default class EmbeddingsModule extends ManagedModule { private database: DatabaseProvider; private queueController: QueueController; - private subscribedSchemas = new Set(); + private subscribedSchemas = new Map(); constructor(peerManifestRoot?: string) { super('embeddings', peerManifestRoot); @@ -107,7 +116,9 @@ export default class EmbeddingsModule extends ManagedModule { } else { await model.create({ ...config, enabled: true }); } - this.subscribeToSchema(config.schemaName); + if (this.currentConfig().enabled) { + this.subscribeToSchema(config.schemaName); + } callback(null, { result: 'Embedding config saved' }); } catch (err) { callback({ code: 13, message: (err as Error).message }); @@ -196,53 +207,103 @@ export default class EmbeddingsModule extends ManagedModule { private async configureRuntime() { const config = this.currentConfig(); - if (!config.enabled) return; this.queueController ??= QueueController.getInstance(this.grpcSdk); - this.queueController.addWorker( + if (!config.enabled) { + await this.queueController.closeWorker(); + this.unsubscribeAll(); + return; + } + await this.queueController.ensureWorker( data => this.processEmbeddingJob(data.schemaName, data.documentId, data.configId), config.queue.concurrency, ); const configs = await EmbeddingConfig.getInstance().findMany({ enabled: true }); - configs.forEach(config => this.subscribeToSchema(config.schemaName)); + const enabledSchemas = new Set(configs.map(item => item.schemaName)); + for (const schemaName of this.subscribedSchemas.keys()) { + if (!enabledSchemas.has(schemaName)) { + this.unsubscribeFromSchema(schemaName); + } + } + configs.forEach(item => this.subscribeToSchema(item.schemaName)); + } + + private schemaSubscriptionIds(schemaName: string): [string, string, string, string] { + const idPrefix = `embeddings:${schemaName}`; + return [ + `${idPrefix}:create`, + `${idPrefix}:update`, + `${idPrefix}:createMany`, + `${idPrefix}:updateMany`, + ]; } private subscribeToSchema(schemaName: string) { if (this.subscribedSchemas.has(schemaName)) return; - this.subscribedSchemas.add(schemaName); - const idPrefix = `embeddings:${schemaName}`; + const [createId, updateId, createManyId, updateManyId] = + this.schemaSubscriptionIds(schemaName); this.grpcSdk.bus?.subscribe( `database:create:${schemaName}`, message => this.enqueueMutation(schemaName, message), - `${idPrefix}:create`, + createId, ); this.grpcSdk.bus?.subscribe( `database:update:${schemaName}`, message => this.enqueueMutation(schemaName, message), - `${idPrefix}:update`, + updateId, ); this.grpcSdk.bus?.subscribe( `database:createMany:${schemaName}`, message => this.enqueueMutation(schemaName, message), - `${idPrefix}:createMany`, + createManyId, ); this.grpcSdk.bus?.subscribe( `database:updateMany:${schemaName}`, message => this.enqueueMutation(schemaName, message), - `${idPrefix}:updateMany`, + updateManyId, + ); + this.subscribedSchemas.set(schemaName, [ + createId, + updateId, + createManyId, + updateManyId, + ]); + } + + private unsubscribeFromSchema(schemaName: string) { + const ids = this.subscribedSchemas.get(schemaName); + if (!ids) return; + ids.forEach(id => this.grpcSdk.bus?.unsubscribe(id)); + this.subscribedSchemas.delete(schemaName); + } + + private unsubscribeAll() { + [...this.subscribedSchemas.keys()].forEach(schemaName => + this.unsubscribeFromSchema(schemaName), ); } private enqueueMutation(schemaName: string, message: string) { + this.enqueueMutationAsync(schemaName, message).catch(err => + ConduitGrpcSdk.Logger.error(err), + ); + } + + private async enqueueMutationAsync(schemaName: string, message: string) { + const parsed = parseMutationEvent(message); + if (!parsed?.ids.length) return; + const configs = await EmbeddingConfig.getInstance().findMany({ + schemaName, + enabled: true, + }); + if (!configs.length) return; + if (isEmbeddingOwnedMutation(parsed.payload, embeddingOwnedFields(configs))) { + return; + } const attempts = this.currentConfig().queue.attempts; - const payload = JSON.parse(message); - const docs = Array.isArray(payload) ? payload : [payload]; - docs - .filter(doc => doc?._id) - .forEach(doc => { - this.queueController - .addEmbeddingJob({ schemaName, documentId: String(doc._id) }, attempts) - .catch(err => ConduitGrpcSdk.Logger.error(err)); - }); + await this.queueController.addBulkEmbeddingJobs( + parsed.ids.map(documentId => ({ schemaName, documentId })), + attempts, + ); } private async processEmbeddingJob( @@ -250,31 +311,30 @@ export default class EmbeddingsModule extends ManagedModule { documentId: string, configId?: string, ) { - const configs = configId - ? [await EmbeddingConfig.getInstance().findOne({ _id: configId })] - : await EmbeddingConfig.getInstance().findMany({ schemaName, enabled: true }); - const doc = await this.database.findOne>(schemaName, { - _id: documentId, - }); + const configs = ( + configId + ? [await EmbeddingConfig.getInstance().findOne({ _id: configId })] + : await EmbeddingConfig.getInstance().findMany({ schemaName, enabled: true }) + ).filter(Boolean) as EmbeddingConfig[]; + if (!configs.length) return; + const doc = await this.database.findOne>( + schemaName, + { _id: documentId }, + { select: buildEmbeddingDocumentSelect(configs) }, + ); if (!doc) return; - for (const config of configs.filter(Boolean) as EmbeddingConfig[]) { - const input = config.sourceFields.map(field => doc[field] ?? '').join('\n'); - const sourceHash = hashEmbeddingInput(input); - if (doc[`${config.targetField}SourceHash`] === sourceHash) continue; - const vector = await getProvider(config.provider).embed( - input, - this.providerConfig(config.provider, config.modelName), - ); - if (vector.length !== config.dimensions) { - throw new Error( - `Embedding provider returned ${vector.length} dimensions; expected ${config.dimensions}`, - ); - } - await this.database.findByIdAndUpdate(schemaName, documentId, { - [config.targetField]: vector, - [`${config.targetField}SourceHash`]: sourceHash, - }); - } + await generateEmbeddingsForDocument({ + doc, + configs, + hashInput: hashEmbeddingInput, + embed: (input, config) => + getProvider(config.provider).embed( + input, + this.providerConfig(config.provider, config.modelName ?? ''), + ), + update: (fields, options) => + this.database.findByIdAndUpdate(schemaName, documentId, fields, options), + }); } private validateConfigRequest(request: EmbeddingConfigRequest) { diff --git a/modules/embeddings/src/controllers/queue.controller.test.ts b/modules/embeddings/src/controllers/queue.controller.test.ts new file mode 100644 index 000000000..9421cc5ed --- /dev/null +++ b/modules/embeddings/src/controllers/queue.controller.test.ts @@ -0,0 +1,118 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { QueueController } from './queue.controller.js'; +import { EmbeddingJobData, embeddingJobId } from '../utils/embeddingJobs.js'; + +type StoredJob = { + name: string; + data: EmbeddingJobData; + opts?: { jobId?: string }; +}; + +class FakeQueue { + jobs: StoredJob[] = []; + closed = false; + + async add(name: string, data: EmbeddingJobData, opts?: { jobId?: string }) { + if (this.jobs.some(job => job.opts?.jobId === opts?.jobId)) { + throw new Error(`Job ${opts?.jobId} already exists`); + } + this.jobs.push({ name, data, opts }); + } + + async addBulk(jobs: StoredJob[]) { + for (const job of jobs) { + await this.add(job.name, job.data, job.opts); + } + } + + async close() { + this.closed = true; + } +} + +class FakeWorker { + static instances: FakeWorker[] = []; + closed = false; + concurrency: number; + + constructor( + _name: string, + _processor: (job: { data: EmbeddingJobData }) => Promise, + opts: { concurrency: number }, + ) { + this.concurrency = opts.concurrency; + FakeWorker.instances.push(this); + } + + on() { + return this; + } + + async close() { + this.closed = true; + } +} + +function createController(queue: FakeQueue = new FakeQueue()) { + return { + queue, + controller: new QueueController(fakeSdk(), { + Queue: class { + constructor() { + return queue; + } + } as never, + Worker: FakeWorker as never, + }), + }; +} + +function fakeSdk() { + return { + redisManager: { + getClient: () => ({ quit: async () => 'OK' }), + }, + } as unknown as ConduitGrpcSdk; +} + +describe('embedding queue worker lifecycle', () => { + it('keeps a single worker, recreates on concurrency change, and closes idempotently', async () => { + FakeWorker.instances = []; + const { controller } = createController(); + + await controller.ensureWorker(async () => undefined, 2); + await controller.ensureWorker(async () => undefined, 2); + assert.equal(FakeWorker.instances.length, 1); + assert.equal(controller.hasWorker, true); + assert.equal(controller.currentConcurrency, 2); + + await controller.ensureWorker(async () => undefined, 4); + assert.equal(FakeWorker.instances.length, 2); + assert.equal(FakeWorker.instances[0].closed, true); + assert.equal(FakeWorker.instances[1].closed, false); + assert.equal(controller.currentConcurrency, 4); + + await controller.closeWorker(); + await controller.closeWorker(); + assert.equal(FakeWorker.instances[1].closed, true); + assert.equal(controller.hasWorker, false); + }); + + it('deduplicates queued jobs by identity', async () => { + FakeWorker.instances = []; + const { queue, controller } = createController(); + const job = { schemaName: 'Article', documentId: 'a' }; + await controller.addEmbeddingJob(job, 3); + await controller.addEmbeddingJob(job, 3); + await controller.addBulkEmbeddingJobs( + [job, { schemaName: 'Article', documentId: 'b' }, job], + 3, + ); + assert.deepEqual( + queue.jobs.map(stored => stored.opts?.jobId), + [embeddingJobId(job), embeddingJobId({ schemaName: 'Article', documentId: 'b' })], + ); + }); +}); diff --git a/modules/embeddings/src/controllers/queue.controller.ts b/modules/embeddings/src/controllers/queue.controller.ts index b6344ac4d..9ff9867d7 100644 --- a/modules/embeddings/src/controllers/queue.controller.ts +++ b/modules/embeddings/src/controllers/queue.controller.ts @@ -1,65 +1,170 @@ import { Queue, Worker } from 'bullmq'; import { Cluster, Redis } from 'ioredis'; -import { randomUUID } from 'node:crypto'; import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { + EmbeddingJobData, + dedupeEmbeddingJobs, + embeddingJobId, + isDuplicateJobError, +} from '../utils/embeddingJobs.js'; -export interface EmbeddingJobData { - schemaName: string; - documentId: string; - configId?: string; +export type { EmbeddingJobData } from '../utils/embeddingJobs.js'; + +type RedisConnection = Redis | Cluster; + +type QueueLike = { + add: ( + name: string, + data: EmbeddingJobData, + opts?: Record, + ) => Promise; + addBulk: ( + jobs: Array<{ name: string; data: EmbeddingJobData; opts?: Record }>, + ) => Promise; + close: () => Promise; +}; + +type WorkerLike = { + on: (event: string, handler: (...args: unknown[]) => void) => unknown; + close: () => Promise; +}; + +export interface QueueControllerDependencies { + createConnection?: () => RedisConnection; + Queue?: new (name: string, opts: { connection: RedisConnection }) => QueueLike; + Worker?: new ( + name: string, + processor: (job: { data: EmbeddingJobData }) => Promise, + opts: { connection: RedisConnection; concurrency: number } & Record, + ) => WorkerLike; } export class QueueController { private static _instance: QueueController; - private readonly redisConnection: Redis | Cluster; - private readonly embeddingQueue: Queue; + private readonly createConnection: () => RedisConnection; + private readonly QueueImpl: NonNullable; + private readonly WorkerImpl: NonNullable; + private readonly queueConnection: RedisConnection; + private readonly embeddingQueue: QueueLike; + private worker?: WorkerLike; + private workerConnection?: RedisConnection; + private workerConcurrency?: number; + private closingWorker = false; - constructor(private readonly grpcSdk: ConduitGrpcSdk) { - this.redisConnection = this.grpcSdk.redisManager.getClient(); - this.embeddingQueue = new Queue('embeddings-generation-queue', { - connection: this.redisConnection, + constructor( + private readonly grpcSdk: ConduitGrpcSdk, + deps: QueueControllerDependencies = {}, + ) { + this.createConnection = + deps.createConnection ?? (() => this.grpcSdk.redisManager.getClient()); + this.QueueImpl = + deps.Queue ?? + (Queue as unknown as NonNullable); + this.WorkerImpl = + deps.Worker ?? + (Worker as unknown as NonNullable); + this.queueConnection = this.createConnection(); + this.embeddingQueue = new this.QueueImpl('embeddings-generation-queue', { + connection: this.queueConnection, }); } - static getInstance(grpcSdk?: ConduitGrpcSdk) { + static getInstance(grpcSdk?: ConduitGrpcSdk, deps?: QueueControllerDependencies) { if (QueueController._instance) return QueueController._instance; if (!grpcSdk) throw new Error('No grpcSdk instance provided!'); - return (QueueController._instance = new QueueController(grpcSdk)); + return (QueueController._instance = new QueueController(grpcSdk, deps)); + } + + static resetInstance() { + QueueController._instance = undefined as unknown as QueueController; + } + + get hasWorker() { + return this.worker !== undefined; } - addWorker(processor: (data: EmbeddingJobData) => Promise, concurrency: number) { - const worker = new Worker( + get currentConcurrency() { + return this.workerConcurrency; + } + + async ensureWorker( + processor: (data: EmbeddingJobData) => Promise, + concurrency: number, + ) { + if (this.worker && this.workerConcurrency === concurrency) { + return this.worker; + } + await this.closeWorker(); + this.workerConnection = this.createConnection(); + const worker = new this.WorkerImpl( 'embeddings-generation-queue', job => processor(job.data), { concurrency, - connection: this.redisConnection, + connection: this.workerConnection, removeOnComplete: { age: 3600, count: 1000 }, removeOnFail: { age: 24 * 3600 }, }, ); - worker.on('failed', (_job, error) => ConduitGrpcSdk.Logger.error(error)); - worker.on('error', error => ConduitGrpcSdk.Logger.error(error)); + worker.on('failed', (_job, error) => ConduitGrpcSdk.Logger.error(error as Error)); + worker.on('error', error => ConduitGrpcSdk.Logger.error(error as Error)); + this.worker = worker; + this.workerConcurrency = concurrency; return worker; } + async closeWorker() { + if (this.closingWorker || !this.worker) return; + this.closingWorker = true; + const worker = this.worker; + const connection = this.workerConnection; + this.worker = undefined; + this.workerConnection = undefined; + this.workerConcurrency = undefined; + try { + await worker.close(); + await connection?.quit(); + } finally { + this.closingWorker = false; + } + } + + async close() { + await this.closeWorker(); + await this.embeddingQueue.close(); + await this.queueConnection.quit(); + } + async addEmbeddingJob(data: EmbeddingJobData, attempts: number) { - await this.embeddingQueue.add(randomUUID(), data, { - attempts, - backoff: { type: 'exponential', delay: 1000 }, - }); + try { + await this.embeddingQueue.add(embeddingJobId(data), data, { + jobId: embeddingJobId(data), + attempts, + backoff: { type: 'exponential', delay: 1000 }, + }); + } catch (err) { + if (!isDuplicateJobError(err)) throw err; + } } async addBulkEmbeddingJobs(data: EmbeddingJobData[], attempts: number) { - await this.embeddingQueue.addBulk( - data.map(job => ({ - name: randomUUID(), - data: job, - opts: { - attempts, - backoff: { type: 'exponential', delay: 1000 }, - }, - })), - ); + const jobs = dedupeEmbeddingJobs(data); + if (!jobs.length) return; + try { + await this.embeddingQueue.addBulk( + jobs.map(job => ({ + name: embeddingJobId(job), + data: job, + opts: { + jobId: embeddingJobId(job), + attempts, + backoff: { type: 'exponential', delay: 1000 }, + }, + })), + ); + } catch (err) { + if (!isDuplicateJobError(err)) throw err; + await Promise.all(jobs.map(job => this.addEmbeddingJob(job, attempts))); + } } } diff --git a/modules/embeddings/src/utils/embeddingJobs.test.ts b/modules/embeddings/src/utils/embeddingJobs.test.ts new file mode 100644 index 000000000..00b3ccc5c --- /dev/null +++ b/modules/embeddings/src/utils/embeddingJobs.test.ts @@ -0,0 +1,28 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + dedupeEmbeddingJobs, + embeddingJobId, + isDuplicateJobError, +} from './embeddingJobs.js'; + +describe('embedding job identity', () => { + it('deduplicates jobs by schema, document, and config identity', () => { + const jobs = dedupeEmbeddingJobs([ + { schemaName: 'Article', documentId: 'a' }, + { schemaName: 'Article', documentId: 'a' }, + { schemaName: 'Article', documentId: 'a', configId: 'c1' }, + { schemaName: 'Article', documentId: 'b', configId: 'c1' }, + { schemaName: 'Article', documentId: 'a', configId: 'c1' }, + ]); + assert.deepEqual( + jobs.map(job => embeddingJobId(job)), + ['Article__a', 'Article__a__c1', 'Article__b__c1'], + ); + }); + + it('detects BullMQ duplicate job errors', () => { + assert.equal(isDuplicateJobError(new Error('Job Article__a already exists')), true); + assert.equal(isDuplicateJobError(new Error('redis timeout')), false); + }); +}); diff --git a/modules/embeddings/src/utils/embeddingJobs.ts b/modules/embeddings/src/utils/embeddingJobs.ts new file mode 100644 index 000000000..b6f9d82f0 --- /dev/null +++ b/modules/embeddings/src/utils/embeddingJobs.ts @@ -0,0 +1,28 @@ +export interface EmbeddingJobData { + schemaName: string; + documentId: string; + configId?: string; +} + +export function embeddingJobId(data: EmbeddingJobData): string { + const parts = [data.schemaName, data.documentId]; + if (data.configId) parts.push(data.configId); + return parts.join('__'); +} + +export function dedupeEmbeddingJobs(jobs: EmbeddingJobData[]): EmbeddingJobData[] { + const seen = new Set(); + const unique: EmbeddingJobData[] = []; + for (const job of jobs) { + const id = embeddingJobId(job); + if (seen.has(id)) continue; + seen.add(id); + unique.push(job); + } + return unique; +} + +export function isDuplicateJobError(err: unknown): boolean { + const message = err instanceof Error ? err.message : String(err); + return /already exists/i.test(message); +} diff --git a/modules/embeddings/src/utils/mutationEvents.test.ts b/modules/embeddings/src/utils/mutationEvents.test.ts new file mode 100644 index 000000000..e45ca6625 --- /dev/null +++ b/modules/embeddings/src/utils/mutationEvents.test.ts @@ -0,0 +1,68 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + embeddingOwnedFields, + extractDocumentIds, + isEmbeddingOwnedMutation, + parseMutationEvent, +} from './mutationEvents.js'; + +describe('embedding mutation event parsing', () => { + it('normalizes create, update, and bulk payloads to unique ids', () => { + assert.deepEqual(parseMutationEvent(JSON.stringify({ _id: 'a', title: 'x' })), { + payload: { _id: 'a', title: 'x' }, + ids: ['a'], + }); + assert.deepEqual(extractDocumentIds([{ _id: 'a' }, { _id: 'b' }, { _id: 'a' }]), [ + 'a', + 'b', + ]); + assert.deepEqual(extractDocumentIds({ ids: ['a', 'b', 'a'] }), ['a', 'b']); + }); + + it('ignores Mongo updateMany result objects that have no document ids', () => { + assert.deepEqual( + extractDocumentIds({ + acknowledged: true, + matchedCount: 4, + modifiedCount: 4, + }), + [], + ); + assert.equal( + parseMutationEvent( + JSON.stringify({ acknowledged: true, matchedCount: 2, modifiedCount: 2 }), + )?.ids.length, + 0, + ); + }); + + it('parses bounded bulk id chunks', () => { + assert.deepEqual(extractDocumentIds([{ _id: '1' }, { _id: '2' }]), ['1', '2']); + }); + + it('treats embedding-owned write-backs as skippable and keeps source updates', () => { + const owned = embeddingOwnedFields([{ targetField: 'embedding' }]); + assert.equal( + isEmbeddingOwnedMutation( + { + _id: 'a', + embedding: [0.1, 0.2], + embeddingSourceHash: 'abc', + updatedAt: 'now', + }, + owned, + ), + true, + ); + assert.equal( + isEmbeddingOwnedMutation({ _id: 'a', title: 'changed', embedding: [0.1] }, owned), + false, + ); + assert.equal(isEmbeddingOwnedMutation({ _id: 'a' }, owned), false); + }); + + it('returns null for malformed payloads', () => { + assert.equal(parseMutationEvent('{not json'), null); + }); +}); diff --git a/modules/embeddings/src/utils/mutationEvents.ts b/modules/embeddings/src/utils/mutationEvents.ts new file mode 100644 index 000000000..344a2b4b3 --- /dev/null +++ b/modules/embeddings/src/utils/mutationEvents.ts @@ -0,0 +1,107 @@ +const META_FIELDS = new Set(['_id', 'id', 'createdAt', 'updatedAt', '__v']); + +export interface ParsedMutationEvent { + payload: unknown; + ids: string[]; +} + +export function parseMutationEvent(message: string): ParsedMutationEvent | null { + let payload: unknown; + try { + payload = JSON.parse(message); + } catch { + return null; + } + return { + payload, + ids: uniqueIds(extractDocumentIds(payload)), + }; +} + +export function extractDocumentIds(payload: unknown): string[] { + if (payload == null) return []; + if (isMongoBulkWriteResult(payload)) return []; + if (isIdEnvelope(payload)) { + return uniqueIds((payload.ids as unknown[]).map(extractId).filter(isPresent)); + } + const docs = normalizeDocs(payload); + return uniqueIds(docs.map(extractId).filter(isPresent)); +} + +export function embeddingOwnedFields(configs: Array<{ targetField: string }>): string[] { + return configs.flatMap(config => [ + config.targetField, + `${config.targetField}SourceHash`, + ]); +} + +export function isEmbeddingOwnedMutation( + payload: unknown, + ownedFields: string[] = [], +): boolean { + const docs = normalizeDocs(payload); + if (!docs.length) return false; + const owned = new Set(ownedFields); + return docs.every(doc => { + const keys = Object.keys(doc).filter(key => !META_FIELDS.has(key)); + if (!keys.length) return false; + return keys.every( + key => owned.has(key) || key.endsWith('SourceHash') || isNumericVector(doc[key]), + ); + }); +} + +function normalizeDocs(payload: unknown): Record[] { + if (payload == null || isMongoBulkWriteResult(payload)) return []; + if (Array.isArray(payload)) { + return payload.filter(isRecord); + } + if (isRecord(payload)) return [payload]; + return []; +} + +function isMongoBulkWriteResult(payload: unknown): boolean { + if (!isRecord(payload) || payload._id !== undefined) return false; + return ( + typeof payload.matchedCount === 'number' || + typeof payload.modifiedCount === 'number' || + typeof payload.nModified === 'number' || + typeof payload.n === 'number' + ); +} + +function isIdEnvelope(payload: unknown): payload is { ids: unknown[] } { + return isRecord(payload) && Array.isArray(payload.ids); +} + +function extractId(value: unknown): string | undefined { + if (value == null) return undefined; + if (typeof value === 'string' || typeof value === 'number') { + const id = String(value); + return id.length ? id : undefined; + } + if (!isRecord(value)) return undefined; + if (value._id !== undefined) return extractId(value._id); + if (value.id !== undefined) return extractId(value.id); + return undefined; +} + +function isNumericVector(value: unknown): boolean { + return ( + Array.isArray(value) && + value.length > 0 && + value.every(item => typeof item === 'number') + ); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function uniqueIds(ids: string[]): string[] { + return [...new Set(ids)]; +} + +function isPresent(value: string | undefined): value is string { + return Boolean(value); +} diff --git a/modules/embeddings/src/utils/processEmbedding.test.ts b/modules/embeddings/src/utils/processEmbedding.test.ts new file mode 100644 index 000000000..bacccddea --- /dev/null +++ b/modules/embeddings/src/utils/processEmbedding.test.ts @@ -0,0 +1,85 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { + buildEmbeddingDocumentSelect, + generateEmbeddingsForDocument, +} from './processEmbedding.js'; + +const config = { + sourceFields: ['title', 'body'], + targetField: 'embedding', + dimensions: 2, + provider: 'openai-compatible', + modelName: 'test', +}; + +function hash(input: string) { + return createHash('sha256').update(input).digest('hex'); +} + +describe('embedding generation loop safety', () => { + it('explicitly selects source fields and the hidden source hash', () => { + assert.equal( + buildEmbeddingDocumentSelect([config]), + '+title +body +embeddingSourceHash', + ); + }); + + it('skips provider calls when the source hash already matches', async () => { + const sourceHash = hash('Hello\nWorld'); + let embedCalls = 0; + let updates = 0; + const result = await generateEmbeddingsForDocument({ + doc: { + _id: 'a', + title: 'Hello', + body: 'World', + embeddingSourceHash: sourceHash, + }, + configs: [config], + hashInput: hash, + embed: async () => { + embedCalls += 1; + return [1, 2]; + }, + update: async () => { + updates += 1; + }, + }); + assert.deepEqual(result, { generated: 0, skipped: 1 }); + assert.equal(embedCalls, 0); + assert.equal(updates, 0); + }); + + it('performs one write with event suppression and does not loop on the write-back', async () => { + const sourceHash = hash('Hello\nWorld'); + let embedCalls = 0; + const updates: Array<{ fields: Record; options: unknown }> = []; + const doc: Record = { _id: 'a', title: 'Hello', body: 'World' }; + + const run = () => + generateEmbeddingsForDocument({ + doc, + configs: [config], + hashInput: hash, + embed: async () => { + embedCalls += 1; + return [1, 2]; + }, + update: async (fields, options) => { + updates.push({ fields, options }); + }, + }); + + const first = await run(); + const second = await run(); + + assert.deepEqual(first, { generated: 1, skipped: 0 }); + assert.deepEqual(second, { generated: 0, skipped: 1 }); + assert.equal(embedCalls, 1); + assert.equal(updates.length, 1); + assert.deepEqual(updates[0].options, { suppressEvent: true }); + assert.equal(updates[0].fields.embeddingSourceHash, sourceHash); + }); +}); diff --git a/modules/embeddings/src/utils/processEmbedding.ts b/modules/embeddings/src/utils/processEmbedding.ts new file mode 100644 index 000000000..8d91b09ce --- /dev/null +++ b/modules/embeddings/src/utils/processEmbedding.ts @@ -0,0 +1,84 @@ +export interface EmbeddingConfigLike { + sourceFields: string[]; + targetField: string; + dimensions: number; + provider: string; + modelName?: string; +} + +export interface EmbeddingGenerationResult { + generated: number; + skipped: number; +} + +export function buildEmbeddingDocumentSelect( + configs: Array<{ sourceFields: string[]; targetField: string }>, +): string { + const fields = new Set(); + for (const config of configs) { + for (const field of config.sourceFields) { + fields.add(`+${field}`); + } + fields.add(`+${config.targetField}SourceHash`); + } + return [...fields].join(' '); +} + +export function sourceHashField(targetField: string): string { + return `${targetField}SourceHash`; +} + +export function buildEmbeddingInput( + doc: Record, + sourceFields: string[], +): string { + return sourceFields.map(field => doc[field] ?? '').join('\n'); +} + +export function shouldSkipEmbedding( + doc: Record, + targetField: string, + sourceHash: string, +): boolean { + return doc[sourceHashField(targetField)] === sourceHash; +} + +export async function generateEmbeddingsForDocument(args: { + doc: Record; + configs: EmbeddingConfigLike[]; + hashInput: (input: string) => string; + embed: (input: string, config: EmbeddingConfigLike) => Promise; + update: ( + fields: Record, + options: { suppressEvent: true }, + ) => Promise; +}): Promise { + let generated = 0; + let skipped = 0; + for (const config of args.configs) { + const input = buildEmbeddingInput(args.doc, config.sourceFields); + const sourceHash = args.hashInput(input); + if (shouldSkipEmbedding(args.doc, config.targetField, sourceHash)) { + skipped += 1; + continue; + } + const vector = await args.embed(input, config); + if (vector.length !== config.dimensions) { + throw new Error( + `Embedding provider returned ${vector.length} dimensions; expected ${config.dimensions}`, + ); + } + const hashField = sourceHashField(config.targetField); + await args.update( + { + [config.targetField]: vector, + [hashField]: sourceHash, + }, + { suppressEvent: true }, + ); + args.doc[hashField] = sourceHash; + args.doc[config.targetField] = vector; + generated += 1; + } + return { generated, skipped }; +} diff --git a/modules/embeddings/tsconfig.test.json b/modules/embeddings/tsconfig.test.json index e6e240235..c3bb42d36 100644 --- a/modules/embeddings/tsconfig.test.json +++ b/modules/embeddings/tsconfig.test.json @@ -8,8 +8,9 @@ "types": ["node"] }, "include": [ - "src/utils/validateEmbeddingConfig.ts", - "src/utils/validateEmbeddingConfig.test.ts" + "src/utils/**/*.ts", + "src/controllers/**/*.ts", + "src/providers/index.ts" ], "exclude": [] } From 2f8ab53aa8be4f988fc5abbbd7aa042cc80bde02 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 18:03:57 +0300 Subject: [PATCH 04/29] feat(embeddings): enforce fail-closed search and provider trust boundaries Vector and semantic search now require a subject, scope, or verified admin operator on authorization-enabled schemas. Embeddings jobs use a module-identity-scoped read/write context, and provider/config/queue paths fail closed on SSRF, secrets, and malformed payloads. --- libraries/grpc-sdk/src/interfaces/Model.ts | 5 +- .../grpc-sdk/src/modules/database/index.ts | 2 + .../grpc-sdk/src/modules/database/types.ts | 2 + .../grpc-sdk/src/modules/embeddings/index.ts | 3 + libraries/grpc-sdk/src/types/options.ts | 5 + .../src/helpers/wrapGrpcFunctions.ts | 14 +- libraries/module-tools/src/utilities/index.ts | 1 + .../src/utilities/redactSensitiveConfig.ts | 47 ++++ modules/database/src/Database.ts | 54 +++- .../src/adapters/mongoose-adapter/index.ts | 33 ++- .../src/adapters/sequelize-adapter/index.ts | 23 +- .../__tests__/embeddingsJobContext.test.ts | 128 ++++++++++ .../utils/__tests__/vectorSearchAuth.test.ts | 63 +++++ .../adapters/utils/embeddingsJobContext.ts | 196 +++++++++++++++ .../database/src/adapters/utils/grpcStatus.ts | 21 ++ modules/database/src/adapters/utils/index.ts | 3 + .../src/adapters/utils/vectorSearchAuth.ts | 34 +++ modules/database/src/admin/schema.admin.ts | 1 + modules/database/src/database.proto | 7 + modules/embeddings/package.json | 2 +- modules/embeddings/src/Embeddings.ts | 236 ++++++++++++++---- modules/embeddings/src/config/index.ts | 73 +++++- .../src/controllers/queue.controller.test.ts | 20 ++ .../src/controllers/queue.controller.ts | 36 ++- modules/embeddings/src/embeddings.proto | 3 + modules/embeddings/src/metrics/index.ts | 14 ++ .../embeddings/src/providers/index.test.ts | 60 +++++ modules/embeddings/src/providers/index.ts | 119 +++++++-- .../src/utils/embeddingJobs.test.ts | 24 ++ modules/embeddings/src/utils/embeddingJobs.ts | 55 ++++ .../src/utils/endpointSecurity.test.ts | 65 +++++ .../embeddings/src/utils/endpointSecurity.ts | 145 +++++++++++ .../src/utils/mutationEvents.test.ts | 16 ++ .../embeddings/src/utils/mutationEvents.ts | 31 ++- .../src/utils/productionSecurity.test.ts | 29 +++ .../src/utils/productionSecurity.ts | 25 ++ .../embeddings/src/utils/redactConfig.test.ts | 46 ++++ modules/embeddings/src/utils/redactConfig.ts | 23 ++ .../embeddings/src/utils/schemaPolicy.test.ts | 117 +++++++++ modules/embeddings/src/utils/schemaPolicy.ts | 164 ++++++++++++ .../src/utils/validateEmbeddingConfig.test.ts | 17 ++ .../src/utils/validateEmbeddingConfig.ts | 34 ++- modules/embeddings/tsconfig.test.json | 2 +- .../src/admin/routes/GetModuleConfig.route.ts | 3 +- .../src/admin/routes/GetMonoConfig.route.ts | 8 +- .../src/admin/routes/SetModuleConfig.route.ts | 3 +- 46 files changed, 1880 insertions(+), 132 deletions(-) create mode 100644 libraries/module-tools/src/utilities/redactSensitiveConfig.ts create mode 100644 modules/database/src/adapters/utils/__tests__/embeddingsJobContext.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/vectorSearchAuth.test.ts create mode 100644 modules/database/src/adapters/utils/embeddingsJobContext.ts create mode 100644 modules/database/src/adapters/utils/grpcStatus.ts create mode 100644 modules/database/src/adapters/utils/vectorSearchAuth.ts create mode 100644 modules/embeddings/src/providers/index.test.ts create mode 100644 modules/embeddings/src/utils/endpointSecurity.test.ts create mode 100644 modules/embeddings/src/utils/endpointSecurity.ts create mode 100644 modules/embeddings/src/utils/productionSecurity.test.ts create mode 100644 modules/embeddings/src/utils/productionSecurity.ts create mode 100644 modules/embeddings/src/utils/redactConfig.test.ts create mode 100644 modules/embeddings/src/utils/redactConfig.ts create mode 100644 modules/embeddings/src/utils/schemaPolicy.test.ts create mode 100644 modules/embeddings/src/utils/schemaPolicy.ts diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 697bf152a..19dcc3e0f 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -87,9 +87,7 @@ export interface ConduitArrayValidation { } export type ConduitValidationRules = - | ConduitStringValidation - | ConduitNumberValidation - | ConduitArrayValidation; + ConduitStringValidation | ConduitNumberValidation | ConduitArrayValidation; type BaseConduitModelField = { type?: TYPE | TYPE[] | ConduitModel | ArrayConduitModel[]; @@ -318,6 +316,7 @@ export interface VectorSearchInput { select?: string; userId?: string; scope?: string; + adminOperator?: boolean; } export interface VectorSearchResult { diff --git a/libraries/grpc-sdk/src/modules/database/index.ts b/libraries/grpc-sdk/src/modules/database/index.ts index 473a3f460..add5d6b6b 100644 --- a/libraries/grpc-sdk/src/modules/database/index.ts +++ b/libraries/grpc-sdk/src/modules/database/index.ts @@ -197,6 +197,7 @@ export class DatabaseProvider extends ConduitModule { return JSON.parse(res.result); }); @@ -395,6 +396,7 @@ export class DatabaseProvider extends ConduitModule JSON.parse(res.result)); } diff --git a/libraries/grpc-sdk/src/modules/database/types.ts b/libraries/grpc-sdk/src/modules/database/types.ts index 7d39f70d3..672e3674d 100644 --- a/libraries/grpc-sdk/src/modules/database/types.ts +++ b/libraries/grpc-sdk/src/modules/database/types.ts @@ -4,6 +4,8 @@ export type FindOneOptions = { userId?: string; scope?: string; readPreference?: string; + embeddingsJob?: boolean; + embeddingsAllowedFields?: string[]; }; export type FindManyOptions = { diff --git a/libraries/grpc-sdk/src/modules/embeddings/index.ts b/libraries/grpc-sdk/src/modules/embeddings/index.ts index bb2e124f0..1de58ae4c 100644 --- a/libraries/grpc-sdk/src/modules/embeddings/index.ts +++ b/libraries/grpc-sdk/src/modules/embeddings/index.ts @@ -10,6 +10,7 @@ export interface EmbeddingConfigInput { model: string; dimensions: number; similarity?: string; + sourceFieldAllowlist?: string[]; } export interface SemanticSearchInput { @@ -20,6 +21,7 @@ export interface SemanticSearchInput { limit?: number; userId?: string; scope?: string; + adminOperator?: boolean; } export class EmbeddingsProvider extends ConduitModule< @@ -59,6 +61,7 @@ export class EmbeddingsProvider extends ConduitModule< limit: input.limit, userId: input.userId, scope: input.scope, + adminOperator: input.adminOperator, }).then(res => JSON.parse(res.result)); } } diff --git a/libraries/grpc-sdk/src/types/options.ts b/libraries/grpc-sdk/src/types/options.ts index 836dee600..90e8228b3 100644 --- a/libraries/grpc-sdk/src/types/options.ts +++ b/libraries/grpc-sdk/src/types/options.ts @@ -10,4 +10,9 @@ export type PopulateAuthzOptions = { * Existing callers omit this and keep the default publish behavior. */ suppressEvent?: boolean; + /** + * When true, Database verifies the caller is the embeddings module + * and restricts the write to embeddings-owned vector/hash fields. + */ + embeddingsJob?: boolean; } & AuthzOptions; diff --git a/libraries/module-tools/src/helpers/wrapGrpcFunctions.ts b/libraries/module-tools/src/helpers/wrapGrpcFunctions.ts index 579f83ce6..711dbde04 100644 --- a/libraries/module-tools/src/helpers/wrapGrpcFunctions.ts +++ b/libraries/module-tools/src/helpers/wrapGrpcFunctions.ts @@ -1,6 +1,6 @@ import { createVerifier } from 'fast-jwt'; import { status } from '@grpc/grpc-js'; -import { ConduitGrpcSdk, GrpcCallback } from '@conduitplatform/grpc-sdk'; +import { ConduitGrpcSdk, GrpcCallback, GrpcError } from '@conduitplatform/grpc-sdk'; interface JWT { moduleName: string; @@ -43,11 +43,19 @@ export function wrapGrpcFunctions( try { invoked = functions[name](call, callback); } catch (error) { - return throwError(callback, (error as Error).message); + return throwError( + callback, + (error as Error).message, + error instanceof GrpcError ? error.code : status.INTERNAL, + ); } if (typeof invoked?.then === 'function') { invoked.then().catch((error: Error) => { - return throwError(callback, error.message); + return throwError( + callback, + error.message, + error instanceof GrpcError ? error.code : status.INTERNAL, + ); }); } }; diff --git a/libraries/module-tools/src/utilities/index.ts b/libraries/module-tools/src/utilities/index.ts index c43b3e854..3d07b84f1 100644 --- a/libraries/module-tools/src/utilities/index.ts +++ b/libraries/module-tools/src/utilities/index.ts @@ -4,3 +4,4 @@ export * from './merge.js'; export * from './exportHelpers.js'; export * from './conduitPeers.js'; export * from './convictConfigParser.js'; +export * from './redactSensitiveConfig.js'; diff --git a/libraries/module-tools/src/utilities/redactSensitiveConfig.ts b/libraries/module-tools/src/utilities/redactSensitiveConfig.ts new file mode 100644 index 000000000..b6211eb5a --- /dev/null +++ b/libraries/module-tools/src/utilities/redactSensitiveConfig.ts @@ -0,0 +1,47 @@ +const WELL_KNOWN_SECRET_KEYS = + /^(apiKey|api_key|password|secret|privateKey|private_key)$/i; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isSchemaLeaf(value: unknown): value is Record { + return ( + isRecord(value) && + ('default' in value || 'format' in value || 'type' in value) && + !isRecord(value._cvtProperties) + ); +} + +function isSensitiveLeaf(value: unknown): boolean { + return isSchemaLeaf(value) && value.sensitive === true; +} + +function unwrapSchema(schema: unknown): unknown { + if (isRecord(schema) && isRecord(schema._cvtProperties)) { + return schema._cvtProperties; + } + return schema; +} + +export function redactSensitiveConfig(config: T, schema?: unknown): T { + if (!isRecord(config)) return config; + const redacted = Array.isArray(config) ? [...config] : { ...config }; + const node = unwrapSchema(schema); + for (const [key, value] of Object.entries(redacted as Record)) { + const childSchema = isRecord(node) ? node[key] : undefined; + if (isSensitiveLeaf(childSchema) || WELL_KNOWN_SECRET_KEYS.test(key)) { + if (typeof value === 'string' && value.length > 0) { + (redacted as Record)[key] = '[REDACTED]'; + } + continue; + } + if (isRecord(value) || Array.isArray(value)) { + (redacted as Record)[key] = redactSensitiveConfig( + value, + childSchema, + ); + } + } + return redacted as T; +} diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index fc289b9f1..3b83cb895 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -74,7 +74,14 @@ import { collectDocumentIds, mutationEventChannel, shouldPublishMutationEvent, -} from './adapters/utils/mutationEvents.js'; + grpcStatusFromError, + callerModuleName, + resolveAdminOperatorContext, + assertVectorSearchAccess, + assertEmbeddingsJobCaller, + assertEmbeddingsJobRead, + assertEmbeddingsJobWrite, +} from './adapters/utils/index.js'; import AppConfigSchema, { Config } from './config/index.js'; import { Empty } from './protoTypes/google/protobuf/empty.js'; import { fileURLToPath } from 'node:url'; @@ -509,6 +516,15 @@ export default class DatabaseModule extends ManagedModule { ) { try { const schemaAdapter = this._activeAdapter.getSchemaModel(call.request.schemaName); + if (call.request.embeddingsJob) { + assertEmbeddingsJobCaller(callerModuleName(call.metadata)); + assertEmbeddingsJobRead({ + query: call.request.query, + select: call.request.select, + allowedFields: call.request.embeddingsAllowedFields, + schema: schemaAdapter.model.originalSchema as ConduitDatabaseSchema, + }); + } const doc = await schemaAdapter.model.findOne(call.request.query, { select: call.request.select, populate: call.request.populate, @@ -518,10 +534,7 @@ export default class DatabaseModule extends ManagedModule { }); callback(null, { result: JSON.stringify(doc) }); } catch (err) { - callback({ - code: status.INTERNAL, - message: (err as Error).message, - }); + callback(grpcStatusFromError(err)); } } @@ -633,13 +646,19 @@ export default class DatabaseModule extends ManagedModule { call: GrpcRequest, callback: GrpcResponse, ) { - const moduleName = call.metadata!.get('module-name')![0] as string; + const moduleName = callerModuleName(call.metadata); const { schemaName } = call.request; try { const schemaAdapter = this._activeAdapter.getSchemaModel(schemaName); - if ( + if (call.request.embeddingsJob) { + assertEmbeddingsJobCaller(moduleName); + assertEmbeddingsJobWrite({ + document: call.request.query, + schema: schemaAdapter.model.originalSchema as ConduitDatabaseSchema, + }); + } else if ( !(await canModify( - moduleName, + moduleName ?? '', schemaAdapter.model, JSON.parse(call.request.query), )) @@ -670,10 +689,7 @@ export default class DatabaseModule extends ManagedModule { callback(null, { result: resultString }); } catch (err) { - callback({ - code: status.INTERNAL, - message: (err as Error).message, - }); + callback(grpcStatusFromError(err)); } } @@ -1100,6 +1116,17 @@ export default class DatabaseModule extends ManagedModule { callback: GrpcResponse, ) { try { + const schemaAdapter = this._activeAdapter.getSchemaModel(call.request.schemaName); + const adminOperator = resolveAdminOperatorContext({ + requested: call.request.adminOperator, + callerModule: callerModuleName(call.metadata), + }); + assertVectorSearchAccess({ + authzEnabled: !!schemaAdapter.model.authzEnabled, + userId: call.request.userId, + scope: call.request.scope, + adminOperator, + }); const result = await this._activeAdapter.vectorSearch({ schemaName: call.request.schemaName, field: call.request.field, @@ -1111,10 +1138,11 @@ export default class DatabaseModule extends ManagedModule { select: call.request.select, userId: call.request.userId, scope: call.request.scope, + adminOperator, }); callback(null, { result: JSON.stringify(result) }); } catch (err) { - callback({ code: status.INTERNAL, message: (err as Error).message }); + callback(grpcStatusFromError(err)); } } diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 3e59c7023..a506087d4 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -23,6 +23,7 @@ import { mongoVectorCapabilities, fromMongoVectorIndex, toMongoVectorIndexDefinition, + assertVectorSearchAccess, } from '../utils/index.js'; import pluralize from '../../utils/pluralize.js'; import { mongoSchemaConverter } from '../../introspection/mongoose/utils.js'; @@ -821,15 +822,12 @@ export class MongooseAdapter extends DatabaseAdapter { ); } - const filter = request.filter ?? {}; - const authorizedQuery = await model.getAuthorizedQuery( - 'read', - filter, - true, - request.userId, - request.scope, - ); - if (isNil(authorizedQuery)) return []; + assertVectorSearchAccess({ + authzEnabled: !!model.authzEnabled, + userId: request.userId, + scope: request.scope, + adminOperator: request.adminOperator, + }); const vectorStage: any = { index: request.indexName ?? `${request.field}_vector`, @@ -838,8 +836,21 @@ export class MongooseAdapter extends DatabaseAdapter { numCandidates: request.numCandidates ?? Math.max((request.limit ?? 10) * 10, 100), limit: request.limit ?? 10, }; - if (Object.keys(authorizedQuery).length > 0) { - vectorStage.filter = authorizedQuery; + const filter = request.filter ?? {}; + if (!request.adminOperator) { + const authorizedQuery = await model.getAuthorizedQuery( + 'read', + filter, + true, + request.userId, + request.scope, + ); + if (isNil(authorizedQuery)) return []; + if (Object.keys(authorizedQuery).length > 0) { + vectorStage.filter = authorizedQuery; + } + } else if (Object.keys(filter).length > 0) { + vectorStage.filter = filter; } const pipeline: any[] = [ diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 739498972..62b92f269 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -48,6 +48,7 @@ import { postgresIndexMethodSql, resolveVectorFieldFromSchema, } from '../utils/vectorMappings.js'; +import { assertVectorSearchAccess } from '../utils/vectorSearchAuth.js'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; @@ -530,13 +531,21 @@ export abstract class SequelizeAdapter extends DatabaseAdapter `Vector dimensions mismatch: expected ${field.dimensions}`, ); } - const authorizedQuery = await schema.getAuthorizedQuery( - 'read', - request.filter ?? {}, - true, - request.userId, - request.scope, - ); + assertVectorSearchAccess({ + authzEnabled: !!schema.authzEnabled, + userId: request.userId, + scope: request.scope, + adminOperator: request.adminOperator, + }); + const authorizedQuery = request.adminOperator + ? (request.filter ?? {}) + : await schema.getAuthorizedQuery( + 'read', + request.filter ?? {}, + true, + request.userId, + request.scope, + ); if (isNil(authorizedQuery)) return []; const tableName = this.getPhysicalTableName(request.schemaName); const distance = pgVectorDistanceOperator(field.similarity ?? 'cosine'); diff --git a/modules/database/src/adapters/utils/__tests__/embeddingsJobContext.test.ts b/modules/database/src/adapters/utils/__tests__/embeddingsJobContext.test.ts new file mode 100644 index 000000000..728fca4c8 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/embeddingsJobContext.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from '@jest/globals'; +import { GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertEmbeddingsJobCaller, + assertEmbeddingsJobRead, + assertEmbeddingsJobWrite, +} from '../embeddingsJobContext.js'; +import { canModify } from '../../../permissions/index.js'; + +const articleSchema = { + name: 'Article', + compiledFields: { + title: { type: TYPE.String }, + body: { type: TYPE.String }, + views: { type: TYPE.Number }, + embedding: { type: TYPE.Vector, dimensions: 2 }, + embeddingSourceHash: { type: TYPE.String, select: false }, + }, + extensions: [ + { + ownerModule: 'embeddings', + fields: { + embedding: { type: TYPE.Vector, dimensions: 2 }, + embeddingSourceHash: { type: TYPE.String, select: false }, + }, + createdAt: new Date(), + updatedAt: new Date(), + }, + ], +}; + +describe('embeddings job context', () => { + it('rejects callers that are not the embeddings module', () => { + try { + assertEmbeddingsJobCaller('database'); + throw new Error('expected failure'); + } catch (err) { + expect((err as GrpcError).code).toBe(status.PERMISSION_DENIED); + } + expect(() => assertEmbeddingsJobCaller('embeddings')).not.toThrow(); + }); + + it('allows configured source and hash reads by id only', () => { + expect(() => + assertEmbeddingsJobRead({ + query: { _id: 'doc-1' }, + select: '+title +body +embeddingSourceHash', + allowedFields: ['title', 'body', 'embeddingSourceHash'], + schema: articleSchema, + }), + ).not.toThrow(); + }); + + it('rejects collection scans, extra selected fields, and non-string sources', () => { + expect(() => + assertEmbeddingsJobRead({ + query: { title: 'x' }, + select: '+title', + allowedFields: ['title'], + schema: articleSchema, + }), + ).toThrow(GrpcError); + expect(() => + assertEmbeddingsJobRead({ + query: { _id: 'doc-1' }, + select: '+title +password', + allowedFields: ['title', 'embeddingSourceHash'], + schema: articleSchema, + }), + ).toThrow(GrpcError); + expect(() => + assertEmbeddingsJobRead({ + query: { _id: 'doc-1' }, + select: '+views', + allowedFields: ['views'], + schema: articleSchema, + }), + ).toThrow(GrpcError); + }); + + it('allows embeddings-owned vector and hash writes and rejects other fields', () => { + expect(() => + assertEmbeddingsJobWrite({ + document: { embedding: [0.1, 0.2], embeddingSourceHash: 'abc' }, + schema: articleSchema, + }), + ).not.toThrow(); + expect(() => + assertEmbeddingsJobWrite({ + document: { $set: { embedding: [0.1, 0.2] } }, + schema: articleSchema, + }), + ).not.toThrow(); + try { + assertEmbeddingsJobWrite({ + document: { title: 'nope' }, + schema: articleSchema, + }); + throw new Error('expected failure'); + } catch (err) { + expect((err as GrpcError).code).toBe(status.PERMISSION_DENIED); + } + expect(() => + assertEmbeddingsJobWrite({ + document: { $unset: { title: 1 } }, + schema: articleSchema, + }), + ).toThrow(GrpcError); + }); + + it('does not change global canModify behavior', async () => { + const schema = { + originalSchema: { + name: 'Article', + ownerModule: 'database', + modelOptions: { conduit: { permissions: { canModify: 'Nothing' } } }, + extensions: articleSchema.extensions, + }, + }; + await expect(canModify('embeddings', schema as never, { title: 'x' })).resolves.toBe( + false, + ); + await expect(canModify('database', schema as never, { title: 'x' })).resolves.toBe( + true, + ); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchAuth.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchAuth.test.ts new file mode 100644 index 000000000..fa3abad6d --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchAuth.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from '@jest/globals'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertVectorSearchAccess, + resolveAdminOperatorContext, +} from '../vectorSearchAuth.js'; + +describe('vector search authorization', () => { + it('allows unscoped search when authorization is disabled', () => { + expect(() => + assertVectorSearchAccess({ + authzEnabled: false, + }), + ).not.toThrow(); + }); + + it('fails closed on authorization-enabled schemas without subject, scope, or admin operator', () => { + try { + assertVectorSearchAccess({ authzEnabled: true }); + throw new Error('expected failure'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.PERMISSION_DENIED); + } + }); + + it('allows a subject or scope on authorization-enabled schemas', () => { + expect(() => + assertVectorSearchAccess({ authzEnabled: true, userId: 'user-1' }), + ).not.toThrow(); + expect(() => + assertVectorSearchAccess({ authzEnabled: true, scope: 'Team:org' }), + ).not.toThrow(); + }); + + it('allows an explicit admin operator context', () => { + expect(() => + assertVectorSearchAccess({ authzEnabled: true, adminOperator: true }), + ).not.toThrow(); + }); + + it('only honors adminOperator from verified platform operator modules', () => { + expect( + resolveAdminOperatorContext({ requested: true, callerModule: 'database' }), + ).toBe(true); + expect(resolveAdminOperatorContext({ requested: true, callerModule: 'core' })).toBe( + true, + ); + expect( + resolveAdminOperatorContext({ requested: true, callerModule: 'embeddings' }), + ).toBe(true); + expect(resolveAdminOperatorContext({ requested: false, callerModule: 'chat' })).toBe( + false, + ); + try { + resolveAdminOperatorContext({ requested: true, callerModule: 'chat' }); + throw new Error('expected failure'); + } catch (err) { + expect((err as GrpcError).code).toBe(status.PERMISSION_DENIED); + } + }); +}); diff --git a/modules/database/src/adapters/utils/embeddingsJobContext.ts b/modules/database/src/adapters/utils/embeddingsJobContext.ts new file mode 100644 index 000000000..78d99940e --- /dev/null +++ b/modules/database/src/adapters/utils/embeddingsJobContext.ts @@ -0,0 +1,196 @@ +import { GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import type { DeclaredSchemaExtension } from '../../interfaces/DeclaredSchemaExtension.js'; + +export const EMBEDDINGS_MODULE_NAME = 'embeddings'; + +export interface EmbeddingsJobSchema { + name: string; + fields?: Record; + compiledFields?: Record; + extensions?: DeclaredSchemaExtension[]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isStringLikeField(field: unknown): boolean { + if (field === TYPE.String || field === 'String') return true; + if (Array.isArray(field) && field.length === 1) return isStringLikeField(field[0]); + if (!isRecord(field)) return false; + if (field.type === TYPE.String || field.type === 'String') return true; + return Array.isArray(field.type) && isStringLikeField(field.type); +} + +function isVectorField(field: unknown): boolean { + if (field === TYPE.Vector || field === 'Vector') return true; + return isRecord(field) && (field.type === TYPE.Vector || field.type === 'Vector'); +} + +function schemaFields(schema: EmbeddingsJobSchema): Record { + return schema.compiledFields ?? schema.fields ?? {}; +} + +export function parseSelectFields(select?: string): string[] { + if (!select) return []; + return select + .split(/\s+/) + .map(part => part.trim()) + .filter(Boolean) + .map(part => part.replace(/^[+-]/, '')); +} + +export function isIdOnlyQuery(query: unknown): boolean { + let parsed = query; + if (typeof query === 'string') { + try { + parsed = JSON.parse(query); + } catch { + return false; + } + } + if (!isRecord(parsed)) return false; + const keys = Object.keys(parsed); + return keys.length === 1 && (keys[0] === '_id' || keys[0] === 'id'); +} + +export function assertEmbeddingsJobCaller(moduleName?: string): void { + if (moduleName === EMBEDDINGS_MODULE_NAME) return; + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embeddings job context is only available to the embeddings module', + ); +} + +export function assertEmbeddingsJobRead(args: { + query: unknown; + select?: string; + allowedFields?: string[]; + schema: EmbeddingsJobSchema; +}): void { + if (!isIdOnlyQuery(args.query)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embeddings jobs may only read documents by id', + ); + } + const allowed = new Set( + (args.allowedFields ?? []).filter(field => typeof field === 'string' && field.length), + ); + if (!allowed.size) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embeddings jobs require an explicit allowed field list', + ); + } + const fields = schemaFields(args.schema); + for (const field of allowed) { + if (field === '_id' || field === 'id') continue; + if (!(field in fields)) { + throw new GrpcError( + status.PERMISSION_DENIED, + `Embeddings jobs cannot read unknown field '${field}'`, + ); + } + const definition = fields[field]; + const hashField = field.endsWith('SourceHash'); + if (!hashField && !isStringLikeField(definition)) { + throw new GrpcError( + status.PERMISSION_DENIED, + `Embeddings jobs cannot read non-string field '${field}'`, + ); + } + } + const selected = parseSelectFields(args.select); + if (!selected.length) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embeddings jobs must select configured source and hash fields', + ); + } + for (const field of selected) { + if (field === '_id' || field === 'id') continue; + if (!allowed.has(field)) { + throw new GrpcError( + status.PERMISSION_DENIED, + `Embeddings jobs cannot select field '${field}'`, + ); + } + } +} + +export function embeddingsOwnedWriteFields(schema: EmbeddingsJobSchema): Set { + const owned = new Set(); + for (const extension of schema.extensions ?? []) { + if (extension.ownerModule !== EMBEDDINGS_MODULE_NAME) continue; + for (const [name, definition] of Object.entries(extension.fields ?? {})) { + if (isVectorField(definition) || name.endsWith('SourceHash')) { + owned.add(name); + } + } + } + return owned; +} + +export function updateDocumentFields(document: unknown): string[] { + let parsed = document; + if (typeof document === 'string') { + try { + parsed = JSON.parse(document); + } catch { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embeddings job write is not valid JSON', + ); + } + } + if (!isRecord(parsed)) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Embeddings job writes must be objects'); + } + if ('$set' in parsed) { + const keys = Object.keys(parsed); + if (keys.some(key => key !== '$set')) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embeddings jobs may only use $set when sending update operators', + ); + } + if (!isRecord(parsed.$set)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embeddings job $set must be an object', + ); + } + return Object.keys(parsed.$set); + } + if (Object.keys(parsed).some(key => key.startsWith('$'))) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embeddings jobs cannot use update operators other than $set', + ); + } + return Object.keys(parsed); +} + +export function assertEmbeddingsJobWrite(args: { + document: unknown; + schema: EmbeddingsJobSchema; +}): void { + const owned = embeddingsOwnedWriteFields(args.schema); + const fields = updateDocumentFields(args.document); + if (!fields.length) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embeddings job writes must include fields', + ); + } + for (const field of fields) { + if (!owned.has(field)) { + throw new GrpcError( + status.PERMISSION_DENIED, + `Embeddings jobs cannot write field '${field}'`, + ); + } + } +} diff --git a/modules/database/src/adapters/utils/grpcStatus.ts b/modules/database/src/adapters/utils/grpcStatus.ts new file mode 100644 index 000000000..6efdf522d --- /dev/null +++ b/modules/database/src/adapters/utils/grpcStatus.ts @@ -0,0 +1,21 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export function grpcStatusFromError(err: unknown): { code: status; message: string } { + if (err instanceof GrpcError) { + return { code: err.code, message: err.message }; + } + return { + code: status.INTERNAL, + message: err instanceof Error ? err.message : String(err), + }; +} + +export function callerModuleName(metadata?: { + get(key: string): Array; +}): string | undefined { + const value = metadata?.get('module-name')?.[0]; + if (typeof value === 'string' && value.length > 0) return value; + if (Buffer.isBuffer(value) && value.length > 0) return value.toString(); + return undefined; +} diff --git a/modules/database/src/adapters/utils/index.ts b/modules/database/src/adapters/utils/index.ts index 1cad8afa5..3b183e7d1 100644 --- a/modules/database/src/adapters/utils/index.ts +++ b/modules/database/src/adapters/utils/index.ts @@ -6,3 +6,6 @@ export * from './vectorField.js'; export * from './vectorCapabilities.js'; export * from './vectorMappings.js'; export * from './mutationEvents.js'; +export * from './grpcStatus.js'; +export * from './vectorSearchAuth.js'; +export * from './embeddingsJobContext.js'; diff --git a/modules/database/src/adapters/utils/vectorSearchAuth.ts b/modules/database/src/adapters/utils/vectorSearchAuth.ts new file mode 100644 index 000000000..426ed0267 --- /dev/null +++ b/modules/database/src/adapters/utils/vectorSearchAuth.ts @@ -0,0 +1,34 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export const VECTOR_SEARCH_OPERATOR_MODULES = ['database', 'core', 'embeddings'] as const; + +export function resolveAdminOperatorContext(args: { + requested?: boolean; + callerModule?: string; + operatorModules?: readonly string[]; +}): boolean { + if (!args.requested) return false; + const operators = args.operatorModules ?? VECTOR_SEARCH_OPERATOR_MODULES; + if (!args.callerModule || !operators.includes(args.callerModule)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Admin operator context is not allowed for this caller', + ); + } + return true; +} + +export function assertVectorSearchAccess(args: { + authzEnabled: boolean; + userId?: string; + scope?: string; + adminOperator?: boolean; +}): void { + if (!args.authzEnabled) return; + if (args.userId || args.scope || args.adminOperator) return; + throw new GrpcError( + status.PERMISSION_DENIED, + 'Vector search on authorization-enabled schemas requires a subject, scope, or admin operator context', + ); +} diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index 4ea29b4e0..9e8ae74cc 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -736,6 +736,7 @@ export class SchemaAdmin { select: call.request.params.select, userId: call.request.context.user?._id, scope: call.request.params.scope, + adminOperator: true, }); return { results }; } diff --git a/modules/database/src/database.proto b/modules/database/src/database.proto index 690d17244..77c87880f 100644 --- a/modules/database/src/database.proto +++ b/modules/database/src/database.proto @@ -45,6 +45,9 @@ message FindOneRequest { optional string userId = 5; optional string scope = 6; optional string readPreference = 7; + // When true, Database verifies the caller is embeddings and restricts the read. + optional bool embeddingsJob = 8; + repeated string embeddingsAllowedFields = 9; } message FindRequest { @@ -125,6 +128,8 @@ message UpdateRequest { optional string scope = 6; // When true, skip bus publication. Existing callers omit this and keep publishing. optional bool suppressEvent = 7; + // When true, Database verifies the caller is embeddings and restricts the write. + optional bool embeddingsJob = 8; } message UpdateManyRequest { @@ -212,6 +217,8 @@ message VectorSearchRequest { optional string select = 8; optional string userId = 9; optional string scope = 10; + // Honored only when the verified caller is a platform operator module. + optional bool adminOperator = 11; } service DatabaseProvider { diff --git a/modules/embeddings/package.json b/modules/embeddings/package.json index 3bfa2b1a7..732148a00 100644 --- a/modules/embeddings/package.json +++ b/modules/embeddings/package.json @@ -21,7 +21,7 @@ "build": "rimraf dist && tsc", "postbuild": "copyfiles -u 1 src/**/*.proto src/*.proto ./dist/", "generateTypes": "sh build.sh", - "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utils/*.test.js dist-test/controllers/*.test.js" + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utils/*.test.js dist-test/controllers/*.test.js dist-test/providers/*.test.js" }, "dependencies": { "@bufbuild/protobuf": "^2.10.2", diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts index 36b063f0a..4ff289150 100644 --- a/modules/embeddings/src/Embeddings.ts +++ b/modules/embeddings/src/Embeddings.ts @@ -3,6 +3,7 @@ import { fileURLToPath } from 'node:url'; import { ConduitGrpcSdk, DatabaseProvider, + GrpcError, GrpcRequest, GrpcResponse, HealthCheckStatus, @@ -13,6 +14,7 @@ import { ConduitActiveSchema, ManagedModule, } from '@conduitplatform/module-tools'; +import { status } from '@grpc/grpc-js'; import AppConfigSchema, { Config } from './config/index.js'; import * as models from './models/index.js'; import { EmbeddingConfig } from './models/index.js'; @@ -22,12 +24,24 @@ import { validateEmbeddingConfigInput } from './utils/validateEmbeddingConfig.js import { embeddingOwnedFields, isEmbeddingOwnedMutation, - parseMutationEvent, + parseBoundedMutationEvent, } from './utils/mutationEvents.js'; import { buildEmbeddingDocumentSelect, generateEmbeddingsForDocument, } from './utils/processEmbedding.js'; +import { MAX_QUEUE_BATCH_SIZE, parseEmbeddingJobData } from './utils/embeddingJobs.js'; +import { + assertCanManageEmbeddingConfig, + assertEmbeddingTargetSchema, + assertSemanticSearchAccess, + resolveAdminOperatorContext, +} from './utils/schemaPolicy.js'; +import { + assertGrpcKeyRequirement, + callerModuleName, +} from './utils/productionSecurity.js'; +import { sanitizeErrorMessage } from './utils/redactConfig.js'; import metricsSchema from './metrics/index.js'; import { BackfillRequest, @@ -64,6 +78,7 @@ export default class EmbeddingsModule extends ManagedModule { } async onServerStart() { + assertGrpcKeyRequirement(process.env); await this.awaitPeersFromManifest(); this.database = this.grpcSdk.database!; await this.registerSchemas(); @@ -72,6 +87,11 @@ export default class EmbeddingsModule extends ManagedModule { this.updateHealth(HealthCheckStatus.SERVING); } + async preConfig(config: Config) { + assertGrpcKeyRequirement(process.env, config); + return config; + } + async onConfig() { if (!this.database) return; await this.configureRuntime(); @@ -82,24 +102,39 @@ export default class EmbeddingsModule extends ManagedModule { callback: GrpcResponse, ) { try { - const config = this.validateConfigRequest(call.request); - const schema = await this.database.getSchema(config.schemaName); - if (!config.sourceFields.every(field => schema.fields[field])) { - return callback({ - code: 3, - message: 'All source fields must exist on the target schema', - }); - } + const schema = await this.database.getSchema(call.request.schemaName); + const declared = await this.declaredSchema(call.request.schemaName); + assertEmbeddingTargetSchema({ + name: schema.name, + ownerModule: declared?.ownerModule, + }); + assertCanManageEmbeddingConfig({ + callerModule: callerModuleName(call.metadata), + ownerModule: declared?.ownerModule, + schemaName: schema.name, + }); + const { sourceFieldAllowlist: _allowlist, ...persisted } = + validateEmbeddingConfigInput( + { + ...call.request, + sourceFieldAllowlist: [ + ...(this.currentConfig().security.sourceFieldAllowlist ?? []), + ...(call.request.sourceFieldAllowlist ?? []), + ], + }, + { provider: this.currentConfig().defaultProvider }, + schema.fields, + ); await this.database.setSchemaExtension({ - schemaName: config.schemaName, + schemaName: persisted.schemaName, fields: { - [config.targetField]: { + [persisted.targetField]: { type: TYPE.Vector, - dimensions: config.dimensions, - similarity: config.similarity, + dimensions: persisted.dimensions, + similarity: persisted.similarity, select: false, }, - [`${config.targetField}SourceHash`]: { + [`${persisted.targetField}SourceHash`]: { type: TYPE.String, required: false, select: false, @@ -108,20 +143,20 @@ export default class EmbeddingsModule extends ManagedModule { }); const model = EmbeddingConfig.getInstance(); const existing = await model.findOne({ - schemaName: config.schemaName, - targetField: config.targetField, + schemaName: persisted.schemaName, + targetField: persisted.targetField, }); if (existing) { - await model.findByIdAndUpdate(existing._id, config); + await model.findByIdAndUpdate(existing._id, persisted); } else { - await model.create({ ...config, enabled: true }); + await model.create({ ...persisted, enabled: true }); } if (this.currentConfig().enabled) { - this.subscribeToSchema(config.schemaName); + this.subscribeToSchema(persisted.schemaName); } callback(null, { result: 'Embedding config saved' }); } catch (err) { - callback({ code: 13, message: (err as Error).message }); + callback(this.grpcError(err)); } } @@ -133,7 +168,7 @@ export default class EmbeddingsModule extends ManagedModule { const configs = await EmbeddingConfig.getInstance().findMany({}); callback(null, { result: JSON.stringify(configs) }); } catch (err) { - callback({ code: 13, message: (err as Error).message }); + callback(this.grpcError(err)); } } @@ -142,11 +177,29 @@ export default class EmbeddingsModule extends ManagedModule { callback: GrpcResponse, ) { try { + const schema = await this.database.getSchema(call.request.schemaName); + const declared = await this.declaredSchema(call.request.schemaName); + assertEmbeddingTargetSchema({ + name: schema.name, + ownerModule: declared?.ownerModule, + }); + assertCanManageEmbeddingConfig({ + callerModule: callerModuleName(call.metadata), + ownerModule: declared?.ownerModule, + schemaName: schema.name, + }); const configs = await EmbeddingConfig.getInstance().findMany({ schemaName: call.request.schemaName, enabled: true, }); - const batchSize = call.request.batchSize ?? 100; + if (!configs.length) { + throw new GrpcError( + status.FAILED_PRECONDITION, + 'No enabled embedding config found for backfill', + ); + } + const maxBatch = this.currentConfig().queue.maxBatchSize ?? MAX_QUEUE_BATCH_SIZE; + const batchSize = Math.min(Math.max(call.request.batchSize ?? 100, 1), maxBatch); const docs = await this.database.findMany>( call.request.schemaName, {}, @@ -167,7 +220,7 @@ export default class EmbeddingsModule extends ManagedModule { result: JSON.stringify({ queued: docs.length * configs.length }), }); } catch (err) { - callback({ code: 13, message: (err as Error).message }); + callback(this.grpcError(err)); } } @@ -176,6 +229,18 @@ export default class EmbeddingsModule extends ManagedModule { callback: GrpcResponse, ) { try { + const adminOperator = resolveAdminOperatorContext({ + requested: call.request.adminOperator, + callerModule: callerModuleName(call.metadata), + }); + const schema = await this.database.getSchema(call.request.schemaName); + if (schema.modelOptions?.conduit?.authorization?.enabled) { + assertSemanticSearchAccess({ + userId: call.request.userId, + scope: call.request.scope, + adminOperator, + }); + } const config = await this.resolveConfig( call.request.schemaName, call.request.targetField, @@ -186,7 +251,8 @@ export default class EmbeddingsModule extends ManagedModule { providerConfig, ); if (vector.length !== config.dimensions) { - throw new Error( + throw new GrpcError( + status.FAILED_PRECONDITION, `Embedding provider returned ${vector.length} dimensions; expected ${config.dimensions}`, ); } @@ -198,10 +264,11 @@ export default class EmbeddingsModule extends ManagedModule { limit: call.request.limit, userId: call.request.userId, scope: call.request.scope, + adminOperator, }); callback(null, { result: JSON.stringify(results) }); } catch (err) { - callback({ code: 13, message: (err as Error).message }); + callback(this.grpcError(err)); } } @@ -284,24 +351,31 @@ export default class EmbeddingsModule extends ManagedModule { private enqueueMutation(schemaName: string, message: string) { this.enqueueMutationAsync(schemaName, message).catch(err => - ConduitGrpcSdk.Logger.error(err), + ConduitGrpcSdk.Logger.error(sanitizeErrorMessage(err)), ); } private async enqueueMutationAsync(schemaName: string, message: string) { - const parsed = parseMutationEvent(message); - if (!parsed?.ids.length) return; + const parsed = parseBoundedMutationEvent( + message, + this.currentConfig().security.maxMutationEventIds, + ); + if (!parsed.ok) { + ConduitGrpcSdk.Metrics?.increment('malformed_embedding_events_total'); + return; + } + if (!parsed.event.ids.length) return; const configs = await EmbeddingConfig.getInstance().findMany({ schemaName, enabled: true, }); if (!configs.length) return; - if (isEmbeddingOwnedMutation(parsed.payload, embeddingOwnedFields(configs))) { + if (isEmbeddingOwnedMutation(parsed.event.payload, embeddingOwnedFields(configs))) { return; } const attempts = this.currentConfig().queue.attempts; await this.queueController.addBulkEmbeddingJobs( - parsed.ids.map(documentId => ({ schemaName, documentId })), + parsed.event.ids.map(documentId => ({ schemaName, documentId })), attempts, ); } @@ -311,21 +385,44 @@ export default class EmbeddingsModule extends ManagedModule { documentId: string, configId?: string, ) { + const parsed = parseEmbeddingJobData({ schemaName, documentId, configId }); + if (!parsed.ok) { + ConduitGrpcSdk.Metrics?.increment('malformed_embedding_jobs_total'); + return; + } const configs = ( - configId - ? [await EmbeddingConfig.getInstance().findOne({ _id: configId })] - : await EmbeddingConfig.getInstance().findMany({ schemaName, enabled: true }) + parsed.data.configId + ? [await EmbeddingConfig.getInstance().findOne({ _id: parsed.data.configId })] + : await EmbeddingConfig.getInstance().findMany({ + schemaName: parsed.data.schemaName, + enabled: true, + }) ).filter(Boolean) as EmbeddingConfig[]; - if (!configs.length) return; + const matching = configs.filter( + config => config.enabled && config.schemaName === parsed.data.schemaName, + ); + if (!matching.length) return; + const allowedFields = [ + ...new Set( + matching.flatMap(config => [ + ...config.sourceFields, + `${config.targetField}SourceHash`, + ]), + ), + ]; const doc = await this.database.findOne>( - schemaName, - { _id: documentId }, - { select: buildEmbeddingDocumentSelect(configs) }, + parsed.data.schemaName, + { _id: parsed.data.documentId }, + { + select: buildEmbeddingDocumentSelect(matching), + embeddingsJob: true, + embeddingsAllowedFields: allowedFields, + }, ); if (!doc) return; await generateEmbeddingsForDocument({ doc, - configs, + configs: matching, hashInput: hashEmbeddingInput, embed: (input, config) => getProvider(config.provider).embed( @@ -333,36 +430,67 @@ export default class EmbeddingsModule extends ManagedModule { this.providerConfig(config.provider, config.modelName ?? ''), ), update: (fields, options) => - this.database.findByIdAndUpdate(schemaName, documentId, fields, options), + this.database.findByIdAndUpdate( + parsed.data.schemaName, + parsed.data.documentId, + fields, + { + ...options, + embeddingsJob: true, + }, + ), }); } - private validateConfigRequest(request: EmbeddingConfigRequest) { - return validateEmbeddingConfigInput(request, { - provider: this.currentConfig().defaultProvider, - }); + private async declaredSchema(schemaName: string) { + return this.database.findOne<{ name: string; ownerModule: string }>( + '_DeclaredSchema', + { name: schemaName }, + { select: 'name ownerModule' }, + ); } - private async resolveConfig(schemaName: string, targetField?: string) { - const query: Record = { schemaName, enabled: true }; - if (targetField) query.targetField = targetField; - const config = await EmbeddingConfig.getInstance().findOne(query); - if (!config) throw new Error('No embedding config found for semantic search'); - return config; + private grpcError(err: unknown) { + if (err instanceof GrpcError) { + return { code: err.code, message: sanitizeErrorMessage(err) }; + } + return { code: status.INTERNAL, message: sanitizeErrorMessage(err) }; } private providerConfig(provider: string, model: string) { - const providers = this.currentConfig().providers as Record< - string, - Record - >; + const config = this.currentConfig(); + const providers = config.providers as Record>; const providerConfig = providers[provider] ?? {}; return { - ...providerConfig, + endpoint: + typeof providerConfig.endpoint === 'string' ? providerConfig.endpoint : undefined, + apiKey: + typeof providerConfig.apiKey === 'string' ? providerConfig.apiKey : undefined, model: String(providerConfig.model ?? model), + allowedHosts: [ + ...new Set( + ((providerConfig.allowedHosts as string[] | undefined) ?? []).filter(Boolean), + ), + ], + timeoutMs: config.security.embedTimeoutMs, + maxInputBytes: config.security.maxEmbedInputBytes, + maxResponseBytes: config.security.maxEmbedResponseBytes, }; } + private async resolveConfig(schemaName: string, targetField?: string) { + const query: Record = { schemaName, enabled: true }; + if (targetField) query.targetField = targetField; + const config = await EmbeddingConfig.getInstance().findOne(query); + if (!config) { + throw new GrpcError( + status.NOT_FOUND, + 'No embedding config found for semantic search', + ); + } + return config; + } + private currentConfig() { return ConfigController.getInstance().config as Config; } diff --git a/modules/embeddings/src/config/index.ts b/modules/embeddings/src/config/index.ts index 15a00d256..ba3a5089c 100644 --- a/modules/embeddings/src/config/index.ts +++ b/modules/embeddings/src/config/index.ts @@ -13,9 +13,29 @@ const AppConfigSchema = { default: 'openai-compatible', }, providers: { - doc: 'Embedding provider configuration keyed by provider name', - format: Object, - default: {}, + 'openai-compatible': { + endpoint: { + doc: 'HTTPS embedding provider endpoint', + format: String, + default: '', + }, + apiKey: { + doc: 'Provider API key', + format: String, + default: '', + sensitive: true, + }, + model: { + doc: 'Default provider model name', + format: String, + default: '', + }, + allowedHosts: { + doc: 'Allowed HTTPS hosts for this provider after DNS resolution', + format: Array, + default: [], + }, + }, }, queue: { concurrency: { @@ -28,12 +48,57 @@ const AppConfigSchema = { format: 'Number', default: 3, }, + maxBatchSize: { + doc: 'Maximum jobs accepted from a single enqueue or backfill request', + format: 'Number', + default: 500, + }, + }, + security: { + requireGrpcKey: { + doc: 'Require GRPC_KEY. Always enforced when NODE_ENV is production.', + format: 'Boolean', + default: false, + }, + sourceFieldAllowlist: { + doc: 'Source fields allowed even when their names look sensitive', + format: Array, + default: [], + }, + maxMutationEventIds: { + doc: 'Maximum document ids accepted from a single mutation bus payload', + format: 'Number', + default: 500, + }, + embedTimeoutMs: { + doc: 'Provider request timeout in milliseconds', + format: 'Number', + default: 10_000, + }, + maxEmbedInputBytes: { + doc: 'Maximum embedding input payload size in bytes', + format: 'Number', + default: 32 * 1024, + }, + maxEmbedResponseBytes: { + doc: 'Maximum embedding provider response size in bytes', + format: 'Number', + default: 1024 * 1024, + }, }, }; const config = convict(AppConfigSchema); const configProperties = config.getProperties(); export type Config = typeof configProperties & { - providers: Record>; + providers: Record< + string, + { + endpoint?: string; + apiKey?: string; + model?: string; + allowedHosts?: string[]; + } + >; }; export default AppConfigSchema; diff --git a/modules/embeddings/src/controllers/queue.controller.test.ts b/modules/embeddings/src/controllers/queue.controller.test.ts index 9421cc5ed..94b728311 100644 --- a/modules/embeddings/src/controllers/queue.controller.test.ts +++ b/modules/embeddings/src/controllers/queue.controller.test.ts @@ -115,4 +115,24 @@ describe('embedding queue worker lifecycle', () => { [embeddingJobId(job), embeddingJobId({ schemaName: 'Article', documentId: 'b' })], ); }); + + it('skips malformed queue payloads instead of throwing', async () => { + FakeWorker.instances = []; + const { queue, controller } = createController(); + await controller.addEmbeddingJob( + { schemaName: '../nope', documentId: 'a' } as never, + 3, + ); + await controller.addBulkEmbeddingJobs( + [ + { schemaName: 'Article', documentId: 'ok' }, + { schemaName: 'Article', documentId: '' } as never, + ], + 3, + ); + assert.deepEqual( + queue.jobs.map(stored => stored.opts?.jobId), + [embeddingJobId({ schemaName: 'Article', documentId: 'ok' })], + ); + }); }); diff --git a/modules/embeddings/src/controllers/queue.controller.ts b/modules/embeddings/src/controllers/queue.controller.ts index 9ff9867d7..4799bbefe 100644 --- a/modules/embeddings/src/controllers/queue.controller.ts +++ b/modules/embeddings/src/controllers/queue.controller.ts @@ -6,6 +6,7 @@ import { dedupeEmbeddingJobs, embeddingJobId, isDuplicateJobError, + parseEmbeddingJobData, } from '../utils/embeddingJobs.js'; export type { EmbeddingJobData } from '../utils/embeddingJobs.js'; @@ -98,7 +99,14 @@ export class QueueController { this.workerConnection = this.createConnection(); const worker = new this.WorkerImpl( 'embeddings-generation-queue', - job => processor(job.data), + job => { + const parsed = parseEmbeddingJobData(job.data); + if (!parsed.ok) { + ConduitGrpcSdk.Metrics?.increment('malformed_embedding_jobs_total'); + return Promise.resolve(); + } + return processor(parsed.data); + }, { concurrency, connection: this.workerConnection, @@ -136,9 +144,14 @@ export class QueueController { } async addEmbeddingJob(data: EmbeddingJobData, attempts: number) { + const parsed = parseEmbeddingJobData(data); + if (!parsed.ok) { + ConduitGrpcSdk.Metrics?.increment('malformed_embedding_jobs_total'); + return; + } try { - await this.embeddingQueue.add(embeddingJobId(data), data, { - jobId: embeddingJobId(data), + await this.embeddingQueue.add(embeddingJobId(parsed.data), parsed.data, { + jobId: embeddingJobId(parsed.data), attempts, backoff: { type: 'exponential', delay: 1000 }, }); @@ -148,11 +161,20 @@ export class QueueController { } async addBulkEmbeddingJobs(data: EmbeddingJobData[], attempts: number) { - const jobs = dedupeEmbeddingJobs(data); - if (!jobs.length) return; + const jobs: EmbeddingJobData[] = []; + for (const [index, item] of data.entries()) { + const parsed = parseEmbeddingJobData(item, index); + if (!parsed.ok) { + ConduitGrpcSdk.Metrics?.increment('malformed_embedding_jobs_total'); + continue; + } + jobs.push(parsed.data); + } + const unique = dedupeEmbeddingJobs(jobs); + if (!unique.length) return; try { await this.embeddingQueue.addBulk( - jobs.map(job => ({ + unique.map(job => ({ name: embeddingJobId(job), data: job, opts: { @@ -164,7 +186,7 @@ export class QueueController { ); } catch (err) { if (!isDuplicateJobError(err)) throw err; - await Promise.all(jobs.map(job => this.addEmbeddingJob(job, attempts))); + await Promise.all(unique.map(job => this.addEmbeddingJob(job, attempts))); } } } diff --git a/modules/embeddings/src/embeddings.proto b/modules/embeddings/src/embeddings.proto index e44a3f2b2..18f7bc526 100644 --- a/modules/embeddings/src/embeddings.proto +++ b/modules/embeddings/src/embeddings.proto @@ -10,6 +10,7 @@ message EmbeddingConfigRequest { string model = 5; int32 dimensions = 6; optional string similarity = 7; + repeated string sourceFieldAllowlist = 8; } message EmbeddingConfigResponse { @@ -29,6 +30,8 @@ message SemanticSearchRequest { optional int32 limit = 5; optional string userId = 6; optional string scope = 7; + // Honored only when the verified caller is a platform operator module. + optional bool adminOperator = 8; } message EmbeddingsQueryResponse { diff --git a/modules/embeddings/src/metrics/index.ts b/modules/embeddings/src/metrics/index.ts index 52cea73ce..807eed7ab 100644 --- a/modules/embeddings/src/metrics/index.ts +++ b/modules/embeddings/src/metrics/index.ts @@ -15,4 +15,18 @@ export default { help: 'Tracks the total number of failed embedding generation attempts', }, }, + malformedEmbeddingEvents: { + type: MetricType.Counter, + config: { + name: 'malformed_embedding_events_total', + help: 'Tracks malformed or oversized embedding bus payloads', + }, + }, + malformedEmbeddingJobs: { + type: MetricType.Counter, + config: { + name: 'malformed_embedding_jobs_total', + help: 'Tracks malformed or oversized embedding queue payloads', + }, + }, }; diff --git a/modules/embeddings/src/providers/index.test.ts b/modules/embeddings/src/providers/index.test.ts new file mode 100644 index 000000000..33afd2513 --- /dev/null +++ b/modules/embeddings/src/providers/index.test.ts @@ -0,0 +1,60 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { OpenAICompatibleEmbeddingProvider } from './index.js'; + +describe('openai-compatible provider security', () => { + const provider = new OpenAICompatibleEmbeddingProvider({ + lookup: async () => [{ address: '104.18.0.1', family: 4 }], + fetch: async () => { + throw new Error('redirect not allowed'); + }, + }); + + it('rejects redirects, oversize input, and missing allowlists', async () => { + await assert.rejects( + () => + provider.embed('hello', { + endpoint: 'https://api.openai.com/v1/embeddings', + allowedHosts: ['api.openai.com'], + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + await assert.rejects( + () => + provider.embed('x'.repeat(100), { + endpoint: 'https://api.openai.com/v1/embeddings', + allowedHosts: ['api.openai.com'], + maxInputBytes: 8, + }), + err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, + ); + await assert.rejects( + () => + provider.embed('hello', { + endpoint: 'https://api.openai.com/v1/embeddings', + allowedHosts: [], + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + }); + + it('returns embeddings when the allowlisted HTTPS endpoint is safe', async () => { + const safe = new OpenAICompatibleEmbeddingProvider({ + lookup: async () => [{ address: '104.18.0.1', family: 4 }], + fetch: async (_url, init) => { + assert.equal(init?.redirect, 'error'); + return new Response(JSON.stringify({ data: [{ embedding: [0.1, 0.2] }] }), { + status: 200, + }); + }, + }); + const vector = await safe.embed('hello', { + endpoint: 'https://api.openai.com/v1/embeddings', + allowedHosts: ['api.openai.com'], + apiKey: 'sk-test', + }); + assert.deepEqual(vector, [0.1, 0.2]); + }); +}); diff --git a/modules/embeddings/src/providers/index.ts b/modules/embeddings/src/providers/index.ts index 2ecf09f16..01eb05862 100644 --- a/modules/embeddings/src/providers/index.ts +++ b/modules/embeddings/src/providers/index.ts @@ -1,46 +1,131 @@ import { createHash } from 'node:crypto'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertSafeEmbeddingEndpoint, + DEFAULT_EMBED_TIMEOUT_MS, + DEFAULT_MAX_EMBED_INPUT_BYTES, + DEFAULT_MAX_EMBED_RESPONSE_BYTES, + readCappedResponse, + type SafeEndpointOptions, +} from '../utils/endpointSecurity.js'; +import { sanitizeErrorMessage } from '../utils/redactConfig.js'; export interface EmbeddingProviderConfig { endpoint?: string; apiKey?: string; model?: string; + allowedHosts?: string[]; + timeoutMs?: number; + maxInputBytes?: number; + maxResponseBytes?: number; } export interface EmbeddingProvider { embed(input: string, config: EmbeddingProviderConfig): Promise; } +export type EmbeddingFetch = ( + input: string | URL, + init?: RequestInit, +) => Promise; + +export interface EmbeddingProviderDependencies { + fetch?: EmbeddingFetch; + lookup?: SafeEndpointOptions['lookup']; +} + export class OpenAICompatibleEmbeddingProvider implements EmbeddingProvider { + constructor(private readonly deps: EmbeddingProviderDependencies = {}) {} + async embed(input: string, config: EmbeddingProviderConfig): Promise { if (!config.endpoint) { - throw new Error('Embedding provider endpoint is not configured'); + throw new GrpcError( + status.FAILED_PRECONDITION, + 'Embedding provider endpoint is not configured', + ); } - const response = await fetch(config.endpoint, { - method: 'POST', - headers: { - 'content-type': 'application/json', - ...(config.apiKey ? { authorization: `Bearer ${config.apiKey}` } : {}), - }, - body: JSON.stringify({ - input, - model: config.model, - }), + const maxInput = config.maxInputBytes ?? DEFAULT_MAX_EMBED_INPUT_BYTES; + if (Buffer.byteLength(input) > maxInput) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding input exceeds the allowed size', + ); + } + await assertSafeEmbeddingEndpoint(config.endpoint, { + allowedHosts: config.allowedHosts ?? [], + lookup: this.deps.lookup, }); + const timeoutMs = config.timeoutMs ?? DEFAULT_EMBED_TIMEOUT_MS; + const fetchImpl = this.deps.fetch ?? fetch; + let response: Response; + try { + response = await fetchImpl(config.endpoint, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(config.apiKey ? { authorization: `Bearer ${config.apiKey}` } : {}), + }, + body: JSON.stringify({ + input, + model: config.model, + }), + redirect: 'error', + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (err) { + if (err instanceof GrpcError) throw err; + const message = sanitizeErrorMessage(err); + if (/redirect/i.test(message)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embedding provider redirects are not allowed', + ); + } + if (err instanceof Error && err.name === 'TimeoutError') { + throw new GrpcError( + status.DEADLINE_EXCEEDED, + 'Embedding provider request timed out', + ); + } + throw new GrpcError(status.UNAVAILABLE, message); + } if (!response.ok) { - throw new Error(`Embedding provider failed with HTTP ${response.status}`); + throw new GrpcError( + status.UNAVAILABLE, + `Embedding provider failed with HTTP ${response.status}`, + ); + } + const bodyText = await readCappedResponse( + response, + config.maxResponseBytes ?? DEFAULT_MAX_EMBED_RESPONSE_BYTES, + ); + let body: { data?: { embedding?: number[] }[] }; + try { + body = JSON.parse(bodyText) as { data?: { embedding?: number[] }[] }; + } catch { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding provider response was not valid JSON', + ); } - const body = (await response.json()) as { data?: { embedding?: number[] }[] }; const embedding = body.data?.[0]?.embedding; if (!embedding?.length) { - throw new Error('Embedding provider response did not include an embedding'); + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding provider response did not include an embedding', + ); } return embedding; } } -export function getProvider(name: string): EmbeddingProvider { - if (name === 'openai-compatible') return new OpenAICompatibleEmbeddingProvider(); - throw new Error(`Unsupported embedding provider: ${name}`); +export function getProvider( + name: string, + deps: EmbeddingProviderDependencies = {}, +): EmbeddingProvider { + if (name === 'openai-compatible') return new OpenAICompatibleEmbeddingProvider(deps); + throw new GrpcError(status.INVALID_ARGUMENT, `Unsupported embedding provider: ${name}`); } export function hashEmbeddingInput(input: string) { diff --git a/modules/embeddings/src/utils/embeddingJobs.test.ts b/modules/embeddings/src/utils/embeddingJobs.test.ts index 00b3ccc5c..9349c1bb3 100644 --- a/modules/embeddings/src/utils/embeddingJobs.test.ts +++ b/modules/embeddings/src/utils/embeddingJobs.test.ts @@ -4,6 +4,8 @@ import { dedupeEmbeddingJobs, embeddingJobId, isDuplicateJobError, + parseEmbeddingJobData, + parseEmbeddingJobBatch, } from './embeddingJobs.js'; describe('embedding job identity', () => { @@ -25,4 +27,26 @@ describe('embedding job identity', () => { assert.equal(isDuplicateJobError(new Error('Job Article__a already exists')), true); assert.equal(isDuplicateJobError(new Error('redis timeout')), false); }); + + it('rejects malformed and oversized queue payloads', () => { + assert.equal( + parseEmbeddingJobData({ schemaName: 'Article', documentId: 'a' }).ok, + true, + ); + assert.equal(parseEmbeddingJobData({ schemaName: 'Article' }).ok, false); + assert.equal( + parseEmbeddingJobData({ schemaName: '../etc', documentId: 'a' }).ok, + false, + ); + assert.equal( + parseEmbeddingJobData({ schemaName: 'Article', documentId: 'a', extra: true }).ok, + false, + ); + assert.equal( + parseEmbeddingJobBatch( + new Array(600).fill({ schemaName: 'Article', documentId: 'a' }), + ).length, + 500, + ); + }); }); diff --git a/modules/embeddings/src/utils/embeddingJobs.ts b/modules/embeddings/src/utils/embeddingJobs.ts index b6f9d82f0..a4bd1a048 100644 --- a/modules/embeddings/src/utils/embeddingJobs.ts +++ b/modules/embeddings/src/utils/embeddingJobs.ts @@ -4,6 +4,13 @@ export interface EmbeddingJobData { configId?: string; } +export const MAX_SCHEMA_NAME_LENGTH = 128; +export const MAX_DOCUMENT_ID_LENGTH = 128; +export const MAX_QUEUE_BATCH_SIZE = 500; + +const IDENTITY = /^[A-Za-z0-9._-]{1,128}$/; +const SCHEMA_NAME = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; + export function embeddingJobId(data: EmbeddingJobData): string { const parts = [data.schemaName, data.documentId]; if (data.configId) parts.push(data.configId); @@ -26,3 +33,51 @@ export function isDuplicateJobError(err: unknown): boolean { const message = err instanceof Error ? err.message : String(err); return /already exists/i.test(message); } + +export type ParsedEmbeddingJob = + { ok: true; data: EmbeddingJobData } | { ok: false; reason: string }; + +export function parseEmbeddingJobData( + value: unknown, + maxBatchIndex?: number, +): ParsedEmbeddingJob { + if (maxBatchIndex !== undefined && maxBatchIndex >= MAX_QUEUE_BATCH_SIZE) { + return { ok: false, reason: 'batch_size' }; + } + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return { ok: false, reason: 'malformed' }; + } + const record = value as Record; + const extraKeys = Object.keys(record).filter( + key => !['schemaName', 'documentId', 'configId'].includes(key), + ); + if (extraKeys.length) return { ok: false, reason: 'malformed' }; + if (typeof record.schemaName !== 'string' || !SCHEMA_NAME.test(record.schemaName)) { + return { ok: false, reason: 'schemaName' }; + } + if (typeof record.documentId !== 'string' || !IDENTITY.test(record.documentId)) { + return { ok: false, reason: 'documentId' }; + } + if ( + record.configId !== undefined && + (typeof record.configId !== 'string' || !IDENTITY.test(record.configId)) + ) { + return { ok: false, reason: 'configId' }; + } + return { + ok: true, + data: { + schemaName: record.schemaName, + documentId: record.documentId, + ...(record.configId ? { configId: record.configId } : {}), + }, + }; +} + +export function parseEmbeddingJobBatch(values: unknown[]): EmbeddingJobData[] { + return values + .slice(0, MAX_QUEUE_BATCH_SIZE) + .map(value => parseEmbeddingJobData(value)) + .filter((parsed): parsed is { ok: true; data: EmbeddingJobData } => parsed.ok) + .map(parsed => parsed.data); +} diff --git a/modules/embeddings/src/utils/endpointSecurity.test.ts b/modules/embeddings/src/utils/endpointSecurity.test.ts new file mode 100644 index 000000000..9d73e3586 --- /dev/null +++ b/modules/embeddings/src/utils/endpointSecurity.test.ts @@ -0,0 +1,65 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertSafeEmbeddingEndpoint, + isBlockedIp, + readCappedResponse, +} from './endpointSecurity.js'; + +describe('embedding endpoint SSRF controls', () => { + it('blocks private, link-local, and metadata addresses', () => { + assert.equal(isBlockedIp('127.0.0.1'), true); + assert.equal(isBlockedIp('10.0.0.5'), true); + assert.equal(isBlockedIp('192.168.1.20'), true); + assert.equal(isBlockedIp('169.254.169.254'), true); + assert.equal(isBlockedIp('::1'), true); + assert.equal(isBlockedIp('::ffff:127.0.0.1'), true); + assert.equal(isBlockedIp('8.8.8.8'), false); + }); + + it('requires HTTPS and an allowlisted host', async () => { + await assert.rejects( + () => + assertSafeEmbeddingEndpoint('http://api.openai.com/v1/embeddings', { + allowedHosts: ['api.openai.com'], + }), + err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, + ); + await assert.rejects( + () => + assertSafeEmbeddingEndpoint('https://evil.example/v1/embeddings', { + allowedHosts: ['api.openai.com'], + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + }); + + it('rejects DNS results that resolve to private or metadata addresses', async () => { + await assert.rejects( + () => + assertSafeEmbeddingEndpoint('https://api.openai.com/v1/embeddings', { + allowedHosts: ['api.openai.com'], + lookup: async () => [{ address: '169.254.169.254', family: 4 }], + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + const url = await assertSafeEmbeddingEndpoint( + 'https://api.openai.com/v1/embeddings', + { + allowedHosts: ['api.openai.com'], + lookup: async () => [{ address: '104.18.0.1', family: 4 }], + }, + ); + assert.equal(url.hostname, 'api.openai.com'); + }); + + it('caps response payload size', async () => { + const response = new Response('x'.repeat(20), { status: 200 }); + await assert.rejects( + () => readCappedResponse(response, 8), + err => err instanceof GrpcError && err.code === status.RESOURCE_EXHAUSTED, + ); + }); +}); diff --git a/modules/embeddings/src/utils/endpointSecurity.ts b/modules/embeddings/src/utils/endpointSecurity.ts new file mode 100644 index 000000000..6069453cb --- /dev/null +++ b/modules/embeddings/src/utils/endpointSecurity.ts @@ -0,0 +1,145 @@ +import { BlockList, isIP } from 'node:net'; +import { lookup as defaultLookup } from 'node:dns/promises'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export const DEFAULT_EMBED_TIMEOUT_MS = 10_000; +export const DEFAULT_MAX_EMBED_INPUT_BYTES = 32 * 1024; +export const DEFAULT_MAX_EMBED_RESPONSE_BYTES = 1024 * 1024; + +const BLOCKED_HOSTNAMES = new Set([ + 'localhost', + 'metadata', + 'metadata.google.internal', + 'metadata.google.com', +]); + +const privateNetworks = new BlockList(); +privateNetworks.addSubnet('0.0.0.0', 8, 'ipv4'); +privateNetworks.addSubnet('10.0.0.0', 8, 'ipv4'); +privateNetworks.addSubnet('100.64.0.0', 10, 'ipv4'); +privateNetworks.addSubnet('127.0.0.0', 8, 'ipv4'); +privateNetworks.addSubnet('169.254.0.0', 16, 'ipv4'); +privateNetworks.addSubnet('172.16.0.0', 12, 'ipv4'); +privateNetworks.addSubnet('192.168.0.0', 16, 'ipv4'); +privateNetworks.addSubnet('::1', 128, 'ipv6'); +privateNetworks.addAddress('::', 'ipv6'); +privateNetworks.addSubnet('fc00::', 7, 'ipv6'); +privateNetworks.addSubnet('fe80::', 10, 'ipv6'); + +export interface SafeEndpointOptions { + allowedHosts: string[]; + lookup?: ( + hostname: string, + options: { all: true; verbatim: true }, + ) => Promise>; +} + +export function isBlockedIp(address: string): boolean { + const mapped = address.startsWith('::ffff:') ? address.slice(7) : address; + const ipVersion = isIP(mapped); + if (ipVersion === 4) return privateNetworks.check(mapped, 'ipv4'); + if (ipVersion === 6) return privateNetworks.check(mapped, 'ipv6'); + return true; +} + +export async function assertSafeEmbeddingEndpoint( + endpoint: string, + options: SafeEndpointOptions, +): Promise { + let url: URL; + try { + url = new URL(endpoint); + } catch { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding provider endpoint is invalid', + ); + } + if (url.protocol !== 'https:') { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding provider endpoint must use HTTPS', + ); + } + if (url.username || url.password) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding provider endpoint must not include credentials', + ); + } + const hostname = url.hostname.toLowerCase(); + if (BLOCKED_HOSTNAMES.has(hostname)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embedding provider endpoint host is not allowed', + ); + } + const allowed = new Set(options.allowedHosts.map(host => host.toLowerCase())); + if (!allowed.has(hostname)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embedding provider endpoint host is not allowlisted', + ); + } + if (isIP(hostname)) { + if (isBlockedIp(hostname)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embedding provider endpoint resolves to a blocked address', + ); + } + return url; + } + const lookup = options.lookup ?? defaultLookup; + const resolved = await lookup(hostname, { all: true, verbatim: true }); + const records = Array.isArray(resolved) ? resolved : [resolved]; + if (!records.length) { + throw new GrpcError( + status.FAILED_PRECONDITION, + 'Embedding provider endpoint host could not be resolved', + ); + } + for (const record of records) { + if (isBlockedIp(record.address)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embedding provider endpoint resolves to a blocked address', + ); + } + } + return url; +} + +export async function readCappedResponse( + response: Response, + maxBytes: number, +): Promise { + if (!response.body) { + const text = await response.text(); + if (Buffer.byteLength(text) > maxBytes) { + throw new GrpcError( + status.RESOURCE_EXHAUSTED, + 'Embedding provider response is too large', + ); + } + return text; + } + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let received = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + received += value.byteLength; + if (received > maxBytes) { + await reader.cancel(); + throw new GrpcError( + status.RESOURCE_EXHAUSTED, + 'Embedding provider response is too large', + ); + } + chunks.push(value); + } + return Buffer.concat(chunks).toString('utf8'); +} diff --git a/modules/embeddings/src/utils/mutationEvents.test.ts b/modules/embeddings/src/utils/mutationEvents.test.ts index e45ca6625..c6cada7bc 100644 --- a/modules/embeddings/src/utils/mutationEvents.test.ts +++ b/modules/embeddings/src/utils/mutationEvents.test.ts @@ -4,6 +4,7 @@ import { embeddingOwnedFields, extractDocumentIds, isEmbeddingOwnedMutation, + parseBoundedMutationEvent, parseMutationEvent, } from './mutationEvents.js'; @@ -65,4 +66,19 @@ describe('embedding mutation event parsing', () => { it('returns null for malformed payloads', () => { assert.equal(parseMutationEvent('{not json'), null); }); + + it('fails closed on oversized bus payloads instead of crashing', () => { + assert.deepEqual(parseBoundedMutationEvent('{not json'), { + ok: false, + reason: 'malformed', + }); + assert.deepEqual(parseBoundedMutationEvent('x'.repeat(300_000)), { + ok: false, + reason: 'capped', + }); + assert.deepEqual( + parseBoundedMutationEvent(JSON.stringify({ ids: ['a', 'b', 'c'] }), 2), + { ok: false, reason: 'capped' }, + ); + }); }); diff --git a/modules/embeddings/src/utils/mutationEvents.ts b/modules/embeddings/src/utils/mutationEvents.ts index 344a2b4b3..9abbb97a9 100644 --- a/modules/embeddings/src/utils/mutationEvents.ts +++ b/modules/embeddings/src/utils/mutationEvents.ts @@ -5,16 +5,41 @@ export interface ParsedMutationEvent { ids: string[]; } +export const MAX_MUTATION_EVENT_BYTES = 256 * 1024; +export const MAX_MUTATION_EVENT_IDS = 500; + +export type MutationEventParseResult = + | { ok: true; event: ParsedMutationEvent } + | { ok: false; reason: 'malformed' | 'capped' }; + export function parseMutationEvent(message: string): ParsedMutationEvent | null { + const parsed = parseBoundedMutationEvent(message); + return parsed.ok ? parsed.event : null; +} + +export function parseBoundedMutationEvent( + message: string, + maxIds: number = MAX_MUTATION_EVENT_IDS, +): MutationEventParseResult { + if (typeof message !== 'string' || !message.length) { + return { ok: false, reason: 'malformed' }; + } + if (message.length > MAX_MUTATION_EVENT_BYTES) { + return { ok: false, reason: 'capped' }; + } let payload: unknown; try { payload = JSON.parse(message); } catch { - return null; + return { ok: false, reason: 'malformed' }; + } + const ids = uniqueIds(extractDocumentIds(payload)); + if (ids.length > maxIds) { + return { ok: false, reason: 'capped' }; } return { - payload, - ids: uniqueIds(extractDocumentIds(payload)), + ok: true, + event: { payload, ids }, }; } diff --git a/modules/embeddings/src/utils/productionSecurity.test.ts b/modules/embeddings/src/utils/productionSecurity.test.ts new file mode 100644 index 000000000..61fe05852 --- /dev/null +++ b/modules/embeddings/src/utils/productionSecurity.test.ts @@ -0,0 +1,29 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { assertGrpcKeyRequirement } from './productionSecurity.js'; + +describe('production GRPC_KEY requirement', () => { + it('requires GRPC_KEY when NODE_ENV is production', () => { + assert.throws( + () => assertGrpcKeyRequirement({ NODE_ENV: 'production' }), + err => err instanceof GrpcError && err.code === status.FAILED_PRECONDITION, + ); + assert.doesNotThrow(() => + assertGrpcKeyRequirement({ NODE_ENV: 'production', GRPC_KEY: 'secret' }), + ); + }); + + it('does not require GRPC_KEY in non-production unless configured', () => { + assert.doesNotThrow(() => assertGrpcKeyRequirement({ NODE_ENV: 'test' })); + assert.throws( + () => + assertGrpcKeyRequirement( + { NODE_ENV: 'development' }, + { security: { requireGrpcKey: true } }, + ), + err => err instanceof GrpcError && err.code === status.FAILED_PRECONDITION, + ); + }); +}); diff --git a/modules/embeddings/src/utils/productionSecurity.ts b/modules/embeddings/src/utils/productionSecurity.ts new file mode 100644 index 000000000..3ee7e3f2f --- /dev/null +++ b/modules/embeddings/src/utils/productionSecurity.ts @@ -0,0 +1,25 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export function assertGrpcKeyRequirement( + env: NodeJS.ProcessEnv = process.env, + config?: { security?: { requireGrpcKey?: boolean } }, +): void { + const required = + env.NODE_ENV === 'production' || config?.security?.requireGrpcKey === true; + if (required && !env.GRPC_KEY) { + throw new GrpcError( + status.FAILED_PRECONDITION, + 'GRPC_KEY is required for embeddings in production', + ); + } +} + +export function callerModuleName(metadata?: { + get(key: string): Array; +}): string | undefined { + const value = metadata?.get('module-name')?.[0]; + if (typeof value === 'string' && value.length > 0) return value; + if (Buffer.isBuffer(value) && value.length > 0) return value.toString(); + return undefined; +} diff --git a/modules/embeddings/src/utils/redactConfig.test.ts b/modules/embeddings/src/utils/redactConfig.test.ts new file mode 100644 index 000000000..a82efe973 --- /dev/null +++ b/modules/embeddings/src/utils/redactConfig.test.ts @@ -0,0 +1,46 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { redactSensitiveConfig } from '@conduitplatform/module-tools'; +import { redactProviderConfig, redactSecretText } from './redactConfig.js'; + +describe('provider secret redaction', () => { + it('redacts API keys from config objects and error text', () => { + assert.equal( + redactProviderConfig({ endpoint: 'https://api.openai.com', apiKey: 'sk-secret' }) + .apiKey, + '[REDACTED]', + ); + assert.match( + redactSecretText('Embedding failed Bearer sk-secret apiKey=sk-secret'), + /\[REDACTED\]/, + ); + assert.doesNotMatch( + redactSecretText('Embedding failed Bearer sk-secret apiKey=sk-secret'), + /sk-secret/, + ); + }); + + it('redacts convict-sensitive and well-known secret keys', () => { + const redacted = redactSensitiveConfig( + { + enabled: true, + providers: { + 'openai-compatible': { endpoint: 'https://api.openai.com', apiKey: 'sk-live' }, + }, + }, + { + providers: { + 'openai-compatible': { + apiKey: { format: 'String', default: '', sensitive: true }, + endpoint: { format: 'String', default: '' }, + }, + }, + }, + ); + assert.equal(redacted.providers['openai-compatible'].apiKey, '[REDACTED]'); + assert.equal( + redacted.providers['openai-compatible'].endpoint, + 'https://api.openai.com', + ); + }); +}); diff --git a/modules/embeddings/src/utils/redactConfig.ts b/modules/embeddings/src/utils/redactConfig.ts new file mode 100644 index 000000000..6a5da9e17 --- /dev/null +++ b/modules/embeddings/src/utils/redactConfig.ts @@ -0,0 +1,23 @@ +export function redactSecretText(value: string): string { + return value + .replace(/Bearer\s+\S+/gi, 'Bearer [REDACTED]') + .replace(/(apiKey|api_key|password|secret)\s*[:=]\s*\S+/gi, '$1:[REDACTED]'); +} + +export function sanitizeErrorMessage(err: unknown): string { + const message = err instanceof Error ? err.message : String(err); + return redactSecretText(message); +} + +export function redactProviderConfig>(config: T): T { + const redacted = { ...config }; + for (const key of Object.keys(redacted)) { + if ( + /^(apiKey|api_key|password|secret)$/i.test(key) && + typeof redacted[key] === 'string' + ) { + (redacted as Record)[key] = '[REDACTED]'; + } + } + return redacted; +} diff --git a/modules/embeddings/src/utils/schemaPolicy.test.ts b/modules/embeddings/src/utils/schemaPolicy.test.ts new file mode 100644 index 000000000..f4070eb23 --- /dev/null +++ b/modules/embeddings/src/utils/schemaPolicy.test.ts @@ -0,0 +1,117 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertCanManageEmbeddingConfig, + assertEmbeddingTargetSchema, + assertSemanticSearchAccess, + assertSourceFields, + isDeniedEmbeddingSchema, + resolveAdminOperatorContext, +} from './schemaPolicy.js'; + +describe('embedding schema and source policies', () => { + it('denies system, auth-secret, and EmbeddingConfig schemas', () => { + assert.equal(isDeniedEmbeddingSchema({ name: 'EmbeddingConfig' }), true); + assert.equal(isDeniedEmbeddingSchema({ name: '_DeclaredSchema' }), true); + assert.equal(isDeniedEmbeddingSchema({ name: 'Views' }), true); + assert.equal(isDeniedEmbeddingSchema({ name: 'AccessToken' }), true); + assert.equal(isDeniedEmbeddingSchema({ name: 'TwoFactorSecret' }), true); + assert.equal(isDeniedEmbeddingSchema({ name: 'Article' }), false); + assert.throws( + () => assertEmbeddingTargetSchema({ name: 'RefreshToken' }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + }); + + it('restricts config and backfill to the schema owner or platform admin', () => { + assert.doesNotThrow(() => + assertCanManageEmbeddingConfig({ + callerModule: 'cms-app', + ownerModule: 'cms-app', + schemaName: 'Article', + }), + ); + assert.doesNotThrow(() => + assertCanManageEmbeddingConfig({ + callerModule: 'database', + ownerModule: 'cms-app', + schemaName: 'Article', + }), + ); + assert.throws( + () => + assertCanManageEmbeddingConfig({ + callerModule: 'chat', + ownerModule: 'cms-app', + schemaName: 'Article', + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + }); + + it('rejects hidden, non-string, and sensitive source fields unless allowlisted', () => { + const schemaFields = { + title: { type: TYPE.String }, + body: { type: TYPE.String }, + password: { type: TYPE.String }, + token: { type: TYPE.String, select: false }, + views: { type: TYPE.Number }, + notes: { type: TYPE.String, select: false }, + }; + assert.doesNotThrow(() => + assertSourceFields({ + sourceFields: ['title', 'body'], + schemaFields, + }), + ); + assert.throws( + () => assertSourceFields({ sourceFields: ['password'], schemaFields }), + /sensitive/, + ); + assert.throws( + () => assertSourceFields({ sourceFields: ['token'], schemaFields }), + /hidden|sensitive/, + ); + assert.throws( + () => assertSourceFields({ sourceFields: ['views'], schemaFields }), + /string-like/, + ); + assert.throws( + () => assertSourceFields({ sourceFields: ['notes'], schemaFields }), + /hidden/, + ); + assert.doesNotThrow(() => + assertSourceFields({ + sourceFields: ['notes'], + schemaFields, + allowlist: ['notes'], + }), + ); + }); + + it('requires subject, scope, or a verified admin operator for semantic search', () => { + assert.throws( + () => assertSemanticSearchAccess({}), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + assert.doesNotThrow(() => assertSemanticSearchAccess({ userId: 'u1' })); + assert.doesNotThrow(() => + assertSemanticSearchAccess({ + adminOperator: resolveAdminOperatorContext({ + requested: true, + callerModule: 'core', + }), + }), + ); + assert.throws( + () => + resolveAdminOperatorContext({ + requested: true, + callerModule: 'chat', + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + }); +}); diff --git a/modules/embeddings/src/utils/schemaPolicy.ts b/modules/embeddings/src/utils/schemaPolicy.ts new file mode 100644 index 000000000..2988180e5 --- /dev/null +++ b/modules/embeddings/src/utils/schemaPolicy.ts @@ -0,0 +1,164 @@ +import { TYPE } from '@conduitplatform/grpc-sdk'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export const EMBEDDING_CONFIG_SCHEMA = 'EmbeddingConfig'; +export const CONFIG_OPERATOR_MODULES = ['database', 'core'] as const; +export const SEARCH_OPERATOR_MODULES = ['database', 'core', 'embeddings'] as const; + +export const AUTH_SECRET_SCHEMA_NAMES = new Set([ + 'AccessToken', + 'RefreshToken', + 'Token', + 'TwoFactorSecret', + 'TwoFactorBackUpCodes', + 'BiometricToken', + 'AdminTwoFactorSecret', + 'AdminApiToken', +]); + +export const SYSTEM_SCHEMA_NAMES = new Set([ + 'Views', + 'Config', + 'MigratedSchemas', + 'PendingSchemas', + 'CustomEndpoints', +]); + +const SENSITIVE_FIELD_NAME = + /(password|secret|token|credential|apikey|api_key|private[_-]?key|authorization|refresh[_-]?token|access[_-]?token)/i; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function isSensitiveFieldName(field: string): boolean { + return SENSITIVE_FIELD_NAME.test(field); +} + +export function isStringLikeField(field: unknown): boolean { + if (field === TYPE.String || field === 'String') return true; + if (Array.isArray(field) && field.length === 1) return isStringLikeField(field[0]); + if (!isRecord(field)) return false; + if (field.type === TYPE.String || field.type === 'String') return true; + return Array.isArray(field.type) && isStringLikeField(field.type); +} + +export function isHiddenField(field: unknown): boolean { + return isRecord(field) && field.select === false; +} + +export function isDeniedEmbeddingSchema(schema: { + name: string; + ownerModule?: string; +}): boolean { + if (!schema.name) return true; + if (schema.name === EMBEDDING_CONFIG_SCHEMA) return true; + if (schema.name.startsWith('_')) return true; + if (SYSTEM_SCHEMA_NAMES.has(schema.name)) return true; + return AUTH_SECRET_SCHEMA_NAMES.has(schema.name); +} + +export function assertEmbeddingTargetSchema(schema: { + name: string; + ownerModule?: string; +}): void { + if (!isDeniedEmbeddingSchema(schema)) return; + throw new GrpcError( + status.PERMISSION_DENIED, + `Schema '${schema.name}' cannot be used as an embedding source`, + ); +} + +export function canManageEmbeddingConfig(args: { + callerModule?: string; + ownerModule?: string; + operatorModules?: readonly string[]; +}): boolean { + if (!args.callerModule) return false; + if (args.ownerModule && args.callerModule === args.ownerModule) return true; + const operators = args.operatorModules ?? CONFIG_OPERATOR_MODULES; + return operators.includes(args.callerModule); +} + +export function assertCanManageEmbeddingConfig(args: { + callerModule?: string; + ownerModule?: string; + schemaName: string; +}): void { + if (canManageEmbeddingConfig(args)) return; + throw new GrpcError( + status.PERMISSION_DENIED, + `Module '${args.callerModule ?? 'unknown'}' is not allowed to manage embeddings for '${args.schemaName}'`, + ); +} + +export function resolveAdminOperatorContext(args: { + requested?: boolean; + callerModule?: string; + operatorModules?: readonly string[]; +}): boolean { + if (!args.requested) return false; + const operators = args.operatorModules ?? SEARCH_OPERATOR_MODULES; + if (!args.callerModule || !operators.includes(args.callerModule)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Admin operator context is not allowed for this caller', + ); + } + return true; +} + +export function assertSemanticSearchAccess(args: { + userId?: string; + scope?: string; + adminOperator?: boolean; +}): void { + if (args.userId || args.scope || args.adminOperator) return; + throw new GrpcError( + status.PERMISSION_DENIED, + 'Semantic search requires a subject, scope, or admin operator context', + ); +} + +export function assertSourceFields(args: { + sourceFields: string[]; + schemaFields: Record; + allowlist?: string[]; +}): void { + const allowlist = new Set(args.allowlist ?? []); + for (const field of args.sourceFields) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(field)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Invalid source field name '${field}'`, + ); + } + const definition = args.schemaFields[field]; + if (definition === undefined) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Source field '${field}' does not exist on the target schema`, + ); + } + if (!isStringLikeField(definition)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Source field '${field}' must be string-like`, + ); + } + const allowed = allowlist.has(field); + if (!allowed && isHiddenField(definition)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Source field '${field}' is hidden and cannot be embedded`, + ); + } + if (!allowed && isSensitiveFieldName(field)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Source field '${field}' looks sensitive and cannot be embedded`, + ); + } + } +} diff --git a/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts b/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts index 4ae68d177..c6654f8b7 100644 --- a/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts +++ b/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts @@ -24,6 +24,7 @@ describe('validateEmbeddingConfigInput', () => { assert.equal(result.similarity, VectorSimilarity.DotProduct); assert.equal(result.provider, 'openai-compatible'); assert.equal(result.dimensions, 1536); + assert.deepEqual(result.sourceFieldAllowlist, []); }); it('defaults omitted similarity to cosine', () => { @@ -55,4 +56,20 @@ describe('validateEmbeddingConfigInput', () => { /Unsupported similarity/, ); }); + + it('validates source fields against the schema when provided', () => { + assert.throws( + () => + validateEmbeddingConfigInput(valid, defaults, { + title: { type: 'String' }, + password: { type: 'String' }, + }), + /does not exist/, + ); + const result = validateEmbeddingConfigInput(valid, defaults, { + title: { type: 'String' }, + body: { type: 'String' }, + }); + assert.deepEqual(result.sourceFields, ['title', 'body']); + }); }); diff --git a/modules/embeddings/src/utils/validateEmbeddingConfig.ts b/modules/embeddings/src/utils/validateEmbeddingConfig.ts index 704d964b9..967e52164 100644 --- a/modules/embeddings/src/utils/validateEmbeddingConfig.ts +++ b/modules/embeddings/src/utils/validateEmbeddingConfig.ts @@ -1,4 +1,6 @@ -import { VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { GrpcError, VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { assertSourceFields } from './schemaPolicy.js'; export interface EmbeddingConfigInput { schemaName?: string; @@ -8,6 +10,7 @@ export interface EmbeddingConfigInput { model?: string; dimensions?: number; similarity?: string; + sourceFieldAllowlist?: string[]; } export interface ValidatedEmbeddingConfig { @@ -18,6 +21,7 @@ export interface ValidatedEmbeddingConfig { modelName: string | undefined; dimensions: number; similarity: VectorSimilarity; + sourceFieldAllowlist: string[]; } const SUPPORTED_SIMILARITY = Object.values(VectorSimilarity); @@ -25,9 +29,19 @@ const SUPPORTED_SIMILARITY = Object.values(VectorSimilarity); export function validateEmbeddingConfigInput( request: EmbeddingConfigInput, defaults: { provider: string }, + schemaFields?: Record, ): ValidatedEmbeddingConfig { if (!request.schemaName || !request.targetField || !request.sourceFields?.length) { - throw new Error('schemaName, targetField, and sourceFields are required'); + throw new GrpcError( + status.INVALID_ARGUMENT, + 'schemaName, targetField, and sourceFields are required', + ); + } + if (!/^[A-Za-z_][A-Za-z0-9_]{0,127}$/.test(request.schemaName)) { + throw new GrpcError(status.INVALID_ARGUMENT, 'schemaName is invalid'); + } + if (!/^[A-Za-z_][A-Za-z0-9_]{0,127}$/.test(request.targetField)) { + throw new GrpcError(status.INVALID_ARGUMENT, 'targetField is invalid'); } const dimensions = request.dimensions; if ( @@ -35,14 +49,25 @@ export function validateEmbeddingConfigInput( !Number.isInteger(dimensions) || dimensions <= 0 ) { - throw new Error('dimensions must be a positive integer'); + throw new GrpcError(status.INVALID_ARGUMENT, 'dimensions must be a positive integer'); } const similarity = request.similarity || VectorSimilarity.Cosine; if (!SUPPORTED_SIMILARITY.includes(similarity as VectorSimilarity)) { - throw new Error( + throw new GrpcError( + status.INVALID_ARGUMENT, `Unsupported similarity '${similarity}'. Supported values: ${SUPPORTED_SIMILARITY.join(', ')}`, ); } + const sourceFieldAllowlist = (request.sourceFieldAllowlist ?? []).filter( + field => typeof field === 'string' && field.length > 0, + ); + if (schemaFields) { + assertSourceFields({ + sourceFields: request.sourceFields, + schemaFields, + allowlist: sourceFieldAllowlist, + }); + } return { schemaName: request.schemaName, sourceFields: request.sourceFields, @@ -51,5 +76,6 @@ export function validateEmbeddingConfigInput( modelName: request.model, dimensions, similarity: similarity as VectorSimilarity, + sourceFieldAllowlist, }; } diff --git a/modules/embeddings/tsconfig.test.json b/modules/embeddings/tsconfig.test.json index c3bb42d36..5179fd978 100644 --- a/modules/embeddings/tsconfig.test.json +++ b/modules/embeddings/tsconfig.test.json @@ -10,7 +10,7 @@ "include": [ "src/utils/**/*.ts", "src/controllers/**/*.ts", - "src/providers/index.ts" + "src/providers/**/*.ts" ], "exclude": [] } diff --git a/packages/core/src/admin/routes/GetModuleConfig.route.ts b/packages/core/src/admin/routes/GetModuleConfig.route.ts index 3c1d4db0a..b71b62168 100644 --- a/packages/core/src/admin/routes/GetModuleConfig.route.ts +++ b/packages/core/src/admin/routes/GetModuleConfig.route.ts @@ -5,6 +5,7 @@ import { ConduitRouteReturnDefinition, } from '@conduitplatform/grpc-sdk'; import { ConduitRoute } from '@conduitplatform/hermes'; +import { redactSensitiveConfig } from '@conduitplatform/module-tools'; import convict from 'convict'; export function getModuleConfigRoute( @@ -32,7 +33,7 @@ export function getModuleConfigRoute( } else { finalConfig = JSON.parse(finalConfig); } - return { config: finalConfig }; + return { config: redactSensitiveConfig(finalConfig, configSchema) }; }, ); } diff --git a/packages/core/src/admin/routes/GetMonoConfig.route.ts b/packages/core/src/admin/routes/GetMonoConfig.route.ts index 25b768621..10a2c7009 100644 --- a/packages/core/src/admin/routes/GetMonoConfig.route.ts +++ b/packages/core/src/admin/routes/GetMonoConfig.route.ts @@ -3,7 +3,7 @@ import { ConduitRouteActions, ConduitRouteReturnDefinition, } from '@conduitplatform/grpc-sdk'; -import { ConduitJson } from '@conduitplatform/module-tools'; +import { ConduitJson, redactSensitiveConfig } from '@conduitplatform/module-tools'; import { ConduitRoute } from '@conduitplatform/hermes'; import { ServiceRegistry } from '../../service-discovery/ServiceRegistry.js'; @@ -27,7 +27,11 @@ export function getMonoConfigRoute(grpcSdk: ConduitGrpcSdk) { ].sort(); for (const moduleName of sortedModules) { const moduleConfig = await grpcSdk.state!.getKey(`moduleConfigs.${moduleName}`); - if (moduleConfig) monoConfig.modules[moduleName] = JSON.parse(moduleConfig); + if (moduleConfig) { + monoConfig.modules[moduleName] = redactSensitiveConfig( + JSON.parse(moduleConfig), + ); + } } return { config: monoConfig }; }, diff --git a/packages/core/src/admin/routes/SetModuleConfig.route.ts b/packages/core/src/admin/routes/SetModuleConfig.route.ts index 1bf6ec2f2..adc8cea73 100644 --- a/packages/core/src/admin/routes/SetModuleConfig.route.ts +++ b/packages/core/src/admin/routes/SetModuleConfig.route.ts @@ -9,6 +9,7 @@ import { } from '@conduitplatform/grpc-sdk'; // Removed ConduitCommons import - now using configManager directly import { ConduitRoute } from '@conduitplatform/hermes'; +import { redactSensitiveConfig } from '@conduitplatform/module-tools'; import convict from 'convict'; type SetConfig = (config: { newConfig: string }) => Promise<{ updatedConfig: string }>; @@ -66,7 +67,7 @@ export function setModuleConfigRoute( updatedConfig = JSON.parse(updatedConfig.updatedConfig); } await configManager.set(moduleName, updatedConfig); - return { config: updatedConfig }; + return { config: redactSensitiveConfig(updatedConfig, configSchema) }; }, ); } From e4f7d52b8f0c672b633ff30dfccc97561d39f657 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 18:40:49 +0300 Subject: [PATCH 05/29] feat(database): bound vector search authorization and normalize similarity scores Replace unbounded authorized-ID materialization with ANN prefilters plus bounded candidate checks, and document a higher-is-better score contract with strict shared filter and limit validation. --- libraries/grpc-sdk/src/interfaces/Model.ts | 14 ++ .../database/src/adapters/SchemaAdapter.ts | 27 +++ .../src/adapters/mongoose-adapter/index.ts | 93 +++----- .../src/adapters/sequelize-adapter/index.ts | 149 +++---------- .../utils/__tests__/grpcStatus.test.ts | 28 +++ .../utils/__tests__/vectorProjection.test.ts | 40 ++++ .../utils/__tests__/vectorScore.test.ts | 86 +++++++ .../__tests__/vectorSearchAdapters.test.ts | 204 +++++++++++++++++ .../utils/__tests__/vectorSearchAuth.test.ts | 36 ++- .../__tests__/vectorSearchFilter.test.ts | 119 ++++++++++ .../__tests__/vectorSearchLimits.test.ts | 45 ++++ .../utils/__tests__/vectorSearchQuery.test.ts | 164 ++++++++++++++ .../utils/__tests__/vectorSearchWhere.test.ts | 43 ++++ modules/database/src/adapters/utils/index.ts | 6 + .../src/adapters/utils/vectorProjection.ts | 73 ++++++ .../src/adapters/utils/vectorScore.ts | 114 ++++++++++ .../src/adapters/utils/vectorSearchAuth.ts | 29 +++ .../src/adapters/utils/vectorSearchFilter.ts | 195 ++++++++++++++++ .../src/adapters/utils/vectorSearchLimits.ts | 44 ++++ .../src/adapters/utils/vectorSearchQuery.ts | 211 ++++++++++++++++++ .../src/adapters/utils/vectorSearchWhere.ts | 134 +++++++++++ 21 files changed, 1675 insertions(+), 179 deletions(-) create mode 100644 modules/database/src/adapters/utils/__tests__/grpcStatus.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/vectorProjection.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/vectorScore.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/vectorSearchAdapters.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/vectorSearchFilter.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/vectorSearchLimits.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/vectorSearchWhere.test.ts create mode 100644 modules/database/src/adapters/utils/vectorProjection.ts create mode 100644 modules/database/src/adapters/utils/vectorScore.ts create mode 100644 modules/database/src/adapters/utils/vectorSearchFilter.ts create mode 100644 modules/database/src/adapters/utils/vectorSearchLimits.ts create mode 100644 modules/database/src/adapters/utils/vectorSearchQuery.ts create mode 100644 modules/database/src/adapters/utils/vectorSearchWhere.ts diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 19dcc3e0f..414ecf6fb 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -38,6 +38,8 @@ export enum VectorIndexMethod { Flat = 'flat', } +export type VectorSearchProvider = 'mongodb' | 'postgres'; + export enum MongoIndexType { Ascending = 1, Descending = -1, @@ -321,5 +323,17 @@ export interface VectorSearchInput { export interface VectorSearchResult { document: T; + /** + * Provider-neutral, higher-is-better similarity. + * + * Cosine is normalized so identical vectors score `1` (Postgres cosine + * distance is converted with `1 - distance`). Euclidean and inner-product + * scores are also higher-is-better ranking values, but they are **not** + * comparable across Mongo Atlas Vector Search and pgvector. + */ score: number; + /** Raw backend distance or provider score before Conduit normalization. */ + distance?: number; + metric?: VectorSimilarity; + provider?: VectorSearchProvider; } diff --git a/modules/database/src/adapters/SchemaAdapter.ts b/modules/database/src/adapters/SchemaAdapter.ts index e3a8559b1..4c6f5b390 100644 --- a/modules/database/src/adapters/SchemaAdapter.ts +++ b/modules/database/src/adapters/SchemaAdapter.ts @@ -218,6 +218,33 @@ export abstract class SchemaAdapter { } } + async lookupAuthorizedCandidateIds( + operation: string, + candidateIds: Array, + userId?: string, + scope?: string, + ): Promise { + const ids = candidateIds.map(id => String(id)).filter(Boolean); + if (!ids.length) return []; + if (!this.authzEnabled || (isNil(userId) && isNil(scope))) { + return ids; + } + const view = await this.permissionCheck(operation, userId, scope); + if (!view) return ids; + const query = + this.adapter.getDatabaseType() === 'MongoDB' + ? { _id: { $in: ids } } + : { _id: { [Op.in]: ids } }; + const docs = await this.runAuthorizedViewQuery(operation, userId, scope, view, v => + v.findMany(query, { + select: '_id', + userId: undefined, + scope: undefined, + }), + ); + return (docs ?? []).map((doc: { _id?: unknown }) => String(doc._id)); + } + async getPaginatedAuthorizedQuery( operation: string, query: Indexable, diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index a506087d4..5d972b54f 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -24,6 +24,10 @@ import { fromMongoVectorIndex, toMongoVectorIndexDefinition, assertVectorSearchAccess, + completeVectorSearch, + declaredVectorIndexes, + mergeVectorIndexes, + planMongoVectorSearch, } from '../utils/index.js'; import pluralize from '../../utils/pluralize.js'; import { mongoSchemaConverter } from '../../introspection/mongoose/utils.js'; @@ -829,46 +833,31 @@ export class MongooseAdapter extends DatabaseAdapter { adminOperator: request.adminOperator, }); - const vectorStage: any = { - index: request.indexName ?? `${request.field}_vector`, - path: request.field, - queryVector: request.vector, - numCandidates: request.numCandidates ?? Math.max((request.limit ?? 10) * 10, 100), - limit: request.limit ?? 10, - }; - const filter = request.filter ?? {}; - if (!request.adminOperator) { - const authorizedQuery = await model.getAuthorizedQuery( - 'read', - filter, - true, - request.userId, - request.scope, - ); - if (isNil(authorizedQuery)) return []; - if (Object.keys(authorizedQuery).length > 0) { - vectorStage.filter = authorizedQuery; - } - } else if (Object.keys(filter).length > 0) { - vectorStage.filter = filter; - } - - const pipeline: any[] = [ - { $vectorSearch: vectorStage }, - { $addFields: { _score: { $meta: 'vectorSearchScore' } } }, - ]; - pipeline.push({ - $project: this.buildVectorProjection(model.originalSchema, request.select), + const schemaFields = (model.originalSchema.compiledFields ?? + model.originalSchema.fields) as Record; + const liveIndexes = await this.getVectorIndexes(request.schemaName).catch(() => []); + const planned = planMongoVectorSearch({ + request, + indexes: mergeVectorIndexes( + declaredVectorIndexes(model.originalSchema), + liveIndexes, + ), + schemaFields, }); - - const docs = await this.mongoose - .model(request.schemaName) - .collection.aggregate(pipeline) - .toArray(); - return docs.map((doc: any) => { - const score = doc._score ?? 0; - delete doc._score; - return { document: doc, score }; + return completeVectorSearch({ + emptyResult: planned.emptyResult, + limit: planned.limits.limit, + authzEnabled: !!model.authzEnabled, + adminOperator: request.adminOperator, + provider: 'mongodb', + metric: schemaField.similarity, + fetchCandidates: async () => + this.mongoose + .model(request.schemaName) + .collection.aggregate(planned.pipeline) + .toArray(), + lookupAuthorizedIds: ids => + model.lookupAuthorizedCandidateIds('read', ids, request.userId, request.scope), }); } @@ -962,32 +951,6 @@ export class MongooseAdapter extends DatabaseAdapter { assertVectorIndexMatchesField(field, index); } - private buildVectorProjection(schema: ConduitDatabaseSchema, select?: string) { - const hiddenFields = new Set( - Object.entries(schema.compiledFields ?? schema.fields) - .filter(([, field]: [string, any]) => field?.select === false) - .map(([field]) => field), - ); - const tokens = select?.split(' ').filter(Boolean) ?? []; - const includeTokens = tokens.filter(token => !token.startsWith('-')); - if (includeTokens.length) { - return includeTokens.reduce( - (projection: any, token: string) => { - if (!hiddenFields.has(token)) projection[token] = 1; - return projection; - }, - { _score: 1 }, - ); - } - return [ - ...hiddenFields, - ...tokens.filter(token => token.startsWith('-')).map(token => token.slice(1)), - ].reduce((projection: any, field: string) => { - projection[field] = 0; - return projection; - }, {}); - } - protected async _createSchemaFromAdapter( schema: ConduitDatabaseSchema, saveToDb: boolean = true, diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 62b92f269..1603917c6 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -43,12 +43,17 @@ import { } from '../utils/vectorCapabilities.js'; import { fromPostgresVectorIndex, - pgVectorDistanceOperator, pgVectorOperator, postgresIndexMethodSql, resolveVectorFieldFromSchema, } from '../utils/vectorMappings.js'; import { assertVectorSearchAccess } from '../utils/vectorSearchAuth.js'; +import { + completeVectorSearch, + declaredVectorIndexes, + mergeVectorIndexes, + planPostgresVectorSearch, +} from '../utils/vectorSearchQuery.js'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; @@ -537,36 +542,36 @@ export abstract class SequelizeAdapter extends DatabaseAdapter scope: request.scope, adminOperator: request.adminOperator, }); - const authorizedQuery = request.adminOperator - ? (request.filter ?? {}) - : await schema.getAuthorizedQuery( - 'read', - request.filter ?? {}, - true, - request.userId, - request.scope, - ); - if (isNil(authorizedQuery)) return []; - const tableName = this.getPhysicalTableName(request.schemaName); - const distance = pgVectorDistanceOperator(field.similarity ?? 'cosine'); - const where = this.renderSimpleWhere(authorizedQuery); - const limit = Math.max(1, Math.min(request.limit ?? 10, 1000)); - const vector = `[${request.vector.join(',')}]`; - const selectedColumns = this.buildVectorSelectList( - schema.originalSchema, - request.select, - ); - const rows = await this.sequelize.query( - `SELECT ${selectedColumns}, (${this.quoteIdentifier(request.field)} ${distance} ${this.sequelize.escape( - vector, - )}::vector) AS _score FROM ${this.quoteIdentifier(tableName)}${where} ORDER BY ${this.quoteIdentifier( - request.field, - )} ${distance} ${this.sequelize.escape(vector)}::vector LIMIT ${limit}`, - ); - return (rows[0] as any[]).map(row => { - const score = Number(row._score ?? 0); - delete row._score; - return { document: row, score }; + const schemaFields = (schema.originalSchema.compiledFields ?? + schema.originalSchema.fields) as Record; + const liveIndexes = await this.getVectorIndexes(request.schemaName).catch(() => []); + const planned = planPostgresVectorSearch({ + request, + indexes: mergeVectorIndexes( + declaredVectorIndexes(schema.originalSchema), + liveIndexes, + ), + schemaFields, + tableName: this.getPhysicalTableName(request.schemaName), + similarity: field.similarity, + renderer: { + quoteIdentifier: identifier => this.quoteIdentifier(identifier), + escape: value => this.sequelize.escape(value as string | number), + }, + }); + return completeVectorSearch({ + emptyResult: planned.emptyResult, + limit: planned.limits.limit, + authzEnabled: !!schema.authzEnabled, + adminOperator: request.adminOperator, + provider: 'postgres', + metric: field.similarity, + fetchCandidates: async () => { + const rows = await this.sequelize.query(planned.sql); + return (rows[0] as Indexable[]) ?? []; + }, + lookupAuthorizedIds: ids => + schema.lookupAuthorizedCandidateIds('read', ids, request.userId, request.scope), }); } @@ -647,88 +652,6 @@ export abstract class SequelizeAdapter extends DatabaseAdapter return ` WITH (${entries.map(([key, value]) => `${key} = ${value}`).join(', ')})`; } - private renderSimpleWhere(query: Indexable): string { - const clauses: string[] = Reflect.ownKeys(query).flatMap((fieldKey): string[] => { - const value = (query as any)[fieldKey as any]; - const field = String(fieldKey); - if (field === '$and' && Array.isArray(value)) { - return value - .map(item => this.renderSimpleWhere(item as Indexable).replace(/^ WHERE /, '')) - .filter(Boolean); - } - if ( - typeof fieldKey === 'symbol' && - fieldKey.description === 'and' && - Array.isArray(value) - ) { - return value - .map(item => this.renderSimpleWhere(item as Indexable).replace(/^ WHERE /, '')) - .filter(Boolean); - } - if (typeof fieldKey === 'symbol') { - throw new GrpcError( - status.INVALID_ARGUMENT, - 'Unsupported vector search filter operator', - ); - } - const inValues = this.extractInValues(value); - if (field === '_id' && inValues) { - const ids = inValues.map(id => this.sequelize.escape(String(id))); - return ids.length - ? [`${this.quoteIdentifier(field)} IN (${ids.join(', ')})`] - : []; - } - if (value === null) { - return [`${this.quoteIdentifier(field)} IS NULL`]; - } - if (typeof value === 'boolean') { - return [`${this.quoteIdentifier(field)} = ${value ? 'TRUE' : 'FALSE'}`]; - } - if (value === null || ['string', 'number', 'boolean'].includes(typeof value)) { - return [ - `${this.quoteIdentifier(field)} = ${this.sequelize.escape(value as string | number)}`, - ]; - } - if (value && typeof value === 'object') { - throw new GrpcError( - status.INVALID_ARGUMENT, - 'Unsupported vector search filter shape', - ); - } - return []; - }); - return clauses.length ? ` WHERE ${clauses.join(' AND ')}` : ''; - } - - private extractInValues(value: unknown): unknown[] | null { - if (!value || typeof value !== 'object') return null; - if ('$in' in value) return (value as { $in: unknown[] }).$in; - const inSymbol = Object.getOwnPropertySymbols(value).find( - symbol => symbol.description === 'in' || symbol.toString() === 'Symbol(in)', - ); - return inSymbol ? ((value as any)[inSymbol] as unknown[]) : null; - } - - private buildVectorSelectList(schema: ConduitDatabaseSchema, select?: string) { - const fields = schema.compiledFields ?? schema.fields; - const hiddenFields = new Set( - Object.entries(fields) - .filter(([, field]: [string, any]) => field?.select === false) - .map(([field]) => field), - ); - const availableFields = Object.keys(fields).filter(field => !hiddenFields.has(field)); - const tokens = select?.split(' ').filter(Boolean) ?? []; - const includeTokens = tokens.filter(token => !token.startsWith('-')); - const selected = new Set(includeTokens.length ? includeTokens : availableFields); - tokens - .filter(token => token.startsWith('-')) - .map(token => token.slice(1)) - .forEach(field => selected.delete(field)); - hiddenFields.forEach(field => selected.delete(field)); - if (!selected.size) selected.add('_id'); - return [...selected].map(field => this.quoteIdentifier(field)).join(', '); - } - private checkAndConvertIndexes( schemaName: string, indexes: ModelOptionsIndexes[], diff --git a/modules/database/src/adapters/utils/__tests__/grpcStatus.test.ts b/modules/database/src/adapters/utils/__tests__/grpcStatus.test.ts new file mode 100644 index 000000000..84795bd4a --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/grpcStatus.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from '@jest/globals'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { grpcStatusFromError } from '../grpcStatus.js'; + +describe('grpcStatusFromError', () => { + it('preserves typed GrpcError codes instead of collapsing them to INTERNAL', () => { + expect( + grpcStatusFromError(new GrpcError(status.INVALID_ARGUMENT, 'bad filter')), + ).toEqual({ + code: status.INVALID_ARGUMENT, + message: 'bad filter', + }); + expect( + grpcStatusFromError(new GrpcError(status.PERMISSION_DENIED, 'no subject')), + ).toEqual({ + code: status.PERMISSION_DENIED, + message: 'no subject', + }); + }); + + it('maps unknown errors to INTERNAL', () => { + expect(grpcStatusFromError(new Error('boom'))).toEqual({ + code: status.INTERNAL, + message: 'boom', + }); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorProjection.test.ts b/modules/database/src/adapters/utils/__tests__/vectorProjection.test.ts new file mode 100644 index 000000000..84426350c --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorProjection.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from '@jest/globals'; +import { TYPE } from '@conduitplatform/grpc-sdk'; +import { mongoVectorProjection, postgresVectorSelectList } from '../vectorProjection.js'; + +const schemaFields = { + _id: { type: TYPE.ObjectId }, + title: { type: TYPE.String }, + embedding: { type: TYPE.Vector, dimensions: 8, select: false }, + sourceHash: { type: TYPE.String, select: false }, +}; + +describe('vector search projections', () => { + it('keeps select:false fields out of Mongo include and exclude projections', () => { + expect(mongoVectorProjection(schemaFields, 'title embedding')).toEqual({ + _id: 1, + _score: 1, + title: 1, + }); + expect(mongoVectorProjection(schemaFields, '-title')).toEqual({ + embedding: 0, + sourceHash: 0, + title: 0, + }); + expect(mongoVectorProjection(schemaFields)).toEqual({ + embedding: 0, + sourceHash: 0, + }); + }); + + it('keeps select:false fields out of Postgres select lists', () => { + const quote = (identifier: string) => `"${identifier}"`; + expect(postgresVectorSelectList(schemaFields, 'title embedding', quote)).toBe( + '"title", "_id"', + ); + expect(postgresVectorSelectList(schemaFields, undefined, quote)).toBe( + '"_id", "title"', + ); + expect(postgresVectorSelectList(schemaFields, '-title', quote)).toBe('"_id"'); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorScore.test.ts b/modules/database/src/adapters/utils/__tests__/vectorScore.test.ts new file mode 100644 index 000000000..d9e2ecca7 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorScore.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from '@jest/globals'; +import { VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { normalizeVectorSearchScore, toVectorSearchResult } from '../vectorScore.js'; + +describe('vector search score contract', () => { + it('keeps Mongo cosine scores as higher-is-better without claiming a raw distance', () => { + const normalized = normalizeVectorSearchScore({ + provider: 'mongodb', + metric: VectorSimilarity.Cosine, + raw: 0.91, + }); + expect(normalized).toEqual({ + score: 0.91, + metric: VectorSimilarity.Cosine, + provider: 'mongodb', + comparable: true, + }); + expect(toVectorSearchResult({ _id: 'a' }, normalized)).toEqual({ + document: { _id: 'a' }, + score: 0.91, + metric: VectorSimilarity.Cosine, + provider: 'mongodb', + }); + }); + + it('converts Postgres cosine distance with 1 - distance', () => { + const identical = normalizeVectorSearchScore({ + provider: 'postgres', + metric: VectorSimilarity.Cosine, + raw: 0, + }); + expect(identical.score).toBe(1); + expect(identical.distance).toBe(0); + expect(identical.comparable).toBe(true); + + const orthogonal = normalizeVectorSearchScore({ + provider: 'postgres', + metric: 'cosine', + raw: 1, + }); + expect(orthogonal.score).toBe(0); + expect(orthogonal.distance).toBe(1); + }); + + it('does not treat Euclidean or inner-product scores as cross-provider equivalent', () => { + const mongoEuclidean = normalizeVectorSearchScore({ + provider: 'mongodb', + metric: VectorSimilarity.Euclidean, + raw: 0.4, + }); + const postgresEuclidean = normalizeVectorSearchScore({ + provider: 'postgres', + metric: VectorSimilarity.Euclidean, + raw: 0.6, + }); + const mongoDot = normalizeVectorSearchScore({ + provider: 'mongodb', + metric: VectorSimilarity.DotProduct, + raw: 12, + }); + const postgresDot = normalizeVectorSearchScore({ + provider: 'postgres', + metric: VectorSimilarity.DotProduct, + raw: -12, + }); + + expect(mongoEuclidean).toMatchObject({ + score: 0.4, + comparable: false, + provider: 'mongodb', + }); + expect(postgresEuclidean).toMatchObject({ + score: -0.6, + distance: 0.6, + comparable: false, + provider: 'postgres', + }); + expect(mongoDot.comparable).toBe(false); + expect(postgresDot).toMatchObject({ + score: 12, + distance: -12, + comparable: false, + }); + expect(mongoEuclidean.score).not.toBe(postgresEuclidean.score); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchAdapters.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchAdapters.test.ts new file mode 100644 index 000000000..5af93be34 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchAdapters.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { TYPE, VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { MongooseAdapter } from '../../mongoose-adapter/index.js'; +import { SequelizeAdapter } from '../../sequelize-adapter/index.js'; + +const schemaFields = { + _id: { type: TYPE.ObjectId }, + title: { type: TYPE.String }, + tenantId: { type: TYPE.String }, + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, +}; + +function articleModel(overrides?: { + authzEnabled?: boolean; + lookupAuthorizedCandidateIds?: (ids: string[]) => Promise; + getAuthorizedQuery?: (...args: unknown[]) => Promise; +}) { + const resolveIds = + overrides?.lookupAuthorizedCandidateIds ?? (async (ids: string[]) => ids); + return { + authzEnabled: overrides?.authzEnabled ?? true, + originalSchema: { + name: 'Article', + fields: schemaFields, + compiledFields: schemaFields, + collectionName: 'cnd_Article', + modelOptions: { + conduit: { authorization: { enabled: overrides?.authzEnabled ?? true } }, + vectorIndexes: [ + { + name: 'embedding_vector', + field: 'embedding', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + filterFields: ['_id', 'tenantId'], + }, + ], + }, + }, + lookupAuthorizedCandidateIds: jest.fn(async (_operation: string, ids: string[]) => + resolveIds(ids), + ), + getAuthorizedQuery: jest.fn( + overrides?.getAuthorizedQuery ?? (async () => ({ _id: { $in: ['all-docs'] } })), + ), + }; +} + +const searchRequest = { + schemaName: 'Article', + field: 'embedding', + vector: [0.1, 0.2, 0.3], + filter: { tenantId: 'org-1' }, + limit: 2, + numCandidates: 4, + select: 'title embedding', + userId: 'user-1', +}; + +describe('mongoose vector search adapter', () => { + it('runs ANN with indexed prefilters then authorizes only candidate ids', async () => { + const model = articleModel({ + lookupAuthorizedCandidateIds: async ids => ids.filter(id => id !== 'denied'), + }); + const aggregate = jest.fn(() => ({ + toArray: async () => [ + { _id: 'a', title: 'one', _score: 0.9 }, + { _id: 'denied', title: 'secret', _score: 0.8 }, + { _id: 'b', title: 'two', _score: 0.7 }, + ], + })); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + Object.assign(adapter, { + models: { Article: model }, + mongoose: { + model: () => ({ collection: { aggregate } }), + }, + getVectorIndexes: async () => model.originalSchema.modelOptions.vectorIndexes, + }); + + const results = await adapter.vectorSearch(searchRequest); + + expect(model.getAuthorizedQuery).not.toHaveBeenCalled(); + expect(model.lookupAuthorizedCandidateIds).toHaveBeenCalledWith( + 'read', + ['a', 'denied', 'b'], + 'user-1', + undefined, + ); + expect(aggregate.mock.calls[0][0][0].$vectorSearch).toMatchObject({ + filter: { tenantId: 'org-1' }, + limit: 4, + numCandidates: 4, + }); + expect(aggregate.mock.calls[0][0][0].$vectorSearch.filter).not.toHaveProperty('_id'); + expect(aggregate.mock.calls[0][0][2].$project).toEqual({ + _id: 1, + _score: 1, + title: 1, + }); + expect(results.map(result => result.document._id)).toEqual(['a', 'b']); + expect(results[0]).toMatchObject({ + score: 0.9, + provider: 'mongodb', + metric: VectorSimilarity.Cosine, + }); + expect(results[0].document).not.toHaveProperty('embedding'); + }); + + it('does not query Mongo when an empty $in filter matches no rows', async () => { + const model = articleModel(); + const aggregate = jest.fn(); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + Object.assign(adapter, { + models: { Article: model }, + mongoose: { model: () => ({ collection: { aggregate } }) }, + getVectorIndexes: async () => model.originalSchema.modelOptions.vectorIndexes, + }); + const results = await adapter.vectorSearch({ + ...searchRequest, + filter: { tenantId: { $in: [] } }, + }); + expect(results).toEqual([]); + expect(aggregate).not.toHaveBeenCalled(); + expect(model.lookupAuthorizedCandidateIds).not.toHaveBeenCalled(); + }); +}); + +describe('postgres vector search adapter', () => { + it('keeps user filters in SQL, hides select:false columns, and authorizes candidates only', async () => { + const model = articleModel({ + lookupAuthorizedCandidateIds: async ids => ids.filter(id => id !== 'denied'), + }); + const query = jest.fn(async () => [ + [ + { _id: 'a', title: 'one', _score: 0.2 }, + { _id: 'denied', title: 'secret', _score: 0.3 }, + { _id: 'b', title: 'two', _score: 0.5 }, + ], + ]); + const adapter = Object.create(SequelizeAdapter.prototype) as SequelizeAdapter; + Object.assign(adapter, { + models: { Article: model }, + sequelize: { + getDialect: () => 'postgres', + escape: (value: unknown) => + typeof value === 'string' ? `'${value}'` : String(value), + query, + }, + getVectorIndexes: async () => model.originalSchema.modelOptions.vectorIndexes, + }); + + const results = await adapter.vectorSearch(searchRequest); + const sql = query.mock.calls[0][0] as string; + + expect(model.getAuthorizedQuery).not.toHaveBeenCalled(); + expect(sql).toContain('WHERE "tenantId" = \'org-1\''); + expect(sql).toContain('LIMIT 4'); + expect(sql).toMatch(/^SELECT "title", "_id",/); + expect(sql).not.toContain('all-docs'); + expect(results).toEqual([ + { + document: { _id: 'a', title: 'one' }, + score: 0.8, + distance: 0.2, + metric: VectorSimilarity.Cosine, + provider: 'postgres', + }, + { + document: { _id: 'b', title: 'two' }, + score: 0.5, + distance: 0.5, + metric: VectorSimilarity.Cosine, + provider: 'postgres', + }, + ]); + }); + + it('does not query Postgres when empty $in matches no rows', async () => { + const model = articleModel(); + const query = jest.fn(); + const adapter = Object.create(SequelizeAdapter.prototype) as SequelizeAdapter; + Object.assign(adapter, { + models: { Article: model }, + sequelize: { + getDialect: () => 'postgres', + escape: (value: unknown) => `'${value}'`, + query, + }, + getVectorIndexes: async () => [], + }); + const results = await adapter.vectorSearch({ + ...searchRequest, + filter: { tenantId: { $in: [] } }, + }); + expect(results).toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchAuth.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchAuth.test.ts index fa3abad6d..1e165469c 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorSearchAuth.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchAuth.test.ts @@ -1,8 +1,10 @@ -import { describe, expect, it } from '@jest/globals'; +import { describe, expect, it, jest } from '@jest/globals'; import { GrpcError } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { + applyBoundedVectorAuthorization, assertVectorSearchAccess, + authorizeBoundedVectorCandidates, resolveAdminOperatorContext, } from '../vectorSearchAuth.js'; @@ -60,4 +62,36 @@ describe('vector search authorization', () => { expect((err as GrpcError).code).toBe(status.PERMISSION_DENIED); } }); + + it('authorizes only the bounded candidate ids instead of materializing every authorized document', async () => { + const lookupAuthorizedIds = jest.fn(async (ids: string[]) => + ids.filter(id => id === 'keep'), + ); + const authorized = await authorizeBoundedVectorCandidates({ + authzEnabled: true, + candidateIds: ['keep', 'drop'], + lookupAuthorizedIds, + }); + expect(lookupAuthorizedIds).toHaveBeenCalledWith(['keep', 'drop']); + expect([...authorized]).toEqual(['keep']); + expect( + applyBoundedVectorAuthorization( + [{ _id: 'keep' }, { _id: 'drop' }, { _id: 'also-keep' }], + new Set(['keep', 'also-keep']), + 1, + ), + ).toEqual([{ _id: 'keep' }]); + }); + + it('skips authorization lookup for admin operators while still capping the result limit', async () => { + const lookupAuthorizedIds = jest.fn(async (ids: string[]) => ids); + const authorized = await authorizeBoundedVectorCandidates({ + authzEnabled: true, + adminOperator: true, + candidateIds: ['a', 'b'], + lookupAuthorizedIds, + }); + expect(lookupAuthorizedIds).not.toHaveBeenCalled(); + expect(authorized.size).toBe(2); + }); }); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchFilter.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchFilter.test.ts new file mode 100644 index 000000000..9215285c6 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchFilter.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from '@jest/globals'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { validateVectorSearchFilter } from '../vectorSearchFilter.js'; + +function expectInvalid(run: () => unknown) { + try { + run(); + throw new Error('expected INVALID_ARGUMENT'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.INVALID_ARGUMENT); + } +} + +describe('vector search filter validation', () => { + const mongoFields = ['_id', 'tenantId', 'status']; + const postgresFields = ['_id', 'tenantId', 'status', 'score']; + + it('accepts Atlas-legal indexed Mongo prefilters', () => { + const validated = validateVectorSearchFilter( + { + tenantId: 'org-1', + status: { $in: ['active', 'draft'] }, + $or: [{ score: { $gte: 1 } }, { score: { $lte: 0 } }], + }, + { provider: 'mongodb', allowedFilterFields: [...mongoFields, 'score'] }, + ); + expect(validated.emptyResult).toBe(false); + expect(validated.filter.tenantId).toBe('org-1'); + }); + + it('rejects Mongo filters that are not declared indexed filter fields', () => { + expectInvalid(() => + validateVectorSearchFilter( + { authorId: 'user-1' }, + { provider: 'mongodb', allowedFilterFields: mongoFields }, + ), + ); + }); + + it('rejects Mongo filters when no indexed filter fields are declared', () => { + expectInvalid(() => + validateVectorSearchFilter({ tenantId: 'org-1' }, { provider: 'mongodb' }), + ); + }); + + it('rejects unsupported Mongo operators that Atlas vector search cannot prefilter', () => { + expectInvalid(() => + validateVectorSearchFilter( + { tenantId: { $regex: '^org' } }, + { provider: 'mongodb', allowedFilterFields: mongoFields }, + ), + ); + expectInvalid(() => + validateVectorSearchFilter( + { tenantId: { $exists: true } }, + { provider: 'mongodb', allowedFilterFields: mongoFields }, + ), + ); + expectInvalid(() => + validateVectorSearchFilter( + { tenantId: { $elemMatch: { a: 1 } } }, + { provider: 'mongodb', allowedFilterFields: mongoFields }, + ), + ); + }); + + it('never silently drops unsupported Postgres filters', () => { + expectInvalid(() => + validateVectorSearchFilter( + { tenantId: { $regex: '^org' } }, + { provider: 'postgres', allowedFilterFields: postgresFields }, + ), + ); + expectInvalid(() => + validateVectorSearchFilter( + { unknownColumn: 'x' }, + { provider: 'postgres', allowedFilterFields: postgresFields }, + ), + ); + }); + + it('treats empty $in as a no-row filter', () => { + expect( + validateVectorSearchFilter( + { status: { $in: [] } }, + { provider: 'mongodb', allowedFilterFields: mongoFields }, + ).emptyResult, + ).toBe(true); + expect( + validateVectorSearchFilter( + { $and: [{ tenantId: 'org-1' }, { status: { $in: [] } }] }, + { provider: 'postgres', allowedFilterFields: postgresFields }, + ).emptyResult, + ).toBe(true); + expect( + validateVectorSearchFilter( + { $or: [{ status: { $in: [] } }, { tenantId: { $in: [] } }] }, + { provider: 'postgres', allowedFilterFields: postgresFields }, + ).emptyResult, + ).toBe(true); + }); + + it('does not treat negated or $nor empty $in as a no-row filter', () => { + expect( + validateVectorSearchFilter( + { status: { $not: { $in: [] } } }, + { provider: 'mongodb', allowedFilterFields: mongoFields }, + ).emptyResult, + ).toBe(false); + expect( + validateVectorSearchFilter( + { $nor: [{ status: { $in: [] } }] }, + { provider: 'postgres', allowedFilterFields: postgresFields }, + ).emptyResult, + ).toBe(false); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchLimits.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchLimits.test.ts new file mode 100644 index 000000000..b31398d11 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchLimits.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from '@jest/globals'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + clampVectorSearchLimits, + VECTOR_SEARCH_MAX_CANDIDATES, + VECTOR_SEARCH_MAX_LIMIT, +} from '../vectorSearchLimits.js'; + +describe('vector search limits', () => { + it('defaults limit and requires candidates to cover the requested limit', () => { + expect(clampVectorSearchLimits({})).toEqual({ limit: 10, numCandidates: 100 }); + expect(clampVectorSearchLimits({ limit: 5 })).toEqual({ + limit: 5, + numCandidates: 100, + }); + expect(clampVectorSearchLimits({ limit: 25, numCandidates: 250 })).toEqual({ + limit: 25, + numCandidates: 250, + }); + }); + + it('clamps oversized limit and candidate values', () => { + expect(clampVectorSearchLimits({ limit: 50_000, numCandidates: 80_000 })).toEqual({ + limit: VECTOR_SEARCH_MAX_LIMIT, + numCandidates: VECTOR_SEARCH_MAX_CANDIDATES, + }); + }); + + it('rejects non-positive values and candidates below the requested limit', () => { + try { + clampVectorSearchLimits({ limit: 0 }); + throw new Error('expected failure'); + } catch (err) { + expect((err as GrpcError).code).toBe(status.INVALID_ARGUMENT); + } + try { + clampVectorSearchLimits({ limit: 20, numCandidates: 5 }); + throw new Error('expected failure'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).message).toMatch(/numCandidates must be at least/); + } + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts new file mode 100644 index 000000000..2b1cb8e12 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { TYPE, VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { + completeVectorSearch, + planMongoVectorSearch, + planPostgresVectorSearch, +} from '../vectorSearchQuery.js'; + +const schemaFields = { + _id: { type: TYPE.ObjectId }, + title: { type: TYPE.String }, + tenantId: { type: TYPE.String }, + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, +}; + +const indexes = [ + { + name: 'embedding_vector', + field: 'embedding', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + filterFields: ['_id', 'tenantId'], + }, +]; + +const request = { + schemaName: 'Article', + field: 'embedding', + vector: [0.1, 0.2, 0.3], + filter: { tenantId: 'org-1' }, + limit: 2, + numCandidates: 5, + select: 'title embedding', +}; + +describe('vector search query planning', () => { + it('builds a Mongo Atlas pipeline with indexed prefilters and hidden-field projection', () => { + const planned = planMongoVectorSearch({ request, indexes, schemaFields }); + expect(planned.emptyResult).toBe(false); + expect(planned.limits).toEqual({ limit: 2, numCandidates: 5 }); + expect(planned.pipeline[0]).toEqual({ + $vectorSearch: { + index: 'embedding_vector', + path: 'embedding', + queryVector: request.vector, + numCandidates: 5, + limit: 5, + filter: { tenantId: 'org-1' }, + }, + }); + expect(planned.pipeline[2]).toEqual({ + $project: { _id: 1, _score: 1, title: 1 }, + }); + }); + + it('short-circuits empty Mongo $in without emitting a pipeline', () => { + const planned = planMongoVectorSearch({ + request: { ...request, filter: { tenantId: { $in: [] } } }, + indexes, + schemaFields, + }); + expect(planned.emptyResult).toBe(true); + expect(planned.pipeline).toEqual([]); + }); + + it('renders Postgres SQL that keeps filters, hides select:false columns, and fetches candidates', () => { + const planned = planPostgresVectorSearch({ + request, + indexes, + schemaFields, + tableName: 'cnd_Article', + similarity: VectorSimilarity.Cosine, + renderer: { + quoteIdentifier: identifier => `"${identifier}"`, + escape: value => (typeof value === 'string' ? `'${value}'` : String(value)), + }, + }); + expect(planned.emptyResult).toBe(false); + expect(planned.sql).toContain('WHERE "tenantId" = \'org-1\''); + expect(planned.sql).toContain('LIMIT 5'); + expect(planned.sql).toMatch(/^SELECT "title", "_id",/); + expect(planned.sql).toContain('<=>'); + }); +}); + +describe('bounded vector search completion', () => { + it('authorizes only ANN candidate ids and returns normalized higher-is-better scores', async () => { + const lookupAuthorizedIds = jest.fn(async (ids: string[]) => + ids.filter(id => id !== 'denied'), + ); + const results = await completeVectorSearch({ + emptyResult: false, + limit: 2, + authzEnabled: true, + provider: 'postgres', + metric: VectorSimilarity.Cosine, + fetchCandidates: async () => [ + { _id: 'a', title: 'one', _score: 0.1 }, + { _id: 'denied', title: 'secret', _score: 0.2 }, + { _id: 'b', title: 'two', _score: 0.4 }, + { _id: 'c', title: 'three', _score: 0.5 }, + ], + lookupAuthorizedIds, + }); + expect(lookupAuthorizedIds).toHaveBeenCalledWith(['a', 'denied', 'b', 'c']); + expect(results).toEqual([ + { + document: { _id: 'a', title: 'one' }, + score: 0.9, + distance: 0.1, + metric: VectorSimilarity.Cosine, + provider: 'postgres', + }, + { + document: { _id: 'b', title: 'two' }, + score: 0.6, + distance: 0.4, + metric: VectorSimilarity.Cosine, + provider: 'postgres', + }, + ]); + }); + + it('does not call authorization lookup for admin operator or empty $in results', async () => { + const lookupAuthorizedIds = jest.fn(async (ids: string[]) => ids); + await completeVectorSearch({ + emptyResult: true, + limit: 2, + authzEnabled: true, + provider: 'mongodb', + metric: VectorSimilarity.Cosine, + fetchCandidates: async () => [{ _id: 'a', _score: 1 }], + lookupAuthorizedIds, + }); + expect(lookupAuthorizedIds).not.toHaveBeenCalled(); + + const results = await completeVectorSearch({ + emptyResult: false, + limit: 1, + authzEnabled: true, + adminOperator: true, + provider: 'mongodb', + metric: VectorSimilarity.Cosine, + fetchCandidates: async () => [ + { _id: 'a', _score: 0.9 }, + { _id: 'b', _score: 0.8 }, + ], + lookupAuthorizedIds, + }); + expect(lookupAuthorizedIds).not.toHaveBeenCalled(); + expect(results).toHaveLength(1); + expect(results[0]).toMatchObject({ + score: 0.9, + provider: 'mongodb', + metric: VectorSimilarity.Cosine, + }); + expect(results[0].distance).toBeUndefined(); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchWhere.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchWhere.test.ts new file mode 100644 index 000000000..7a2a7c14e --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchWhere.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from '@jest/globals'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { renderPostgresVectorWhere } from '../vectorSearchWhere.js'; + +const renderer = { + quoteIdentifier: (identifier: string) => `"${identifier.replace(/"/g, '""')}"`, + escape: (value: unknown) => + typeof value === 'string' ? `'${value.replace(/'/g, "''")}'` : String(value), +}; + +describe('postgres vector search where rendering', () => { + it('renders supported comparison and membership filters', () => { + expect( + renderPostgresVectorWhere( + { + tenantId: 'org-1', + status: { $in: ['active', 'draft'] }, + score: { $gte: 1, $lt: 10 }, + }, + renderer, + ), + ).toBe( + ` WHERE "tenantId" = 'org-1' AND "status" IN ('active', 'draft') AND ("score" >= 1 AND "score" < 10)`, + ); + }); + + it('returns no rows for empty $in instead of dropping the predicate', () => { + expect(renderPostgresVectorWhere({ status: { $in: [] } }, renderer)).toBe( + ' WHERE FALSE', + ); + }); + + it('fails instead of silently dropping unsupported operators', () => { + try { + renderPostgresVectorWhere({ tenantId: { $regex: '^org' } }, renderer); + throw new Error('expected failure'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.INVALID_ARGUMENT); + } + }); +}); diff --git a/modules/database/src/adapters/utils/index.ts b/modules/database/src/adapters/utils/index.ts index 3b183e7d1..97aebccc3 100644 --- a/modules/database/src/adapters/utils/index.ts +++ b/modules/database/src/adapters/utils/index.ts @@ -9,3 +9,9 @@ export * from './mutationEvents.js'; export * from './grpcStatus.js'; export * from './vectorSearchAuth.js'; export * from './embeddingsJobContext.js'; +export * from './vectorScore.js'; +export * from './vectorSearchLimits.js'; +export * from './vectorSearchFilter.js'; +export * from './vectorSearchWhere.js'; +export * from './vectorSearchQuery.js'; +export * from './vectorProjection.js'; diff --git a/modules/database/src/adapters/utils/vectorProjection.ts b/modules/database/src/adapters/utils/vectorProjection.ts new file mode 100644 index 000000000..758381636 --- /dev/null +++ b/modules/database/src/adapters/utils/vectorProjection.ts @@ -0,0 +1,73 @@ +export function hiddenSelectFalseFields( + schemaFields: Record, +): Set { + return new Set( + Object.entries(schemaFields) + .filter(([, field]) => { + return ( + typeof field === 'object' && + field !== null && + (field as { select?: boolean }).select === false + ); + }) + .map(([field]) => field), + ); +} + +export function mongoVectorProjection( + schemaFields: Record, + select?: string, +): Record { + const hiddenFields = hiddenSelectFalseFields(schemaFields); + const tokens = select?.split(' ').filter(Boolean) ?? []; + const includeTokens = tokens.filter(token => !token.startsWith('-')); + if (includeTokens.length) { + const projection: Record = { _id: 1, _score: 1 }; + for (const token of includeTokens) { + if (token !== '_id' && !hiddenFields.has(token)) { + projection[token] = 1; + } + } + return projection; + } + const projection: Record = {}; + for (const field of hiddenFields) { + projection[field] = 0; + } + for (const token of tokens + .filter(token => token.startsWith('-')) + .map(token => token.slice(1))) { + if (token !== '_id') { + projection[token] = 0; + } + } + return projection; +} + +export function postgresVectorSelectList( + schemaFields: Record, + select: string | undefined, + quoteIdentifier: (identifier: string) => string, +): string { + const hiddenFields = hiddenSelectFalseFields(schemaFields); + const availableFields = Object.keys(schemaFields).filter( + field => !hiddenFields.has(field), + ); + if (!availableFields.includes('_id')) { + availableFields.unshift('_id'); + } + const tokens = select?.split(' ').filter(Boolean) ?? []; + const includeTokens = tokens.filter( + token => !token.startsWith('-') && !hiddenFields.has(token), + ); + const selected = new Set(includeTokens.length ? includeTokens : availableFields); + tokens + .filter(token => token.startsWith('-')) + .map(token => token.slice(1)) + .forEach(field => { + if (field !== '_id') selected.delete(field); + }); + hiddenFields.forEach(field => selected.delete(field)); + selected.add('_id'); + return [...selected].map(field => quoteIdentifier(field)).join(', '); +} diff --git a/modules/database/src/adapters/utils/vectorScore.ts b/modules/database/src/adapters/utils/vectorScore.ts new file mode 100644 index 000000000..64248e942 --- /dev/null +++ b/modules/database/src/adapters/utils/vectorScore.ts @@ -0,0 +1,114 @@ +import { + VectorSearchProvider, + VectorSearchResult, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import type { Indexable } from '@conduitplatform/grpc-sdk'; +import { parseVectorSimilarity } from './vectorField.js'; + +export interface NormalizedVectorScore { + score: number; + distance?: number; + metric: VectorSimilarity; + provider: VectorSearchProvider; + comparable: boolean; +} + +function finiteNumber(value: unknown, fallback = 0): number { + const numeric = typeof value === 'number' ? value : Number(value); + return Number.isFinite(numeric) ? numeric : fallback; +} + +/** + * Convert a backend raw score/distance into the documented result contract. + * + * Mongo Atlas `vectorSearchScore` is already higher-is-better. + * Postgres `<=>` is cosine *distance* (lower-is-better) and is inverted with + * `1 - distance`. Euclidean and inner-product values are higher-is-better + * rankings only and must not be treated as comparable across providers. + */ +export function normalizeVectorSearchScore(args: { + provider: VectorSearchProvider; + metric: VectorSimilarity | string | undefined; + raw: unknown; +}): NormalizedVectorScore { + const raw = finiteNumber(args.raw); + const metric = parseVectorSimilarity(args.metric); + switch (metric) { + case VectorSimilarity.Cosine: { + if (args.provider === 'postgres') { + return { + score: 1 - raw, + distance: raw, + metric, + provider: args.provider, + comparable: true, + }; + } + return { + score: raw, + metric, + provider: args.provider, + comparable: true, + }; + } + case VectorSimilarity.Euclidean: { + if (args.provider === 'postgres') { + return { + score: -raw, + distance: raw, + metric, + provider: args.provider, + comparable: false, + }; + } + return { + score: raw, + metric, + provider: args.provider, + comparable: false, + }; + } + case VectorSimilarity.DotProduct: { + if (args.provider === 'postgres') { + // pgvector `<#>` stores the negative inner product. + return { + score: -raw, + distance: raw, + metric, + provider: args.provider, + comparable: false, + }; + } + return { + score: raw, + metric, + provider: args.provider, + comparable: false, + }; + } + default: { + const exhaustive: never = metric; + throw new Error(`Unsupported vector similarity '${String(exhaustive)}'`); + } + } +} + +export function stripVectorScoreField(document: T): T { + const next = { ...document }; + delete next._score; + return next; +} + +export function toVectorSearchResult( + document: T, + normalized: NormalizedVectorScore, +): VectorSearchResult { + return { + document, + score: normalized.score, + ...(normalized.distance !== undefined ? { distance: normalized.distance } : {}), + metric: normalized.metric, + provider: normalized.provider, + }; +} diff --git a/modules/database/src/adapters/utils/vectorSearchAuth.ts b/modules/database/src/adapters/utils/vectorSearchAuth.ts index 426ed0267..2457b908c 100644 --- a/modules/database/src/adapters/utils/vectorSearchAuth.ts +++ b/modules/database/src/adapters/utils/vectorSearchAuth.ts @@ -32,3 +32,32 @@ export function assertVectorSearchAccess(args: { 'Vector search on authorization-enabled schemas requires a subject, scope, or admin operator context', ); } + +export async function authorizeBoundedVectorCandidates(args: { + authzEnabled: boolean; + adminOperator?: boolean; + candidateIds: Array; + lookupAuthorizedIds: (ids: string[]) => Promise; +}): Promise> { + const ids = args.candidateIds.map(id => String(id)).filter(Boolean); + if (!ids.length) return new Set(); + if (!args.authzEnabled || args.adminOperator) { + return new Set(ids); + } + const authorized = await args.lookupAuthorizedIds(ids); + return new Set(authorized.map(id => String(id))); +} + +export function applyBoundedVectorAuthorization< + T extends { _id?: unknown; id?: unknown }, +>(documents: T[], authorizedIds: Set, limit: number): T[] { + const next: T[] = []; + for (const document of documents) { + const id = document._id ?? document.id; + if (id === undefined || id === null) continue; + if (!authorizedIds.has(String(id))) continue; + next.push(document); + if (next.length >= limit) break; + } + return next; +} diff --git a/modules/database/src/adapters/utils/vectorSearchFilter.ts b/modules/database/src/adapters/utils/vectorSearchFilter.ts new file mode 100644 index 000000000..f1f12fcce --- /dev/null +++ b/modules/database/src/adapters/utils/vectorSearchFilter.ts @@ -0,0 +1,195 @@ +import { GrpcError, Indexable } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export type VectorFilterProvider = 'mongodb' | 'postgres'; + +const COMPARISON_OPERATORS = new Set(['$eq', '$ne', '$gt', '$gte', '$lt', '$lte']); +const MEMBERSHIP_OPERATORS = new Set(['$in', '$nin']); +const LOGICAL_OPERATORS = new Set(['$and', '$or', '$nor']); +const MONGO_FIELD_PATTERN = /^[A-Za-z_][A-Za-z0-9_.]*$/; +const POSTGRES_FIELD_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; +const RESERVED_FIELD_NAMES = new Set(['__proto__', 'prototype', 'constructor']); + +export interface ValidatedVectorSearchFilter { + filter: Indexable; + emptyResult: boolean; +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isScalar(value: unknown): boolean { + return ( + value === null || + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' + ); +} + +function invalid(message: string): never { + throw new GrpcError(status.INVALID_ARGUMENT, message); +} + +function assertAllowedField( + field: string, + provider: VectorFilterProvider, + allowedFields?: readonly string[], +): void { + const pattern = provider === 'postgres' ? POSTGRES_FIELD_PATTERN : MONGO_FIELD_PATTERN; + if (!pattern.test(field) || field.startsWith('$') || RESERVED_FIELD_NAMES.has(field)) { + invalid(`Unsupported vector search filter field '${field}'`); + } + if (allowedFields && !allowedFields.includes(field)) { + invalid( + provider === 'mongodb' + ? `Vector search filter field '${field}' is not an indexed filter field. ` + + `Allowed fields: ${allowedFields.length ? allowedFields.join(', ') : '(none)'}` + : `Vector search filter field '${field}' is not a schema field. ` + + `Allowed fields: ${allowedFields.join(', ')}`, + ); + } +} + +function validateMembership(value: unknown, operator: string): { empty: boolean } { + if (!Array.isArray(value)) { + invalid(`Vector search operator ${operator} requires an array`); + } + for (const item of value) { + if (!isScalar(item)) { + invalid(`Vector search operator ${operator} only accepts scalar values`); + } + } + if (operator === '$in' && value.length === 0) { + return { empty: true }; + } + return { empty: false }; +} + +function validateComparisonValue(value: unknown, operator: string): void { + if (operator === '$eq' || operator === '$ne') { + if (!isScalar(value)) { + invalid(`Vector search operator ${operator} only accepts scalar values`); + } + return; + } + if (typeof value !== 'number' && typeof value !== 'string') { + invalid(`Vector search operator ${operator} only accepts number or string values`); + } +} + +function validateFieldPredicate( + field: string, + value: unknown, + provider: VectorFilterProvider, + allowedFields?: readonly string[], +): { empty: boolean } { + assertAllowedField(field, provider, allowedFields); + if (isScalar(value)) { + return { empty: false }; + } + if (!isPlainObject(value)) { + invalid(`Unsupported vector search filter value for '${field}'`); + } + const keys = Object.keys(value); + if (!keys.length) { + invalid(`Unsupported vector search filter value for '${field}'`); + } + let empty = false; + for (const operator of keys) { + if (operator === '$not') { + const nested = value[operator]; + if (!isPlainObject(nested)) { + invalid(`Vector search operator $not requires a comparison object`); + } + // Negating a match-nothing predicate matches everything. + validateFieldPredicate(field, nested, provider, allowedFields); + continue; + } + if (COMPARISON_OPERATORS.has(operator)) { + validateComparisonValue(value[operator], operator); + continue; + } + if (MEMBERSHIP_OPERATORS.has(operator)) { + const result = validateMembership(value[operator], operator); + if (result.empty) empty = true; + continue; + } + invalid(`Unsupported vector search filter operator '${operator}'`); + } + return { empty }; +} + +function validateLogical( + operator: string, + value: unknown, + provider: VectorFilterProvider, + allowedFields?: readonly string[], +): { empty: boolean } { + if (!Array.isArray(value) || value.length === 0) { + invalid(`Vector search operator ${operator} requires a non-empty array`); + } + const branchEmpty = value.map(branch => + validateFilterNode(branch, provider, allowedFields), + ); + if (operator === '$and') { + return { empty: branchEmpty.some(branch => branch.empty) }; + } + if (operator === '$or') { + return { empty: branchEmpty.every(branch => branch.empty) }; + } + return { empty: false }; +} + +function validateFilterNode( + node: unknown, + provider: VectorFilterProvider, + allowedFields?: readonly string[], +): { empty: boolean } { + if (!isPlainObject(node)) { + invalid('Vector search filter must be an object'); + } + const keys = Object.keys(node); + if (!keys.length) { + return { empty: false }; + } + let emptyAnd = false; + const orEmpty: boolean[] = []; + for (const key of keys) { + if (LOGICAL_OPERATORS.has(key)) { + const result = validateLogical(key, node[key], provider, allowedFields); + if (key === '$and' && result.empty) emptyAnd = true; + if (key === '$or') orEmpty.push(result.empty); + continue; + } + const result = validateFieldPredicate(key, node[key], provider, allowedFields); + if (result.empty) emptyAnd = true; + } + if (emptyAnd) return { empty: true }; + if (orEmpty.length && orEmpty.every(Boolean) && keys.every(key => key === '$or')) { + return { empty: true }; + } + return { empty: false }; +} + +export function validateVectorSearchFilter( + filter: Indexable | undefined, + options: { + provider: VectorFilterProvider; + allowedFilterFields?: readonly string[]; + }, +): ValidatedVectorSearchFilter { + if (filter === undefined || filter === null) { + return { filter: {}, emptyResult: false }; + } + if (!isPlainObject(filter)) { + invalid('Vector search filter must be an object'); + } + const allowedFields = + options.provider === 'mongodb' + ? (options.allowedFilterFields ?? []) + : options.allowedFilterFields; + const emptyResult = validateFilterNode(filter, options.provider, allowedFields).empty; + return { filter, emptyResult }; +} diff --git a/modules/database/src/adapters/utils/vectorSearchLimits.ts b/modules/database/src/adapters/utils/vectorSearchLimits.ts new file mode 100644 index 000000000..3be08185d --- /dev/null +++ b/modules/database/src/adapters/utils/vectorSearchLimits.ts @@ -0,0 +1,44 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export const VECTOR_SEARCH_DEFAULT_LIMIT = 10; +export const VECTOR_SEARCH_MAX_LIMIT = 1000; +export const VECTOR_SEARCH_MAX_CANDIDATES = 10_000; + +export interface VectorSearchLimits { + limit: number; + numCandidates: number; +} + +function parsePositiveInt(value: unknown, field: string): number | undefined { + if (value === undefined || value === null || value === '') return undefined; + const numeric = typeof value === 'number' ? value : Number(value); + if (!Number.isFinite(numeric) || !Number.isInteger(numeric) || numeric < 1) { + throw new GrpcError(status.INVALID_ARGUMENT, `${field} must be a positive integer`); + } + return numeric; +} + +export function clampVectorSearchLimits(input: { + limit?: number; + numCandidates?: number; +}): VectorSearchLimits { + const parsedLimit = parsePositiveInt(input.limit, 'limit'); + const parsedCandidates = parsePositiveInt(input.numCandidates, 'numCandidates'); + const limit = Math.min( + parsedLimit ?? VECTOR_SEARCH_DEFAULT_LIMIT, + VECTOR_SEARCH_MAX_LIMIT, + ); + const defaultCandidates = Math.max(limit * 10, 100); + const numCandidates = Math.min( + parsedCandidates ?? defaultCandidates, + VECTOR_SEARCH_MAX_CANDIDATES, + ); + if (numCandidates < limit) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `numCandidates must be at least the requested limit (${limit})`, + ); + } + return { limit, numCandidates }; +} diff --git a/modules/database/src/adapters/utils/vectorSearchQuery.ts b/modules/database/src/adapters/utils/vectorSearchQuery.ts new file mode 100644 index 000000000..ccad4abbf --- /dev/null +++ b/modules/database/src/adapters/utils/vectorSearchQuery.ts @@ -0,0 +1,211 @@ +import { + Indexable, + VectorIndexDefinition, + VectorSearchInput, + VectorSearchProvider, + VectorSearchResult, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { clampVectorSearchLimits } from './vectorSearchLimits.js'; +import { validateVectorSearchFilter } from './vectorSearchFilter.js'; +import { renderPostgresVectorWhere, PostgresWhereRenderer } from './vectorSearchWhere.js'; +import { pgVectorDistanceOperator } from './vectorMappings.js'; +import { mongoVectorProjection, postgresVectorSelectList } from './vectorProjection.js'; +import { + applyBoundedVectorAuthorization, + authorizeBoundedVectorCandidates, +} from './vectorSearchAuth.js'; +import { + normalizeVectorSearchScore, + stripVectorScoreField, + toVectorSearchResult, +} from './vectorScore.js'; +import { parseVectorSimilarity } from './vectorField.js'; + +export interface PlannedMongoVectorSearch { + emptyResult: boolean; + limits: { limit: number; numCandidates: number }; + index?: VectorIndexDefinition; + pipeline: Indexable[]; +} + +export interface PlannedPostgresVectorSearch { + emptyResult: boolean; + limits: { limit: number; numCandidates: number }; + index?: VectorIndexDefinition; + sql: string; + distanceOperator: string; +} + +export function schemaFieldNames(schemaFields: Record): string[] { + const fields = Object.keys(schemaFields); + if (!fields.includes('_id')) { + fields.unshift('_id'); + } + return fields; +} + +export function declaredVectorIndexes(schema: { + modelOptions?: { vectorIndexes?: ReadonlyArray }; +}): VectorIndexDefinition[] { + return [...(schema.modelOptions?.vectorIndexes ?? [])]; +} + +export function findVectorIndexForSearch( + indexes: VectorIndexDefinition[], + request: { indexName?: string; field: string }, +): VectorIndexDefinition | undefined { + if (request.indexName) { + return indexes.find(item => item.name === request.indexName); + } + return ( + indexes.find( + item => item.field === request.field && item.name === `${request.field}_vector`, + ) ?? indexes.find(item => item.field === request.field) + ); +} + +export function mergeVectorIndexes( + declared: VectorIndexDefinition[], + live: VectorIndexDefinition[], +): VectorIndexDefinition[] { + if (!live.length) return declared; + if (!declared.length) return live; + const merged = new Map(); + for (const index of declared) { + merged.set(index.name ?? `${index.field}_vector`, index); + } + for (const index of live) { + merged.set(index.name ?? `${index.field}_vector`, index); + } + return [...merged.values()]; +} + +export function buildMongoVectorSearchPipeline(args: { + indexName: string; + field: string; + vector: number[]; + numCandidates: number; + limit: number; + filter?: Indexable; + projection: Indexable; +}): Indexable[] { + const vectorStage: Indexable = { + index: args.indexName, + path: args.field, + queryVector: args.vector, + numCandidates: args.numCandidates, + limit: args.limit, + }; + if (args.filter && Object.keys(args.filter).length > 0) { + vectorStage.filter = args.filter; + } + return [ + { $vectorSearch: vectorStage }, + { $addFields: { _score: { $meta: 'vectorSearchScore' } } }, + { $project: args.projection }, + ]; +} + +export function planMongoVectorSearch(args: { + request: VectorSearchInput; + indexes: VectorIndexDefinition[]; + schemaFields: Record; +}): PlannedMongoVectorSearch { + const limits = clampVectorSearchLimits({ + limit: args.request.limit, + numCandidates: args.request.numCandidates, + }); + const index = findVectorIndexForSearch(args.indexes, args.request); + const validated = validateVectorSearchFilter(args.request.filter, { + provider: 'mongodb', + allowedFilterFields: index?.filterFields ?? [], + }); + if (validated.emptyResult) { + return { emptyResult: true, limits, index, pipeline: [] }; + } + return { + emptyResult: false, + limits, + index, + pipeline: buildMongoVectorSearchPipeline({ + indexName: args.request.indexName ?? index?.name ?? `${args.request.field}_vector`, + field: args.request.field, + vector: args.request.vector, + numCandidates: limits.numCandidates, + limit: limits.numCandidates, + filter: validated.filter, + projection: mongoVectorProjection(args.schemaFields, args.request.select), + }), + }; +} + +export function planPostgresVectorSearch(args: { + request: VectorSearchInput; + indexes: VectorIndexDefinition[]; + schemaFields: Record; + tableName: string; + similarity: VectorSimilarity | string | undefined; + renderer: PostgresWhereRenderer; +}): PlannedPostgresVectorSearch { + const limits = clampVectorSearchLimits({ + limit: args.request.limit, + numCandidates: args.request.numCandidates, + }); + const index = findVectorIndexForSearch(args.indexes, args.request); + const validated = validateVectorSearchFilter(args.request.filter, { + provider: 'postgres', + allowedFilterFields: schemaFieldNames(args.schemaFields), + }); + const distanceOperator = pgVectorDistanceOperator( + parseVectorSimilarity(args.similarity), + ); + if (validated.emptyResult) { + return { emptyResult: true, limits, index, sql: '', distanceOperator }; + } + const where = renderPostgresVectorWhere(validated.filter, args.renderer); + const selectedColumns = postgresVectorSelectList( + args.schemaFields, + args.request.select, + args.renderer.quoteIdentifier, + ); + const vectorLiteral = args.renderer.escape(`[${args.request.vector.join(',')}]`); + const fieldSql = args.renderer.quoteIdentifier(args.request.field); + const sql = + `SELECT ${selectedColumns}, (${fieldSql} ${distanceOperator} ${vectorLiteral}::vector) AS _score ` + + `FROM ${args.renderer.quoteIdentifier(args.tableName)}${where} ` + + `ORDER BY ${fieldSql} ${distanceOperator} ${vectorLiteral}::vector ` + + `LIMIT ${limits.numCandidates}`; + return { emptyResult: false, limits, index, sql, distanceOperator }; +} + +export async function completeVectorSearch(args: { + emptyResult: boolean; + limit: number; + authzEnabled: boolean; + adminOperator?: boolean; + provider: VectorSearchProvider; + metric: VectorSimilarity | string | undefined; + fetchCandidates: () => Promise; + lookupAuthorizedIds: (ids: string[]) => Promise; +}): Promise[]> { + if (args.emptyResult) return []; + const documents = await args.fetchCandidates(); + const authorizedIds = await authorizeBoundedVectorCandidates({ + authzEnabled: args.authzEnabled, + adminOperator: args.adminOperator, + candidateIds: documents + .map(document => document._id ?? document.id) + .filter((id): id is string | { toString(): string } => id != null), + lookupAuthorizedIds: args.lookupAuthorizedIds, + }); + const allowed = applyBoundedVectorAuthorization(documents, authorizedIds, args.limit); + return allowed.map(document => { + const normalized = normalizeVectorSearchScore({ + provider: args.provider, + metric: args.metric, + raw: document._score, + }); + return toVectorSearchResult(stripVectorScoreField(document), normalized); + }); +} diff --git a/modules/database/src/adapters/utils/vectorSearchWhere.ts b/modules/database/src/adapters/utils/vectorSearchWhere.ts new file mode 100644 index 000000000..e3761a93b --- /dev/null +++ b/modules/database/src/adapters/utils/vectorSearchWhere.ts @@ -0,0 +1,134 @@ +import { GrpcError, Indexable } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export interface PostgresWhereRenderer { + quoteIdentifier: (identifier: string) => string; + escape: (value: unknown) => string; +} + +function invalid(message: string): never { + throw new GrpcError(status.INVALID_ARGUMENT, message); +} + +function renderComparison( + fieldSql: string, + operator: string, + value: unknown, + renderer: PostgresWhereRenderer, +): string { + switch (operator) { + case '$eq': + return value === null + ? `${fieldSql} IS NULL` + : `${fieldSql} = ${renderer.escape(value)}`; + case '$ne': + return value === null + ? `${fieldSql} IS NOT NULL` + : `${fieldSql} <> ${renderer.escape(value)}`; + case '$gt': + return `${fieldSql} > ${renderer.escape(value)}`; + case '$gte': + return `${fieldSql} >= ${renderer.escape(value)}`; + case '$lt': + return `${fieldSql} < ${renderer.escape(value)}`; + case '$lte': + return `${fieldSql} <= ${renderer.escape(value)}`; + default: { + const exhaustive: never = operator as never; + invalid(`Unsupported vector search filter operator '${String(exhaustive)}'`); + } + } +} + +function renderFieldPredicate( + field: string, + value: unknown, + renderer: PostgresWhereRenderer, +): string { + const fieldSql = renderer.quoteIdentifier(field); + if (value === null) { + return `${fieldSql} IS NULL`; + } + if (typeof value === 'boolean') { + return `${fieldSql} = ${value ? 'TRUE' : 'FALSE'}`; + } + if (typeof value === 'string' || typeof value === 'number') { + return `${fieldSql} = ${renderer.escape(value)}`; + } + if (!value || typeof value !== 'object' || Array.isArray(value)) { + invalid('Unsupported vector search filter shape'); + } + const clauses: string[] = []; + for (const [operator, operand] of Object.entries(value as Record)) { + if (operator === '$in') { + const values = operand as unknown[]; + if (!values.length) { + return 'FALSE'; + } + clauses.push( + `${fieldSql} IN (${values.map(item => renderer.escape(item)).join(', ')})`, + ); + continue; + } + if (operator === '$nin') { + const values = operand as unknown[]; + if (!values.length) { + continue; + } + clauses.push( + `${fieldSql} NOT IN (${values.map(item => renderer.escape(item)).join(', ')})`, + ); + continue; + } + if (operator === '$not') { + const nested = renderFieldPredicate(field, operand, renderer); + clauses.push(`NOT (${nested})`); + continue; + } + clauses.push(renderComparison(fieldSql, operator, operand, renderer)); + } + if (!clauses.length) { + return 'TRUE'; + } + return clauses.length === 1 ? clauses[0] : `(${clauses.join(' AND ')})`; +} + +function renderNode(node: Indexable, renderer: PostgresWhereRenderer): string { + const clauses: string[] = []; + for (const [key, value] of Object.entries(node)) { + if (key === '$and' && Array.isArray(value)) { + const nested = value + .map(item => renderNode(item as Indexable, renderer)) + .filter(Boolean); + if (nested.length) clauses.push(`(${nested.join(' AND ')})`); + continue; + } + if (key === '$or' && Array.isArray(value)) { + const nested = value.map(item => renderNode(item as Indexable, renderer)); + clauses.push(`(${nested.join(' OR ')})`); + continue; + } + if (key === '$nor' && Array.isArray(value)) { + const nested = value.map(item => renderNode(item as Indexable, renderer)); + clauses.push(`NOT (${nested.join(' OR ')})`); + continue; + } + clauses.push(renderFieldPredicate(key, value, renderer)); + } + return clauses.filter(Boolean).join(' AND '); +} + +export function renderPostgresVectorWhere( + filter: Indexable | undefined, + renderer: PostgresWhereRenderer, +): string { + if (!filter || !Object.keys(filter).length) { + return ''; + } + const body = renderNode(filter, renderer); + if (!body) return ''; + if (body === 'FALSE') { + return ' WHERE FALSE'; + } + return ` WHERE ${body}`; +} From 1f6dd60946cbb802f31024e29eb2f332a054dd50 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 18:56:56 +0300 Subject: [PATCH 06/29] feat(database): harden Mongo and Postgres vector index lifecycle Prevent silent index mismatches, expose queryability, and apply declared vector indexes so search fails closed until indexes are ready. --- libraries/grpc-sdk/src/interfaces/Model.ts | 8 + .../grpc-sdk/src/modules/database/index.ts | 2 + modules/database/src/Database.ts | 8 +- .../database/src/adapters/DatabaseAdapter.ts | 15 + .../src/adapters/mongoose-adapter/index.ts | 29 +- .../src/adapters/sequelize-adapter/index.ts | 136 +++--- .../__tests__/vectorIndexAdapters.test.ts | 251 +++++++++++ .../__tests__/vectorIndexLifecycle.test.ts | 294 +++++++++++++ .../utils/__tests__/vectorMappings.test.ts | 29 +- .../utils/__tests__/vectorSearchQuery.test.ts | 42 +- modules/database/src/adapters/utils/index.ts | 1 + .../adapters/utils/vectorIndexLifecycle.ts | 409 ++++++++++++++++++ .../src/adapters/utils/vectorMappings.ts | 37 +- .../src/adapters/utils/vectorSearchQuery.ts | 19 +- modules/database/src/database.proto | 2 + 15 files changed, 1193 insertions(+), 89 deletions(-) create mode 100644 modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts create mode 100644 modules/database/src/adapters/utils/vectorIndexLifecycle.ts diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 414ecf6fb..957d3afb3 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -38,6 +38,12 @@ export enum VectorIndexMethod { Flat = 'flat', } +export enum VectorIndexStatus { + Pending = 'pending', + Ready = 'ready', + Failed = 'failed', +} + export type VectorSearchProvider = 'mongodb' | 'postgres'; export enum MongoIndexType { @@ -281,6 +287,8 @@ export interface VectorIndexDefinition { similarity: VectorSimilarity; method?: VectorIndexMethod; filterFields?: string[]; + status?: VectorIndexStatus; + queryable?: boolean; options?: { numCandidates?: number; quantization?: 'none' | 'scalar' | 'binary'; diff --git a/libraries/grpc-sdk/src/modules/database/index.ts b/libraries/grpc-sdk/src/modules/database/index.ts index add5d6b6b..2b3b42ffb 100644 --- a/libraries/grpc-sdk/src/modules/database/index.ts +++ b/libraries/grpc-sdk/src/modules/database/index.ts @@ -372,6 +372,8 @@ export class DatabaseProvider extends ConduitModule { ); callback(null, { result: JSON.stringify(result) }); } catch (err) { - callback({ code: status.INTERNAL, message: (err as Error).message }); + callback(grpcStatusFromError(err)); } } @@ -1081,10 +1081,12 @@ export default class DatabaseModule extends ManagedModule { method: index.method, filterFields: [...(index.filterFields ?? [])], options: index.options ? JSON.stringify(index.options) : undefined, + status: index.status, + queryable: index.queryable, })), }); } catch (err) { - callback({ code: status.INTERNAL, message: (err as Error).message }); + callback(grpcStatusFromError(err)); } } @@ -1107,7 +1109,7 @@ export default class DatabaseModule extends ManagedModule { ); callback(null, { result: JSON.stringify(result) }); } catch (err) { - callback({ code: status.INTERNAL, message: (err as Error).message }); + callback(grpcStatusFromError(err)); } } diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index e9328bf76..d8c93bda5 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -16,6 +16,7 @@ import { import { ConfigController } from '@conduitplatform/module-tools'; import type { Config } from '../config/index.js'; import { unsupportedVectorCapabilities } from './utils/vectorCapabilities.js'; +import { declaredVectorIndexes } from './utils/vectorSearchQuery.js'; import { _ConduitSchema, ConduitDatabaseSchema, @@ -323,6 +324,20 @@ export abstract class DatabaseAdapter { ); } + protected async applyDeclaredVectorIndexes( + schemaName: string, + isInstanceSync: boolean, + ): Promise { + if (isInstanceSync) return; + const declared = declaredVectorIndexes(this.models[schemaName]?.originalSchema ?? {}); + if (!declared.length) return; + const capabilities = await this.getVectorCapabilities(schemaName); + if (!capabilities.indexing) return; + for (const index of declared) { + await this.createVectorIndex(schemaName, index); + } + } + vectorSearch(_request: VectorSearchInput): Promise { throw new GrpcError( status.UNIMPLEMENTED, diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 5d972b54f..f2d45e0f7 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -18,8 +18,6 @@ import { DatabaseAdapter } from '../DatabaseAdapter.js'; import { validateFieldChanges, validateFieldConstraints, - assertVectorIndexContract, - assertVectorIndexMatchesField, mongoVectorCapabilities, fromMongoVectorIndex, toMongoVectorIndexDefinition, @@ -28,6 +26,8 @@ import { declaredVectorIndexes, mergeVectorIndexes, planMongoVectorSearch, + bindVectorIndexToField, + planMongoVectorIndexCreate, } from '../utils/index.js'; import pluralize from '../../utils/pluralize.js'; import { mongoSchemaConverter } from '../../introspection/mongoose/utils.js'; @@ -767,8 +767,13 @@ export class MongooseAdapter extends DatabaseAdapter { ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - this.validateVectorField(schemaName, index); - assertVectorIndexContract('mongodb', index); + const schema = this.models[schemaName].originalSchema as any; + const field = schema.compiledFields?.[index.field] ?? schema.fields?.[index.field]; + const bound = bindVectorIndexToField({ + provider: 'mongodb', + index, + field, + }); const collection: any = this.mongoose.model(schemaName).collection; if (typeof collection.createSearchIndex !== 'function') { throw new GrpcError( @@ -776,10 +781,13 @@ export class MongooseAdapter extends DatabaseAdapter { 'MongoDB Vector Search index commands are not available for this deployment', ); } + const existing = await this.getVectorIndexes(schemaName); + const plan = planMongoVectorIndexCreate({ requested: bound, existing }); + if (plan.action === 'reuse') return 'Vector index created!'; await collection.createSearchIndex({ - name: index.name ?? `${index.field}_vector`, + name: bound.name, type: 'vectorSearch', - definition: toMongoVectorIndexDefinition(index), + definition: toMongoVectorIndexDefinition(bound), }); return 'Vector index created!'; } @@ -835,7 +843,7 @@ export class MongooseAdapter extends DatabaseAdapter { const schemaFields = (model.originalSchema.compiledFields ?? model.originalSchema.fields) as Record; - const liveIndexes = await this.getVectorIndexes(request.schemaName).catch(() => []); + const liveIndexes = await this.getVectorIndexes(request.schemaName); const planned = planMongoVectorSearch({ request, indexes: mergeVectorIndexes( @@ -945,12 +953,6 @@ export class MongooseAdapter extends DatabaseAdapter { ); } - private validateVectorField(schemaName: string, index: VectorIndexDefinition) { - const schema = this.models[schemaName].originalSchema as any; - const field = schema.compiledFields?.[index.field] ?? schema.fields?.[index.field]; - assertVectorIndexMatchesField(field, index); - } - protected async _createSchemaFromAdapter( schema: ConduitDatabaseSchema, saveToDb: boolean = true, @@ -994,6 +996,7 @@ export class MongooseAdapter extends DatabaseAdapter { if (!isInstanceSync) { await this.createMongooseFieldIndexes(schema.name); } + await this.applyDeclaredVectorIndexes(schema.name, isInstanceSync); return this.models[schema.name]; } diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 1603917c6..3afa84535 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -34,26 +34,22 @@ import { sqlSchemaConverter } from './sql-adapter/SqlSchemaConverter.js'; import { pgSchemaConverter } from './postgres-adapter/PgSchemaConverter.js'; import { isEqual, isNil } from 'lodash-es'; import { - assertVectorIndexContract, - assertVectorIndexMatchesField, -} from '../utils/vectorField.js'; -import { - postgresVectorCapabilities, - sqlFallbackVectorCapabilities, -} from '../utils/vectorCapabilities.js'; -import { - fromPostgresVectorIndex, - pgVectorOperator, - postgresIndexMethodSql, - resolveVectorFieldFromSchema, -} from '../utils/vectorMappings.js'; -import { assertVectorSearchAccess } from '../utils/vectorSearchAuth.js'; -import { + assertVectorSearchAccess, + bindVectorIndexToField, completeVectorSearch, declaredVectorIndexes, + fromPostgresVectorIndex, mergeVectorIndexes, + pgVectorOperator, + planPostgresVectorIndexCreate, planPostgresVectorSearch, -} from '../utils/vectorSearchQuery.js'; + postgresIndexMethodSql, + postgresVectorCapabilities, + resolveVectorFieldFromSchema, + sqlFallbackVectorCapabilities, + assertPostgresVectorIndexDropTarget, + type PostgresCatalogIndex, +} from '../utils/index.js'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; @@ -307,6 +303,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter await this.compareAndStoreMigratedSchema(schema); await this.saveSchemaToDatabase(schema); } + await this.applyDeclaredVectorIndexes(schema.name, isInstanceSync); return this.models[schema.name]; } @@ -468,57 +465,76 @@ export abstract class SequelizeAdapter extends DatabaseAdapter index: VectorIndexDefinition, ): Promise { this.ensurePostgresVectorSupport(schemaName); - this.validateVectorField(schemaName, index); - assertVectorIndexContract('postgres', index); + const schema = this.models[schemaName].originalSchema; + const field = (schema.compiledFields?.[index.field] ?? + schema.fields?.[index.field]) as unknown; const tableName = this.getPhysicalTableName(schemaName); - const indexName = this.quoteIdentifier( - index.name ?? `${tableName}_${index.field}_vector`, - ); - const method = postgresIndexMethodSql(index.method); - const operator = pgVectorOperator(index.similarity); + const bound = bindVectorIndexToField({ + provider: 'postgres', + index, + field, + physicalTableName: tableName, + }); + const existing = await this.findPostgresCatalogIndex(bound.name!); + const method = postgresIndexMethodSql(bound.method); + const operator = pgVectorOperator(bound.similarity); const withOptions = method === 'ivfflat' - ? this.renderWithOptions({ lists: index.options?.ivfflat?.lists }) + ? this.renderWithOptions({ lists: bound.options?.ivfflat?.lists }) : this.renderWithOptions({ - m: index.options?.hnsw?.m, - ef_construction: index.options?.hnsw?.efConstruction, + m: bound.options?.hnsw?.m, + ef_construction: bound.options?.hnsw?.efConstruction, }); - await this.sequelize.query( - `CREATE INDEX IF NOT EXISTS ${indexName} ON ${this.quoteIdentifier( - tableName, - )} USING ${method} (${this.quoteIdentifier(index.field)} ${operator})${withOptions}`, - ); + const plan = planPostgresVectorIndexCreate({ + indexName: bound.name!, + tableName, + field: bound.field, + method, + operator, + withOptions, + existing, + quoteIdentifier: identifier => this.quoteIdentifier(identifier), + }); + if (plan.action === 'reuse') return 'Vector index created!'; + await this.sequelize.query(plan.sql); return 'Vector index created!'; } async getVectorIndexes(schemaName: string): Promise { this.ensurePostgresVectorSupport(schemaName); const tableName = this.getPhysicalTableName(schemaName); - const rows = await this.sequelize.query( - `SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = current_schema() AND tablename = ${this.sequelize.escape( - tableName, - )}`, - ); + const rows = await this.listPostgresCatalogIndexes(tableName); const schema = this.models[schemaName]?.originalSchema; const schemaFields = (schema?.compiledFields ?? schema?.fields) as Record | undefined; - return (rows[0] as any[]) + const declared = declaredVectorIndexes(schema ?? {}); + return rows .filter(row => /USING (hnsw|ivfflat)/i.test(row.indexdef)) .map(row => { const mapped = fromPostgresVectorIndex(row.indexname, row.indexdef); const field = resolveVectorFieldFromSchema(schemaFields, mapped.field); - if (!field) return mapped; - return { - ...mapped, - dimensions: field.dimensions, - similarity: field.similarity ?? mapped.similarity, - }; + const matchingDeclared = declared.find( + item => item.name === row.indexname || item.field === mapped.field, + ); + return fromPostgresVectorIndex( + row.indexname, + row.indexdef, + field, + matchingDeclared, + ); }); } async deleteVectorIndex(schemaName: string, indexName: string): Promise { this.ensurePostgresVectorSupport(schemaName); - await this.sequelize.query(`DROP INDEX IF EXISTS ${this.quoteIdentifier(indexName)}`); + const tableName = this.getPhysicalTableName(schemaName); + const existing = await this.findPostgresCatalogIndex(indexName); + assertPostgresVectorIndexDropTarget({ + indexName, + tableName, + existing, + }); + await this.sequelize.query(`DROP INDEX ${this.quoteIdentifier(indexName)}`); return 'Vector index deleted'; } @@ -544,7 +560,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter }); const schemaFields = (schema.originalSchema.compiledFields ?? schema.originalSchema.fields) as Record; - const liveIndexes = await this.getVectorIndexes(request.schemaName).catch(() => []); + const liveIndexes = await this.getVectorIndexes(request.schemaName); const planned = planPostgresVectorSearch({ request, indexes: mergeVectorIndexes( @@ -629,11 +645,31 @@ export abstract class SequelizeAdapter extends DatabaseAdapter } } - private validateVectorField(schemaName: string, index: VectorIndexDefinition) { - const schema = this.models[schemaName].originalSchema; - const field = (schema.compiledFields?.[index.field] ?? - schema.fields?.[index.field]) as any; - assertVectorIndexMatchesField(field, index); + private async listPostgresCatalogIndexes( + tableName?: string, + ): Promise { + const tableFilter = tableName + ? ` AND tablename = ${this.sequelize.escape(tableName)}` + : ''; + const rows = await this.sequelize.query( + `SELECT indexname, tablename, indexdef FROM pg_indexes WHERE schemaname = current_schema()${tableFilter}`, + ); + return ((rows[0] as PostgresCatalogIndex[]) ?? []).map(row => ({ + indexname: row.indexname, + tablename: row.tablename, + indexdef: row.indexdef, + })); + } + + private async findPostgresCatalogIndex( + indexName: string, + ): Promise { + const rows = await this.sequelize.query( + `SELECT indexname, tablename, indexdef FROM pg_indexes WHERE schemaname = current_schema() AND indexname = ${this.sequelize.escape( + indexName, + )}`, + ); + return ((rows[0] as PostgresCatalogIndex[]) ?? [])[0]; } private getPhysicalTableName(schemaName: string) { diff --git a/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts b/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts new file mode 100644 index 000000000..b503f2fcb --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts @@ -0,0 +1,251 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { + GrpcError, + TYPE, + VectorIndexMethod, + VectorIndexStatus, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { DatabaseAdapter } from '../../DatabaseAdapter.js'; +import { MongooseAdapter } from '../../mongoose-adapter/index.js'; +import { SequelizeAdapter } from '../../sequelize-adapter/index.js'; + +const schemaFields = { + _id: { type: TYPE.ObjectId }, + title: { type: TYPE.String }, + tenantId: { type: TYPE.String }, + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, +}; + +const declaredIndex = { + field: 'embedding', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.HNSW, + filterFields: ['tenantId'], +}; + +function articleModel() { + return { + originalSchema: { + name: 'Article', + fields: schemaFields, + compiledFields: schemaFields, + collectionName: 'cnd_Article', + modelOptions: { vectorIndexes: [declaredIndex] }, + }, + }; +} + +describe('mongoose vector index lifecycle', () => { + it('creates search indexes with _id filters and default names, reusing matches', async () => { + const createSearchIndex = jest.fn(async () => undefined); + const listSearchIndexes = jest.fn(() => ({ + toArray: async () => [], + })); + const createIndex = jest.fn(async () => 'title_1'); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + mongoose: { + model: () => ({ + collection: { createSearchIndex, listSearchIndexes, createIndex }, + }), + }, + }); + + await adapter.createVectorIndex('Article', declaredIndex); + expect(createSearchIndex).toHaveBeenCalledWith({ + name: 'embedding_vector', + type: 'vectorSearch', + definition: { + fields: [ + { + type: 'vector', + path: 'embedding', + numDimensions: 3, + similarity: VectorSimilarity.Cosine, + indexingMethod: VectorIndexMethod.HNSW, + }, + { type: 'filter', path: '_id' }, + { type: 'filter', path: 'tenantId' }, + ], + }, + }); + + listSearchIndexes.mockImplementation(() => ({ + toArray: async () => [ + { + name: 'embedding_vector', + type: 'vectorSearch', + status: 'READY', + queryable: true, + latestDefinition: createSearchIndex.mock.calls[0][0].definition, + }, + ], + })); + await adapter.createVectorIndex('Article', declaredIndex); + expect(createSearchIndex).toHaveBeenCalledTimes(1); + + await adapter.createIndexes('Article', [{ fields: ['title'] }], 'database'); + expect(createIndex).toHaveBeenCalled(); + expect(createSearchIndex).toHaveBeenCalledTimes(1); + }); + + it('rejects vector search against a pending Mongo index', async () => { + const aggregate = jest.fn(); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + mongoose: { model: () => ({ collection: { aggregate } }) }, + getVectorIndexes: async () => [ + { + name: 'embedding_vector', + field: 'embedding', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + filterFields: ['_id', 'tenantId'], + status: VectorIndexStatus.Pending, + queryable: false, + }, + ], + }); + try { + await adapter.vectorSearch({ + schemaName: 'Article', + field: 'embedding', + vector: [0.1, 0.2, 0.3], + limit: 2, + }); + throw new Error('expected not-ready error'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.FAILED_PRECONDITION); + expect((err as GrpcError).message).toMatch(/not queryable/); + } + expect(aggregate).not.toHaveBeenCalled(); + }); +}); + +describe('postgres vector index lifecycle', () => { + it('creates vector indexes without IF NOT EXISTS and restores catalog options', async () => { + const query = jest.fn(async (sql: string) => { + if (sql.includes('pg_indexes') && sql.includes('indexname =')) return [[]]; + if (sql.includes('pg_indexes')) { + return [ + [ + { + indexname: 'cnd_Article_embedding_vector', + tablename: 'cnd_Article', + indexdef: + 'CREATE INDEX cnd_Article_embedding_vector ON cnd_Article USING hnsw ("embedding" vector_cosine_ops) WITH (m=16, ef_construction=64)', + }, + ], + ]; + } + return [[]]; + }); + const adapter = Object.create(SequelizeAdapter.prototype) as SequelizeAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + sequelize: { + getDialect: () => 'postgres', + escape: (value: unknown) => `'${value}'`, + query, + }, + }); + + await adapter.createVectorIndex('Article', { + ...declaredIndex, + options: { hnsw: { m: 16, efConstruction: 64 } }, + }); + const createSql = query.mock.calls.find(call => + String(call[0]).startsWith('CREATE INDEX'), + )?.[0] as string; + expect(createSql).toContain('CREATE INDEX "cnd_Article_embedding_vector"'); + expect(createSql).not.toMatch(/IF NOT EXISTS/i); + + const indexes = await adapter.getVectorIndexes('Article'); + expect(indexes[0]).toMatchObject({ + dimensions: 3, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.HNSW, + status: VectorIndexStatus.Ready, + queryable: true, + options: { hnsw: { m: 16, efConstruction: 64 } }, + }); + }); + + it('does not drop a vector index that belongs to another table', async () => { + const query = jest.fn(async () => [ + [ + { + indexname: 'embedding_vector', + tablename: 'cnd_Other', + indexdef: + 'CREATE INDEX embedding_vector ON cnd_Other USING hnsw ("embedding" vector_cosine_ops)', + }, + ], + ]); + const adapter = Object.create(SequelizeAdapter.prototype) as SequelizeAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + sequelize: { + getDialect: () => 'postgres', + escape: (value: unknown) => `'${value}'`, + query, + }, + }); + await expect( + adapter.deleteVectorIndex('Article', 'embedding_vector'), + ).rejects.toThrow(/was not found on table/); + expect(query.mock.calls.some(call => String(call[0]).startsWith('DROP INDEX'))).toBe( + false, + ); + }); +}); + +describe('declared vectorIndexes application', () => { + it('applies modelOptions.vectorIndexes through createVectorIndex, not regular indexes', async () => { + const createVectorIndex = jest.fn(async () => 'Vector index created!'); + const createIndexes = jest.fn(async () => 'Indexes created!'); + const adapter = Object.create(DatabaseAdapter.prototype) as DatabaseAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + createVectorIndex, + createIndexes, + getVectorCapabilities: async () => ({ + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'mongodb', + }), + }); + + await (adapter as any).applyDeclaredVectorIndexes('Article', false); + expect(createVectorIndex).toHaveBeenCalledWith('Article', declaredIndex); + expect(createIndexes).not.toHaveBeenCalled(); + + await (adapter as any).applyDeclaredVectorIndexes('Article', true); + expect(createVectorIndex).toHaveBeenCalledTimes(1); + + Object.assign(adapter, { + getVectorCapabilities: async () => ({ + supported: false, + storage: true, + indexing: false, + search: false, + provider: 'unsupported', + }), + }); + await (adapter as any).applyDeclaredVectorIndexes('Article', false); + expect(createVectorIndex).toHaveBeenCalledTimes(1); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts b/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts new file mode 100644 index 000000000..f68e6dab1 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts @@ -0,0 +1,294 @@ +import { describe, expect, it } from '@jest/globals'; +import { + GrpcError, + TYPE, + VectorIndexMethod, + VectorIndexStatus, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertPostgresVectorIndexDropTarget, + assertVectorIndexQueryable, + bindVectorIndexToField, + defaultVectorIndexName, + hydratePostgresVectorIndex, + mongoSearchIndexReadiness, + mongoVectorFilterFields, + planMongoVectorIndexCreate, + planPostgresVectorIndexCreate, + postgresVectorIndexDefinitionMatches, + renderPostgresCreateVectorIndexSql, +} from '../vectorIndexLifecycle.js'; + +const vectorField = { + type: TYPE.Vector, + dimensions: 1536, + similarity: VectorSimilarity.Cosine, +}; + +const quote = (identifier: string) => `"${identifier}"`; + +describe('vector index lifecycle', () => { + it('uses consistent default names and always includes Mongo _id filter fields', () => { + expect(defaultVectorIndexName('embedding')).toBe('embedding_vector'); + expect(defaultVectorIndexName('embedding', 'cnd_Article')).toBe( + 'cnd_Article_embedding_vector', + ); + expect(mongoVectorFilterFields(['tenantId'])).toEqual(['_id', 'tenantId']); + expect(mongoVectorFilterFields(['_id', 'tenantId'])).toEqual(['_id', 'tenantId']); + expect(mongoVectorFilterFields()).toEqual(['_id']); + }); + + it('binds declared indexes to field dimensions/similarity and rejects mismatches', () => { + const bound = bindVectorIndexToField({ + provider: 'mongodb', + field: vectorField, + index: { + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + filterFields: ['tenantId'], + }, + }); + expect(bound.name).toBe('embedding_vector'); + expect(bound.filterFields).toEqual(['_id', 'tenantId']); + + expect(() => + bindVectorIndexToField({ + provider: 'mongodb', + field: vectorField, + index: { + field: 'embedding', + dimensions: 768, + similarity: VectorSimilarity.Cosine, + }, + }), + ).toThrow(/dimensions mismatch/); + expect(() => + bindVectorIndexToField({ + provider: 'postgres', + field: vectorField, + index: { + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.Flat, + }, + }), + ).toThrow(/Unsupported vector index method/); + }); + + it('maps Atlas search index status to ready/pending/failed queryability', () => { + expect(mongoSearchIndexReadiness({ status: 'READY' })).toEqual({ + status: VectorIndexStatus.Ready, + queryable: true, + }); + expect(mongoSearchIndexReadiness({ status: 'PENDING' })).toEqual({ + status: VectorIndexStatus.Pending, + queryable: false, + }); + expect(mongoSearchIndexReadiness({ status: 'FAILED' })).toEqual({ + status: VectorIndexStatus.Failed, + queryable: false, + }); + expect(mongoSearchIndexReadiness({ status: 'STALE', queryable: true })).toEqual({ + status: VectorIndexStatus.Ready, + queryable: true, + }); + }); + + it('fails clearly when a vector index is missing or not queryable', () => { + try { + assertVectorIndexQueryable(undefined, { field: 'embedding' }); + throw new Error('expected missing index error'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.FAILED_PRECONDITION); + expect((err as GrpcError).message).toMatch(/No vector index is available/); + } + try { + assertVectorIndexQueryable( + { + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + status: VectorIndexStatus.Pending, + queryable: false, + }, + { field: 'embedding' }, + ); + throw new Error('expected not-ready error'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.FAILED_PRECONDITION); + expect((err as GrpcError).message).toMatch(/not queryable \(status: pending\)/); + } + }); + + it('reuses matching Mongo indexes and rejects silent definition changes', () => { + const requested = bindVectorIndexToField({ + provider: 'mongodb', + field: vectorField, + index: { + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.HNSW, + filterFields: ['tenantId'], + }, + }); + expect( + planMongoVectorIndexCreate({ + requested, + existing: [requested], + }), + ).toEqual({ action: 'reuse' }); + expect(planMongoVectorIndexCreate({ requested, existing: [] }).action).toBe('create'); + expect(() => + planMongoVectorIndexCreate({ + requested, + existing: [{ ...requested, dimensions: 768 }], + }), + ).toThrow(/different definition/); + }); + + it('creates Postgres vector indexes without IF NOT EXISTS and detects mismatches', () => { + const sql = renderPostgresCreateVectorIndexSql({ + indexName: 'cnd_Article_embedding_vector', + tableName: 'cnd_Article', + field: 'embedding', + method: 'hnsw', + operator: 'vector_cosine_ops', + withOptions: ' WITH (m = 16, ef_construction = 64)', + quoteIdentifier: quote, + }); + expect(sql).toContain('CREATE INDEX "cnd_Article_embedding_vector"'); + expect(sql).not.toMatch(/IF NOT EXISTS/i); + + expect( + planPostgresVectorIndexCreate({ + indexName: 'cnd_Article_embedding_vector', + tableName: 'cnd_Article', + field: 'embedding', + method: 'hnsw', + operator: 'vector_cosine_ops', + withOptions: '', + quoteIdentifier: quote, + }).action, + ).toBe('create'); + + expect( + planPostgresVectorIndexCreate({ + indexName: 'cnd_Article_embedding_vector', + tableName: 'cnd_Article', + field: 'embedding', + method: 'hnsw', + operator: 'vector_cosine_ops', + withOptions: '', + existing: { + indexname: 'cnd_Article_embedding_vector', + tablename: 'cnd_Article', + indexdef: + 'CREATE INDEX cnd_Article_embedding_vector ON cnd_Article USING hnsw ("embedding" vector_cosine_ops) WITH (m=16, ef_construction=64)', + }, + quoteIdentifier: quote, + }), + ).toEqual({ action: 'reuse' }); + + expect(() => + planPostgresVectorIndexCreate({ + indexName: 'cnd_Article_embedding_vector', + tableName: 'cnd_Article', + field: 'embedding', + method: 'hnsw', + operator: 'vector_cosine_ops', + withOptions: '', + existing: { + indexname: 'cnd_Article_embedding_vector', + tablename: 'cnd_Article', + indexdef: + 'CREATE INDEX cnd_Article_embedding_vector ON cnd_Article USING ivfflat ("embedding" vector_cosine_ops) WITH (lists=100)', + }, + quoteIdentifier: quote, + }), + ).toThrow(/different definition/); + }); + + it('scopes Postgres vector-index deletion to the requested table', () => { + expect(() => + assertPostgresVectorIndexDropTarget({ + indexName: 'embedding_vector', + tableName: 'cnd_Article', + existing: { + indexname: 'embedding_vector', + tablename: 'cnd_Other', + indexdef: + 'CREATE INDEX embedding_vector ON cnd_Other USING hnsw ("embedding" vector_cosine_ops)', + }, + }), + ).toThrow(/was not found on table/); + expect(() => + assertPostgresVectorIndexDropTarget({ + indexName: 'title_idx', + tableName: 'cnd_Article', + existing: { + indexname: 'title_idx', + tablename: 'cnd_Article', + indexdef: 'CREATE INDEX title_idx ON cnd_Article USING btree (title)', + }, + }), + ).toThrow(/is not a vector index/); + expect( + assertPostgresVectorIndexDropTarget({ + indexName: 'cnd_Article_embedding_vector', + tableName: 'cnd_Article', + existing: { + indexname: 'cnd_Article_embedding_vector', + tablename: 'cnd_Article', + indexdef: + 'CREATE INDEX cnd_Article_embedding_vector ON cnd_Article USING hnsw ("embedding" vector_cosine_ops)', + }, + }).indexname, + ).toBe('cnd_Article_embedding_vector'); + }); + + it('restores Postgres dimensions and WITH options during catalog read-back', () => { + const hydrated = hydratePostgresVectorIndex({ + name: 'cnd_Article_embedding_vector', + indexdef: + 'CREATE INDEX cnd_Article_embedding_vector ON cnd_Article USING hnsw ("embedding" vector_cosine_ops) WITH (m=16, ef_construction=64)', + field: vectorField, + declared: { + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + options: { hnsw: { m: 16 } }, + }, + }); + expect(hydrated).toMatchObject({ + name: 'cnd_Article_embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.HNSW, + status: VectorIndexStatus.Ready, + queryable: true, + options: { hnsw: { m: 16, efConstruction: 64 } }, + }); + expect( + postgresVectorIndexDefinitionMatches( + 'CREATE INDEX cnd_Article_embedding_vector ON cnd_Article USING ivfflat ("embedding" vector_l2_ops) WITH (lists=100)', + { + tableName: 'cnd_Article', + field: 'embedding', + method: 'ivfflat', + operator: 'vector_l2_ops', + options: { ivfflat: { lists: 100 } }, + }, + ), + ).toBe(true); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts b/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts index 0b076edb1..4ee20e0b7 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts @@ -95,6 +95,27 @@ describe('vector field and index mappings', () => { expect(converted.fields.embedding.type).toBe(DataTypes.JSON); }); + it('always includes _id as a Mongo vector index filter field', () => { + expect( + toMongoVectorIndexDefinition({ + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + filterFields: ['tenantId'], + }).fields, + ).toEqual([ + { + type: 'vector', + path: 'embedding', + numDimensions: 1536, + similarity: VectorSimilarity.Cosine, + }, + { type: 'filter', path: '_id' }, + { type: 'filter', path: 'tenantId' }, + ]); + }); + it('round-trips Mongo vector index definitions', () => { const definition = toMongoVectorIndexDefinition({ name: 'embedding_vector', @@ -123,6 +144,8 @@ describe('vector field and index mappings', () => { expect( fromMongoVectorIndex({ name: 'embedding_vector', + status: 'READY', + queryable: true, latestDefinition: definition, }), ).toMatchObject({ @@ -132,6 +155,8 @@ describe('vector field and index mappings', () => { similarity: VectorSimilarity.Cosine, method: VectorIndexMethod.HNSW, filterFields: ['_id', 'tenantId'], + status: 'ready', + queryable: true, }); }); @@ -159,7 +184,7 @@ describe('vector field and index mappings', () => { expect( fromPostgresVectorIndex( 'cnd_article_embedding_vector', - 'CREATE INDEX cnd_article_embedding_vector ON cnd_article USING ivfflat (embedding vector_l2_ops)', + 'CREATE INDEX cnd_article_embedding_vector ON cnd_article USING ivfflat (embedding vector_l2_ops) WITH (lists=100)', { dimensions: 1536, similarity: VectorSimilarity.Euclidean }, ), ).toMatchObject({ @@ -167,6 +192,8 @@ describe('vector field and index mappings', () => { dimensions: 1536, similarity: VectorSimilarity.Euclidean, method: 'ivfflat', + queryable: true, + options: { ivfflat: { lists: 100 } }, }); }); }); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts index 2b1cb8e12..2327b9faf 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it, jest } from '@jest/globals'; -import { TYPE, VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { + GrpcError, + TYPE, + VectorIndexStatus, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; import { completeVectorSearch, planMongoVectorSearch, @@ -86,6 +92,40 @@ describe('vector search query planning', () => { expect(planned.sql).toMatch(/^SELECT "title", "_id",/); expect(planned.sql).toContain('<=>'); }); + + it('fails clearly when the selected vector index is not queryable', () => { + expect(() => + planMongoVectorSearch({ + request, + indexes: [ + { + ...indexes[0], + status: VectorIndexStatus.Pending, + queryable: false, + }, + ], + schemaFields, + }), + ).toThrow(GrpcError); + try { + planMongoVectorSearch({ + request, + indexes: [ + { + ...indexes[0], + status: VectorIndexStatus.Failed, + queryable: false, + }, + ], + schemaFields, + }); + throw new Error('expected failed index error'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.FAILED_PRECONDITION); + expect((err as GrpcError).message).toMatch(/not queryable/); + } + }); }); describe('bounded vector search completion', () => { diff --git a/modules/database/src/adapters/utils/index.ts b/modules/database/src/adapters/utils/index.ts index 97aebccc3..a514cd7ad 100644 --- a/modules/database/src/adapters/utils/index.ts +++ b/modules/database/src/adapters/utils/index.ts @@ -15,3 +15,4 @@ export * from './vectorSearchFilter.js'; export * from './vectorSearchWhere.js'; export * from './vectorSearchQuery.js'; export * from './vectorProjection.js'; +export * from './vectorIndexLifecycle.js'; diff --git a/modules/database/src/adapters/utils/vectorIndexLifecycle.ts b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts new file mode 100644 index 000000000..44c3e897f --- /dev/null +++ b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts @@ -0,0 +1,409 @@ +import { + GrpcError, + VectorIndexDefinition, + VectorIndexMethod, + VectorIndexStatus, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertVectorIndexContract, + assertVectorIndexMatchesField, + isObjectFormVectorField, + VectorIndexProvider, +} from './vectorField.js'; + +export const MONGO_VECTOR_ID_FILTER_FIELD = '_id'; + +export interface PostgresCatalogIndex { + indexname: string; + tablename: string; + indexdef: string; +} + +export interface ParsedPostgresVectorIndex { + name?: string; + tableName?: string; + field: string; + method?: string; + similarity: VectorSimilarity; + options?: VectorIndexDefinition['options']; +} + +export type PostgresVectorIndexCreatePlan = + { action: 'create'; sql: string } | { action: 'reuse' }; + +export function defaultVectorIndexName( + field: string, + physicalTableName?: string, +): string { + return physicalTableName ? `${physicalTableName}_${field}_vector` : `${field}_vector`; +} + +export function mongoVectorFilterFields(filterFields?: readonly string[]): string[] { + const fields: string[] = []; + for (const field of [MONGO_VECTOR_ID_FILTER_FIELD, ...(filterFields ?? [])]) { + if (!fields.includes(field)) { + fields.push(field); + } + } + return fields; +} + +export function bindVectorIndexToField(args: { + provider: VectorIndexProvider; + index: VectorIndexDefinition; + field: unknown; + physicalTableName?: string; +}): VectorIndexDefinition { + const field = isObjectFormVectorField(args.field) ? args.field : undefined; + const bound: VectorIndexDefinition = { + ...args.index, + name: + args.index.name ?? defaultVectorIndexName(args.index.field, args.physicalTableName), + dimensions: args.index.dimensions ?? field?.dimensions ?? args.index.dimensions, + similarity: args.index.similarity ?? field?.similarity ?? args.index.similarity, + filterFields: + args.provider === 'mongodb' + ? mongoVectorFilterFields(args.index.filterFields) + : args.index.filterFields, + }; + assertVectorIndexContract(args.provider, bound); + assertVectorIndexMatchesField(args.field, bound); + return bound; +} + +export function mongoSearchIndexReadiness(index: { + status?: string; + queryable?: boolean; +}): { status: VectorIndexStatus; queryable: boolean } { + const raw = (index.status ?? '').toUpperCase(); + switch (raw) { + case 'READY': + return { status: VectorIndexStatus.Ready, queryable: true }; + case 'STALE': + return { + status: VectorIndexStatus.Ready, + queryable: index.queryable !== false, + }; + case 'FAILED': + case 'DOES_NOT_EXIST': + return { status: VectorIndexStatus.Failed, queryable: false }; + case 'PENDING': + case 'BUILDING': + case 'DELETING': + case '': + return { + status: VectorIndexStatus.Pending, + queryable: index.queryable === true, + }; + default: + if (index.queryable === true) { + return { status: VectorIndexStatus.Ready, queryable: true }; + } + return { status: VectorIndexStatus.Pending, queryable: false }; + } +} + +export function isVectorIndexQueryable(index?: VectorIndexDefinition): boolean { + if (!index) return false; + if (index.queryable === false) return false; + if (index.status === VectorIndexStatus.Failed) return false; + if (index.status === VectorIndexStatus.Pending && index.queryable !== true) { + return false; + } + return true; +} + +export function assertVectorIndexQueryable( + index: VectorIndexDefinition | undefined, + request: { field: string; indexName?: string }, +): asserts index is VectorIndexDefinition { + if (!index) { + const named = request.indexName ? ` (index '${request.indexName}')` : ''; + throw new GrpcError( + status.FAILED_PRECONDITION, + `No vector index is available for field '${request.field}'${named}. ` + + 'Create the index and wait until it is ready before searching.', + ); + } + if (isVectorIndexQueryable(index)) return; + const indexName = index.name ?? defaultVectorIndexName(index.field); + const statusLabel = index.status ?? 'unknown'; + throw new GrpcError( + status.FAILED_PRECONDITION, + `Vector index '${indexName}' is not queryable (status: ${statusLabel}). ` + + 'Wait until the index is ready before searching.', + ); +} + +export function vectorIndexesEquivalent( + left: VectorIndexDefinition, + right: VectorIndexDefinition, + provider: VectorIndexProvider, +): boolean { + if (left.field !== right.field) return false; + if (left.dimensions !== right.dimensions) return false; + if (left.similarity !== right.similarity) return false; + const leftMethod = left.method ?? VectorIndexMethod.HNSW; + const rightMethod = right.method ?? VectorIndexMethod.HNSW; + if (leftMethod !== rightMethod) return false; + if (provider === 'mongodb') { + return sameStringSet( + mongoVectorFilterFields(left.filterFields), + mongoVectorFilterFields(right.filterFields), + ); + } + return true; +} + +export function planMongoVectorIndexCreate(args: { + requested: VectorIndexDefinition; + existing: VectorIndexDefinition[]; +}): { action: 'create' } | { action: 'reuse' } { + const existing = args.existing.find(index => index.name === args.requested.name); + if (!existing) return { action: 'create' }; + if (vectorIndexesEquivalent(args.requested, existing, 'mongodb')) { + return { action: 'reuse' }; + } + throw new GrpcError( + status.FAILED_PRECONDITION, + `Vector index '${args.requested.name}' already exists with a different definition. ` + + 'Drop it before recreating.', + ); +} + +export function parsePostgresVectorIndexDef(indexdef: string): ParsedPostgresVectorIndex { + const tableMatch = /ON\s+(?:(?:"[^"]+"|\w+)\.)?(?:"([^"]+)"|(\w+))/i.exec(indexdef); + const method = /USING\s+(\w+)/i.exec(indexdef)?.[1]?.toLowerCase(); + const fieldMatch = /\((?:"([^"]+)"|(\w+))\s+vector_/i.exec(indexdef); + const operator = /vector_(l2|cosine|ip)_ops/i.exec(indexdef)?.[1]; + const similarity = + operator === 'l2' + ? VectorSimilarity.Euclidean + : operator === 'ip' + ? VectorSimilarity.DotProduct + : VectorSimilarity.Cosine; + return { + tableName: tableMatch?.[1] ?? tableMatch?.[2], + field: fieldMatch?.[1] ?? fieldMatch?.[2] ?? '', + method, + similarity, + options: parsePostgresIndexOptions(indexdef, method), + }; +} + +export function postgresVectorIndexDefinitionMatches( + indexdef: string, + expected: { + tableName: string; + field: string; + method: string; + operator: string; + options?: VectorIndexDefinition['options']; + }, +): boolean { + const parsed = parsePostgresVectorIndexDef(indexdef); + if (parsed.tableName && parsed.tableName !== expected.tableName) return false; + if (parsed.field !== expected.field) return false; + if ((parsed.method ?? '').toLowerCase() !== expected.method.toLowerCase()) { + return false; + } + const expectedSimilarity = + expected.operator === 'vector_l2_ops' + ? VectorSimilarity.Euclidean + : expected.operator === 'vector_ip_ops' + ? VectorSimilarity.DotProduct + : VectorSimilarity.Cosine; + if (parsed.similarity !== expectedSimilarity) return false; + return postgresRequestedOptionsMatch(expected.options, parsed.options, expected.method); +} + +export function planPostgresVectorIndexCreate(args: { + indexName: string; + tableName: string; + field: string; + method: string; + operator: string; + withOptions: string; + existing?: PostgresCatalogIndex; + quoteIdentifier: (identifier: string) => string; +}): PostgresVectorIndexCreatePlan { + if (!args.existing) { + return { + action: 'create', + sql: renderPostgresCreateVectorIndexSql(args), + }; + } + if (args.existing.tablename !== args.tableName) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Vector index name '${args.indexName}' already exists on table '${args.existing.tablename}'.`, + ); + } + if (!isPostgresVectorIndexDef(args.existing.indexdef)) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Index '${args.indexName}' exists on '${args.tableName}' but is not a vector index.`, + ); + } + if ( + !postgresVectorIndexDefinitionMatches(args.existing.indexdef, { + tableName: args.tableName, + field: args.field, + method: args.method, + operator: args.operator, + options: withOptionsToDefinition(args.method, args.withOptions), + }) + ) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Vector index '${args.indexName}' already exists on '${args.tableName}' with a different definition. ` + + 'Drop it before recreating.', + ); + } + return { action: 'reuse' }; +} + +export function renderPostgresCreateVectorIndexSql(args: { + indexName: string; + tableName: string; + field: string; + method: string; + operator: string; + withOptions: string; + quoteIdentifier: (identifier: string) => string; +}): string { + return ( + `CREATE INDEX ${args.quoteIdentifier(args.indexName)} ON ${args.quoteIdentifier( + args.tableName, + )} USING ${args.method} (${args.quoteIdentifier(args.field)} ${args.operator})` + + args.withOptions + ); +} + +export function assertPostgresVectorIndexDropTarget(args: { + indexName: string; + tableName: string; + existing?: PostgresCatalogIndex; +}): PostgresCatalogIndex { + if (!args.existing || args.existing.tablename !== args.tableName) { + throw new GrpcError( + status.NOT_FOUND, + `Vector index '${args.indexName}' was not found on table '${args.tableName}'.`, + ); + } + if (!isPostgresVectorIndexDef(args.existing.indexdef)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Index '${args.indexName}' on '${args.tableName}' is not a vector index.`, + ); + } + return args.existing; +} + +export function hydratePostgresVectorIndex(args: { + name: string; + indexdef: string; + field?: { dimensions?: number; similarity?: VectorSimilarity }; + declared?: VectorIndexDefinition; +}): VectorIndexDefinition { + const parsed = parsePostgresVectorIndexDef(args.indexdef); + const method = + (parsed.method as VectorIndexMethod | undefined) ?? args.declared?.method; + return { + name: args.name, + field: parsed.field || args.declared?.field || '', + dimensions: args.field?.dimensions ?? args.declared?.dimensions ?? 0, + similarity: args.field?.similarity ?? parsed.similarity, + method: method ?? VectorIndexMethod.HNSW, + options: mergeVectorIndexOptions(args.declared?.options, parsed.options), + status: VectorIndexStatus.Ready, + queryable: true, + }; +} + +export function isPostgresVectorIndexDef(indexdef: string): boolean { + return /USING\s+(hnsw|ivfflat)\b/i.test(indexdef); +} + +function sameStringSet(left: string[], right: string[]): boolean { + if (left.length !== right.length) return false; + const rightSet = new Set(right); + return left.every(value => rightSet.has(value)); +} + +function parsePostgresIndexOptions( + indexdef: string, + method?: string, +): VectorIndexDefinition['options'] | undefined { + const match = /WITH\s*\(([^)]*)\)/i.exec(indexdef); + if (!match) return undefined; + const values: Record = {}; + for (const part of match[1].split(',')) { + const [rawKey, rawValue] = part.split('=').map(item => item.trim()); + if (!rawKey || rawValue === undefined) continue; + const value = Number(rawValue.replace(/^['"]|['"]$/g, '')); + if (!Number.isFinite(value)) continue; + values[rawKey.toLowerCase()] = value; + } + if (method === 'ivfflat') { + return values.lists !== undefined ? { ivfflat: { lists: values.lists } } : undefined; + } + const hnsw: NonNullable['hnsw'] = {}; + if (values.m !== undefined) hnsw.m = values.m; + if (values.ef_construction !== undefined) { + hnsw.efConstruction = values.ef_construction; + } + return Object.keys(hnsw).length ? { hnsw } : undefined; +} + +function withOptionsToDefinition( + method: string, + withOptions: string, +): VectorIndexDefinition['options'] | undefined { + if (!withOptions.trim()) return undefined; + return parsePostgresIndexOptions(` ${withOptions}`, method.toLowerCase()); +} + +function postgresRequestedOptionsMatch( + requested: VectorIndexDefinition['options'] | undefined, + actual: VectorIndexDefinition['options'] | undefined, + method: string, +): boolean { + if (method === 'ivfflat') { + if (requested?.ivfflat?.lists === undefined) return true; + return requested.ivfflat.lists === actual?.ivfflat?.lists; + } + if (requested?.hnsw?.m === undefined && requested?.hnsw?.efConstruction === undefined) { + return true; + } + if (requested?.hnsw?.m !== undefined && requested.hnsw.m !== actual?.hnsw?.m) { + return false; + } + if ( + requested?.hnsw?.efConstruction !== undefined && + requested.hnsw.efConstruction !== actual?.hnsw?.efConstruction + ) { + return false; + } + return true; +} + +function mergeVectorIndexOptions( + declared?: VectorIndexDefinition['options'], + catalog?: VectorIndexDefinition['options'], +): VectorIndexDefinition['options'] | undefined { + if (!declared && !catalog) return undefined; + const hnsw = { ...declared?.hnsw, ...catalog?.hnsw }; + const ivfflat = { ...declared?.ivfflat, ...catalog?.ivfflat }; + const merged: VectorIndexDefinition['options'] = { + ...declared, + ...catalog, + }; + if (Object.keys(hnsw).length) merged.hnsw = hnsw; + else delete merged.hnsw; + if (Object.keys(ivfflat).length) merged.ivfflat = ivfflat; + else delete merged.ivfflat; + return Object.values(merged).some(value => value !== undefined) ? merged : undefined; +} diff --git a/modules/database/src/adapters/utils/vectorMappings.ts b/modules/database/src/adapters/utils/vectorMappings.ts index 08f6cd19b..30322cd69 100644 --- a/modules/database/src/adapters/utils/vectorMappings.ts +++ b/modules/database/src/adapters/utils/vectorMappings.ts @@ -5,6 +5,11 @@ import { VectorSimilarity, } from '@conduitplatform/grpc-sdk'; import { isObjectFormVectorField } from './vectorField.js'; +import { + hydratePostgresVectorIndex, + mongoSearchIndexReadiness, + mongoVectorFilterFields, +} from './vectorIndexLifecycle.js'; export type VectorStorageBackend = 'mongodb' | 'postgres' | 'sql'; @@ -116,7 +121,10 @@ export function toMongoVectorIndexDefinition(index: VectorIndexDefinition) { return { fields: [ vectorField, - ...(index.filterFields ?? []).map((path: string) => ({ type: 'filter', path })), + ...mongoVectorFilterFields(index.filterFields).map((path: string) => ({ + type: 'filter', + path, + })), ], ...(index.options?.storedSource !== undefined && { storedSource: index.options.storedSource, @@ -126,11 +134,14 @@ export function toMongoVectorIndexDefinition(index: VectorIndexDefinition) { export function fromMongoVectorIndex(index: { name?: string; + status?: string; + queryable?: boolean; latestDefinition?: { fields?: Array> }; definition?: { fields?: Array> }; }): VectorIndexDefinition { const fields = index.latestDefinition?.fields ?? index.definition?.fields ?? []; const vectorField = fields.find(field => field.type === 'vector') ?? {}; + const readiness = mongoSearchIndexReadiness(index); return { name: index.name, field: vectorField.path, @@ -140,6 +151,8 @@ export function fromMongoVectorIndex(index: { filterFields: fields .filter(field => field.type === 'filter') .map(field => field.path), + status: readiness.status, + queryable: readiness.queryable, }; } @@ -147,24 +160,14 @@ export function fromPostgresVectorIndex( name: string, definition: string, field?: { dimensions?: number; similarity?: VectorSimilarity }, + declared?: VectorIndexDefinition, ): VectorIndexDefinition { - const method = /USING\s+(\w+)/i.exec(definition)?.[1]; - const fieldMatch = /\((?:"([^"]+)"|(\w+))\s+vector_/i.exec(definition); - const operator = /vector_(l2|cosine|ip)_ops/i.exec(definition)?.[1]; - const similarity = - field?.similarity ?? - (operator === 'l2' - ? VectorSimilarity.Euclidean - : operator === 'ip' - ? VectorSimilarity.DotProduct - : VectorSimilarity.Cosine); - return { + return hydratePostgresVectorIndex({ name, - field: fieldMatch?.[1] ?? fieldMatch?.[2] ?? '', - dimensions: field?.dimensions ?? 0, - similarity, - method: method as VectorIndexMethod | undefined, - }; + indexdef: definition, + field, + declared, + }); } export function postgresIndexMethodSql(method?: VectorIndexMethod | string) { diff --git a/modules/database/src/adapters/utils/vectorSearchQuery.ts b/modules/database/src/adapters/utils/vectorSearchQuery.ts index ccad4abbf..18c00c765 100644 --- a/modules/database/src/adapters/utils/vectorSearchQuery.ts +++ b/modules/database/src/adapters/utils/vectorSearchQuery.ts @@ -21,6 +21,10 @@ import { toVectorSearchResult, } from './vectorScore.js'; import { parseVectorSimilarity } from './vectorField.js'; +import { + assertVectorIndexQueryable, + defaultVectorIndexName, +} from './vectorIndexLifecycle.js'; export interface PlannedMongoVectorSearch { emptyResult: boolean; @@ -60,7 +64,9 @@ export function findVectorIndexForSearch( } return ( indexes.find( - item => item.field === request.field && item.name === `${request.field}_vector`, + item => + item.field === request.field && + item.name === defaultVectorIndexName(request.field), ) ?? indexes.find(item => item.field === request.field) ); } @@ -73,10 +79,10 @@ export function mergeVectorIndexes( if (!declared.length) return live; const merged = new Map(); for (const index of declared) { - merged.set(index.name ?? `${index.field}_vector`, index); + merged.set(index.name ?? defaultVectorIndexName(index.field), index); } for (const index of live) { - merged.set(index.name ?? `${index.field}_vector`, index); + merged.set(index.name ?? defaultVectorIndexName(index.field), index); } return [...merged.values()]; } @@ -124,12 +130,16 @@ export function planMongoVectorSearch(args: { if (validated.emptyResult) { return { emptyResult: true, limits, index, pipeline: [] }; } + assertVectorIndexQueryable(index, args.request); return { emptyResult: false, limits, index, pipeline: buildMongoVectorSearchPipeline({ - indexName: args.request.indexName ?? index?.name ?? `${args.request.field}_vector`, + indexName: + args.request.indexName ?? + index.name ?? + defaultVectorIndexName(args.request.field), field: args.request.field, vector: args.request.vector, numCandidates: limits.numCandidates, @@ -163,6 +173,7 @@ export function planPostgresVectorSearch(args: { if (validated.emptyResult) { return { emptyResult: true, limits, index, sql: '', distanceOperator }; } + assertVectorIndexQueryable(index, args.request); const where = renderPostgresVectorWhere(validated.filter, args.renderer); const selectedColumns = postgresVectorSelectList( args.schemaFields, diff --git a/modules/database/src/database.proto b/modules/database/src/database.proto index 77c87880f..276d2d232 100644 --- a/modules/database/src/database.proto +++ b/modules/database/src/database.proto @@ -186,6 +186,8 @@ message VectorIndex { optional string method = 5; repeated string filterFields = 6; optional string options = 7; + optional string status = 8; + optional bool queryable = 9; } message VectorIndexRequest { From ac66c3eebe752f3c54d9d8a8e6fb693c14391e2b Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 19:26:44 +0300 Subject: [PATCH 07/29] feat(embeddings): persist cursor-based backfill run state Add an embeddings-owned BackfillRun schema and a pure transition/pagination helper so operational backfills can resume, cancel, and report progress without scanning in the gRPC thread. --- .../src/models/BackfillRun.schema.ts | 80 ++++ modules/embeddings/src/models/index.ts | 1 + .../embeddings/src/utils/backfillRun.test.ts | 313 ++++++++++++++ modules/embeddings/src/utils/backfillRun.ts | 391 ++++++++++++++++++ .../embeddings/src/utils/schemaPolicy.test.ts | 3 +- modules/embeddings/src/utils/schemaPolicy.ts | 8 +- 6 files changed, 794 insertions(+), 2 deletions(-) create mode 100644 modules/embeddings/src/models/BackfillRun.schema.ts create mode 100644 modules/embeddings/src/utils/backfillRun.test.ts create mode 100644 modules/embeddings/src/utils/backfillRun.ts diff --git a/modules/embeddings/src/models/BackfillRun.schema.ts b/modules/embeddings/src/models/BackfillRun.schema.ts new file mode 100644 index 000000000..7815bec3f --- /dev/null +++ b/modules/embeddings/src/models/BackfillRun.schema.ts @@ -0,0 +1,80 @@ +import { + ConduitModel, + DatabaseProvider, + Indexable, + TYPE, +} from '@conduitplatform/grpc-sdk'; +import { ConduitActiveSchema } from '@conduitplatform/module-tools'; +import { BACKFILL_RUN_STATES, BackfillRunState } from '../utils/backfillRun.js'; + +const schema: ConduitModel = { + _id: TYPE.ObjectId, + schemaName: { type: TYPE.String, required: true }, + configId: { type: TYPE.String, required: false }, + state: { + type: TYPE.String, + enum: [...BACKFILL_RUN_STATES], + required: true, + default: 'queued', + }, + cursor: { type: TYPE.String, required: false }, + batchSize: { type: TYPE.Number, required: true }, + onlyMissing: { type: TYPE.Boolean, default: false }, + filter: { type: TYPE.JSON, required: false }, + scannedCount: { type: TYPE.Number, default: 0 }, + queuedCount: { type: TYPE.Number, default: 0 }, + processedCount: { type: TYPE.Number, default: 0 }, + failedCount: { type: TYPE.Number, default: 0 }, + startedAt: { type: TYPE.Date, required: false }, + finishedAt: { type: TYPE.Date, required: false }, + error: { type: TYPE.String, required: false }, + createdAt: TYPE.Date, + updatedAt: TYPE.Date, +}; + +const modelOptions = { + timestamps: true, + indexes: [{ fields: ['schemaName', 'state'] }, { fields: ['configId', 'state'] }], + conduit: { + permissions: { + extendable: false, + canCreate: false, + canModify: 'Nothing', + canDelete: false, + }, + }, +} as const; + +export class BackfillRun extends ConduitActiveSchema { + private static _instance: BackfillRun; + _id: string; + schemaName: string; + configId?: string; + state: BackfillRunState; + cursor?: string; + batchSize: number; + onlyMissing: boolean; + filter?: Indexable; + scannedCount: number; + queuedCount: number; + processedCount: number; + failedCount: number; + startedAt?: Date; + finishedAt?: Date; + error?: string; + createdAt: Date; + updatedAt: Date; + + private constructor(database: DatabaseProvider) { + super(database, BackfillRun.name, schema, modelOptions); + } + + static getInstance(database?: DatabaseProvider) { + if (BackfillRun._instance) return BackfillRun._instance; + if (!database) { + throw new Error('No database instance provided!'); + } + BackfillRun._instance = new BackfillRun(database); + return BackfillRun._instance; + } +} diff --git a/modules/embeddings/src/models/index.ts b/modules/embeddings/src/models/index.ts index 9b3e3b4e4..27cb5ba7b 100644 --- a/modules/embeddings/src/models/index.ts +++ b/modules/embeddings/src/models/index.ts @@ -1 +1,2 @@ export * from './EmbeddingConfig.schema.js'; +export * from './BackfillRun.schema.js'; diff --git a/modules/embeddings/src/utils/backfillRun.test.ts b/modules/embeddings/src/utils/backfillRun.test.ts new file mode 100644 index 000000000..f593a7224 --- /dev/null +++ b/modules/embeddings/src/utils/backfillRun.test.ts @@ -0,0 +1,313 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + applyBackfillJobCounts, + applyBackfillPage, + BACKFILL_RUN_STATES, + boundBackfillBatchSize, + boundBackfillPage, + buildBackfillPageQuery, + canCancelBackfill, + cancelBackfillRun, + completeBackfillRun, + createQueuedBackfill, + DEFAULT_BACKFILL_BATCH_SIZE, + failBackfillRun, + isLegalBackfillTransition, + isResumeEligible, + LEGAL_BACKFILL_TRANSITIONS, + MAX_BACKFILL_BATCH_SIZE, + MAX_BACKFILL_ERROR_LENGTH, + MAX_BACKFILL_FILTER_BYTES, + MIN_BACKFILL_BATCH_SIZE, + resumeBackfillRun, + sanitizeBackfillError, + startBackfillRun, +} from './backfillRun.js'; + +const now = new Date('2026-09-06T16:00:00.000Z'); + +function queuedRun() { + const created = createQueuedBackfill({ + schemaName: 'Article', + configId: 'cfg1', + batchSize: 2, + onlyMissing: true, + filter: { published: true }, + }); + assert.equal(created.ok, true); + if (!created.ok) throw new Error('expected queued run'); + return created.run; +} + +function runningRun() { + const started = startBackfillRun(queuedRun(), now); + assert.equal(started.ok, true); + if (!started.ok) throw new Error('expected running run'); + return started.run; +} + +describe('backfill run state transitions', () => { + it('allows only the documented legal transitions', () => { + assert.deepEqual(BACKFILL_RUN_STATES, [ + 'queued', + 'running', + 'completed', + 'failed', + 'canceled', + ]); + assert.equal(isLegalBackfillTransition('queued', 'running'), true); + assert.equal(isLegalBackfillTransition('queued', 'canceled'), true); + assert.equal(isLegalBackfillTransition('running', 'completed'), true); + assert.equal(isLegalBackfillTransition('running', 'failed'), true); + assert.equal(isLegalBackfillTransition('running', 'canceled'), true); + assert.equal(isLegalBackfillTransition('failed', 'queued'), true); + assert.equal(isLegalBackfillTransition('canceled', 'queued'), true); + assert.equal(isLegalBackfillTransition('queued', 'completed'), false); + assert.equal(isLegalBackfillTransition('queued', 'failed'), false); + assert.equal(isLegalBackfillTransition('running', 'queued'), false); + assert.equal(isLegalBackfillTransition('completed', 'queued'), false); + assert.equal(isLegalBackfillTransition('completed', 'running'), false); + assert.equal(isLegalBackfillTransition('failed', 'running'), false); + assert.equal(isLegalBackfillTransition('canceled', 'running'), false); + assert.deepEqual(LEGAL_BACKFILL_TRANSITIONS.completed, []); + }); + + it('starts a queued run and records startedAt', () => { + const started = startBackfillRun(queuedRun(), now); + assert.equal(started.ok, true); + if (!started.ok) return; + assert.equal(started.run.state, 'running'); + assert.equal(started.run.startedAt?.toISOString(), now.toISOString()); + assert.equal(started.run.finishedAt, null); + assert.equal(started.run.error, null); + }); + + it('rejects illegal transitions without mutating counters', () => { + const completed = completeBackfillRun(runningRun(), now); + assert.equal(completed.ok, true); + if (!completed.ok) return; + const resumed = resumeBackfillRun(completed.run); + assert.equal(resumed.ok, false); + if (resumed.ok) return; + assert.equal(resumed.reason, 'illegal_transition'); + const failedFromQueued = failBackfillRun(queuedRun(), 'boom', now); + assert.equal(failedFromQueued.ok, false); + }); +}); + +describe('backfill pagination and cursor progression', () => { + it('advances the cursor through ordered pages and marks exhaustion', () => { + const first = applyBackfillPage(runningRun(), [{ _id: 'a' }, { _id: 'b' }]); + assert.equal(first.ok, true); + if (!first.ok) return; + assert.equal(first.exhausted, false); + assert.equal(first.run.cursor, 'b'); + assert.equal(first.run.scannedCount, 2); + assert.equal(first.run.queuedCount, 2); + + const query = buildBackfillPageQuery(first.run, 'embedding'); + assert.equal(query.ok, true); + if (!query.ok) return; + assert.deepEqual(query.page, { + query: { published: true, embedding: null, _id: { $gt: 'b' } }, + sort: { _id: 1 }, + limit: 2, + }); + + const last = applyBackfillPage(first.run, [{ _id: 'c' }]); + assert.equal(last.ok, true); + if (!last.ok) return; + assert.equal(last.exhausted, true); + assert.equal(last.run.cursor, 'c'); + assert.equal(last.run.scannedCount, 3); + }); + + it('does not move the cursor on an empty exhausted page', () => { + const empty = applyBackfillPage(runningRun(), []); + assert.equal(empty.ok, true); + if (!empty.ok) return; + assert.equal(empty.exhausted, true); + assert.equal(empty.run.cursor, null); + assert.equal(empty.run.scannedCount, 0); + const completed = completeBackfillRun(empty.run, now); + assert.equal(completed.ok, true); + if (!completed.ok) return; + assert.equal(completed.run.state, 'completed'); + }); + + it('rejects a page that would not advance the cursor', () => { + const first = applyBackfillPage(runningRun(), [{ _id: 'a' }, { _id: 'b' }]); + assert.equal(first.ok, true); + if (!first.ok) return; + const stuck = applyBackfillPage(first.run, [{ _id: 'b' }]); + assert.equal(stuck.ok, false); + if (stuck.ok) return; + assert.equal(stuck.reason, 'cursor'); + }); + + it('owns pagination _id and requires a target field for onlyMissing', () => { + const run = runningRun(); + const missingTarget = buildBackfillPageQuery(run); + assert.equal(missingTarget.ok, false); + const ownedId = buildBackfillPageQuery( + { ...run, cursor: 'doc1', filter: { _id: 'ignored', published: true } }, + 'embedding', + ); + assert.equal(ownedId.ok, true); + if (!ownedId.ok) return; + assert.deepEqual(ownedId.page.query._id, { $gt: 'doc1' }); + assert.equal(ownedId.page.query.published, true); + }); +}); + +describe('backfill counters', () => { + it('tracks scanned, queued, processed, and failed counts', () => { + const paged = applyBackfillPage(runningRun(), [{ _id: 'a' }, { _id: 'b' }], 2); + assert.equal(paged.ok, true); + if (!paged.ok) return; + const progressed = applyBackfillJobCounts(paged.run, { processed: 1, failed: 1 }); + assert.equal(progressed.ok, true); + if (!progressed.ok) return; + assert.equal(progressed.run.scannedCount, 2); + assert.equal(progressed.run.queuedCount, 2); + assert.equal(progressed.run.processedCount, 1); + assert.equal(progressed.run.failedCount, 1); + }); + + it('rejects job counts that exceed queued work or run while not running', () => { + const paged = applyBackfillPage(runningRun(), [{ _id: 'a' }]); + assert.equal(paged.ok, true); + if (!paged.ok) return; + const overflow = applyBackfillJobCounts(paged.run, { processed: 2 }); + assert.equal(overflow.ok, false); + const negative = applyBackfillJobCounts(paged.run, { failed: -1 }); + assert.equal(negative.ok, false); + const queuedCounts = applyBackfillJobCounts(queuedRun(), { processed: 1 }); + assert.equal(queuedCounts.ok, false); + }); +}); + +describe('backfill cancellation and resume', () => { + it('cancels queued and running runs, then allows resume from canceled or failed', () => { + assert.equal(canCancelBackfill('queued'), true); + assert.equal(canCancelBackfill('running'), true); + assert.equal(canCancelBackfill('completed'), false); + assert.equal(isResumeEligible('canceled'), true); + assert.equal(isResumeEligible('failed'), true); + assert.equal(isResumeEligible('completed'), false); + assert.equal(isResumeEligible('running'), false); + + const canceled = cancelBackfillRun(runningRun(), now); + assert.equal(canceled.ok, true); + if (!canceled.ok) return; + assert.equal(canceled.run.state, 'canceled'); + assert.equal(canceled.run.finishedAt?.toISOString(), now.toISOString()); + + const resumed = resumeBackfillRun(canceled.run); + assert.equal(resumed.ok, true); + if (!resumed.ok) return; + assert.equal(resumed.run.state, 'queued'); + assert.equal(resumed.run.finishedAt, null); + assert.equal(resumed.run.error, null); + assert.equal(resumed.run.cursor, canceled.run.cursor); + assert.equal(resumed.run.scannedCount, canceled.run.scannedCount); + + const failed = failBackfillRun(runningRun(), 'provider timeout', now); + assert.equal(failed.ok, true); + if (!failed.ok) return; + const resumedFailed = resumeBackfillRun(failed.run); + assert.equal(resumedFailed.ok, true); + if (!resumedFailed.ok) return; + assert.equal(resumedFailed.run.state, 'queued'); + assert.equal(resumedFailed.run.error, null); + }); + + it('preserves cursor across cancel and resume so paging can continue', () => { + const paged = applyBackfillPage(runningRun(), [{ _id: 'a' }, { _id: 'b' }]); + assert.equal(paged.ok, true); + if (!paged.ok) return; + const canceled = cancelBackfillRun(paged.run, now); + assert.equal(canceled.ok, true); + if (!canceled.ok) return; + const resumed = resumeBackfillRun(canceled.run); + assert.equal(resumed.ok, true); + if (!resumed.ok) return; + const restarted = startBackfillRun(resumed.run, now); + assert.equal(restarted.ok, true); + if (!restarted.ok) return; + assert.equal(restarted.run.cursor, 'b'); + assert.equal(restarted.run.startedAt?.toISOString(), now.toISOString()); + }); +}); + +describe('backfill bounds', () => { + it('clamps batch size into the configured inclusive range', () => { + assert.deepEqual(boundBackfillBatchSize(undefined), { + ok: true, + batchSize: DEFAULT_BACKFILL_BATCH_SIZE, + }); + assert.deepEqual(boundBackfillBatchSize(0), { + ok: true, + batchSize: MIN_BACKFILL_BATCH_SIZE, + }); + assert.deepEqual(boundBackfillBatchSize(10_000), { + ok: true, + batchSize: MAX_BACKFILL_BATCH_SIZE, + }); + assert.deepEqual(boundBackfillBatchSize(50, 25), { ok: true, batchSize: 25 }); + assert.equal(boundBackfillBatchSize(1.5).ok, false); + assert.equal(boundBackfillBatchSize(Number.NaN).ok, false); + assert.equal(boundBackfillBatchSize(10, 0).ok, false); + }); + + it('bounds pages to batch size and rejects oversized apply calls', () => { + const docs = [{ _id: 'a' }, { _id: 'b' }, { _id: 'c' }]; + assert.deepEqual(boundBackfillPage(docs, 2), [{ _id: 'a' }, { _id: 'b' }]); + const oversized = applyBackfillPage(runningRun(), docs); + assert.equal(oversized.ok, false); + if (oversized.ok) return; + assert.equal(oversized.reason, 'page_size'); + }); + + it('rejects oversized filters, dangerous operators, and invalid identity', () => { + assert.equal(createQueuedBackfill({ schemaName: '../etc' }).ok, false); + assert.equal( + createQueuedBackfill({ schemaName: 'Article', configId: 'bad id' }).ok, + false, + ); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { $where: 'this.password' }, + }).ok, + false, + ); + assert.equal(createQueuedBackfill({ schemaName: 'Article', filter: [] }).ok, false); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { body: 'x'.repeat(MAX_BACKFILL_FILTER_BYTES) }, + }).ok, + false, + ); + }); +}); + +describe('backfill sanitized errors', () => { + it('redacts secrets and truncates stored failure text', () => { + const failed = failBackfillRun( + runningRun(), + new Error('provider failed apiKey=sk-secret Bearer tok-live'), + now, + ); + assert.equal(failed.ok, true); + if (!failed.ok) return; + assert.equal(failed.run.state, 'failed'); + assert.match(failed.run.error ?? '', /\[REDACTED\]/); + assert.doesNotMatch(failed.run.error ?? '', /sk-secret|tok-live/); + + const long = sanitizeBackfillError('e'.repeat(MAX_BACKFILL_ERROR_LENGTH + 50)); + assert.equal(long.length, MAX_BACKFILL_ERROR_LENGTH); + }); +}); diff --git a/modules/embeddings/src/utils/backfillRun.ts b/modules/embeddings/src/utils/backfillRun.ts new file mode 100644 index 000000000..9dc765db2 --- /dev/null +++ b/modules/embeddings/src/utils/backfillRun.ts @@ -0,0 +1,391 @@ +import { sanitizeErrorMessage } from './redactConfig.js'; +import { MAX_QUEUE_BATCH_SIZE } from './embeddingJobs.js'; + +export const BACKFILL_RUN_SCHEMA = 'BackfillRun'; + +export const BACKFILL_RUN_STATES = [ + 'queued', + 'running', + 'completed', + 'failed', + 'canceled', +] as const; + +export type BackfillRunState = (typeof BACKFILL_RUN_STATES)[number]; + +export const LEGAL_BACKFILL_TRANSITIONS: Record< + BackfillRunState, + readonly BackfillRunState[] +> = { + queued: ['running', 'canceled'], + running: ['completed', 'failed', 'canceled'], + completed: [], + failed: ['queued'], + canceled: ['queued'], +}; + +export const MIN_BACKFILL_BATCH_SIZE = 1; +export const DEFAULT_BACKFILL_BATCH_SIZE = 100; +export const MAX_BACKFILL_BATCH_SIZE = MAX_QUEUE_BATCH_SIZE; +export const MAX_BACKFILL_ERROR_LENGTH = 1024; +export const MAX_BACKFILL_FILTER_BYTES = 4 * 1024; + +const SCHEMA_NAME = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; +const IDENTITY = /^[A-Za-z0-9._-]{1,128}$/; +const DANGEROUS_FILTER_KEY = /^(\$where|\$function|\$accumulator|__proto__|constructor)$/; + +export interface BackfillRunProgress { + state: BackfillRunState; + schemaName: string; + configId?: string; + cursor?: string | null; + batchSize: number; + onlyMissing: boolean; + filter?: Record | null; + scannedCount: number; + queuedCount: number; + processedCount: number; + failedCount: number; + startedAt?: Date | null; + finishedAt?: Date | null; + error?: string | null; +} + +export interface CreateBackfillRunInput { + schemaName: string; + configId?: string; + batchSize?: number; + onlyMissing?: boolean; + filter?: unknown; + maxBatchSize?: number; +} + +export type BackfillRunResult = + { ok: true; run: BackfillRunProgress } | { ok: false; reason: string }; + +export interface BackfillPageQuery { + query: Record; + sort: { _id: 1 }; + limit: number; +} + +function unexpectedState(state: never): never { + throw new Error(`Unhandled backfill state: ${String(state)}`); +} + +export function isBackfillRunState(value: unknown): value is BackfillRunState { + return ( + typeof value === 'string' && + (BACKFILL_RUN_STATES as readonly string[]).includes(value) + ); +} + +export function isLegalBackfillTransition( + from: BackfillRunState, + to: BackfillRunState, +): boolean { + return LEGAL_BACKFILL_TRANSITIONS[from].includes(to); +} + +export function canCancelBackfill(state: BackfillRunState): boolean { + switch (state) { + case 'queued': + case 'running': + return true; + case 'completed': + case 'failed': + case 'canceled': + return false; + default: + return unexpectedState(state); + } +} + +export function isResumeEligible(state: BackfillRunState): boolean { + switch (state) { + case 'failed': + case 'canceled': + return true; + case 'queued': + case 'running': + case 'completed': + return false; + default: + return unexpectedState(state); + } +} + +export function boundBackfillBatchSize( + requested: number | undefined, + maxBatchSize: number = MAX_BACKFILL_BATCH_SIZE, +): { ok: true; batchSize: number } | { ok: false; reason: 'batch_size' } { + if (!Number.isInteger(maxBatchSize) || maxBatchSize < MIN_BACKFILL_BATCH_SIZE) { + return { ok: false, reason: 'batch_size' }; + } + const cappedMax = Math.min(maxBatchSize, MAX_BACKFILL_BATCH_SIZE); + const value = requested ?? DEFAULT_BACKFILL_BATCH_SIZE; + if (typeof value !== 'number' || !Number.isInteger(value) || !Number.isFinite(value)) { + return { ok: false, reason: 'batch_size' }; + } + if (value < MIN_BACKFILL_BATCH_SIZE) { + return { ok: true, batchSize: MIN_BACKFILL_BATCH_SIZE }; + } + return { ok: true, batchSize: Math.min(value, cappedMax) }; +} + +export function boundBackfillPage(docs: readonly T[], batchSize: number): T[] { + if (!Number.isInteger(batchSize) || batchSize < MIN_BACKFILL_BATCH_SIZE) return []; + return docs.slice(0, Math.min(batchSize, MAX_BACKFILL_BATCH_SIZE)); +} + +export function isBackfillPageExhausted(pageLength: number, batchSize: number): boolean { + return pageLength < batchSize; +} + +export function sanitizeBackfillError(err: unknown): string { + return sanitizeErrorMessage(err).slice(0, MAX_BACKFILL_ERROR_LENGTH); +} + +export function createQueuedBackfill(input: CreateBackfillRunInput): BackfillRunResult { + if (typeof input.schemaName !== 'string' || !SCHEMA_NAME.test(input.schemaName)) { + return { ok: false, reason: 'schemaName' }; + } + if ( + input.configId !== undefined && + (typeof input.configId !== 'string' || !IDENTITY.test(input.configId)) + ) { + return { ok: false, reason: 'configId' }; + } + const bounded = boundBackfillBatchSize(input.batchSize, input.maxBatchSize); + if (!bounded.ok) return bounded; + const filter = normalizeFilter(input.filter); + if (!filter.ok) return filter; + return { + ok: true, + run: { + state: 'queued', + schemaName: input.schemaName, + ...(input.configId ? { configId: input.configId } : {}), + cursor: null, + batchSize: bounded.batchSize, + onlyMissing: input.onlyMissing === true, + filter: filter.filter, + scannedCount: 0, + queuedCount: 0, + processedCount: 0, + failedCount: 0, + startedAt: null, + finishedAt: null, + error: null, + }, + }; +} + +export function startBackfillRun( + run: BackfillRunProgress, + now: Date = new Date(), +): BackfillRunResult { + return transition(run, 'running', { + startedAt: run.startedAt ?? now, + finishedAt: null, + error: null, + }); +} + +export function cancelBackfillRun( + run: BackfillRunProgress, + now: Date = new Date(), +): BackfillRunResult { + if (!canCancelBackfill(run.state)) { + return { ok: false, reason: 'illegal_transition' }; + } + return transition(run, 'canceled', { + finishedAt: now, + }); +} + +export function failBackfillRun( + run: BackfillRunProgress, + err: unknown, + now: Date = new Date(), +): BackfillRunResult { + return transition(run, 'failed', { + finishedAt: now, + error: sanitizeBackfillError(err), + }); +} + +export function completeBackfillRun( + run: BackfillRunProgress, + now: Date = new Date(), +): BackfillRunResult { + return transition(run, 'completed', { + finishedAt: now, + error: null, + }); +} + +export function resumeBackfillRun(run: BackfillRunProgress): BackfillRunResult { + if (!isResumeEligible(run.state)) { + return { ok: false, reason: 'illegal_transition' }; + } + return transition(run, 'queued', { + finishedAt: null, + error: null, + }); +} + +export function applyBackfillPage( + run: BackfillRunProgress, + docs: ReadonlyArray<{ _id?: unknown }>, + queuedDelta: number = docs.length, +): BackfillRunResult & { exhausted?: boolean } { + if (run.state !== 'running') { + return { ok: false, reason: 'not_running' }; + } + if (docs.length > run.batchSize) { + return { ok: false, reason: 'page_size' }; + } + if (!Number.isInteger(queuedDelta) || queuedDelta < 0 || queuedDelta > docs.length) { + return { ok: false, reason: 'queued' }; + } + const exhausted = isBackfillPageExhausted(docs.length, run.batchSize); + if (docs.length === 0) { + return { ok: true, run, exhausted }; + } + const ids: string[] = []; + for (const doc of docs) { + if (typeof doc._id !== 'string' || !IDENTITY.test(doc._id)) { + return { ok: false, reason: 'cursor' }; + } + ids.push(doc._id); + } + const cursor = ids[ids.length - 1]; + if (run.cursor && cursor === run.cursor) { + return { ok: false, reason: 'cursor' }; + } + return { + ok: true, + exhausted, + run: { + ...run, + cursor, + scannedCount: run.scannedCount + docs.length, + queuedCount: run.queuedCount + queuedDelta, + }, + }; +} + +export function applyBackfillJobCounts( + run: BackfillRunProgress, + counts: { processed?: number; failed?: number }, +): BackfillRunResult { + if (run.state !== 'running') { + return { ok: false, reason: 'not_running' }; + } + const processedDelta = counts.processed ?? 0; + const failedDelta = counts.failed ?? 0; + if ( + !Number.isInteger(processedDelta) || + processedDelta < 0 || + !Number.isInteger(failedDelta) || + failedDelta < 0 + ) { + return { ok: false, reason: 'counts' }; + } + const processedCount = run.processedCount + processedDelta; + const failedCount = run.failedCount + failedDelta; + if (processedCount + failedCount > run.queuedCount) { + return { ok: false, reason: 'counts' }; + } + return { + ok: true, + run: { + ...run, + processedCount, + failedCount, + }, + }; +} + +export function buildBackfillPageQuery( + run: Pick, + targetField?: string, +): { ok: true; page: BackfillPageQuery } | { ok: false; reason: string } { + if (run.onlyMissing) { + if (typeof targetField !== 'string' || !SCHEMA_NAME.test(targetField)) { + return { ok: false, reason: 'targetField' }; + } + } + const query: Record = { ...(run.filter ?? {}) }; + delete query._id; + if (run.onlyMissing && targetField) { + query[targetField] = null; + } + if (run.cursor) { + if (!IDENTITY.test(run.cursor)) { + return { ok: false, reason: 'cursor' }; + } + query._id = { $gt: run.cursor }; + } + return { + ok: true, + page: { + query, + sort: { _id: 1 }, + limit: run.batchSize, + }, + }; +} + +function transition( + run: BackfillRunProgress, + to: BackfillRunState, + patch: Partial, +): BackfillRunResult { + if (!isLegalBackfillTransition(run.state, to)) { + return { ok: false, reason: 'illegal_transition' }; + } + return { + ok: true, + run: { + ...run, + ...patch, + state: to, + }, + }; +} + +function normalizeFilter( + filter: unknown, +): { ok: true; filter: Record | null } | { ok: false; reason: string } { + if (filter == null) return { ok: true, filter: null }; + if (typeof filter !== 'object' || Array.isArray(filter)) { + return { ok: false, reason: 'filter' }; + } + let serialized: string; + try { + serialized = JSON.stringify(filter); + } catch { + return { ok: false, reason: 'filter' }; + } + if (serialized.length > MAX_BACKFILL_FILTER_BYTES) { + return { ok: false, reason: 'filter' }; + } + if (hasDangerousFilterKey(filter)) { + return { ok: false, reason: 'filter' }; + } + return { ok: true, filter: JSON.parse(serialized) as Record }; +} + +function hasDangerousFilterKey(value: unknown): boolean { + if (value == null || typeof value !== 'object') return false; + if (Array.isArray(value)) { + return value.some(hasDangerousFilterKey); + } + for (const [key, nested] of Object.entries(value as Record)) { + if (DANGEROUS_FILTER_KEY.test(key)) return true; + if (hasDangerousFilterKey(nested)) return true; + } + return false; +} diff --git a/modules/embeddings/src/utils/schemaPolicy.test.ts b/modules/embeddings/src/utils/schemaPolicy.test.ts index f4070eb23..de9b83a64 100644 --- a/modules/embeddings/src/utils/schemaPolicy.test.ts +++ b/modules/embeddings/src/utils/schemaPolicy.test.ts @@ -12,8 +12,9 @@ import { } from './schemaPolicy.js'; describe('embedding schema and source policies', () => { - it('denies system, auth-secret, and EmbeddingConfig schemas', () => { + it('denies system, auth-secret, and embeddings-owned schemas', () => { assert.equal(isDeniedEmbeddingSchema({ name: 'EmbeddingConfig' }), true); + assert.equal(isDeniedEmbeddingSchema({ name: 'BackfillRun' }), true); assert.equal(isDeniedEmbeddingSchema({ name: '_DeclaredSchema' }), true); assert.equal(isDeniedEmbeddingSchema({ name: 'Views' }), true); assert.equal(isDeniedEmbeddingSchema({ name: 'AccessToken' }), true); diff --git a/modules/embeddings/src/utils/schemaPolicy.ts b/modules/embeddings/src/utils/schemaPolicy.ts index 2988180e5..636e525ed 100644 --- a/modules/embeddings/src/utils/schemaPolicy.ts +++ b/modules/embeddings/src/utils/schemaPolicy.ts @@ -1,8 +1,14 @@ import { TYPE } from '@conduitplatform/grpc-sdk'; import { GrpcError } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; +import { BACKFILL_RUN_SCHEMA } from './backfillRun.js'; export const EMBEDDING_CONFIG_SCHEMA = 'EmbeddingConfig'; +export { BACKFILL_RUN_SCHEMA }; +export const EMBEDDING_OWNED_SCHEMA_NAMES = new Set([ + EMBEDDING_CONFIG_SCHEMA, + BACKFILL_RUN_SCHEMA, +]); export const CONFIG_OPERATOR_MODULES = ['database', 'core'] as const; export const SEARCH_OPERATOR_MODULES = ['database', 'core', 'embeddings'] as const; @@ -53,7 +59,7 @@ export function isDeniedEmbeddingSchema(schema: { ownerModule?: string; }): boolean { if (!schema.name) return true; - if (schema.name === EMBEDDING_CONFIG_SCHEMA) return true; + if (EMBEDDING_OWNED_SCHEMA_NAMES.has(schema.name)) return true; if (schema.name.startsWith('_')) return true; if (SYSTEM_SCHEMA_NAMES.has(schema.name)) return true; return AUTH_SECRET_SCHEMA_NAMES.has(schema.name); From 3d9324138ab64098427351578315f20f7e488548 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 19:43:21 +0300 Subject: [PATCH 08/29] feat(embeddings): queue cursor-based backfill with observability Persist a queued BackfillRun and return after enqueueing a controller job, then scan bounded pages through continuation jobs with gated execution, accurate counters, and unlabeled queue metrics. --- modules/embeddings/src/Embeddings.ts | 164 +++++-- .../src/controllers/queue.controller.test.ts | 131 +++++- .../src/controllers/queue.controller.ts | 277 ++++++++++-- modules/embeddings/src/metrics/index.ts | 21 + .../src/utils/backfillExecution.test.ts | 388 ++++++++++++++++ .../embeddings/src/utils/backfillExecution.ts | 427 ++++++++++++++++++ .../src/utils/backfillGates.test.ts | 192 ++++++++ modules/embeddings/src/utils/backfillGates.ts | 147 ++++++ .../src/utils/embeddingJobs.test.ts | 9 + modules/embeddings/src/utils/embeddingJobs.ts | 10 +- .../embeddings/src/utils/embeddingMetrics.ts | 21 + 11 files changed, 1699 insertions(+), 88 deletions(-) create mode 100644 modules/embeddings/src/utils/backfillExecution.test.ts create mode 100644 modules/embeddings/src/utils/backfillExecution.ts create mode 100644 modules/embeddings/src/utils/backfillGates.test.ts create mode 100644 modules/embeddings/src/utils/backfillGates.ts create mode 100644 modules/embeddings/src/utils/embeddingMetrics.ts diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts index 4ff289150..7e59b9325 100644 --- a/modules/embeddings/src/Embeddings.ts +++ b/modules/embeddings/src/Embeddings.ts @@ -17,7 +17,7 @@ import { import { status } from '@grpc/grpc-js'; import AppConfigSchema, { Config } from './config/index.js'; import * as models from './models/index.js'; -import { EmbeddingConfig } from './models/index.js'; +import { BackfillRun, EmbeddingConfig } from './models/index.js'; import { QueueController } from './controllers/queue.controller.js'; import { getProvider, hashEmbeddingInput } from './providers/index.js'; import { validateEmbeddingConfigInput } from './utils/validateEmbeddingConfig.js'; @@ -30,7 +30,11 @@ import { buildEmbeddingDocumentSelect, generateEmbeddingsForDocument, } from './utils/processEmbedding.js'; -import { MAX_QUEUE_BATCH_SIZE, parseEmbeddingJobData } from './utils/embeddingJobs.js'; +import { + MAX_QUEUE_BATCH_SIZE, + parseEmbeddingJobData, + type EmbeddingJobData, +} from './utils/embeddingJobs.js'; import { assertCanManageEmbeddingConfig, assertEmbeddingTargetSchema, @@ -42,6 +46,16 @@ import { callerModuleName, } from './utils/productionSecurity.js'; import { sanitizeErrorMessage } from './utils/redactConfig.js'; +import { + applyBackfillJobOutcome, + backfillRunFromDocument, + persistableBackfillRun, + processBackfillControllerJob, + queueBackfillRuns, + type BackfillControllerJobData, +} from './utils/backfillExecution.js'; +import { BackfillGateError, grpcErrorFromBackfillGate } from './utils/backfillGates.js'; +import { incrementEmbeddingMetric } from './utils/embeddingMetrics.js'; import metricsSchema from './metrics/index.js'; import { BackfillRequest, @@ -188,37 +202,35 @@ export default class EmbeddingsModule extends ManagedModule { ownerModule: declared?.ownerModule, schemaName: schema.name, }); - const configs = await EmbeddingConfig.getInstance().findMany({ - schemaName: call.request.schemaName, - enabled: true, - }); - if (!configs.length) { - throw new GrpcError( - status.FAILED_PRECONDITION, - 'No enabled embedding config found for backfill', - ); - } - const maxBatch = this.currentConfig().queue.maxBatchSize ?? MAX_QUEUE_BATCH_SIZE; - const batchSize = Math.min(Math.max(call.request.batchSize ?? 100, 1), maxBatch); - const docs = await this.database.findMany>( - call.request.schemaName, - {}, - { limit: batchSize, select: '_id' }, - ); - const attempts = this.currentConfig().queue.attempts; - await this.queueController.addBulkEmbeddingJobs( - docs.flatMap(doc => - configs.map(config => ({ - schemaName: config.schemaName, - documentId: String(doc._id), - configId: config._id, - })), - ), - attempts, + const [configs, capabilities, indexes] = await Promise.all([ + EmbeddingConfig.getInstance().findMany({ + schemaName: call.request.schemaName, + enabled: true, + }), + this.database.getVectorCapabilities(call.request.schemaName), + this.database.getVectorIndexes(call.request.schemaName), + ]); + const queued = await queueBackfillRuns( + { + schemaName: call.request.schemaName, + batchSize: call.request.batchSize, + maxBatchSize: this.currentConfig().queue.maxBatchSize ?? MAX_QUEUE_BATCH_SIZE, + }, + { + moduleEnabled: this.currentConfig().enabled, + capabilities, + configs, + indexes, + createRun: async run => { + const created = await BackfillRun.getInstance().create( + persistableBackfillRun(run), + ); + return { _id: created._id }; + }, + enqueueController: job => this.queueController.addBackfillControllerJob(job), + }, ); - callback(null, { - result: JSON.stringify({ queued: docs.length * configs.length }), - }); + callback(null, { result: JSON.stringify(queued) }); } catch (err) { callback(this.grpcError(err)); } @@ -280,10 +292,17 @@ export default class EmbeddingsModule extends ManagedModule { this.unsubscribeAll(); return; } + this.queueController.setBackfillJobOutcomeHandler((runId, outcome) => + this.recordBackfillJobOutcome(runId, outcome), + ); await this.queueController.ensureWorker( - data => this.processEmbeddingJob(data.schemaName, data.documentId, data.configId), + data => this.processEmbeddingJob(data), config.queue.concurrency, ); + await this.queueController.ensureBackfillWorker( + data => this.processBackfillJob(data), + 1, + ); const configs = await EmbeddingConfig.getInstance().findMany({ enabled: true }); const enabledSchemas = new Set(configs.map(item => item.schemaName)); for (const schemaName of this.subscribedSchemas.keys()) { @@ -361,7 +380,7 @@ export default class EmbeddingsModule extends ManagedModule { this.currentConfig().security.maxMutationEventIds, ); if (!parsed.ok) { - ConduitGrpcSdk.Metrics?.increment('malformed_embedding_events_total'); + incrementEmbeddingMetric('malformedEvents'); return; } if (!parsed.event.ids.length) return; @@ -380,14 +399,40 @@ export default class EmbeddingsModule extends ManagedModule { ); } - private async processEmbeddingJob( - schemaName: string, - documentId: string, - configId?: string, - ) { - const parsed = parseEmbeddingJobData({ schemaName, documentId, configId }); + private async processBackfillJob(data: BackfillControllerJobData) { + await processBackfillControllerJob(data, { + maxBatchSize: this.currentConfig().queue.maxBatchSize ?? MAX_QUEUE_BATCH_SIZE, + moduleEnabled: this.currentConfig().enabled, + getRun: async id => { + const doc = await BackfillRun.getInstance().findOne({ _id: id }); + return doc ? backfillRunFromDocument(doc) : null; + }, + saveRun: (id, run) => + BackfillRun.getInstance() + .findByIdAndUpdate(id, persistableBackfillRun(run)) + .then(() => undefined), + findPage: (schemaName, page) => + this.database.findMany<{ _id?: unknown }>(schemaName, page.query, { + sort: page.sort, + limit: page.limit, + select: '_id', + }), + enqueueEmbeddingJobs: jobs => + this.queueController.addBulkEmbeddingJobs( + jobs, + this.currentConfig().queue.attempts, + ), + enqueueContinuation: job => this.queueController.addBackfillControllerJob(job), + getCapabilities: schemaName => this.database.getVectorCapabilities(schemaName), + getConfig: id => EmbeddingConfig.getInstance().findOne({ _id: id }), + getIndexes: schemaName => this.database.getVectorIndexes(schemaName), + }); + } + + private async processEmbeddingJob(data: EmbeddingJobData) { + const parsed = parseEmbeddingJobData(data); if (!parsed.ok) { - ConduitGrpcSdk.Metrics?.increment('malformed_embedding_jobs_total'); + incrementEmbeddingMetric('malformedJobs'); return; } const configs = ( @@ -401,7 +446,10 @@ export default class EmbeddingsModule extends ManagedModule { const matching = configs.filter( config => config.enabled && config.schemaName === parsed.data.schemaName, ); - if (!matching.length) return; + if (!matching.length) { + await this.recordBackfillJobOutcome(parsed.data.backfillRunId, 'processed'); + return; + } const allowedFields = [ ...new Set( matching.flatMap(config => [ @@ -419,8 +467,11 @@ export default class EmbeddingsModule extends ManagedModule { embeddingsAllowedFields: allowedFields, }, ); - if (!doc) return; - await generateEmbeddingsForDocument({ + if (!doc) { + await this.recordBackfillJobOutcome(parsed.data.backfillRunId, 'processed'); + return; + } + const result = await generateEmbeddingsForDocument({ doc, configs: matching, hashInput: hashEmbeddingInput, @@ -440,6 +491,27 @@ export default class EmbeddingsModule extends ManagedModule { }, ), }); + incrementEmbeddingMetric('generated', result.generated); + incrementEmbeddingMetric('skipped', result.skipped); + await this.recordBackfillJobOutcome(parsed.data.backfillRunId, 'processed'); + } + + private async recordBackfillJobOutcome( + runId: string | undefined, + outcome: 'processed' | 'failed', + ) { + if (!runId) return; + const doc = await BackfillRun.getInstance().findOne({ _id: runId }); + if (!doc) return; + const run = backfillRunFromDocument(doc); + await applyBackfillJobOutcome({ + run, + outcome, + saveRun: (id, next) => + BackfillRun.getInstance() + .findByIdAndUpdate(id, persistableBackfillRun(next)) + .then(() => undefined), + }); } private async declaredSchema(schemaName: string) { @@ -451,6 +523,10 @@ export default class EmbeddingsModule extends ManagedModule { } private grpcError(err: unknown) { + if (err instanceof BackfillGateError) { + const mapped = grpcErrorFromBackfillGate(err); + return { code: mapped.code, message: sanitizeErrorMessage(mapped) }; + } if (err instanceof GrpcError) { return { code: err.code, message: sanitizeErrorMessage(err) }; } diff --git a/modules/embeddings/src/controllers/queue.controller.test.ts b/modules/embeddings/src/controllers/queue.controller.test.ts index 94b728311..4a83fb977 100644 --- a/modules/embeddings/src/controllers/queue.controller.test.ts +++ b/modules/embeddings/src/controllers/queue.controller.test.ts @@ -2,21 +2,22 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import { QueueController } from './queue.controller.js'; -import { EmbeddingJobData, embeddingJobId } from '../utils/embeddingJobs.js'; +import { embeddingJobId } from '../utils/embeddingJobs.js'; +import { EMBEDDING_METRICS } from '../utils/embeddingMetrics.js'; type StoredJob = { name: string; - data: EmbeddingJobData; - opts?: { jobId?: string }; + data: Record; + opts?: { jobId?: string; delay?: number; attempts?: number }; }; class FakeQueue { jobs: StoredJob[] = []; closed = false; - async add(name: string, data: EmbeddingJobData, opts?: { jobId?: string }) { - if (this.jobs.some(job => job.opts?.jobId === opts?.jobId)) { - throw new Error(`Job ${opts?.jobId} already exists`); + async add(name: string, data: Record, opts?: StoredJob['opts']) { + if (opts?.jobId && this.jobs.some(job => job.opts?.jobId === opts.jobId)) { + throw new Error(`Job ${opts.jobId} already exists`); } this.jobs.push({ name, data, opts }); } @@ -27,6 +28,17 @@ class FakeQueue { } } + async getJobCounts() { + return { + waiting: this.jobs.length, + active: 0, + completed: 0, + failed: 0, + delayed: this.jobs.filter(job => (job.opts?.delay ?? 0) > 0).length, + paused: 0, + }; + } + async close() { this.closed = true; } @@ -36,17 +48,21 @@ class FakeWorker { static instances: FakeWorker[] = []; closed = false; concurrency: number; + name: string; + handlers: Record void> = {}; constructor( - _name: string, - _processor: (job: { data: EmbeddingJobData }) => Promise, + name: string, + _processor: (job: { data: unknown }) => Promise, opts: { concurrency: number }, ) { + this.name = name; this.concurrency = opts.concurrency; FakeWorker.instances.push(this); } - on() { + on(event: string, handler: (...args: unknown[]) => void) { + this.handlers[event] = handler; return this; } @@ -55,13 +71,17 @@ class FakeWorker { } } -function createController(queue: FakeQueue = new FakeQueue()) { +function createController( + queue: FakeQueue = new FakeQueue(), + backfillQueue: FakeQueue = new FakeQueue(), +) { return { queue, + backfillQueue, controller: new QueueController(fakeSdk(), { Queue: class { - constructor() { - return queue; + constructor(name: string) { + return name.includes('backfill') ? backfillQueue : queue; } } as never, Worker: FakeWorker as never, @@ -77,6 +97,22 @@ function fakeSdk() { } as unknown as ConduitGrpcSdk; } +function withMetrics() { + const seen: Array<{ name: string; amount?: number; labels?: unknown }> = []; + const previous = ConduitGrpcSdk.Metrics; + ConduitGrpcSdk.Metrics = { + increment(name: string, amount?: number, labels?: unknown) { + seen.push({ name, amount, labels }); + }, + } as never; + return { + seen, + restore() { + ConduitGrpcSdk.Metrics = previous; + }, + }; +} + describe('embedding queue worker lifecycle', () => { it('keeps a single worker, recreates on concurrency change, and closes idempotently', async () => { FakeWorker.instances = []; @@ -100,6 +136,20 @@ describe('embedding queue worker lifecycle', () => { assert.equal(controller.hasWorker, false); }); + it('does not recreate the backfill worker when generation concurrency changes', async () => { + FakeWorker.instances = []; + const { controller } = createController(); + await controller.ensureWorker(async () => undefined, 2); + await controller.ensureBackfillWorker(async () => undefined, 1); + assert.equal(controller.hasBackfillWorker, true); + await controller.ensureWorker(async () => undefined, 3); + const backfill = FakeWorker.instances.find( + worker => worker.name === 'embeddings-backfill-queue', + ); + assert.equal(backfill?.closed, false); + assert.equal(controller.hasBackfillWorker, true); + }); + it('deduplicates queued jobs by identity', async () => { FakeWorker.instances = []; const { queue, controller } = createController(); @@ -136,3 +186,60 @@ describe('embedding queue worker lifecycle', () => { ); }); }); + +describe('embedding queue status and backfill jobs', () => { + it('reports generation and backfill counts separately', async () => { + const { controller, queue, backfillQueue } = createController(); + await controller.addEmbeddingJob({ schemaName: 'Article', documentId: 'a' }, 3); + await controller.addBackfillControllerJob({ runId: 'run1', cursor: null }); + const status = await controller.getQueueStatus(); + assert.equal(status.generation.waiting, queue.jobs.length); + assert.equal(status.backfill.waiting, backfillQueue.jobs.length); + assert.equal((await controller.getJobCounts('generation')).waiting, 1); + assert.equal((await controller.getJobCounts('backfill')).waiting, 1); + }); + + it('enqueues lightweight backfill controller jobs with cursor identity', async () => { + const { backfillQueue, controller } = createController(); + await controller.addBackfillControllerJob({ runId: 'run1', cursor: null }); + await controller.addBackfillControllerJob({ runId: 'run1', cursor: null }); + await controller.addBackfillControllerJob({ runId: 'run1', cursor: 'b' }); + await controller.addBackfillControllerJob({ runId: 'run1', drain: true }); + assert.deepEqual( + backfillQueue.jobs.map(job => job.opts?.jobId), + ['backfill:run1:start', 'backfill:run1:b', undefined], + ); + assert.equal( + backfillQueue.jobs.some(job => job.opts?.delay === 1000), + true, + ); + }); + + it('increments retried then failed metrics without job payload labels', async () => { + FakeWorker.instances = []; + const metrics = withMetrics(); + try { + const { controller } = createController(); + await controller.ensureWorker(async () => undefined, 1); + const worker = FakeWorker.instances[0]; + const job = { + data: { schemaName: 'Article', documentId: 'a', backfillRunId: 'run1' }, + attemptsMade: 1, + opts: { attempts: 3 }, + }; + worker.handlers.failed?.(job, new Error('provider timeout apiKey=sk-secret')); + job.attemptsMade = 3; + worker.handlers.failed?.(job, new Error('provider timeout')); + assert.deepEqual( + metrics.seen.map(item => item.name), + [EMBEDDING_METRICS.retried, EMBEDDING_METRICS.failed], + ); + assert.equal( + metrics.seen.every(item => item.labels === undefined), + true, + ); + } finally { + metrics.restore(); + } + }); +}); diff --git a/modules/embeddings/src/controllers/queue.controller.ts b/modules/embeddings/src/controllers/queue.controller.ts index 4799bbefe..6507e49f8 100644 --- a/modules/embeddings/src/controllers/queue.controller.ts +++ b/modules/embeddings/src/controllers/queue.controller.ts @@ -8,21 +8,54 @@ import { isDuplicateJobError, parseEmbeddingJobData, } from '../utils/embeddingJobs.js'; +import { + BackfillControllerJobData, + parseBackfillControllerJob, + BACKFILL_DRAIN_DELAY_MS, +} from '../utils/backfillExecution.js'; +import { incrementEmbeddingMetric } from '../utils/embeddingMetrics.js'; +import { sanitizeErrorMessage } from '../utils/redactConfig.js'; export type { EmbeddingJobData } from '../utils/embeddingJobs.js'; +export type { BackfillControllerJobData } from '../utils/backfillExecution.js'; type RedisConnection = Redis | Cluster; +export interface QueueJobCounts { + waiting: number; + active: number; + completed: number; + failed: number; + delayed: number; + paused: number; +} + +export interface EmbeddingQueueStatus { + generation: QueueJobCounts; + backfill: QueueJobCounts; +} + type QueueLike = { add: ( name: string, - data: EmbeddingJobData, + data: Record, opts?: Record, ) => Promise; addBulk: ( - jobs: Array<{ name: string; data: EmbeddingJobData; opts?: Record }>, + jobs: Array<{ + name: string; + data: Record; + opts?: Record; + }>, ) => Promise; close: () => Promise; + getJobCounts: () => Promise & Record>; +}; + +type WorkerJob = { + data?: unknown; + attemptsMade?: number; + opts?: { attempts?: number }; }; type WorkerLike = { @@ -35,11 +68,20 @@ export interface QueueControllerDependencies { Queue?: new (name: string, opts: { connection: RedisConnection }) => QueueLike; Worker?: new ( name: string, - processor: (job: { data: EmbeddingJobData }) => Promise, + processor: (job: WorkerJob) => Promise, opts: { connection: RedisConnection; concurrency: number } & Record, ) => WorkerLike; } +const EMPTY_COUNTS: QueueJobCounts = { + waiting: 0, + active: 0, + completed: 0, + failed: 0, + delayed: 0, + paused: 0, +}; + export class QueueController { private static _instance: QueueController; private readonly createConnection: () => RedisConnection; @@ -47,10 +89,19 @@ export class QueueController { private readonly WorkerImpl: NonNullable; private readonly queueConnection: RedisConnection; private readonly embeddingQueue: QueueLike; + private readonly backfillQueue: QueueLike; private worker?: WorkerLike; private workerConnection?: RedisConnection; private workerConcurrency?: number; private closingWorker = false; + private backfillWorker?: WorkerLike; + private backfillWorkerConnection?: RedisConnection; + private backfillWorkerConcurrency?: number; + private closingBackfillWorker = false; + private onBackfillJobOutcome?: ( + runId: string, + outcome: 'processed' | 'failed', + ) => Promise; constructor( private readonly grpcSdk: ConduitGrpcSdk, @@ -68,6 +119,9 @@ export class QueueController { this.embeddingQueue = new this.QueueImpl('embeddings-generation-queue', { connection: this.queueConnection, }); + this.backfillQueue = new this.QueueImpl('embeddings-backfill-queue', { + connection: this.queueConnection, + }); } static getInstance(grpcSdk?: ConduitGrpcSdk, deps?: QueueControllerDependencies) { @@ -84,10 +138,24 @@ export class QueueController { return this.worker !== undefined; } + get hasBackfillWorker() { + return this.backfillWorker !== undefined; + } + get currentConcurrency() { return this.workerConcurrency; } + get currentBackfillConcurrency() { + return this.backfillWorkerConcurrency; + } + + setBackfillJobOutcomeHandler( + handler?: (runId: string, outcome: 'processed' | 'failed') => Promise, + ) { + this.onBackfillJobOutcome = handler; + } + async ensureWorker( processor: (data: EmbeddingJobData) => Promise, concurrency: number, @@ -95,14 +163,14 @@ export class QueueController { if (this.worker && this.workerConcurrency === concurrency) { return this.worker; } - await this.closeWorker(); + await this.closeGenerationWorker(); this.workerConnection = this.createConnection(); const worker = new this.WorkerImpl( 'embeddings-generation-queue', job => { const parsed = parseEmbeddingJobData(job.data); if (!parsed.ok) { - ConduitGrpcSdk.Metrics?.increment('malformed_embedding_jobs_total'); + incrementEmbeddingMetric('malformedJobs'); return Promise.resolve(); } return processor(parsed.data); @@ -114,49 +182,99 @@ export class QueueController { removeOnFail: { age: 24 * 3600 }, }, ); - worker.on('failed', (_job, error) => ConduitGrpcSdk.Logger.error(error as Error)); - worker.on('error', error => ConduitGrpcSdk.Logger.error(error as Error)); + worker.on('failed', (job, error) => + this.handleGenerationFailure(job as WorkerJob | undefined, error), + ); + worker.on('error', error => ConduitGrpcSdk.Logger.error(sanitizeErrorMessage(error))); this.worker = worker; this.workerConcurrency = concurrency; return worker; } - async closeWorker() { - if (this.closingWorker || !this.worker) return; - this.closingWorker = true; - const worker = this.worker; - const connection = this.workerConnection; - this.worker = undefined; - this.workerConnection = undefined; - this.workerConcurrency = undefined; - try { - await worker.close(); - await connection?.quit(); - } finally { - this.closingWorker = false; + async ensureBackfillWorker( + processor: (data: BackfillControllerJobData) => Promise, + concurrency: number, + ) { + if (this.backfillWorker && this.backfillWorkerConcurrency === concurrency) { + return this.backfillWorker; } + await this.closeBackfillWorker(); + this.backfillWorkerConnection = this.createConnection(); + const worker = new this.WorkerImpl( + 'embeddings-backfill-queue', + job => { + const parsed = parseBackfillControllerJob(job.data); + if (!parsed.ok) { + incrementEmbeddingMetric('malformedJobs'); + return Promise.resolve(); + } + return processor(parsed.data); + }, + { + concurrency, + connection: this.backfillWorkerConnection, + removeOnComplete: { age: 3600, count: 1000 }, + removeOnFail: { age: 24 * 3600 }, + }, + ); + worker.on('failed', (_job, error) => + ConduitGrpcSdk.Logger.error(sanitizeErrorMessage(error)), + ); + worker.on('error', error => ConduitGrpcSdk.Logger.error(sanitizeErrorMessage(error))); + this.backfillWorker = worker; + this.backfillWorkerConcurrency = concurrency; + return worker; + } + + async closeWorker() { + await Promise.all([this.closeGenerationWorker(), this.closeBackfillWorker()]); } async close() { await this.closeWorker(); await this.embeddingQueue.close(); + await this.backfillQueue.close(); await this.queueConnection.quit(); } + async getJobCounts( + queue: 'generation' | 'backfill' = 'generation', + ): Promise { + const counts = + queue === 'backfill' + ? await this.backfillQueue.getJobCounts() + : await this.embeddingQueue.getJobCounts(); + return normalizeJobCounts(counts); + } + + async getQueueStatus(): Promise { + const [generation, backfill] = await Promise.all([ + this.getJobCounts('generation'), + this.getJobCounts('backfill'), + ]); + return { generation, backfill }; + } + async addEmbeddingJob(data: EmbeddingJobData, attempts: number) { const parsed = parseEmbeddingJobData(data); if (!parsed.ok) { - ConduitGrpcSdk.Metrics?.increment('malformed_embedding_jobs_total'); - return; + incrementEmbeddingMetric('malformedJobs'); + return 0; } try { - await this.embeddingQueue.add(embeddingJobId(parsed.data), parsed.data, { - jobId: embeddingJobId(parsed.data), - attempts, - backoff: { type: 'exponential', delay: 1000 }, - }); + await this.embeddingQueue.add( + embeddingJobId(parsed.data), + { ...parsed.data }, + { + jobId: embeddingJobId(parsed.data), + attempts, + backoff: { type: 'exponential', delay: 1000 }, + }, + ); + return 1; } catch (err) { if (!isDuplicateJobError(err)) throw err; + return 0; } } @@ -165,18 +283,18 @@ export class QueueController { for (const [index, item] of data.entries()) { const parsed = parseEmbeddingJobData(item, index); if (!parsed.ok) { - ConduitGrpcSdk.Metrics?.increment('malformed_embedding_jobs_total'); + incrementEmbeddingMetric('malformedJobs'); continue; } jobs.push(parsed.data); } const unique = dedupeEmbeddingJobs(jobs); - if (!unique.length) return; + if (!unique.length) return 0; try { await this.embeddingQueue.addBulk( unique.map(job => ({ name: embeddingJobId(job), - data: job, + data: { ...job }, opts: { jobId: embeddingJobId(job), attempts, @@ -184,9 +302,106 @@ export class QueueController { }, })), ); + return unique.length; } catch (err) { if (!isDuplicateJobError(err)) throw err; - await Promise.all(unique.map(job => this.addEmbeddingJob(job, attempts))); + const added = await Promise.all( + unique.map(job => this.addEmbeddingJob(job, attempts)), + ); + let queued = 0; + for (const count of added) queued += count; + return queued; + } + } + + async addBackfillControllerJob( + data: BackfillControllerJobData, + opts?: { delay?: number }, + ) { + const parsed = parseBackfillControllerJob(data); + if (!parsed.ok) { + incrementEmbeddingMetric('malformedJobs'); + return; + } + const delay = + parsed.data.drain === true ? (opts?.delay ?? BACKFILL_DRAIN_DELAY_MS) : opts?.delay; + const jobId = parsed.data.drain + ? undefined + : `backfill:${parsed.data.runId}:${parsed.data.cursor ?? 'start'}`; + try { + await this.backfillQueue.add( + 'backfill-page', + { ...parsed.data }, + { + ...(jobId ? { jobId } : {}), + attempts: 3, + backoff: { type: 'exponential', delay: 1000 }, + ...(delay ? { delay } : {}), + }, + ); + } catch (err) { + if (!isDuplicateJobError(err)) throw err; + } + } + + private async closeGenerationWorker() { + if (this.closingWorker || !this.worker) return; + this.closingWorker = true; + const worker = this.worker; + const connection = this.workerConnection; + this.worker = undefined; + this.workerConnection = undefined; + this.workerConcurrency = undefined; + try { + await worker.close(); + await connection?.quit(); + } finally { + this.closingWorker = false; } } + + private async closeBackfillWorker() { + if (this.closingBackfillWorker || !this.backfillWorker) return; + this.closingBackfillWorker = true; + const worker = this.backfillWorker; + const connection = this.backfillWorkerConnection; + this.backfillWorker = undefined; + this.backfillWorkerConnection = undefined; + this.backfillWorkerConcurrency = undefined; + try { + await worker.close(); + await connection?.quit(); + } finally { + this.closingBackfillWorker = false; + } + } + + private handleGenerationFailure(job: WorkerJob | undefined, error: unknown) { + ConduitGrpcSdk.Logger.error(sanitizeErrorMessage(error)); + const attempts = job?.opts?.attempts ?? 1; + const made = job?.attemptsMade ?? 1; + if (made < attempts) { + incrementEmbeddingMetric('retried'); + return; + } + incrementEmbeddingMetric('failed'); + const parsed = parseEmbeddingJobData(job?.data); + if (!parsed.ok || !parsed.data.backfillRunId || !this.onBackfillJobOutcome) return; + this.onBackfillJobOutcome(parsed.data.backfillRunId, 'failed').catch(err => + ConduitGrpcSdk.Logger.error(sanitizeErrorMessage(err)), + ); + } +} + +function normalizeJobCounts( + counts: Partial & Record, +): QueueJobCounts { + return { + waiting: counts.waiting ?? 0, + active: counts.active ?? 0, + completed: counts.completed ?? 0, + failed: counts.failed ?? 0, + delayed: counts.delayed ?? 0, + paused: counts.paused ?? 0, + }; } diff --git a/modules/embeddings/src/metrics/index.ts b/modules/embeddings/src/metrics/index.ts index 807eed7ab..b93f7e713 100644 --- a/modules/embeddings/src/metrics/index.ts +++ b/modules/embeddings/src/metrics/index.ts @@ -15,6 +15,27 @@ export default { help: 'Tracks the total number of failed embedding generation attempts', }, }, + skippedEmbeddings: { + type: MetricType.Counter, + config: { + name: 'skipped_embeddings_total', + help: 'Tracks embeddings skipped because the source hash already matched', + }, + }, + retriedEmbeddings: { + type: MetricType.Counter, + config: { + name: 'retried_embeddings_total', + help: 'Tracks embedding generation retries before a terminal outcome', + }, + }, + embeddingBackfillJobs: { + type: MetricType.Counter, + config: { + name: 'embedding_backfill_jobs_total', + help: 'Tracks embedding jobs queued by backfill scans', + }, + }, malformedEmbeddingEvents: { type: MetricType.Counter, config: { diff --git a/modules/embeddings/src/utils/backfillExecution.test.ts b/modules/embeddings/src/utils/backfillExecution.test.ts new file mode 100644 index 000000000..cd46efc5e --- /dev/null +++ b/modules/embeddings/src/utils/backfillExecution.test.ts @@ -0,0 +1,388 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { VectorIndexStatus } from '@conduitplatform/grpc-sdk'; +import { + applyBackfillJobOutcome, + backfillRunFromDocument, + cancelBackfillExecution, + parseBackfillControllerJob, + persistableBackfillRun, + processBackfillControllerJob, + queueBackfillRuns, + resumeBackfillExecution, + type BackfillControllerJobData, + type PersistedBackfillRun, + type ProcessBackfillDeps, +} from './backfillExecution.js'; +import { BackfillGateError } from './backfillGates.js'; +import type { EmbeddingJobData } from './embeddingJobs.js'; +import type { BackfillRunProgress } from './backfillRun.js'; + +const now = new Date('2026-09-06T18:00:00.000Z'); +const capabilities = { + supported: true, + storage: true, + provider: 'mongodb' as const, +}; +const config = { + _id: 'cfg1', + enabled: true, + schemaName: 'Article', + targetField: 'embedding', +}; +const readyIndex = { + field: 'embedding', + name: 'embedding_vector', + status: VectorIndexStatus.Ready, + queryable: true, +}; + +function memoryStore(initial: PersistedBackfillRun[] = []) { + const runs = new Map( + initial.map(run => [run._id, { ...run }]), + ); + return { + runs, + createRun: async (run: BackfillRunProgress) => { + const created: PersistedBackfillRun = { ...run, _id: `run${runs.size + 1}` }; + runs.set(created._id, created); + return { _id: created._id }; + }, + getRun: async (id: string) => { + const run = runs.get(id); + return run ? { ...run } : null; + }, + saveRun: async (id: string, run: BackfillRunProgress) => { + runs.set(id, { ...run, _id: id }); + }, + }; +} + +function deps( + overrides: Partial & { store: ReturnType }, +): ProcessBackfillDeps { + const pages: Array<{ _id: string }>[] = overrides.findPage + ? [] + : [[{ _id: 'a' }, { _id: 'b' }], [{ _id: 'c' }]]; + let page = 0; + const embeddingJobs: EmbeddingJobData[] = []; + const continuations: BackfillControllerJobData[] = []; + return { + now, + maxBatchSize: 500, + moduleEnabled: true, + getRun: overrides.store.getRun, + saveRun: overrides.store.saveRun, + findPage: async () => pages[page++] ?? [], + enqueueEmbeddingJobs: async jobs => { + embeddingJobs.push(...jobs); + return jobs.length; + }, + enqueueContinuation: async job => { + continuations.push(job); + }, + getCapabilities: async () => capabilities, + getConfig: async id => (id === config._id ? config : null), + getIndexes: async () => [readyIndex], + ...overrides, + embeddingJobs, + continuations, + } as ProcessBackfillDeps & { + embeddingJobs: EmbeddingJobData[]; + continuations: BackfillControllerJobData[]; + }; +} + +describe('queued backfill start', () => { + it('persists queued config-specific runs and enqueues controller jobs without scanning', async () => { + const store = memoryStore(); + const controllerJobs: BackfillControllerJobData[] = []; + const queued = await queueBackfillRuns( + { schemaName: 'Article', batchSize: 2, onlyMissing: true }, + { + moduleEnabled: true, + capabilities, + configs: [config, { ...config, _id: 'cfg2', targetField: 'other' }], + indexes: [readyIndex, { ...readyIndex, field: 'other', name: 'other_vector' }], + createRun: store.createRun, + enqueueController: async job => { + controllerJobs.push(job); + }, + }, + ); + assert.equal(queued.queued, 2); + assert.equal(queued.runs[0].state, 'queued'); + assert.equal(queued.runs[0].configId, 'cfg1'); + assert.equal(queued.runs[1].configId, 'cfg2'); + assert.deepEqual( + controllerJobs.map(job => job.runId), + queued.runs.map(run => run.id), + ); + const persisted = [...store.runs.values()]; + assert.equal( + persisted.every(run => run.state === 'queued'), + true, + ); + assert.equal( + persisted.every(run => run.scannedCount === 0), + true, + ); + assert.equal( + persisted.every(run => run.onlyMissing === true), + true, + ); + }); + + it('fails closed before persisting when a gate is not met', async () => { + const store = memoryStore(); + await assert.rejects( + () => + queueBackfillRuns( + { schemaName: 'Article' }, + { + moduleEnabled: false, + capabilities, + configs: [config], + indexes: [readyIndex], + createRun: store.createRun, + enqueueController: async () => undefined, + }, + ), + (err: unknown) => + err instanceof BackfillGateError && err.reason === 'module_disabled', + ); + assert.equal(store.runs.size, 0); + }); +}); + +describe('cursor-based backfill continuation', () => { + it('scans bounded pages, caps enqueue, and continues from the cursor', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'queued', + batchSize: 2, + onlyMissing: false, + scannedCount: 0, + queuedCount: 0, + processedCount: 0, + failedCount: 0, + }), + ); + const harness = deps({ + store, + findPage: async (_schema, page) => { + if (page.query._id) return [{ _id: 'c' }]; + return [{ _id: 'a' }, { _id: 'b' }, { _id: 'extra' }]; + }, + }) as ProcessBackfillDeps & { + embeddingJobs: EmbeddingJobData[]; + continuations: BackfillControllerJobData[]; + }; + + const first = await processBackfillControllerJob( + { runId: created._id, cursor: null }, + harness, + ); + assert.equal(first.action, 'continue'); + assert.equal(first.run?.state, 'running'); + assert.equal(first.run?.cursor, 'b'); + assert.equal(first.run?.scannedCount, 2); + assert.equal(first.run?.queuedCount, 2); + assert.equal(harness.embeddingJobs.length, 2); + assert.equal(harness.embeddingJobs[0].configId, 'cfg1'); + assert.equal(harness.embeddingJobs[0].backfillRunId, created._id); + assert.deepEqual(harness.continuations, [{ runId: created._id, cursor: 'b' }]); + + const second = await processBackfillControllerJob(harness.continuations[0], { + ...harness, + enqueueContinuation: async job => { + harness.continuations.push(job); + }, + }); + assert.equal(second.action, 'drain'); + assert.equal(second.run?.cursor, 'c'); + assert.equal(second.run?.scannedCount, 3); + assert.equal(second.run?.queuedCount, 3); + assert.equal(second.run?.state, 'running'); + for (let i = 0; i < 3; i += 1) { + await applyBackfillJobOutcome({ + run: (await store.getRun(created._id))!, + outcome: 'processed', + saveRun: store.saveRun, + }); + } + const drained = await processBackfillControllerJob( + { runId: created._id, cursor: 'c', drain: true }, + harness, + ); + assert.equal(drained.action, 'completed'); + assert.equal(drained.run?.processedCount, 3); + }); + + it('uses onlyMissing target-field queries and config-specific jobs', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'queued', + batchSize: 1, + onlyMissing: true, + }), + ); + let observedQuery: Record | undefined; + const harness = deps({ + store, + findPage: async (_schema, page) => { + observedQuery = page.query; + return [{ _id: 'a' }]; + }, + }) as ProcessBackfillDeps & { embeddingJobs: EmbeddingJobData[] }; + await processBackfillControllerJob({ runId: created._id, cursor: null }, harness); + assert.equal(observedQuery?.embedding, null); + assert.equal(harness.embeddingJobs[0]?.configId, 'cfg1'); + }); +}); + +describe('backfill cancellation, resume, and counters', () => { + it('stops at cancellation checks and resumes from the saved cursor', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'queued', + batchSize: 2, + }), + ); + const first = deps({ store }); + await processBackfillControllerJob({ runId: created._id, cursor: null }, first); + const running = (await store.getRun(created._id))!; + const canceled = await cancelBackfillExecution({ + run: running, + saveRun: store.saveRun, + now, + }); + assert.equal(canceled.ok, true); + const canceledProcess = await processBackfillControllerJob( + { runId: created._id, cursor: 'b' }, + deps({ store }), + ); + assert.equal(canceledProcess.action, 'canceled'); + + const resumed = await resumeBackfillExecution({ + run: (await store.getRun(created._id))!, + saveRun: store.saveRun, + enqueueController: async () => undefined, + }); + assert.equal(resumed.ok, true); + if (!resumed.ok) return; + assert.equal(resumed.run.state, 'queued'); + assert.equal(resumed.run.cursor, 'b'); + + let query: Record | undefined; + const restarted = await processBackfillControllerJob( + { runId: created._id, cursor: 'b' }, + deps({ + store, + findPage: async (_schema, page) => { + query = page.query; + return [{ _id: 'c' }]; + }, + }), + ); + assert.deepEqual(query?._id, { $gt: 'b' }); + assert.equal(restarted.run?.cursor, 'c'); + assert.equal(restarted.run?.state, 'running'); + assert.equal(restarted.run?.startedAt?.toISOString(), now.toISOString()); + }); + + it('persists processed and failed job counts without exceeding queued work', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'queued', + batchSize: 2, + }), + ); + await processBackfillControllerJob( + { runId: created._id, cursor: null }, + deps({ store }), + ); + const afterPage = (await store.getRun(created._id))!; + const processed = await applyBackfillJobOutcome({ + run: afterPage, + outcome: 'processed', + saveRun: store.saveRun, + }); + assert.equal(processed.ok, true); + const failed = await applyBackfillJobOutcome({ + run: (await store.getRun(created._id))!, + outcome: 'failed', + saveRun: store.saveRun, + }); + assert.equal(failed.ok, true); + if (!failed.ok) return; + assert.equal(failed.run.scannedCount, 2); + assert.equal(failed.run.queuedCount, 2); + assert.equal(failed.run.processedCount, 1); + assert.equal(failed.run.failedCount, 1); + }); + + it('fails the running run when the vector index is not queryable', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'queued', + batchSize: 1, + }), + ); + const result = await processBackfillControllerJob( + { runId: created._id, cursor: null }, + deps({ + store, + getIndexes: async () => [ + { + field: 'embedding', + status: VectorIndexStatus.Pending, + queryable: false, + }, + ], + }), + ); + assert.equal(result.action, 'failed'); + assert.equal(result.run?.state, 'failed'); + assert.match(result.run?.error ?? '', /not queryable/); + assert.doesNotMatch(result.run?.error ?? '', /apiKey|Bearer /); + }); +}); + +describe('backfill controller job parsing and persistence mapping', () => { + it('rejects malformed controller payloads and round-trips run documents', () => { + assert.equal(parseBackfillControllerJob({ runId: 'run1', cursor: null }).ok, true); + assert.equal(parseBackfillControllerJob({ runId: '../nope' }).ok, false); + assert.equal(parseBackfillControllerJob({ runId: 'run1', extra: true }).ok, false); + const progress = backfillRunFromDocument({ + _id: 'run1', + schemaName: 'Article', + configId: 'cfg1', + state: 'queued', + batchSize: 10, + onlyMissing: true, + }); + assert.equal(progress.cursor, null); + assert.equal(persistableBackfillRun(progress).onlyMissing, true); + }); +}); diff --git a/modules/embeddings/src/utils/backfillExecution.ts b/modules/embeddings/src/utils/backfillExecution.ts new file mode 100644 index 000000000..eb78cdd96 --- /dev/null +++ b/modules/embeddings/src/utils/backfillExecution.ts @@ -0,0 +1,427 @@ +import type { VectorCapabilities } from '@conduitplatform/grpc-sdk'; +import { + applyBackfillJobCounts, + applyBackfillPage, + boundBackfillPage, + buildBackfillPageQuery, + cancelBackfillRun, + completeBackfillRun, + createQueuedBackfill, + failBackfillRun, + resumeBackfillRun, + startBackfillRun, + type BackfillPageQuery, + type BackfillRunProgress, + type BackfillRunResult, +} from './backfillRun.js'; +import { + assertBackfillExecutable, + BackfillGateError, + type BackfillConfigGate, + type VectorIndexGate, +} from './backfillGates.js'; +import { EmbeddingJobData, MAX_QUEUE_BATCH_SIZE } from './embeddingJobs.js'; +import { incrementEmbeddingMetric } from './embeddingMetrics.js'; + +const IDENTITY = /^[A-Za-z0-9._-]{1,128}$/; + +export const BACKFILL_DRAIN_DELAY_MS = 1000; + +export interface PersistedBackfillRun extends BackfillRunProgress { + _id: string; +} + +export interface BackfillControllerJobData { + runId: string; + cursor?: string | null; + drain?: boolean; +} + +export type ParsedBackfillControllerJob = + { ok: true; data: BackfillControllerJobData } | { ok: false; reason: string }; + +export interface QueueBackfillInput { + schemaName: string; + batchSize?: number; + configId?: string; + onlyMissing?: boolean; + filter?: unknown; + maxBatchSize?: number; +} + +export interface QueueBackfillDeps { + moduleEnabled: boolean; + capabilities: Pick; + configs: BackfillConfigGate[]; + indexes: readonly VectorIndexGate[]; + createRun: (run: BackfillRunProgress) => Promise<{ _id: string }>; + enqueueController: (job: BackfillControllerJobData) => Promise; +} + +export interface ProcessBackfillDeps { + now?: Date; + maxBatchSize: number; + moduleEnabled: boolean; + getRun: (id: string) => Promise; + saveRun: (id: string, run: BackfillRunProgress) => Promise; + findPage: ( + schemaName: string, + page: BackfillPageQuery, + ) => Promise>; + enqueueEmbeddingJobs: (jobs: EmbeddingJobData[]) => Promise; + enqueueContinuation: (job: BackfillControllerJobData) => Promise; + getCapabilities: ( + schemaName: string, + ) => Promise>; + getConfig: (id: string) => Promise; + getIndexes: (schemaName: string) => Promise; +} + +export function backfillRunFromDocument(doc: { + _id: string; + schemaName: string; + configId?: string; + state: BackfillRunProgress['state']; + cursor?: string; + batchSize: number; + onlyMissing?: boolean; + filter?: Record; + scannedCount?: number; + queuedCount?: number; + processedCount?: number; + failedCount?: number; + startedAt?: Date; + finishedAt?: Date; + error?: string; +}): PersistedBackfillRun { + return { + _id: doc._id, + state: doc.state, + schemaName: doc.schemaName, + ...(doc.configId ? { configId: doc.configId } : {}), + cursor: doc.cursor ?? null, + batchSize: doc.batchSize, + onlyMissing: doc.onlyMissing === true, + filter: doc.filter ?? null, + scannedCount: doc.scannedCount ?? 0, + queuedCount: doc.queuedCount ?? 0, + processedCount: doc.processedCount ?? 0, + failedCount: doc.failedCount ?? 0, + startedAt: doc.startedAt ?? null, + finishedAt: doc.finishedAt ?? null, + error: doc.error ?? null, + }; +} + +export function persistableBackfillRun( + run: BackfillRunProgress, +): Record { + return { + state: run.state, + schemaName: run.schemaName, + configId: run.configId, + cursor: run.cursor ?? undefined, + batchSize: run.batchSize, + onlyMissing: run.onlyMissing, + filter: run.filter ?? undefined, + scannedCount: run.scannedCount, + queuedCount: run.queuedCount, + processedCount: run.processedCount, + failedCount: run.failedCount, + startedAt: run.startedAt ?? undefined, + finishedAt: run.finishedAt ?? undefined, + error: run.error ?? undefined, + }; +} + +export function parseBackfillControllerJob(value: unknown): ParsedBackfillControllerJob { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return { ok: false, reason: 'malformed' }; + } + const record = value as Record; + const extraKeys = Object.keys(record).filter( + key => !['runId', 'cursor', 'drain'].includes(key), + ); + if (extraKeys.length) return { ok: false, reason: 'malformed' }; + if (typeof record.runId !== 'string' || !IDENTITY.test(record.runId)) { + return { ok: false, reason: 'runId' }; + } + if (record.drain !== undefined && typeof record.drain !== 'boolean') { + return { ok: false, reason: 'drain' }; + } + if ( + record.cursor != null && + (typeof record.cursor !== 'string' || !IDENTITY.test(record.cursor)) + ) { + return { ok: false, reason: 'cursor' }; + } + return { + ok: true, + data: { + runId: record.runId, + cursor: record.cursor ?? null, + ...(record.drain === true ? { drain: true } : {}), + }, + }; +} + +export function selectedBackfillConfigs( + configs: BackfillConfigGate[], + configId?: string, +): BackfillConfigGate[] { + if (configId) { + return configs.filter(config => config._id === configId); + } + return configs.filter(config => config.enabled !== false && config._id); +} + +export async function queueBackfillRuns( + input: QueueBackfillInput, + deps: QueueBackfillDeps, +): Promise<{ + queued: number; + runs: Array<{ id: string; configId?: string; state: string }>; +}> { + const selected = selectedBackfillConfigs(deps.configs, input.configId); + if (!selected.length) { + assertBackfillExecutable({ + moduleEnabled: deps.moduleEnabled, + capabilities: deps.capabilities, + config: null, + indexes: deps.indexes, + }); + } + for (const config of selected) { + assertBackfillExecutable({ + moduleEnabled: deps.moduleEnabled, + capabilities: deps.capabilities, + config, + indexes: deps.indexes, + }); + } + const runs: Array<{ id: string; configId?: string; state: string }> = []; + for (const config of selected) { + const created = createQueuedBackfill({ + schemaName: input.schemaName, + configId: config._id, + batchSize: input.batchSize, + onlyMissing: input.onlyMissing, + filter: input.filter, + maxBatchSize: input.maxBatchSize, + }); + if (!created.ok) { + throw new BackfillGateError( + 'config_not_found', + `Invalid backfill request: ${created.reason}`, + ); + } + const persisted = await deps.createRun(created.run); + await deps.enqueueController({ runId: persisted._id, cursor: null }); + runs.push({ + id: persisted._id, + ...(config._id ? { configId: config._id } : {}), + state: created.run.state, + }); + } + return { queued: runs.length, runs }; +} + +export async function resumeBackfillExecution(args: { + run: PersistedBackfillRun; + saveRun: (id: string, run: BackfillRunProgress) => Promise; + enqueueController: (job: BackfillControllerJobData) => Promise; +}): Promise { + const resumed = resumeBackfillRun(args.run); + if (!resumed.ok) return resumed; + await args.saveRun(args.run._id, resumed.run); + await args.enqueueController({ + runId: args.run._id, + cursor: resumed.run.cursor ?? null, + }); + return resumed; +} + +export async function cancelBackfillExecution(args: { + run: PersistedBackfillRun; + saveRun: (id: string, run: BackfillRunProgress) => Promise; + now?: Date; +}): Promise { + const canceled = cancelBackfillRun(args.run, args.now); + if (!canceled.ok) return canceled; + await args.saveRun(args.run._id, canceled.run); + return canceled; +} + +export async function applyBackfillJobOutcome(args: { + run: PersistedBackfillRun; + outcome: 'processed' | 'failed'; + saveRun: (id: string, run: BackfillRunProgress) => Promise; +}): Promise { + const counted = applyBackfillJobCounts( + args.run, + args.outcome === 'processed' ? { processed: 1 } : { failed: 1 }, + ); + if (!counted.ok) return counted; + await args.saveRun(args.run._id, counted.run); + return counted; +} + +export async function processBackfillControllerJob( + rawJob: unknown, + deps: ProcessBackfillDeps, +): Promise<{ action: string; run?: BackfillRunProgress }> { + const parsed = parseBackfillControllerJob(rawJob); + if (!parsed.ok) { + incrementEmbeddingMetric('malformedJobs'); + return { action: 'malformed' }; + } + const persisted = await deps.getRun(parsed.data.runId); + if (!persisted) { + return { action: 'missing' }; + } + if (persisted.state === 'canceled' || persisted.state === 'completed') { + return { action: persisted.state, run: persisted }; + } + if (parsed.data.drain) { + return drainBackfill(persisted, deps); + } + return scanBackfillPage(parsed.data, persisted, deps); +} + +async function scanBackfillPage( + job: BackfillControllerJobData, + persisted: PersistedBackfillRun, + deps: ProcessBackfillDeps, +): Promise<{ action: string; run?: BackfillRunProgress }> { + const now = deps.now ?? new Date(); + let run: BackfillRunProgress = persisted; + if (run.state === 'queued') { + const started = startBackfillRun(run, now); + if (!started.ok) return { action: started.reason, run }; + run = started.run; + await deps.saveRun(persisted._id, run); + } + if (run.state !== 'running') { + return { action: run.state, run }; + } + const jobCursor = job.cursor ?? null; + const runCursor = run.cursor ?? null; + if (jobCursor !== runCursor) { + await deps.enqueueContinuation({ + runId: persisted._id, + cursor: runCursor, + }); + return { action: 'stale', run }; + } + + try { + if (!run.configId) { + throw new BackfillGateError( + 'config_not_found', + 'Backfill run is missing a config id', + ); + } + const config = await deps.getConfig(run.configId); + assertBackfillExecutable({ + moduleEnabled: deps.moduleEnabled, + capabilities: await deps.getCapabilities(run.schemaName), + config, + indexes: await deps.getIndexes(run.schemaName), + }); + const pageQuery = buildBackfillPageQuery(run, config?.targetField); + if (!pageQuery.ok) { + throw new BackfillGateError( + 'config_not_found', + `Invalid backfill page: ${pageQuery.reason}`, + ); + } + const docs = boundBackfillPage( + await deps.findPage(run.schemaName, pageQuery.page), + run.batchSize, + ); + const jobs = docs + .map(doc => ({ + schemaName: run.schemaName, + documentId: String(doc._id), + ...(run.configId ? { configId: run.configId } : {}), + backfillRunId: persisted._id, + })) + .slice(0, Math.min(run.batchSize, deps.maxBatchSize, MAX_QUEUE_BATCH_SIZE)); + const queuedDelta = jobs.length ? await deps.enqueueEmbeddingJobs(jobs) : 0; + incrementEmbeddingMetric('backfill', queuedDelta); + const applied = applyBackfillPage( + run, + docs.map(doc => ({ _id: String(doc._id) })), + queuedDelta, + ); + if (!applied.ok) { + return failPersistedRun(persisted._id, run, applied.reason, deps, now); + } + run = applied.run; + await deps.saveRun(persisted._id, run); + const canceled = await deps.getRun(persisted._id); + if (!canceled || canceled.state === 'canceled') { + return { action: 'canceled', run: canceled ?? run }; + } + if (applied.exhausted) { + return finishOrDrain(persisted._id, run, deps, now); + } + await deps.enqueueContinuation({ + runId: persisted._id, + cursor: run.cursor ?? null, + }); + return { action: 'continue', run }; + } catch (err) { + if (err instanceof BackfillGateError) { + return failPersistedRun(persisted._id, run, err, deps, now); + } + return failPersistedRun(persisted._id, run, err, deps, now); + } +} + +async function drainBackfill( + persisted: PersistedBackfillRun, + deps: ProcessBackfillDeps, +): Promise<{ action: string; run?: BackfillRunProgress }> { + const latest = (await deps.getRun(persisted._id)) ?? persisted; + if (latest.state === 'canceled') { + return { action: 'canceled', run: latest }; + } + if (latest.state !== 'running') { + return { action: latest.state, run: latest }; + } + return finishOrDrain(latest._id, latest, deps, deps.now ?? new Date()); +} + +async function finishOrDrain( + id: string, + run: BackfillRunProgress, + deps: ProcessBackfillDeps, + now: Date, +): Promise<{ action: string; run?: BackfillRunProgress }> { + if (run.processedCount + run.failedCount >= run.queuedCount) { + const completed = completeBackfillRun(run, now); + if (!completed.ok) return { action: completed.reason, run }; + await deps.saveRun(id, completed.run); + return { action: 'completed', run: completed.run }; + } + await deps.enqueueContinuation({ + runId: id, + cursor: run.cursor ?? null, + drain: true, + }); + return { action: 'drain', run }; +} + +async function failPersistedRun( + id: string, + run: BackfillRunProgress, + err: unknown, + deps: ProcessBackfillDeps, + now: Date, +): Promise<{ action: string; run?: BackfillRunProgress }> { + const failed = failBackfillRun(run, err, now); + if (!failed.ok) return { action: failed.reason, run }; + await deps.saveRun(id, failed.run); + return { action: 'failed', run: failed.run }; +} diff --git a/modules/embeddings/src/utils/backfillGates.test.ts b/modules/embeddings/src/utils/backfillGates.test.ts new file mode 100644 index 000000000..ba415b89d --- /dev/null +++ b/modules/embeddings/src/utils/backfillGates.test.ts @@ -0,0 +1,192 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { ConduitGrpcSdk, VectorIndexStatus } from '@conduitplatform/grpc-sdk'; +import { + assertBackfillExecutable, + BackfillGateError, + findTargetVectorIndex, + grpcErrorFromBackfillGate, + isEmbeddingVectorIndexQueryable, +} from './backfillGates.js'; + +const capabilities = { + supported: true, + storage: true, + provider: 'mongodb' as const, +}; + +const config = { + _id: 'cfg1', + enabled: true, + schemaName: 'Article', + targetField: 'embedding', +}; + +const readyIndex = { + field: 'embedding', + name: 'embedding_vector', + status: VectorIndexStatus.Ready, + queryable: true, +}; + +describe('backfill execution gates', () => { + it('blocks disabled modules, unsupported storage, and disabled configs', () => { + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: false, + capabilities, + config, + indexes: [readyIndex], + }), + (err: unknown) => + err instanceof BackfillGateError && err.reason === 'module_disabled', + ); + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities: { + supported: false, + storage: false, + provider: 'unsupported', + reason: 'mysql does not support Conduit vector search', + }, + config, + indexes: [readyIndex], + }), + (err: unknown) => + err instanceof BackfillGateError && + err.reason === 'vector_unsupported' && + /mysql/.test(err.message), + ); + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities: { + supported: true, + storage: false, + provider: 'postgres', + reason: 'pgvector is not available', + }, + config, + indexes: [readyIndex], + }), + (err: unknown) => + err instanceof BackfillGateError && err.reason === 'vector_storage_unavailable', + ); + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities, + config: { ...config, enabled: false }, + indexes: [readyIndex], + }), + (err: unknown) => + err instanceof BackfillGateError && err.reason === 'config_disabled', + ); + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities, + config: null, + indexes: [readyIndex], + }), + (err: unknown) => + err instanceof BackfillGateError && err.reason === 'config_not_found', + ); + }); + + it('requires a queryable vector index and keeps the status actionable', () => { + assert.equal(isEmbeddingVectorIndexQueryable(readyIndex), true); + assert.equal( + isEmbeddingVectorIndexQueryable({ + field: 'embedding', + status: VectorIndexStatus.Pending, + }), + false, + ); + assert.deepEqual(findTargetVectorIndex([readyIndex], 'embedding'), readyIndex); + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities, + config, + indexes: [ + { + field: 'embedding', + name: 'embedding_vector', + status: VectorIndexStatus.Pending, + queryable: false, + }, + ], + }), + (err: unknown) => + err instanceof BackfillGateError && + err.reason === 'index_not_queryable' && + err.indexStatus === VectorIndexStatus.Pending && + /Wait until the index is ready/.test(err.message), + ); + const mapped = grpcErrorFromBackfillGate( + new BackfillGateError( + 'index_not_queryable', + "Vector index for field 'embedding' is not queryable (status: pending). Wait until the index is ready before running a backfill.", + 'pending', + ), + ); + assert.equal(mapped.code, 9); + }); +}); + +describe('embedding metric increments do not accept labels', () => { + it('increments named counters without attaching payload data', async () => { + const { incrementEmbeddingMetric, EMBEDDING_METRICS } = + await import('./embeddingMetrics.js'); + const seen: Array<{ name: string; amount?: number; labels?: unknown }> = []; + const previous = ConduitGrpcSdk.Metrics; + ConduitGrpcSdk.Metrics = { + increment(name: string, amount?: number, labels?: unknown) { + seen.push({ name, amount, labels }); + }, + } as never; + try { + incrementEmbeddingMetric('generated', 2); + incrementEmbeddingMetric('failed'); + incrementEmbeddingMetric('skipped', 1); + incrementEmbeddingMetric('retried', 1); + incrementEmbeddingMetric('backfill', 3); + incrementEmbeddingMetric('malformedEvents'); + incrementEmbeddingMetric('malformedJobs'); + incrementEmbeddingMetric('generated', 0); + } finally { + ConduitGrpcSdk.Metrics = previous; + } + assert.deepEqual( + seen.map(item => item.name), + [ + EMBEDDING_METRICS.generated, + EMBEDDING_METRICS.failed, + EMBEDDING_METRICS.skipped, + EMBEDDING_METRICS.retried, + EMBEDDING_METRICS.backfill, + EMBEDDING_METRICS.malformedEvents, + EMBEDDING_METRICS.malformedJobs, + ], + ); + assert.equal( + seen.every(item => item.labels === undefined), + true, + ); + assert.equal( + seen.some( + item => + JSON.stringify(item).includes('sk-') || JSON.stringify(item).includes('doc'), + ), + false, + ); + }); +}); diff --git a/modules/embeddings/src/utils/backfillGates.ts b/modules/embeddings/src/utils/backfillGates.ts new file mode 100644 index 000000000..4efdca6d9 --- /dev/null +++ b/modules/embeddings/src/utils/backfillGates.ts @@ -0,0 +1,147 @@ +import { + GrpcError, + VectorCapabilities, + VectorIndexStatus, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export const BACKFILL_GATE_REASONS = [ + 'module_disabled', + 'vector_unsupported', + 'vector_storage_unavailable', + 'config_not_found', + 'config_disabled', + 'index_not_queryable', +] as const; + +export type BackfillGateReason = (typeof BACKFILL_GATE_REASONS)[number]; + +export class BackfillGateError extends Error { + readonly code = 'BACKFILL_GATE' as const; + + constructor( + readonly reason: BackfillGateReason, + message: string, + readonly indexStatus?: string, + ) { + super(message); + this.name = 'BackfillGateError'; + } +} + +export interface BackfillConfigGate { + _id?: string; + enabled?: boolean; + schemaName?: string; + targetField?: string; +} + +export interface VectorIndexGate { + field?: string; + name?: string; + queryable?: boolean; + status?: string; +} + +export function isEmbeddingVectorIndexQueryable(index?: VectorIndexGate): boolean { + if (!index) return false; + if (index.queryable === false) return false; + const indexStatus = index.status?.toLowerCase(); + if (indexStatus === VectorIndexStatus.Failed || indexStatus === 'failed') { + return false; + } + if ( + (indexStatus === VectorIndexStatus.Pending || indexStatus === 'pending') && + index.queryable !== true + ) { + return false; + } + return true; +} + +export function findTargetVectorIndex( + indexes: readonly VectorIndexGate[], + targetField: string, +): VectorIndexGate | undefined { + return ( + indexes.find( + index => index.field === targetField && index.name === `${targetField}_vector`, + ) ?? indexes.find(index => index.field === targetField) + ); +} + +export function assertBackfillExecutable(args: { + moduleEnabled: boolean; + capabilities?: Pick< + VectorCapabilities, + 'supported' | 'storage' | 'provider' | 'reason' + >; + config?: BackfillConfigGate | null; + indexes?: readonly VectorIndexGate[]; +}): void { + if (!args.moduleEnabled) { + throw new BackfillGateError( + 'module_disabled', + 'Embeddings module is disabled; enable it before starting a backfill', + ); + } + const capabilities = args.capabilities; + if (!capabilities?.supported) { + throw new BackfillGateError( + 'vector_unsupported', + capabilities?.reason ?? + 'Database does not support Conduit vector storage; use MongoDB Atlas Vector Search or Postgres pgvector', + ); + } + if (!capabilities.storage) { + throw new BackfillGateError( + 'vector_storage_unavailable', + capabilities.reason ?? + `Vector storage is unavailable for provider '${capabilities.provider}'`, + ); + } + if (!args.config) { + throw new BackfillGateError( + 'config_not_found', + 'No enabled embedding config found for backfill', + ); + } + if (args.config.enabled === false) { + throw new BackfillGateError( + 'config_disabled', + `Embedding config '${args.config._id ?? 'unknown'}' is disabled`, + ); + } + const targetField = args.config.targetField; + if (typeof targetField !== 'string' || !targetField.length) { + throw new BackfillGateError( + 'config_not_found', + 'Embedding config is missing a target vector field', + ); + } + const index = findTargetVectorIndex(args.indexes ?? [], targetField); + if (isEmbeddingVectorIndexQueryable(index)) return; + const indexStatus = index?.status ?? 'missing'; + throw new BackfillGateError( + 'index_not_queryable', + `Vector index for field '${targetField}' is not queryable (status: ${indexStatus}). ` + + 'Wait until the index is ready before running a backfill.', + indexStatus, + ); +} + +export function grpcErrorFromBackfillGate(err: BackfillGateError): GrpcError { + switch (err.reason) { + case 'module_disabled': + case 'vector_unsupported': + case 'vector_storage_unavailable': + case 'config_not_found': + case 'config_disabled': + case 'index_not_queryable': + return new GrpcError(status.FAILED_PRECONDITION, err.message); + default: { + const unexpected: never = err.reason; + return new GrpcError(status.INTERNAL, String(unexpected)); + } + } +} diff --git a/modules/embeddings/src/utils/embeddingJobs.test.ts b/modules/embeddings/src/utils/embeddingJobs.test.ts index 9349c1bb3..6ec768273 100644 --- a/modules/embeddings/src/utils/embeddingJobs.test.ts +++ b/modules/embeddings/src/utils/embeddingJobs.test.ts @@ -38,6 +38,15 @@ describe('embedding job identity', () => { parseEmbeddingJobData({ schemaName: '../etc', documentId: 'a' }).ok, false, ); + assert.equal( + parseEmbeddingJobData({ + schemaName: 'Article', + documentId: 'a', + configId: 'c1', + backfillRunId: 'run1', + }).ok, + true, + ); assert.equal( parseEmbeddingJobData({ schemaName: 'Article', documentId: 'a', extra: true }).ok, false, diff --git a/modules/embeddings/src/utils/embeddingJobs.ts b/modules/embeddings/src/utils/embeddingJobs.ts index a4bd1a048..c298e4ed4 100644 --- a/modules/embeddings/src/utils/embeddingJobs.ts +++ b/modules/embeddings/src/utils/embeddingJobs.ts @@ -2,6 +2,7 @@ export interface EmbeddingJobData { schemaName: string; documentId: string; configId?: string; + backfillRunId?: string; } export const MAX_SCHEMA_NAME_LENGTH = 128; @@ -49,7 +50,7 @@ export function parseEmbeddingJobData( } const record = value as Record; const extraKeys = Object.keys(record).filter( - key => !['schemaName', 'documentId', 'configId'].includes(key), + key => !['schemaName', 'documentId', 'configId', 'backfillRunId'].includes(key), ); if (extraKeys.length) return { ok: false, reason: 'malformed' }; if (typeof record.schemaName !== 'string' || !SCHEMA_NAME.test(record.schemaName)) { @@ -64,12 +65,19 @@ export function parseEmbeddingJobData( ) { return { ok: false, reason: 'configId' }; } + if ( + record.backfillRunId !== undefined && + (typeof record.backfillRunId !== 'string' || !IDENTITY.test(record.backfillRunId)) + ) { + return { ok: false, reason: 'backfillRunId' }; + } return { ok: true, data: { schemaName: record.schemaName, documentId: record.documentId, ...(record.configId ? { configId: record.configId } : {}), + ...(record.backfillRunId ? { backfillRunId: record.backfillRunId } : {}), }, }; } diff --git a/modules/embeddings/src/utils/embeddingMetrics.ts b/modules/embeddings/src/utils/embeddingMetrics.ts new file mode 100644 index 000000000..6a80009e7 --- /dev/null +++ b/modules/embeddings/src/utils/embeddingMetrics.ts @@ -0,0 +1,21 @@ +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; + +export const EMBEDDING_METRICS = { + generated: 'generated_embeddings_total', + failed: 'failed_embeddings_total', + skipped: 'skipped_embeddings_total', + retried: 'retried_embeddings_total', + backfill: 'embedding_backfill_jobs_total', + malformedEvents: 'malformed_embedding_events_total', + malformedJobs: 'malformed_embedding_jobs_total', +} as const; + +export type EmbeddingMetric = keyof typeof EMBEDDING_METRICS; + +export function incrementEmbeddingMetric( + metric: EmbeddingMetric, + amount: number = 1, +): void { + if (!Number.isFinite(amount) || amount <= 0) return; + ConduitGrpcSdk.Metrics?.increment(EMBEDDING_METRICS[metric], amount); +} From 881ed10489649f81d71e34531ca0249fac3dd83e Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 20:04:06 +0300 Subject: [PATCH 09/29] feat(embeddings): add typed gRPC, Admin, and MCP APIs Replace unreleased JSON-string proto responses with typed config, status, backfill, and search messages so operators can manage embeddings through gRPC, Admin, and Hermes MCP without exposing config or backfill on client routes. --- .../grpc-sdk/src/modules/embeddings/index.ts | 215 +++++- modules/embeddings/README.md | 27 +- modules/embeddings/package.json | 2 +- modules/embeddings/src/Embeddings.ts | 385 ++++++----- modules/embeddings/src/admin/index.ts | 334 +++++++++ modules/embeddings/src/admin/routes.ts | 117 ++++ .../embeddings/src/api/embeddingsApi.test.ts | 416 ++++++++++++ modules/embeddings/src/api/embeddingsApi.ts | 638 ++++++++++++++++++ modules/embeddings/src/embeddings.proto | 169 ++++- modules/embeddings/src/routes/index.ts | 89 +++ .../src/utils/clientSearchContext.test.ts | 25 + .../src/utils/clientSearchContext.ts | 27 + .../embeddings/src/utils/mcpToolNames.test.ts | 73 ++ modules/embeddings/src/utils/mcpToolNames.ts | 21 + .../src/utils/operationalStatus.test.ts | 114 ++++ .../embeddings/src/utils/operationalStatus.ts | 217 ++++++ modules/embeddings/src/utils/protoMappers.ts | 173 +++++ .../test/embedding-contract.test.mjs | 78 ++- modules/embeddings/tsconfig.test.json | 7 +- 19 files changed, 2919 insertions(+), 208 deletions(-) create mode 100644 modules/embeddings/src/admin/index.ts create mode 100644 modules/embeddings/src/admin/routes.ts create mode 100644 modules/embeddings/src/api/embeddingsApi.test.ts create mode 100644 modules/embeddings/src/api/embeddingsApi.ts create mode 100644 modules/embeddings/src/routes/index.ts create mode 100644 modules/embeddings/src/utils/clientSearchContext.test.ts create mode 100644 modules/embeddings/src/utils/clientSearchContext.ts create mode 100644 modules/embeddings/src/utils/mcpToolNames.test.ts create mode 100644 modules/embeddings/src/utils/mcpToolNames.ts create mode 100644 modules/embeddings/src/utils/operationalStatus.test.ts create mode 100644 modules/embeddings/src/utils/operationalStatus.ts create mode 100644 modules/embeddings/src/utils/protoMappers.ts diff --git a/libraries/grpc-sdk/src/modules/embeddings/index.ts b/libraries/grpc-sdk/src/modules/embeddings/index.ts index 1de58ae4c..57656817e 100644 --- a/libraries/grpc-sdk/src/modules/embeddings/index.ts +++ b/libraries/grpc-sdk/src/modules/embeddings/index.ts @@ -1,6 +1,10 @@ import { ConduitModule } from '../../classes/index.js'; import { EmbeddingsProviderDefinition } from '../../protoUtils/embeddings.js'; -import type { Indexable, VectorSearchResult } from '../../interfaces/index.js'; +import type { + Indexable, + VectorCapabilities, + VectorSearchResult, +} from '../../interfaces/index.js'; export interface EmbeddingConfigInput { schemaName: string; @@ -11,6 +15,67 @@ export interface EmbeddingConfigInput { dimensions: number; similarity?: string; sourceFieldAllowlist?: string[]; + enabled?: boolean; +} + +export interface EmbeddingConfigRecord { + id: string; + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + model: string; + dimensions: number; + similarity: string; + enabled: boolean; + createdAt?: string; + updatedAt?: string; +} + +export interface QueueCounts { + waiting: number; + active: number; + completed: number; + failed: number; + delayed: number; + paused: number; +} + +export interface EmbeddingsStatus { + enabled: boolean; + ready: boolean; + capabilities: VectorCapabilities; + generationQueue: QueueCounts; + backfillQueue: QueueCounts; + warnings: string[]; +} + +export interface BackfillRunRecord { + id: string; + schemaName: string; + configId?: string; + state: string; + cursor?: string; + batchSize: number; + onlyMissing: boolean; + filter?: Indexable; + scannedCount: number; + queuedCount: number; + processedCount: number; + failedCount: number; + startedAt?: string; + finishedAt?: string; + error?: string; + createdAt?: string; + updatedAt?: string; +} + +export interface StartBackfillInput { + schemaName: string; + batchSize?: number; + configId?: string; + onlyMissing?: boolean; + filter?: Indexable; } export interface SemanticSearchInput { @@ -24,6 +89,54 @@ export interface SemanticSearchInput { adminOperator?: boolean; } +function parseOptionalJson(value?: string): Indexable | undefined { + if (!value) return undefined; + return JSON.parse(value) as Indexable; +} + +function mapBackfillRun(run: { + id: string; + schemaName: string; + configId?: string; + state: string; + cursor?: string; + batchSize: number; + onlyMissing: boolean; + filter?: string; + scannedCount: number; + queuedCount: number; + processedCount: number; + failedCount: number; + startedAt?: string; + finishedAt?: string; + error?: string; + createdAt?: string; + updatedAt?: string; +}): BackfillRunRecord { + return { + ...run, + filter: parseOptionalJson(run.filter), + }; +} + +function mapCapabilities(capabilities: { + supported: boolean; + storage: boolean; + indexing: boolean; + search: boolean; + provider: string; + reason?: string; +}): VectorCapabilities { + return { + supported: capabilities.supported, + storage: capabilities.storage, + indexing: capabilities.indexing, + search: capabilities.search, + provider: capabilities.provider as VectorCapabilities['provider'], + reason: capabilities.reason, + }; +} + export class EmbeddingsProvider extends ConduitModule< typeof EmbeddingsProviderDefinition > { @@ -36,18 +149,92 @@ export class EmbeddingsProvider extends ConduitModule< this.initializeClient(EmbeddingsProviderDefinition); } - upsertConfig(config: EmbeddingConfigInput): Promise { - return this.client!.upsertConfig(config).then(res => res.result); + upsertConfig( + config: EmbeddingConfigInput, + ): Promise<{ config: EmbeddingConfigRecord; warnings: string[] }> { + return this.client!.upsertConfig(config).then(res => ({ + config: res.config!, + warnings: res.warnings, + })); } - getConfigs(): Promise { - return this.client!.getConfigs({}).then(res => JSON.parse(res.result)); + getConfigs(query?: { + schemaName?: string; + id?: string; + }): Promise { + return this.client!.getConfigs(query ?? {}).then(res => res.configs); } - startBackfill(schemaName: string, batchSize?: number): Promise<{ queued: number }> { - return this.client!.startBackfill({ schemaName, batchSize }).then(res => - JSON.parse(res.result), - ); + deleteConfig(query: { + id?: string; + schemaName?: string; + targetField?: string; + }): Promise { + return this.client!.deleteConfig(query).then(res => res.config!); + } + + getCapabilities(schemaName?: string): Promise<{ + capabilities: VectorCapabilities; + warnings: string[]; + }> { + return this.client!.getCapabilities({ schemaName }).then(res => ({ + capabilities: mapCapabilities(res.capabilities!), + warnings: res.warnings, + })); + } + + getStatus(schemaName?: string): Promise { + return this.client!.getStatus({ schemaName }).then(res => ({ + enabled: res.enabled, + ready: res.ready, + capabilities: mapCapabilities(res.capabilities!), + generationQueue: res.generationQueue!, + backfillQueue: res.backfillQueue!, + warnings: res.warnings, + })); + } + + startBackfill(input: StartBackfillInput): Promise<{ + queued: number; + runs: BackfillRunRecord[]; + warnings: string[]; + }> { + return this.client!.startBackfill({ + schemaName: input.schemaName, + batchSize: input.batchSize, + configId: input.configId, + onlyMissing: input.onlyMissing, + filter: input.filter ? JSON.stringify(input.filter) : undefined, + }).then(res => ({ + queued: res.queued, + runs: res.runs.map(mapBackfillRun), + warnings: res.warnings, + })); + } + + getBackfill(id: string): Promise { + return this.client!.getBackfill({ id }).then(mapBackfillRun); + } + + listBackfills(query?: { + schemaName?: string; + state?: string; + configId?: string; + skip?: number; + limit?: number; + }): Promise<{ runs: BackfillRunRecord[]; count: number }> { + return this.client!.listBackfills(query ?? {}).then(res => ({ + runs: res.runs.map(mapBackfillRun), + count: res.count, + })); + } + + cancelBackfill(id: string): Promise { + return this.client!.cancelBackfill({ id }).then(res => mapBackfillRun(res.run!)); + } + + resumeBackfill(id: string): Promise { + return this.client!.resumeBackfill({ id }).then(res => mapBackfillRun(res.run!)); } semanticSearch( @@ -62,6 +249,14 @@ export class EmbeddingsProvider extends ConduitModule< userId: input.userId, scope: input.scope, adminOperator: input.adminOperator, - }).then(res => JSON.parse(res.result)); + }).then(res => + res.hits.map(hit => ({ + document: JSON.parse(hit.document) as T, + score: hit.score, + distance: hit.distance, + metric: hit.metric as VectorSearchResult['metric'], + provider: hit.provider as VectorSearchResult['provider'], + })), + ); } } diff --git a/modules/embeddings/README.md b/modules/embeddings/README.md index cdd112e81..96365846e 100644 --- a/modules/embeddings/README.md +++ b/modules/embeddings/README.md @@ -16,7 +16,8 @@ provider: "providers": { "openai-compatible": { "endpoint": "https://api.openai.com/v1/embeddings", - "apiKey": "..." + "apiKey": "...", + "allowedHosts": ["api.openai.com"] } }, "queue": { @@ -29,13 +30,33 @@ provider: ## Workflow 1. Create an embedding config with `schemaName`, `sourceFields`, `targetField`, - `provider`, `model`, and `dimensions`. + `provider`, `model`, and `dimensions`. Save `enabled: false` until Database + reports vector storage and a queryable index. 2. The module adds a vector schema extension for the target field and a source hash field used to skip unchanged documents. 3. Start a backfill, or rely on database create/update events to enqueue - incremental embedding jobs. + incremental embedding jobs. Backfills persist `BackfillRun` state and can be + canceled or resumed from the stored cursor. 4. Use `semanticSearch` to generate a query embedding and delegate search to the Database module. Provider output dimensions must match the configured vector dimensions. Mismatches fail before vectors are written or searched. + +## Admin and MCP + +Operator-only Admin routes are registered under `/embeddings/*` and become MCP +tools through Hermes: + +- `GET /embeddings/configs` +- `POST /embeddings/configs` +- `GET /embeddings/capabilities` +- `GET /embeddings/status` +- `POST /embeddings/backfills` +- `POST /embeddings/backfills/:id/cancel` +- `POST /embeddings/backfills/:id/resume` +- `POST /embeddings/search` + +Config and backfill APIs are never exposed as client routes. Client +`POST /embeddings/search` accepts text only and takes user/scope from the +authenticated router context. diff --git a/modules/embeddings/package.json b/modules/embeddings/package.json index 732148a00..525e27b15 100644 --- a/modules/embeddings/package.json +++ b/modules/embeddings/package.json @@ -21,7 +21,7 @@ "build": "rimraf dist && tsc", "postbuild": "copyfiles -u 1 src/**/*.proto src/*.proto ./dist/", "generateTypes": "sh build.sh", - "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utils/*.test.js dist-test/controllers/*.test.js dist-test/providers/*.test.js" + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utils/*.test.js dist-test/controllers/*.test.js dist-test/providers/*.test.js dist-test/api/*.test.js test/embedding-contract.test.mjs" }, "dependencies": { "@bufbuild/protobuf": "^2.10.2", diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts index 7e59b9325..d366d87d9 100644 --- a/modules/embeddings/src/Embeddings.ts +++ b/modules/embeddings/src/Embeddings.ts @@ -3,24 +3,20 @@ import { fileURLToPath } from 'node:url'; import { ConduitGrpcSdk, DatabaseProvider, - GrpcError, GrpcRequest, GrpcResponse, HealthCheckStatus, - TYPE, } from '@conduitplatform/grpc-sdk'; import { ConfigController, ConduitActiveSchema, ManagedModule, } from '@conduitplatform/module-tools'; -import { status } from '@grpc/grpc-js'; import AppConfigSchema, { Config } from './config/index.js'; import * as models from './models/index.js'; import { BackfillRun, EmbeddingConfig } from './models/index.js'; import { QueueController } from './controllers/queue.controller.js'; import { getProvider, hashEmbeddingInput } from './providers/index.js'; -import { validateEmbeddingConfigInput } from './utils/validateEmbeddingConfig.js'; import { embeddingOwnedFields, isEmbeddingOwnedMutation, @@ -35,12 +31,6 @@ import { parseEmbeddingJobData, type EmbeddingJobData, } from './utils/embeddingJobs.js'; -import { - assertCanManageEmbeddingConfig, - assertEmbeddingTargetSchema, - assertSemanticSearchAccess, - resolveAdminOperatorContext, -} from './utils/schemaPolicy.js'; import { assertGrpcKeyRequirement, callerModuleName, @@ -51,18 +41,35 @@ import { backfillRunFromDocument, persistableBackfillRun, processBackfillControllerJob, - queueBackfillRuns, type BackfillControllerJobData, } from './utils/backfillExecution.js'; -import { BackfillGateError, grpcErrorFromBackfillGate } from './utils/backfillGates.js'; import { incrementEmbeddingMetric } from './utils/embeddingMetrics.js'; import metricsSchema from './metrics/index.js'; +import { EmbeddingsApi } from './api/embeddingsApi.js'; +import { AdminHandlers } from './admin/index.js'; +import { EmbeddingsRoutes } from './routes/index.js'; import { - BackfillRequest, - EmbeddingConfigRequest, - EmbeddingConfigResponse, - EmbeddingsQueryResponse, + CancelBackfillRequest, + DeleteEmbeddingConfigRequest, + DeleteEmbeddingConfigResponse, + GetBackfillRequest, + GetCapabilitiesRequest, + GetCapabilitiesResponse, + GetConfigsRequest, + GetConfigsResponse, + GetStatusRequest, + GetStatusResponse, + ListBackfillsRequest, + ListBackfillsResponse, + ResumeBackfillRequest, SemanticSearchRequest, + SemanticSearchResponse, + StartBackfillRequest, + StartBackfillResponse, + UpsertConfigRequest, + UpsertConfigResponse, + BackfillMutationResponse, + BackfillRun as BackfillRunMessage, } from './protoTypes/embeddings.js'; const __filename = fileURLToPath(import.meta.url); @@ -77,7 +84,14 @@ export default class EmbeddingsModule extends ManagedModule { functions: { upsertConfig: this.upsertConfig.bind(this), getConfigs: this.getConfigs.bind(this), + deleteConfig: this.deleteConfig.bind(this), + getCapabilities: this.getCapabilities.bind(this), + getStatus: this.getStatus.bind(this), startBackfill: this.startBackfill.bind(this), + getBackfill: this.getBackfill.bind(this), + listBackfills: this.listBackfills.bind(this), + cancelBackfill: this.cancelBackfill.bind(this), + resumeBackfill: this.resumeBackfill.bind(this), semanticSearch: this.semanticSearch.bind(this), }, }; @@ -85,6 +99,10 @@ export default class EmbeddingsModule extends ManagedModule { private database: DatabaseProvider; private queueController: QueueController; private subscribedSchemas = new Map(); + private api: EmbeddingsApi; + private adminRouter?: AdminHandlers; + private clientRouter?: EmbeddingsRoutes; + private routerWatchDispose?: () => void; constructor(peerManifestRoot?: string) { super('embeddings', peerManifestRoot); @@ -97,6 +115,8 @@ export default class EmbeddingsModule extends ManagedModule { this.database = this.grpcSdk.database!; await this.registerSchemas(); this.queueController = QueueController.getInstance(this.grpcSdk); + this.api = this.createApi(); + this.adminRouter = new AdminHandlers(this.grpcServer, this.grpcSdk, this.api); await this.configureRuntime(); this.updateHealth(HealthCheckStatus.SERVING); } @@ -111,179 +131,216 @@ export default class EmbeddingsModule extends ManagedModule { await this.configureRuntime(); } + async onRegister() { + this.routerWatchDispose = this.grpcSdk.watchPeer( + 'router', + serving => { + if (serving) void this.ensureClientRoutes(); + }, + { edge: 'rising', syncInitialState: true }, + ); + } + async upsertConfig( - call: GrpcRequest, - callback: GrpcResponse, + call: GrpcRequest, + callback: GrpcResponse, ) { try { - const schema = await this.database.getSchema(call.request.schemaName); - const declared = await this.declaredSchema(call.request.schemaName); - assertEmbeddingTargetSchema({ - name: schema.name, - ownerModule: declared?.ownerModule, - }); - assertCanManageEmbeddingConfig({ + const result = await this.api.upsertConfig(call.request, { callerModule: callerModuleName(call.metadata), - ownerModule: declared?.ownerModule, - schemaName: schema.name, }); - const { sourceFieldAllowlist: _allowlist, ...persisted } = - validateEmbeddingConfigInput( - { - ...call.request, - sourceFieldAllowlist: [ - ...(this.currentConfig().security.sourceFieldAllowlist ?? []), - ...(call.request.sourceFieldAllowlist ?? []), - ], - }, - { provider: this.currentConfig().defaultProvider }, - schema.fields, - ); - await this.database.setSchemaExtension({ - schemaName: persisted.schemaName, - fields: { - [persisted.targetField]: { - type: TYPE.Vector, - dimensions: persisted.dimensions, - similarity: persisted.similarity, - select: false, - }, - [`${persisted.targetField}SourceHash`]: { - type: TYPE.String, - required: false, - select: false, - }, - }, + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async getConfigs( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.getConfigs(call.request, { + callerModule: callerModuleName(call.metadata), }); - const model = EmbeddingConfig.getInstance(); - const existing = await model.findOne({ - schemaName: persisted.schemaName, - targetField: persisted.targetField, + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async deleteConfig( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.deleteConfig(call.request, { + callerModule: callerModuleName(call.metadata), }); - if (existing) { - await model.findByIdAndUpdate(existing._id, persisted); - } else { - await model.create({ ...persisted, enabled: true }); - } - if (this.currentConfig().enabled) { - this.subscribeToSchema(persisted.schemaName); - } - callback(null, { result: 'Embedding config saved' }); + callback(null, result); } catch (err) { - callback(this.grpcError(err)); + callback(this.api.mapGrpcError(err)); } } - async getConfigs( - _call: GrpcRequest, - callback: GrpcResponse, + async getCapabilities( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.getCapabilities(call.request.schemaName); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async getStatus( + call: GrpcRequest, + callback: GrpcResponse, ) { try { - const configs = await EmbeddingConfig.getInstance().findMany({}); - callback(null, { result: JSON.stringify(configs) }); + const result = await this.api.getStatus(call.request.schemaName); + callback(null, result); } catch (err) { - callback(this.grpcError(err)); + callback(this.api.mapGrpcError(err)); } } async startBackfill( - call: GrpcRequest, - callback: GrpcResponse, + call: GrpcRequest, + callback: GrpcResponse, ) { try { - const schema = await this.database.getSchema(call.request.schemaName); - const declared = await this.declaredSchema(call.request.schemaName); - assertEmbeddingTargetSchema({ - name: schema.name, - ownerModule: declared?.ownerModule, + const result = await this.api.startBackfill(call.request, { + callerModule: callerModuleName(call.metadata), }); - assertCanManageEmbeddingConfig({ + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async getBackfill( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.getBackfill(call.request.id, { callerModule: callerModuleName(call.metadata), - ownerModule: declared?.ownerModule, - schemaName: schema.name, }); - const [configs, capabilities, indexes] = await Promise.all([ - EmbeddingConfig.getInstance().findMany({ - schemaName: call.request.schemaName, - enabled: true, - }), - this.database.getVectorCapabilities(call.request.schemaName), - this.database.getVectorIndexes(call.request.schemaName), - ]); - const queued = await queueBackfillRuns( - { - schemaName: call.request.schemaName, - batchSize: call.request.batchSize, - maxBatchSize: this.currentConfig().queue.maxBatchSize ?? MAX_QUEUE_BATCH_SIZE, - }, - { - moduleEnabled: this.currentConfig().enabled, - capabilities, - configs, - indexes, - createRun: async run => { - const created = await BackfillRun.getInstance().create( - persistableBackfillRun(run), - ); - return { _id: created._id }; - }, - enqueueController: job => this.queueController.addBackfillControllerJob(job), - }, - ); - callback(null, { result: JSON.stringify(queued) }); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async listBackfills( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.listBackfills(call.request, { + callerModule: callerModuleName(call.metadata), + }); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async cancelBackfill( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.cancelBackfill(call.request.id, { + callerModule: callerModuleName(call.metadata), + }); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async resumeBackfill( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.resumeBackfill(call.request.id, { + callerModule: callerModuleName(call.metadata), + }); + callback(null, result); } catch (err) { - callback(this.grpcError(err)); + callback(this.api.mapGrpcError(err)); } } async semanticSearch( call: GrpcRequest, - callback: GrpcResponse, + callback: GrpcResponse, ) { try { - const adminOperator = resolveAdminOperatorContext({ - requested: call.request.adminOperator, + const result = await this.api.semanticSearch(call.request, { callerModule: callerModuleName(call.metadata), }); - const schema = await this.database.getSchema(call.request.schemaName); - if (schema.modelOptions?.conduit?.authorization?.enabled) { - assertSemanticSearchAccess({ - userId: call.request.userId, - scope: call.request.scope, - adminOperator, - }); - } - const config = await this.resolveConfig( - call.request.schemaName, - call.request.targetField, - ); - const providerConfig = this.providerConfig(config.provider, config.modelName); - const vector = await getProvider(config.provider).embed( - call.request.text, - providerConfig, - ); - if (vector.length !== config.dimensions) { - throw new GrpcError( - status.FAILED_PRECONDITION, - `Embedding provider returned ${vector.length} dimensions; expected ${config.dimensions}`, - ); - } - const results = await this.database.vectorSearch({ - schemaName: call.request.schemaName, - field: config.targetField, - vector, - filter: call.request.filter ? JSON.parse(call.request.filter) : undefined, - limit: call.request.limit, - userId: call.request.userId, - scope: call.request.scope, - adminOperator, - }); - callback(null, { result: JSON.stringify(results) }); + callback(null, result); } catch (err) { - callback(this.grpcError(err)); + callback(this.api.mapGrpcError(err)); } } + private async ensureClientRoutes() { + if (!this.api || !this.grpcSdk.router) return; + this.clientRouter ??= new EmbeddingsRoutes(this.grpcServer, this.grpcSdk, this.api); + await this.clientRouter.registerRoutes(); + } + + private createApi() { + return new EmbeddingsApi({ + currentConfig: () => this.currentConfig(), + getSchema: schemaName => this.database.getSchema(schemaName), + declaredSchema: schemaName => this.declaredSchema(schemaName), + setSchemaExtension: extension => this.database.setSchemaExtension(extension), + getVectorCapabilities: schemaName => + this.database.getVectorCapabilities(schemaName), + getVectorIndexes: schemaName => this.database.getVectorIndexes(schemaName), + vectorSearch: input => this.database.vectorSearch(input), + configs: { + findMany: query => EmbeddingConfig.getInstance().findMany(query), + findOne: query => EmbeddingConfig.getInstance().findOne(query), + create: doc => EmbeddingConfig.getInstance().create(doc), + findByIdAndUpdate: (id, doc) => + EmbeddingConfig.getInstance().findByIdAndUpdate(id, doc), + deleteOne: query => EmbeddingConfig.getInstance().deleteOne(query), + }, + backfills: { + findMany: (query, options) => BackfillRun.getInstance().findMany(query, options), + findOne: query => BackfillRun.getInstance().findOne(query), + countDocuments: query => BackfillRun.getInstance().countDocuments(query), + create: doc => BackfillRun.getInstance().create(doc), + findByIdAndUpdate: (id, doc) => + BackfillRun.getInstance().findByIdAndUpdate(id, doc), + }, + getQueueStatus: () => this.queueController.getQueueStatus(), + enqueueBackfill: job => this.queueController.addBackfillControllerJob(job), + embed: (input, provider, model) => + getProvider(provider).embed(input, this.providerConfig(provider, model)), + onConfigChanged: async schemaName => { + const enabled = await EmbeddingConfig.getInstance().findMany({ + schemaName, + enabled: true, + }); + if (this.currentConfig().enabled && enabled.length) { + this.subscribeToSchema(schemaName); + } else { + this.unsubscribeFromSchema(schemaName); + } + }, + }); + } + private async configureRuntime() { const config = this.currentConfig(); this.queueController ??= QueueController.getInstance(this.grpcSdk); @@ -522,17 +579,6 @@ export default class EmbeddingsModule extends ManagedModule { ); } - private grpcError(err: unknown) { - if (err instanceof BackfillGateError) { - const mapped = grpcErrorFromBackfillGate(err); - return { code: mapped.code, message: sanitizeErrorMessage(mapped) }; - } - if (err instanceof GrpcError) { - return { code: err.code, message: sanitizeErrorMessage(err) }; - } - return { code: status.INTERNAL, message: sanitizeErrorMessage(err) }; - } - private providerConfig(provider: string, model: string) { const config = this.currentConfig(); const providers = config.providers as Record>; @@ -554,19 +600,6 @@ export default class EmbeddingsModule extends ManagedModule { }; } - private async resolveConfig(schemaName: string, targetField?: string) { - const query: Record = { schemaName, enabled: true }; - if (targetField) query.targetField = targetField; - const config = await EmbeddingConfig.getInstance().findOne(query); - if (!config) { - throw new GrpcError( - status.NOT_FOUND, - 'No embedding config found for semantic search', - ); - } - return config; - } - private currentConfig() { return ConfigController.getInstance().config as Config; } diff --git a/modules/embeddings/src/admin/index.ts b/modules/embeddings/src/admin/index.ts new file mode 100644 index 000000000..3a9f45f93 --- /dev/null +++ b/modules/embeddings/src/admin/index.ts @@ -0,0 +1,334 @@ +import { + ConduitGrpcSdk, + ConduitRouteActions, + ConduitRouteReturnDefinition, + ParsedRouterRequest, + TYPE, + UnparsedRouterResponse, +} from '@conduitplatform/grpc-sdk'; +import { + ConduitBoolean, + ConduitJson, + ConduitNumber, + ConduitString, + GrpcServer, + RoutingManager, +} from '@conduitplatform/module-tools'; +import { EmbeddingsApi } from '../api/embeddingsApi.js'; +import { CONFIG_BODY, EMBEDDINGS_ADMIN_ROUTES } from './routes.js'; + +const ADMIN_CALLER = { platformAdmin: true as const }; + +export class AdminHandlers { + private readonly routingManager: RoutingManager; + + constructor( + private readonly server: GrpcServer, + private readonly grpcSdk: ConduitGrpcSdk, + private readonly api: EmbeddingsApi, + ) { + this.routingManager = new RoutingManager(this.grpcSdk.admin, this.server); + this.registerAdminRoutes(); + } + + async listConfigs(call: ParsedRouterRequest): Promise { + return this.api.getConfigs( + { + schemaName: call.request.params.schemaName, + id: call.request.params.id, + }, + ADMIN_CALLER, + ); + } + + async getConfig(call: ParsedRouterRequest): Promise { + const result = await this.api.getConfigs( + { id: call.request.params.id }, + ADMIN_CALLER, + ); + return result.configs[0]; + } + + async upsertConfig(call: ParsedRouterRequest): Promise { + const params = call.request.params as { + schemaName: string; + sourceFields: string[]; + targetField: string; + provider?: string; + model?: string; + dimensions: number; + similarity?: string; + sourceFieldAllowlist?: string[]; + enabled?: boolean; + }; + return this.api.upsertConfig(params, ADMIN_CALLER); + } + + async deleteConfig(call: ParsedRouterRequest): Promise { + return this.api.deleteConfig({ id: call.request.params.id }, ADMIN_CALLER); + } + + async getCapabilities(call: ParsedRouterRequest): Promise { + return this.api.getCapabilities(call.request.params.schemaName); + } + + async getStatus(call: ParsedRouterRequest): Promise { + return this.api.getStatus(call.request.params.schemaName); + } + + async listBackfills(call: ParsedRouterRequest): Promise { + return this.api.listBackfills( + { + schemaName: call.request.params.schemaName, + state: call.request.params.state, + configId: call.request.params.configId, + skip: call.request.params.skip, + limit: call.request.params.limit, + }, + ADMIN_CALLER, + ); + } + + async startBackfill(call: ParsedRouterRequest): Promise { + const filter = call.request.params.filter; + return this.api.startBackfill( + { + schemaName: call.request.params.schemaName, + batchSize: call.request.params.batchSize, + configId: call.request.params.configId, + onlyMissing: call.request.params.onlyMissing, + filter: + filter == null + ? undefined + : typeof filter === 'string' + ? filter + : JSON.stringify(filter), + }, + ADMIN_CALLER, + ); + } + + async getBackfill(call: ParsedRouterRequest): Promise { + return this.api.getBackfill(call.request.params.id, ADMIN_CALLER); + } + + async cancelBackfill(call: ParsedRouterRequest): Promise { + return this.api.cancelBackfill(call.request.params.id, ADMIN_CALLER); + } + + async resumeBackfill(call: ParsedRouterRequest): Promise { + return this.api.resumeBackfill(call.request.params.id, ADMIN_CALLER); + } + + async semanticSearch(call: ParsedRouterRequest): Promise { + const filter = call.request.params.filter; + const result = await this.api.semanticSearch( + { + schemaName: call.request.params.schemaName, + text: call.request.params.text, + targetField: call.request.params.targetField, + limit: call.request.params.limit, + filter: + filter == null + ? undefined + : typeof filter === 'string' + ? filter + : JSON.stringify(filter), + adminOperator: true, + }, + ADMIN_CALLER, + ); + return { + hits: result.hits.map(hit => ({ + ...hit, + document: JSON.parse(hit.document), + })), + }; + } + + private registerAdminRoutes() { + this.routingManager.clear(); + const descriptions = new Map( + EMBEDDINGS_ADMIN_ROUTES.map(route => [ + `${route.action}:${route.path}`, + route.description, + ]), + ); + this.routingManager.route( + { + path: '/configs', + action: ConduitRouteActions.GET, + description: descriptions.get(`${ConduitRouteActions.GET}:/configs`), + queryParams: { + schemaName: ConduitString.Optional, + id: ConduitString.Optional, + }, + }, + new ConduitRouteReturnDefinition('GetEmbeddingConfigs', { + configs: [ConduitJson.Required], + }), + this.listConfigs.bind(this), + ); + this.routingManager.route( + { + path: '/configs', + action: ConduitRouteActions.POST, + description: descriptions.get(`${ConduitRouteActions.POST}:/configs`), + bodyParams: CONFIG_BODY as never, + }, + new ConduitRouteReturnDefinition('UpsertEmbeddingConfig', { + config: ConduitJson.Required, + warnings: [ConduitString.Required], + }), + this.upsertConfig.bind(this), + ); + this.routingManager.route( + { + path: '/configs/:id', + action: ConduitRouteActions.GET, + description: descriptions.get(`${ConduitRouteActions.GET}:/configs/:id`), + urlParams: { id: ConduitString.Required }, + }, + new ConduitRouteReturnDefinition('GetEmbeddingConfig', TYPE.JSON), + this.getConfig.bind(this), + ); + this.routingManager.route( + { + path: '/configs/:id', + action: ConduitRouteActions.DELETE, + description: descriptions.get(`${ConduitRouteActions.DELETE}:/configs/:id`), + urlParams: { id: ConduitString.Required }, + }, + new ConduitRouteReturnDefinition('DeleteEmbeddingConfig', { + config: ConduitJson.Required, + }), + this.deleteConfig.bind(this), + ); + this.routingManager.route( + { + path: '/capabilities', + action: ConduitRouteActions.GET, + description: descriptions.get(`${ConduitRouteActions.GET}:/capabilities`), + queryParams: { schemaName: ConduitString.Optional }, + }, + new ConduitRouteReturnDefinition('GetEmbeddingCapabilities', { + capabilities: ConduitJson.Required, + warnings: [ConduitString.Required], + }), + this.getCapabilities.bind(this), + ); + this.routingManager.route( + { + path: '/status', + action: ConduitRouteActions.GET, + description: descriptions.get(`${ConduitRouteActions.GET}:/status`), + queryParams: { schemaName: ConduitString.Optional }, + }, + new ConduitRouteReturnDefinition('GetEmbeddingStatus', { + enabled: ConduitBoolean.Required, + ready: ConduitBoolean.Required, + capabilities: ConduitJson.Required, + generationQueue: ConduitJson.Required, + backfillQueue: ConduitJson.Required, + warnings: [ConduitString.Required], + }), + this.getStatus.bind(this), + ); + this.routingManager.route( + { + path: '/backfills', + action: ConduitRouteActions.GET, + description: descriptions.get(`${ConduitRouteActions.GET}:/backfills`), + queryParams: { + schemaName: ConduitString.Optional, + state: ConduitString.Optional, + configId: ConduitString.Optional, + skip: ConduitNumber.Optional, + limit: ConduitNumber.Optional, + }, + }, + new ConduitRouteReturnDefinition('ListEmbeddingBackfills', { + runs: [ConduitJson.Required], + count: ConduitNumber.Required, + }), + this.listBackfills.bind(this), + ); + this.routingManager.route( + { + path: '/backfills', + action: ConduitRouteActions.POST, + description: descriptions.get(`${ConduitRouteActions.POST}:/backfills`), + bodyParams: { + schemaName: ConduitString.Required, + batchSize: ConduitNumber.Optional, + configId: ConduitString.Optional, + onlyMissing: ConduitBoolean.Optional, + filter: ConduitJson.Optional, + }, + }, + new ConduitRouteReturnDefinition('StartEmbeddingBackfill', { + queued: ConduitNumber.Required, + runs: [ConduitJson.Required], + warnings: [ConduitString.Required], + }), + this.startBackfill.bind(this), + ); + this.routingManager.route( + { + path: '/backfills/:id', + action: ConduitRouteActions.GET, + description: descriptions.get(`${ConduitRouteActions.GET}:/backfills/:id`), + urlParams: { id: ConduitString.Required }, + }, + new ConduitRouteReturnDefinition('GetEmbeddingBackfill', TYPE.JSON), + this.getBackfill.bind(this), + ); + this.routingManager.route( + { + path: '/backfills/:id/cancel', + action: ConduitRouteActions.POST, + description: descriptions.get( + `${ConduitRouteActions.POST}:/backfills/:id/cancel`, + ), + urlParams: { id: ConduitString.Required }, + }, + new ConduitRouteReturnDefinition('CancelEmbeddingBackfill', { + run: ConduitJson.Required, + }), + this.cancelBackfill.bind(this), + ); + this.routingManager.route( + { + path: '/backfills/:id/resume', + action: ConduitRouteActions.POST, + description: descriptions.get( + `${ConduitRouteActions.POST}:/backfills/:id/resume`, + ), + urlParams: { id: ConduitString.Required }, + }, + new ConduitRouteReturnDefinition('ResumeEmbeddingBackfill', { + run: ConduitJson.Required, + }), + this.resumeBackfill.bind(this), + ); + this.routingManager.route( + { + path: '/search', + action: ConduitRouteActions.POST, + description: descriptions.get(`${ConduitRouteActions.POST}:/search`), + bodyParams: { + schemaName: ConduitString.Required, + text: ConduitString.Required, + targetField: ConduitString.Optional, + filter: ConduitJson.Optional, + limit: ConduitNumber.Optional, + }, + }, + new ConduitRouteReturnDefinition('AdminSemanticSearch', { + hits: [ConduitJson.Required], + }), + this.semanticSearch.bind(this), + ); + void this.routingManager.registerRoutes(); + } +} diff --git a/modules/embeddings/src/admin/routes.ts b/modules/embeddings/src/admin/routes.ts new file mode 100644 index 000000000..ba7ef698e --- /dev/null +++ b/modules/embeddings/src/admin/routes.ts @@ -0,0 +1,117 @@ +import { ConduitRouteActions, TYPE } from '@conduitplatform/grpc-sdk'; +import { + ConduitBoolean, + ConduitNumber, + ConduitString, +} from '@conduitplatform/module-tools'; +import { embeddingsMcpToolName, embeddingsPublicPath } from '../utils/mcpToolNames.js'; + +export interface EmbeddingsAdminRouteContract { + path: string; + action: ConduitRouteActions; + description: string; + publicPath: string; + mcpName: string; + clientExposed: false; +} + +const CONFIG_BODY = { + schemaName: ConduitString.Required, + sourceFields: { type: [TYPE.String], required: true }, + targetField: ConduitString.Required, + provider: ConduitString.Optional, + model: ConduitString.Optional, + dimensions: ConduitNumber.Required, + similarity: ConduitString.Optional, + sourceFieldAllowlist: { type: [TYPE.String], required: false }, + enabled: ConduitBoolean.Optional, +}; + +function contract( + path: string, + action: ConduitRouteActions, + description: string, +): EmbeddingsAdminRouteContract { + const publicPath = embeddingsPublicPath(path); + return { + path, + action, + description, + publicPath, + mcpName: embeddingsMcpToolName(action, publicPath), + clientExposed: false, + }; +} + +export const EMBEDDINGS_ADMIN_ROUTES: EmbeddingsAdminRouteContract[] = [ + contract( + '/configs', + ConduitRouteActions.GET, + 'Lists embedding configurations. Operator-only. Filter by schemaName or id. Never expose this as a client route.', + ), + contract( + '/configs', + ConduitRouteActions.POST, + 'Creates or updates an embedding config for a schema. Operator-only. Saving enabled=false succeeds with capability warnings; enabling requires vector storage, a queryable index, and an enabled module.', + ), + contract( + '/configs/:id', + ConduitRouteActions.GET, + 'Returns one embedding configuration by id. Operator-only.', + ), + contract( + '/configs/:id', + ConduitRouteActions.DELETE, + 'Deletes an embedding configuration by id. Operator-only. Does not drop vector fields or indexes.', + ), + contract( + '/capabilities', + ConduitRouteActions.GET, + 'Returns Database vector storage, index, and search capabilities plus readiness warnings. Operator-only.', + ), + contract( + '/status', + ConduitRouteActions.GET, + 'Returns embeddings module readiness, provider/index warnings, and generation/backfill queue counts. Operator-only.', + ), + contract( + '/backfills', + ConduitRouteActions.GET, + 'Lists persisted BackfillRun records with scanned/queued/processed/failed counts, cursor, onlyMissing, and state. Operator-only.', + ), + contract( + '/backfills', + ConduitRouteActions.POST, + 'Starts a queued cursor-based backfill. Operator-only. Supports onlyMissing, configId, and bounded batchSize. Never scans in the request thread.', + ), + contract( + '/backfills/:id', + ConduitRouteActions.GET, + 'Returns one persisted BackfillRun including counts, cursor, onlyMissing, and sanitized error. Operator-only.', + ), + contract( + '/backfills/:id/cancel', + ConduitRouteActions.POST, + 'Cancels a queued or running BackfillRun. Operator-only.', + ), + contract( + '/backfills/:id/resume', + ConduitRouteActions.POST, + 'Resumes a failed or canceled BackfillRun from its persisted cursor. Operator-only. Re-checks capability and index readiness.', + ), + contract( + '/search', + ConduitRouteActions.POST, + 'Runs operator semantic search by text. Generates a query embedding and delegates vector search to Database. Operator-only; does not accept raw vectors.', + ), +]; + +export const EMBEDDINGS_CLIENT_SEARCH_PATH = '/search'; +export const EMBEDDINGS_CLIENT_FORBIDDEN_PATHS = [ + '/configs', + '/backfills', + '/capabilities', + '/status', +]; + +export { CONFIG_BODY }; diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts new file mode 100644 index 000000000..e74ead1b7 --- /dev/null +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -0,0 +1,416 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + GrpcError, + TYPE, + VectorCapabilities, + VectorIndexStatus, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + EmbeddingsApi, + type EmbeddingsApiDeps, + type EmbeddingConfigRecord, + type BackfillRunRecord, + type SchemaInfo, +} from './embeddingsApi.js'; +import type { Config } from '../config/index.js'; +import type { QueueJobCounts } from '../controllers/queue.controller.js'; + +const articleSchema = { + name: 'Article', + fields: { title: { type: TYPE.String }, body: { type: TYPE.String } }, + modelOptions: { conduit: { authorization: { enabled: true } } }, +}; + +const readyCapabilities = { + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'mongodb' as const, +}; + +const readyIndex = { + field: 'embedding', + name: 'embedding_vector', + queryable: true, + status: VectorIndexStatus.Ready, +}; + +const moduleConfig = { + enabled: true, + defaultProvider: 'openai-compatible', + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-test', + allowedHosts: ['api.openai.com'], + }, + }, + queue: { concurrency: 1, attempts: 3, maxBatchSize: 50 }, + security: { + requireGrpcKey: false, + sourceFieldAllowlist: [], + maxMutationEventIds: 10, + embedTimeoutMs: 1000, + maxEmbedInputBytes: 1024, + maxEmbedResponseBytes: 1024, + }, +} as Config; + +function emptyCounts(): QueueJobCounts { + return { waiting: 0, active: 0, completed: 0, failed: 0, delayed: 0, paused: 0 }; +} + +function createApi(overrides?: { + configs?: EmbeddingConfigRecord[]; + runs?: BackfillRunRecord[]; + capabilities?: VectorCapabilities; + indexes?: Array<{ + field?: string; + name?: string; + queryable?: boolean; + status?: string; + }>; + schemas?: Record; + declared?: Record; + embed?: EmbeddingsApiDeps['embed']; + vectorSearch?: EmbeddingsApiDeps['vectorSearch']; + enqueue?: string[]; + config?: Config; + queue?: { generation: QueueJobCounts; backfill: QueueJobCounts }; +}) { + const configs = [...(overrides?.configs ?? [])]; + const runs = [...(overrides?.runs ?? [])]; + const enqueued = overrides?.enqueue ?? []; + const deps: EmbeddingsApiDeps = { + currentConfig: () => overrides?.config ?? moduleConfig, + getSchema: async name => { + const schema = + overrides?.schemas?.[name] ?? (name === 'Article' ? articleSchema : undefined); + if (!schema) throw new GrpcError(status.NOT_FOUND, `Schema ${name} not found`); + return schema; + }, + declaredSchema: async name => + overrides?.declared?.[name] ?? { name, ownerModule: 'database' }, + setSchemaExtension: async () => undefined, + getVectorCapabilities: async () => overrides?.capabilities ?? readyCapabilities, + getVectorIndexes: async () => overrides?.indexes ?? [readyIndex], + vectorSearch: overrides?.vectorSearch ?? (async () => []), + configs: { + findMany: async query => + configs.filter(config => + Object.entries(query).every(([key, value]) => (config as never)[key] === value), + ), + findOne: async query => + configs.find(config => + Object.entries(query).every(([key, value]) => (config as never)[key] === value), + ) ?? null, + create: async doc => { + const created = { + _id: `cfg${configs.length + 1}`, + ...doc, + } as EmbeddingConfigRecord; + configs.push(created); + return created; + }, + findByIdAndUpdate: async (id, doc) => { + const index = configs.findIndex(config => config._id === id); + if (index < 0) return null; + configs[index] = { ...configs[index], ...doc }; + return configs[index]; + }, + deleteOne: async query => { + const index = configs.findIndex(config => + Object.entries(query).every(([key, value]) => (config as never)[key] === value), + ); + if (index >= 0) configs.splice(index, 1); + }, + }, + backfills: { + findMany: async (query, options) => { + const matched = runs.filter(run => + Object.entries(query).every(([key, value]) => (run as never)[key] === value), + ); + const skip = options?.skip ?? 0; + const limit = options?.limit ?? matched.length; + return matched.slice(skip, skip + limit); + }, + findOne: async query => + runs.find(run => + Object.entries(query).every(([key, value]) => (run as never)[key] === value), + ) ?? null, + countDocuments: async query => + runs.filter(run => + Object.entries(query).every(([key, value]) => (run as never)[key] === value), + ).length, + create: async doc => { + const created = { + _id: `run${runs.length + 1}`, + ...doc, + } as BackfillRunRecord; + runs.push(created); + return { _id: created._id }; + }, + findByIdAndUpdate: async (id, doc) => { + const index = runs.findIndex(run => run._id === id); + if (index < 0) return null; + runs[index] = { ...runs[index], ...doc } as BackfillRunRecord; + return runs[index]; + }, + }, + getQueueStatus: async () => + overrides?.queue ?? { + generation: { ...emptyCounts(), waiting: 2 }, + backfill: emptyCounts(), + }, + enqueueBackfill: async job => { + enqueued.push(job.runId); + }, + embed: + overrides?.embed ?? + (async () => Array.from({ length: 3 }, (_, index) => index + 0.1)), + }; + return { api: new EmbeddingsApi(deps), configs, runs, enqueued }; +} + +const enabledConfig: EmbeddingConfigRecord = { + _id: 'cfg1', + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + modelName: 'text-embedding-3-small', + dimensions: 3, + similarity: 'cosine', + enabled: true, +}; + +describe('typed embeddings API handlers', () => { + it('upserts a typed config and refuses to enable without a queryable index', async () => { + const { api, configs } = createApi({ indexes: [] }); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + enabled: true, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && err.code === status.FAILED_PRECONDITION, + ); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + enabled: false, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(saved.config.model, 'text-embedding-3-small'); + assert.equal(typeof saved.config.id, 'string'); + assert.equal( + saved.warnings.some(warning => /not queryable/.test(warning)), + true, + ); + assert.equal(configs.length, 1); + }); + + it('gates system schemas and owner policies on config and backfill', async () => { + const { api } = createApi({ + declared: { Article: { name: 'Article', ownerModule: 'cms-app' } }, + schemas: { + Article: articleSchema, + AccessToken: { + name: 'AccessToken', + fields: { token: { type: TYPE.String } }, + }, + }, + }); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'AccessToken', + sourceFields: ['token'], + targetField: 'embedding', + model: 'm', + dimensions: 3, + enabled: false, + }, + { platformAdmin: true }, + ), + (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + await assert.rejects( + () => api.startBackfill({ schemaName: 'Article' }, { callerModule: 'chat' }), + (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + await assert.rejects( + () => api.getConfigs({}, { callerModule: 'chat' }), + (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + }); + + it('starts, lists, cancels, and resumes persisted backfill runs with onlyMissing', async () => { + const { api, runs, enqueued } = createApi({ configs: [enabledConfig] }); + const started = await api.startBackfill( + { schemaName: 'Article', onlyMissing: true, batchSize: 10 }, + { callerModule: 'database' }, + ); + assert.equal(started.queued, 1); + assert.equal(started.runs[0].onlyMissing, true); + assert.equal(started.runs[0].state, 'queued'); + assert.equal(enqueued.length, 1); + const listed = await api.listBackfills( + { schemaName: 'Article' }, + { callerModule: 'database' }, + ); + assert.equal(listed.count, 1); + const gotten = await api.getBackfill(started.runs[0].id, { + callerModule: 'database', + }); + assert.equal(gotten.queuedCount, 0); + const canceled = await api.cancelBackfill(started.runs[0].id, { + callerModule: 'database', + }); + assert.equal(canceled.run.state, 'canceled'); + const resumed = await api.resumeBackfill(started.runs[0].id, { + callerModule: 'database', + }); + assert.equal(resumed.run.state, 'queued'); + assert.equal(runs[0].state, 'queued'); + await assert.rejects( + () => api.cancelBackfill('missing', { platformAdmin: true }), + (err: unknown) => err instanceof GrpcError && err.code === status.NOT_FOUND, + ); + }); + + it('returns typed status, queue counts, and capability warnings', async () => { + const { api } = createApi({ + capabilities: { + supported: false, + storage: false, + indexing: false, + search: false, + provider: 'unsupported', + reason: 'mysql is storage-only', + }, + config: { ...moduleConfig, enabled: false }, + queue: { + generation: { ...emptyCounts(), waiting: 4, failed: 1 }, + backfill: { ...emptyCounts(), active: 1 }, + }, + }); + const statusResult = await api.getStatus(); + assert.equal(statusResult.enabled, false); + assert.equal(statusResult.ready, false); + assert.equal(statusResult.generationQueue.waiting, 4); + assert.equal(statusResult.backfillQueue.active, 1); + assert.equal( + statusResult.warnings.some(warning => /disabled/.test(warning)), + true, + ); + assert.equal( + statusResult.warnings.some(warning => /mysql is storage-only/.test(warning)), + true, + ); + const capabilities = await api.getCapabilities('Article'); + assert.equal(capabilities.capabilities.search, false); + }); + + it('runs semantic search with typed hits and fail-closed auth', async () => { + const { api } = createApi({ + configs: [enabledConfig], + vectorSearch: async input => { + assert.equal(input.userId, 'user-1'); + assert.equal(input.adminOperator, false); + assert.deepEqual(input.vector, [0.1, 1.1, 2.1]); + return [ + { + document: { _id: 'doc1', title: 'Hello' }, + score: 0.91, + distance: 0.09, + metric: VectorSimilarity.Cosine, + provider: 'mongodb', + }, + ]; + }, + }); + await assert.rejects( + () => + api.semanticSearch( + { schemaName: 'Article', text: 'hello' }, + { callerModule: 'database' }, + ), + (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + const result = await api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1' }, + { callerModule: 'database' }, + ); + assert.equal(result.hits.length, 1); + assert.equal(JSON.parse(result.hits[0].document)._id, 'doc1'); + assert.equal(result.hits[0].score, 0.91); + const mapped = api.mapGrpcError( + new GrpcError(status.FAILED_PRECONDITION, 'index not ready'), + ); + assert.equal(mapped.code, status.FAILED_PRECONDITION); + }); + + it('maps provider dimension mismatches and illegal backfill transitions to typed statuses', async () => { + const { api } = createApi({ + configs: [enabledConfig], + embed: async () => [1, 2], + }); + await assert.rejects( + () => + api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1' }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && err.code === status.FAILED_PRECONDITION, + ); + const started = await createApi({ configs: [enabledConfig] }).api.startBackfill( + { schemaName: 'Article' }, + { callerModule: 'database' }, + ); + const { api: resumeApi } = createApi({ + configs: [enabledConfig], + runs: [ + { + _id: started.runs[0].id, + schemaName: 'Article', + configId: 'cfg1', + state: 'completed', + batchSize: 10, + onlyMissing: false, + scannedCount: 1, + queuedCount: 1, + processedCount: 1, + failedCount: 0, + }, + ], + }); + await assert.rejects( + () => resumeApi.resumeBackfill(started.runs[0].id, { callerModule: 'database' }), + (err: unknown) => + err instanceof GrpcError && err.code === status.FAILED_PRECONDITION, + ); + }); +}); diff --git a/modules/embeddings/src/api/embeddingsApi.ts b/modules/embeddings/src/api/embeddingsApi.ts new file mode 100644 index 000000000..4ce407d8b --- /dev/null +++ b/modules/embeddings/src/api/embeddingsApi.ts @@ -0,0 +1,638 @@ +import { + GrpcError, + TYPE, + VectorCapabilities, + VectorSearchResult, + type ConduitModel, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { Config } from '../config/index.js'; +import { QueueJobCounts } from '../controllers/queue.controller.js'; +import { + cancelBackfillExecution, + persistableBackfillRun, + queueBackfillRuns, + resumeBackfillExecution, + type BackfillControllerJobData, + type PersistedBackfillRun, +} from '../utils/backfillExecution.js'; +import { BackfillGateError, grpcErrorFromBackfillGate } from '../utils/backfillGates.js'; +import { MAX_QUEUE_BATCH_SIZE } from '../utils/embeddingJobs.js'; +import { + assertCanManageEmbeddingConfig, + assertEmbeddingTargetSchema, + assertSemanticSearchAccess, + canManageEmbeddingConfig, + resolveAdminOperatorContext, +} from '../utils/schemaPolicy.js'; +import { validateEmbeddingConfigInput } from '../utils/validateEmbeddingConfig.js'; +import { + assertConfigActivation, + assertSearchExecutable, + capabilityWarnings, + emptyQueueCounts, + indexReadinessWarnings, + isEmbeddingsReady, + providerReadinessWarnings, + SearchGateError, + grpcErrorFromSearchGate, +} from '../utils/operationalStatus.js'; +import { + mapBackfillRun, + mapCapabilities, + mapEmbeddingConfig, + mapQueueCounts, + mapSearchHits, + parseJsonObject, + type MappedBackfillRun, + type MappedEmbeddingConfig, +} from '../utils/protoMappers.js'; +import { sanitizeErrorMessage } from '../utils/redactConfig.js'; + +export interface DeclaredSchemaInfo { + name: string; + ownerModule?: string; +} + +export interface SchemaInfo { + name: string; + fields: Record; + modelOptions?: { conduit?: { authorization?: { enabled?: boolean } } }; +} + +export interface EmbeddingConfigRecord { + _id: string; + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + modelName: string; + dimensions: number; + similarity: string; + enabled: boolean; + createdAt?: Date | string; + updatedAt?: Date | string; +} + +export interface BackfillRunRecord extends PersistedBackfillRun { + createdAt?: Date | string; + updatedAt?: Date | string; +} + +export interface ConfigStore { + findMany: (query: Record) => Promise; + findOne: (query: Record) => Promise; + create: (doc: Record) => Promise; + findByIdAndUpdate: ( + id: string, + doc: Record, + ) => Promise; + deleteOne: (query: Record) => Promise; +} + +export interface BackfillStore { + findMany: ( + query: Record, + options?: { skip?: number; limit?: number; sort?: Record }, + ) => Promise; + findOne: (query: Record) => Promise; + countDocuments: (query: Record) => Promise; + create: (doc: Record) => Promise<{ _id: string }>; + findByIdAndUpdate: ( + id: string, + doc: Record, + ) => Promise; +} + +export interface EmbeddingsApiCaller { + callerModule?: string; + platformAdmin?: boolean; +} + +export interface EmbeddingsApiDeps { + currentConfig: () => Config; + getSchema: (schemaName: string) => Promise; + declaredSchema: (schemaName: string) => Promise; + setSchemaExtension: (args: { + schemaName: string; + fields: ConduitModel; + }) => Promise; + getVectorCapabilities: (schemaName?: string) => Promise; + getVectorIndexes: ( + schemaName: string, + ) => Promise< + Array<{ field?: string; name?: string; queryable?: boolean; status?: string }> + >; + vectorSearch: (input: { + schemaName: string; + field: string; + vector: number[]; + filter?: Record; + limit?: number; + userId?: string; + scope?: string; + adminOperator?: boolean; + }) => Promise; + configs: ConfigStore; + backfills: BackfillStore; + getQueueStatus: () => Promise<{ generation: QueueJobCounts; backfill: QueueJobCounts }>; + enqueueBackfill: (job: BackfillControllerJobData) => Promise; + embed: (input: string, provider: string, model: string) => Promise; + onConfigChanged?: (schemaName: string) => Promise | void; +} + +const DEFAULT_LIST_LIMIT = 25; +const MAX_LIST_LIMIT = 100; + +export class EmbeddingsApi { + constructor(private readonly deps: EmbeddingsApiDeps) {} + + async upsertConfig( + request: { + schemaName: string; + sourceFields: string[]; + targetField: string; + provider?: string; + model?: string; + dimensions: number; + similarity?: string; + sourceFieldAllowlist?: string[]; + enabled?: boolean; + }, + caller: EmbeddingsApiCaller, + ): Promise<{ config: MappedEmbeddingConfig; warnings: string[] }> { + const schema = await this.loadTargetSchema(request.schemaName, caller); + const configDefaults = this.deps.currentConfig(); + const { sourceFieldAllowlist: _allowlist, ...persisted } = + validateEmbeddingConfigInput( + { + ...request, + sourceFieldAllowlist: [ + ...(configDefaults.security.sourceFieldAllowlist ?? []), + ...(request.sourceFieldAllowlist ?? []), + ], + }, + { provider: configDefaults.defaultProvider }, + schema.fields, + ); + const enabled = request.enabled ?? true; + const capabilities = await this.deps.getVectorCapabilities(persisted.schemaName); + const indexes = await this.deps.getVectorIndexes(persisted.schemaName); + const warnings = [ + ...capabilityWarnings(capabilities), + ...indexReadinessWarnings( + [ + { + targetField: persisted.targetField, + enabled, + schemaName: persisted.schemaName, + }, + ], + indexes, + ), + ...providerReadinessWarnings( + configDefaults.providers[persisted.provider] ?? + configDefaults.providers[configDefaults.defaultProvider], + ), + ]; + if (enabled) { + assertConfigActivation({ + moduleEnabled: configDefaults.enabled, + capabilities, + config: { + enabled, + schemaName: persisted.schemaName, + targetField: persisted.targetField, + }, + indexes, + }); + } + if (capabilities.storage) { + await this.deps.setSchemaExtension({ + schemaName: persisted.schemaName, + fields: { + [persisted.targetField]: { + type: TYPE.Vector, + dimensions: persisted.dimensions, + similarity: persisted.similarity, + select: false, + }, + [`${persisted.targetField}SourceHash`]: { + type: TYPE.String, + required: false, + select: false, + }, + }, + }); + } + const existing = await this.deps.configs.findOne({ + schemaName: persisted.schemaName, + targetField: persisted.targetField, + }); + const saved = existing + ? await this.deps.configs.findByIdAndUpdate(existing._id, { ...persisted, enabled }) + : await this.deps.configs.create({ ...persisted, enabled }); + if (!saved) { + throw new GrpcError(status.INTERNAL, 'Failed to persist embedding config'); + } + await this.deps.onConfigChanged?.(saved.schemaName); + return { config: mapEmbeddingConfig(saved), warnings }; + } + + async getConfigs( + request: { schemaName?: string; id?: string }, + caller: EmbeddingsApiCaller, + ): Promise<{ configs: MappedEmbeddingConfig[] }> { + if (request.id) { + const config = await this.requireConfig({ id: request.id }); + await this.loadTargetSchema(config.schemaName, caller); + return { configs: [mapEmbeddingConfig(config)] }; + } + if (request.schemaName) { + await this.loadTargetSchema(request.schemaName, caller); + const configs = await this.deps.configs.findMany({ + schemaName: request.schemaName, + }); + return { configs: configs.map(mapEmbeddingConfig) }; + } + this.assertOperatorListAccess(caller); + const configs = await this.deps.configs.findMany({}); + return { configs: configs.map(mapEmbeddingConfig) }; + } + + async deleteConfig( + request: { id?: string; schemaName?: string; targetField?: string }, + caller: EmbeddingsApiCaller, + ): Promise<{ config: MappedEmbeddingConfig }> { + const existing = await this.requireConfig(request); + await this.loadTargetSchema(existing.schemaName, caller); + await this.deps.configs.deleteOne({ _id: existing._id }); + await this.deps.onConfigChanged?.(existing.schemaName); + return { config: mapEmbeddingConfig(existing) }; + } + + async getCapabilities(schemaName?: string): Promise<{ + capabilities: ReturnType; + warnings: string[]; + }> { + const capabilities = await this.deps.getVectorCapabilities(schemaName); + return { + capabilities: mapCapabilities(capabilities), + warnings: capabilityWarnings(capabilities), + }; + } + + async getStatus(schemaName?: string): Promise<{ + enabled: boolean; + ready: boolean; + capabilities: ReturnType; + generationQueue: ReturnType; + backfillQueue: ReturnType; + warnings: string[]; + }> { + const config = this.deps.currentConfig(); + const capabilities = await this.deps.getVectorCapabilities(schemaName); + const queue = await this.deps.getQueueStatus().catch(() => ({ + generation: emptyQueueCounts(), + backfill: emptyQueueCounts(), + })); + const warnings = [ + ...(config.enabled ? [] : ['Embeddings module is disabled']), + ...capabilityWarnings(capabilities), + ...providerReadinessWarnings( + config.providers[config.defaultProvider] ?? Object.values(config.providers)[0], + ), + ]; + if (schemaName) { + const configs = await this.deps.configs.findMany({ schemaName, enabled: true }); + const indexes = await this.deps.getVectorIndexes(schemaName); + warnings.push(...indexReadinessWarnings(configs, indexes)); + } + return { + enabled: config.enabled, + ready: isEmbeddingsReady({ moduleEnabled: config.enabled, warnings }), + capabilities: mapCapabilities(capabilities), + generationQueue: mapQueueCounts(queue.generation), + backfillQueue: mapQueueCounts(queue.backfill), + warnings, + }; + } + + async startBackfill( + request: { + schemaName: string; + batchSize?: number; + configId?: string; + onlyMissing?: boolean; + filter?: string; + }, + caller: EmbeddingsApiCaller, + ): Promise<{ queued: number; runs: MappedBackfillRun[]; warnings: string[] }> { + await this.loadTargetSchema(request.schemaName, caller); + const filter = this.parseOptionalFilter(request.filter); + const [configs, capabilities, indexes] = await Promise.all([ + this.deps.configs.findMany({ + schemaName: request.schemaName, + ...(request.configId ? { _id: request.configId } : { enabled: true }), + }), + this.deps.getVectorCapabilities(request.schemaName), + this.deps.getVectorIndexes(request.schemaName), + ]); + const queued = await queueBackfillRuns( + { + schemaName: request.schemaName, + batchSize: request.batchSize, + configId: request.configId, + onlyMissing: request.onlyMissing, + filter, + maxBatchSize: + this.deps.currentConfig().queue.maxBatchSize ?? MAX_QUEUE_BATCH_SIZE, + }, + { + moduleEnabled: this.deps.currentConfig().enabled, + capabilities, + configs, + indexes, + createRun: async run => this.deps.backfills.create(persistableBackfillRun(run)), + enqueueController: job => this.deps.enqueueBackfill(job), + }, + ); + const runs = await Promise.all( + queued.runs.map(async item => { + const persisted = await this.deps.backfills.findOne({ _id: item.id }); + if (!persisted) { + throw new GrpcError(status.INTERNAL, 'Failed to load queued backfill run'); + } + return mapBackfillRun(persisted); + }), + ); + return { + queued: queued.queued, + runs, + warnings: [ + ...capabilityWarnings(capabilities), + ...indexReadinessWarnings(configs, indexes), + ], + }; + } + + async getBackfill(id: string, caller: EmbeddingsApiCaller): Promise { + const run = await this.requireBackfill(id); + await this.loadTargetSchema(run.schemaName, caller); + return mapBackfillRun(run); + } + + async listBackfills( + request: { + schemaName?: string; + state?: string; + configId?: string; + skip?: number; + limit?: number; + }, + caller: EmbeddingsApiCaller, + ): Promise<{ runs: MappedBackfillRun[]; count: number }> { + if (request.schemaName) { + await this.loadTargetSchema(request.schemaName, caller); + } else { + this.assertOperatorListAccess(caller); + } + const query: Record = {}; + if (request.schemaName) query.schemaName = request.schemaName; + if (request.state) query.state = request.state; + if (request.configId) query.configId = request.configId; + const skip = Math.max(0, request.skip ?? 0); + const limit = Math.min( + MAX_LIST_LIMIT, + Math.max(1, request.limit ?? DEFAULT_LIST_LIMIT), + ); + const [runs, count] = await Promise.all([ + this.deps.backfills.findMany(query, { skip, limit, sort: { createdAt: -1 } }), + this.deps.backfills.countDocuments(query), + ]); + return { runs: runs.map(mapBackfillRun), count }; + } + + async cancelBackfill( + id: string, + caller: EmbeddingsApiCaller, + ): Promise<{ run: MappedBackfillRun }> { + const existing = await this.requireBackfill(id); + await this.loadTargetSchema(existing.schemaName, caller); + const result = await cancelBackfillExecution({ + run: existing, + saveRun: async (runId, run) => { + await this.deps.backfills.findByIdAndUpdate(runId, persistableBackfillRun(run)); + }, + }); + if (!result.ok) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Backfill run '${id}' cannot be canceled from state '${existing.state}'`, + ); + } + return { run: mapBackfillRun({ ...existing, ...result.run }) }; + } + + async resumeBackfill( + id: string, + caller: EmbeddingsApiCaller, + ): Promise<{ run: MappedBackfillRun }> { + const existing = await this.requireBackfill(id); + await this.loadTargetSchema(existing.schemaName, caller); + const config = existing.configId + ? await this.deps.configs.findOne({ _id: existing.configId }) + : await this.deps.configs.findOne({ + schemaName: existing.schemaName, + enabled: true, + }); + const [capabilities, indexes] = await Promise.all([ + this.deps.getVectorCapabilities(existing.schemaName), + this.deps.getVectorIndexes(existing.schemaName), + ]); + assertConfigActivation({ + moduleEnabled: this.deps.currentConfig().enabled, + capabilities, + config: config ?? null, + indexes, + }); + const result = await resumeBackfillExecution({ + run: existing, + saveRun: async (runId, run) => { + await this.deps.backfills.findByIdAndUpdate(runId, persistableBackfillRun(run)); + }, + enqueueController: job => this.deps.enqueueBackfill(job), + }); + if (!result.ok) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Backfill run '${id}' cannot be resumed from state '${existing.state}'`, + ); + } + return { run: mapBackfillRun({ ...existing, ...result.run }) }; + } + + async semanticSearch( + request: { + schemaName: string; + text: string; + targetField?: string; + filter?: string; + limit?: number; + userId?: string; + scope?: string; + adminOperator?: boolean; + }, + caller: EmbeddingsApiCaller, + ): Promise<{ hits: ReturnType }> { + if (typeof request.text !== 'string' || request.text.trim().length === 0) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Search text is required'); + } + const adminOperator = caller.platformAdmin + ? true + : resolveAdminOperatorContext({ + requested: request.adminOperator, + callerModule: caller.callerModule, + }); + const schema = await this.deps.getSchema(request.schemaName); + const declared = await this.deps.declaredSchema(request.schemaName); + assertEmbeddingTargetSchema({ + name: schema.name, + ownerModule: declared?.ownerModule, + }); + if (schema.modelOptions?.conduit?.authorization?.enabled) { + assertSemanticSearchAccess({ + userId: request.userId, + scope: request.scope, + adminOperator, + }); + } + const config = await this.resolveEnabledConfig( + request.schemaName, + request.targetField, + ); + const [capabilities, indexes] = await Promise.all([ + this.deps.getVectorCapabilities(request.schemaName), + this.deps.getVectorIndexes(request.schemaName), + ]); + assertSearchExecutable({ + capabilities, + config, + indexes, + }); + const vector = await this.deps.embed(request.text, config.provider, config.modelName); + if (vector.length !== config.dimensions) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Embedding provider returned ${vector.length} dimensions; expected ${config.dimensions}`, + ); + } + const results = await this.deps.vectorSearch({ + schemaName: request.schemaName, + field: config.targetField, + vector, + filter: this.parseOptionalFilter(request.filter), + limit: request.limit, + userId: request.userId, + scope: request.scope, + adminOperator, + }); + return { hits: mapSearchHits(results) }; + } + + mapGrpcError(err: unknown): { code: number; message: string } { + if (err instanceof BackfillGateError) { + const mapped = grpcErrorFromBackfillGate(err); + return { code: mapped.code, message: sanitizeErrorMessage(mapped) }; + } + if (err instanceof SearchGateError) { + const mapped = grpcErrorFromSearchGate(err); + return { code: mapped.code, message: sanitizeErrorMessage(mapped) }; + } + if (err instanceof GrpcError) { + return { code: err.code, message: sanitizeErrorMessage(err) }; + } + return { code: status.INTERNAL, message: sanitizeErrorMessage(err) }; + } + + private parseOptionalFilter(filter?: string): Record | undefined { + try { + return parseJsonObject(filter, 'filter'); + } catch { + throw new GrpcError(status.INVALID_ARGUMENT, 'filter must be a JSON object'); + } + } + + private async loadTargetSchema(schemaName: string, caller: EmbeddingsApiCaller) { + const schema = await this.deps.getSchema(schemaName); + const declared = await this.deps.declaredSchema(schemaName); + assertEmbeddingTargetSchema({ + name: schema.name, + ownerModule: declared?.ownerModule, + }); + if (!caller.platformAdmin) { + assertCanManageEmbeddingConfig({ + callerModule: caller.callerModule, + ownerModule: declared?.ownerModule, + schemaName: schema.name, + }); + } + return schema; + } + + private assertOperatorListAccess(caller: EmbeddingsApiCaller) { + if (caller.platformAdmin) return; + if (canManageEmbeddingConfig({ callerModule: caller.callerModule })) return; + throw new GrpcError( + status.PERMISSION_DENIED, + 'Listing embedding resources requires the schema owner or a platform operator', + ); + } + + private async requireConfig(request: { + id?: string; + schemaName?: string; + targetField?: string; + }): Promise { + const query: Record = {}; + if (request.id) query._id = request.id; + else if (request.schemaName && request.targetField) { + query.schemaName = request.schemaName; + query.targetField = request.targetField; + } else { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'id or schemaName and targetField are required', + ); + } + const existing = await this.deps.configs.findOne(query); + if (!existing) { + throw new GrpcError(status.NOT_FOUND, 'Embedding config not found'); + } + return existing; + } + + private async requireBackfill(id: string): Promise { + if (!id) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Backfill id is required'); + } + const run = await this.deps.backfills.findOne({ _id: id }); + if (!run) { + throw new GrpcError(status.NOT_FOUND, 'Backfill run not found'); + } + return run; + } + + private async resolveEnabledConfig(schemaName: string, targetField?: string) { + const query: Record = { schemaName, enabled: true }; + if (targetField) query.targetField = targetField; + const config = await this.deps.configs.findOne(query); + if (!config) { + throw new GrpcError( + status.NOT_FOUND, + 'No embedding config found for semantic search', + ); + } + return config; + } +} diff --git a/modules/embeddings/src/embeddings.proto b/modules/embeddings/src/embeddings.proto index 18f7bc526..177cdb926 100644 --- a/modules/embeddings/src/embeddings.proto +++ b/modules/embeddings/src/embeddings.proto @@ -1,8 +1,21 @@ syntax = 'proto3'; -import "google/protobuf/empty.proto"; package embeddings; -message EmbeddingConfigRequest { +message EmbeddingConfig { + string id = 1; + string schemaName = 2; + repeated string sourceFields = 3; + string targetField = 4; + string provider = 5; + string model = 6; + int32 dimensions = 7; + string similarity = 8; + bool enabled = 9; + optional string createdAt = 10; + optional string updatedAt = 11; +} + +message UpsertConfigRequest { string schemaName = 1; repeated string sourceFields = 2; string targetField = 3; @@ -11,15 +24,134 @@ message EmbeddingConfigRequest { int32 dimensions = 6; optional string similarity = 7; repeated string sourceFieldAllowlist = 8; + optional bool enabled = 9; +} + +message UpsertConfigResponse { + EmbeddingConfig config = 1; + repeated string warnings = 2; +} + +message GetConfigsRequest { + optional string schemaName = 1; + optional string id = 2; +} + +message GetConfigsResponse { + repeated EmbeddingConfig configs = 1; +} + +message DeleteEmbeddingConfigRequest { + optional string id = 1; + optional string schemaName = 2; + optional string targetField = 3; +} + +message DeleteEmbeddingConfigResponse { + EmbeddingConfig config = 1; +} + +message VectorCapabilities { + bool supported = 1; + bool storage = 2; + bool indexing = 3; + bool search = 4; + string provider = 5; + optional string reason = 6; +} + +message GetCapabilitiesRequest { + optional string schemaName = 1; +} + +message GetCapabilitiesResponse { + VectorCapabilities capabilities = 1; + repeated string warnings = 2; +} + +message QueueCounts { + int32 waiting = 1; + int32 active = 2; + int32 completed = 3; + int32 failed = 4; + int32 delayed = 5; + int32 paused = 6; } -message EmbeddingConfigResponse { - string result = 1; +message GetStatusRequest { + optional string schemaName = 1; } -message BackfillRequest { +message GetStatusResponse { + bool enabled = 1; + bool ready = 2; + VectorCapabilities capabilities = 3; + QueueCounts generationQueue = 4; + QueueCounts backfillQueue = 5; + repeated string warnings = 6; +} + +message BackfillRun { + string id = 1; + string schemaName = 2; + optional string configId = 3; + string state = 4; + optional string cursor = 5; + int32 batchSize = 6; + bool onlyMissing = 7; + optional string filter = 8; + int32 scannedCount = 9; + int32 queuedCount = 10; + int32 processedCount = 11; + int32 failedCount = 12; + optional string startedAt = 13; + optional string finishedAt = 14; + optional string error = 15; + optional string createdAt = 16; + optional string updatedAt = 17; +} + +message StartBackfillRequest { string schemaName = 1; optional int32 batchSize = 2; + optional string configId = 3; + optional bool onlyMissing = 4; + optional string filter = 5; +} + +message StartBackfillResponse { + int32 queued = 1; + repeated BackfillRun runs = 2; + repeated string warnings = 3; +} + +message GetBackfillRequest { + string id = 1; +} + +message ListBackfillsRequest { + optional string schemaName = 1; + optional string state = 2; + optional string configId = 3; + optional int32 skip = 4; + optional int32 limit = 5; +} + +message ListBackfillsResponse { + repeated BackfillRun runs = 1; + int32 count = 2; +} + +message CancelBackfillRequest { + string id = 1; +} + +message ResumeBackfillRequest { + string id = 1; +} + +message BackfillMutationResponse { + BackfillRun run = 1; } message SemanticSearchRequest { @@ -34,13 +166,28 @@ message SemanticSearchRequest { optional bool adminOperator = 8; } -message EmbeddingsQueryResponse { - string result = 1; +message SemanticSearchHit { + string document = 1; + double score = 2; + optional double distance = 3; + optional string metric = 4; + optional string provider = 5; +} + +message SemanticSearchResponse { + repeated SemanticSearchHit hits = 1; } service EmbeddingsProvider { - rpc upsertConfig(EmbeddingConfigRequest) returns (EmbeddingConfigResponse); - rpc getConfigs(google.protobuf.Empty) returns (EmbeddingsQueryResponse); - rpc startBackfill(BackfillRequest) returns (EmbeddingsQueryResponse); - rpc semanticSearch(SemanticSearchRequest) returns (EmbeddingsQueryResponse); + rpc upsertConfig(UpsertConfigRequest) returns (UpsertConfigResponse); + rpc getConfigs(GetConfigsRequest) returns (GetConfigsResponse); + rpc deleteConfig(DeleteEmbeddingConfigRequest) returns (DeleteEmbeddingConfigResponse); + rpc getCapabilities(GetCapabilitiesRequest) returns (GetCapabilitiesResponse); + rpc getStatus(GetStatusRequest) returns (GetStatusResponse); + rpc startBackfill(StartBackfillRequest) returns (StartBackfillResponse); + rpc getBackfill(GetBackfillRequest) returns (BackfillRun); + rpc listBackfills(ListBackfillsRequest) returns (ListBackfillsResponse); + rpc cancelBackfill(CancelBackfillRequest) returns (BackfillMutationResponse); + rpc resumeBackfill(ResumeBackfillRequest) returns (BackfillMutationResponse); + rpc semanticSearch(SemanticSearchRequest) returns (SemanticSearchResponse); } diff --git a/modules/embeddings/src/routes/index.ts b/modules/embeddings/src/routes/index.ts new file mode 100644 index 000000000..e317c135a --- /dev/null +++ b/modules/embeddings/src/routes/index.ts @@ -0,0 +1,89 @@ +import { + ConduitGrpcSdk, + ConduitRouteActions, + ConduitRouteReturnDefinition, + ParsedRouterRequest, + UnparsedRouterResponse, +} from '@conduitplatform/grpc-sdk'; +import { + ConduitJson, + ConduitNumber, + ConduitString, + GrpcServer, + RoutingManager, +} from '@conduitplatform/module-tools'; +import { EmbeddingsApi } from '../api/embeddingsApi.js'; +import { EMBEDDINGS_CLIENT_FORBIDDEN_PATHS } from '../admin/routes.js'; +import { + assertClientSearchSubject, + clientSearchSubject, +} from '../utils/clientSearchContext.js'; + +export class EmbeddingsRoutes { + private readonly routingManager: RoutingManager; + + constructor( + private readonly server: GrpcServer, + private readonly grpcSdk: ConduitGrpcSdk, + private readonly api: EmbeddingsApi, + ) { + this.routingManager = new RoutingManager(this.grpcSdk.router!, this.server); + } + + static clientForbiddenPaths(): string[] { + return [...EMBEDDINGS_CLIENT_FORBIDDEN_PATHS]; + } + + async semanticSearch(call: ParsedRouterRequest): Promise { + const subject = assertClientSearchSubject(clientSearchSubject(call.request.context)); + const filter = call.request.params.filter; + const result = await this.api.semanticSearch( + { + schemaName: call.request.params.schemaName, + text: call.request.params.text, + targetField: call.request.params.targetField, + limit: call.request.params.limit, + filter: + filter == null + ? undefined + : typeof filter === 'string' + ? filter + : JSON.stringify(filter), + userId: subject.userId, + scope: subject.scope, + }, + { callerModule: 'router' }, + ); + return { + hits: result.hits.map(hit => ({ + ...hit, + document: JSON.parse(hit.document), + })), + }; + } + + async registerRoutes() { + this.routingManager.clear(); + this.routingManager.route( + { + path: '/search', + action: ConduitRouteActions.POST, + description: + 'Client semantic search by text. User and scope are taken from the authenticated router context; raw vectors, userId, scope, and adminOperator are not accepted.', + bodyParams: { + schemaName: ConduitString.Required, + text: ConduitString.Required, + targetField: ConduitString.Optional, + filter: ConduitJson.Optional, + limit: ConduitNumber.Optional, + }, + middlewares: ['authMiddleware'], + }, + new ConduitRouteReturnDefinition('ClientSemanticSearch', { + hits: [ConduitJson.Required], + }), + this.semanticSearch.bind(this), + ); + await this.routingManager.registerRoutes(); + } +} diff --git a/modules/embeddings/src/utils/clientSearchContext.test.ts b/modules/embeddings/src/utils/clientSearchContext.test.ts new file mode 100644 index 000000000..c49e98b4c --- /dev/null +++ b/modules/embeddings/src/utils/clientSearchContext.test.ts @@ -0,0 +1,25 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { assertClientSearchSubject, clientSearchSubject } from './clientSearchContext.js'; + +describe('client semantic search context', () => { + it('accepts text-only search subjects from router context and fail-closes otherwise', () => { + assert.deepEqual( + clientSearchSubject({ user: { _id: 'user-1' }, scope: 'Team:org' }), + { + userId: 'user-1', + scope: 'Team:org', + }, + ); + assert.deepEqual(clientSearchSubject({ user: { _id: 1 } }), {}); + assert.throws( + () => assertClientSearchSubject(clientSearchSubject({})), + (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + assert.doesNotThrow(() => + assertClientSearchSubject(clientSearchSubject({ user: { _id: 'user-1' } })), + ); + }); +}); diff --git a/modules/embeddings/src/utils/clientSearchContext.ts b/modules/embeddings/src/utils/clientSearchContext.ts new file mode 100644 index 000000000..69d8f6d6d --- /dev/null +++ b/modules/embeddings/src/utils/clientSearchContext.ts @@ -0,0 +1,27 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export function clientSearchSubject(context?: { + user?: { _id?: unknown }; + scope?: unknown; +}): { userId?: string; scope?: string } { + const userId = context?.user?._id; + const scope = context?.scope; + return { + ...(typeof userId === 'string' && userId.length > 0 ? { userId } : {}), + ...(typeof scope === 'string' && scope.length > 0 ? { scope } : {}), + }; +} + +export function assertClientSearchSubject(subject: { userId?: string; scope?: string }): { + userId?: string; + scope?: string; +} { + if (!subject.userId && !subject.scope) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Semantic search requires an authenticated user or scope from router context', + ); + } + return subject; +} diff --git a/modules/embeddings/src/utils/mcpToolNames.test.ts b/modules/embeddings/src/utils/mcpToolNames.test.ts new file mode 100644 index 000000000..826bfc364 --- /dev/null +++ b/modules/embeddings/src/utils/mcpToolNames.test.ts @@ -0,0 +1,73 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { ConduitRouteActions } from '@conduitplatform/grpc-sdk'; +import { embeddingsMcpToolName, embeddingsPublicPath } from './mcpToolNames.js'; +import { + EMBEDDINGS_ADMIN_ROUTES, + EMBEDDINGS_CLIENT_FORBIDDEN_PATHS, + EMBEDDINGS_CLIENT_SEARCH_PATH, +} from '../admin/routes.js'; +import { EmbeddingsRoutes } from '../routes/index.js'; + +describe('embeddings MCP tool names', () => { + it('mirrors Hermes route-to-tool naming after the module prefix', () => { + assert.equal(embeddingsPublicPath('/configs'), '/embeddings/configs'); + assert.equal( + embeddingsMcpToolName('GET', '/embeddings/configs'), + 'get_embeddings_configs', + ); + assert.equal( + embeddingsMcpToolName('POST', '/embeddings/backfills/id/cancel'), + 'post_embeddings_backfills_id_cancel', + ); + }); + + it('exposes operator /embeddings/* admin routes with descriptions and MCP names', () => { + const names = EMBEDDINGS_ADMIN_ROUTES.map(route => route.mcpName); + assert.deepEqual(names, [ + 'get_embeddings_configs', + 'post_embeddings_configs', + 'get_embeddings_configs_id', + 'delete_embeddings_configs_id', + 'get_embeddings_capabilities', + 'get_embeddings_status', + 'get_embeddings_backfills', + 'post_embeddings_backfills', + 'get_embeddings_backfills_id', + 'post_embeddings_backfills_id_cancel', + 'post_embeddings_backfills_id_resume', + 'post_embeddings_search', + ]); + for (const route of EMBEDDINGS_ADMIN_ROUTES) { + assert.equal(route.publicPath.startsWith('/embeddings/'), true); + assert.equal(route.clientExposed, false); + assert.equal(route.description.length > 20, true); + assert.match(route.description, /Operator-only/); + assert.equal(route.mcpName, embeddingsMcpToolName(route.action, route.publicPath)); + } + assert.equal( + EMBEDDINGS_ADMIN_ROUTES.some( + route => route.path === '/backfills' && route.action === ConduitRouteActions.POST, + ), + true, + ); + }); + + it('never exposes config, backfill, capabilities, or status as client routes', () => { + assert.deepEqual(EmbeddingsRoutes.clientForbiddenPaths(), [ + '/configs', + '/backfills', + '/capabilities', + '/status', + ]); + for (const path of EMBEDDINGS_CLIENT_FORBIDDEN_PATHS) { + assert.equal( + EMBEDDINGS_ADMIN_ROUTES.some( + route => route.path === path || route.path.startsWith(`${path}/`), + ), + true, + ); + } + assert.equal(EMBEDDINGS_CLIENT_SEARCH_PATH, '/search'); + }); +}); diff --git a/modules/embeddings/src/utils/mcpToolNames.ts b/modules/embeddings/src/utils/mcpToolNames.ts new file mode 100644 index 000000000..6e08496f8 --- /dev/null +++ b/modules/embeddings/src/utils/mcpToolNames.ts @@ -0,0 +1,21 @@ +export function embeddingsMcpToolName(action: string, publicPath: string): string { + const cleanPath = publicPath + .replace(/^\/admin\//, '') + .replace(/\//g, '_') + .replace(/[^a-zA-Z0-9_]/g, '') + .toLowerCase(); + return `${action.toLowerCase()}${cleanPath}`; +} + +export function embeddingsPublicPath( + routePath: string, + moduleName = 'embeddings', +): string { + if ( + routePath.startsWith(`/${moduleName}/`) || + routePath.startsWith(`/hook/${moduleName}/`) + ) { + return routePath; + } + return `/${moduleName}${routePath}`; +} diff --git a/modules/embeddings/src/utils/operationalStatus.test.ts b/modules/embeddings/src/utils/operationalStatus.test.ts new file mode 100644 index 000000000..2804f8f2f --- /dev/null +++ b/modules/embeddings/src/utils/operationalStatus.test.ts @@ -0,0 +1,114 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { VectorIndexStatus } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertSearchExecutable, + capabilityWarnings, + grpcErrorFromSearchGate, + isEmbeddingsReady, + providerReadinessWarnings, + SearchGateError, +} from './operationalStatus.js'; +import { mapBackfillRun, mapEmbeddingConfig, parseSearchHits } from './protoMappers.js'; + +describe('embeddings operational warnings and search gates', () => { + it('warns on unsupported capabilities and missing provider settings without leaking secrets', () => { + const warnings = [ + ...capabilityWarnings({ + supported: false, + storage: false, + indexing: false, + search: false, + provider: 'unsupported', + reason: 'mysql is storage-only', + }), + ...providerReadinessWarnings({ + endpoint: '', + apiKey: 'sk-secret', + allowedHosts: [], + }), + ]; + assert.equal( + warnings.some(warning => /mysql is storage-only/.test(warning)), + true, + ); + assert.equal( + warnings.some(warning => /allowlist is empty/.test(warning)), + true, + ); + assert.equal(warnings.join(' ').includes('sk-secret'), false); + assert.equal( + isEmbeddingsReady({ + moduleEnabled: false, + warnings: ['Embeddings module is disabled'], + }), + false, + ); + }); + + it('blocks semantic search when the vector index is not queryable', () => { + assert.throws( + () => + assertSearchExecutable({ + capabilities: { + supported: true, + search: true, + provider: 'mongodb', + }, + config: { _id: 'cfg1', enabled: true, targetField: 'embedding' }, + indexes: [ + { + field: 'embedding', + status: VectorIndexStatus.Pending, + queryable: false, + }, + ], + }), + (err: unknown) => + err instanceof SearchGateError && err.reason === 'index_not_queryable', + ); + const mapped = grpcErrorFromSearchGate( + new SearchGateError('index_not_queryable', 'index pending', 'pending'), + ); + assert.equal(mapped.code, status.FAILED_PRECONDITION); + }); +}); + +describe('typed proto mappers', () => { + it('maps persisted configs and backfills without JSON-string envelopes', () => { + const config = mapEmbeddingConfig({ + _id: 'cfg1', + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + modelName: 'text-embedding-3-small', + dimensions: 3, + similarity: 'cosine', + enabled: true, + createdAt: new Date('2026-01-02T00:00:00.000Z'), + }); + assert.equal(config.model, 'text-embedding-3-small'); + assert.equal(config.createdAt, '2026-01-02T00:00:00.000Z'); + const run = mapBackfillRun({ + _id: 'run1', + schemaName: 'Article', + state: 'queued', + batchSize: 10, + onlyMissing: true, + scannedCount: 0, + queuedCount: 0, + processedCount: 0, + failedCount: 0, + filter: { status: 'draft' }, + }); + assert.equal(run.onlyMissing, true); + assert.equal(run.filter, '{"status":"draft"}'); + const hits = parseSearchHits<{ _id: string }>([ + { document: '{"_id":"doc1"}', score: 0.5, metric: 'cosine', provider: 'mongodb' }, + ]); + assert.equal(hits[0].document._id, 'doc1'); + assert.equal(hits[0].score, 0.5); + }); +}); diff --git a/modules/embeddings/src/utils/operationalStatus.ts b/modules/embeddings/src/utils/operationalStatus.ts new file mode 100644 index 000000000..d30fa39bf --- /dev/null +++ b/modules/embeddings/src/utils/operationalStatus.ts @@ -0,0 +1,217 @@ +import { GrpcError, VectorCapabilities } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertBackfillExecutable, + BackfillGateError, + findTargetVectorIndex, + isEmbeddingVectorIndexQueryable, + type BackfillConfigGate, + type VectorIndexGate, +} from './backfillGates.js'; +import type { QueueJobCounts } from '../controllers/queue.controller.js'; + +export const SEARCH_GATE_REASONS = [ + 'vector_unsupported', + 'vector_search_unavailable', + 'config_not_found', + 'config_disabled', + 'index_not_queryable', +] as const; + +export type SearchGateReason = (typeof SEARCH_GATE_REASONS)[number]; + +export class SearchGateError extends Error { + readonly code = 'SEARCH_GATE' as const; + + constructor( + readonly reason: SearchGateReason, + message: string, + readonly indexStatus?: string, + ) { + super(message); + this.name = 'SearchGateError'; + } +} + +export function assertSearchExecutable(args: { + capabilities?: Pick; + config?: BackfillConfigGate | null; + indexes?: readonly VectorIndexGate[]; +}): void { + const capabilities = args.capabilities; + if (!capabilities?.supported) { + throw new SearchGateError( + 'vector_unsupported', + capabilities?.reason ?? + 'Database does not support Conduit vector search; use MongoDB Atlas Vector Search or Postgres pgvector', + ); + } + if (!capabilities.search) { + throw new SearchGateError( + 'vector_search_unavailable', + capabilities.reason ?? + `Vector search is unavailable for provider '${capabilities.provider}'`, + ); + } + if (!args.config) { + throw new SearchGateError( + 'config_not_found', + 'No enabled embedding config found for semantic search', + ); + } + if (args.config.enabled === false) { + throw new SearchGateError( + 'config_disabled', + `Embedding config '${args.config._id ?? 'unknown'}' is disabled`, + ); + } + const targetField = args.config.targetField; + if (typeof targetField !== 'string' || !targetField.length) { + throw new SearchGateError( + 'config_not_found', + 'Embedding config is missing a target vector field', + ); + } + const index = findTargetVectorIndex(args.indexes ?? [], targetField); + if (isEmbeddingVectorIndexQueryable(index)) return; + const indexStatus = index?.status ?? 'missing'; + throw new SearchGateError( + 'index_not_queryable', + `Vector index for field '${targetField}' is not queryable (status: ${indexStatus}). ` + + 'Wait until the index is ready before running semantic search.', + indexStatus, + ); +} + +export function grpcErrorFromSearchGate(err: SearchGateError): GrpcError { + switch (err.reason) { + case 'vector_unsupported': + case 'vector_search_unavailable': + case 'config_not_found': + case 'config_disabled': + case 'index_not_queryable': + return new GrpcError(status.FAILED_PRECONDITION, err.message); + default: { + const unexpected: never = err.reason; + return new GrpcError(status.INTERNAL, String(unexpected)); + } + } +} + +export function capabilityWarnings( + capabilities?: Pick< + VectorCapabilities, + 'supported' | 'storage' | 'indexing' | 'search' | 'provider' | 'reason' + >, +): string[] { + if (!capabilities) { + return ['Vector capabilities are unavailable']; + } + const warnings: string[] = []; + if (!capabilities.supported) { + warnings.push( + capabilities.reason ?? + `Vector storage is unsupported for provider '${capabilities.provider}'`, + ); + return warnings; + } + if (!capabilities.storage) { + warnings.push( + capabilities.reason ?? + `Vector storage is unavailable for provider '${capabilities.provider}'`, + ); + } + if (!capabilities.indexing) { + warnings.push( + capabilities.reason ?? + `Vector indexing is unavailable for provider '${capabilities.provider}'`, + ); + } + if (!capabilities.search) { + warnings.push( + capabilities.reason ?? + `Vector search is unavailable for provider '${capabilities.provider}'`, + ); + } + return warnings; +} + +export function indexReadinessWarnings( + configs: readonly BackfillConfigGate[], + indexes: readonly VectorIndexGate[], +): string[] { + const warnings: string[] = []; + for (const config of configs) { + const targetField = config.targetField; + if (!targetField) continue; + const index = findTargetVectorIndex(indexes, targetField); + if (isEmbeddingVectorIndexQueryable(index)) continue; + warnings.push( + `Vector index for field '${targetField}' is not queryable (status: ${index?.status ?? 'missing'})`, + ); + } + return warnings; +} + +export function providerReadinessWarnings(provider?: { + endpoint?: string; + apiKey?: string; + allowedHosts?: string[]; +}): string[] { + const warnings: string[] = []; + if (!provider?.endpoint) { + warnings.push('Embedding provider endpoint is not configured'); + } + if (!provider?.apiKey) { + warnings.push('Embedding provider API key is not configured'); + } + if (!provider?.allowedHosts?.length) { + warnings.push('Embedding provider host allowlist is empty'); + } + return warnings; +} + +export function assertConfigActivation(args: { + moduleEnabled: boolean; + capabilities?: Pick< + VectorCapabilities, + 'supported' | 'storage' | 'provider' | 'reason' + >; + config?: BackfillConfigGate | null; + indexes?: readonly VectorIndexGate[]; +}): void { + try { + assertBackfillExecutable({ + moduleEnabled: args.moduleEnabled, + capabilities: args.capabilities, + config: args.config, + indexes: args.indexes, + }); + } catch (err) { + if (err instanceof BackfillGateError && err.reason === 'index_not_queryable') { + throw new GrpcError( + status.FAILED_PRECONDITION, + `${err.message} Save the config with enabled=false until the index is ready.`, + ); + } + throw err; + } +} + +export function emptyQueueCounts(): QueueJobCounts { + return { + waiting: 0, + active: 0, + completed: 0, + failed: 0, + delayed: 0, + paused: 0, + }; +} + +export function isEmbeddingsReady(args: { + moduleEnabled: boolean; + warnings: string[]; +}): boolean { + return args.moduleEnabled && args.warnings.length === 0; +} diff --git a/modules/embeddings/src/utils/protoMappers.ts b/modules/embeddings/src/utils/protoMappers.ts new file mode 100644 index 000000000..ed3a70c50 --- /dev/null +++ b/modules/embeddings/src/utils/protoMappers.ts @@ -0,0 +1,173 @@ +import type { VectorCapabilities, VectorSearchResult } from '@conduitplatform/grpc-sdk'; +import type { QueueJobCounts } from '../controllers/queue.controller.js'; +import type { PersistedBackfillRun } from './backfillExecution.js'; + +export function toIsoString(value?: Date | string | null): string | undefined { + if (value == null) return undefined; + if (value instanceof Date) { + return Number.isNaN(value.getTime()) ? undefined : value.toISOString(); + } + if (typeof value === 'string' && value.length > 0) return value; + return undefined; +} + +export function parseJsonObject( + value: string | undefined, + field: string, +): Record | undefined { + if (value == null || value === '') return undefined; + try { + const parsed = JSON.parse(value); + if (parsed == null) return undefined; + if (typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('not-object'); + } + return parsed as Record; + } catch { + throw new Error(field); + } +} + +export interface MappedEmbeddingConfig { + id: string; + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + model: string; + dimensions: number; + similarity: string; + enabled: boolean; + createdAt?: string; + updatedAt?: string; +} + +export interface MappedBackfillRun { + id: string; + schemaName: string; + configId?: string; + state: string; + cursor?: string; + batchSize: number; + onlyMissing: boolean; + filter?: string; + scannedCount: number; + queuedCount: number; + processedCount: number; + failedCount: number; + startedAt?: string; + finishedAt?: string; + error?: string; + createdAt?: string; + updatedAt?: string; +} + +export function mapEmbeddingConfig(doc: { + _id: string; + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + modelName?: string; + dimensions: number; + similarity: string; + enabled?: boolean; + createdAt?: Date | string; + updatedAt?: Date | string; +}): MappedEmbeddingConfig { + return { + id: doc._id, + schemaName: doc.schemaName, + sourceFields: [...doc.sourceFields], + targetField: doc.targetField, + provider: doc.provider, + model: doc.modelName ?? '', + dimensions: doc.dimensions, + similarity: doc.similarity, + enabled: doc.enabled !== false, + createdAt: toIsoString(doc.createdAt), + updatedAt: toIsoString(doc.updatedAt), + }; +} + +export function mapBackfillRun( + run: PersistedBackfillRun & { createdAt?: Date | string; updatedAt?: Date | string }, +): MappedBackfillRun { + return { + id: run._id, + schemaName: run.schemaName, + ...(run.configId ? { configId: run.configId } : {}), + state: run.state, + ...(run.cursor ? { cursor: run.cursor } : {}), + batchSize: run.batchSize, + onlyMissing: run.onlyMissing === true, + ...(run.filter ? { filter: JSON.stringify(run.filter) } : {}), + scannedCount: run.scannedCount, + queuedCount: run.queuedCount, + processedCount: run.processedCount, + failedCount: run.failedCount, + startedAt: toIsoString(run.startedAt), + finishedAt: toIsoString(run.finishedAt), + ...(run.error ? { error: run.error } : {}), + createdAt: toIsoString(run.createdAt), + updatedAt: toIsoString(run.updatedAt), + }; +} + +export function mapQueueCounts(counts: QueueJobCounts) { + return { + waiting: counts.waiting, + active: counts.active, + completed: counts.completed, + failed: counts.failed, + delayed: counts.delayed, + paused: counts.paused, + }; +} + +export function mapCapabilities(capabilities: VectorCapabilities) { + return { + supported: capabilities.supported, + storage: capabilities.storage, + indexing: capabilities.indexing, + search: capabilities.search, + provider: capabilities.provider, + ...(capabilities.reason ? { reason: capabilities.reason } : {}), + }; +} + +export function mapSearchHits(results: VectorSearchResult[]): Array<{ + document: string; + score: number; + distance?: number; + metric?: string; + provider?: string; +}> { + return results.map(result => ({ + document: JSON.stringify(result.document ?? {}), + score: result.score, + ...(result.distance != null ? { distance: result.distance } : {}), + ...(result.metric ? { metric: result.metric } : {}), + ...(result.provider ? { provider: result.provider } : {}), + })); +} + +export function parseSearchHits( + hits: Array<{ + document: string; + score: number; + distance?: number; + metric?: string; + provider?: string; + }>, +): VectorSearchResult[] { + return hits.map(hit => ({ + document: JSON.parse(hit.document) as T, + score: hit.score, + ...(hit.distance != null ? { distance: hit.distance } : {}), + ...(hit.metric ? { metric: hit.metric as VectorSearchResult['metric'] } : {}), + ...(hit.provider + ? { provider: hit.provider as VectorSearchResult['provider'] } + : {}), + })); +} diff --git a/modules/embeddings/test/embedding-contract.test.mjs b/modules/embeddings/test/embedding-contract.test.mjs index c0f985e26..f51b41a3d 100644 --- a/modules/embeddings/test/embedding-contract.test.mjs +++ b/modules/embeddings/test/embedding-contract.test.mjs @@ -2,18 +2,84 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { test } from 'node:test'; -const protoSource = readFileSync(new URL('../src/embeddings.proto', import.meta.url), 'utf8'); +const protoSource = readFileSync( + new URL('../src/embeddings.proto', import.meta.url), + 'utf8', +); const readmeSource = readFileSync(new URL('../README.md', import.meta.url), 'utf8'); +const sdkSource = readFileSync( + new URL('../../../libraries/grpc-sdk/src/modules/embeddings/index.ts', import.meta.url), + 'utf8', +); -test('embeddings module exposes configuration, backfill, and semantic search RPCs', () => { - assert.match(protoSource, /rpc upsertConfig/); - assert.match(protoSource, /rpc getConfigs/); - assert.match(protoSource, /rpc startBackfill/); - assert.match(protoSource, /rpc semanticSearch/); +test('embeddings proto exposes typed config, status, backfill, and search RPCs', () => { + assert.match( + protoSource, + /rpc upsertConfig\(UpsertConfigRequest\) returns \(UpsertConfigResponse\)/, + ); + assert.match( + protoSource, + /rpc getConfigs\(GetConfigsRequest\) returns \(GetConfigsResponse\)/, + ); + assert.match( + protoSource, + /rpc deleteConfig\(DeleteEmbeddingConfigRequest\) returns \(DeleteEmbeddingConfigResponse\)/, + ); + assert.match( + protoSource, + /rpc getCapabilities\(GetCapabilitiesRequest\) returns \(GetCapabilitiesResponse\)/, + ); + assert.match( + protoSource, + /rpc getStatus\(GetStatusRequest\) returns \(GetStatusResponse\)/, + ); + assert.match( + protoSource, + /rpc startBackfill\(StartBackfillRequest\) returns \(StartBackfillResponse\)/, + ); + assert.match( + protoSource, + /rpc getBackfill\(GetBackfillRequest\) returns \(BackfillRun\)/, + ); + assert.match( + protoSource, + /rpc listBackfills\(ListBackfillsRequest\) returns \(ListBackfillsResponse\)/, + ); + assert.match( + protoSource, + /rpc cancelBackfill\(CancelBackfillRequest\) returns \(BackfillMutationResponse\)/, + ); + assert.match( + protoSource, + /rpc resumeBackfill\(ResumeBackfillRequest\) returns \(BackfillMutationResponse\)/, + ); + assert.match( + protoSource, + /rpc semanticSearch\(SemanticSearchRequest\) returns \(SemanticSearchResponse\)/, + ); + assert.doesNotMatch( + protoSource, + /message EmbeddingConfigResponse \{\n string result = 1;/, + ); + assert.doesNotMatch(protoSource, /message EmbeddingsQueryResponse/); +}); + +test('grpc-sdk embeddings client maps typed proto messages instead of JSON-string envelopes', () => { + assert.match(sdkSource, /upsertConfig\(/); + assert.match(sdkSource, /deleteConfig\(/); + assert.match(sdkSource, /getCapabilities\(/); + assert.match(sdkSource, /getStatus\(/); + assert.match(sdkSource, /getBackfill\(/); + assert.match(sdkSource, /listBackfills\(/); + assert.match(sdkSource, /cancelBackfill\(/); + assert.match(sdkSource, /resumeBackfill\(/); + assert.doesNotMatch(sdkSource, /JSON\.parse\(res\.result\)/); + assert.match(sdkSource, /JSON\.parse\(hit\.document\)/); }); test('deployment docs describe provider configuration and rollout workflow', () => { assert.match(readmeSource, /openai-compatible/); assert.match(readmeSource, /backfill/); assert.match(readmeSource, /semanticSearch/); + assert.match(readmeSource, /\/embeddings\//); }); diff --git a/modules/embeddings/tsconfig.test.json b/modules/embeddings/tsconfig.test.json index 5179fd978..fb6350ab5 100644 --- a/modules/embeddings/tsconfig.test.json +++ b/modules/embeddings/tsconfig.test.json @@ -10,7 +10,12 @@ "include": [ "src/utils/**/*.ts", "src/controllers/**/*.ts", - "src/providers/**/*.ts" + "src/providers/**/*.ts", + "src/api/**/*.ts", + "src/admin/**/*.ts", + "src/routes/**/*.ts", + "src/models/**/*.ts", + "src/config/**/*.ts" ], "exclude": [] } From 427429717f2c9fbc31755683f2a5ec94e8a71d34 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 20:14:12 +0300 Subject: [PATCH 10/29] feat(hermes): map Vector to numeric arrays and omit CMS embedding writes Teach TYPE.Vector as a finite number array in Zod, OpenAPI, GraphQL, and MCP so clients never see a phantom Vector type, and keep managed vector/hash/select:false fields out of CMS create/update bodies. --- libraries/hermes/README.mdx | 2 + libraries/hermes/package.json | 1 + libraries/hermes/src/GraphQl/GraphQlParser.ts | 2 + .../src/GraphQl/utils/SimpleTypeParamUtils.ts | 8 +- .../hermes/src/GraphQl/vectorGraphQl.test.ts | 46 +++++++ libraries/hermes/src/MCP/vectorMcp.test.ts | 51 ++++++++ .../hermes/src/Rest/SimpleTypeParamUtils.ts | 29 ++++- libraries/hermes/src/Rest/SwaggerParser.ts | 17 ++- .../hermes/src/Rest/vectorOpenApi.test.ts | 73 +++++++++++ libraries/hermes/src/classes/ConduitParser.ts | 2 +- libraries/hermes/src/classes/ParserUtils.ts | 46 +++++++ libraries/hermes/src/classes/ZodParser.ts | 24 +++- .../hermes/src/classes/vectorParser.test.ts | 83 ++++++++++++ libraries/hermes/tsconfig.json | 3 +- libraries/hermes/tsconfig.test.json | 13 ++ modules/database/README.md | 4 + .../cms/__tests__/assignableFields.test.ts | 121 ++++++++++++++++++ modules/database/src/controllers/cms/utils.ts | 64 ++++++++- 18 files changed, 567 insertions(+), 22 deletions(-) create mode 100644 libraries/hermes/src/GraphQl/vectorGraphQl.test.ts create mode 100644 libraries/hermes/src/MCP/vectorMcp.test.ts create mode 100644 libraries/hermes/src/Rest/vectorOpenApi.test.ts create mode 100644 libraries/hermes/src/classes/vectorParser.test.ts create mode 100644 libraries/hermes/tsconfig.test.json create mode 100644 modules/database/src/controllers/cms/__tests__/assignableFields.test.ts diff --git a/libraries/hermes/README.mdx b/libraries/hermes/README.mdx index c43299e68..ea2a72656 100644 --- a/libraries/hermes/README.mdx +++ b/libraries/hermes/README.mdx @@ -12,3 +12,5 @@ It is utilized by [Admin](../../packages/admin) and [Router](../../modules/route - WebSockets (via Socket.io) - Support for middleware - Auto-generated API documentation + +`TYPE.Vector` is documented and validated as an array of finite numbers (`[Number]` in GraphQL, OpenAPI `array` of `number`). It is never emitted as a referenced `Vector` schema or GraphQL type. Dimension constraints are applied in Zod/OpenAPI when `dimensions` is present; GraphQL cannot express array length. diff --git a/libraries/hermes/package.json b/libraries/hermes/package.json index 743552033..45d49f4c1 100644 --- a/libraries/hermes/package.json +++ b/libraries/hermes/package.json @@ -11,6 +11,7 @@ "scripts": { "prepublish": "npm run build", "build": "rimraf dist && tsc", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/classes/vectorParser.test.js dist-test/Rest/vectorOpenApi.test.js dist-test/GraphQl/vectorGraphQl.test.js dist-test/MCP/vectorMcp.test.js", "publish": "npm publish", "postbuild": "copyfiles -u 1 src/*.proto src/**/*.json ./dist/" }, diff --git a/libraries/hermes/src/GraphQl/GraphQlParser.ts b/libraries/hermes/src/GraphQl/GraphQlParser.ts index 2ba7c0782..138a1f615 100644 --- a/libraries/hermes/src/GraphQl/GraphQlParser.ts +++ b/libraries/hermes/src/GraphQl/GraphQlParser.ts @@ -74,6 +74,8 @@ export class GraphQlParser extends ConduitParser return 'ID'; case 'JSON': return 'JSONObject'; + case 'Vector': + return '[Number]'; default: this.requestedTypes.add(conduitType); return conduitType; diff --git a/libraries/hermes/src/GraphQl/utils/SimpleTypeParamUtils.ts b/libraries/hermes/src/GraphQl/utils/SimpleTypeParamUtils.ts index 14d4f8ddf..5bf02869f 100644 --- a/libraries/hermes/src/GraphQl/utils/SimpleTypeParamUtils.ts +++ b/libraries/hermes/src/GraphQl/utils/SimpleTypeParamUtils.ts @@ -1,4 +1,4 @@ -import { ConduitModel, Indexable } from '@conduitplatform/grpc-sdk'; +import { ConduitModel, Indexable, TYPE } from '@conduitplatform/grpc-sdk'; const GQL_PRIMITIVES = ['Number', 'Boolean', 'Date', 'String']; @@ -7,6 +7,8 @@ function extractParam(param: string, required: boolean = false) { return 'ID' + (required ? '!' : ''); } else if (param === 'JSON') { return 'JSONObject' + (required ? '!' : ''); + } else if (param === TYPE.Vector || param === 'Vector') { + return '[Number]' + (required ? '!' : ''); } else { return param + (required ? '!' : ''); } @@ -17,7 +19,9 @@ function extractArrayParam( required: boolean = false, originalParam?: any, ) { - if (GQL_PRIMITIVES.indexOf(param) !== -1) { + if (param === TYPE.Vector || param === 'Vector') { + return '[[Number]]' + (required ? '!' : ''); + } else if (GQL_PRIMITIVES.indexOf(param) !== -1) { return `[${param}]` + (required ? '!' : ''); } else if (param === 'ObjectId') { return '[ID]' + (required ? '!' : ''); diff --git a/libraries/hermes/src/GraphQl/vectorGraphQl.test.ts b/libraries/hermes/src/GraphQl/vectorGraphQl.test.ts new file mode 100644 index 000000000..78746e5da --- /dev/null +++ b/libraries/hermes/src/GraphQl/vectorGraphQl.test.ts @@ -0,0 +1,46 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { TYPE } from '@conduitplatform/grpc-sdk'; +import { GraphQlParser } from './GraphQlParser.js'; +import { processParams } from './utils/SimpleTypeParamUtils.js'; + +describe('GraphQL Vector mapping', () => { + it('renders Vector fields as [Number] and does not request a Vector type', () => { + const parser = new GraphQlParser(); + const result = parser.extractTypes( + 'Doc', + { + title: TYPE.String, + count: TYPE.Number, + embedding: { type: TYPE.Vector, dimensions: 8 }, + owner: { type: TYPE.Relation, model: 'User' }, + payload: TYPE.JSON, + }, + false, + ); + + assert.equal(parser.requestedTypes.has('Vector'), false); + assert.equal(parser.requestedTypes.has('User'), true); + assert.match(result.typeString, /embedding: \[Number]/); + assert.match(result.typeString, /title: String/); + assert.match(result.typeString, /count: Number/); + assert.match(result.typeString, /payload: JSONObject/); + assert.equal(/\btype Vector\b/.test(result.typeString), false); + assert.equal(/: Vector\b/.test(result.typeString), false); + }); + + it('maps simple Vector parameters to [Number] rather than a Vector named type', () => { + const params = processParams( + { + q: TYPE.String, + embedding: { type: TYPE.Vector, dimensions: 3, required: true }, + ids: [TYPE.ObjectId], + }, + '', + ); + assert.match(params, /q:String/); + assert.match(params, /embedding:\[Number]!/); + assert.match(params, /ids:\[ID]/); + assert.equal(params.includes('Vector'), false); + }); +}); diff --git a/libraries/hermes/src/MCP/vectorMcp.test.ts b/libraries/hermes/src/MCP/vectorMcp.test.ts new file mode 100644 index 000000000..c0066ab3a --- /dev/null +++ b/libraries/hermes/src/MCP/vectorMcp.test.ts @@ -0,0 +1,51 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { z } from 'zod'; +import { + ConduitRouteActions, + ConduitRouteReturnDefinition, + TYPE, +} from '@conduitplatform/grpc-sdk'; +import { ConduitRoute } from '../classes/index.js'; +import { ConduitRouter } from '../Router.js'; +import { RouteToToolConverter } from './RouteToTool.js'; + +describe('MCP Vector tool schemas', () => { + it('validates Vector body params as finite numeric arrays without a Vector named type', () => { + const converter = new RouteToToolConverter({} as ConduitRouter); + const route = new ConduitRoute( + { + path: '/search', + action: ConduitRouteActions.POST, + bodyParams: { + query: { type: TYPE.String, required: true }, + embedding: { type: TYPE.Vector, dimensions: 3, required: true }, + owner: { type: TYPE.Relation, model: 'User' }, + }, + }, + new ConduitRouteReturnDefinition('Search', { hits: TYPE.JSON }), + async () => ({}), + ); + + const tool = converter.convertRouteToTool(route); + const schema = z.object(tool.inputSchema); + const ok = schema.safeParse({ + query: 'hello', + embedding: [0.1, 0.2, 0.3], + owner: 'user-1', + }); + assert.equal(ok.success, true); + assert.equal( + schema.safeParse({ query: 'hello', embedding: [0.1, 0.2] }).success, + false, + ); + assert.equal( + schema.safeParse({ + query: 'hello', + embedding: [0.1, 0.2, Number.POSITIVE_INFINITY], + }).success, + false, + ); + assert.equal(JSON.stringify(tool.inputSchema).includes('Vector'), false); + }); +}); diff --git a/libraries/hermes/src/Rest/SimpleTypeParamUtils.ts b/libraries/hermes/src/Rest/SimpleTypeParamUtils.ts index eb98fde75..fd58bdc8f 100644 --- a/libraries/hermes/src/Rest/SimpleTypeParamUtils.ts +++ b/libraries/hermes/src/Rest/SimpleTypeParamUtils.ts @@ -1,4 +1,5 @@ import { ConduitModel, ConduitValidationRules, TYPE } from '@conduitplatform/grpc-sdk'; +import { ParserUtils } from '../classes/index.js'; export function applyOpenApiFieldValidation( res: Record, @@ -24,6 +25,7 @@ function extractParam( param: string, required: boolean = false, validate?: ConduitValidationRules, + sourceField?: unknown, ) { const res: Record = { type: 'string' }; switch (param) { @@ -38,6 +40,10 @@ function extractParam( case TYPE.Relation: res.type = 'string'; break; + case TYPE.Vector: + res.type = 'array'; + res.items = { type: 'number' }; + break; case 'String': case 'Number': case 'Boolean': @@ -45,6 +51,9 @@ function extractParam( break; } applyOpenApiFieldValidation(res, validate); + if (param === TYPE.Vector) { + ParserUtils.applyVectorOpenApiConstraints(res, sourceField); + } return res; } @@ -67,13 +76,13 @@ export function processSwaggerParams(paramObj: any) { let params: Record = {}; if (typeof paramObj === 'string') { - params = extractParam(paramObj); + params = extractParam(paramObj, false, undefined, paramObj); } else if (Array.isArray(paramObj)) { const elementZero = paramObj[0]; if (typeof elementZero === 'string') { params = { type: 'array', - items: { ...extractParam(elementZero, false) }, + items: { ...extractParam(elementZero, false, undefined, elementZero) }, minItems: 0, }; } else { @@ -84,7 +93,12 @@ export function processSwaggerParams(paramObj: any) { params = { type: 'array', items: { - ...extractParam(typeZero! as string, typeZeroRequired, itemValidate), + ...extractParam( + typeZero! as string, + typeZeroRequired, + itemValidate, + elementZero, + ), }, minItems: typeZeroRequired ? 1 : 0, }; @@ -94,7 +108,7 @@ export function processSwaggerParams(paramObj: any) { const typeZeroRequired = (paramObj as ConduitModel).required; const validate = (paramObj as { validate?: ConduitValidationRules }).validate; if (typeof typeZero === 'string') { - params = extractParam(typeZero, typeZeroRequired, validate); + params = extractParam(typeZero, typeZeroRequired, validate, paramObj); } else if (Array.isArray(typeZero)) { const elementZero = typeZero[0]; if (typeof elementZero === 'string') { @@ -107,7 +121,12 @@ export function processSwaggerParams(paramObj: any) { params = { type: 'array', items: { - ...extractParam(typeZeroTwo! as string, typeZeroTwoRequired, itemValidate), + ...extractParam( + typeZeroTwo! as string, + typeZeroTwoRequired, + itemValidate, + elementZero, + ), }, minItems: typeZeroTwoRequired ? 1 : 0, }; diff --git a/libraries/hermes/src/Rest/SwaggerParser.ts b/libraries/hermes/src/Rest/SwaggerParser.ts index 9464555d7..7005fcc13 100644 --- a/libraries/hermes/src/Rest/SwaggerParser.ts +++ b/libraries/hermes/src/Rest/SwaggerParser.ts @@ -5,7 +5,7 @@ import { TYPE, UntypedArray, } from '@conduitplatform/grpc-sdk'; -import { ConduitParser } from '../classes/index.js'; +import { ConduitParser, ParserUtils } from '../classes/index.js'; import { applyOpenApiFieldValidation } from './SimpleTypeParamUtils.js'; export interface ParseResult { @@ -65,6 +65,9 @@ export class SwaggerParser extends ConduitParser $ref?: string; format?: string; properties?: object; + items?: { type?: string }; + minItems?: number; + maxItems?: number; } = {}; switch (conduitType) { case TYPE.JSON: @@ -79,6 +82,10 @@ export class SwaggerParser extends ConduitParser case TYPE.Relation: res.type = 'string'; break; + case TYPE.Vector: + res.type = 'array'; + res.items = { type: 'number' }; + break; case 'String': case 'Number': case 'Boolean': @@ -129,6 +136,10 @@ export class SwaggerParser extends ConduitParser processingObject as unknown as Record, v as any, ); + ParserUtils.applyVectorOpenApiConstraints( + processingObject as unknown as Record, + sourceField, + ); } else { if (!processingObject.properties) { processingObject.properties = {}; @@ -140,6 +151,10 @@ export class SwaggerParser extends ConduitParser processingObject.properties[name] as Record, v as any, ); + ParserUtils.applyVectorOpenApiConstraints( + processingObject.properties[name] as Record, + sourceField, + ); } this.addFieldToRequired(processingObject, name, isRequired); } diff --git a/libraries/hermes/src/Rest/vectorOpenApi.test.ts b/libraries/hermes/src/Rest/vectorOpenApi.test.ts new file mode 100644 index 000000000..ca7fa50e7 --- /dev/null +++ b/libraries/hermes/src/Rest/vectorOpenApi.test.ts @@ -0,0 +1,73 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { TYPE } from '@conduitplatform/grpc-sdk'; +import { SwaggerParser } from './SwaggerParser.js'; +import { processSwaggerParams } from './SimpleTypeParamUtils.js'; + +function assertNoPhantomVector(value: unknown) { + const serialized = JSON.stringify(value); + assert.equal(serialized.includes('#/components/schemas/Vector'), false); + assert.equal(serialized.includes('"Vector"'), false); +} + +describe('OpenAPI Vector mapping', () => { + it('emits a numeric array with dimensions and never a Vector $ref', () => { + const parser = new SwaggerParser(); + const result = parser.extractTypes( + 'Doc', + { + title: TYPE.String, + embedding: { type: TYPE.Vector, dimensions: 8, required: true }, + owner: { type: TYPE.Relation, model: 'User' }, + payload: TYPE.JSON, + }, + false, + ); + + assert.equal(parser.requestedTypes.has('Vector'), false); + assert.equal(parser.requestedTypes.has('User'), true); + assertNoPhantomVector(result); + assert.deepEqual( + (result as { properties: Record }).properties.embedding, + { + type: 'array', + items: { type: 'number' }, + minItems: 8, + maxItems: 8, + }, + ); + assert.equal( + ( + (result as { properties: Record }).properties + .title as { type?: string } + ).type, + 'string', + ); + assert.equal( + ( + (result as { properties: Record }).properties + .payload as { type?: string } + ).type, + 'object', + ); + }); + + it('maps simple Vector params to numeric arrays with dimensions where present', () => { + assert.deepEqual(processSwaggerParams(TYPE.Vector), { + type: 'array', + items: { type: 'number' }, + }); + assert.deepEqual( + processSwaggerParams({ type: TYPE.Vector, dimensions: 4, required: true }), + { + type: 'array', + items: { type: 'number' }, + minItems: 4, + maxItems: 4, + }, + ); + assert.deepEqual(processSwaggerParams(TYPE.Number), { type: 'number' }); + assert.deepEqual(processSwaggerParams(TYPE.JSON), { type: 'object' }); + assertNoPhantomVector(processSwaggerParams({ type: TYPE.Vector, dimensions: 2 })); + }); +}); diff --git a/libraries/hermes/src/classes/ConduitParser.ts b/libraries/hermes/src/classes/ConduitParser.ts index a847953dd..500475634 100644 --- a/libraries/hermes/src/classes/ConduitParser.ts +++ b/libraries/hermes/src/classes/ConduitParser.ts @@ -9,7 +9,7 @@ import { } from '@conduitplatform/grpc-sdk'; import { ParserUtils } from './ParserUtils.js'; -const baseTypes = ['String', 'Number', 'Boolean', 'Date', 'ObjectId', 'JSON']; +const baseTypes = ['String', 'Number', 'Boolean', 'Date', 'ObjectId', 'JSON', 'Vector']; export abstract class ConduitParser { result!: ParseResult; diff --git a/libraries/hermes/src/classes/ParserUtils.ts b/libraries/hermes/src/classes/ParserUtils.ts index b0e217435..c1c0579f0 100644 --- a/libraries/hermes/src/classes/ParserUtils.ts +++ b/libraries/hermes/src/classes/ParserUtils.ts @@ -60,6 +60,52 @@ export class ParserUtils { return baseType === TYPE.Relation || baseType === 'Relation'; } + /** + * True for TYPE.Vector / 'Vector'. Never treat this as a named schema/reference. + */ + static isVectorTypeName(value: unknown): boolean { + return value === TYPE.Vector || value === 'Vector'; + } + + /** + * Check if a field is a Vector type (shorthand or object form). + */ + static isVectorType(field: unknown): boolean { + return ParserUtils.isVectorTypeName(ParserUtils.getBaseType(field)); + } + + /** + * Positive integer dimensions from a Vector field (or a raw dimensions value). + */ + static getVectorDimensions(fieldOrDimensions: unknown): number | undefined { + const value = + typeof fieldOrDimensions === 'number' + ? fieldOrDimensions + : typeof fieldOrDimensions === 'object' && + fieldOrDimensions !== null && + 'dimensions' in fieldOrDimensions + ? (fieldOrDimensions as { dimensions?: unknown }).dimensions + : undefined; + if (typeof value === 'number' && Number.isInteger(value) && value > 0) { + return value; + } + return undefined; + } + + /** + * OpenAPI/Swagger: constrain a numeric array to the Vector field's dimensions. + */ + static applyVectorOpenApiConstraints( + schema: Record, + sourceField?: unknown, + ): void { + if (schema.type !== 'array') return; + const dimensions = ParserUtils.getVectorDimensions(sourceField); + if (dimensions === undefined) return; + schema.minItems = dimensions; + schema.maxItems = dimensions; + } + /** * Get the model name for a Relation field */ diff --git a/libraries/hermes/src/classes/ZodParser.ts b/libraries/hermes/src/classes/ZodParser.ts index 4f553a5e9..7037065eb 100644 --- a/libraries/hermes/src/classes/ZodParser.ts +++ b/libraries/hermes/src/classes/ZodParser.ts @@ -165,7 +165,17 @@ export class ZodParser { ); } - private getZodType(conduitType: TYPE): z.ZodTypeAny { + private vectorZodType(sourceField?: unknown): z.ZodTypeAny { + const item = this.useCoercion ? z.coerce.number().finite() : z.number().finite(); + let arrayType = z.array(item); + const dimensions = ParserUtils.getVectorDimensions(sourceField); + if (dimensions !== undefined) { + arrayType = arrayType.length(dimensions); + } + return arrayType; + } + + private getZodType(conduitType: TYPE, sourceField?: unknown): z.ZodTypeAny { switch (conduitType) { case TYPE.String: return z.string(); @@ -181,6 +191,8 @@ export class ZodParser { return this.conduitJsonZodType(); case TYPE.Relation: return z.string(); + case TYPE.Vector: + return this.vectorZodType(sourceField); default: return z.any(); } @@ -194,7 +206,7 @@ export class ZodParser { if (typeof fields === 'string') { const t = fields as TYPE; - let zodType = this.getZodType(t); + let zodType = this.getZodType(t, fields); if (t === TYPE.JSON) { zodType = this.finalizeJsonRouteParam(zodType, true); } @@ -218,7 +230,7 @@ export class ZodParser { if (typeof field === 'string') { const t = field as TYPE; - let zodType = this.getZodType(t); + let zodType = this.getZodType(t, field); if (t === TYPE.JSON) { zodType = this.finalizeJsonRouteParam(zodType, !isRequired); } else if (!isRequired) { @@ -252,13 +264,13 @@ export class ZodParser { let itemType: z.ZodTypeAny; if (typeof firstItem === 'string') { - itemType = this.getZodType(firstItem as TYPE); + itemType = this.getZodType(firstItem as TYPE, firstItem); } else if (typeof firstItem === 'object' && firstItem !== null) { if (firstItem.type) { if (firstItem.type === TYPE.Relation) { itemType = z.string(); } else if (typeof firstItem.type === 'string') { - itemType = this.getZodType(firstItem.type as TYPE); + itemType = this.getZodType(firstItem.type as TYPE, firstItem); } else { const nestedResult = this.extractTypesInternal( fieldName, @@ -316,7 +328,7 @@ export class ZodParser { zodType = z.string(); } else if (typeof field.type === 'string') { const t = field.type as TYPE; - zodType = this.getZodType(t); + zodType = this.getZodType(t, field); if (t === TYPE.JSON) { zodType = this.finalizeJsonRouteParam(zodType, !isRequired); } diff --git a/libraries/hermes/src/classes/vectorParser.test.ts b/libraries/hermes/src/classes/vectorParser.test.ts new file mode 100644 index 000000000..988647980 --- /dev/null +++ b/libraries/hermes/src/classes/vectorParser.test.ts @@ -0,0 +1,83 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { TYPE } from '@conduitplatform/grpc-sdk'; +import { ParserUtils } from './ParserUtils.js'; +import { ZodParser } from './ZodParser.js'; + +const vectorField = { + type: TYPE.Vector, + dimensions: 3, + required: true, +}; + +describe('ParserUtils Vector helpers', () => { + it('recognizes Vector shorthand and object form without treating it as a relation', () => { + assert.equal(ParserUtils.isVectorTypeName(TYPE.Vector), true); + assert.equal(ParserUtils.isVectorTypeName('Vector'), true); + assert.equal(ParserUtils.isVectorType(TYPE.Vector), true); + assert.equal(ParserUtils.isVectorType(vectorField), true); + assert.equal(ParserUtils.isVectorType(TYPE.JSON), false); + assert.equal(ParserUtils.isRelationType(vectorField), false); + assert.equal(ParserUtils.getVectorDimensions(vectorField), 3); + assert.equal(ParserUtils.getVectorDimensions({ type: TYPE.Vector }), undefined); + assert.equal(ParserUtils.getVectorDimensions({ dimensions: 1.5 }), undefined); + }); +}); + +describe('ZodParser Vector validation', () => { + const parser = new ZodParser(); + + it('accepts finite numeric arrays of the declared dimensions', () => { + const schema = parser.buildZodSchema({ + title: TYPE.String, + count: TYPE.Number, + embedding: vectorField, + }); + const parsed = schema.parse({ + title: 'doc', + count: 2, + embedding: [0.1, 0.2, 0.3], + }); + assert.deepEqual(parsed.embedding, [0.1, 0.2, 0.3]); + assert.equal(parsed.title, 'doc'); + assert.equal(parsed.count, 2); + }); + + it('rejects non-finite values, wrong length, and shorthand still as a numeric array', () => { + const schema = parser.buildZodSchema({ + embedding: vectorField, + raw: TYPE.Vector, + }); + assert.equal(schema.safeParse({ embedding: [0.1, 0.2] }).success, false); + assert.equal(schema.safeParse({ embedding: [0.1, 0.2, Number.NaN] }).success, false); + assert.equal( + schema.safeParse({ embedding: [0.1, 0.2, Number.POSITIVE_INFINITY] }).success, + false, + ); + const shorthand = schema.safeParse({ embedding: [1, 2, 3], raw: [1, 2, 3, 4] }); + assert.equal(shorthand.success, true); + assert.equal( + schema.safeParse({ embedding: [1, 2, 3], raw: 'Vector' }).success, + false, + ); + }); + + it('preserves existing non-vector types including JSON and Relation', () => { + const schema = parser.buildZodSchema({ + name: TYPE.String, + active: TYPE.Boolean, + created: TYPE.Date, + owner: { type: TYPE.Relation, model: 'User', required: true }, + meta: TYPE.JSON, + }); + const parsed = schema.parse({ + name: 'ok', + active: true, + created: '2026-01-01T00:00:00.000Z', + owner: '507f1f77bcf86cd799439011', + meta: { a: 1 }, + }); + assert.equal(parsed.owner, '507f1f77bcf86cd799439011'); + assert.deepEqual(parsed.meta, { a: 1 }); + }); +}); diff --git a/libraries/hermes/tsconfig.json b/libraries/hermes/tsconfig.json index cd32b1ca8..b15700386 100644 --- a/libraries/hermes/tsconfig.json +++ b/libraries/hermes/tsconfig.json @@ -65,5 +65,6 @@ /* Advanced Options */ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ - } + }, + "exclude": ["node_modules", "dist", "dist-test", "**/*.test.ts"] } diff --git a/libraries/hermes/tsconfig.test.json b/libraries/hermes/tsconfig.test.json new file mode 100644 index 000000000..99f8d1331 --- /dev/null +++ b/libraries/hermes/tsconfig.test.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "removeComments": false, + "types": ["node"] + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "dist-test"] +} diff --git a/modules/database/README.md b/modules/database/README.md index adbff147f..4eced2d94 100644 --- a/modules/database/README.md +++ b/modules/database/README.md @@ -39,3 +39,7 @@ responses distinguish storage support from index/search support: filter fields. 4. Backfill embeddings. 5. Run `vectorSearch` with a query vector. + +CMS create/update bodies omit `TYPE.Vector` fields, `*SourceHash` fields, and +any `select: false` field so clients cannot write managed embeddings. Read/return +projections still include those schema fields. diff --git a/modules/database/src/controllers/cms/__tests__/assignableFields.test.ts b/modules/database/src/controllers/cms/__tests__/assignableFields.test.ts new file mode 100644 index 000000000..239049e92 --- /dev/null +++ b/modules/database/src/controllers/cms/__tests__/assignableFields.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { + ConduitRouteActions, + ConduitSchema, + TYPE, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { CmsHandlers } from '../../../handlers/cms/crud.handler.js'; +import { getAssignableCmsFields, getOps, isCmsWriteOmittedField } from '../utils.js'; + +const fields = { + _id: { type: TYPE.ObjectId }, + title: { type: TYPE.String, required: true }, + body: TYPE.String, + owner: { type: TYPE.Relation, model: 'User' }, + meta: TYPE.JSON, + secret: { type: TYPE.String, select: false }, + embedding: { + type: TYPE.Vector, + dimensions: 8, + similarity: VectorSimilarity.Cosine, + select: false, + }, + embeddingSourceHash: { type: TYPE.String, select: false }, + createdAt: TYPE.Date, + updatedAt: TYPE.Date, +}; + +function enabledCmsSchema() { + return new ConduitSchema('Article', fields, { + conduit: { + cms: { + enabled: true, + crudOperations: { + create: { enabled: true, authenticated: false }, + read: { enabled: true, authenticated: false }, + update: { enabled: true, authenticated: false }, + delete: { enabled: true, authenticated: false }, + }, + }, + authorization: { enabled: false }, + }, + }); +} + +describe('CMS assignable fields', () => { + it('omits vector, hash, select:false, and system fields while keeping writable types', () => { + const assignable = getAssignableCmsFields(fields); + expect(Object.keys(assignable).sort()).toEqual(['body', 'meta', 'owner', 'title']); + expect(isCmsWriteOmittedField('embedding', fields.embedding)).toBe(true); + expect( + isCmsWriteOmittedField('embeddingSourceHash', fields.embeddingSourceHash), + ).toBe(true); + expect(isCmsWriteOmittedField('secret', fields.secret)).toBe(true); + expect( + isCmsWriteOmittedField('embedding', { + type: TYPE.Vector, + dimensions: 2, + }), + ).toBe(true); + expect(isCmsWriteOmittedField('title', fields.title)).toBe(false); + }); + + it('keeps vector fields on CMS return projections but not create/update bodies', () => { + const handlers = { + getDocuments: jest.fn(), + getDocumentById: jest.fn(), + createDocument: jest.fn(), + createManyDocuments: jest.fn(), + updateManyDocuments: jest.fn(), + patchManyDocuments: jest.fn(), + updateDocument: jest.fn(), + patchDocument: jest.fn(), + deleteDocument: jest.fn(), + } as unknown as CmsHandlers; + + const routes = getOps('Article', enabledCmsSchema(), handlers); + const create = routes.find( + route => + route.input.action === ConduitRouteActions.POST && + route.input.path === '/Article', + ); + const update = routes.find( + route => + route.input.action === ConduitRouteActions.UPDATE && + route.input.path === '/Article/:id', + ); + const getById = routes.find( + route => + route.input.action === ConduitRouteActions.GET && + route.input.path === '/Article/:id', + ); + + expect(create?.input.bodyParams).toMatchObject({ + title: { type: TYPE.String, required: true }, + body: TYPE.String, + owner: { type: TYPE.Relation, model: 'User' }, + meta: TYPE.JSON, + }); + expect(Object.keys(create?.input.bodyParams ?? {}).sort()).toEqual([ + 'body', + 'meta', + 'owner', + 'title', + ]); + expect(create?.input.bodyParams).not.toHaveProperty('embedding'); + expect(create?.input.bodyParams).not.toHaveProperty('embeddingSourceHash'); + expect(create?.input.bodyParams).not.toHaveProperty('secret'); + expect(update?.input.bodyParams).not.toHaveProperty('embedding'); + expect(getById?.returnType.fields).toMatchObject({ + title: { type: TYPE.String, required: true }, + embedding: { + type: TYPE.Vector, + dimensions: 8, + select: false, + }, + embeddingSourceHash: { type: TYPE.String, select: false }, + }); + expect(fields.title.required).toBe(true); + }); +}); diff --git a/modules/database/src/controllers/cms/utils.ts b/modules/database/src/controllers/cms/utils.ts index 2efa4106f..183809a65 100644 --- a/modules/database/src/controllers/cms/utils.ts +++ b/modules/database/src/controllers/cms/utils.ts @@ -57,6 +57,61 @@ export function compareFunction(schemaA: ConduitModel, schemaB: ConduitModel): n } } +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isVectorField(field: unknown): boolean { + if (field === TYPE.Vector || field === 'Vector') return true; + if (Array.isArray(field) && field.length > 0) return isVectorField(field[0]); + if (!isPlainObject(field)) return false; + return isVectorField(field.type); +} + +function isHiddenSelectField(field: unknown): boolean { + return isPlainObject(field) && field.select === false; +} + +function isManagedHashField(name: string): boolean { + return name.endsWith('SourceHash'); +} + +export function isCmsWriteOmittedField(name: string, field: unknown): boolean { + if (name === '_id' || name === 'createdAt' || name === 'updatedAt') return true; + if (isManagedHashField(name)) return true; + if (isHiddenSelectField(field)) return true; + return isVectorField(field); +} + +export function getAssignableCmsFields(sourceFields: ConduitModel): ConduitModel { + const assignable: ConduitModel = {}; + for (const [name, field] of Object.entries(sourceFields)) { + if (isCmsWriteOmittedField(name, field)) continue; + assignable[name] = cloneAssignableValue(field); + } + return assignable; +} + +function cloneAssignableValue(field: unknown): ConduitModel[string] { + if (Array.isArray(field)) { + return field.map(item => + isPlainObject(item) ? { ...item } : item, + ) as ConduitModel[string]; + } + if (isPlainObject(field)) { + return { ...field } as ConduitModel[string]; + } + return field as ConduitModel[string]; +} + +function cloneConduitModel(fields: ConduitModel): ConduitModel { + const cloned: ConduitModel = {}; + for (const [name, field] of Object.entries(fields)) { + cloned[name] = cloneAssignableValue(field); + } + return cloned; +} + function removeRequiredFields(fields: ConduitModel) { for (const field in fields) { const modelField = fields[field] as ConduitModelField; @@ -133,10 +188,7 @@ export function getOps( const sourceFields = (actualSchema as unknown as { compiledFields?: ConduitModel }).compiledFields ?? actualSchema.fields; - const assignableFields: ConduitModel = Object.assign({}, sourceFields); - delete assignableFields._id; - delete assignableFields.createdAt; - delete assignableFields.updatedAt; + const assignableFields: ConduitModel = getAssignableCmsFields(sourceFields); if (createIsEnabled) { let route = new RouteBuilder() .path(`/${schemaName}`) @@ -210,7 +262,7 @@ export function getOps( docs: { type: [ { - ...removeRequiredFields(Object.assign({}, assignableFields)), + ...removeRequiredFields(cloneConduitModel(assignableFields)), _id: { type: 'String', unique: true }, } as unknown as ArrayConduitModel, ], @@ -254,7 +306,7 @@ export function getOps( }) .bodyParams( removeRequiredFields( - Object.assign({}, assignableFields), + cloneConduitModel(assignableFields), ) as unknown as ConduitModel, ) .return(`patch${schemaName}`, actualSchema.fields) From 6398b8fc9a5b7085370f996e7c155a0082635b06 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 20:28:16 +0300 Subject: [PATCH 11/29] fix(embeddings): close remaining backfill and config-change operations gaps Material EmbeddingConfig changes now invalidate hashes, recreate indexes when needed, and schedule an explicit backfill instead of reusing stale vectors. Backfill start/cancel/resume is idempotent, enqueue failures cannot leave queued orphans, and drain polling fails closed on timeout with sanitized diagnostics. --- .../cms/__tests__/assignableFields.test.ts | 44 ++++ modules/database/src/controllers/cms/utils.ts | 65 +++++- modules/embeddings/src/Embeddings.ts | 15 ++ .../embeddings/src/api/embeddingsApi.test.ts | 109 ++++++++- modules/embeddings/src/api/embeddingsApi.ts | 210 ++++++++++++++++-- modules/embeddings/src/config/index.ts | 5 + .../src/models/BackfillRun.schema.ts | 2 + .../src/utils/backfillExecution.test.ts | 176 +++++++++++++++ .../embeddings/src/utils/backfillExecution.ts | 92 +++++++- .../embeddings/src/utils/backfillRun.test.ts | 27 ++- modules/embeddings/src/utils/backfillRun.ts | 17 +- .../embeddings/src/utils/configChange.test.ts | 97 ++++++++ modules/embeddings/src/utils/configChange.ts | 127 +++++++++++ .../src/utils/processEmbedding.test.ts | 26 ++- .../embeddings/src/utils/processEmbedding.ts | 5 +- 15 files changed, 978 insertions(+), 39 deletions(-) create mode 100644 modules/embeddings/src/utils/configChange.test.ts create mode 100644 modules/embeddings/src/utils/configChange.ts diff --git a/modules/database/src/controllers/cms/__tests__/assignableFields.test.ts b/modules/database/src/controllers/cms/__tests__/assignableFields.test.ts index 239049e92..e2df454c8 100644 --- a/modules/database/src/controllers/cms/__tests__/assignableFields.test.ts +++ b/modules/database/src/controllers/cms/__tests__/assignableFields.test.ts @@ -61,6 +61,50 @@ describe('CMS assignable fields', () => { expect(isCmsWriteOmittedField('title', fields.title)).toBe(false); }); + it('recursively omits nested Vector, select:false, and source-hash fields from embedded objects', () => { + const nested = { + title: { type: TYPE.String, required: true }, + profile: { + type: { + bio: TYPE.String, + embedding: { + type: TYPE.Vector, + dimensions: 4, + similarity: VectorSimilarity.Cosine, + }, + embeddingSourceHash: { type: TYPE.String, select: false }, + secret: { type: TYPE.String, select: false }, + }, + }, + items: [ + { + name: TYPE.String, + embedding: { + type: TYPE.Vector, + dimensions: 2, + similarity: VectorSimilarity.Cosine, + }, + hidden: { type: TYPE.String, select: false }, + }, + ], + }; + const assignable = getAssignableCmsFields(nested); + expect(Object.keys(assignable).sort()).toEqual(['items', 'profile', 'title']); + expect(assignable.profile).toMatchObject({ + type: { bio: TYPE.String }, + }); + expect( + (assignable.profile as { type: Record }).type, + ).not.toHaveProperty('embedding'); + expect( + (assignable.profile as { type: Record }).type, + ).not.toHaveProperty('embeddingSourceHash'); + expect( + (assignable.profile as { type: Record }).type, + ).not.toHaveProperty('secret'); + expect(assignable.items).toEqual([{ name: TYPE.String }]); + }); + it('keeps vector fields on CMS return projections but not create/update bodies', () => { const handlers = { getDocuments: jest.fn(), diff --git a/modules/database/src/controllers/cms/utils.ts b/modules/database/src/controllers/cms/utils.ts index 183809a65..8bd26e02a 100644 --- a/modules/database/src/controllers/cms/utils.ts +++ b/modules/database/src/controllers/cms/utils.ts @@ -84,24 +84,75 @@ export function isCmsWriteOmittedField(name: string, field: unknown): boolean { } export function getAssignableCmsFields(sourceFields: ConduitModel): ConduitModel { + return stripAssignableModel(sourceFields); +} + +const FIELD_DESCRIPTOR_KEYS = new Set([ + 'type', + 'sqlType', + 'default', + 'description', + 'required', + 'select', + 'unique', + 'index', + 'enum', + 'model', + 'validate', + 'dimensions', + 'similarity', + 'provider', +]); + +function isNestedConduitModel(field: unknown): field is ConduitModel { + if (!isPlainObject(field)) return false; + if (isVectorField(field) || isHiddenSelectField(field)) return false; + if ('type' in field || 'enum' in field || 'model' in field) return false; + return Object.keys(field).some(key => !FIELD_DESCRIPTOR_KEYS.has(key)); +} + +function stripAssignableModel(source: ConduitModel): ConduitModel { const assignable: ConduitModel = {}; - for (const [name, field] of Object.entries(sourceFields)) { + for (const [name, field] of Object.entries(source)) { if (isCmsWriteOmittedField(name, field)) continue; assignable[name] = cloneAssignableValue(field); } return assignable; } +function cloneAssignableArrayItem(item: unknown): unknown { + if (Array.isArray(item)) { + return item.map(cloneAssignableArrayItem); + } + if (!isPlainObject(item)) return item; + if (isCmsWriteOmittedField('item', item) && isVectorField(item)) { + return { ...item }; + } + if (isNestedConduitModel(item)) { + return stripAssignableModel(item); + } + return cloneAssignableValue(item); +} + function cloneAssignableValue(field: unknown): ConduitModel[string] { if (Array.isArray(field)) { - return field.map(item => - isPlainObject(item) ? { ...item } : item, - ) as ConduitModel[string]; + return field.map(cloneAssignableArrayItem) as ConduitModel[string]; } - if (isPlainObject(field)) { - return { ...field } as ConduitModel[string]; + if (!isPlainObject(field)) { + return field as ConduitModel[string]; + } + if (isNestedConduitModel(field)) { + return stripAssignableModel(field) as ConduitModel[string]; + } + const cloned: Record = { ...field }; + if (Array.isArray(cloned.type)) { + cloned.type = cloned.type.map(cloneAssignableArrayItem); + } else if (isPlainObject(cloned.type) && !isVectorField(cloned)) { + if (isNestedConduitModel(cloned.type) || isPlainObject(cloned.type)) { + cloned.type = stripAssignableModel(cloned.type as ConduitModel); + } } - return field as ConduitModel[string]; + return cloned as ConduitModel[string]; } function cloneConduitModel(fields: ConduitModel): ConduitModel { diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts index d366d87d9..2e482d37a 100644 --- a/modules/embeddings/src/Embeddings.ts +++ b/modules/embeddings/src/Embeddings.ts @@ -306,6 +306,20 @@ export default class EmbeddingsModule extends ManagedModule { getVectorCapabilities: schemaName => this.database.getVectorCapabilities(schemaName), getVectorIndexes: schemaName => this.database.getVectorIndexes(schemaName), + createVectorIndex: (schemaName, index) => + this.database.createVectorIndex(schemaName, index), + deleteVectorIndex: (schemaName, indexName) => + this.database.deleteVectorIndex(schemaName, indexName), + invalidateHashes: async (schemaName, hashFields) => { + for (const field of hashFields) { + await this.database.updateMany( + schemaName, + {}, + { [field]: null }, + { suppressEvent: true }, + ); + } + }, vectorSearch: input => this.database.vectorSearch(input), configs: { findMany: query => EmbeddingConfig.getInstance().findMany(query), @@ -459,6 +473,7 @@ export default class EmbeddingsModule extends ManagedModule { private async processBackfillJob(data: BackfillControllerJobData) { await processBackfillControllerJob(data, { maxBatchSize: this.currentConfig().queue.maxBatchSize ?? MAX_QUEUE_BATCH_SIZE, + drainTimeoutMs: this.currentConfig().queue.drainTimeoutMs, moduleEnabled: this.currentConfig().enabled, getRun: async id => { const doc = await BackfillRun.getInstance().findOne({ _id: id }); diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts index e74ead1b7..361584a25 100644 --- a/modules/embeddings/src/api/embeddingsApi.test.ts +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -79,12 +79,18 @@ function createApi(overrides?: { embed?: EmbeddingsApiDeps['embed']; vectorSearch?: EmbeddingsApiDeps['vectorSearch']; enqueue?: string[]; + invalidated?: string[]; + deletedIndexes?: string[]; + createdIndexes?: string[]; config?: Config; queue?: { generation: QueueJobCounts; backfill: QueueJobCounts }; }) { const configs = [...(overrides?.configs ?? [])]; const runs = [...(overrides?.runs ?? [])]; const enqueued = overrides?.enqueue ?? []; + const invalidated = overrides?.invalidated ?? []; + const deletedIndexes = overrides?.deletedIndexes ?? []; + const createdIndexes = overrides?.createdIndexes ?? []; const deps: EmbeddingsApiDeps = { currentConfig: () => overrides?.config ?? moduleConfig, getSchema: async name => { @@ -169,11 +175,30 @@ function createApi(overrides?: { enqueueBackfill: async job => { enqueued.push(job.runId); }, + createVectorIndex: async (_schema, index) => { + createdIndexes.push(index.name ?? index.field); + return 'created'; + }, + deleteVectorIndex: async (_schema, indexName) => { + deletedIndexes.push(indexName); + return 'deleted'; + }, + invalidateHashes: async (schemaName, hashFields) => { + invalidated.push(...hashFields.map(field => `${schemaName}.${field}`)); + }, embed: overrides?.embed ?? (async () => Array.from({ length: 3 }, (_, index) => index + 0.1)), }; - return { api: new EmbeddingsApi(deps), configs, runs, enqueued }; + return { + api: new EmbeddingsApi(deps), + configs, + runs, + enqueued, + invalidated, + deletedIndexes, + createdIndexes, + }; } const enabledConfig: EmbeddingConfigRecord = { @@ -413,4 +438,86 @@ describe('typed embeddings API handlers', () => { err instanceof GrpcError && err.code === status.FAILED_PRECONDITION, ); }); + + it('invalidates hashes, recreates the index, and schedules a backfill on material config changes', async () => { + const { api, invalidated, deletedIndexes, createdIndexes, enqueued, runs } = + createApi({ + configs: [enabledConfig], + }); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 8, + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /dimensions/.test(err.message), + ); + + const updated = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title', 'body'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-large', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(updated.config.model, 'text-embedding-3-large'); + assert.deepEqual(invalidated, ['Article.embeddingSourceHash']); + assert.deepEqual(deletedIndexes, ['embedding_vector']); + assert.equal(createdIndexes.includes('embedding_vector'), true); + assert.equal(enqueued.length, 1); + assert.equal(runs[0].state, 'queued'); + assert.equal(runs[0].onlyMissing, false); + assert.equal( + updated.warnings.some(warning => /explicit backfill was scheduled/.test(warning)), + true, + ); + }); + + it('is idempotent for start, cancel, and resume and does not duplicate active runs', async () => { + const { api, runs, enqueued } = createApi({ configs: [enabledConfig] }); + const first = await api.startBackfill( + { schemaName: 'Article', configId: 'cfg1' }, + { callerModule: 'database' }, + ); + const second = await api.startBackfill( + { schemaName: 'Article', configId: 'cfg1' }, + { callerModule: 'database' }, + ); + assert.equal(first.runs[0].id, second.runs[0].id); + assert.equal(runs.filter(run => run.state === 'queued').length, 1); + const canceled = await api.cancelBackfill(first.runs[0].id, { + callerModule: 'database', + }); + assert.equal(canceled.run.state, 'canceled'); + const canceledAgain = await api.cancelBackfill(first.runs[0].id, { + callerModule: 'database', + }); + assert.equal(canceledAgain.run.state, 'canceled'); + const resumed = await api.resumeBackfill(first.runs[0].id, { + callerModule: 'database', + }); + assert.equal(resumed.run.state, 'queued'); + const resumedAgain = await api.resumeBackfill(first.runs[0].id, { + callerModule: 'database', + }); + assert.equal(resumedAgain.run.state, 'queued'); + assert.equal(enqueued.length >= 2, true); + }); }); diff --git a/modules/embeddings/src/api/embeddingsApi.ts b/modules/embeddings/src/api/embeddingsApi.ts index 4ce407d8b..0ecf9c54a 100644 --- a/modules/embeddings/src/api/embeddingsApi.ts +++ b/modules/embeddings/src/api/embeddingsApi.ts @@ -4,6 +4,7 @@ import { VectorCapabilities, VectorSearchResult, type ConduitModel, + type VectorIndexDefinition, } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { Config } from '../config/index.js'; @@ -16,8 +17,21 @@ import { type BackfillControllerJobData, type PersistedBackfillRun, } from '../utils/backfillExecution.js'; -import { BackfillGateError, grpcErrorFromBackfillGate } from '../utils/backfillGates.js'; +import { + BackfillGateError, + findTargetVectorIndex, + grpcErrorFromBackfillGate, +} from '../utils/backfillGates.js'; +import { + defaultEmbeddingVectorIndexName, + diffMaterialEmbeddingConfig, + hashFieldsToInvalidate, + isInPlaceDimensionChange, + materialChangeWarnings, + requiresIndexRecreation, +} from '../utils/configChange.js'; import { MAX_QUEUE_BATCH_SIZE } from '../utils/embeddingJobs.js'; +import { ACTIVE_BACKFILL_STATES } from '../utils/backfillRun.js'; import { assertCanManageEmbeddingConfig, assertEmbeddingTargetSchema, @@ -137,6 +151,12 @@ export interface EmbeddingsApiDeps { backfills: BackfillStore; getQueueStatus: () => Promise<{ generation: QueueJobCounts; backfill: QueueJobCounts }>; enqueueBackfill: (job: BackfillControllerJobData) => Promise; + createVectorIndex: ( + schemaName: string, + index: VectorIndexDefinition, + ) => Promise; + deleteVectorIndex: (schemaName: string, indexName: string) => Promise; + invalidateHashes: (schemaName: string, hashFields: string[]) => Promise; embed: (input: string, provider: string, model: string) => Promise; onConfigChanged?: (schemaName: string) => Promise | void; } @@ -177,7 +197,7 @@ export class EmbeddingsApi { ); const enabled = request.enabled ?? true; const capabilities = await this.deps.getVectorCapabilities(persisted.schemaName); - const indexes = await this.deps.getVectorIndexes(persisted.schemaName); + let indexes = await this.deps.getVectorIndexes(persisted.schemaName); const warnings = [ ...capabilityWarnings(capabilities), ...indexReadinessWarnings( @@ -195,17 +215,20 @@ export class EmbeddingsApi { configDefaults.providers[configDefaults.defaultProvider], ), ]; - if (enabled) { - assertConfigActivation({ - moduleEnabled: configDefaults.enabled, - capabilities, - config: { - enabled, - schemaName: persisted.schemaName, - targetField: persisted.targetField, - }, - indexes, - }); + const existing = await this.deps.configs.findOne({ + schemaName: persisted.schemaName, + targetField: persisted.targetField, + }); + const changed = existing ? diffMaterialEmbeddingConfig(existing, persisted) : []; + if (existing && isInPlaceDimensionChange(existing, persisted)) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Changing vector field '${existing.targetField}' dimensions from ${existing.dimensions} to ${persisted.dimensions} is not allowed. Create a new targetField and run an explicit backfill.`, + ); + } + if (existing && changed.length && requiresIndexRecreation(changed)) { + await this.recreateVectorIndex(existing, persisted, indexes); + indexes = await this.deps.getVectorIndexes(persisted.schemaName); } if (capabilities.storage) { await this.deps.setSchemaExtension({ @@ -225,16 +248,54 @@ export class EmbeddingsApi { }, }); } - const existing = await this.deps.configs.findOne({ - schemaName: persisted.schemaName, - targetField: persisted.targetField, - }); + let persistEnabled = enabled; + if (enabled) { + try { + assertConfigActivation({ + moduleEnabled: configDefaults.enabled, + capabilities, + config: { + enabled, + schemaName: persisted.schemaName, + targetField: persisted.targetField, + }, + indexes, + }); + } catch (err) { + if (existing && changed.length && requiresIndexRecreation(changed)) { + persistEnabled = false; + warnings.push( + 'Config was saved disabled until the recreated vector index is queryable. Enable it and start an explicit backfill once the index is ready.', + ); + } else { + throw err; + } + } + } const saved = existing - ? await this.deps.configs.findByIdAndUpdate(existing._id, { ...persisted, enabled }) - : await this.deps.configs.create({ ...persisted, enabled }); + ? await this.deps.configs.findByIdAndUpdate(existing._id, { + ...persisted, + enabled: persistEnabled, + }) + : await this.deps.configs.create({ ...persisted, enabled: persistEnabled }); if (!saved) { throw new GrpcError(status.INTERNAL, 'Failed to persist embedding config'); } + if (existing && changed.length) { + await this.deps.invalidateHashes( + saved.schemaName, + hashFieldsToInvalidate(existing, saved), + ); + await this.supersedeActiveBackfills(saved._id); + let scheduledBackfill = false; + if (persistEnabled) { + scheduledBackfill = await this.scheduleExplicitBackfill(saved, { + capabilities, + indexes, + }); + } + warnings.push(...materialChangeWarnings(changed, scheduledBackfill)); + } await this.deps.onConfigChanged?.(saved.schemaName); return { config: mapEmbeddingConfig(saved), warnings }; } @@ -354,6 +415,10 @@ export class EmbeddingsApi { configs, indexes, createRun: async run => this.deps.backfills.create(persistableBackfillRun(run)), + saveRun: async (id, run) => { + await this.deps.backfills.findByIdAndUpdate(id, persistableBackfillRun(run)); + }, + findActiveRuns: configId => this.findActiveBackfills(configId), enqueueController: job => this.deps.enqueueBackfill(job), }, ); @@ -635,4 +700,111 @@ export class EmbeddingsApi { } return config; } + + private async findActiveBackfills(configId: string): Promise { + const runs: PersistedBackfillRun[] = []; + for (const state of ACTIVE_BACKFILL_STATES) { + const found = await this.deps.backfills.findMany({ configId, state }); + runs.push(...found); + } + return runs; + } + + private async recreateVectorIndex( + existing: EmbeddingConfigRecord, + next: + | EmbeddingConfigRecord + | { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + }, + indexes: Array<{ + field?: string; + name?: string; + queryable?: boolean; + status?: string; + }>, + ): Promise { + const current = findTargetVectorIndex(indexes, existing.targetField); + if (current?.name) { + try { + await this.deps.deleteVectorIndex(existing.schemaName, current.name); + } catch (err) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Failed to delete vector index '${current.name}': ${sanitizeErrorMessage(err)}`, + ); + } + } + try { + await this.deps.createVectorIndex(next.schemaName, { + field: next.targetField, + dimensions: next.dimensions, + similarity: next.similarity as VectorIndexDefinition['similarity'], + name: defaultEmbeddingVectorIndexName(next.targetField), + }); + } catch (err) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Failed to recreate vector index for '${next.targetField}': ${sanitizeErrorMessage(err)}`, + ); + } + } + + private async supersedeActiveBackfills(configId: string): Promise { + const active = await this.findActiveBackfills(configId); + for (const run of active) { + await cancelBackfillExecution({ + run, + saveRun: async (id, next) => { + await this.deps.backfills.findByIdAndUpdate(id, persistableBackfillRun(next)); + }, + }); + } + } + + private async scheduleExplicitBackfill( + config: EmbeddingConfigRecord, + args: { + capabilities: VectorCapabilities; + indexes: Array<{ + field?: string; + name?: string; + queryable?: boolean; + status?: string; + }>; + }, + ): Promise { + try { + const queued = await queueBackfillRuns( + { + schemaName: config.schemaName, + configId: config._id, + onlyMissing: false, + maxBatchSize: + this.deps.currentConfig().queue.maxBatchSize ?? MAX_QUEUE_BATCH_SIZE, + }, + { + moduleEnabled: this.deps.currentConfig().enabled, + capabilities: args.capabilities, + configs: [config], + indexes: args.indexes, + createRun: async run => this.deps.backfills.create(persistableBackfillRun(run)), + saveRun: async (id, run) => { + await this.deps.backfills.findByIdAndUpdate(id, persistableBackfillRun(run)); + }, + findActiveRuns: configId => this.findActiveBackfills(configId), + enqueueController: job => this.deps.enqueueBackfill(job), + }, + ); + return queued.queued > 0; + } catch (err) { + if (err instanceof BackfillGateError) { + return false; + } + throw err; + } + } } diff --git a/modules/embeddings/src/config/index.ts b/modules/embeddings/src/config/index.ts index ba3a5089c..e4b313c90 100644 --- a/modules/embeddings/src/config/index.ts +++ b/modules/embeddings/src/config/index.ts @@ -53,6 +53,11 @@ const AppConfigSchema = { format: 'Number', default: 500, }, + drainTimeoutMs: { + doc: 'Maximum time a backfill may wait for generation jobs during drain before failing', + format: 'Number', + default: 15 * 60 * 1000, + }, }, security: { requireGrpcKey: { diff --git a/modules/embeddings/src/models/BackfillRun.schema.ts b/modules/embeddings/src/models/BackfillRun.schema.ts index 7815bec3f..108629a8b 100644 --- a/modules/embeddings/src/models/BackfillRun.schema.ts +++ b/modules/embeddings/src/models/BackfillRun.schema.ts @@ -27,6 +27,7 @@ const schema: ConduitModel = { failedCount: { type: TYPE.Number, default: 0 }, startedAt: { type: TYPE.Date, required: false }, finishedAt: { type: TYPE.Date, required: false }, + drainStartedAt: { type: TYPE.Date, required: false }, error: { type: TYPE.String, required: false }, createdAt: TYPE.Date, updatedAt: TYPE.Date, @@ -61,6 +62,7 @@ export class BackfillRun extends ConduitActiveSchema { failedCount: number; startedAt?: Date; finishedAt?: Date; + drainStartedAt?: Date; error?: string; createdAt: Date; updatedAt: Date; diff --git a/modules/embeddings/src/utils/backfillExecution.test.ts b/modules/embeddings/src/utils/backfillExecution.test.ts index cd46efc5e..2c20c12dd 100644 --- a/modules/embeddings/src/utils/backfillExecution.test.ts +++ b/modules/embeddings/src/utils/backfillExecution.test.ts @@ -105,6 +105,13 @@ describe('queued backfill start', () => { configs: [config, { ...config, _id: 'cfg2', targetField: 'other' }], indexes: [readyIndex, { ...readyIndex, field: 'other', name: 'other_vector' }], createRun: store.createRun, + saveRun: store.saveRun, + findActiveRuns: async configId => + [...store.runs.values()].filter( + run => + run.configId === configId && + (run.state === 'queued' || run.state === 'running'), + ), enqueueController: async job => { controllerJobs.push(job); }, @@ -145,6 +152,8 @@ describe('queued backfill start', () => { configs: [config], indexes: [readyIndex], createRun: store.createRun, + saveRun: store.saveRun, + findActiveRuns: async () => [], enqueueController: async () => undefined, }, ), @@ -386,3 +395,170 @@ describe('backfill controller job parsing and persistence mapping', () => { assert.equal(persistableBackfillRun(progress).onlyMissing, true); }); }); + +describe('backfill enqueue failures, drain timeout, and start idempotency', () => { + function queueDeps( + store: ReturnType, + extras: { + enqueue?: (job: BackfillControllerJobData) => Promise; + configs?: Array<{ + _id: string; + enabled?: boolean; + schemaName?: string; + targetField?: string; + }>; + } = {}, + ) { + return { + moduleEnabled: true, + capabilities, + configs: extras.configs ?? [config], + indexes: [readyIndex], + createRun: store.createRun, + saveRun: store.saveRun, + findActiveRuns: async (configId: string) => + [...store.runs.values()].filter( + run => + run.configId === configId && + (run.state === 'queued' || run.state === 'running'), + ), + enqueueController: extras.enqueue ?? (async () => undefined), + }; + } + + it('fails a newly created run when controller enqueue throws instead of leaving it queued', async () => { + const store = memoryStore(); + await assert.rejects( + () => + queueBackfillRuns( + { schemaName: 'Article' }, + queueDeps(store, { + enqueue: async () => { + throw new Error('redis down apiKey=sk-secret'); + }, + }), + ), + /redis down/, + ); + const persisted = [...store.runs.values()]; + assert.equal(persisted.length, 1); + assert.equal(persisted[0].state, 'failed'); + assert.match(persisted[0].error ?? '', /redis down/); + assert.doesNotMatch(persisted[0].error ?? '', /sk-secret/); + }); + + it('reuses an active run for the same config instead of creating a duplicate', async () => { + const store = memoryStore(); + const first = await queueBackfillRuns( + { schemaName: 'Article', batchSize: 2 }, + queueDeps(store), + ); + const second = await queueBackfillRuns( + { schemaName: 'Article', batchSize: 50 }, + queueDeps(store), + ); + assert.equal(first.queued, 1); + assert.equal(second.queued, 1); + assert.equal(second.runs[0].id, first.runs[0].id); + assert.equal(store.runs.size, 1); + }); + + it('fails drain polling with a sanitized timeout instead of looping forever', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'running', + batchSize: 2, + queuedCount: 2, + processedCount: 0, + failedCount: 0, + scannedCount: 2, + cursor: 'b', + drainStartedAt: new Date('2026-09-06T17:00:00.000Z'), + }), + ); + const result = await processBackfillControllerJob( + { runId: created._id, cursor: 'b', drain: true }, + deps({ + store, + drainTimeoutMs: 60_000, + now: new Date('2026-09-06T18:00:00.000Z'), + }), + ); + assert.equal(result.action, 'failed'); + assert.equal(result.run?.state, 'failed'); + assert.match(result.run?.error ?? '', /timed out waiting for generation jobs/); + assert.doesNotMatch(result.run?.error ?? '', /apiKey|Bearer /); + }); + + it('records drainStartedAt on the first drain poll then fails after the timeout', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'running', + batchSize: 2, + queuedCount: 2, + processedCount: 0, + failedCount: 0, + scannedCount: 2, + cursor: 'b', + }), + ); + const startedAt = new Date('2026-09-06T18:00:00.000Z'); + const first = await processBackfillControllerJob( + { runId: created._id, cursor: 'b', drain: true }, + deps({ + store, + drainTimeoutMs: 60_000, + now: startedAt, + }), + ); + assert.equal(first.action, 'drain'); + assert.equal(first.run?.drainStartedAt?.toISOString(), startedAt.toISOString()); + const timedOut = await processBackfillControllerJob( + { runId: created._id, cursor: 'b', drain: true }, + deps({ + store, + drainTimeoutMs: 60_000, + now: new Date('2026-09-06T18:01:00.000Z'), + }), + ); + assert.equal(timedOut.action, 'failed'); + assert.match(timedOut.run?.error ?? '', /timed out waiting for generation jobs/); + }); + + it('fails a resumed run when controller enqueue throws', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'canceled', + batchSize: 2, + cursor: 'b', + }), + ); + const existing = (await store.getRun(created._id))!; + await assert.rejects( + () => + resumeBackfillExecution({ + run: existing, + saveRun: store.saveRun, + enqueueController: async () => { + throw new Error('queue unavailable Bearer sk-secret'); + }, + }), + /queue unavailable/, + ); + const persisted = (await store.getRun(created._id))!; + assert.equal(persisted.state, 'failed'); + assert.doesNotMatch(persisted.error ?? '', /sk-secret/); + }); +}); diff --git a/modules/embeddings/src/utils/backfillExecution.ts b/modules/embeddings/src/utils/backfillExecution.ts index eb78cdd96..5dd633508 100644 --- a/modules/embeddings/src/utils/backfillExecution.ts +++ b/modules/embeddings/src/utils/backfillExecution.ts @@ -8,6 +8,7 @@ import { completeBackfillRun, createQueuedBackfill, failBackfillRun, + isActiveBackfillState, resumeBackfillRun, startBackfillRun, type BackfillPageQuery, @@ -22,10 +23,14 @@ import { } from './backfillGates.js'; import { EmbeddingJobData, MAX_QUEUE_BATCH_SIZE } from './embeddingJobs.js'; import { incrementEmbeddingMetric } from './embeddingMetrics.js'; +import { sanitizeErrorMessage } from './redactConfig.js'; const IDENTITY = /^[A-Za-z0-9._-]{1,128}$/; export const BACKFILL_DRAIN_DELAY_MS = 1000; +export const DEFAULT_BACKFILL_DRAIN_TIMEOUT_MS = 15 * 60 * 1000; +export const BACKFILL_DRAIN_TIMEOUT_MESSAGE = + 'Backfill drain timed out waiting for generation jobs'; export interface PersistedBackfillRun extends BackfillRunProgress { _id: string; @@ -55,6 +60,8 @@ export interface QueueBackfillDeps { configs: BackfillConfigGate[]; indexes: readonly VectorIndexGate[]; createRun: (run: BackfillRunProgress) => Promise<{ _id: string }>; + saveRun: (id: string, run: BackfillRunProgress) => Promise; + findActiveRuns: (configId: string) => Promise; enqueueController: (job: BackfillControllerJobData) => Promise; } @@ -75,6 +82,7 @@ export interface ProcessBackfillDeps { ) => Promise>; getConfig: (id: string) => Promise; getIndexes: (schemaName: string) => Promise; + drainTimeoutMs?: number; } export function backfillRunFromDocument(doc: { @@ -92,6 +100,7 @@ export function backfillRunFromDocument(doc: { failedCount?: number; startedAt?: Date; finishedAt?: Date; + drainStartedAt?: Date; error?: string; }): PersistedBackfillRun { return { @@ -109,6 +118,7 @@ export function backfillRunFromDocument(doc: { failedCount: doc.failedCount ?? 0, startedAt: doc.startedAt ?? null, finishedAt: doc.finishedAt ?? null, + drainStartedAt: doc.drainStartedAt ?? null, error: doc.error ?? null, }; } @@ -130,6 +140,7 @@ export function persistableBackfillRun( failedCount: run.failedCount, startedAt: run.startedAt ?? undefined, finishedAt: run.finishedAt ?? undefined, + drainStartedAt: run.drainStartedAt ?? undefined, error: run.error ?? undefined, }; } @@ -215,8 +226,44 @@ export async function queueBackfillRuns( `Invalid backfill request: ${created.reason}`, ); } + if (config._id) { + const active = (await deps.findActiveRuns(config._id)).filter(run => + isActiveBackfillState(run.state), + ); + const existing = active[0]; + if (existing) { + if (existing.state === 'queued') { + await enqueueOrFailRun( + existing._id, + existing, + deps.saveRun, + deps.enqueueController, + { + runId: existing._id, + cursor: existing.cursor ?? null, + }, + ); + } + runs.push({ + id: existing._id, + ...(config._id ? { configId: config._id } : {}), + state: existing.state, + }); + continue; + } + } const persisted = await deps.createRun(created.run); - await deps.enqueueController({ runId: persisted._id, cursor: null }); + const queuedRun: PersistedBackfillRun = { ...created.run, _id: persisted._id }; + await enqueueOrFailRun( + persisted._id, + queuedRun, + deps.saveRun, + deps.enqueueController, + { + runId: persisted._id, + cursor: null, + }, + ); runs.push({ id: persisted._id, ...(config._id ? { configId: config._id } : {}), @@ -226,6 +273,24 @@ export async function queueBackfillRuns( return { queued: runs.length, runs }; } +async function enqueueOrFailRun( + id: string, + run: BackfillRunProgress, + saveRun: (id: string, run: BackfillRunProgress) => Promise, + enqueue: (job: BackfillControllerJobData) => Promise, + job: BackfillControllerJobData, +): Promise { + try { + await enqueue(job); + } catch (err) { + const failed = failBackfillRun(run, sanitizeErrorMessage(err)); + if (failed.ok) { + await saveRun(id, failed.run); + } + throw err; + } +} + export async function resumeBackfillExecution(args: { run: PersistedBackfillRun; saveRun: (id: string, run: BackfillRunProgress) => Promise; @@ -234,10 +299,18 @@ export async function resumeBackfillExecution(args: { const resumed = resumeBackfillRun(args.run); if (!resumed.ok) return resumed; await args.saveRun(args.run._id, resumed.run); - await args.enqueueController({ - runId: args.run._id, - cursor: resumed.run.cursor ?? null, - }); + try { + await args.enqueueController({ + runId: args.run._id, + cursor: resumed.run.cursor ?? null, + }); + } catch (err) { + const failed = failBackfillRun(resumed.run, sanitizeErrorMessage(err)); + if (failed.ok) { + await args.saveRun(args.run._id, failed.run); + } + throw err; + } return resumed; } @@ -405,6 +478,15 @@ async function finishOrDrain( await deps.saveRun(id, completed.run); return { action: 'completed', run: completed.run }; } + const drainStartedAt = run.drainStartedAt ?? now; + const timeoutMs = deps.drainTimeoutMs ?? DEFAULT_BACKFILL_DRAIN_TIMEOUT_MS; + if (now.getTime() - drainStartedAt.getTime() >= timeoutMs) { + return failPersistedRun(id, run, BACKFILL_DRAIN_TIMEOUT_MESSAGE, deps, now); + } + if (!run.drainStartedAt) { + run = { ...run, drainStartedAt }; + await deps.saveRun(id, run); + } await deps.enqueueContinuation({ runId: id, cursor: run.cursor ?? null, diff --git a/modules/embeddings/src/utils/backfillRun.test.ts b/modules/embeddings/src/utils/backfillRun.test.ts index f593a7224..fbd4c6452 100644 --- a/modules/embeddings/src/utils/backfillRun.test.ts +++ b/modules/embeddings/src/utils/backfillRun.test.ts @@ -63,8 +63,8 @@ describe('backfill run state transitions', () => { assert.equal(isLegalBackfillTransition('running', 'canceled'), true); assert.equal(isLegalBackfillTransition('failed', 'queued'), true); assert.equal(isLegalBackfillTransition('canceled', 'queued'), true); + assert.equal(isLegalBackfillTransition('queued', 'failed'), true); assert.equal(isLegalBackfillTransition('queued', 'completed'), false); - assert.equal(isLegalBackfillTransition('queued', 'failed'), false); assert.equal(isLegalBackfillTransition('running', 'queued'), false); assert.equal(isLegalBackfillTransition('completed', 'queued'), false); assert.equal(isLegalBackfillTransition('completed', 'running'), false); @@ -91,8 +91,8 @@ describe('backfill run state transitions', () => { assert.equal(resumed.ok, false); if (resumed.ok) return; assert.equal(resumed.reason, 'illegal_transition'); - const failedFromQueued = failBackfillRun(queuedRun(), 'boom', now); - assert.equal(failedFromQueued.ok, false); + const failedFromCompleted = failBackfillRun(completed.run, 'boom', now); + assert.equal(failedFromCompleted.ok, false); }); }); @@ -213,6 +213,27 @@ describe('backfill cancellation and resume', () => { assert.equal(resumed.run.cursor, canceled.run.cursor); assert.equal(resumed.run.scannedCount, canceled.run.scannedCount); + assert.equal(canCancelBackfill('canceled'), false); + const alreadyCanceled = cancelBackfillRun(canceled.run, now); + assert.equal(alreadyCanceled.ok, true); + if (!alreadyCanceled.ok) return; + assert.equal(alreadyCanceled.run.state, 'canceled'); + + const alreadyQueued = resumeBackfillRun(resumed.run); + assert.equal(alreadyQueued.ok, true); + if (!alreadyQueued.ok) return; + assert.equal(alreadyQueued.run.state, 'queued'); + + const queuedFailed = failBackfillRun( + queuedRun(), + 'enqueue failed apiKey=sk-test', + now, + ); + assert.equal(queuedFailed.ok, true); + if (!queuedFailed.ok) return; + assert.equal(queuedFailed.run.state, 'failed'); + assert.doesNotMatch(queuedFailed.run.error ?? '', /sk-test|apiKey=/); + const failed = failBackfillRun(runningRun(), 'provider timeout', now); assert.equal(failed.ok, true); if (!failed.ok) return; diff --git a/modules/embeddings/src/utils/backfillRun.ts b/modules/embeddings/src/utils/backfillRun.ts index 9dc765db2..2cb3e2784 100644 --- a/modules/embeddings/src/utils/backfillRun.ts +++ b/modules/embeddings/src/utils/backfillRun.ts @@ -17,13 +17,15 @@ export const LEGAL_BACKFILL_TRANSITIONS: Record< BackfillRunState, readonly BackfillRunState[] > = { - queued: ['running', 'canceled'], + queued: ['running', 'failed', 'canceled'], running: ['completed', 'failed', 'canceled'], completed: [], failed: ['queued'], canceled: ['queued'], }; +export const ACTIVE_BACKFILL_STATES: readonly BackfillRunState[] = ['queued', 'running']; + export const MIN_BACKFILL_BATCH_SIZE = 1; export const DEFAULT_BACKFILL_BATCH_SIZE = 100; export const MAX_BACKFILL_BATCH_SIZE = MAX_QUEUE_BATCH_SIZE; @@ -48,6 +50,7 @@ export interface BackfillRunProgress { failedCount: number; startedAt?: Date | null; finishedAt?: Date | null; + drainStartedAt?: Date | null; error?: string | null; } @@ -87,6 +90,10 @@ export function isLegalBackfillTransition( return LEGAL_BACKFILL_TRANSITIONS[from].includes(to); } +export function isActiveBackfillState(state: BackfillRunState): boolean { + return ACTIVE_BACKFILL_STATES.includes(state); +} + export function canCancelBackfill(state: BackfillRunState): boolean { switch (state) { case 'queued': @@ -176,6 +183,7 @@ export function createQueuedBackfill(input: CreateBackfillRunInput): BackfillRun failedCount: 0, startedAt: null, finishedAt: null, + drainStartedAt: null, error: null, }, }; @@ -196,6 +204,9 @@ export function cancelBackfillRun( run: BackfillRunProgress, now: Date = new Date(), ): BackfillRunResult { + if (run.state === 'canceled') { + return { ok: true, run }; + } if (!canCancelBackfill(run.state)) { return { ok: false, reason: 'illegal_transition' }; } @@ -226,12 +237,16 @@ export function completeBackfillRun( } export function resumeBackfillRun(run: BackfillRunProgress): BackfillRunResult { + if (run.state === 'queued' || run.state === 'running') { + return { ok: true, run }; + } if (!isResumeEligible(run.state)) { return { ok: false, reason: 'illegal_transition' }; } return transition(run, 'queued', { finishedAt: null, error: null, + drainStartedAt: null, }); } diff --git a/modules/embeddings/src/utils/configChange.test.ts b/modules/embeddings/src/utils/configChange.test.ts new file mode 100644 index 000000000..48f7504eb --- /dev/null +++ b/modules/embeddings/src/utils/configChange.test.ts @@ -0,0 +1,97 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + diffMaterialEmbeddingConfig, + embeddingConfigFingerprint, + hashFieldsToInvalidate, + hashedEmbeddingSource, + isInPlaceDimensionChange, + materialChangeWarnings, + requiresIndexRecreation, +} from './configChange.js'; + +const base = { + provider: 'openai-compatible', + modelName: 'text-embedding-3-small', + dimensions: 1536, + sourceFields: ['title', 'body'], + targetField: 'embedding', + similarity: 'cosine', +}; + +describe('material embedding config changes', () => { + it('detects provider, model, dimensions, source fields, target, and similarity changes', () => { + assert.deepEqual(diffMaterialEmbeddingConfig(base, base), []); + assert.deepEqual( + diffMaterialEmbeddingConfig(base, { ...base, sourceFields: ['body', 'title'] }), + [], + ); + assert.deepEqual(diffMaterialEmbeddingConfig(base, { ...base, modelName: 'large' }), [ + 'modelName', + ]); + assert.deepEqual( + diffMaterialEmbeddingConfig(base, { + ...base, + provider: 'other', + dimensions: 768, + sourceFields: ['title'], + targetField: 'vector', + similarity: 'euclidean', + }), + ['provider', 'dimensions', 'sourceFields', 'targetField', 'similarity'], + ); + }); + + it('requires index recreation for dimensions, target field, and similarity', () => { + assert.equal(requiresIndexRecreation(['modelName', 'provider']), false); + assert.equal(requiresIndexRecreation(['sourceFields']), false); + assert.equal(requiresIndexRecreation(['similarity']), true); + assert.equal(requiresIndexRecreation(['dimensions']), true); + assert.equal(requiresIndexRecreation(['targetField']), true); + assert.equal(isInPlaceDimensionChange(base, { ...base, dimensions: 768 }), true); + assert.equal( + isInPlaceDimensionChange(base, { + ...base, + targetField: 'other', + dimensions: 768, + }), + false, + ); + }); + + it('invalidates old hashes via fingerprint and names hash fields to clear', () => { + const hash = (input: string) => input; + const original = hashedEmbeddingSource(hash, 'Hello', base); + const changedModel = hashedEmbeddingSource(hash, 'Hello', { + ...base, + modelName: 'large', + }); + assert.notEqual(original, changedModel); + assert.equal( + embeddingConfigFingerprint(base).includes('text-embedding-3-small'), + true, + ); + assert.deepEqual(hashFieldsToInvalidate(base, { ...base, targetField: 'other' }), [ + 'embeddingSourceHash', + 'otherSourceHash', + ]); + }); + + it('warns that stale vectors require an explicit backfill', () => { + const warnings = materialChangeWarnings(['modelName'], false); + assert.equal( + warnings.some(warning => /invalidated stored source hashes/.test(warning)), + true, + ); + assert.equal( + warnings.some(warning => /explicit backfill/.test(warning)), + true, + ); + assert.equal( + materialChangeWarnings(['similarity'], true).some(warning => + /index recreation is required/i.test(warning), + ), + true, + ); + }); +}); diff --git a/modules/embeddings/src/utils/configChange.ts b/modules/embeddings/src/utils/configChange.ts new file mode 100644 index 000000000..21d833a56 --- /dev/null +++ b/modules/embeddings/src/utils/configChange.ts @@ -0,0 +1,127 @@ +export const MATERIAL_EMBEDDING_CONFIG_FIELDS = [ + 'provider', + 'modelName', + 'dimensions', + 'sourceFields', + 'targetField', + 'similarity', +] as const; + +export type MaterialEmbeddingConfigField = + (typeof MATERIAL_EMBEDDING_CONFIG_FIELDS)[number]; + +export interface MaterialEmbeddingConfig { + provider: string; + modelName?: string; + dimensions: number; + sourceFields: readonly string[]; + targetField: string; + similarity?: string; +} + +export function normalizeSourceFields(fields: readonly string[]): string[] { + return [...fields].map(field => field.trim()).sort(); +} + +export function embeddingConfigFingerprint(config: MaterialEmbeddingConfig): string { + return JSON.stringify({ + provider: config.provider, + model: config.modelName ?? '', + dimensions: config.dimensions, + sourceFields: normalizeSourceFields(config.sourceFields), + targetField: config.targetField, + similarity: config.similarity ?? '', + }); +} + +export function hashedEmbeddingSource( + hashInput: (input: string) => string, + input: string, + config: MaterialEmbeddingConfig, +): string { + return hashInput(`${embeddingConfigFingerprint(config)}\n${input}`); +} + +export function sameSourceFields( + left: readonly string[] | undefined, + right: readonly string[] | undefined, +): boolean { + return ( + JSON.stringify(normalizeSourceFields(left ?? [])) === + JSON.stringify(normalizeSourceFields(right ?? [])) + ); +} + +export function diffMaterialEmbeddingConfig( + existing: MaterialEmbeddingConfig, + next: MaterialEmbeddingConfig, +): MaterialEmbeddingConfigField[] { + const changed: MaterialEmbeddingConfigField[] = []; + if (existing.provider !== next.provider) changed.push('provider'); + if ((existing.modelName ?? '') !== (next.modelName ?? '')) changed.push('modelName'); + if (existing.dimensions !== next.dimensions) changed.push('dimensions'); + if (!sameSourceFields(existing.sourceFields, next.sourceFields)) { + changed.push('sourceFields'); + } + if (existing.targetField !== next.targetField) changed.push('targetField'); + if ((existing.similarity ?? '') !== (next.similarity ?? '')) { + changed.push('similarity'); + } + return changed; +} + +export function requiresIndexRecreation( + changed: readonly MaterialEmbeddingConfigField[], +): boolean { + return changed.some( + field => field === 'dimensions' || field === 'targetField' || field === 'similarity', + ); +} + +export function isInPlaceDimensionChange( + existing: MaterialEmbeddingConfig, + next: MaterialEmbeddingConfig, +): boolean { + return ( + existing.targetField === next.targetField && existing.dimensions !== next.dimensions + ); +} + +export function hashFieldsToInvalidate( + existing: MaterialEmbeddingConfig, + next: MaterialEmbeddingConfig, +): string[] { + const fields = new Set([ + `${existing.targetField}SourceHash`, + `${next.targetField}SourceHash`, + ]); + return [...fields]; +} + +export function defaultEmbeddingVectorIndexName(field: string): string { + return `${field}_vector`; +} + +export function materialChangeWarnings( + changed: readonly MaterialEmbeddingConfigField[], + scheduledBackfill: boolean, +): string[] { + if (!changed.length) return []; + const warnings = [ + `Material embedding config change (${changed.join(', ')}) invalidated stored source hashes. ` + + 'Existing vectors are stale until an explicit backfill completes.', + ]; + if (requiresIndexRecreation(changed)) { + warnings.push( + 'Vector index recreation is required for this change. Wait until the index is queryable before searching or backfilling.', + ); + } + if (!scheduledBackfill) { + warnings.push( + 'Start an explicit backfill after the vector index is queryable. Stale vectors will not be reused by hash skip.', + ); + } else { + warnings.push('An explicit backfill was scheduled for this config.'); + } + return warnings; +} diff --git a/modules/embeddings/src/utils/processEmbedding.test.ts b/modules/embeddings/src/utils/processEmbedding.test.ts index bacccddea..5bfd7280d 100644 --- a/modules/embeddings/src/utils/processEmbedding.test.ts +++ b/modules/embeddings/src/utils/processEmbedding.test.ts @@ -1,6 +1,7 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; +import { hashedEmbeddingSource } from './configChange.js'; import { buildEmbeddingDocumentSelect, generateEmbeddingsForDocument, @@ -27,7 +28,7 @@ describe('embedding generation loop safety', () => { }); it('skips provider calls when the source hash already matches', async () => { - const sourceHash = hash('Hello\nWorld'); + const sourceHash = hashedEmbeddingSource(hash, 'Hello\nWorld', config); let embedCalls = 0; let updates = 0; const result = await generateEmbeddingsForDocument({ @@ -53,7 +54,7 @@ describe('embedding generation loop safety', () => { }); it('performs one write with event suppression and does not loop on the write-back', async () => { - const sourceHash = hash('Hello\nWorld'); + const sourceHash = hashedEmbeddingSource(hash, 'Hello\nWorld', config); let embedCalls = 0; const updates: Array<{ fields: Record; options: unknown }> = []; const doc: Record = { _id: 'a', title: 'Hello', body: 'World' }; @@ -82,4 +83,25 @@ describe('embedding generation loop safety', () => { assert.deepEqual(updates[0].options, { suppressEvent: true }); assert.equal(updates[0].fields.embeddingSourceHash, sourceHash); }); + + it('does not skip when stored hashes were computed without the material config fingerprint', async () => { + let embedCalls = 0; + const result = await generateEmbeddingsForDocument({ + doc: { + _id: 'a', + title: 'Hello', + body: 'World', + embeddingSourceHash: hash('Hello\nWorld'), + }, + configs: [{ ...config, modelName: 'text-embedding-3-large' }], + hashInput: hash, + embed: async () => { + embedCalls += 1; + return [1, 2]; + }, + update: async () => undefined, + }); + assert.deepEqual(result, { generated: 1, skipped: 0 }); + assert.equal(embedCalls, 1); + }); }); diff --git a/modules/embeddings/src/utils/processEmbedding.ts b/modules/embeddings/src/utils/processEmbedding.ts index 8d91b09ce..8e0929e3d 100644 --- a/modules/embeddings/src/utils/processEmbedding.ts +++ b/modules/embeddings/src/utils/processEmbedding.ts @@ -1,9 +1,12 @@ +import { hashedEmbeddingSource } from './configChange.js'; + export interface EmbeddingConfigLike { sourceFields: string[]; targetField: string; dimensions: number; provider: string; modelName?: string; + similarity?: string; } export interface EmbeddingGenerationResult { @@ -57,7 +60,7 @@ export async function generateEmbeddingsForDocument(args: { let skipped = 0; for (const config of args.configs) { const input = buildEmbeddingInput(args.doc, config.sourceFields); - const sourceHash = args.hashInput(input); + const sourceHash = hashedEmbeddingSource(args.hashInput, input, config); if (shouldSkipEmbedding(args.doc, config.targetField, sourceHash)) { skipped += 1; continue; From 43d914fc904c63cc69d8a629462e546b60dc5b03 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 20:39:26 +0300 Subject: [PATCH 12/29] feat(embeddings): add bundle image and opt-in compose profile Package embeddings as a BullMQ service bundle with bake/CI wiring while keeping it out of standalone v1. Production still requires GRPC_KEY and capability/index readiness before activation. --- .github/CONTRIBUTING.md | 1 + .github/workflows/embeddings-test.yml | 54 + .github/workflows/service-bundle-verify.yml | 4 +- Dockerfile | 2 + README.md | 1 + deploy/docker/README.md | 3 + deploy/embeddings.md | 68 + deploy/k8s/README.md | 4 + docker-bake.hcl | 19 + docker/.env | 2 + docker/docker-compose.standalone.yml | 2 + docker/docker-compose.yml | 28 + docker/prometheus.cfg.yml | 3 + libraries/grpc-sdk/src/index.ts | 44 +- modules/database/README.md | 4 + modules/embeddings/Dockerfile | 18 + modules/embeddings/README.md | 22 +- modules/embeddings/package.bundle-lock.json | 2629 +++++++++++++++++ modules/embeddings/package.bundle.json | 49 + modules/embeddings/package.json | 8 +- modules/embeddings/service-bundle.config.json | 5 + modules/embeddings/tsup.config.ts | 6 + pnpm-lock.yaml | 6 + scripts/docker-build.sh | 5 +- scripts/resolve-docker-targets.mjs | 13 + scripts/verify-service-bundle.sh | 5 + scripts/verify-standalone-bundle.sh | 1 + standalone.Dockerfile | 2 +- 28 files changed, 2996 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/embeddings-test.yml create mode 100644 deploy/embeddings.md create mode 100644 modules/embeddings/Dockerfile create mode 100644 modules/embeddings/package.bundle-lock.json create mode 100644 modules/embeddings/package.bundle.json create mode 100644 modules/embeddings/service-bundle.config.json create mode 100644 modules/embeddings/tsup.config.ts diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b17b13166..55304fdc3 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -95,6 +95,7 @@ You may find additional scripts in the `scripts` section of the `package.json` f - `authorization`: resource authorization based on Google Zanzibar - `chat`: chat room functionality - `database`: database engine (MongoDB, PostgreSQL), CMS, CRUD/functional endpoint generation + - `embeddings`: opt-in text-to-vector generation and semantic search (disabled by default; not in standalone v1) - `email`: email sending with templates support - `forms`: form generation and submission - `push-notifications`: provides support for push notifications diff --git a/.github/workflows/embeddings-test.yml b/.github/workflows/embeddings-test.yml new file mode 100644 index 000000000..0f0c71cc9 --- /dev/null +++ b/.github/workflows/embeddings-test.yml @@ -0,0 +1,54 @@ +name: Embeddings offline tests + +on: + workflow_dispatch: + pull_request: + paths: + - 'modules/embeddings/**' + - 'libraries/grpc-sdk/**' + - 'libraries/module-tools/**' + - '.github/workflows/embeddings-test.yml' + push: + branches: + - main + paths: + - 'modules/embeddings/**' + - 'libraries/grpc-sdk/**' + - 'libraries/module-tools/**' + - '.github/workflows/embeddings-test.yml' + +permissions: + contents: read + pull-requests: read + +jobs: + test: + runs-on: ubuntu-24.04 + name: Embeddings unit and contract tests + steps: + - name: Checkout + uses: actions/checkout@v7 + + - uses: pnpm/action-setup@v6 + with: + version: 11.5.0 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + + - name: Install Protoc + uses: arduino/setup-protoc@v3 + with: + version: '29.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: pnpm install --frozen-lockfile --ignore-scripts + + - name: Build embeddings and workspace dependencies + run: pnpm exec turbo run build --filter=@conduitplatform/embeddings... + + - name: Run embeddings offline tests + run: pnpm --filter @conduitplatform/embeddings test diff --git a/.github/workflows/service-bundle-verify.yml b/.github/workflows/service-bundle-verify.yml index 9117de15c..83175c7e2 100644 --- a/.github/workflows/service-bundle-verify.yml +++ b/.github/workflows/service-bundle-verify.yml @@ -11,6 +11,7 @@ on: - 'modules/authorization/**' - 'modules/communications/**' - 'modules/database/**' + - 'modules/embeddings/**' - 'modules/router/**' - 'packages/core/**' - 'libraries/service-bundle/**' @@ -39,6 +40,7 @@ on: - 'modules/authorization/**' - 'modules/communications/**' - 'modules/database/**' + - 'modules/embeddings/**' - 'modules/router/**' - 'packages/core/**' - 'libraries/service-bundle/**' @@ -67,7 +69,7 @@ jobs: strategy: fail-fast: false matrix: - service: [chat, functions, storage, authentication, authorization, communications, database, router, core] + service: [chat, functions, storage, authentication, authorization, communications, database, embeddings, router, core] name: Verify ${{ matrix.service }} bundle steps: - name: Checkout diff --git a/Dockerfile b/Dockerfile index de7013193..9ed902748 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,8 @@ RUN pnpm install --frozen-lockfile --ignore-scripts && \ # Compile first, then bundle. Router/authentication turbo branches must not skip # build:bundle or COPY --from=conduit-base .../bundle fails in image CI. +# Standalone v1 (empty BUILDING_SERVICE) bundles core + database/router/authentication/ +# authorization/communications/storage/chat only. Embeddings is a separate image. RUN pnpm --filter @conduitplatform/service-bundle run build && \ if [ -z "$BUILDING_SERVICE" ] ; then npx turbo run build ; \ elif [ "$BUILDING_SERVICE" = "conduit" ] ; then npx turbo run build --filter=@conduitplatform/core --filter=@conduitplatform/hermes \ diff --git a/README.md b/README.md index 8cf88427c..960924966 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Can't find what you're interested in? Shoot us a message [on Discord](https://di - [Chat](https://getconduit.dev/docs/modules/chat) - Build realtime chat applications. - [Database](https://getconduit.dev/docs/modules/database) - Create schemas with auto-generated CRUD and Query-based functional endpoints. Supports MongoDB and PostgreSQL. - [Email](https://getconduit.dev/docs/modules/email) - Send emails using multiple supported providers. +- [Embeddings](modules/embeddings) - Opt-in text-to-vector generation and semantic search. Disabled by default; not included in standalone v1. See the [rollout runbook](deploy/embeddings.md). - [Forms](https://getconduit.dev/docs/modules/forms) - Submit forms and have responses forwarded to an email address. - [PushNotifications](https://getconduit.dev/docs/modules/push-notifications) - Send push notifications to your users. - [Router](https://getconduit.dev/docs/modules/router) - Seamlessly expose REST, GraphQL and WebSockets APIs with auto-generated endpoint documentation. diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 7d8ede759..8463bffe7 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -29,3 +29,6 @@ To run the microservices version: - Open the admin panel in your browser at [http://localhost:8080](http://localhost:8080) - Open the router in your browser at [http://localhost:8081](http://localhost:8081) - You can inject `--profile {profile_name}` command on compose to configure more services + (`mongodb` / `postgres` for the database engine, `embeddings` for the embeddings module). + Embeddings is disabled by default, omitted from standalone v1, and requires `GRPC_KEY` in + production. See the [embeddings rollout runbook](../embeddings.md). diff --git a/deploy/embeddings.md b/deploy/embeddings.md new file mode 100644 index 000000000..93c8fa620 --- /dev/null +++ b/deploy/embeddings.md @@ -0,0 +1,68 @@ +# Embeddings rollout and rollback + +Embeddings is a separate, disabled-by-default module image. It is not part of +standalone v1. Live MongoDB Atlas, pgvector, Redis, and provider suites are +**not** covered by CI; operators must complete the capability and index +readiness checks below before enabling generation or search. + +Production containers set `NODE_ENV=production` and **require `GRPC_KEY`**. +The module stays disabled (`enabled: false`) until an operator enables it +through Core config after peer health and vector capabilities are confirmed. + +## Compose (opt-in) + +```bash +# Set GRPC_KEY in docker/.env before starting embeddings. +docker compose --profile mongodb --profile embeddings up +``` + +- gRPC: `55165` (`EMBEDDINGS_GRPC_PORT`) +- Metrics: `9192` (Prometheus scrapes `conduit-embeddings:9192`) +- Image: `docker.io/conduitplatform/embeddings:${IMAGE_TAG}` + +Helm values (`install.embeddings`) are documented in the charts repository +and remain disabled by default. + +## Rollout order + +1. Release compatible Core, Database, grpc-sdk, and the embeddings image. +2. Deploy embeddings **disabled**. Confirm the process is serving and waiting + on / registered with Core. Health stays serving while disabled so operators + can configure the module. +3. Set `GRPC_KEY` (required in production). Confirm gRPC peer health. +4. Call `GET /embeddings/capabilities` (or gRPC `getCapabilities`) and verify + Database `getVectorCapabilities`: storage, indexing, and search must be + true for the target backend (MongoDB Atlas Vector Search or Postgres + pgvector). Saving a disabled config may succeed with capability warnings; + activation must not. +5. Configure the HTTPS provider (`endpoint`, `apiKey`, `allowedHosts`). Check + `GET /embeddings/status` for provider/index warnings. +6. Create an embedding config with `enabled: false`. Wait until the vector + index for `targetField` is queryable (`status` ready, not pending/failed). +7. Enable the config only after index readiness. Start a **bounded** backfill + (`onlyMissing` recommended). Watch `GET /embeddings/backfills/:id` and + `GET /embeddings/status` queue counts. Do not scan collections in the + request thread; backfills are queued. +8. Run a scoped canary semantic search (`POST /embeddings/search` as an + operator, or client search with authenticated user/scope). Confirm + fail-closed behavior on authorization-enabled schemas. +9. Enable workers/search for normal traffic (`enabled: true` on module config). + +## Rollback + +1. Disable workers and embedding configs (`enabled: false`). Generation and + search stop; existing vectors remain. +2. Scale down or stop the embeddings service: + - Compose: omit `--profile embeddings` / `docker compose stop embeddings` + - Helm: `install.embeddings: false` (charts repo) +3. Roll back the embeddings image and/or chart to the previous version. +4. Do **not** automatically delete vector fields, indexes, `EmbeddingConfig` + documents, `BackfillRun` records, or Redis/BullMQ state. Data and index + removal is a separate explicit operator action. + +## Residual validation + +Offline CI covers unit/contract tests, bundle smoke (`Waiting for Core`), +image target discovery, and compose rendering. It does not prove Atlas, +pgvector, Redis queue behavior, or a live provider. Repeat capability and +index readiness checks in the target environment before activation. diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index dee03b64c..94891887a 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -6,3 +6,7 @@ We've included some basic instructions to get you started on a local k8s cluster Current setup includes: - [Minikube](minikube.md) - [AKS](aks.md) + +Embeddings is not part of the standalone image. For a disabled-by-default +embeddings rollout, capability/index readiness, and rollback, see +[embeddings.md](../embeddings.md). Helm values live in the charts repository. diff --git a/docker-bake.hcl b/docker-bake.hcl index 7360ed224..e28038b3f 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -159,6 +159,24 @@ target "database" { } } +target "conduit-base-bundle-embeddings" { + inherits = ["conduit-base"] + args = { + BUILDING_SERVICE = "modules/embeddings" + BUILD_BUNDLE = "1" + } +} + +target "embeddings" { + inherits = ["_runtime"] + context = "modules/embeddings" + dockerfile = "Dockerfile" + contexts = { + conduit-base = "target:conduit-base-bundle-embeddings" + conduit-builder = "target:conduit-builder" + } +} + target "functions" { inherits = ["_runtime"] context = "modules/functions" @@ -230,6 +248,7 @@ group "all" { "chat", "communications", "database", + "embeddings", "functions", "router", "storage", diff --git a/docker/.env b/docker/.env index 66dad89bd..c4194ff64 100644 --- a/docker/.env +++ b/docker/.env @@ -10,6 +10,7 @@ AUTHN_GRPC_PORT="55162" AUTHZ_GRPC_PORT="55169" CHAT_GRPC_PORT="55163" COMMS_GRPC_PORT="55164" +EMBEDDINGS_GRPC_PORT="55165" STORAGE_GRPC_PORT="55168" @@ -30,6 +31,7 @@ DB_CONN_URI="mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin #DB_CONN_URI="postgres://conduit:pass@conduit-postgres:5432/conduit" # profile: postgres # Security +# Embeddings (NODE_ENV=production in the image) requires a non-empty GRPC_KEY. CORE_MASTER_KEY="M4ST3RK3Y" GRPC_KEY="" diff --git a/docker/docker-compose.standalone.yml b/docker/docker-compose.standalone.yml index 8cd0b0e9b..b061788dd 100644 --- a/docker/docker-compose.standalone.yml +++ b/docker/docker-compose.standalone.yml @@ -2,6 +2,8 @@ # This compose file deploys a "standalone" version of conduit with most modules # packaged in a single image. Loki and Prometheus are not deployed, since # metrics and logs can be viewed directly from the Docker daemon. +# Embeddings is not included in standalone v1; use docker-compose.yml with +# --profile embeddings after publishing a compatible embeddings image. #------------------------------------------------------------------------------------------- version: '3.9' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 83b3e9b55..610f81a17 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,9 +11,11 @@ # Otherwise, simply update the env values with any available port. # 3. Specify at least '--profile mongodb' or '--profile postgres' # If you're going to use PostgreSQL, swap out 'DB_CONN_URI' in the '.env' file +# 4. Embeddings is opt-in: add '--profile embeddings'. Production requires GRPC_KEY. # # Examples: # docker compose --profile mongodb up +# docker compose --profile mongodb --profile embeddings up # ---------------------------------------------------------------------------------------------- version: '3.9' @@ -226,6 +228,32 @@ services: extra_hosts: - host.docker.internal:host-gateway + embeddings: + container_name: 'conduit-embeddings' + image: 'docker.io/conduitplatform/embeddings:${IMAGE_TAG}' + restart: unless-stopped + profiles: ['embeddings'] + depends_on: + - core + - database + - prometheus + - loki + ports: + - '${EMBEDDINGS_GRPC_PORT:-55165}:${EMBEDDINGS_GRPC_PORT:-55165}' + environment: + CONDUIT_SERVER: 'conduit:${CORE_GRPC_PORT:-55152}' + SERVICE_URL: 'conduit-embeddings:${EMBEDDINGS_GRPC_PORT:-55165}' + GRPC_PORT: '55165' + METRICS_PORT: '9192' + LOKI_URL: 'http://conduit-loki:3100' + GRPC_KEY: '${GRPC_KEY}' + networks: + default: + aliases: + - conduit-embeddings + extra_hosts: + - host.docker.internal:host-gateway + storage: container_name: 'conduit-storage' image: 'docker.io/conduitplatform/storage:${IMAGE_TAG}' diff --git a/docker/prometheus.cfg.yml b/docker/prometheus.cfg.yml index 286b307b1..3d3dc687e 100644 --- a/docker/prometheus.cfg.yml +++ b/docker/prometheus.cfg.yml @@ -38,6 +38,9 @@ scrape_configs: - labels: module: 'Communications' targets: ['conduit-communications:9096'] + - labels: + module: 'Embeddings' + targets: ['conduit-embeddings:9192'] - labels: module: 'Storage' targets: ['conduit-storage:9190'] diff --git a/libraries/grpc-sdk/src/index.ts b/libraries/grpc-sdk/src/index.ts index 31a709991..d43ebdffa 100644 --- a/libraries/grpc-sdk/src/index.ts +++ b/libraries/grpc-sdk/src/index.ts @@ -160,12 +160,10 @@ class ConduitGrpcSdk { } private _redisDetails?: - | RedisOptions - | { nodes: { host: string; port: number }[]; options: ClusterOptions }; + RedisOptions | { nodes: { host: string; port: number }[]; options: ClusterOptions }; get redisDetails(): - | RedisOptions - | { nodes: { host: string; port: number }[]; options: ClusterOptions } { + RedisOptions | { nodes: { host: string; port: number }[]; options: ClusterOptions } { if (this._redisDetails) { return this._redisDetails; } else { @@ -823,5 +821,41 @@ export * from './classes/index.js'; export * from './modules/index.js'; export * from './constants/index.js'; export * from './types/index.js'; -export * from './protoUtils/index.js'; +export * from './protoUtils/authentication.js'; +export * from './protoUtils/authorization.js'; +export * from './protoUtils/chat.js'; +export * from './protoUtils/communications.js'; +export * from './protoUtils/core.js'; +export * from './protoUtils/database.js'; +export * from './protoUtils/grpc_health_check.js'; +export * from './protoUtils/module.js'; +export * from './protoUtils/router.js'; +export * from './protoUtils/storage.js'; +// Embeddings proto VectorCapabilities/QueueCounts collide with the public SDK types. +export { + BackfillMutationResponse, + BackfillRun, + CancelBackfillRequest, + DeleteEmbeddingConfigRequest, + DeleteEmbeddingConfigResponse, + EmbeddingConfig, + EmbeddingsProviderDefinition, + GetBackfillRequest, + GetCapabilitiesRequest, + GetCapabilitiesResponse, + GetConfigsRequest, + GetConfigsResponse, + GetStatusRequest, + GetStatusResponse, + ListBackfillsRequest, + ListBackfillsResponse, + ResumeBackfillRequest, + SemanticSearchHit, + SemanticSearchRequest, + SemanticSearchResponse, + StartBackfillRequest, + StartBackfillResponse, + UpsertConfigRequest, + UpsertConfigResponse, +} from './protoUtils/embeddings.js'; export * from '@grpc/grpc-js'; diff --git a/modules/database/README.md b/modules/database/README.md index 4eced2d94..950630943 100644 --- a/modules/database/README.md +++ b/modules/database/README.md @@ -40,6 +40,10 @@ responses distinguish storage support from index/search support: 4. Backfill embeddings. 5. Run `vectorSearch` with a query vector. +The Embeddings module is a separate opt-in image (not standalone v1). See +[deploy/embeddings.md](../../deploy/embeddings.md) before enabling generation +or search. Live Atlas/pgvector validation is an operator runbook step, not CI. + CMS create/update bodies omit `TYPE.Vector` fields, `*SourceHash` fields, and any `select: false` field so clients cannot write managed embeddings. Read/return projections still include those schema fields. diff --git a/modules/embeddings/Dockerfile b/modules/embeddings/Dockerfile new file mode 100644 index 000000000..3d7742e9f --- /dev/null +++ b/modules/embeddings/Dockerfile @@ -0,0 +1,18 @@ +FROM conduit-builder + +WORKDIR /app/modules/embeddings + +COPY --from=conduit-base /app/modules/embeddings/bundle /app/modules/embeddings/bundle +COPY --from=conduit-base /app/modules/embeddings/package.bundle.json /app/modules/embeddings/package.json +COPY --from=conduit-base /app/modules/embeddings/package.bundle-lock.json /app/modules/embeddings/package-lock.json + +RUN npm ci --omit=dev --ignore-scripts && npm cache clean --force + +ENV NODE_ENV=production +ENV CONDUIT_SERVER=conduit_server +ENV SERVICE_URL=0.0.0.0:5000 +ENV GRPC_PORT=5000 + +EXPOSE 5000 + +CMD ["node", "bundle/index.js"] diff --git a/modules/embeddings/README.md b/modules/embeddings/README.md index 96365846e..2979e46f4 100644 --- a/modules/embeddings/README.md +++ b/modules/embeddings/README.md @@ -6,8 +6,12 @@ for vector storage, index creation, and vector-in/vector-out search. ## Configuration -The module is disabled by default. Enable it and configure an OpenAI-compatible -provider: +The module is disabled by default. Production deployments require `GRPC_KEY` +(`NODE_ENV=production` in the published image). The module is **not** included +in the standalone image for the first production release; run it as a separate +opt-in compose profile or Helm service. + +Enable it and configure an OpenAI-compatible provider: ```json { @@ -60,3 +64,17 @@ tools through Hermes: Config and backfill APIs are never exposed as client routes. Client `POST /embeddings/search` accepts text only and takes user/scope from the authenticated router context. + +## Packaging + +- Bundle image: `docker.io/conduitplatform/embeddings` (BullMQ is an extra + bundle dependency). Bake target: `embeddings`. +- Compose: `docker compose --profile embeddings up` (gRPC `55165`, metrics + `9192`). Set `GRPC_KEY` before starting. +- Standalone v1 does not ship embeddings. + +Operator rollout, capability/index readiness, and rollback: +[deploy/embeddings.md](../../deploy/embeddings.md). + +Live Atlas/pgvector/provider behavior is not covered by CI. Repeat the +capability and index checks in the target environment before activation. diff --git a/modules/embeddings/package.bundle-lock.json b/modules/embeddings/package.bundle-lock.json new file mode 100644 index 000000000..d90b14ac4 --- /dev/null +++ b/modules/embeddings/package.bundle-lock.json @@ -0,0 +1,2629 @@ +{ + "name": "@conduitplatform/embeddings", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@conduitplatform/embeddings", + "version": "1.0.0", + "dependencies": { + "@bufbuild/protobuf": "^2.10.2", + "@grpc/grpc-js": "^1.14.3", + "@grpc/proto-loader": "^0.8.0", + "@sesamecare-oss/redlock": "^1.4.0", + "abort-controller-x": "^0.5.0", + "axios": "1.18.0", + "bullmq": "^5.21.2", + "convict": "^6.2.5", + "escape-string-regexp": "1.0.5", + "express": "^5.2.1", + "fast-jwt": "^6.2.4", + "fs-extra": "^11.3.5", + "ioredis": "^5.10.1", + "lodash": "^4.18.1", + "lodash-es": "^4.18.1", + "nice-grpc": "^2.1.17", + "nice-grpc-client-middleware-retry": "^3.1.16", + "nice-grpc-common": "^2.0.4", + "prom-client": "^15.1.3", + "protobufjs": "^8.7.2", + "snappy": "7.4.1", + "uuid": "14.0.2", + "winston": "^3.19.0", + "winston-loki": "^6.1.7" + }, + "engines": { + "node": ">=24" + } + }, + "node_modules/@bufbuild/protobuf": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.14.1.tgz", + "integrity": "sha512-agRJn3+EJDUe8AvxTx/LnHA/GErvLE62pSaSk7+MwFOtOv8eWBu/qCq2qoZjBjVZ3C2aiFJCveuSs17KMkYGOw==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, + "node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "license": "MIT", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@dabh/diagnostics": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.8.tgz", + "integrity": "sha512-R4MSXTVnuMzGD7bzHdW2ZhhdPC/igELENcq5IjEverBvq5hn1SXCWcsi6eSsdWP0/Ur+SItRRjAktmdoX/8R/Q==", + "license": "MIT", + "dependencies": { + "@so-ric/colorspace": "^1.1.6", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.4.tgz", + "integrity": "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@grpc/proto-loader/node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "license": "MIT" + }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/@lukeed/ms": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@lukeed/ms/-/ms-2.0.2.tgz", + "integrity": "sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.4.tgz", + "integrity": "sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.4.tgz", + "integrity": "sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.4.tgz", + "integrity": "sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.4.tgz", + "integrity": "sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.4.tgz", + "integrity": "sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.4.tgz", + "integrity": "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@napi-rs/snappy-android-arm-eabi": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.4.1.tgz", + "integrity": "sha512-7siGMYnpi4pjI07XXoIgXlBkIbI/XsmXYMi+dSPEZz/7V/f/4MvK9OGGKT4cgHLiszgxkTSTZzL+AjCF7OACtQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-android-arm64": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.4.1.tgz", + "integrity": "sha512-8tcrG2V3LSzCS3OdppuvalNbsgsvyr8hIkUliPBaFzjOiAjRpCOyS05s7KZe7YdkJyAht/OCTjIV1XqKdAUJXw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-darwin-arm64": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.4.1.tgz", + "integrity": "sha512-imzAEKEySv3dmzMFCnVj0SeaVbaZptlVGjQXLuIQ1n157/DpqJYuG4Yj1pPIRh3S/VzniC3ewPmYQWfxWepRsA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-darwin-x64": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.4.1.tgz", + "integrity": "sha512-Q9LDalgq5uqpd1JwpWxKb34rmheOmk7BhlwUMadaECAZU/v5HwArAZjsWomRXiYm4tfXFnQeL3ItFGW0Irv6+A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-freebsd-x64": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.4.1.tgz", + "integrity": "sha512-O1B/ynTjOOlnr6dnVHdj/xrYcPcWLjZPC3dNWyOy7xGbkjQHHYiTt0EXDxmfANALt9YEa1rSfzICMmKcyFCtnQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm-gnueabihf": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.4.1.tgz", + "integrity": "sha512-8vY+IGu1qm2EEuqEvanMh73edIFlRD19HeXMSEZZLOzJeIhlOcMGCAqKApPJSgdwMf7nxNmnBYyXnH+QM1ryQA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm64-gnu": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.4.1.tgz", + "integrity": "sha512-FRjLCPfbtmr4B4OwR0gIEAKnnaczyLKL7QtG3iAn5+n4g284Fur1RoIGTCloq7zGKAuPbKBRhrARCOrXr7nx3A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm64-musl": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-musl/-/snappy-linux-arm64-musl-7.4.1.tgz", + "integrity": "sha512-seZSp/mCSOMZ/ve29s3bAhnLTPpwUZmf9N7+HCCndVckbS6lzRfasstn4RDWP2fVn0MoGbCDFAHADrGP55dzPQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-ppc64-gnu": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-ppc64-gnu/-/snappy-linux-ppc64-gnu-7.4.1.tgz", + "integrity": "sha512-vBguLu+Dc3J7pwn6QXYDK1iPgGSoqfSig7O1FQMg7v9iUxexR7IAna7vKMTprMHP8FDM9szxcYOzEMrNGmcdaA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-riscv64-gnu": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-riscv64-gnu/-/snappy-linux-riscv64-gnu-7.4.1.tgz", + "integrity": "sha512-UmgLvreYF6NWV/cBOwZp25dOm5uxiOTC8Rj+vFdOAx8whi/jiwui/pbxjjdUfwBQutzxFk3nMOSC5Jq6TgAZTA==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-s390x-gnu": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-s390x-gnu/-/snappy-linux-s390x-gnu-7.4.1.tgz", + "integrity": "sha512-mXTqUnLUMZeYiSMT6wRiDEmqLDm9VJXJhhtKExqK2sYiTPRGQfn1NR/qzv192LlOsXikIRvY6jWMq4Ron0LA1Q==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-x64-gnu": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.4.1.tgz", + "integrity": "sha512-D+2LJTgAAv10SRZcX9rdmdx/rgqU+Lp1oJvZAMkiL7fhkFeHmKRHND8DOkMyAdcVi2ES7EpZnUJBXdcYqxWxjQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-x64-musl": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-musl/-/snappy-linux-x64-musl-7.4.1.tgz", + "integrity": "sha512-2DgFW7mQ9cy0EtiddIOuEk9rPUCnt8xv875AYtEjjPrQArXbgVmB/Lt6ngbTlckv5Xgb0Cx4lt7R7ahWXcTbxw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-openharmony-arm64": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-openharmony-arm64/-/snappy-openharmony-arm64-7.4.1.tgz", + "integrity": "sha512-sF1LJRTvZn2A3AyrZUEr6zpNEQW5mm9GO09clzd0nQJlNKoPbWaLusv+nhZYTBxEVLFApISp1ggL8yPNkIpjvA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-win32-arm64-msvc": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.4.1.tgz", + "integrity": "sha512-sqZbbT1yKlKp1LPO4Yqkja6He0txXOvWKANKZJM98vvHqQikLayq4X6ogovzVXOFYzmpkHFdT10NfGwh4hZbog==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-win32-ia32-msvc": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.4.1.tgz", + "integrity": "sha512-RrgbOcrH52k+dFfuey2wm98OxNmNayI4qyGZ1J36pGWMEOemft+g10zsVzHkN86J5D1RWcD6z25SzB87eW3SEQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-win32-x64-msvc": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.4.1.tgz", + "integrity": "sha512-n8OpxSCvNr1QPmXquDkMatYQaG80pjXrmgPuwvbXbGTr8BbZay6wZWlAE0OW32mEBrGWGqLUs4uheIJdF1cLVA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@sesamecare-oss/redlock": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@sesamecare-oss/redlock/-/redlock-1.4.0.tgz", + "integrity": "sha512-2z589R+yxKLN4CgKxP1oN4dsg6Y548SE4bVYam/R0kHk7Q9VrQ9l66q+k1ehhSLLY4or9hcchuF9/MhuuZdjJg==", + "license": "UNLICENSED", + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "ioredis": ">=5" + } + }, + "node_modules/@so-ric/colorspace": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@so-ric/colorspace/-/colorspace-1.1.6.tgz", + "integrity": "sha512-/KiKkpHNOBgkFJwu9sh48LkHSMYGyuTcSFK/qMBdnOAlrRJzRSXAOFB5qwzaVQuDl8wAvHVMkaASQDReTahxuw==", + "license": "MIT", + "dependencies": { + "color": "^5.0.2", + "text-hex": "1.0.x" + } + }, + "node_modules/@types/node": { + "version": "26.4.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.1.tgz", + "integrity": "sha512-k97ENvZWtvA6yqz5/FS6a7duDgOPEeOQOc2iKS/nY6mX6qJUKtLnWzQS+Xj6tXweyj6ZcTAK2Qecetnvi9nCLA==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/@types/triple-beam": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", + "license": "MIT" + }, + "node_modules/abort-controller-x": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/abort-controller-x/-/abort-controller-x-0.5.0.tgz", + "integrity": "sha512-yTt9CI0x+nRfX6BFMenEGP8ooPvErGH6AbFz20C2IeOLIlDsrw/VHpgne3GsCEuTA410IiFiaLVFKmgM4bKEPQ==", + "license": "MIT" + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/ansi-styles/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/ansi-styles/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "license": "MIT", + "dependencies": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/async-exit-hook": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/async-exit-hook/-/async-exit-hook-2.0.1.tgz", + "integrity": "sha512-NW2cX8m1Q7KPA7a5M2ULQeZ2wR5qI5PAbw5L0UOMxdioVk9PMZ0h1TmyZEkPYrCvYjDlFICusOu1dlEKAAeXBw==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.18.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.18.0.tgz", + "integrity": "sha512-E32NzpYKp++W7XRe52rHiXV2ehxmh3wbdgO7MHeFM+vqxLBYHzt0ElkiImtOBxtOmyp0yoC8C6uESVV84Y2/hw==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.16.0", + "form-data": "^4.0.5", + "https-proxy-agent": "^5.0.1", + "proxy-from-env": "^2.1.0" + } + }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, + "node_modules/bn.js": { + "version": "4.12.5", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.5.tgz", + "integrity": "sha512-3aRg6/JxfffFD+OlOjOFR3Vo79l39ooBTFucxx+MT3dhCtzn3EmiUPQo+6/OZuI2jbXi3YKgmiTFBgChQMwIRQ==", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/btoa": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", + "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==", + "license": "(MIT OR Apache-2.0)", + "bin": { + "btoa": "bin/btoa.js" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/bullmq": { + "version": "5.81.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.81.4.tgz", + "integrity": "sha512-n+WHSzz20KooBGoJyASre6oJNz/p5f1IJRRN2ibD+NWPQaNpNQmDrwYuPO+bsXrQeM1MQzUxXGbdjmyOFKP2xQ==", + "license": "MIT", + "dependencies": { + "cron-parser": "4.9.0", + "ioredis": "5.11.1", + "msgpackr": "2.0.5", + "node-abort-controller": "3.1.1", + "semver": "7.8.5", + "tslib": "2.8.1" + }, + "engines": { + "node": ">=12.22.0" + }, + "peerDependencies": { + "redis": ">=5.0.0" + }, + "peerDependenciesMeta": { + "redis": { + "optional": true + } + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/color/-/color-5.0.3.tgz", + "integrity": "sha512-ezmVcLR3xAVp8kYOm4GS45ZLLgIE6SPAFoduLr6hTDajwb3KZ2F46gulK3XpcwRFb5KKGCSezCBAY4Dw4HsyXA==", + "license": "MIT", + "dependencies": { + "color-convert": "^3.1.3", + "color-string": "^2.1.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/color-convert": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-3.1.3.tgz", + "integrity": "sha512-fasDH2ont2GqF5HpyO4w0+BcewlhHEZOFn9c1ckZdHpJ56Qb7MHhH/IcJZbBGgvdtwdwNbLvxiBEdg336iA9Sg==", + "license": "MIT", + "dependencies": { + "color-name": "^2.0.0" + }, + "engines": { + "node": ">=14.6" + } + }, + "node_modules/color-name": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-2.1.1.tgz", + "integrity": "sha512-p2FdgwVx1a9yWBHP2wI0VgShkDpgN4kZISkxdNipGBJWpa5G6b04OINlVWCyJj0JmfvcPrgqt95E9k8yvaOJFg==", + "license": "MIT", + "engines": { + "node": ">=12.20" + } + }, + "node_modules/color-string": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-2.1.4.tgz", + "integrity": "sha512-Bb6Cq8oq0IjDOe8wJmi4JeNn763Xs9cfrBcaylK1tPypWzyoy2G3l90v9k64kjphl/ZJjPIShFztenRomi8WTg==", + "license": "MIT", + "dependencies": { + "color-name": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/convict": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/convict/-/convict-6.2.5.tgz", + "integrity": "sha512-JtXpxqDqJ8P0UwEHwhxLzCIXQy97vlYBZR222Sbzb1q1Erex9ASrztJ29SyhWFQjod1AeFBaPzEEC8YvtZMIYg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.clonedeep": "^4.5.0", + "yargs-parser": "^20.2.7" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "deprecated": "v4 is no longer maintained, upgrade to v5", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/fast-jwt": { + "version": "6.3.3", + "resolved": "https://registry.npmjs.org/fast-jwt/-/fast-jwt-6.3.3.tgz", + "integrity": "sha512-pQDXx7IHeZT4jSmpE9o80RrBqfrG4fPrl8anazSM5vErIdK1iCc13z/EWX+H0j7liWSRnwTpHswIKMeLYGAckw==", + "license": "Apache-2.0", + "dependencies": { + "@lukeed/ms": "^2.0.2", + "asn1.js": "^5.4.1", + "ecdsa-sig-formatter": "^1.0.11", + "mnemonist": "^0.40.0", + "safe-regex2": "^5.1.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", + "license": "MIT" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==", + "license": "MIT" + }, + "node_modules/follow-redirects": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.4", + "mime-types": "^2.1.35" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/form-data/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/fs-extra": { + "version": "11.4.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.0.tgz", + "integrity": "sha512-EQsFzMUJkCKGr1ePqlYADkIUmHW1s3ZXr5Yqy6wbGrfUCphpl2maM/kyOIRA2HpP3AaFQTZXD4ldjek+nccddA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jsonfile": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", + "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", + "license": "MIT" + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, + "node_modules/lodash-es": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", + "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", + "license": "MIT" + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==", + "license": "MIT" + }, + "node_modules/logform": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.7.0.tgz", + "integrity": "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ==", + "license": "MIT", + "dependencies": { + "@colors/colors": "1.6.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", + "license": "ISC" + }, + "node_modules/mnemonist": { + "version": "0.40.4", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.40.4.tgz", + "integrity": "sha512-ZAv+KNavneRVzu4tUeOgzkScI3W5BGwZ3rkxIpKtzzVgfTtWQFN1CgX0U72cyvyh3iTuHL3SiSmrQxTlryEIcw==", + "license": "MIT", + "dependencies": { + "obliterator": "^2.0.4" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/msgpackr": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-2.0.5.tgz", + "integrity": "sha512-cef05H/dSYpLpqp3sj/qyZh5vhUYCalnaLO7j1yOmpsR0y/XwLVtK7r5gn+U/F7CTEfMowcGhlUQJDLcLf7jcA==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.4" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.4.tgz", + "integrity": "sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.4" + } + }, + "node_modules/negotiator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/negotiator/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/nice-grpc": { + "version": "2.1.17", + "resolved": "https://registry.npmjs.org/nice-grpc/-/nice-grpc-2.1.17.tgz", + "integrity": "sha512-pu9xYPlWSeqoYQOCqb2ftQqZi/P2DaI70PSuwnxvoyIRiilaOVtktyPe6LmuXegWB4Ic1sPuDGCnCCASbwzK0w==", + "license": "MIT", + "dependencies": { + "@grpc/grpc-js": "^1.14.0", + "abort-controller-x": "^0.5.0", + "nice-grpc-common": "^2.0.4" + } + }, + "node_modules/nice-grpc-client-middleware-retry": { + "version": "3.1.16", + "resolved": "https://registry.npmjs.org/nice-grpc-client-middleware-retry/-/nice-grpc-client-middleware-retry-3.1.16.tgz", + "integrity": "sha512-8EbOCUZZ1Uq1pCfTD8dB2zMU1AYrilRvm9al5VSUi444eqhPhxElnI7iqP6ZY/ixb1NGsvMpfbU5FW/3L7DdFw==", + "license": "MIT", + "dependencies": { + "abort-controller-x": "^0.5.0", + "nice-grpc-common": "^2.0.4" + } + }, + "node_modules/nice-grpc-common": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/nice-grpc-common/-/nice-grpc-common-2.0.4.tgz", + "integrity": "sha512-gOEXlD6ShXZMZ8k+49wm/bA2j1+3IKbEFV9WYREJcvZV4kM5L1KkILYTX9VYBzZF2OuYCe1wp8c82bQkvR0fHw==", + "license": "MIT", + "dependencies": { + "ts-error": "^1.0.6" + } + }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", + "license": "MIT" + }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/obliterator": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.5.tgz", + "integrity": "sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==", + "license": "MIT" + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "license": "MIT", + "dependencies": { + "fn.name": "1.x.x" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "deprecated": "prom-client has been replaced by @prometheus-io/client", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, + "node_modules/protobufjs": { + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.8.0.tgz", + "integrity": "sha512-N3xhQ5yyBx3vQq4gubBfASzYhJGNzeDbjqBpu61g7UVylsN/qyffU96TKWD3GbbLOKF82VGNRNvv1+BFgE31Eg==", + "license": "BSD-3-Clause", + "dependencies": { + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/proxy-from-env": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", + "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/qs": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ret": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.5.0.tgz", + "integrity": "sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-regex2": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/safe-regex2/-/safe-regex2-5.1.1.tgz", + "integrity": "sha512-mOSBvHGDZMuIEZMdOz/aCEYDCv0E7nfcNsIhUF+/P+xC7Hyf3FkvymqgPbg9D1EdSGu+uKbJgy09K/RKKc7kJA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "ret": "~0.5.0" + }, + "bin": { + "safe-regex2": "bin/safe-regex2.js" + } + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/snappy": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/snappy/-/snappy-7.4.1.tgz", + "integrity": "sha512-Em7vzkTe3d2K4BHhEoKqU3Xz6cZnVuAmx66/iMlly7KZDBIviIRcNRejsvWCQg+/snpalbh1mWZfIKzCfFyiHg==", + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/snappy-android-arm-eabi": "7.4.1", + "@napi-rs/snappy-android-arm64": "7.4.1", + "@napi-rs/snappy-darwin-arm64": "7.4.1", + "@napi-rs/snappy-darwin-x64": "7.4.1", + "@napi-rs/snappy-freebsd-x64": "7.4.1", + "@napi-rs/snappy-linux-arm-gnueabihf": "7.4.1", + "@napi-rs/snappy-linux-arm64-gnu": "7.4.1", + "@napi-rs/snappy-linux-arm64-musl": "7.4.1", + "@napi-rs/snappy-linux-ppc64-gnu": "7.4.1", + "@napi-rs/snappy-linux-riscv64-gnu": "7.4.1", + "@napi-rs/snappy-linux-s390x-gnu": "7.4.1", + "@napi-rs/snappy-linux-x64-gnu": "7.4.1", + "@napi-rs/snappy-linux-x64-musl": "7.4.1", + "@napi-rs/snappy-openharmony-arm64": "7.4.1", + "@napi-rs/snappy-win32-arm64-msvc": "7.4.1", + "@napi-rs/snappy-win32-ia32-msvc": "7.4.1", + "@napi-rs/snappy-win32-x64-msvc": "7.4.1" + } + }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tdigest": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.3.tgz", + "integrity": "sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + }, + "node_modules/text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", + "license": "MIT" + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/triple-beam": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/ts-error": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/ts-error/-/ts-error-1.0.6.tgz", + "integrity": "sha512-tLJxacIQUM82IR7JO1UUkKlYuUTmoY9HBJAmNWFzheSlDS5SPMcNIepejHJa4BpPQLAcbRhRf3GDJzyj6rbKvA==", + "license": "MIT" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "license": "MIT" + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/url-polyfill": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/url-polyfill/-/url-polyfill-1.1.14.tgz", + "integrity": "sha512-p4f3TTAG6ADVF3mwbXw7hGw+QJyw5CnNGvYh5fCuQQZIiuKUswqcznyV3pGDP9j0TSmC4UvRKm8kl1QsX1diiQ==", + "license": "MIT" + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/uuid": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.2.tgz", + "integrity": "sha512-xZe/16rV4aa+HGSOCiY2YeLT1OybRLrrkL/Rqaq7p7GMVXjFh+6wN4oMYgjFmnSnhY8t6Xpdl2l9qmnHYuMHwQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/winston": { + "version": "3.19.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz", + "integrity": "sha512-LZNJgPzfKR+/J3cHkxcpHKpKKvGfDZVPS4hfJCc4cCG0CgYzvlD6yE/S3CIL/Yt91ak327YCpiF/0MyeZHEHKA==", + "license": "MIT", + "dependencies": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.8", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.7.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.9.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-loki": { + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/winston-loki/-/winston-loki-6.1.7.tgz", + "integrity": "sha512-QxwyQJezn3ZmDx1AsngecxwnZ8SoVuJ8hW4LtK5CA+9eyswgH4+AE93fY7t/Ub6s/zu4KDP1qB9Lxb8tO8vvAw==", + "license": "MIT", + "dependencies": { + "async-exit-hook": "2.0.1", + "btoa": "^1.2.1", + "protobufjs": "^7.2.4", + "url-polyfill": "^1.1.12", + "winston-transport": "^4.3.0" + }, + "optionalDependencies": { + "snappy": "^7.2.2" + } + }, + "node_modules/winston-loki/node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/winston-transport": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.9.0.tgz", + "integrity": "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A==", + "license": "MIT", + "dependencies": { + "logform": "^2.7.0", + "readable-stream": "^3.6.2", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + } + } +} diff --git a/modules/embeddings/package.bundle.json b/modules/embeddings/package.bundle.json new file mode 100644 index 000000000..d3bc4bdeb --- /dev/null +++ b/modules/embeddings/package.bundle.json @@ -0,0 +1,49 @@ +{ + "name": "@conduitplatform/embeddings", + "version": "1.0.0", + "private": true, + "type": "module", + "main": "bundle/index.js", + "engines": { + "node": ">=24" + }, + "conduit": { + "peers": { + "await": [ + "database" + ], + "watch": [ + { + "module": "database", + "edge": "rising" + } + ] + } + }, + "dependencies": { + "@bufbuild/protobuf": "^2.10.2", + "@grpc/grpc-js": "^1.14.3", + "@grpc/proto-loader": "^0.8.0", + "@sesamecare-oss/redlock": "^1.4.0", + "abort-controller-x": "^0.5.0", + "axios": "1.18.0", + "convict": "^6.2.5", + "escape-string-regexp": "1.0.5", + "express": "^5.2.1", + "fast-jwt": "^6.2.4", + "fs-extra": "^11.3.5", + "ioredis": "^5.10.1", + "lodash": "^4.18.1", + "lodash-es": "^4.18.1", + "nice-grpc": "^2.1.17", + "nice-grpc-client-middleware-retry": "^3.1.16", + "nice-grpc-common": "^2.0.4", + "prom-client": "^15.1.3", + "protobufjs": "^8.7.2", + "snappy": "7.4.1", + "uuid": "14.0.2", + "winston": "^3.19.0", + "winston-loki": "^6.1.7", + "bullmq": "^5.21.2" + } +} diff --git a/modules/embeddings/package.json b/modules/embeddings/package.json index 525e27b15..a1845dae1 100644 --- a/modules/embeddings/package.json +++ b/modules/embeddings/package.json @@ -17,11 +17,15 @@ }, "scripts": { "start": "node dist/index.js", + "start:bundle": "node bundle/index.js", "prebuild": "npm run generateTypes", "build": "rimraf dist && tsc", "postbuild": "copyfiles -u 1 src/**/*.proto src/*.proto ./dist/", + "prebuild:bundle": "pnpm --filter @conduitplatform/service-bundle run build", + "build:bundle": "rimraf bundle && node ../../libraries/service-bundle/dist/cli.js generate-manifest && tsup && node ../../libraries/service-bundle/dist/cli.js copy-assets && node ../../libraries/service-bundle/dist/cli.js generate-lockfile", "generateTypes": "sh build.sh", - "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utils/*.test.js dist-test/controllers/*.test.js dist-test/providers/*.test.js dist-test/api/*.test.js test/embedding-contract.test.mjs" + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utils/*.test.js dist-test/controllers/*.test.js dist-test/providers/*.test.js dist-test/api/*.test.js test/embedding-contract.test.mjs", + "build:docker": "docker build -t ghcr.io/conduitplatform/embeddings:latest -f ./Dockerfile ../../ && docker push ghcr.io/conduitplatform/embeddings:latest" }, "dependencies": { "@bufbuild/protobuf": "^2.10.2", @@ -35,12 +39,14 @@ "lodash-es": "^4.18.1" }, "devDependencies": { + "@conduitplatform/service-bundle": "workspace:*", "@types/convict": "^6.1.6", "@types/lodash-es": "^4.17.12", "@types/node": "24.9.1", "copyfiles": "^2.4.1", "rimraf": "^6.1.3", "ts-proto": "^2.11.6", + "tsup": "^8.5.1", "typescript": "~6.0.2" } } diff --git a/modules/embeddings/service-bundle.config.json b/modules/embeddings/service-bundle.config.json new file mode 100644 index 000000000..18feb0810 --- /dev/null +++ b/modules/embeddings/service-bundle.config.json @@ -0,0 +1,5 @@ +{ + "extraDependencies": [ + "bullmq" + ] +} diff --git a/modules/embeddings/tsup.config.ts b/modules/embeddings/tsup.config.ts new file mode 100644 index 000000000..c21167400 --- /dev/null +++ b/modules/embeddings/tsup.config.ts @@ -0,0 +1,6 @@ +import { createServiceTsupConfig } from '@conduitplatform/service-bundle/tsup'; +import bundleConfig from './service-bundle.config.json' with { type: 'json' }; + +export default createServiceTsupConfig({ + extraExternal: bundleConfig.extraDependencies, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5a6b5425c..fdfa38163 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -969,6 +969,9 @@ importers: specifier: ^4.18.1 version: 4.18.1 devDependencies: + '@conduitplatform/service-bundle': + specifier: workspace:* + version: link:../../libraries/service-bundle '@types/convict': specifier: ^6.1.6 version: 6.1.6 @@ -987,6 +990,9 @@ importers: ts-proto: specifier: ^2.11.6 version: 2.12.1 + tsup: + specifier: ^8.5.1 + version: 8.5.1(jiti@2.6.1)(postcss@8.5.15)(typescript@6.0.3)(yaml@2.9.0) typescript: specifier: ~6.0.2 version: 6.0.3 diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index d78ebdef7..0694f5968 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -13,6 +13,7 @@ case "$TARGET" in chat) BUILDING_SERVICE="modules/chat" ;; communications) BUILDING_SERVICE="modules/communications" ;; database) BUILDING_SERVICE="modules/database" ;; + embeddings) BUILDING_SERVICE="modules/embeddings" ;; functions) BUILDING_SERVICE="modules/functions" ;; router) BUILDING_SERVICE="modules/router" ;; storage) BUILDING_SERVICE="modules/storage" ;; @@ -21,7 +22,7 @@ case "$TARGET" in all) ;; *) echo "Unknown target: $TARGET" >&2 - echo "Usage: $0 [conduit|authentication|authorization|chat|communications|database|functions|router|storage|conduit-standalone|all]" >&2 + echo "Usage: $0 [conduit|authentication|authorization|chat|communications|database|embeddings|functions|router|storage|conduit-standalone|all]" >&2 exit 1 ;; esac @@ -41,7 +42,7 @@ fi if [ "$TARGET" = "all" ]; then docker buildx bake --file docker-bake.hcl all --set "*.platform=linux/amd64,linux/arm64" -elif [ "$TARGET" = "conduit" ] || [ "$TARGET" = "chat" ] || [ "$TARGET" = "functions" ] || [ "$TARGET" = "storage" ] || [ "$TARGET" = "authentication" ] || [ "$TARGET" = "authorization" ] || [ "$TARGET" = "communications" ] || [ "$TARGET" = "database" ] || [ "$TARGET" = "router" ] || [ "$TARGET" = "conduit-standalone" ]; then +elif [ "$TARGET" = "conduit" ] || [ "$TARGET" = "chat" ] || [ "$TARGET" = "functions" ] || [ "$TARGET" = "storage" ] || [ "$TARGET" = "authentication" ] || [ "$TARGET" = "authorization" ] || [ "$TARGET" = "communications" ] || [ "$TARGET" = "database" ] || [ "$TARGET" = "embeddings" ] || [ "$TARGET" = "router" ] || [ "$TARGET" = "conduit-standalone" ]; then # Bundle-based targets: bake HCL wires conduit-base-bundle-* (BUILD_BUNDLE=1). docker buildx bake --file docker-bake.hcl "$TARGET" \ --set "*.platform=linux/amd64,linux/arm64" diff --git a/scripts/resolve-docker-targets.mjs b/scripts/resolve-docker-targets.mjs index 000ea5ddb..3c7763b87 100644 --- a/scripts/resolve-docker-targets.mjs +++ b/scripts/resolve-docker-targets.mjs @@ -98,6 +98,19 @@ const IMAGE_TARGETS = [ ...SHARED_BUILD_PATHS, ], }, + { + target: 'embeddings', + image: 'embeddings', + name: 'Build embeddings', + buildingService: 'modules/embeddings', + isBundle: true, + paths: [ + 'modules/embeddings/**', + ...SERVICE_BUNDLE_PATHS, + ...LIBRARY_BUILD_PATHS, + ...SHARED_BUILD_PATHS, + ], + }, { target: 'functions', image: 'functions', diff --git a/scripts/verify-service-bundle.sh b/scripts/verify-service-bundle.sh index 509e13357..fe5455337 100755 --- a/scripts/verify-service-bundle.sh +++ b/scripts/verify-service-bundle.sh @@ -51,6 +51,11 @@ case "$SERVICE" in SERVICE_DIR="$ROOT/modules/$SERVICE" SERVICE_PROTOS="database.proto" ;; + embeddings) + PKG="@conduitplatform/embeddings" + SERVICE_DIR="$ROOT/modules/$SERVICE" + SERVICE_PROTOS="embeddings.proto" + ;; router) PKG="@conduitplatform/router" SERVICE_DIR="$ROOT/modules/$SERVICE" diff --git a/scripts/verify-standalone-bundle.sh b/scripts/verify-standalone-bundle.sh index dc8e83f3d..67ec7f911 100755 --- a/scripts/verify-standalone-bundle.sh +++ b/scripts/verify-standalone-bundle.sh @@ -15,6 +15,7 @@ STANDALONE_SERVICES=( "modules/storage:@conduitplatform/storage" "modules/chat:@conduitplatform/chat" ) +# embeddings is not part of standalone v1; it ships as a separate opt-in image. TMP="$(mktemp -d)" trap 'cleanup_all' EXIT diff --git a/standalone.Dockerfile b/standalone.Dockerfile index 248c59beb..94ca75d56 100644 --- a/standalone.Dockerfile +++ b/standalone.Dockerfile @@ -16,7 +16,7 @@ COPY --from=conduit-base /app/packages/core/bundle /app/packages/core/bundle COPY --from=conduit-base /app/packages/core/package.bundle.json /app/packages/core/package.json COPY --from=conduit-base /app/packages/core/package.bundle-lock.json /app/packages/core/package-lock.json -# Modules (standalone PM2 set — functions excluded) +# Modules (standalone PM2 set — functions and embeddings excluded from v1) COPY --from=conduit-base /app/modules/database/bundle /app/modules/database/bundle COPY --from=conduit-base /app/modules/database/package.bundle.json /app/modules/database/package.json COPY --from=conduit-base /app/modules/database/package.bundle-lock.json /app/modules/database/package-lock.json From 571d7ee9557b10727973bd007ec275392a82d8b9 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 20:47:58 +0300 Subject: [PATCH 13/29] refactor: remove unused helpers and redundant type dodges from embedding-support --- libraries/hermes/src/classes/ParserUtils.ts | 16 +++--- modules/database/src/Database.ts | 9 ++-- .../mongoose-adapter/SchemaConverter.ts | 5 +- .../src/adapters/mongoose-adapter/index.ts | 4 +- .../src/adapters/sequelize-adapter/index.ts | 16 +++--- .../postgres-adapter/PgSchemaConverter.ts | 8 ++- .../sql-adapter/SqlSchemaConverter.ts | 7 --- .../adapters/utils/vectorIndexLifecycle.ts | 31 +++++------ .../src/adapters/utils/vectorMappings.ts | 5 -- modules/database/src/controllers/cms/utils.ts | 2 +- modules/embeddings/src/Embeddings.ts | 51 +++++-------------- .../src/utils/mutationEvents.test.ts | 28 +++++----- .../embeddings/src/utils/mutationEvents.ts | 5 -- .../embeddings/src/utils/redactConfig.test.ts | 4 +- modules/embeddings/src/utils/redactConfig.ts | 13 ----- modules/embeddings/src/utils/schemaPolicy.ts | 3 +- 16 files changed, 76 insertions(+), 131 deletions(-) diff --git a/libraries/hermes/src/classes/ParserUtils.ts b/libraries/hermes/src/classes/ParserUtils.ts index c1c0579f0..c9143065b 100644 --- a/libraries/hermes/src/classes/ParserUtils.ts +++ b/libraries/hermes/src/classes/ParserUtils.ts @@ -78,14 +78,14 @@ export class ParserUtils { * Positive integer dimensions from a Vector field (or a raw dimensions value). */ static getVectorDimensions(fieldOrDimensions: unknown): number | undefined { - const value = - typeof fieldOrDimensions === 'number' - ? fieldOrDimensions - : typeof fieldOrDimensions === 'object' && - fieldOrDimensions !== null && - 'dimensions' in fieldOrDimensions - ? (fieldOrDimensions as { dimensions?: unknown }).dimensions - : undefined; + let value: unknown = fieldOrDimensions; + if ( + typeof fieldOrDimensions === 'object' && + fieldOrDimensions !== null && + 'dimensions' in fieldOrDimensions + ) { + value = (fieldOrDimensions as { dimensions?: unknown }).dimensions; + } if (typeof value === 'number' && Number.isInteger(value) && value > 0) { return value; } diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index f484b9966..4c87b9dd9 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -6,6 +6,9 @@ import { GrpcRequest, GrpcResponse, HealthCheckStatus, + VectorIndexDefinition, + VectorIndexMethod, + VectorSimilarity, } from '@conduitplatform/grpc-sdk'; import { AdminHandlers } from './admin/index.js'; import { SchemaAdmin } from './admin/schema.admin.js'; @@ -1182,13 +1185,13 @@ export default class DatabaseModule extends ManagedModule { callback(null, { result }); } - private parseVectorIndex(index: VectorIndex) { + private parseVectorIndex(index: VectorIndex): VectorIndexDefinition { return { field: index.field, dimensions: index.dimensions, - similarity: index.similarity as any, + similarity: index.similarity as VectorSimilarity, name: index.name, - method: index.method as any, + method: index.method as VectorIndexMethod, filterFields: index.filterFields, options: index.options ? JSON.parse(index.options) : undefined, }; diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index dead468dd..fe9dab7ba 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -8,7 +8,8 @@ import { } from '@conduitplatform/grpc-sdk'; import { cloneDeep, isArray, isNil, isObject } from 'lodash-es'; import { checkIfMongoOptions } from './utils.js'; -import { applyMongoVectorField, isVectorSchemaType } from '../utils/vectorMappings.js'; +import { applyMongoVectorField } from '../utils/vectorMappings.js'; +import { isVectorTypeName } from '../utils/vectorField.js'; import * as deepdash from 'deepdash-es/standalone'; @@ -88,7 +89,7 @@ function convert(value: any, key: any, parentValue: any) { parentValue[key].type = Schema.Types.Mixed; } - if (isVectorSchemaType(parentValue[key]?.type)) { + if (isVectorTypeName(parentValue[key]?.type)) { parentValue[key] = applyMongoVectorField(parentValue[key]); } diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index f2d45e0f7..9cc2da711 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -37,7 +37,7 @@ import { ConduitDatabaseSchema, introspectedSchemaCmsOptionsDefaults, } from '../../interfaces/index.js'; -import { isArray, isEqual, isNil } from 'lodash-es'; +import { isArray, isEqual } from 'lodash-es'; import { parseSchema } from 'mongodb-schema'; const VIEW_LOCK_TTL_MS = 60_000; @@ -767,7 +767,7 @@ export class MongooseAdapter extends DatabaseAdapter { ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - const schema = this.models[schemaName].originalSchema as any; + const schema = this.models[schemaName].originalSchema; const field = schema.compiledFields?.[index.field] ?? schema.fields?.[index.field]; const bound = bindVectorIndexToField({ provider: 'mongodb', diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 3afa84535..5c3a95048 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -45,6 +45,7 @@ import { planPostgresVectorSearch, postgresIndexMethodSql, postgresVectorCapabilities, + parsePostgresVectorIndexDef, resolveVectorFieldFromSchema, sqlFallbackVectorCapabilities, assertPostgresVectorIndexDropTarget, @@ -511,10 +512,10 @@ export abstract class SequelizeAdapter extends DatabaseAdapter return rows .filter(row => /USING (hnsw|ivfflat)/i.test(row.indexdef)) .map(row => { - const mapped = fromPostgresVectorIndex(row.indexname, row.indexdef); - const field = resolveVectorFieldFromSchema(schemaFields, mapped.field); + const parsed = parsePostgresVectorIndexDef(row.indexdef); + const field = resolveVectorFieldFromSchema(schemaFields, parsed.field); const matchingDeclared = declared.find( - item => item.name === row.indexname || item.field === mapped.field, + item => item.name === row.indexname || item.field === parsed.field, ); return fromPostgresVectorIndex( row.indexname, @@ -541,9 +542,10 @@ export abstract class SequelizeAdapter extends DatabaseAdapter async vectorSearch(request: VectorSearchInput): Promise { this.ensurePostgresVectorSupport(request.schemaName); const schema = this.models[request.schemaName]; - const field = (schema.originalSchema.compiledFields?.[request.field] ?? - schema.originalSchema.fields?.[request.field]) as any; - if (field?.type !== 'Vector') { + const schemaFields = (schema.originalSchema.compiledFields ?? + schema.originalSchema.fields) as Record; + const field = resolveVectorFieldFromSchema(schemaFields, request.field); + if (!field) { throw new GrpcError(status.INVALID_ARGUMENT, 'Requested field is not a vector'); } if (request.vector.length !== field.dimensions) { @@ -558,8 +560,6 @@ export abstract class SequelizeAdapter extends DatabaseAdapter scope: request.scope, adminOperator: request.adminOperator, }); - const schemaFields = (schema.originalSchema.compiledFields ?? - schema.originalSchema.fields) as Record; const liveIndexes = await this.getVectorIndexes(request.schemaName); const planned = planPostgresVectorSearch({ request, diff --git a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts index a9b7939c3..6504e65ba 100644 --- a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts @@ -18,10 +18,8 @@ import { extractRelations, RelationType, } from '../utils/extractors/index.js'; -import { - isVectorSchemaType, - vectorFieldStorageMapping, -} from '../../utils/vectorMappings.js'; +import { vectorFieldStorageMapping } from '../../utils/vectorMappings.js'; +import { isVectorTypeName } from '../../utils/vectorField.js'; /** * This function should take as an input a JSON schema and convert it to the sequelize equivalent @@ -158,7 +156,7 @@ function extractObjectType(objectField: Indexable): res.type = extractArrayType(objectField.type).type; } else { res.type = extractType(objectField.type, objectField.sqlType); - if (isVectorSchemaType(objectField.type)) { + if (isVectorTypeName(objectField.type)) { const mapping = vectorFieldStorageMapping('postgres', { dimensions: objectField.dimensions, }); diff --git a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts index 26106fce8..7f418d43a 100644 --- a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts @@ -18,10 +18,6 @@ import { extractRelations, RelationType, } from '../utils/extractors/index.js'; -import { - isVectorSchemaType, - vectorFieldStorageMapping, -} from '../../utils/vectorMappings.js'; /** * This function should take as an input a JSON schema and convert it to the sequelize equivalent @@ -167,9 +163,6 @@ function extractObjectType(objectField: Indexable, field: string) { res.type = extractType(objectField.type, objectField.sqlType); } res.type = extractType(objectField.type, objectField.sqlType); - if (isVectorSchemaType(objectField.type)) { - vectorFieldStorageMapping('sql', { dimensions: objectField.dimensions }); - } if (objectField.hasOwnProperty('default')) { res.defaultValue = checkDefaultValue(objectField.type, objectField.default); } diff --git a/modules/database/src/adapters/utils/vectorIndexLifecycle.ts b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts index 44c3e897f..ec4c7218f 100644 --- a/modules/database/src/adapters/utils/vectorIndexLifecycle.ts +++ b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts @@ -61,8 +61,8 @@ export function bindVectorIndexToField(args: { ...args.index, name: args.index.name ?? defaultVectorIndexName(args.index.field, args.physicalTableName), - dimensions: args.index.dimensions ?? field?.dimensions ?? args.index.dimensions, - similarity: args.index.similarity ?? field?.similarity ?? args.index.similarity, + dimensions: args.index.dimensions ?? field?.dimensions, + similarity: args.index.similarity ?? field?.similarity, filterFields: args.provider === 'mongodb' ? mongoVectorFilterFields(args.index.filterFields) @@ -173,17 +173,22 @@ export function planMongoVectorIndexCreate(args: { ); } +function postgresSimilarityFromOperator(operator?: string): VectorSimilarity { + if (operator === 'l2' || operator === 'vector_l2_ops') { + return VectorSimilarity.Euclidean; + } + if (operator === 'ip' || operator === 'vector_ip_ops') { + return VectorSimilarity.DotProduct; + } + return VectorSimilarity.Cosine; +} + export function parsePostgresVectorIndexDef(indexdef: string): ParsedPostgresVectorIndex { const tableMatch = /ON\s+(?:(?:"[^"]+"|\w+)\.)?(?:"([^"]+)"|(\w+))/i.exec(indexdef); const method = /USING\s+(\w+)/i.exec(indexdef)?.[1]?.toLowerCase(); const fieldMatch = /\((?:"([^"]+)"|(\w+))\s+vector_/i.exec(indexdef); const operator = /vector_(l2|cosine|ip)_ops/i.exec(indexdef)?.[1]; - const similarity = - operator === 'l2' - ? VectorSimilarity.Euclidean - : operator === 'ip' - ? VectorSimilarity.DotProduct - : VectorSimilarity.Cosine; + const similarity = postgresSimilarityFromOperator(operator); return { tableName: tableMatch?.[1] ?? tableMatch?.[2], field: fieldMatch?.[1] ?? fieldMatch?.[2] ?? '', @@ -209,13 +214,9 @@ export function postgresVectorIndexDefinitionMatches( if ((parsed.method ?? '').toLowerCase() !== expected.method.toLowerCase()) { return false; } - const expectedSimilarity = - expected.operator === 'vector_l2_ops' - ? VectorSimilarity.Euclidean - : expected.operator === 'vector_ip_ops' - ? VectorSimilarity.DotProduct - : VectorSimilarity.Cosine; - if (parsed.similarity !== expectedSimilarity) return false; + if (parsed.similarity !== postgresSimilarityFromOperator(expected.operator)) { + return false; + } return postgresRequestedOptionsMatch(expected.options, parsed.options, expected.method); } diff --git a/modules/database/src/adapters/utils/vectorMappings.ts b/modules/database/src/adapters/utils/vectorMappings.ts index 30322cd69..f7f8473f6 100644 --- a/modules/database/src/adapters/utils/vectorMappings.ts +++ b/modules/database/src/adapters/utils/vectorMappings.ts @@ -1,5 +1,4 @@ import { - TYPE, VectorIndexDefinition, VectorIndexMethod, VectorSimilarity, @@ -184,7 +183,3 @@ export function resolveVectorFieldFromSchema( if (!isObjectFormVectorField(field)) return undefined; return field; } - -export function isVectorSchemaType(type: unknown) { - return type === TYPE.Vector || type === 'Vector'; -} diff --git a/modules/database/src/controllers/cms/utils.ts b/modules/database/src/controllers/cms/utils.ts index 8bd26e02a..27162718f 100644 --- a/modules/database/src/controllers/cms/utils.ts +++ b/modules/database/src/controllers/cms/utils.ts @@ -125,7 +125,7 @@ function cloneAssignableArrayItem(item: unknown): unknown { return item.map(cloneAssignableArrayItem); } if (!isPlainObject(item)) return item; - if (isCmsWriteOmittedField('item', item) && isVectorField(item)) { + if (isVectorField(item)) { return { ...item }; } if (isNestedConduitModel(item)) { diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts index 2e482d37a..d363b9070 100644 --- a/modules/embeddings/src/Embeddings.ts +++ b/modules/embeddings/src/Embeddings.ts @@ -384,46 +384,19 @@ export default class EmbeddingsModule extends ManagedModule { configs.forEach(item => this.subscribeToSchema(item.schemaName)); } - private schemaSubscriptionIds(schemaName: string): [string, string, string, string] { - const idPrefix = `embeddings:${schemaName}`; - return [ - `${idPrefix}:create`, - `${idPrefix}:update`, - `${idPrefix}:createMany`, - `${idPrefix}:updateMany`, - ]; - } - private subscribeToSchema(schemaName: string) { if (this.subscribedSchemas.has(schemaName)) return; - const [createId, updateId, createManyId, updateManyId] = - this.schemaSubscriptionIds(schemaName); - this.grpcSdk.bus?.subscribe( - `database:create:${schemaName}`, - message => this.enqueueMutation(schemaName, message), - createId, - ); - this.grpcSdk.bus?.subscribe( - `database:update:${schemaName}`, - message => this.enqueueMutation(schemaName, message), - updateId, - ); - this.grpcSdk.bus?.subscribe( - `database:createMany:${schemaName}`, - message => this.enqueueMutation(schemaName, message), - createManyId, - ); - this.grpcSdk.bus?.subscribe( - `database:updateMany:${schemaName}`, - message => this.enqueueMutation(schemaName, message), - updateManyId, - ); - this.subscribedSchemas.set(schemaName, [ - createId, - updateId, - createManyId, - updateManyId, - ]); + const events = ['create', 'update', 'createMany', 'updateMany'] as const; + const ids = events.map(event => { + const id = `embeddings:${schemaName}:${event}`; + this.grpcSdk.bus?.subscribe( + `database:${event}:${schemaName}`, + message => this.enqueueMutation(schemaName, message), + id, + ); + return id; + }); + this.subscribedSchemas.set(schemaName, ids); } private unsubscribeFromSchema(schemaName: string) { @@ -514,7 +487,7 @@ export default class EmbeddingsModule extends ManagedModule { schemaName: parsed.data.schemaName, enabled: true, }) - ).filter(Boolean) as EmbeddingConfig[]; + ).filter((config): config is EmbeddingConfig => Boolean(config)); const matching = configs.filter( config => config.enabled && config.schemaName === parsed.data.schemaName, ); diff --git a/modules/embeddings/src/utils/mutationEvents.test.ts b/modules/embeddings/src/utils/mutationEvents.test.ts index c6cada7bc..e74144e7d 100644 --- a/modules/embeddings/src/utils/mutationEvents.test.ts +++ b/modules/embeddings/src/utils/mutationEvents.test.ts @@ -5,15 +5,18 @@ import { extractDocumentIds, isEmbeddingOwnedMutation, parseBoundedMutationEvent, - parseMutationEvent, } from './mutationEvents.js'; describe('embedding mutation event parsing', () => { it('normalizes create, update, and bulk payloads to unique ids', () => { - assert.deepEqual(parseMutationEvent(JSON.stringify({ _id: 'a', title: 'x' })), { - payload: { _id: 'a', title: 'x' }, - ids: ['a'], - }); + const parsed = parseBoundedMutationEvent(JSON.stringify({ _id: 'a', title: 'x' })); + assert.equal(parsed.ok, true); + if (parsed.ok) { + assert.deepEqual(parsed.event, { + payload: { _id: 'a', title: 'x' }, + ids: ['a'], + }); + } assert.deepEqual(extractDocumentIds([{ _id: 'a' }, { _id: 'b' }, { _id: 'a' }]), [ 'a', 'b', @@ -30,12 +33,13 @@ describe('embedding mutation event parsing', () => { }), [], ); - assert.equal( - parseMutationEvent( - JSON.stringify({ acknowledged: true, matchedCount: 2, modifiedCount: 2 }), - )?.ids.length, - 0, + const parsed = parseBoundedMutationEvent( + JSON.stringify({ acknowledged: true, matchedCount: 2, modifiedCount: 2 }), ); + assert.equal(parsed.ok, true); + if (parsed.ok) { + assert.equal(parsed.event.ids.length, 0); + } }); it('parses bounded bulk id chunks', () => { @@ -63,10 +67,6 @@ describe('embedding mutation event parsing', () => { assert.equal(isEmbeddingOwnedMutation({ _id: 'a' }, owned), false); }); - it('returns null for malformed payloads', () => { - assert.equal(parseMutationEvent('{not json'), null); - }); - it('fails closed on oversized bus payloads instead of crashing', () => { assert.deepEqual(parseBoundedMutationEvent('{not json'), { ok: false, diff --git a/modules/embeddings/src/utils/mutationEvents.ts b/modules/embeddings/src/utils/mutationEvents.ts index 9abbb97a9..169f8f5e9 100644 --- a/modules/embeddings/src/utils/mutationEvents.ts +++ b/modules/embeddings/src/utils/mutationEvents.ts @@ -12,11 +12,6 @@ export type MutationEventParseResult = | { ok: true; event: ParsedMutationEvent } | { ok: false; reason: 'malformed' | 'capped' }; -export function parseMutationEvent(message: string): ParsedMutationEvent | null { - const parsed = parseBoundedMutationEvent(message); - return parsed.ok ? parsed.event : null; -} - export function parseBoundedMutationEvent( message: string, maxIds: number = MAX_MUTATION_EVENT_IDS, diff --git a/modules/embeddings/src/utils/redactConfig.test.ts b/modules/embeddings/src/utils/redactConfig.test.ts index a82efe973..dabc601e5 100644 --- a/modules/embeddings/src/utils/redactConfig.test.ts +++ b/modules/embeddings/src/utils/redactConfig.test.ts @@ -1,12 +1,12 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { redactSensitiveConfig } from '@conduitplatform/module-tools'; -import { redactProviderConfig, redactSecretText } from './redactConfig.js'; +import { redactSecretText } from './redactConfig.js'; describe('provider secret redaction', () => { it('redacts API keys from config objects and error text', () => { assert.equal( - redactProviderConfig({ endpoint: 'https://api.openai.com', apiKey: 'sk-secret' }) + redactSensitiveConfig({ endpoint: 'https://api.openai.com', apiKey: 'sk-secret' }) .apiKey, '[REDACTED]', ); diff --git a/modules/embeddings/src/utils/redactConfig.ts b/modules/embeddings/src/utils/redactConfig.ts index 6a5da9e17..131d54295 100644 --- a/modules/embeddings/src/utils/redactConfig.ts +++ b/modules/embeddings/src/utils/redactConfig.ts @@ -8,16 +8,3 @@ export function sanitizeErrorMessage(err: unknown): string { const message = err instanceof Error ? err.message : String(err); return redactSecretText(message); } - -export function redactProviderConfig>(config: T): T { - const redacted = { ...config }; - for (const key of Object.keys(redacted)) { - if ( - /^(apiKey|api_key|password|secret)$/i.test(key) && - typeof redacted[key] === 'string' - ) { - (redacted as Record)[key] = '[REDACTED]'; - } - } - return redacted; -} diff --git a/modules/embeddings/src/utils/schemaPolicy.ts b/modules/embeddings/src/utils/schemaPolicy.ts index 636e525ed..c7479c8cd 100644 --- a/modules/embeddings/src/utils/schemaPolicy.ts +++ b/modules/embeddings/src/utils/schemaPolicy.ts @@ -1,5 +1,4 @@ -import { TYPE } from '@conduitplatform/grpc-sdk'; -import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { BACKFILL_RUN_SCHEMA } from './backfillRun.js'; From c76114eb780e32e86371c9341adedcc077e90000 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 21:11:32 +0300 Subject: [PATCH 14/29] fix(embeddings): stop retained jobs and races from dropping work Dedupe BullMQ jobs only while waiting/active so completed/failed ids cannot block later updates. Increment backfill counts atomically, provision vector indexes on first upsert, and bound updateMany mutation id collection. --- .github/workflows/embeddings-test.yml | 29 ++++- deploy/embeddings.md | 5 +- modules/database/src/Database.ts | 22 ++-- .../utils/__tests__/mutationEvents.test.ts | 40 +++++++ .../__tests__/vectorIndexLifecycle.test.ts | 9 ++ .../__tests__/vectorSearchAdapters.test.ts | 4 +- .../utils/__tests__/vectorSearchQuery.test.ts | 33 ++++++ .../src/adapters/utils/mutationEvents.ts | 43 +++++++ .../adapters/utils/vectorIndexLifecycle.ts | 3 +- .../src/adapters/utils/vectorSearchQuery.ts | 22 ++-- modules/embeddings/README.md | 6 +- modules/embeddings/src/Embeddings.ts | 13 +-- modules/embeddings/src/admin/routes.ts | 2 +- .../embeddings/src/api/embeddingsApi.test.ts | 63 ++++++---- modules/embeddings/src/api/embeddingsApi.ts | 110 +++++++++++++----- .../src/controllers/queue.controller.test.ts | 54 ++++++++- .../src/controllers/queue.controller.ts | 53 ++++++++- .../src/utils/backfillExecution.test.ts | 51 +++++++- .../embeddings/src/utils/backfillExecution.ts | 21 ++-- .../src/utils/backfillGates.test.ts | 7 ++ modules/embeddings/src/utils/backfillGates.ts | 3 +- .../embeddings/src/utils/backfillRun.test.ts | 38 ++++++ modules/embeddings/src/utils/backfillRun.ts | 22 ++++ .../src/utils/embeddingJobs.test.ts | 16 +++ modules/embeddings/src/utils/embeddingJobs.ts | 26 +++++ 25 files changed, 591 insertions(+), 104 deletions(-) diff --git a/.github/workflows/embeddings-test.yml b/.github/workflows/embeddings-test.yml index 0f0c71cc9..bf54fae2b 100644 --- a/.github/workflows/embeddings-test.yml +++ b/.github/workflows/embeddings-test.yml @@ -5,16 +5,22 @@ on: pull_request: paths: - 'modules/embeddings/**' + - 'modules/database/**' + - 'libraries/hermes/**' - 'libraries/grpc-sdk/**' - 'libraries/module-tools/**' + - 'packages/core/**' - '.github/workflows/embeddings-test.yml' push: branches: - main paths: - 'modules/embeddings/**' + - 'modules/database/**' + - 'libraries/hermes/**' - 'libraries/grpc-sdk/**' - 'libraries/module-tools/**' + - 'packages/core/**' - '.github/workflows/embeddings-test.yml' permissions: @@ -47,8 +53,27 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts - - name: Build embeddings and workspace dependencies - run: pnpm exec turbo run build --filter=@conduitplatform/embeddings... + - name: Build embeddings, database, hermes, grpc-sdk, module-tools, and Core tsc + run: > + pnpm exec turbo run build + --filter=@conduitplatform/embeddings... + --filter=@conduitplatform/database... + --filter=@conduitplatform/hermes... + --filter=@conduitplatform/grpc-sdk... + --filter=@conduitplatform/module-tools... + --filter=@conduitplatform/core + + # Core `build:bundle` still aliases @conduitplatform/node-2fa to its + # TypeScript source because the library's CommonJS dist breaks ESM + # bundling. That pre-existing packaging issue is out of this embeddings + # offline gate; this job only typechecks Core (`tsc`), it does not run + # authentication/Core service-bundle verification. - name: Run embeddings offline tests run: pnpm --filter @conduitplatform/embeddings test + + - name: Run database vector and mutation offline tests + run: pnpm --filter @conduitplatform/database test --testPathIgnorePatterns=integration + + - name: Run hermes vector offline tests + run: pnpm --filter @conduitplatform/hermes test diff --git a/deploy/embeddings.md b/deploy/embeddings.md index 93c8fa620..6484a3e32 100644 --- a/deploy/embeddings.md +++ b/deploy/embeddings.md @@ -37,8 +37,11 @@ and remain disabled by default. activation must not. 5. Configure the HTTPS provider (`endpoint`, `apiKey`, `allowedHosts`). Check `GET /embeddings/status` for provider/index warnings. -6. Create an embedding config with `enabled: false`. Wait until the vector +6. Create an embedding config. The first upsert provisions the vector index + when Database indexing is available. The config stays disabled until the index for `targetField` is queryable (`status` ready, not pending/failed). + If indexing is unavailable, status reports a manual lifecycle warning and + the operator must create the index before enabling. 7. Enable the config only after index readiness. Start a **bounded** backfill (`onlyMissing` recommended). Watch `GET /embeddings/backfills/:id` and `GET /embeddings/status` queue counts. Do not scan collections in the diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index 4c87b9dd9..9719b29df 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -74,7 +74,7 @@ import { import { QueueController } from './controllers/queue.controller.js'; import { buildMutationEventChunks, - collectDocumentIds, + collectBoundedMutationIds, mutationEventChannel, shouldPublishMutationEvent, grpcStatusFromError, @@ -877,10 +877,7 @@ export default class DatabaseModule extends ManagedModule { callback(null, { result: resultString }); } catch (err) { - callback({ - code: status.INTERNAL, - message: (err as Error).message, - }); + callback(grpcStatusFromError(err)); } } @@ -1272,11 +1269,16 @@ export default class DatabaseModule extends ManagedModule { filterQuery: string, options: { userId?: string; scope?: string }, ): Promise { - const docs = await model.findMany(filterQuery, { - select: '_id', - userId: options.userId, - scope: options.scope, + return collectBoundedMutationIds({ + findPage: (skip, limit) => + model.findMany(filterQuery, { + select: '_id', + skip, + limit, + sort: { _id: 1 }, + userId: options.userId, + scope: options.scope, + }), }); - return collectDocumentIds(docs); } } diff --git a/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts b/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts index 0fb4da174..81a6a5c53 100644 --- a/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts +++ b/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts @@ -1,8 +1,13 @@ import { describe, expect, it } from '@jest/globals'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; import { buildMutationEventChunks, + collectBoundedMutationIds, collectDocumentIds, + MAX_MUTATION_EVENT_COLLECT_IDS, mutationEventChannel, + mutationIdCollectionExhaustedError, shouldPublishMutationEvent, } from '../mutationEvents.js'; @@ -52,4 +57,39 @@ describe('mutation event helpers', () => { ]); expect(ids).toEqual(['1', '2', '3', '4', '5']); }); + + it('pages and caps updateMany mutation id collection instead of materializing unlimited ids', async () => { + const docs = Array.from({ length: 7 }, (_, index) => ({ _id: String(index) })); + const seen: Array<{ skip: number; limit: number }> = []; + const ids = await collectBoundedMutationIds({ + findPage: async (skip, limit) => { + seen.push({ skip, limit }); + return docs.slice(skip, skip + limit); + }, + cap: 10, + pageSize: 3, + }); + expect(ids).toEqual(['0', '1', '2', '3', '4', '5', '6']); + expect(seen).toEqual([ + { skip: 0, limit: 3 }, + { skip: 3, limit: 3 }, + { skip: 6, limit: 3 }, + ]); + + await expect( + collectBoundedMutationIds({ + findPage: async (skip, limit) => + Array.from({ length: limit }, (_, index) => ({ + _id: String(skip + index), + })), + cap: 4, + pageSize: 3, + }), + ).rejects.toMatchObject({ + code: status.RESOURCE_EXHAUSTED, + message: mutationIdCollectionExhaustedError(4).message, + }); + expect(mutationIdCollectionExhaustedError().code).toBe(status.RESOURCE_EXHAUSTED); + expect(MAX_MUTATION_EVENT_COLLECT_IDS).toBe(10_000); + }); }); diff --git a/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts b/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts index f68e6dab1..0072220e7 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts @@ -13,6 +13,7 @@ import { bindVectorIndexToField, defaultVectorIndexName, hydratePostgresVectorIndex, + isVectorIndexQueryable, mongoSearchIndexReadiness, mongoVectorFilterFields, planMongoVectorIndexCreate, @@ -125,6 +126,14 @@ describe('vector index lifecycle', () => { expect((err as GrpcError).code).toBe(status.FAILED_PRECONDITION); expect((err as GrpcError).message).toMatch(/not queryable \(status: pending\)/); } + expect( + isVectorIndexQueryable({ + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + }), + ).toBe(false); }); it('reuses matching Mongo indexes and rejects silent definition changes', () => { diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchAdapters.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchAdapters.test.ts index 5af93be34..22a989c11 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorSearchAdapters.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchAdapters.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, jest } from '@jest/globals'; -import { TYPE, VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { TYPE, VectorIndexStatus, VectorSimilarity } from '@conduitplatform/grpc-sdk'; import { MongooseAdapter } from '../../mongoose-adapter/index.js'; import { SequelizeAdapter } from '../../sequelize-adapter/index.js'; @@ -38,6 +38,8 @@ function articleModel(overrides?: { dimensions: 3, similarity: VectorSimilarity.Cosine, filterFields: ['_id', 'tenantId'], + status: VectorIndexStatus.Ready, + queryable: true, }, ], }, diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts index 2327b9faf..2f3c9e9d9 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts @@ -8,6 +8,7 @@ import { import { status } from '@grpc/grpc-js'; import { completeVectorSearch, + mergeVectorIndexes, planMongoVectorSearch, planPostgresVectorSearch, } from '../vectorSearchQuery.js'; @@ -31,6 +32,8 @@ const indexes = [ dimensions: 3, similarity: VectorSimilarity.Cosine, filterFields: ['_id', 'tenantId'], + status: VectorIndexStatus.Ready, + queryable: true, }, ]; @@ -126,6 +129,36 @@ describe('vector search query planning', () => { expect((err as GrpcError).message).toMatch(/not queryable/); } }); + + it('does not treat declared-only modelOptions vector indexes as live or queryable', () => { + const declaredOnly = mergeVectorIndexes(indexes, []); + expect(declaredOnly).toEqual([]); + expect(() => + planMongoVectorSearch({ + request, + indexes: declaredOnly, + schemaFields, + }), + ).toThrow(GrpcError); + const livePending = mergeVectorIndexes(indexes, [ + { + ...indexes[0], + status: VectorIndexStatus.Pending, + queryable: false, + }, + ]); + expect(livePending[0]).toMatchObject({ + status: VectorIndexStatus.Pending, + queryable: false, + }); + expect(() => + planMongoVectorSearch({ + request, + indexes: livePending, + schemaFields, + }), + ).toThrow(/not queryable/); + }); }); describe('bounded vector search completion', () => { diff --git a/modules/database/src/adapters/utils/mutationEvents.ts b/modules/database/src/adapters/utils/mutationEvents.ts index d947bfeb2..7caf4856b 100644 --- a/modules/database/src/adapters/utils/mutationEvents.ts +++ b/modules/database/src/adapters/utils/mutationEvents.ts @@ -1,4 +1,9 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + export const MUTATION_EVENT_ID_CHUNK_SIZE = 500; +export const MUTATION_EVENT_ID_PAGE_SIZE = MUTATION_EVENT_ID_CHUNK_SIZE; +export const MAX_MUTATION_EVENT_COLLECT_IDS = 10_000; export type MutationOperation = 'create' | 'createMany' | 'update' | 'updateMany' | 'delete'; @@ -38,6 +43,44 @@ export function collectDocumentIds(docs: unknown): string[] { return [...ids]; } +export function mutationIdCollectionExhaustedError( + cap: number = MAX_MUTATION_EVENT_COLLECT_IDS, +): GrpcError { + return new GrpcError( + status.RESOURCE_EXHAUSTED, + `updateMany matched more than ${cap} documents; refuse unbounded mutation event collection`, + ); +} + +export async function collectBoundedMutationIds(args: { + findPage: (skip: number, limit: number) => Promise; + cap?: number; + pageSize?: number; +}): Promise { + const cap = args.cap ?? MAX_MUTATION_EVENT_COLLECT_IDS; + const pageSize = args.pageSize ?? MUTATION_EVENT_ID_PAGE_SIZE; + if (!Number.isInteger(cap) || cap < 1 || !Number.isInteger(pageSize) || pageSize < 1) { + throw mutationIdCollectionExhaustedError(cap); + } + const ids: string[] = []; + let skip = 0; + while (ids.length <= cap) { + const remainingWithOverflowProbe = cap - ids.length + 1; + const limit = Math.min(pageSize, remainingWithOverflowProbe); + const page = await args.findPage(skip, limit); + const pageLength = Array.isArray(page) ? page.length : page == null ? 0 : 1; + if (!pageLength) break; + skip += pageLength; + const pageIds = collectDocumentIds(page); + if (ids.length + pageIds.length > cap) { + throw mutationIdCollectionExhaustedError(cap); + } + ids.push(...pageIds); + if (pageLength < limit) break; + } + return ids; +} + export function toIdEventPayload(ids: string[]): { _id: string }[] { return ids.map(_id => ({ _id })); } diff --git a/modules/database/src/adapters/utils/vectorIndexLifecycle.ts b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts index ec4c7218f..0eec21a47 100644 --- a/modules/database/src/adapters/utils/vectorIndexLifecycle.ts +++ b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts @@ -112,7 +112,8 @@ export function isVectorIndexQueryable(index?: VectorIndexDefinition): boolean { if (index.status === VectorIndexStatus.Pending && index.queryable !== true) { return false; } - return true; + if (index.queryable === true) return true; + return index.status === VectorIndexStatus.Ready; } export function assertVectorIndexQueryable( diff --git a/modules/database/src/adapters/utils/vectorSearchQuery.ts b/modules/database/src/adapters/utils/vectorSearchQuery.ts index 18c00c765..266f71752 100644 --- a/modules/database/src/adapters/utils/vectorSearchQuery.ts +++ b/modules/database/src/adapters/utils/vectorSearchQuery.ts @@ -75,16 +75,24 @@ export function mergeVectorIndexes( declared: VectorIndexDefinition[], live: VectorIndexDefinition[], ): VectorIndexDefinition[] { - if (!live.length) return declared; + if (!live.length) return []; if (!declared.length) return live; - const merged = new Map(); + const declaredByKey = new Map(); for (const index of declared) { - merged.set(index.name ?? defaultVectorIndexName(index.field), index); + declaredByKey.set(index.name ?? defaultVectorIndexName(index.field), index); } - for (const index of live) { - merged.set(index.name ?? defaultVectorIndexName(index.field), index); - } - return [...merged.values()]; + return live.map(liveIndex => { + const key = liveIndex.name ?? defaultVectorIndexName(liveIndex.field); + const declaredIndex = + declaredByKey.get(key) ?? declared.find(item => item.field === liveIndex.field); + if (!declaredIndex) return liveIndex; + return { + ...declaredIndex, + ...liveIndex, + status: liveIndex.status, + queryable: liveIndex.queryable, + }; + }); } export function buildMongoVectorSearchPipeline(args: { diff --git a/modules/embeddings/README.md b/modules/embeddings/README.md index 2979e46f4..4a22beaaa 100644 --- a/modules/embeddings/README.md +++ b/modules/embeddings/README.md @@ -34,8 +34,10 @@ Enable it and configure an OpenAI-compatible provider: ## Workflow 1. Create an embedding config with `schemaName`, `sourceFields`, `targetField`, - `provider`, `model`, and `dimensions`. Save `enabled: false` until Database - reports vector storage and a queryable index. + `provider`, `model`, and `dimensions`. The first upsert provisions the vector + index when Database indexing is available. The config stays disabled until + Database reports a queryable index. If indexing is unavailable, status + returns a manual index lifecycle warning. 2. The module adds a vector schema extension for the target field and a source hash field used to skip unchanged documents. 3. Start a backfill, or rely on database create/update events to enqueue diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts index d363b9070..a2900b85d 100644 --- a/modules/embeddings/src/Embeddings.ts +++ b/modules/embeddings/src/Embeddings.ts @@ -546,16 +546,13 @@ export default class EmbeddingsModule extends ManagedModule { outcome: 'processed' | 'failed', ) { if (!runId) return; - const doc = await BackfillRun.getInstance().findOne({ _id: runId }); - if (!doc) return; - const run = backfillRunFromDocument(doc); await applyBackfillJobOutcome({ - run, + runId, outcome, - saveRun: (id, next) => - BackfillRun.getInstance() - .findByIdAndUpdate(id, persistableBackfillRun(next)) - .then(() => undefined), + incrementCounts: async (id, patch) => { + const updated = await BackfillRun.getInstance().findByIdAndUpdate(id, patch); + return updated ? backfillRunFromDocument(updated) : null; + }, }); } diff --git a/modules/embeddings/src/admin/routes.ts b/modules/embeddings/src/admin/routes.ts index ba7ef698e..54c580ee9 100644 --- a/modules/embeddings/src/admin/routes.ts +++ b/modules/embeddings/src/admin/routes.ts @@ -52,7 +52,7 @@ export const EMBEDDINGS_ADMIN_ROUTES: EmbeddingsAdminRouteContract[] = [ contract( '/configs', ConduitRouteActions.POST, - 'Creates or updates an embedding config for a schema. Operator-only. Saving enabled=false succeeds with capability warnings; enabling requires vector storage, a queryable index, and an enabled module.', + 'Creates or updates an embedding config for a schema. Operator-only. The first upsert provisions the vector index when Database indexing is available. Saving enabled=true while the index is pending stores the config disabled until it is queryable. If indexing is unavailable, status reports a manual index lifecycle warning.', ), contract( '/configs/:id', diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts index 361584a25..6f2779d15 100644 --- a/modules/embeddings/src/api/embeddingsApi.test.ts +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -214,25 +214,8 @@ const enabledConfig: EmbeddingConfigRecord = { }; describe('typed embeddings API handlers', () => { - it('upserts a typed config and refuses to enable without a queryable index', async () => { - const { api, configs } = createApi({ indexes: [] }); - await assert.rejects( - () => - api.upsertConfig( - { - schemaName: 'Article', - sourceFields: ['title'], - targetField: 'embedding', - provider: 'openai-compatible', - model: 'text-embedding-3-small', - dimensions: 3, - enabled: true, - }, - { callerModule: 'database' }, - ), - (err: unknown) => - err instanceof GrpcError && err.code === status.FAILED_PRECONDITION, - ); + it('upserts a typed config and provisions a missing vector index on first save', async () => { + const { api, configs, createdIndexes } = createApi({ indexes: [] }); const saved = await api.upsertConfig( { schemaName: 'Article', @@ -241,18 +224,58 @@ describe('typed embeddings API handlers', () => { provider: 'openai-compatible', model: 'text-embedding-3-small', dimensions: 3, - enabled: false, + enabled: true, }, { callerModule: 'database' }, ); assert.equal(saved.config.enabled, false); assert.equal(saved.config.model, 'text-embedding-3-small'); assert.equal(typeof saved.config.id, 'string'); + assert.equal(createdIndexes.includes('embedding_vector'), true); assert.equal( saved.warnings.some(warning => /not queryable/.test(warning)), true, ); + assert.equal( + saved.warnings.some(warning => + /saved disabled until the provisioned vector index/.test(warning), + ), + true, + ); assert.equal(configs.length, 1); + assert.equal(configs[0].enabled, false); + }); + + it('reports a manual index lifecycle when Database indexing is unavailable', async () => { + const { api, createdIndexes } = createApi({ + indexes: [], + capabilities: { + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + reason: 'indexing unavailable', + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(createdIndexes.length, 0); + assert.equal( + saved.warnings.some(warning => /Create the index manually/.test(warning)), + true, + ); }); it('gates system schemas and owner policies on config and backfill', async () => { diff --git a/modules/embeddings/src/api/embeddingsApi.ts b/modules/embeddings/src/api/embeddingsApi.ts index 0ecf9c54a..16973ba26 100644 --- a/modules/embeddings/src/api/embeddingsApi.ts +++ b/modules/embeddings/src/api/embeddingsApi.ts @@ -198,23 +198,6 @@ export class EmbeddingsApi { const enabled = request.enabled ?? true; const capabilities = await this.deps.getVectorCapabilities(persisted.schemaName); let indexes = await this.deps.getVectorIndexes(persisted.schemaName); - const warnings = [ - ...capabilityWarnings(capabilities), - ...indexReadinessWarnings( - [ - { - targetField: persisted.targetField, - enabled, - schemaName: persisted.schemaName, - }, - ], - indexes, - ), - ...providerReadinessWarnings( - configDefaults.providers[persisted.provider] ?? - configDefaults.providers[configDefaults.defaultProvider], - ), - ]; const existing = await this.deps.configs.findOne({ schemaName: persisted.schemaName, targetField: persisted.targetField, @@ -226,10 +209,6 @@ export class EmbeddingsApi { `Changing vector field '${existing.targetField}' dimensions from ${existing.dimensions} to ${persisted.dimensions} is not allowed. Create a new targetField and run an explicit backfill.`, ); } - if (existing && changed.length && requiresIndexRecreation(changed)) { - await this.recreateVectorIndex(existing, persisted, indexes); - indexes = await this.deps.getVectorIndexes(persisted.schemaName); - } if (capabilities.storage) { await this.deps.setSchemaExtension({ schemaName: persisted.schemaName, @@ -248,6 +227,42 @@ export class EmbeddingsApi { }, }); } + let provisionedIndex = false; + if (existing && changed.length && requiresIndexRecreation(changed)) { + await this.recreateVectorIndex(existing, persisted, indexes); + indexes = await this.deps.getVectorIndexes(persisted.schemaName); + provisionedIndex = true; + } else { + provisionedIndex = await this.ensureVectorIndex(persisted, indexes, capabilities); + if (provisionedIndex) { + indexes = await this.deps.getVectorIndexes(persisted.schemaName); + } + } + const warnings = [ + ...capabilityWarnings(capabilities), + ...indexReadinessWarnings( + [ + { + targetField: persisted.targetField, + enabled, + schemaName: persisted.schemaName, + }, + ], + indexes, + ), + ...providerReadinessWarnings( + configDefaults.providers[persisted.provider] ?? + configDefaults.providers[configDefaults.defaultProvider], + ), + ]; + if ( + !findTargetVectorIndex(indexes, persisted.targetField) && + !capabilities.indexing + ) { + warnings.push( + `Vector index for field '${persisted.targetField}' was not provisioned automatically because Database indexing is unavailable. Create the index manually and wait until it is queryable before enabling this config.`, + ); + } let persistEnabled = enabled; if (enabled) { try { @@ -262,14 +277,18 @@ export class EmbeddingsApi { indexes, }); } catch (err) { - if (existing && changed.length && requiresIndexRecreation(changed)) { - persistEnabled = false; - warnings.push( - 'Config was saved disabled until the recreated vector index is queryable. Enable it and start an explicit backfill once the index is ready.', - ); - } else { - throw err; - } + const indexPending = + (err instanceof BackfillGateError && err.reason === 'index_not_queryable') || + (err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not queryable/.test(err.message)); + if (!indexPending) throw err; + persistEnabled = false; + warnings.push( + provisionedIndex + ? 'Config was saved disabled until the provisioned vector index is queryable. Enable it once Database reports the index ready.' + : 'Config was saved disabled until the vector index is queryable. Enable it once Database reports the index ready.', + ); } } const saved = existing @@ -710,6 +729,39 @@ export class EmbeddingsApi { return runs; } + private async ensureVectorIndex( + next: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + }, + indexes: Array<{ + field?: string; + name?: string; + queryable?: boolean; + status?: string; + }>, + capabilities: VectorCapabilities, + ): Promise { + if (findTargetVectorIndex(indexes, next.targetField)) return false; + if (!capabilities.indexing) return false; + try { + await this.deps.createVectorIndex(next.schemaName, { + field: next.targetField, + dimensions: next.dimensions, + similarity: next.similarity as VectorIndexDefinition['similarity'], + name: defaultEmbeddingVectorIndexName(next.targetField), + }); + } catch (err) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Failed to provision vector index for '${next.targetField}': ${sanitizeErrorMessage(err)}`, + ); + } + return true; + } + private async recreateVectorIndex( existing: EmbeddingConfigRecord, next: diff --git a/modules/embeddings/src/controllers/queue.controller.test.ts b/modules/embeddings/src/controllers/queue.controller.test.ts index 4a83fb977..c6ab25f58 100644 --- a/modules/embeddings/src/controllers/queue.controller.test.ts +++ b/modules/embeddings/src/controllers/queue.controller.test.ts @@ -9,6 +9,7 @@ type StoredJob = { name: string; data: Record; opts?: { jobId?: string; delay?: number; attempts?: number }; + state?: string; }; class FakeQueue { @@ -19,7 +20,7 @@ class FakeQueue { if (opts?.jobId && this.jobs.some(job => job.opts?.jobId === opts.jobId)) { throw new Error(`Job ${opts.jobId} already exists`); } - this.jobs.push({ name, data, opts }); + this.jobs.push({ name, data, opts, state: 'waiting' }); } async addBulk(jobs: StoredJob[]) { @@ -28,6 +29,22 @@ class FakeQueue { } } + async getJob(jobId: string) { + const job = this.jobs.find(stored => stored.opts?.jobId === jobId); + if (!job) return undefined; + return { + getState: async () => job.state ?? 'waiting', + remove: async () => { + this.jobs = this.jobs.filter(stored => stored !== job); + }, + }; + } + + markState(jobId: string, state: string) { + const job = this.jobs.find(stored => stored.opts?.jobId === jobId); + if (job) job.state = state; + } + async getJobCounts() { return { waiting: this.jobs.length, @@ -166,6 +183,29 @@ describe('embedding queue worker lifecycle', () => { ); }); + it('re-enqueues the same identity after a retained completed or failed job', async () => { + FakeWorker.instances = []; + const { queue, controller } = createController(); + const job = { schemaName: 'Article', documentId: 'a' }; + assert.equal(await controller.addEmbeddingJob(job, 3), 1); + queue.markState(embeddingJobId(job), 'completed'); + assert.equal(await controller.addEmbeddingJob(job, 3), 1); + assert.equal(queue.jobs.length, 1); + assert.equal(queue.jobs[0].state, 'waiting'); + queue.markState(embeddingJobId(job), 'failed'); + assert.equal( + await controller.addBulkEmbeddingJobs( + [job, { schemaName: 'Article', documentId: 'b' }], + 3, + ), + 2, + ); + assert.deepEqual( + queue.jobs.map(stored => stored.opts?.jobId), + [embeddingJobId(job), embeddingJobId({ schemaName: 'Article', documentId: 'b' })], + ); + }); + it('skips malformed queue payloads instead of throwing', async () => { FakeWorker.instances = []; const { queue, controller } = createController(); @@ -215,6 +255,18 @@ describe('embedding queue status and backfill jobs', () => { ); }); + it('does not let a completed backfill page job block a later scan of the same cursor', async () => { + const { backfillQueue, controller } = createController(); + await controller.addBackfillControllerJob({ runId: 'run1', cursor: 'b' }); + backfillQueue.markState('backfill:run1:b', 'completed'); + await controller.addBackfillControllerJob({ runId: 'run1', cursor: 'b' }); + assert.deepEqual( + backfillQueue.jobs.map(job => job.opts?.jobId), + ['backfill:run1:b'], + ); + assert.equal(backfillQueue.jobs[0].state, 'waiting'); + }); + it('increments retried then failed metrics without job payload labels', async () => { FakeWorker.instances = []; const metrics = withMetrics(); diff --git a/modules/embeddings/src/controllers/queue.controller.ts b/modules/embeddings/src/controllers/queue.controller.ts index 6507e49f8..9d306e609 100644 --- a/modules/embeddings/src/controllers/queue.controller.ts +++ b/modules/embeddings/src/controllers/queue.controller.ts @@ -6,7 +6,9 @@ import { dedupeEmbeddingJobs, embeddingJobId, isDuplicateJobError, + isInFlightQueueJobState, parseEmbeddingJobData, + shouldReplaceRetainedQueueJob, } from '../utils/embeddingJobs.js'; import { BackfillControllerJobData, @@ -35,6 +37,11 @@ export interface EmbeddingQueueStatus { backfill: QueueJobCounts; } +type QueueJobHandle = { + getState: () => Promise; + remove: () => Promise; +}; + type QueueLike = { add: ( name: string, @@ -49,6 +56,7 @@ type QueueLike = { }>, ) => Promise; close: () => Promise; + getJob?: (jobId: string) => Promise; getJobCounts: () => Promise & Record>; }; @@ -261,12 +269,15 @@ export class QueueController { incrementEmbeddingMetric('malformedJobs'); return 0; } + const jobId = embeddingJobId(parsed.data); + const decision = await resolveExistingQueueJob(this.embeddingQueue, jobId); + if (decision === 'skip') return 0; try { await this.embeddingQueue.add( - embeddingJobId(parsed.data), + jobId, { ...parsed.data }, { - jobId: embeddingJobId(parsed.data), + jobId, attempts, backoff: { type: 'exponential', delay: 1000 }, }, @@ -290,9 +301,19 @@ export class QueueController { } const unique = dedupeEmbeddingJobs(jobs); if (!unique.length) return 0; + const enqueueable: EmbeddingJobData[] = []; + for (const job of unique) { + const decision = await resolveExistingQueueJob( + this.embeddingQueue, + embeddingJobId(job), + ); + if (decision === 'skip') continue; + enqueueable.push(job); + } + if (!enqueueable.length) return 0; try { await this.embeddingQueue.addBulk( - unique.map(job => ({ + enqueueable.map(job => ({ name: embeddingJobId(job), data: { ...job }, opts: { @@ -302,11 +323,11 @@ export class QueueController { }, })), ); - return unique.length; + return enqueueable.length; } catch (err) { if (!isDuplicateJobError(err)) throw err; const added = await Promise.all( - unique.map(job => this.addEmbeddingJob(job, attempts)), + enqueueable.map(job => this.addEmbeddingJob(job, attempts)), ); let queued = 0; for (const count of added) queued += count; @@ -328,6 +349,10 @@ export class QueueController { const jobId = parsed.data.drain ? undefined : `backfill:${parsed.data.runId}:${parsed.data.cursor ?? 'start'}`; + if (jobId) { + const decision = await resolveExistingQueueJob(this.backfillQueue, jobId); + if (decision === 'skip') return; + } try { await this.backfillQueue.add( 'backfill-page', @@ -405,3 +430,21 @@ function normalizeJobCounts( paused: counts.paused ?? 0, }; } + +async function resolveExistingQueueJob( + queue: QueueLike, + jobId: string, +): Promise<'enqueue' | 'skip'> { + if (!queue.getJob) return 'enqueue'; + const existing = await queue.getJob(jobId); + if (!existing) return 'enqueue'; + const state = await existing.getState(); + if (isInFlightQueueJobState(state)) return 'skip'; + if (!shouldReplaceRetainedQueueJob(state)) return 'skip'; + try { + await existing.remove(); + } catch { + return 'skip'; + } + return 'enqueue'; +} diff --git a/modules/embeddings/src/utils/backfillExecution.test.ts b/modules/embeddings/src/utils/backfillExecution.test.ts index 2c20c12dd..fe7b0a1ac 100644 --- a/modules/embeddings/src/utils/backfillExecution.test.ts +++ b/modules/embeddings/src/utils/backfillExecution.test.ts @@ -55,6 +55,16 @@ function memoryStore(initial: PersistedBackfillRun[] = []) { saveRun: async (id: string, run: BackfillRunProgress) => { runs.set(id, { ...run, _id: id }); }, + incrementCounts: async ( + id: string, + patch: { $inc: { processedCount?: number; failedCount?: number } }, + ) => { + const run = runs.get(id); + if (!run || run.state !== 'running') return null; + run.processedCount += patch.$inc.processedCount ?? 0; + run.failedCount += patch.$inc.failedCount ?? 0; + return { ...run }; + }, }; } @@ -219,9 +229,9 @@ describe('cursor-based backfill continuation', () => { assert.equal(second.run?.state, 'running'); for (let i = 0; i < 3; i += 1) { await applyBackfillJobOutcome({ - run: (await store.getRun(created._id))!, + runId: created._id, outcome: 'processed', - saveRun: store.saveRun, + incrementCounts: store.incrementCounts, }); } const drained = await processBackfillControllerJob( @@ -329,15 +339,15 @@ describe('backfill cancellation, resume, and counters', () => { ); const afterPage = (await store.getRun(created._id))!; const processed = await applyBackfillJobOutcome({ - run: afterPage, + runId: created._id, outcome: 'processed', - saveRun: store.saveRun, + incrementCounts: store.incrementCounts, }); assert.equal(processed.ok, true); const failed = await applyBackfillJobOutcome({ - run: (await store.getRun(created._id))!, + runId: created._id, outcome: 'failed', - saveRun: store.saveRun, + incrementCounts: store.incrementCounts, }); assert.equal(failed.ok, true); if (!failed.ok) return; @@ -347,6 +357,35 @@ describe('backfill cancellation, resume, and counters', () => { assert.equal(failed.run.failedCount, 1); }); + it('keeps concurrent processed and failed increments without lost updates', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'running', + batchSize: 2, + queuedCount: 40, + }), + ); + await Promise.all( + Array.from({ length: 40 }, (_, index) => + applyBackfillJobOutcome({ + runId: created._id, + outcome: index % 5 === 0 ? 'failed' : 'processed', + incrementCounts: async (id, patch) => { + await Promise.resolve(); + return store.incrementCounts(id, patch); + }, + }), + ), + ); + const latest = (await store.getRun(created._id))!; + assert.equal(latest.processedCount, 32); + assert.equal(latest.failedCount, 8); + }); + it('fails the running run when the vector index is not queryable', async () => { const store = memoryStore(); const created = await store.createRun( diff --git a/modules/embeddings/src/utils/backfillExecution.ts b/modules/embeddings/src/utils/backfillExecution.ts index 5dd633508..9536849cb 100644 --- a/modules/embeddings/src/utils/backfillExecution.ts +++ b/modules/embeddings/src/utils/backfillExecution.ts @@ -1,7 +1,7 @@ import type { VectorCapabilities } from '@conduitplatform/grpc-sdk'; import { - applyBackfillJobCounts, applyBackfillPage, + backfillCountIncrementPatch, boundBackfillPage, buildBackfillPageQuery, cancelBackfillRun, @@ -11,6 +11,7 @@ import { isActiveBackfillState, resumeBackfillRun, startBackfillRun, + type BackfillCountIncrementPatch, type BackfillPageQuery, type BackfillRunProgress, type BackfillRunResult, @@ -326,17 +327,19 @@ export async function cancelBackfillExecution(args: { } export async function applyBackfillJobOutcome(args: { - run: PersistedBackfillRun; + runId: string; outcome: 'processed' | 'failed'; - saveRun: (id: string, run: BackfillRunProgress) => Promise; + incrementCounts: ( + id: string, + patch: BackfillCountIncrementPatch, + ) => Promise; }): Promise { - const counted = applyBackfillJobCounts( - args.run, - args.outcome === 'processed' ? { processed: 1 } : { failed: 1 }, + const run = await args.incrementCounts( + args.runId, + backfillCountIncrementPatch(args.outcome), ); - if (!counted.ok) return counted; - await args.saveRun(args.run._id, counted.run); - return counted; + if (!run) return { ok: false, reason: 'not_found' }; + return { ok: true, run }; } export async function processBackfillControllerJob( diff --git a/modules/embeddings/src/utils/backfillGates.test.ts b/modules/embeddings/src/utils/backfillGates.test.ts index ba415b89d..f15631c6f 100644 --- a/modules/embeddings/src/utils/backfillGates.test.ts +++ b/modules/embeddings/src/utils/backfillGates.test.ts @@ -102,6 +102,13 @@ describe('backfill execution gates', () => { it('requires a queryable vector index and keeps the status actionable', () => { assert.equal(isEmbeddingVectorIndexQueryable(readyIndex), true); + assert.equal( + isEmbeddingVectorIndexQueryable({ + field: 'embedding', + name: 'embedding_vector', + }), + false, + ); assert.equal( isEmbeddingVectorIndexQueryable({ field: 'embedding', diff --git a/modules/embeddings/src/utils/backfillGates.ts b/modules/embeddings/src/utils/backfillGates.ts index 4efdca6d9..09c43d336 100644 --- a/modules/embeddings/src/utils/backfillGates.ts +++ b/modules/embeddings/src/utils/backfillGates.ts @@ -56,7 +56,8 @@ export function isEmbeddingVectorIndexQueryable(index?: VectorIndexGate): boolea ) { return false; } - return true; + if (index.queryable === true) return true; + return indexStatus === VectorIndexStatus.Ready || indexStatus === 'ready'; } export function findTargetVectorIndex( diff --git a/modules/embeddings/src/utils/backfillRun.test.ts b/modules/embeddings/src/utils/backfillRun.test.ts index fbd4c6452..81ca9e734 100644 --- a/modules/embeddings/src/utils/backfillRun.test.ts +++ b/modules/embeddings/src/utils/backfillRun.test.ts @@ -1,9 +1,11 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { + applyAtomicBackfillCountDelta, applyBackfillJobCounts, applyBackfillPage, BACKFILL_RUN_STATES, + backfillCountIncrementPatch, boundBackfillBatchSize, boundBackfillPage, buildBackfillPageQuery, @@ -186,6 +188,42 @@ describe('backfill counters', () => { const queuedCounts = applyBackfillJobCounts(queuedRun(), { processed: 1 }); assert.equal(queuedCounts.ok, false); }); + + it('loses concurrent updates when counts are applied via read-modify-write', () => { + const paged = applyBackfillPage(runningRun(), [{ _id: 'a' }, { _id: 'b' }], 2); + assert.equal(paged.ok, true); + if (!paged.ok) return; + const snapshot = { ...paged.run }; + const first = applyBackfillJobCounts(snapshot, { processed: 1 }); + const second = applyBackfillJobCounts(snapshot, { processed: 1 }); + assert.equal(first.ok, true); + assert.equal(second.ok, true); + if (!first.ok || !second.ok) return; + assert.equal(first.run.processedCount, 1); + assert.equal(second.run.processedCount, 1); + }); + + it('keeps concurrent processed and failed increments with an atomic delta', async () => { + const counters = { processedCount: 0, failedCount: 0 }; + assert.deepEqual(backfillCountIncrementPatch('processed'), { + $inc: { processedCount: 1 }, + }); + assert.deepEqual(backfillCountIncrementPatch('failed'), { + $inc: { failedCount: 1 }, + }); + await Promise.all( + Array.from({ length: 40 }, (_, index) => + Promise.resolve( + applyAtomicBackfillCountDelta( + counters, + index % 4 === 0 ? 'failed' : 'processed', + ), + ), + ), + ); + assert.equal(counters.processedCount, 30); + assert.equal(counters.failedCount, 10); + }); }); describe('backfill cancellation and resume', () => { diff --git a/modules/embeddings/src/utils/backfillRun.ts b/modules/embeddings/src/utils/backfillRun.ts index 2cb3e2784..b2ed528d6 100644 --- a/modules/embeddings/src/utils/backfillRun.ts +++ b/modules/embeddings/src/utils/backfillRun.ts @@ -291,6 +291,28 @@ export function applyBackfillPage( }; } +export type BackfillCountIncrementPatch = { + $inc: { processedCount?: number; failedCount?: number }; +}; + +export function backfillCountIncrementPatch( + outcome: 'processed' | 'failed', +): BackfillCountIncrementPatch { + return { + $inc: outcome === 'processed' ? { processedCount: 1 } : { failedCount: 1 }, + }; +} + +export function applyAtomicBackfillCountDelta( + counters: { processedCount: number; failedCount: number }, + outcome: 'processed' | 'failed', +): { processedCount: number; failedCount: number } { + const patch = backfillCountIncrementPatch(outcome).$inc; + counters.processedCount += patch.processedCount ?? 0; + counters.failedCount += patch.failedCount ?? 0; + return counters; +} + export function applyBackfillJobCounts( run: BackfillRunProgress, counts: { processed?: number; failed?: number }, diff --git a/modules/embeddings/src/utils/embeddingJobs.test.ts b/modules/embeddings/src/utils/embeddingJobs.test.ts index 6ec768273..800c3f804 100644 --- a/modules/embeddings/src/utils/embeddingJobs.test.ts +++ b/modules/embeddings/src/utils/embeddingJobs.test.ts @@ -4,8 +4,11 @@ import { dedupeEmbeddingJobs, embeddingJobId, isDuplicateJobError, + isInFlightQueueJobState, + isTerminalQueueJobState, parseEmbeddingJobData, parseEmbeddingJobBatch, + shouldReplaceRetainedQueueJob, } from './embeddingJobs.js'; describe('embedding job identity', () => { @@ -28,6 +31,19 @@ describe('embedding job identity', () => { assert.equal(isDuplicateJobError(new Error('redis timeout')), false); }); + it('replaces retained completed or failed jobs and only dedupes in-flight work', () => { + assert.equal(isInFlightQueueJobState('waiting'), true); + assert.equal(isInFlightQueueJobState('active'), true); + assert.equal(isInFlightQueueJobState('delayed'), true); + assert.equal(isInFlightQueueJobState('completed'), false); + assert.equal(isTerminalQueueJobState('completed'), true); + assert.equal(isTerminalQueueJobState('failed'), true); + assert.equal(shouldReplaceRetainedQueueJob('completed'), true); + assert.equal(shouldReplaceRetainedQueueJob('failed'), true); + assert.equal(shouldReplaceRetainedQueueJob('waiting'), false); + assert.equal(shouldReplaceRetainedQueueJob('active'), false); + }); + it('rejects malformed and oversized queue payloads', () => { assert.equal( parseEmbeddingJobData({ schemaName: 'Article', documentId: 'a' }).ok, diff --git a/modules/embeddings/src/utils/embeddingJobs.ts b/modules/embeddings/src/utils/embeddingJobs.ts index c298e4ed4..4f542d99c 100644 --- a/modules/embeddings/src/utils/embeddingJobs.ts +++ b/modules/embeddings/src/utils/embeddingJobs.ts @@ -12,12 +12,38 @@ export const MAX_QUEUE_BATCH_SIZE = 500; const IDENTITY = /^[A-Za-z0-9._-]{1,128}$/; const SCHEMA_NAME = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; +export const IN_FLIGHT_QUEUE_JOB_STATES = [ + 'waiting', + 'active', + 'delayed', + 'paused', + 'waiting-children', + 'prioritized', +] as const; + +export const TERMINAL_QUEUE_JOB_STATES = ['completed', 'failed'] as const; + +export type InFlightQueueJobState = (typeof IN_FLIGHT_QUEUE_JOB_STATES)[number]; +export type TerminalQueueJobState = (typeof TERMINAL_QUEUE_JOB_STATES)[number]; + export function embeddingJobId(data: EmbeddingJobData): string { const parts = [data.schemaName, data.documentId]; if (data.configId) parts.push(data.configId); return parts.join('__'); } +export function isInFlightQueueJobState(state: string): state is InFlightQueueJobState { + return (IN_FLIGHT_QUEUE_JOB_STATES as readonly string[]).includes(state); +} + +export function isTerminalQueueJobState(state: string): state is TerminalQueueJobState { + return (TERMINAL_QUEUE_JOB_STATES as readonly string[]).includes(state); +} + +export function shouldReplaceRetainedQueueJob(state: string): boolean { + return isTerminalQueueJobState(state); +} + export function dedupeEmbeddingJobs(jobs: EmbeddingJobData[]): EmbeddingJobData[] { const seen = new Set(); const unique: EmbeddingJobData[] = []; From 38f4aff89a47d9f98e837db294059227aab3fb0b Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 21:21:46 +0300 Subject: [PATCH 15/29] fix(embeddings): restrict source allowlists to platform-admin upserts Prevent schema-owner gRPC callers from bypassing hidden or sensitive source rejection, and add defense-in-depth bounds for job reads, backfill filters, Mongo index drops, and client search limits. --- .../src/adapters/mongoose-adapter/index.ts | 11 +- .../__tests__/embeddingsJobContext.test.ts | 27 ++++ .../__tests__/vectorIndexAdapters.test.ts | 28 ++++ .../__tests__/vectorIndexLifecycle.test.ts | 21 +++ .../adapters/utils/embeddingsJobContext.ts | 11 +- .../adapters/utils/vectorIndexLifecycle.ts | 29 ++++ modules/embeddings/src/admin/routes.ts | 2 +- .../embeddings/src/api/embeddingsApi.test.ts | 140 ++++++++++++++++++ modules/embeddings/src/api/embeddingsApi.ts | 21 ++- modules/embeddings/src/config/index.ts | 2 +- modules/embeddings/src/routes/index.ts | 5 +- .../embeddings/src/utils/backfillRun.test.ts | 49 ++++++ modules/embeddings/src/utils/backfillRun.ts | 134 +++++++++++++++-- .../src/utils/clientSearchContext.test.ts | 18 ++- .../src/utils/clientSearchContext.ts | 10 ++ .../embeddings/src/utils/schemaPolicy.test.ts | 52 ++++++- modules/embeddings/src/utils/schemaPolicy.ts | 29 ++++ 17 files changed, 557 insertions(+), 32 deletions(-) diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 9cc2da711..47c3b1a4b 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -28,6 +28,7 @@ import { planMongoVectorSearch, bindVectorIndexToField, planMongoVectorIndexCreate, + assertMongoVectorSearchIndexDropTarget, } from '../utils/index.js'; import pluralize from '../../utils/pluralize.js'; import { mongoSchemaConverter } from '../../introspection/mongoose/utils.js'; @@ -807,12 +808,20 @@ export class MongooseAdapter extends DatabaseAdapter { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collection: any = this.mongoose.model(schemaName).collection; - if (typeof collection.dropSearchIndex !== 'function') { + if ( + typeof collection.dropSearchIndex !== 'function' || + typeof collection.listSearchIndexes !== 'function' + ) { throw new GrpcError( status.FAILED_PRECONDITION, 'MongoDB Vector Search index commands are not available for this deployment', ); } + const indexes = await collection.listSearchIndexes().toArray(); + const existing = (indexes as Array<{ name?: string; type?: string }>).find( + index => index.name === indexName, + ); + assertMongoVectorSearchIndexDropTarget({ indexName, existing }); await collection.dropSearchIndex(indexName); return 'Vector index deleted'; } diff --git a/modules/database/src/adapters/utils/__tests__/embeddingsJobContext.test.ts b/modules/database/src/adapters/utils/__tests__/embeddingsJobContext.test.ts index 728fca4c8..2c5e1b576 100644 --- a/modules/database/src/adapters/utils/__tests__/embeddingsJobContext.test.ts +++ b/modules/database/src/adapters/utils/__tests__/embeddingsJobContext.test.ts @@ -50,6 +50,33 @@ describe('embeddings job context', () => { schema: articleSchema, }), ).not.toThrow(); + expect(() => + assertEmbeddingsJobRead({ + query: JSON.stringify({ id: 'doc-1' }), + select: '+title', + allowedFields: ['title'], + schema: articleSchema, + }), + ).not.toThrow(); + }); + + it('rejects embeddingsJob reads that use operator objects instead of scalar ids', () => { + const denied = (query: unknown) => { + expect(() => + assertEmbeddingsJobRead({ + query, + select: '+title', + allowedFields: ['title'], + schema: articleSchema, + }), + ).toThrow(GrpcError); + }; + denied({ _id: { $gt: '' } }); + denied({ _id: { $ne: null } }); + denied({ _id: { $in: ['doc-1'] } }); + denied({ _id: 123 }); + denied({ _id: '' }); + denied({ id: { $regex: '.*' } }); }); it('rejects collection scans, extra selected fields, and non-string sources', () => { diff --git a/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts b/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts index b503f2fcb..76f61b393 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts @@ -131,6 +131,34 @@ describe('mongoose vector index lifecycle', () => { } expect(aggregate).not.toHaveBeenCalled(); }); + + it('drops Mongo search indexes only after verifying type vectorSearch', async () => { + const dropSearchIndex = jest.fn(async () => undefined); + const listSearchIndexes = jest.fn(() => ({ + toArray: async () => [{ name: 'article_text', type: 'search' }], + })); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + mongoose: { + model: () => ({ + collection: { dropSearchIndex, listSearchIndexes }, + }), + }, + }); + await expect(adapter.deleteVectorIndex('Article', 'article_text')).rejects.toThrow( + /is not a vectorSearch index/, + ); + expect(dropSearchIndex).not.toHaveBeenCalled(); + + listSearchIndexes.mockImplementation(() => ({ + toArray: async () => [{ name: 'embedding_vector', type: 'vectorSearch' }], + })); + await expect(adapter.deleteVectorIndex('Article', 'embedding_vector')).resolves.toBe( + 'Vector index deleted', + ); + expect(dropSearchIndex).toHaveBeenCalledWith('embedding_vector'); + }); }); describe('postgres vector index lifecycle', () => { diff --git a/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts b/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts index 0072220e7..d43a205e7 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts @@ -8,6 +8,7 @@ import { } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { + assertMongoVectorSearchIndexDropTarget, assertPostgresVectorIndexDropTarget, assertVectorIndexQueryable, bindVectorIndexToField, @@ -264,6 +265,26 @@ describe('vector index lifecycle', () => { ).toBe('cnd_Article_embedding_vector'); }); + it('refuses to drop Mongo search indexes that are not vectorSearch', () => { + expect(() => + assertMongoVectorSearchIndexDropTarget({ + indexName: 'article_text', + }), + ).toThrow(/was not found/); + expect(() => + assertMongoVectorSearchIndexDropTarget({ + indexName: 'article_text', + existing: { name: 'article_text', type: 'search' }, + }), + ).toThrow(/is not a vectorSearch index/); + expect( + assertMongoVectorSearchIndexDropTarget({ + indexName: 'embedding_vector', + existing: { name: 'embedding_vector', type: 'vectorSearch' }, + }), + ).toEqual({ name: 'embedding_vector', type: 'vectorSearch' }); + }); + it('restores Postgres dimensions and WITH options during catalog read-back', () => { const hydrated = hydratePostgresVectorIndex({ name: 'cnd_Article_embedding_vector', diff --git a/modules/database/src/adapters/utils/embeddingsJobContext.ts b/modules/database/src/adapters/utils/embeddingsJobContext.ts index 78d99940e..f13fee743 100644 --- a/modules/database/src/adapters/utils/embeddingsJobContext.ts +++ b/modules/database/src/adapters/utils/embeddingsJobContext.ts @@ -41,6 +41,12 @@ export function parseSelectFields(select?: string): string[] { .map(part => part.replace(/^[+-]/, '')); } +export const EMBEDDINGS_JOB_DOCUMENT_ID = /^[A-Za-z0-9._-]{1,128}$/; + +export function isScalarDocumentId(value: unknown): value is string { + return typeof value === 'string' && EMBEDDINGS_JOB_DOCUMENT_ID.test(value); +} + export function isIdOnlyQuery(query: unknown): boolean { let parsed = query; if (typeof query === 'string') { @@ -52,7 +58,10 @@ export function isIdOnlyQuery(query: unknown): boolean { } if (!isRecord(parsed)) return false; const keys = Object.keys(parsed); - return keys.length === 1 && (keys[0] === '_id' || keys[0] === 'id'); + if (keys.length !== 1) return false; + const key = keys[0]; + if (key !== '_id' && key !== 'id') return false; + return isScalarDocumentId(parsed[key]); } export function assertEmbeddingsJobCaller(moduleName?: string): void { diff --git a/modules/database/src/adapters/utils/vectorIndexLifecycle.ts b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts index 0eec21a47..211501a61 100644 --- a/modules/database/src/adapters/utils/vectorIndexLifecycle.ts +++ b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts @@ -304,6 +304,35 @@ export function assertPostgresVectorIndexDropTarget(args: { return args.existing; } +export function isMongoVectorSearchIndex(index?: { + name?: string; + type?: string; +}): boolean { + return index?.type === 'vectorSearch'; +} + +export function assertMongoVectorSearchIndexDropTarget(args: { + indexName: string; + existing?: { name?: string; type?: string }; +}): { name: string; type: 'vectorSearch' } { + if (!args.existing) { + throw new GrpcError( + status.NOT_FOUND, + `Vector search index '${args.indexName}' was not found.`, + ); + } + if (!isMongoVectorSearchIndex(args.existing)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Search index '${args.indexName}' is not a vectorSearch index.`, + ); + } + return { + name: args.existing.name ?? args.indexName, + type: 'vectorSearch', + }; +} + export function hydratePostgresVectorIndex(args: { name: string; indexdef: string; diff --git a/modules/embeddings/src/admin/routes.ts b/modules/embeddings/src/admin/routes.ts index 54c580ee9..4ac27f594 100644 --- a/modules/embeddings/src/admin/routes.ts +++ b/modules/embeddings/src/admin/routes.ts @@ -52,7 +52,7 @@ export const EMBEDDINGS_ADMIN_ROUTES: EmbeddingsAdminRouteContract[] = [ contract( '/configs', ConduitRouteActions.POST, - 'Creates or updates an embedding config for a schema. Operator-only. The first upsert provisions the vector index when Database indexing is available. Saving enabled=true while the index is pending stores the config disabled until it is queryable. If indexing is unavailable, status reports a manual index lifecycle warning.', + 'Creates or updates an embedding config for a schema. Operator-only. The first upsert provisions the vector index when Database indexing is available. Saving enabled=true while the index is pending stores the config disabled until it is queryable. If indexing is unavailable, status reports a manual index lifecycle warning. Caller-supplied sourceFieldAllowlist is honored only for platform-admin upserts; schema-owner gRPC callers use operator config allowlists.', ), contract( '/configs/:id', diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts index 6f2779d15..4a42b929e 100644 --- a/modules/embeddings/src/api/embeddingsApi.test.ts +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -314,6 +314,122 @@ describe('typed embeddings API handlers', () => { ); }); + it('ignores caller-supplied sourceFieldAllowlist for schema-owner gRPC callers', async () => { + const sensitiveSchema = { + name: 'Article', + fields: { + title: { type: TYPE.String }, + password: { type: TYPE.String }, + notes: { type: TYPE.String, select: false }, + }, + }; + const owner = { callerModule: 'cms-app' }; + const declared = { Article: { name: 'Article', ownerModule: 'cms-app' } }; + const { api: ownerApi } = createApi({ + schemas: { Article: sensitiveSchema }, + declared, + indexes: [readyIndex], + }); + await assert.rejects( + () => + ownerApi.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['password'], + targetField: 'embedding', + model: 'm', + dimensions: 3, + sourceFieldAllowlist: ['password'], + enabled: false, + }, + owner, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /sensitive/.test(err.message), + ); + await assert.rejects( + () => + ownerApi.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['notes'], + targetField: 'embedding', + model: 'm', + dimensions: 3, + sourceFieldAllowlist: ['notes'], + enabled: false, + }, + owner, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /hidden/.test(err.message), + ); + await assert.rejects( + () => + ownerApi.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['password'], + targetField: 'embedding', + model: 'm', + dimensions: 3, + sourceFieldAllowlist: ['password'], + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /sensitive/.test(err.message), + ); + + const { api: operatorAllowlistApi } = createApi({ + schemas: { Article: sensitiveSchema }, + declared, + indexes: [readyIndex], + config: { + ...moduleConfig, + security: { ...moduleConfig.security, sourceFieldAllowlist: ['notes'] }, + } as Config, + }); + const operatorSaved = await operatorAllowlistApi.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['notes'], + targetField: 'embedding', + model: 'm', + dimensions: 3, + enabled: false, + }, + owner, + ); + assert.equal(operatorSaved.config.enabled, false); + + const { api: adminApi } = createApi({ + schemas: { Article: sensitiveSchema }, + declared, + indexes: [readyIndex], + }); + const adminSaved = await adminApi.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['notes'], + targetField: 'embedding', + model: 'm', + dimensions: 3, + sourceFieldAllowlist: ['notes'], + enabled: false, + }, + { platformAdmin: true }, + ); + assert.equal(adminSaved.config.enabled, false); + }); + it('starts, lists, cancels, and resumes persisted backfill runs with onlyMissing', async () => { const { api, runs, enqueued } = createApi({ configs: [enabledConfig] }); const started = await api.startBackfill( @@ -420,6 +536,30 @@ describe('typed embeddings API handlers', () => { assert.equal(mapped.code, status.FAILED_PRECONDITION); }); + it('caps client semantic-search limit below admin and gRPC callers', async () => { + const seen: number[] = []; + const { api } = createApi({ + configs: [enabledConfig], + vectorSearch: async input => { + seen.push(input.limit ?? -1); + return []; + }, + }); + await api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1', limit: 1000 }, + { callerModule: 'router' }, + ); + await api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1', limit: 1000 }, + { callerModule: 'database' }, + ); + await api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1', limit: 1000 }, + { platformAdmin: true }, + ); + assert.deepEqual(seen, [50, 1000, 1000]); + }); + it('maps provider dimension mismatches and illegal backfill transitions to typed statuses', async () => { const { api } = createApi({ configs: [enabledConfig], diff --git a/modules/embeddings/src/api/embeddingsApi.ts b/modules/embeddings/src/api/embeddingsApi.ts index 16973ba26..896e07663 100644 --- a/modules/embeddings/src/api/embeddingsApi.ts +++ b/modules/embeddings/src/api/embeddingsApi.ts @@ -38,7 +38,9 @@ import { assertSemanticSearchAccess, canManageEmbeddingConfig, resolveAdminOperatorContext, + resolveSourceFieldAllowlist, } from '../utils/schemaPolicy.js'; +import { clampClientSearchLimit } from '../utils/clientSearchContext.js'; import { validateEmbeddingConfigInput } from '../utils/validateEmbeddingConfig.js'; import { assertConfigActivation, @@ -187,10 +189,11 @@ export class EmbeddingsApi { validateEmbeddingConfigInput( { ...request, - sourceFieldAllowlist: [ - ...(configDefaults.security.sourceFieldAllowlist ?? []), - ...(request.sourceFieldAllowlist ?? []), - ], + sourceFieldAllowlist: resolveSourceFieldAllowlist({ + operatorAllowlist: configDefaults.security.sourceFieldAllowlist, + requestAllowlist: request.sourceFieldAllowlist, + platformAdmin: caller.platformAdmin === true, + }), }, { provider: configDefaults.defaultProvider }, schema.fields, @@ -616,7 +619,7 @@ export class EmbeddingsApi { field: config.targetField, vector, filter: this.parseOptionalFilter(request.filter), - limit: request.limit, + limit: this.clampSemanticSearchLimit(request.limit, caller), userId: request.userId, scope: request.scope, adminOperator, @@ -639,6 +642,14 @@ export class EmbeddingsApi { return { code: status.INTERNAL, message: sanitizeErrorMessage(err) }; } + private clampSemanticSearchLimit( + limit: number | undefined, + caller: EmbeddingsApiCaller, + ): number | undefined { + if (caller.platformAdmin || caller.callerModule !== 'router') return limit; + return clampClientSearchLimit(limit); + } + private parseOptionalFilter(filter?: string): Record | undefined { try { return parseJsonObject(filter, 'filter'); diff --git a/modules/embeddings/src/config/index.ts b/modules/embeddings/src/config/index.ts index e4b313c90..4caded7bb 100644 --- a/modules/embeddings/src/config/index.ts +++ b/modules/embeddings/src/config/index.ts @@ -66,7 +66,7 @@ const AppConfigSchema = { default: false, }, sourceFieldAllowlist: { - doc: 'Source fields allowed even when their names look sensitive', + doc: 'Operator-configured source fields allowed even when hidden or sensitive-named. Caller-supplied allowlists are honored only for platform-admin upserts.', format: Array, default: [], }, diff --git a/modules/embeddings/src/routes/index.ts b/modules/embeddings/src/routes/index.ts index e317c135a..a8672b4a1 100644 --- a/modules/embeddings/src/routes/index.ts +++ b/modules/embeddings/src/routes/index.ts @@ -16,6 +16,7 @@ import { EmbeddingsApi } from '../api/embeddingsApi.js'; import { EMBEDDINGS_CLIENT_FORBIDDEN_PATHS } from '../admin/routes.js'; import { assertClientSearchSubject, + clampClientSearchLimit, clientSearchSubject, } from '../utils/clientSearchContext.js'; @@ -42,7 +43,7 @@ export class EmbeddingsRoutes { schemaName: call.request.params.schemaName, text: call.request.params.text, targetField: call.request.params.targetField, - limit: call.request.params.limit, + limit: clampClientSearchLimit(call.request.params.limit), filter: filter == null ? undefined @@ -69,7 +70,7 @@ export class EmbeddingsRoutes { path: '/search', action: ConduitRouteActions.POST, description: - 'Client semantic search by text. User and scope are taken from the authenticated router context; raw vectors, userId, scope, and adminOperator are not accepted.', + 'Client semantic search by text. User and scope are taken from the authenticated router context; raw vectors, userId, scope, and adminOperator are not accepted. Client limit is capped below the admin/gRPC vector-search maximum.', bodyParams: { schemaName: ConduitString.Required, text: ConduitString.Required, diff --git a/modules/embeddings/src/utils/backfillRun.test.ts b/modules/embeddings/src/utils/backfillRun.test.ts index 81ca9e734..fa590fe57 100644 --- a/modules/embeddings/src/utils/backfillRun.test.ts +++ b/modules/embeddings/src/utils/backfillRun.test.ts @@ -17,10 +17,12 @@ import { failBackfillRun, isLegalBackfillTransition, isResumeEligible, + isSafeBackfillFilter, LEGAL_BACKFILL_TRANSITIONS, MAX_BACKFILL_BATCH_SIZE, MAX_BACKFILL_ERROR_LENGTH, MAX_BACKFILL_FILTER_BYTES, + MAX_BACKFILL_FILTER_IN_VALUES, MIN_BACKFILL_BATCH_SIZE, resumeBackfillRun, sanitizeBackfillError, @@ -342,6 +344,34 @@ describe('backfill bounds', () => { }).ok, false, ); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { title: { $regex: 'a+' } }, + }).ok, + false, + ); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { $or: [{ published: true }] }, + }).ok, + false, + ); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { title: { $exists: true } }, + }).ok, + false, + ); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { body: { $like: '%secret%' } }, + }).ok, + false, + ); assert.equal(createQueuedBackfill({ schemaName: 'Article', filter: [] }).ok, false); assert.equal( createQueuedBackfill({ @@ -350,6 +380,25 @@ describe('backfill bounds', () => { }).ok, false, ); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { + status: { + $in: Array.from({ length: MAX_BACKFILL_FILTER_IN_VALUES + 1 }, () => 'a'), + }, + }, + }).ok, + false, + ); + assert.equal( + isSafeBackfillFilter({ published: true, status: { $in: ['draft', 'live'] } }), + true, + ); + assert.equal( + isSafeBackfillFilter({ $and: [{ published: true }, { views: { $gte: 1 } }] }), + true, + ); }); }); diff --git a/modules/embeddings/src/utils/backfillRun.ts b/modules/embeddings/src/utils/backfillRun.ts index b2ed528d6..3aa420a32 100644 --- a/modules/embeddings/src/utils/backfillRun.ts +++ b/modules/embeddings/src/utils/backfillRun.ts @@ -34,7 +34,32 @@ export const MAX_BACKFILL_FILTER_BYTES = 4 * 1024; const SCHEMA_NAME = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; const IDENTITY = /^[A-Za-z0-9._-]{1,128}$/; -const DANGEROUS_FILTER_KEY = /^(\$where|\$function|\$accumulator|__proto__|constructor)$/; +const BACKFILL_FILTER_FIELD = /^[A-Za-z_][A-Za-z0-9_]*$/; +const BACKFILL_FILTER_RESERVED_FIELDS = new Set([ + '__proto__', + 'prototype', + 'constructor', +]); + +/** + * Documented safe backfill filter subset. Equality, comparisons, bounded + * `$in`/`$nin`, and `$and` are allowed. Regex, existence, expression, `$or`, + * `$not`, `$like`, and other expensive/operator-injection shapes are rejected. + */ +export const BACKFILL_FILTER_COMPARISON_OPERATORS = [ + '$eq', + '$ne', + '$gt', + '$gte', + '$lt', + '$lte', +] as const; +export const BACKFILL_FILTER_MEMBERSHIP_OPERATORS = ['$in', '$nin'] as const; +export const BACKFILL_FILTER_LOGICAL_OPERATORS = ['$and'] as const; +export const MAX_BACKFILL_FILTER_DEPTH = 3; +export const MAX_BACKFILL_FILTER_KEYS = 16; +export const MAX_BACKFILL_FILTER_IN_VALUES = 32; +export const MAX_BACKFILL_FILTER_AND_BRANCHES = 8; export interface BackfillRunProgress { state: BackfillRunState; @@ -393,11 +418,99 @@ function transition( }; } +function isPlainFilterObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isBackfillFilterScalar(value: unknown): boolean { + return ( + value === null || + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' + ); +} + +function isSafeBackfillFieldName(field: string): boolean { + return ( + BACKFILL_FILTER_FIELD.test(field) && + !field.startsWith('$') && + !BACKFILL_FILTER_RESERVED_FIELDS.has(field) + ); +} + +function isComparisonOperator( + operator: string, +): operator is (typeof BACKFILL_FILTER_COMPARISON_OPERATORS)[number] { + return (BACKFILL_FILTER_COMPARISON_OPERATORS as readonly string[]).includes(operator); +} + +function isMembershipOperator( + operator: string, +): operator is (typeof BACKFILL_FILTER_MEMBERSHIP_OPERATORS)[number] { + return (BACKFILL_FILTER_MEMBERSHIP_OPERATORS as readonly string[]).includes(operator); +} + +function isSafeBackfillPredicate(value: unknown, depth: number): boolean { + if (depth > MAX_BACKFILL_FILTER_DEPTH) return false; + if (isBackfillFilterScalar(value)) return true; + if (!isPlainFilterObject(value)) return false; + const operators = Object.keys(value); + if (!operators.length || operators.length > MAX_BACKFILL_FILTER_KEYS) return false; + for (const operator of operators) { + if (isComparisonOperator(operator)) { + const comparison = value[operator]; + if (operator === '$eq' || operator === '$ne') { + if (!isBackfillFilterScalar(comparison)) return false; + continue; + } + if (typeof comparison !== 'number' && typeof comparison !== 'string') return false; + continue; + } + if (isMembershipOperator(operator)) { + const items = value[operator]; + if (!Array.isArray(items) || items.length > MAX_BACKFILL_FILTER_IN_VALUES) { + return false; + } + if (!items.every(isBackfillFilterScalar)) return false; + continue; + } + return false; + } + return true; +} + +export function isSafeBackfillFilter(value: unknown, depth = 1): boolean { + if (depth > MAX_BACKFILL_FILTER_DEPTH) return false; + if (!isPlainFilterObject(value)) return false; + const keys = Object.keys(value); + if (keys.length > MAX_BACKFILL_FILTER_KEYS) return false; + for (const key of keys) { + if (key === '$and') { + const branches = value[key]; + if ( + !Array.isArray(branches) || + branches.length === 0 || + branches.length > MAX_BACKFILL_FILTER_AND_BRANCHES + ) { + return false; + } + if (!branches.every(branch => isSafeBackfillFilter(branch, depth + 1))) { + return false; + } + continue; + } + if (key.startsWith('$') || !isSafeBackfillFieldName(key)) return false; + if (!isSafeBackfillPredicate(value[key], depth + 1)) return false; + } + return true; +} + function normalizeFilter( filter: unknown, ): { ok: true; filter: Record | null } | { ok: false; reason: string } { if (filter == null) return { ok: true, filter: null }; - if (typeof filter !== 'object' || Array.isArray(filter)) { + if (!isPlainFilterObject(filter)) { return { ok: false, reason: 'filter' }; } let serialized: string; @@ -409,20 +522,9 @@ function normalizeFilter( if (serialized.length > MAX_BACKFILL_FILTER_BYTES) { return { ok: false, reason: 'filter' }; } - if (hasDangerousFilterKey(filter)) { + const parsed = JSON.parse(serialized) as Record; + if (!isSafeBackfillFilter(parsed)) { return { ok: false, reason: 'filter' }; } - return { ok: true, filter: JSON.parse(serialized) as Record }; -} - -function hasDangerousFilterKey(value: unknown): boolean { - if (value == null || typeof value !== 'object') return false; - if (Array.isArray(value)) { - return value.some(hasDangerousFilterKey); - } - for (const [key, nested] of Object.entries(value as Record)) { - if (DANGEROUS_FILTER_KEY.test(key)) return true; - if (hasDangerousFilterKey(nested)) return true; - } - return false; + return { ok: true, filter: parsed }; } diff --git a/modules/embeddings/src/utils/clientSearchContext.test.ts b/modules/embeddings/src/utils/clientSearchContext.test.ts index c49e98b4c..bf2f23e41 100644 --- a/modules/embeddings/src/utils/clientSearchContext.test.ts +++ b/modules/embeddings/src/utils/clientSearchContext.test.ts @@ -2,7 +2,12 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { GrpcError } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; -import { assertClientSearchSubject, clientSearchSubject } from './clientSearchContext.js'; +import { + assertClientSearchSubject, + clampClientSearchLimit, + clientSearchSubject, + CLIENT_SEMANTIC_SEARCH_MAX_LIMIT, +} from './clientSearchContext.js'; describe('client semantic search context', () => { it('accepts text-only search subjects from router context and fail-closes otherwise', () => { @@ -22,4 +27,15 @@ describe('client semantic search context', () => { assertClientSearchSubject(clientSearchSubject({ user: { _id: 'user-1' } })), ); }); + + it('caps client semantic-search limit below the admin/gRPC maximum', () => { + assert.equal(clampClientSearchLimit(undefined), undefined); + assert.equal(clampClientSearchLimit(10), 10); + assert.equal(clampClientSearchLimit(1000), CLIENT_SEMANTIC_SEARCH_MAX_LIMIT); + assert.equal(CLIENT_SEMANTIC_SEARCH_MAX_LIMIT, 50); + assert.throws( + () => clampClientSearchLimit(0), + (err: unknown) => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, + ); + }); }); diff --git a/modules/embeddings/src/utils/clientSearchContext.ts b/modules/embeddings/src/utils/clientSearchContext.ts index 69d8f6d6d..970547da8 100644 --- a/modules/embeddings/src/utils/clientSearchContext.ts +++ b/modules/embeddings/src/utils/clientSearchContext.ts @@ -1,6 +1,8 @@ import { GrpcError } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; +export const CLIENT_SEMANTIC_SEARCH_MAX_LIMIT = 50; + export function clientSearchSubject(context?: { user?: { _id?: unknown }; scope?: unknown; @@ -25,3 +27,11 @@ export function assertClientSearchSubject(subject: { userId?: string; scope?: st } return subject; } + +export function clampClientSearchLimit(limit?: number): number | undefined { + if (limit === undefined || limit === null) return undefined; + if (!Number.isInteger(limit) || limit < 1) { + throw new GrpcError(status.INVALID_ARGUMENT, 'limit must be a positive integer'); + } + return Math.min(limit, CLIENT_SEMANTIC_SEARCH_MAX_LIMIT); +} diff --git a/modules/embeddings/src/utils/schemaPolicy.test.ts b/modules/embeddings/src/utils/schemaPolicy.test.ts index de9b83a64..794ef2ab3 100644 --- a/modules/embeddings/src/utils/schemaPolicy.test.ts +++ b/modules/embeddings/src/utils/schemaPolicy.test.ts @@ -9,17 +9,42 @@ import { assertSourceFields, isDeniedEmbeddingSchema, resolveAdminOperatorContext, + resolveSourceFieldAllowlist, } from './schemaPolicy.js'; describe('embedding schema and source policies', () => { it('denies system, auth-secret, and embeddings-owned schemas', () => { assert.equal(isDeniedEmbeddingSchema({ name: 'EmbeddingConfig' }), true); assert.equal(isDeniedEmbeddingSchema({ name: 'BackfillRun' }), true); + assert.equal( + isDeniedEmbeddingSchema({ name: 'CustomOps', ownerModule: 'embeddings' }), + true, + ); assert.equal(isDeniedEmbeddingSchema({ name: '_DeclaredSchema' }), true); - assert.equal(isDeniedEmbeddingSchema({ name: 'Views' }), true); - assert.equal(isDeniedEmbeddingSchema({ name: 'AccessToken' }), true); - assert.equal(isDeniedEmbeddingSchema({ name: 'TwoFactorSecret' }), true); - assert.equal(isDeniedEmbeddingSchema({ name: 'Article' }), false); + assert.equal( + isDeniedEmbeddingSchema({ name: 'Views', ownerModule: 'database' }), + true, + ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'AccessToken', ownerModule: 'authentication' }), + true, + ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'TwoFactorSecret', ownerModule: 'authentication' }), + true, + ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'Article', ownerModule: 'cms-app' }), + false, + ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'User', ownerModule: 'authentication' }), + false, + ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'File', ownerModule: 'storage' }), + false, + ); assert.throws( () => assertEmbeddingTargetSchema({ name: 'RefreshToken' }), err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, @@ -92,6 +117,25 @@ describe('embedding schema and source policies', () => { ); }); + it('honors caller-supplied sourceFieldAllowlist only for platform-admin context', () => { + assert.deepEqual( + resolveSourceFieldAllowlist({ + operatorAllowlist: ['summary'], + requestAllowlist: ['password', 'notes'], + platformAdmin: false, + }), + ['summary'], + ); + assert.deepEqual( + resolveSourceFieldAllowlist({ + operatorAllowlist: ['summary'], + requestAllowlist: ['password', 'notes'], + platformAdmin: true, + }), + ['summary', 'password', 'notes'], + ); + }); + it('requires subject, scope, or a verified admin operator for semantic search', () => { assert.throws( () => assertSemanticSearchAccess({}), diff --git a/modules/embeddings/src/utils/schemaPolicy.ts b/modules/embeddings/src/utils/schemaPolicy.ts index c7479c8cd..cea44a1fe 100644 --- a/modules/embeddings/src/utils/schemaPolicy.ts +++ b/modules/embeddings/src/utils/schemaPolicy.ts @@ -4,6 +4,7 @@ import { BACKFILL_RUN_SCHEMA } from './backfillRun.js'; export const EMBEDDING_CONFIG_SCHEMA = 'EmbeddingConfig'; export { BACKFILL_RUN_SCHEMA }; +export const EMBEDDINGS_OWNER_MODULE = 'embeddings'; export const EMBEDDING_OWNED_SCHEMA_NAMES = new Set([ EMBEDDING_CONFIG_SCHEMA, BACKFILL_RUN_SCHEMA, @@ -53,11 +54,16 @@ export function isHiddenField(field: unknown): boolean { return isRecord(field) && field.select === false; } +/** + * Explicit denylist for embeddings sources. Owner-controlled business schemas + * (including authentication User/Team) are not denied by ownerModule alone. + */ export function isDeniedEmbeddingSchema(schema: { name: string; ownerModule?: string; }): boolean { if (!schema.name) return true; + if (schema.ownerModule === EMBEDDINGS_OWNER_MODULE) return true; if (EMBEDDING_OWNED_SCHEMA_NAMES.has(schema.name)) return true; if (schema.name.startsWith('_')) return true; if (SYSTEM_SCHEMA_NAMES.has(schema.name)) return true; @@ -126,6 +132,29 @@ export function assertSemanticSearchAccess(args: { ); } +export function normalizeSourceFieldAllowlist(fields?: string[]): string[] { + return [ + ...new Set( + (fields ?? []).filter(field => typeof field === 'string' && field.length > 0), + ), + ]; +} + +export function resolveSourceFieldAllowlist(args: { + operatorAllowlist?: string[]; + requestAllowlist?: string[]; + platformAdmin?: boolean; +}): string[] { + const operatorAllowlist = normalizeSourceFieldAllowlist(args.operatorAllowlist); + if (!args.platformAdmin) return operatorAllowlist; + return [ + ...new Set([ + ...operatorAllowlist, + ...normalizeSourceFieldAllowlist(args.requestAllowlist), + ]), + ]; +} + export function assertSourceFields(args: { sourceFields: string[]; schemaFields: Record; From fd4cfbd24386f79f0825609cff9f213916439aba Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 21:29:34 +0300 Subject: [PATCH 16/29] fix(embeddings): type atomic backfill counts as Query $inc updates Keep concurrent backfill counters on the Database $inc contract. Production tsc can then accept findByIdAndUpdate without casts. --- libraries/grpc-sdk/src/types/db.ts | 9 +++++++++ modules/embeddings/src/Embeddings.ts | 6 +++++- modules/embeddings/src/utils/backfillRun.test.ts | 16 ++++++++++++++++ modules/embeddings/src/utils/backfillRun.ts | 16 +++++++++++++--- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/libraries/grpc-sdk/src/types/db.ts b/libraries/grpc-sdk/src/types/db.ts index dd0396c58..55d4f5117 100644 --- a/libraries/grpc-sdk/src/types/db.ts +++ b/libraries/grpc-sdk/src/types/db.ts @@ -37,9 +37,18 @@ type setQuery = { $set: simpleQuery; }; +type numericDocumentKeys = { + [K in keyof T]-?: NonNullable extends number ? K : never; +}[keyof T]; + +type incQuery = { + $inc: { [K in numericDocumentKeys]?: number }; +}; + export type Query = | simpleQuery | pushQuery | setQuery + | incQuery // | arrayQuery | conditionalQuery; diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts index a2900b85d..ecbeb4e67 100644 --- a/modules/embeddings/src/Embeddings.ts +++ b/modules/embeddings/src/Embeddings.ts @@ -43,6 +43,7 @@ import { processBackfillControllerJob, type BackfillControllerJobData, } from './utils/backfillExecution.js'; +import { toBackfillCountUpdateQuery } from './utils/backfillRun.js'; import { incrementEmbeddingMetric } from './utils/embeddingMetrics.js'; import metricsSchema from './metrics/index.js'; import { EmbeddingsApi } from './api/embeddingsApi.js'; @@ -550,7 +551,10 @@ export default class EmbeddingsModule extends ManagedModule { runId, outcome, incrementCounts: async (id, patch) => { - const updated = await BackfillRun.getInstance().findByIdAndUpdate(id, patch); + const updated = await BackfillRun.getInstance().findByIdAndUpdate( + id, + toBackfillCountUpdateQuery(patch), + ); return updated ? backfillRunFromDocument(updated) : null; }, }); diff --git a/modules/embeddings/src/utils/backfillRun.test.ts b/modules/embeddings/src/utils/backfillRun.test.ts index fa590fe57..031ccf2bf 100644 --- a/modules/embeddings/src/utils/backfillRun.test.ts +++ b/modules/embeddings/src/utils/backfillRun.test.ts @@ -1,5 +1,7 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; +import type { Query } from '@conduitplatform/grpc-sdk'; +import type { BackfillRun } from '../models/BackfillRun.schema.js'; import { applyAtomicBackfillCountDelta, applyBackfillJobCounts, @@ -27,6 +29,7 @@ import { resumeBackfillRun, sanitizeBackfillError, startBackfillRun, + toBackfillCountUpdateQuery, } from './backfillRun.js'; const now = new Date('2026-09-06T16:00:00.000Z'); @@ -226,6 +229,19 @@ describe('backfill counters', () => { assert.equal(counters.processedCount, 30); assert.equal(counters.failedCount, 10); }); + + it('types atomic count patches as Query-compatible $inc updates', () => { + const processed: Query = toBackfillCountUpdateQuery( + backfillCountIncrementPatch('processed'), + ); + const failed: Query = toBackfillCountUpdateQuery( + backfillCountIncrementPatch('failed'), + ); + assert.deepEqual(processed, { $inc: { processedCount: 1 } }); + assert.deepEqual(failed, { $inc: { failedCount: 1 } }); + assert.equal('$set' in processed, false); + assert.equal('$set' in failed, false); + }); }); describe('backfill cancellation and resume', () => { diff --git a/modules/embeddings/src/utils/backfillRun.ts b/modules/embeddings/src/utils/backfillRun.ts index 3aa420a32..d7914511f 100644 --- a/modules/embeddings/src/utils/backfillRun.ts +++ b/modules/embeddings/src/utils/backfillRun.ts @@ -1,3 +1,4 @@ +import type { Query } from '@conduitplatform/grpc-sdk'; import { sanitizeErrorMessage } from './redactConfig.js'; import { MAX_QUEUE_BATCH_SIZE } from './embeddingJobs.js'; @@ -316,9 +317,12 @@ export function applyBackfillPage( }; } -export type BackfillCountIncrementPatch = { - $inc: { processedCount?: number; failedCount?: number }; -}; +type BackfillCountDocument = Pick; + +export type BackfillCountIncrementPatch = Extract< + Query, + { $inc: unknown } +>; export function backfillCountIncrementPatch( outcome: 'processed' | 'failed', @@ -328,6 +332,12 @@ export function backfillCountIncrementPatch( }; } +export function toBackfillCountUpdateQuery( + patch: BackfillCountIncrementPatch, +): Query { + return patch; +} + export function applyAtomicBackfillCountDelta( counters: { processedCount: number; failedCount: number }, outcome: 'processed' | 'failed', From f0b9a4023cd046761bab07941f798478e1b8f504 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 21:37:11 +0300 Subject: [PATCH 17/29] fix(embeddings): close remaining opt-in deploy review gaps Require a non-empty GRPC_KEY in profile enablement examples, distinguish Helm workload install.embeddings.enabled from convict enabled, and keep embeddings-only image rebuilds off standalone v1. --- .github/workflows/embeddings-test.yml | 58 +++++++ deploy/docker/README.md | 6 +- deploy/embeddings.md | 71 +++++--- deploy/k8s/README.md | 10 +- docker/.env | 3 +- docker/docker-compose.standalone.yml | 6 +- docker/docker-compose.yml | 7 +- modules/embeddings/README.md | 24 ++- modules/embeddings/package.json | 2 +- .../test/deployment-contract.test.mjs | 154 ++++++++++++++++++ scripts/resolve-docker-targets.mjs | 71 +++++--- 11 files changed, 348 insertions(+), 64 deletions(-) create mode 100644 modules/embeddings/test/deployment-contract.test.mjs diff --git a/.github/workflows/embeddings-test.yml b/.github/workflows/embeddings-test.yml index bf54fae2b..598d5539c 100644 --- a/.github/workflows/embeddings-test.yml +++ b/.github/workflows/embeddings-test.yml @@ -10,6 +10,9 @@ on: - 'libraries/grpc-sdk/**' - 'libraries/module-tools/**' - 'packages/core/**' + - 'docker/**' + - 'deploy/**' + - 'scripts/resolve-docker-targets.mjs' - '.github/workflows/embeddings-test.yml' push: branches: @@ -21,6 +24,9 @@ on: - 'libraries/grpc-sdk/**' - 'libraries/module-tools/**' - 'packages/core/**' + - 'docker/**' + - 'deploy/**' + - 'scripts/resolve-docker-targets.mjs' - '.github/workflows/embeddings-test.yml' permissions: @@ -77,3 +83,55 @@ jobs: - name: Run hermes vector offline tests run: pnpm --filter @conduitplatform/hermes test + + deploy-contracts: + runs-on: ubuntu-24.04 + name: Compose render and target discovery + steps: + - name: Checkout + uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + + - name: Render compose with embeddings profile + working-directory: docker + env: + GRPC_KEY: ci-nonempty-grpc-key + run: | + docker compose --profile mongodb --profile embeddings config --services > /tmp/compose-embeddings-services.txt + grep -qx 'embeddings' /tmp/compose-embeddings-services.txt + docker compose --profile mongodb --profile embeddings config > /tmp/compose-embeddings.yml + grep -q 'ci-nonempty-grpc-key' /tmp/compose-embeddings.yml + grep -q '55165' /tmp/compose-embeddings.yml + + - name: Render compose without embeddings profile + working-directory: docker + run: | + docker compose --profile mongodb config --services > /tmp/compose-default-services.txt + if grep -qx 'embeddings' /tmp/compose-default-services.txt; then + echo 'embeddings service must stay profile-gated' >&2 + exit 1 + fi + + - name: Discover docker targets for embeddings-only changes + env: + CHANGED_FILES: modules/embeddings/src/index.ts + run: | + node scripts/resolve-docker-targets.mjs > /tmp/targets.json + node -e ' + const fs = require("fs"); + const data = JSON.parse(fs.readFileSync("/tmp/targets.json", "utf8")); + const matrix = JSON.parse(data.matrix); + const targets = matrix.include.map((row) => row.target); + if (!targets.includes("embeddings")) { + throw new Error("expected embeddings target, got " + targets.join(",")); + } + if (targets.includes("conduit-standalone")) { + throw new Error("embeddings-only changes must not select standalone: " + targets.join(",")); + } + ' + + - name: Run deployment contract tests + run: node --test modules/embeddings/test/deployment-contract.test.mjs diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 8463bffe7..c858071dd 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -30,5 +30,7 @@ To run the microservices version: - Open the router in your browser at [http://localhost:8081](http://localhost:8081) - You can inject `--profile {profile_name}` command on compose to configure more services (`mongodb` / `postgres` for the database engine, `embeddings` for the embeddings module). - Embeddings is disabled by default, omitted from standalone v1, and requires `GRPC_KEY` in - production. See the [embeddings rollout runbook](../embeddings.md). + Embeddings stays omitted until you pass `--profile embeddings` **and** export a + non-empty `GRPC_KEY`. The embeddings image is not published until a compatible + release tag exists. Helm workload `install.embeddings.enabled` is separate from + module convict `enabled`. See the [embeddings rollout runbook](../embeddings.md). diff --git a/deploy/embeddings.md b/deploy/embeddings.md index 6484a3e32..bd5d71886 100644 --- a/deploy/embeddings.md +++ b/deploy/embeddings.md @@ -1,35 +1,48 @@ # Embeddings rollout and rollback Embeddings is a separate, disabled-by-default module image. It is not part of -standalone v1. Live MongoDB Atlas, pgvector, Redis, and provider suites are -**not** covered by CI; operators must complete the capability and index -readiness checks below before enabling generation or search. +standalone v1. No embeddings image is published until a compatible release tag +exists; do not enable the compose profile or Helm workload against `latest` +until that tag is published. Live MongoDB Atlas, pgvector, Redis, and provider +suites are **not** covered by CI; operators must complete the capability and +index readiness checks below before enabling generation or search. -Production containers set `NODE_ENV=production` and **require `GRPC_KEY`**. -The module stays disabled (`enabled: false`) until an operator enables it -through Core config after peer health and vector capabilities are confirmed. +Production containers set `NODE_ENV=production` and **require a non-empty +`GRPC_KEY`**. Two independent enablement flags exist: + +- Helm workload `install.embeddings.enabled` (charts repo, default `false`) + only deploys or removes the embeddings process. It does not start workers. +- Module convict `enabled` (Core config, default `false`) turns on embedding + generation workers, mutation subscriptions, and search. Keep this `false` + until peer health and vector capabilities are confirmed. ## Compose (opt-in) ```bash -# Set GRPC_KEY in docker/.env before starting embeddings. +# A non-empty GRPC_KEY is required; empty values fail in production. +export GRPC_KEY='replace-with-a-non-empty-key' docker compose --profile mongodb --profile embeddings up ``` -- gRPC: `55165` (`EMBEDDINGS_GRPC_PORT`) +- gRPC: `${EMBEDDINGS_GRPC_PORT:-55165}` (container `GRPC_PORT` uses the same value) - Metrics: `9192` (Prometheus scrapes `conduit-embeddings:9192`) -- Image: `docker.io/conduitplatform/embeddings:${IMAGE_TAG}` +- Image name after a compatible release: `docker.io/conduitplatform/embeddings:`. + Compose interpolates `${IMAGE_TAG}`; that tag is not published by this change. -Helm values (`install.embeddings`) are documented in the charts repository -and remain disabled by default. +Helm workload `install.embeddings.enabled` is documented in the charts +repository and remains `false` by default. Setting it to `true` deploys the +pod with module convict `enabled` still false. ## Rollout order -1. Release compatible Core, Database, grpc-sdk, and the embeddings image. -2. Deploy embeddings **disabled**. Confirm the process is serving and waiting - on / registered with Core. Health stays serving while disabled so operators - can configure the module. -3. Set `GRPC_KEY` (required in production). Confirm gRPC peer health. +1. Publish compatible Core, Database, grpc-sdk, **and** the embeddings image + tag you will run. Do not start the workload before that tag exists. +2. Deploy the embeddings **workload** with convict `enabled: false` + (`install.embeddings.enabled=true` in Helm, or the compose embeddings + profile with a non-empty `GRPC_KEY`). Confirm the process is serving and + waiting on / registered with Core. Health stays serving while workers are + disabled so operators can configure the module. +3. Confirm `GRPC_KEY` is set and gRPC peer health is good. 4. Call `GET /embeddings/capabilities` (or gRPC `getCapabilities`) and verify Database `getVectorCapabilities`: storage, indexing, and search must be true for the target backend (MongoDB Atlas Vector Search or Postgres @@ -49,23 +62,29 @@ and remain disabled by default. 8. Run a scoped canary semantic search (`POST /embeddings/search` as an operator, or client search with authenticated user/scope). Confirm fail-closed behavior on authorization-enabled schemas. -9. Enable workers/search for normal traffic (`enabled: true` on module config). +9. Enable workers/search for normal traffic (module convict `enabled: true` + through Core config). This is not `install.embeddings.enabled`. ## Rollback -1. Disable workers and embedding configs (`enabled: false`). Generation and - search stop; existing vectors remain. -2. Scale down or stop the embeddings service: +1. Disable workers and embedding configs (module convict `enabled: false` + and per-config `enabled: false`). Generation and search stop; existing + vectors remain. +2. Scale down or stop the embeddings **workload**: - Compose: omit `--profile embeddings` / `docker compose stop embeddings` - - Helm: `install.embeddings: false` (charts repo) -3. Roll back the embeddings image and/or chart to the previous version. -4. Do **not** automatically delete vector fields, indexes, `EmbeddingConfig` - documents, `BackfillRun` records, or Redis/BullMQ state. Data and index + - Helm: `install.embeddings.enabled=false` (charts repo). This is the + workload flag, not module convict `enabled`. +3. Roll back the embeddings image and/or chart to the previous **published** + version, if any. +4. Rollback **retains** vector fields, indexes, `EmbeddingConfig` documents, + `BackfillRun` records, and Redis/BullMQ queue state. Data and index removal is a separate explicit operator action. ## Residual validation Offline CI covers unit/contract tests, bundle smoke (`Waiting for Core`), image target discovery, and compose rendering. It does not prove Atlas, -pgvector, Redis queue behavior, or a live provider. Repeat capability and -index readiness checks in the target environment before activation. +pgvector, Redis queue behavior, a live provider, or a published embeddings +image. Repeat capability and index readiness checks in the target +environment before activation. The remaining release prerequisite is +publishing the first compatible embeddings image tag. diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index 94891887a..72584d306 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -7,6 +7,10 @@ Current setup includes: - [Minikube](minikube.md) - [AKS](aks.md) -Embeddings is not part of the standalone image. For a disabled-by-default -embeddings rollout, capability/index readiness, and rollback, see -[embeddings.md](../embeddings.md). Helm values live in the charts repository. +Embeddings is not part of the standalone image. Helm workload +`install.embeddings.enabled` (charts repo, default `false`) deploys the +process; module convict `enabled` is a separate Core config switch. For a +disabled-by-default embeddings rollout, capability/index readiness, and +rollback (`install.embeddings.enabled=false`, retained vector/index/config/ +Redis state), see [embeddings.md](../embeddings.md). No embeddings image is +published until a compatible release tag exists. diff --git a/docker/.env b/docker/.env index c4194ff64..10f1e39d6 100644 --- a/docker/.env +++ b/docker/.env @@ -31,7 +31,8 @@ DB_CONN_URI="mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin #DB_CONN_URI="postgres://conduit:pass@conduit-postgres:5432/conduit" # profile: postgres # Security -# Embeddings (NODE_ENV=production in the image) requires a non-empty GRPC_KEY. +# Leave GRPC_KEY empty for default profiles. Enabling '--profile embeddings' +# requires exporting a non-empty GRPC_KEY; production images refuse empty values. CORE_MASTER_KEY="M4ST3RK3Y" GRPC_KEY="" diff --git a/docker/docker-compose.standalone.yml b/docker/docker-compose.standalone.yml index b061788dd..aaf8e5db8 100644 --- a/docker/docker-compose.standalone.yml +++ b/docker/docker-compose.standalone.yml @@ -2,8 +2,10 @@ # This compose file deploys a "standalone" version of conduit with most modules # packaged in a single image. Loki and Prometheus are not deployed, since # metrics and logs can be viewed directly from the Docker daemon. -# Embeddings is not included in standalone v1; use docker-compose.yml with -# --profile embeddings after publishing a compatible embeddings image. +# Embeddings is not included in standalone v1. After a compatible embeddings +# image is published, use docker-compose.yml with a non-empty GRPC_KEY: +# export GRPC_KEY='replace-with-a-non-empty-key' +# docker compose --profile embeddings up #------------------------------------------------------------------------------------------- version: '3.9' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 610f81a17..102f68d58 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,10 +11,13 @@ # Otherwise, simply update the env values with any available port. # 3. Specify at least '--profile mongodb' or '--profile postgres' # If you're going to use PostgreSQL, swap out 'DB_CONN_URI' in the '.env' file -# 4. Embeddings is opt-in: add '--profile embeddings'. Production requires GRPC_KEY. +# 4. Embeddings is opt-in and disabled by default. Enabling the profile requires +# exporting a non-empty GRPC_KEY (production images refuse to start without it). +# The embeddings image tag is not published until a compatible release exists. # # Examples: # docker compose --profile mongodb up +# export GRPC_KEY='replace-with-a-non-empty-key' # docker compose --profile mongodb --profile embeddings up # ---------------------------------------------------------------------------------------------- @@ -243,7 +246,7 @@ services: environment: CONDUIT_SERVER: 'conduit:${CORE_GRPC_PORT:-55152}' SERVICE_URL: 'conduit-embeddings:${EMBEDDINGS_GRPC_PORT:-55165}' - GRPC_PORT: '55165' + GRPC_PORT: '${EMBEDDINGS_GRPC_PORT:-55165}' METRICS_PORT: '9192' LOKI_URL: 'http://conduit-loki:3100' GRPC_KEY: '${GRPC_KEY}' diff --git a/modules/embeddings/README.md b/modules/embeddings/README.md index 4a22beaaa..455be2acd 100644 --- a/modules/embeddings/README.md +++ b/modules/embeddings/README.md @@ -6,10 +6,13 @@ for vector storage, index creation, and vector-in/vector-out search. ## Configuration -The module is disabled by default. Production deployments require `GRPC_KEY` -(`NODE_ENV=production` in the published image). The module is **not** included -in the standalone image for the first production release; run it as a separate -opt-in compose profile or Helm service. +The module convict `enabled` setting is `false` by default. Production +deployments require a non-empty `GRPC_KEY` (`NODE_ENV=production` in the +image). The module is **not** included in the standalone image for the first +production release; run it as a separate opt-in compose profile or Helm +workload after a compatible image tag is published. Helm +`install.embeddings.enabled` only deploys the process; it does not set convict +`enabled`. Enable it and configure an OpenAI-compatible provider: @@ -69,11 +72,16 @@ authenticated router context. ## Packaging -- Bundle image: `docker.io/conduitplatform/embeddings` (BullMQ is an extra - bundle dependency). Bake target: `embeddings`. -- Compose: `docker compose --profile embeddings up` (gRPC `55165`, metrics - `9192`). Set `GRPC_KEY` before starting. +- Bake target: `embeddings` (BullMQ is an extra bundle dependency). The image + is not published until a compatible release; do not pull + `docker.io/conduitplatform/embeddings:latest` until that tag exists. +- Compose: export a non-empty `GRPC_KEY`, then + `docker compose --profile embeddings up` (gRPC + `${EMBEDDINGS_GRPC_PORT:-55165}`, metrics `9192`). - Standalone v1 does not ship embeddings. +- Helm `install.embeddings.enabled` (charts repo) deploys the workload only. + Module convict `enabled` (default false) is a separate Core config switch + for workers and search. Operator rollout, capability/index readiness, and rollback: [deploy/embeddings.md](../../deploy/embeddings.md). diff --git a/modules/embeddings/package.json b/modules/embeddings/package.json index a1845dae1..c76d3f0cb 100644 --- a/modules/embeddings/package.json +++ b/modules/embeddings/package.json @@ -24,7 +24,7 @@ "prebuild:bundle": "pnpm --filter @conduitplatform/service-bundle run build", "build:bundle": "rimraf bundle && node ../../libraries/service-bundle/dist/cli.js generate-manifest && tsup && node ../../libraries/service-bundle/dist/cli.js copy-assets && node ../../libraries/service-bundle/dist/cli.js generate-lockfile", "generateTypes": "sh build.sh", - "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utils/*.test.js dist-test/controllers/*.test.js dist-test/providers/*.test.js dist-test/api/*.test.js test/embedding-contract.test.mjs", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utils/*.test.js dist-test/controllers/*.test.js dist-test/providers/*.test.js dist-test/api/*.test.js test/embedding-contract.test.mjs test/deployment-contract.test.mjs", "build:docker": "docker build -t ghcr.io/conduitplatform/embeddings:latest -f ./Dockerfile ../../ && docker push ghcr.io/conduitplatform/embeddings:latest" }, "dependencies": { diff --git a/modules/embeddings/test/deployment-contract.test.mjs b/modules/embeddings/test/deployment-contract.test.mjs new file mode 100644 index 000000000..a63207627 --- /dev/null +++ b/modules/embeddings/test/deployment-contract.test.mjs @@ -0,0 +1,154 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { test } from 'node:test'; +import { + IMAGE_TARGETS, + resolveTargets, +} from '../../../scripts/resolve-docker-targets.mjs'; + +const repo = new URL('../../..', import.meta.url); +const readRepo = relativePath => + readFileSync(new URL(relativePath, repo), 'utf8'); + +const runbook = readRepo('deploy/embeddings.md'); +const moduleReadme = readRepo('modules/embeddings/README.md'); +const composeSource = readRepo('docker/docker-compose.yml'); +const standaloneCompose = readRepo('docker/docker-compose.standalone.yml'); +const dockerReadme = readRepo('deploy/docker/README.md'); +const k8sReadme = readRepo('deploy/k8s/README.md'); +const convictConfig = readRepo('modules/embeddings/src/config/index.ts'); +const workflow = readRepo('.github/workflows/embeddings-test.yml'); + +const PROFILE_ENABLEMENT_DOCS = [ + ['deploy/embeddings.md', runbook], + ['modules/embeddings/README.md', moduleReadme], + ['docker/docker-compose.yml', composeSource], + ['docker/docker-compose.standalone.yml', standaloneCompose], + ['deploy/docker/README.md', dockerReadme], +]; + +function assertProfileExamplesRequireGrpcKey(source, label) { + const lines = source.split('\n'); + let seen = 0; + for (const [index, line] of lines.entries()) { + if (!line.includes('--profile embeddings')) { + continue; + } + if (/omit `--profile embeddings`|omit --profile embeddings/.test(line)) { + continue; + } + seen += 1; + const window = lines.slice(Math.max(0, index - 3), index + 3).join('\n'); + assert.match( + window, + /export GRPC_KEY=|non-empty `GRPC_KEY`|non-empty GRPC_KEY/, + `${label}:${index + 1} profile enablement must require/export a non-empty GRPC_KEY`, + ); + } + assert.ok(seen > 0, `${label} must document --profile embeddings`); +} + +test('compose profile enablement examples require a non-empty GRPC_KEY', () => { + for (const [label, source] of PROFILE_ENABLEMENT_DOCS) { + assertProfileExamplesRequireGrpcKey(source, label); + } +}); + +test('runbook distinguishes Helm workload install.embeddings.enabled from convict enabled', () => { + assert.match(runbook, /Helm workload `install\.embeddings\.enabled`/); + assert.match(runbook, /Module convict `enabled`/); + assert.match(runbook, /install\.embeddings\.enabled=false/); + assert.match( + runbook, + /This is not `install\.embeddings\.enabled`/, + ); + assert.doesNotMatch(runbook, /Helm: `install\.embeddings: false`/); + assert.match(k8sReadme, /install\.embeddings\.enabled/); + assert.match(moduleReadme, /install\.embeddings\.enabled/); +}); + +test('rollback retains vector, index, config, and Redis state', () => { + assert.match(runbook, /Rollback \*\*retains\*\*/); + assert.match(runbook, /vector fields, indexes, `EmbeddingConfig` documents/); + assert.match(runbook, /Redis\/BullMQ queue state/); + assert.match(k8sReadme, /retained vector\/index\/config\/\nRedis state|retained vector/); +}); + +test('docs stay default-off and do not claim a published embeddings image', () => { + assert.match(convictConfig, /default: false/); + assert.match(runbook, /not published/); + assert.match(moduleReadme, /not published until a compatible release/); + assert.doesNotMatch( + runbook, + /Image: `docker\.io\/conduitplatform\/embeddings:\$\{IMAGE_TAG\}`/, + ); + assert.match(composeSource, /profiles: \['embeddings'\]/); +}); + +test('compose maps container GRPC_PORT through EMBEDDINGS_GRPC_PORT', () => { + assert.match( + composeSource, + /GRPC_PORT: '\$\{EMBEDDINGS_GRPC_PORT:-55165\}'/, + ); + assert.match( + composeSource, + /SERVICE_URL: 'conduit-embeddings:\$\{EMBEDDINGS_GRPC_PORT:-55165\}'/, + ); + assert.match( + composeSource, + /'\$\{EMBEDDINGS_GRPC_PORT:-55165\}:\$\{EMBEDDINGS_GRPC_PORT:-55165\}'/, + ); +}); + +test('PR CI runs compose render and target discovery', () => { + assert.match( + workflow, + /docker compose --profile mongodb --profile embeddings config --services/, + ); + assert.match(workflow, /docker compose --profile mongodb config --services/); + assert.match(workflow, /node scripts\/resolve-docker-targets\.mjs/); + assert.match(workflow, /docker\/\*\*/); + assert.match(workflow, /scripts\/resolve-docker-targets\.mjs/); +}); + +test('embeddings-only changes select embeddings and exclude standalone', () => { + const standalone = IMAGE_TARGETS.find(entry => entry.target === 'conduit-standalone'); + assert.deepEqual(standalone?.excludePaths, ['modules/embeddings/**']); + + const selected = resolveTargets({ + changedFiles: ['modules/embeddings/src/index.ts'], + forceAll: false, + }).map(entry => entry.target); + + assert.ok(selected.includes('embeddings')); + assert.ok(!selected.includes('conduit-standalone')); + assert.ok(!selected.includes('chat')); +}); + +test('other module rebuilds still select standalone', () => { + const chatSelected = resolveTargets({ + changedFiles: ['modules/chat/src/Chat.ts'], + forceAll: false, + }).map(entry => entry.target); + assert.ok(chatSelected.includes('chat')); + assert.ok(chatSelected.includes('conduit-standalone')); + assert.ok(!chatSelected.includes('embeddings')); + + const mixed = resolveTargets({ + changedFiles: [ + 'modules/embeddings/src/index.ts', + 'modules/storage/src/Storage.ts', + ], + forceAll: false, + }).map(entry => entry.target); + assert.ok(mixed.includes('embeddings')); + assert.ok(mixed.includes('storage')); + assert.ok(mixed.includes('conduit-standalone')); + + const shared = resolveTargets({ + changedFiles: ['docker-bake.hcl'], + forceAll: false, + }).map(entry => entry.target); + assert.ok(shared.includes('embeddings')); + assert.ok(shared.includes('conduit-standalone')); +}); diff --git a/scripts/resolve-docker-targets.mjs b/scripts/resolve-docker-targets.mjs index 3c7763b87..74dadae83 100644 --- a/scripts/resolve-docker-targets.mjs +++ b/scripts/resolve-docker-targets.mjs @@ -1,6 +1,8 @@ #!/usr/bin/env node import { appendFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; const SHARED_BUILD_PATHS = [ 'Dockerfile', @@ -170,6 +172,9 @@ const IMAGE_TARGETS = [ ...SERVICE_BUNDLE_PATHS, ...SHARED_BUILD_PATHS, ], + // Embeddings is intentionally omitted from standalone v1. Keep other + // module path matches so chat/storage/etc. still rebuild standalone. + excludePaths: ['modules/embeddings/**'], }, ]; @@ -191,6 +196,13 @@ function matchesAnyPath(file, patterns) { return patterns.some((pattern) => globMatch(file, pattern)); } +function matchesTarget(file, entry) { + if (Array.isArray(entry.excludePaths) && matchesAnyPath(file, entry.excludePaths)) { + return false; + } + return matchesAnyPath(file, entry.paths); +} + function parseChangedFiles() { const raw = process.env.CHANGED_FILES ?? ''; if (!raw.trim()) { @@ -206,21 +218,19 @@ function shouldBuildAll() { return process.env.FORCE_ALL === 'true'; } -function resolveTargets() { - if (shouldBuildAll()) { +function resolveTargets({ changedFiles, forceAll } = {}) { + if (forceAll ?? shouldBuildAll()) { return IMAGE_TARGETS; } - const changed = parseChangedFiles(); + const changed = changedFiles ?? parseChangedFiles(); if (changed.length === 0) { return IMAGE_TARGETS; } - const selected = IMAGE_TARGETS.filter((entry) => - changed.some((file) => matchesAnyPath(file, entry.paths)), + return IMAGE_TARGETS.filter((entry) => + changed.some((file) => matchesTarget(file, entry)), ); - - return selected; } function writeOutput(matrix, channel) { @@ -235,17 +245,40 @@ function writeOutput(matrix, channel) { } } -const channel = - process.env.GITHUB_EVENT_NAME === 'release' ? 'release' : 'dev'; +function isMainModule() { + const entry = process.argv[1]; + if (!entry) { + return false; + } + try { + return import.meta.url === pathToFileURL(resolve(entry)).href; + } catch { + return false; + } +} + +if (isMainModule()) { + const channel = + process.env.GITHUB_EVENT_NAME === 'release' ? 'release' : 'dev'; -const matrix = resolveTargets().map( - ({ target, image, name, buildingService, isBundle }) => ({ - target, - image, - name, - building_service: buildingService, - is_bundle: isBundle === true, - }), -); + const matrix = resolveTargets().map( + ({ target, image, name, buildingService, isBundle }) => ({ + target, + image, + name, + building_service: buildingService, + is_bundle: isBundle === true, + }), + ); + + writeOutput(matrix, channel); +} -writeOutput(matrix, channel); +export { + IMAGE_TARGETS, + globMatch, + matchesAnyPath, + matchesTarget, + parseChangedFiles, + resolveTargets, +}; From 18c8101b41232be01ce053d59be87d2cc6f8843c Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 21:52:36 +0300 Subject: [PATCH 18/29] fix(embeddings): authorize ExtensionOnly indexes and isolate backfill counters Evaluate createVectorIndex canModify against the requested vector field so embeddings can index its own extension fields, persist failed provisioning as disabled, and keep page/state saves from overwriting atomic processed/failed counts. --- modules/database/src/Database.ts | 18 ++++- .../__tests__/canModify.vectorIndex.test.ts | 72 +++++++++++++++++ modules/database/src/permissions/index.ts | 5 ++ modules/embeddings/src/admin/routes.ts | 2 +- .../embeddings/src/api/embeddingsApi.test.ts | 78 ++++++++++++++++++- modules/embeddings/src/api/embeddingsApi.ts | 34 +++++--- .../src/utils/backfillExecution.test.ts | 64 ++++++++++++++- .../embeddings/src/utils/backfillExecution.ts | 12 ++- 8 files changed, 263 insertions(+), 22 deletions(-) create mode 100644 modules/database/src/permissions/__tests__/canModify.vectorIndex.test.ts diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index 9719b29df..cefe805bc 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -53,7 +53,12 @@ import { MongooseAdapter } from './adapters/mongoose-adapter/index.js'; import { MongooseSchema } from './adapters/mongoose-adapter/MongooseSchema.js'; import { SequelizeSchema } from './adapters/sequelize-adapter/SequelizeSchema.js'; import { ConduitDatabaseSchema, IView, Schema } from './interfaces/index.js'; -import { canCreate, canDelete, canModify } from './permissions/index.js'; +import { + canCreate, + canDelete, + canModify, + vectorIndexMutationData, +} from './permissions/index.js'; import { runMigrations } from './migrations/index.js'; import { SchemaController } from './controllers/cms/schema.controller.js'; import { CustomEndpointController } from './controllers/customEndpoints/customEndpoint.controller.js'; @@ -1050,7 +1055,14 @@ export default class DatabaseModule extends ManagedModule { } const moduleName = call.metadata!.get('module-name')![0] as string; const schemaAdapter = this._activeAdapter.getSchemaModel(call.request.schemaName); - if (!(await canModify(moduleName, schemaAdapter.model))) { + const index = this.parseVectorIndex(call.request.index); + if ( + !(await canModify( + moduleName, + schemaAdapter.model, + vectorIndexMutationData(index.field), + )) + ) { return callback({ code: status.PERMISSION_DENIED, message: `Module ${moduleName} is not authorized to create vector indexes for ${call.request.schemaName}!`, @@ -1058,7 +1070,7 @@ export default class DatabaseModule extends ManagedModule { } const result = await this._activeAdapter.createVectorIndex( call.request.schemaName, - this.parseVectorIndex(call.request.index), + index, ); callback(null, { result: JSON.stringify(result) }); } catch (err) { diff --git a/modules/database/src/permissions/__tests__/canModify.vectorIndex.test.ts b/modules/database/src/permissions/__tests__/canModify.vectorIndex.test.ts new file mode 100644 index 000000000..91275efc2 --- /dev/null +++ b/modules/database/src/permissions/__tests__/canModify.vectorIndex.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from '@jest/globals'; +import { TYPE } from '@conduitplatform/grpc-sdk'; +import { canModify, vectorIndexMutationData } from '../index.js'; + +const embeddingExtension = { + ownerModule: 'embeddings', + fields: { + embedding: { type: TYPE.Vector, dimensions: 3 }, + embeddingSourceHash: { type: TYPE.String, select: false }, + }, + createdAt: new Date(), + updatedAt: new Date(), +}; + +function schema(args: { + ownerModule: string; + canModify: 'Everything' | 'Nothing' | 'ExtensionOnly'; + name?: string; +}) { + return { + originalSchema: { + name: args.name ?? 'User', + ownerModule: args.ownerModule, + modelOptions: { conduit: { permissions: { canModify: args.canModify } } }, + extensions: [embeddingExtension], + }, + }; +} + +describe('createVectorIndex canModify field evaluation', () => { + it('allows embeddings to index its own extension field on ExtensionOnly schemas', async () => { + const user = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + await expect( + canModify('embeddings', user as never, vectorIndexMutationData('embedding')), + ).resolves.toBe(true); + }); + + it('denies embeddings indexing unrelated fields on ExtensionOnly schemas', async () => { + const user = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + await expect( + canModify('embeddings', user as never, vectorIndexMutationData('email')), + ).resolves.toBe(false); + await expect( + canModify('chat', user as never, vectorIndexMutationData('embedding')), + ).resolves.toBe(false); + }); + + it('preserves owner and Everything authorization without field data', async () => { + const owned = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + const open = schema({ + ownerModule: 'database', + canModify: 'Everything', + name: 'Article', + }); + await expect(canModify('authentication', owned as never)).resolves.toBe(true); + await expect( + canModify('authentication', owned as never, vectorIndexMutationData('email')), + ).resolves.toBe(true); + await expect(canModify('embeddings', open as never)).resolves.toBe(true); + await expect( + canModify('chat', open as never, vectorIndexMutationData('title')), + ).resolves.toBe(true); + }); + + it('denies ExtensionOnly callers when the vector field is missing', async () => { + const user = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + await expect(canModify('embeddings', user as never)).resolves.toBe(false); + await expect( + canModify('embeddings', user as never, vectorIndexMutationData('')), + ).resolves.toBe(false); + }); +}); diff --git a/modules/database/src/permissions/index.ts b/modules/database/src/permissions/index.ts index 2c5452902..0a205c16b 100644 --- a/modules/database/src/permissions/index.ts +++ b/modules/database/src/permissions/index.ts @@ -17,6 +17,11 @@ export async function canCreate(moduleName: string, schema: Schema) { ); } +export function vectorIndexMutationData(field?: string): Indexable | undefined { + if (typeof field !== 'string' || field.length === 0) return undefined; + return { [field]: true }; +} + export async function canModify(moduleName: string, schema: Schema, data?: Indexable) { if (moduleName === 'database' && schema.originalSchema.name === '_DeclaredSchema') return true; diff --git a/modules/embeddings/src/admin/routes.ts b/modules/embeddings/src/admin/routes.ts index 4ac27f594..ce7b4c543 100644 --- a/modules/embeddings/src/admin/routes.ts +++ b/modules/embeddings/src/admin/routes.ts @@ -52,7 +52,7 @@ export const EMBEDDINGS_ADMIN_ROUTES: EmbeddingsAdminRouteContract[] = [ contract( '/configs', ConduitRouteActions.POST, - 'Creates or updates an embedding config for a schema. Operator-only. The first upsert provisions the vector index when Database indexing is available. Saving enabled=true while the index is pending stores the config disabled until it is queryable. If indexing is unavailable, status reports a manual index lifecycle warning. Caller-supplied sourceFieldAllowlist is honored only for platform-admin upserts; schema-owner gRPC callers use operator config allowlists.', + 'Creates or updates an embedding config for a schema. Operator-only. The first upsert provisions the vector index when Database indexing is available. Saving enabled=true while the index is pending, or if provisioning fails, stores the config disabled until it is queryable. If indexing is unavailable, status reports a manual index lifecycle warning. Caller-supplied sourceFieldAllowlist is honored only for platform-admin upserts; schema-owner gRPC callers use operator config allowlists.', ), contract( '/configs/:id', diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts index 4a42b929e..c1d52fc1a 100644 --- a/modules/embeddings/src/api/embeddingsApi.test.ts +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -78,6 +78,7 @@ function createApi(overrides?: { declared?: Record; embed?: EmbeddingsApiDeps['embed']; vectorSearch?: EmbeddingsApiDeps['vectorSearch']; + createVectorIndex?: EmbeddingsApiDeps['createVectorIndex']; enqueue?: string[]; invalidated?: string[]; deletedIndexes?: string[]; @@ -175,10 +176,12 @@ function createApi(overrides?: { enqueueBackfill: async job => { enqueued.push(job.runId); }, - createVectorIndex: async (_schema, index) => { - createdIndexes.push(index.name ?? index.field); - return 'created'; - }, + createVectorIndex: + overrides?.createVectorIndex ?? + (async (_schema, index) => { + createdIndexes.push(index.name ?? index.field); + return 'created'; + }), deleteVectorIndex: async (_schema, indexName) => { deletedIndexes.push(indexName); return 'deleted'; @@ -246,6 +249,73 @@ describe('typed embeddings API handlers', () => { assert.equal(configs[0].enabled, false); }); + it('saves the config disabled when vector index provisioning fails', async () => { + const { api, configs, createdIndexes } = createApi({ + indexes: [], + createVectorIndex: async () => { + throw new Error('atlas search index rejected'); + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(configs.length, 1); + assert.equal(configs[0].enabled, false); + assert.equal(createdIndexes.length, 0); + assert.equal( + saved.warnings.some( + warning => + /saved disabled because vector index provisioning failed/.test(warning) && + /atlas search index rejected/.test(warning) && + /queryable/.test(warning), + ), + true, + ); + }); + + it('saves the updated config disabled when index recreation fails', async () => { + const { api, configs, deletedIndexes } = createApi({ + configs: [enabledConfig], + createVectorIndex: async () => { + throw new Error('recreate rejected'); + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(configs[0].enabled, false); + assert.deepEqual(deletedIndexes, ['embedding_vector']); + assert.equal( + saved.warnings.some( + warning => + /saved disabled because vector index provisioning failed/.test(warning) && + /recreate rejected/.test(warning), + ), + true, + ); + }); + it('reports a manual index lifecycle when Database indexing is unavailable', async () => { const { api, createdIndexes } = createApi({ indexes: [], diff --git a/modules/embeddings/src/api/embeddingsApi.ts b/modules/embeddings/src/api/embeddingsApi.ts index 896e07663..33159b008 100644 --- a/modules/embeddings/src/api/embeddingsApi.ts +++ b/modules/embeddings/src/api/embeddingsApi.ts @@ -12,6 +12,7 @@ import { QueueJobCounts } from '../controllers/queue.controller.js'; import { cancelBackfillExecution, persistableBackfillRun, + persistableNewBackfillRun, queueBackfillRuns, resumeBackfillExecution, type BackfillControllerJobData, @@ -230,16 +231,25 @@ export class EmbeddingsApi { }, }); } + let persistEnabled = enabled; let provisionedIndex = false; - if (existing && changed.length && requiresIndexRecreation(changed)) { - await this.recreateVectorIndex(existing, persisted, indexes); - indexes = await this.deps.getVectorIndexes(persisted.schemaName); - provisionedIndex = true; - } else { - provisionedIndex = await this.ensureVectorIndex(persisted, indexes, capabilities); - if (provisionedIndex) { + const provisionWarnings: string[] = []; + try { + if (existing && changed.length && requiresIndexRecreation(changed)) { + await this.recreateVectorIndex(existing, persisted, indexes); indexes = await this.deps.getVectorIndexes(persisted.schemaName); + provisionedIndex = true; + } else { + provisionedIndex = await this.ensureVectorIndex(persisted, indexes, capabilities); + if (provisionedIndex) { + indexes = await this.deps.getVectorIndexes(persisted.schemaName); + } } + } catch (err) { + persistEnabled = false; + provisionWarnings.push( + `Config was saved disabled because vector index provisioning failed for '${persisted.targetField}': ${sanitizeErrorMessage(err)}. Repair or create the index and enable the config once Database reports it queryable.`, + ); } const warnings = [ ...capabilityWarnings(capabilities), @@ -257,6 +267,7 @@ export class EmbeddingsApi { configDefaults.providers[persisted.provider] ?? configDefaults.providers[configDefaults.defaultProvider], ), + ...provisionWarnings, ]; if ( !findTargetVectorIndex(indexes, persisted.targetField) && @@ -266,8 +277,7 @@ export class EmbeddingsApi { `Vector index for field '${persisted.targetField}' was not provisioned automatically because Database indexing is unavailable. Create the index manually and wait until it is queryable before enabling this config.`, ); } - let persistEnabled = enabled; - if (enabled) { + if (enabled && persistEnabled) { try { assertConfigActivation({ moduleEnabled: configDefaults.enabled, @@ -436,7 +446,8 @@ export class EmbeddingsApi { capabilities, configs, indexes, - createRun: async run => this.deps.backfills.create(persistableBackfillRun(run)), + createRun: async run => + this.deps.backfills.create(persistableNewBackfillRun(run)), saveRun: async (id, run) => { await this.deps.backfills.findByIdAndUpdate(id, persistableBackfillRun(run)); }, @@ -854,7 +865,8 @@ export class EmbeddingsApi { capabilities: args.capabilities, configs: [config], indexes: args.indexes, - createRun: async run => this.deps.backfills.create(persistableBackfillRun(run)), + createRun: async run => + this.deps.backfills.create(persistableNewBackfillRun(run)), saveRun: async (id, run) => { await this.deps.backfills.findByIdAndUpdate(id, persistableBackfillRun(run)); }, diff --git a/modules/embeddings/src/utils/backfillExecution.test.ts b/modules/embeddings/src/utils/backfillExecution.test.ts index fe7b0a1ac..67576bd83 100644 --- a/modules/embeddings/src/utils/backfillExecution.test.ts +++ b/modules/embeddings/src/utils/backfillExecution.test.ts @@ -7,6 +7,7 @@ import { cancelBackfillExecution, parseBackfillControllerJob, persistableBackfillRun, + persistableNewBackfillRun, processBackfillControllerJob, queueBackfillRuns, resumeBackfillExecution, @@ -53,7 +54,16 @@ function memoryStore(initial: PersistedBackfillRun[] = []) { return run ? { ...run } : null; }, saveRun: async (id: string, run: BackfillRunProgress) => { - runs.set(id, { ...run, _id: id }); + const existing = runs.get(id); + if (!existing) { + runs.set(id, { + ...run, + ...persistableNewBackfillRun(run), + _id: id, + } as PersistedBackfillRun); + return; + } + Object.assign(existing, persistableBackfillRun(run)); }, incrementCounts: async ( id: string, @@ -386,6 +396,54 @@ describe('backfill cancellation, resume, and counters', () => { assert.equal(latest.failedCount, 8); }); + it('does not overwrite atomic processed/failed counts when a page save races with workers', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'running', + batchSize: 2, + queuedCount: 0, + }), + ); + const originalSave = store.saveRun; + store.saveRun = async (id, run) => { + await Promise.resolve(); + await originalSave(id, run); + }; + const page = processBackfillControllerJob( + { runId: created._id, cursor: null }, + deps({ + store, + findPage: async () => { + await Promise.resolve(); + return [{ _id: 'a' }, { _id: 'b' }]; + }, + }), + ); + const workers = Promise.all( + Array.from({ length: 10 }, (_, index) => + applyBackfillJobOutcome({ + runId: created._id, + outcome: index % 2 === 0 ? 'processed' : 'failed', + incrementCounts: async (id, patch) => { + await Promise.resolve(); + return store.incrementCounts(id, patch); + }, + }), + ), + ); + await Promise.all([page, workers]); + const latest = (await store.getRun(created._id))!; + assert.equal(latest.processedCount, 5); + assert.equal(latest.failedCount, 5); + assert.equal(latest.scannedCount, 2); + assert.equal(latest.queuedCount, 2); + assert.equal(latest.cursor, 'b'); + }); + it('fails the running run when the vector index is not queryable', async () => { const store = memoryStore(); const created = await store.createRun( @@ -432,6 +490,10 @@ describe('backfill controller job parsing and persistence mapping', () => { }); assert.equal(progress.cursor, null); assert.equal(persistableBackfillRun(progress).onlyMissing, true); + assert.equal('processedCount' in persistableBackfillRun(progress), false); + assert.equal('failedCount' in persistableBackfillRun(progress), false); + assert.equal(persistableNewBackfillRun(progress).processedCount, 0); + assert.equal(persistableNewBackfillRun(progress).failedCount, 0); }); }); diff --git a/modules/embeddings/src/utils/backfillExecution.ts b/modules/embeddings/src/utils/backfillExecution.ts index 9536849cb..f2b382fd9 100644 --- a/modules/embeddings/src/utils/backfillExecution.ts +++ b/modules/embeddings/src/utils/backfillExecution.ts @@ -137,8 +137,6 @@ export function persistableBackfillRun( filter: run.filter ?? undefined, scannedCount: run.scannedCount, queuedCount: run.queuedCount, - processedCount: run.processedCount, - failedCount: run.failedCount, startedAt: run.startedAt ?? undefined, finishedAt: run.finishedAt ?? undefined, drainStartedAt: run.drainStartedAt ?? undefined, @@ -146,6 +144,16 @@ export function persistableBackfillRun( }; } +export function persistableNewBackfillRun( + run: BackfillRunProgress, +): Record { + return { + ...persistableBackfillRun(run), + processedCount: run.processedCount, + failedCount: run.failedCount, + }; +} + export function parseBackfillControllerJob(value: unknown): ParsedBackfillControllerJob { if (!value || typeof value !== 'object' || Array.isArray(value)) { return { ok: false, reason: 'malformed' }; From b5fc10a16ce26f2cc3014a570ad575740b04dde0 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 22:03:54 +0300 Subject: [PATCH 19/29] fix(embeddings): close residual vector index mutation gaps Authorize ExtensionOnly deletes from the live index field, and provision a versioned replacement before retiring the previous index so pending builds cannot strand search. --- modules/database/src/Database.ts | 12 +- .../__tests__/vectorIndexLifecycle.test.ts | 23 +++ .../utils/__tests__/vectorSearchQuery.test.ts | 39 +++++ .../adapters/utils/vectorIndexLifecycle.ts | 25 +++ .../src/adapters/utils/vectorSearchQuery.ts | 9 +- .../__tests__/canModify.vectorIndex.test.ts | 67 +++++++- modules/database/src/permissions/index.ts | 9 ++ .../embeddings/src/api/embeddingsApi.test.ts | 151 +++++++++++++++++- modules/embeddings/src/api/embeddingsApi.ts | 96 +++++++++-- .../src/utils/backfillGates.test.ts | 15 ++ modules/embeddings/src/utils/backfillGates.ts | 7 +- .../embeddings/src/utils/configChange.test.ts | 35 ++++ modules/embeddings/src/utils/configChange.ts | 64 ++++++++ 13 files changed, 518 insertions(+), 34 deletions(-) diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index cefe805bc..a70a85ec5 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -57,6 +57,7 @@ import { canCreate, canDelete, canModify, + vectorIndexDeleteMutationData, vectorIndexMutationData, } from './permissions/index.js'; import { runMigrations } from './migrations/index.js'; @@ -1109,7 +1110,16 @@ export default class DatabaseModule extends ManagedModule { try { const moduleName = call.metadata!.get('module-name')![0] as string; const schemaAdapter = this._activeAdapter.getSchemaModel(call.request.schemaName); - if (!(await canModify(moduleName, schemaAdapter.model))) { + const liveIndexes = await this._activeAdapter.getVectorIndexes( + call.request.schemaName, + ); + if ( + !(await canModify( + moduleName, + schemaAdapter.model, + vectorIndexDeleteMutationData(liveIndexes, call.request.indexName), + )) + ) { return callback({ code: status.PERMISSION_DENIED, message: `Module ${moduleName} is not authorized to delete vector indexes for ${call.request.schemaName}!`, diff --git a/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts b/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts index d43a205e7..862fba56d 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts @@ -21,6 +21,7 @@ import { planPostgresVectorIndexCreate, postgresVectorIndexDefinitionMatches, renderPostgresCreateVectorIndexSql, + selectLiveVectorIndexForField, } from '../vectorIndexLifecycle.js'; const vectorField = { @@ -42,6 +43,28 @@ describe('vector index lifecycle', () => { expect(mongoVectorFilterFields()).toEqual(['_id']); }); + it('selects the highest generation live index for a field', () => { + expect( + selectLiveVectorIndexForField( + [ + { field: 'embedding', name: 'embedding_vector' }, + { field: 'embedding', name: 'embedding_vector_v2' }, + { field: 'title', name: 'title_vector_v4' }, + ], + 'embedding', + )?.name, + ).toBe('embedding_vector_v2'); + expect( + selectLiveVectorIndexForField( + [ + { field: 'embedding', name: 'cnd_Article_embedding_vector' }, + { field: 'embedding', name: 'embedding_vector' }, + ], + 'embedding', + )?.name, + ).toBe('embedding_vector'); + }); + it('binds declared indexes to field dimensions/similarity and rejects mismatches', () => { const bound = bindVectorIndexToField({ provider: 'mongodb', diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts index 2f3c9e9d9..4fccc30bb 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts @@ -67,6 +67,45 @@ describe('vector search query planning', () => { }); }); + it('selects the highest generation live index when a named request is not provided', () => { + const planned = planMongoVectorSearch({ + request, + indexes: [ + indexes[0], + { + ...indexes[0], + name: 'embedding_vector_v2', + }, + ], + schemaFields, + }); + expect(planned.pipeline[0]).toEqual({ + $vectorSearch: { + index: 'embedding_vector_v2', + path: 'embedding', + queryVector: request.vector, + numCandidates: 5, + limit: 5, + filter: { tenantId: 'org-1' }, + }, + }); + expect(() => + planMongoVectorSearch({ + request, + indexes: [ + indexes[0], + { + ...indexes[0], + name: 'embedding_vector_v2', + status: VectorIndexStatus.Pending, + queryable: false, + }, + ], + schemaFields, + }), + ).toThrow(/not queryable/); + }); + it('short-circuits empty Mongo $in without emitting a pipeline', () => { const planned = planMongoVectorSearch({ request: { ...request, filter: { tenantId: { $in: [] } } }, diff --git a/modules/database/src/adapters/utils/vectorIndexLifecycle.ts b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts index 211501a61..feaabb945 100644 --- a/modules/database/src/adapters/utils/vectorIndexLifecycle.ts +++ b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts @@ -40,6 +40,31 @@ export function defaultVectorIndexName( return physicalTableName ? `${physicalTableName}_${field}_vector` : `${field}_vector`; } +export function vectorIndexGeneration(name?: string): number { + if (typeof name !== 'string' || name.length === 0) return 0; + const match = /_v(\d+)$/.exec(name); + if (match) return Number(match[1]); + return 1; +} + +export function selectLiveVectorIndexForField< + T extends { field?: string; name?: string }, +>(indexes: readonly T[], field: string): T | undefined { + const matches = indexes.filter(index => index.field === field); + if (!matches.length) return undefined; + const defaultName = defaultVectorIndexName(field); + return matches.reduce((best, current) => { + const bestGeneration = vectorIndexGeneration(best.name); + const currentGeneration = vectorIndexGeneration(current.name); + if (currentGeneration !== bestGeneration) { + return currentGeneration > bestGeneration ? current : best; + } + if (current.name === defaultName) return current; + if (best.name === defaultName) return best; + return best; + }); +} + export function mongoVectorFilterFields(filterFields?: readonly string[]): string[] { const fields: string[] = []; for (const field of [MONGO_VECTOR_ID_FILTER_FIELD, ...(filterFields ?? [])]) { diff --git a/modules/database/src/adapters/utils/vectorSearchQuery.ts b/modules/database/src/adapters/utils/vectorSearchQuery.ts index 266f71752..cd9d3ed08 100644 --- a/modules/database/src/adapters/utils/vectorSearchQuery.ts +++ b/modules/database/src/adapters/utils/vectorSearchQuery.ts @@ -24,6 +24,7 @@ import { parseVectorSimilarity } from './vectorField.js'; import { assertVectorIndexQueryable, defaultVectorIndexName, + selectLiveVectorIndexForField, } from './vectorIndexLifecycle.js'; export interface PlannedMongoVectorSearch { @@ -62,13 +63,7 @@ export function findVectorIndexForSearch( if (request.indexName) { return indexes.find(item => item.name === request.indexName); } - return ( - indexes.find( - item => - item.field === request.field && - item.name === defaultVectorIndexName(request.field), - ) ?? indexes.find(item => item.field === request.field) - ); + return selectLiveVectorIndexForField(indexes, request.field); } export function mergeVectorIndexes( diff --git a/modules/database/src/permissions/__tests__/canModify.vectorIndex.test.ts b/modules/database/src/permissions/__tests__/canModify.vectorIndex.test.ts index 91275efc2..f2891b90d 100644 --- a/modules/database/src/permissions/__tests__/canModify.vectorIndex.test.ts +++ b/modules/database/src/permissions/__tests__/canModify.vectorIndex.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from '@jest/globals'; import { TYPE } from '@conduitplatform/grpc-sdk'; -import { canModify, vectorIndexMutationData } from '../index.js'; +import { + canModify, + vectorIndexDeleteMutationData, + vectorIndexMutationData, +} from '../index.js'; const embeddingExtension = { ownerModule: 'embeddings', @@ -70,3 +74,64 @@ describe('createVectorIndex canModify field evaluation', () => { ).resolves.toBe(false); }); }); + +describe('deleteVectorIndex canModify field evaluation', () => { + const liveIndexes = [ + { name: 'embedding_vector', field: 'embedding' }, + { name: 'email_1', field: 'email' }, + ]; + + it('allows embeddings to delete its own live extension index on ExtensionOnly schemas', async () => { + const user = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + await expect( + canModify( + 'embeddings', + user as never, + vectorIndexDeleteMutationData(liveIndexes, 'embedding_vector'), + ), + ).resolves.toBe(true); + }); + + it('denies unknown and non-vector index names on ExtensionOnly schemas', async () => { + const user = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + await expect( + canModify( + 'embeddings', + user as never, + vectorIndexDeleteMutationData(liveIndexes, 'missing_vector'), + ), + ).resolves.toBe(false); + await expect( + canModify( + 'embeddings', + user as never, + vectorIndexDeleteMutationData([], 'embedding_vector'), + ), + ).resolves.toBe(false); + await expect( + canModify( + 'embeddings', + user as never, + vectorIndexDeleteMutationData([{ name: 'title_idx' }], 'title_idx'), + ), + ).resolves.toBe(false); + }); + + it('denies unowned live vector indexes on ExtensionOnly schemas', async () => { + const user = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + await expect( + canModify( + 'embeddings', + user as never, + vectorIndexDeleteMutationData(liveIndexes, 'email_1'), + ), + ).resolves.toBe(false); + await expect( + canModify( + 'chat', + user as never, + vectorIndexDeleteMutationData(liveIndexes, 'embedding_vector'), + ), + ).resolves.toBe(false); + }); +}); diff --git a/modules/database/src/permissions/index.ts b/modules/database/src/permissions/index.ts index 0a205c16b..8e7058e6b 100644 --- a/modules/database/src/permissions/index.ts +++ b/modules/database/src/permissions/index.ts @@ -22,6 +22,15 @@ export function vectorIndexMutationData(field?: string): Indexable | undefined { return { [field]: true }; } +export function vectorIndexDeleteMutationData( + liveIndexes: ReadonlyArray<{ name?: string; field?: string }>, + indexName: string, +) { + if (typeof indexName !== 'string' || indexName.length === 0) return undefined; + const live = liveIndexes.find(index => index.name === indexName); + return vectorIndexMutationData(live?.field); +} + export async function canModify(moduleName: string, schema: Schema, data?: Indexable) { if (moduleName === 'database' && schema.originalSchema.name === '_DeclaredSchema') return true; diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts index c1d52fc1a..be9927acc 100644 --- a/modules/embeddings/src/api/embeddingsApi.test.ts +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -79,6 +79,7 @@ function createApi(overrides?: { embed?: EmbeddingsApiDeps['embed']; vectorSearch?: EmbeddingsApiDeps['vectorSearch']; createVectorIndex?: EmbeddingsApiDeps['createVectorIndex']; + createdIndexQueryable?: boolean; enqueue?: string[]; invalidated?: string[]; deletedIndexes?: string[]; @@ -88,6 +89,7 @@ function createApi(overrides?: { }) { const configs = [...(overrides?.configs ?? [])]; const runs = [...(overrides?.runs ?? [])]; + const indexes = [...(overrides?.indexes ?? [readyIndex])]; const enqueued = overrides?.enqueue ?? []; const invalidated = overrides?.invalidated ?? []; const deletedIndexes = overrides?.deletedIndexes ?? []; @@ -104,7 +106,7 @@ function createApi(overrides?: { overrides?.declared?.[name] ?? { name, ownerModule: 'database' }, setSchemaExtension: async () => undefined, getVectorCapabilities: async () => overrides?.capabilities ?? readyCapabilities, - getVectorIndexes: async () => overrides?.indexes ?? [readyIndex], + getVectorIndexes: async () => indexes, vectorSearch: overrides?.vectorSearch ?? (async () => []), configs: { findMany: async query => @@ -179,11 +181,25 @@ function createApi(overrides?: { createVectorIndex: overrides?.createVectorIndex ?? (async (_schema, index) => { - createdIndexes.push(index.name ?? index.field); + const name = index.name ?? `${index.field}_vector`; + createdIndexes.push(name); + if (!indexes.some(item => item.name === name)) { + indexes.push({ + field: index.field, + name, + queryable: overrides?.createdIndexQueryable === true, + status: + overrides?.createdIndexQueryable === true + ? VectorIndexStatus.Ready + : VectorIndexStatus.Pending, + }); + } return 'created'; }), deleteVectorIndex: async (_schema, indexName) => { deletedIndexes.push(indexName); + const index = indexes.findIndex(item => item.name === indexName); + if (index >= 0) indexes.splice(index, 1); return 'deleted'; }, invalidateHashes: async (schemaName, hashFields) => { @@ -201,6 +217,7 @@ function createApi(overrides?: { invalidated, deletedIndexes, createdIndexes, + indexes, }; } @@ -305,7 +322,7 @@ describe('typed embeddings API handlers', () => { ); assert.equal(saved.config.enabled, false); assert.equal(configs[0].enabled, false); - assert.deepEqual(deletedIndexes, ['embedding_vector']); + assert.deepEqual(deletedIndexes, []); assert.equal( saved.warnings.some( warning => @@ -676,6 +693,7 @@ describe('typed embeddings API handlers', () => { const { api, invalidated, deletedIndexes, createdIndexes, enqueued, runs } = createApi({ configs: [enabledConfig], + createdIndexQueryable: true, }); await assert.rejects( () => @@ -712,8 +730,8 @@ describe('typed embeddings API handlers', () => { ); assert.equal(updated.config.model, 'text-embedding-3-large'); assert.deepEqual(invalidated, ['Article.embeddingSourceHash']); + assert.deepEqual(createdIndexes, ['embedding_vector_v2']); assert.deepEqual(deletedIndexes, ['embedding_vector']); - assert.equal(createdIndexes.includes('embedding_vector'), true); assert.equal(enqueued.length, 1); assert.equal(runs[0].state, 'queued'); assert.equal(runs[0].onlyMissing, false); @@ -723,6 +741,131 @@ describe('typed embeddings API handlers', () => { ); }); + it('keeps the previous index when replacement provisioning fails', async () => { + const { api, configs, deletedIndexes, createdIndexes, indexes } = createApi({ + configs: [enabledConfig], + createVectorIndex: async () => { + throw new Error('atlas rejected replacement'); + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(configs[0].enabled, false); + assert.deepEqual(createdIndexes, []); + assert.deepEqual(deletedIndexes, []); + assert.equal( + indexes.some(index => index.name === 'embedding_vector' && index.queryable), + true, + ); + assert.equal( + saved.warnings.some( + warning => + /saved disabled because vector index provisioning failed/.test(warning) && + /atlas rejected replacement/.test(warning), + ), + true, + ); + }); + + it('keeps the previous index while a versioned replacement is not queryable', async () => { + const { api, configs, deletedIndexes, createdIndexes, indexes, enqueued } = createApi( + { + configs: [enabledConfig], + }, + ); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(configs[0].enabled, false); + assert.deepEqual(createdIndexes, ['embedding_vector_v2']); + assert.deepEqual(deletedIndexes, []); + assert.equal( + indexes.some(index => index.name === 'embedding_vector' && index.queryable), + true, + ); + assert.equal( + indexes.some( + index => index.name === 'embedding_vector_v2' && index.queryable !== true, + ), + true, + ); + assert.equal(enqueued.length, 0); + assert.equal( + saved.warnings.some(warning => + /saved disabled until the provisioned vector index/.test(warning), + ), + true, + ); + }); + + it('retires the previous index after a pending replacement becomes queryable', async () => { + const pendingReplacement = { + field: 'embedding', + name: 'embedding_vector_v2', + queryable: false, + status: VectorIndexStatus.Pending, + }; + const { api, configs, deletedIndexes, createdIndexes, indexes, enqueued } = createApi( + { + configs: [ + { ...enabledConfig, similarity: VectorSimilarity.Euclidean, enabled: false }, + ], + indexes: [readyIndex, pendingReplacement], + }, + ); + pendingReplacement.queryable = true; + pendingReplacement.status = VectorIndexStatus.Ready; + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, true); + assert.equal(configs[0].enabled, true); + assert.deepEqual(createdIndexes, []); + assert.deepEqual(deletedIndexes, ['embedding_vector']); + assert.equal( + indexes.some(index => index.name === 'embedding_vector'), + false, + ); + assert.equal( + indexes.some(index => index.name === 'embedding_vector_v2' && index.queryable), + true, + ); + assert.equal(enqueued.length, 0); + }); + it('is idempotent for start, cancel, and resume and does not duplicate active runs', async () => { const { api, runs, enqueued } = createApi({ configs: [enabledConfig] }); const first = await api.startBackfill( diff --git a/modules/embeddings/src/api/embeddingsApi.ts b/modules/embeddings/src/api/embeddingsApi.ts index 33159b008..40711b1bf 100644 --- a/modules/embeddings/src/api/embeddingsApi.ts +++ b/modules/embeddings/src/api/embeddingsApi.ts @@ -22,6 +22,7 @@ import { BackfillGateError, findTargetVectorIndex, grpcErrorFromBackfillGate, + isEmbeddingVectorIndexQueryable, } from '../utils/backfillGates.js'; import { defaultEmbeddingVectorIndexName, @@ -29,7 +30,9 @@ import { hashFieldsToInvalidate, isInPlaceDimensionChange, materialChangeWarnings, + nextEmbeddingVectorIndexName, requiresIndexRecreation, + sameEmbeddingVectorIndexFamily, } from '../utils/configChange.js'; import { MAX_QUEUE_BATCH_SIZE } from '../utils/embeddingJobs.js'; import { ACTIVE_BACKFILL_STATES } from '../utils/backfillRun.js'; @@ -233,10 +236,15 @@ export class EmbeddingsApi { } let persistEnabled = enabled; let provisionedIndex = false; + let replacementIndexName: string | undefined; const provisionWarnings: string[] = []; try { if (existing && changed.length && requiresIndexRecreation(changed)) { - await this.recreateVectorIndex(existing, persisted, indexes); + replacementIndexName = await this.recreateVectorIndex( + existing, + persisted, + indexes, + ); indexes = await this.deps.getVectorIndexes(persisted.schemaName); provisionedIndex = true; } else { @@ -245,6 +253,12 @@ export class EmbeddingsApi { indexes = await this.deps.getVectorIndexes(persisted.schemaName); } } + indexes = await this.retireSupersededVectorIndexes({ + schemaName: persisted.schemaName, + targetField: persisted.targetField, + previousField: existing?.targetField, + indexes, + }); } catch (err) { persistEnabled = false; provisionWarnings.push( @@ -279,6 +293,18 @@ export class EmbeddingsApi { } if (enabled && persistEnabled) { try { + if (replacementIndexName) { + const replacement = indexes.find(index => index.name === replacementIndexName); + if (!isEmbeddingVectorIndexQueryable(replacement)) { + throw new BackfillGateError( + 'index_not_queryable', + `Vector index '${replacementIndexName}' is not queryable (status: ${ + replacement?.status ?? 'missing' + }). Wait until the index is ready before enabling this config.`, + replacement?.status ?? 'missing', + ); + } + } assertConfigActivation({ moduleEnabled: configDefaults.enabled, capabilities, @@ -785,7 +811,7 @@ export class EmbeddingsApi { } private async recreateVectorIndex( - existing: EmbeddingConfigRecord, + _existing: EmbeddingConfigRecord, next: | EmbeddingConfigRecord | { @@ -800,31 +826,69 @@ export class EmbeddingsApi { queryable?: boolean; status?: string; }>, - ): Promise { - const current = findTargetVectorIndex(indexes, existing.targetField); - if (current?.name) { - try { - await this.deps.deleteVectorIndex(existing.schemaName, current.name); - } catch (err) { - throw new GrpcError( - status.FAILED_PRECONDITION, - `Failed to delete vector index '${current.name}': ${sanitizeErrorMessage(err)}`, - ); - } - } + ): Promise { + const replacementName = nextEmbeddingVectorIndexName(next.targetField, indexes); try { await this.deps.createVectorIndex(next.schemaName, { field: next.targetField, dimensions: next.dimensions, similarity: next.similarity as VectorIndexDefinition['similarity'], - name: defaultEmbeddingVectorIndexName(next.targetField), + name: replacementName, }); } catch (err) { throw new GrpcError( status.FAILED_PRECONDITION, - `Failed to recreate vector index for '${next.targetField}': ${sanitizeErrorMessage(err)}`, + `Failed to provision replacement vector index for '${next.targetField}': ${sanitizeErrorMessage(err)}`, ); } + return replacementName; + } + + private async retireSupersededVectorIndexes(args: { + schemaName: string; + targetField: string; + previousField?: string; + indexes: Array<{ + field?: string; + name?: string; + queryable?: boolean; + status?: string; + }>; + }): Promise< + Array<{ field?: string; name?: string; queryable?: boolean; status?: string }> + > { + const selected = findTargetVectorIndex(args.indexes, args.targetField); + if (!selected?.name || !isEmbeddingVectorIndexQueryable(selected)) { + return args.indexes; + } + const retireNames = new Set(); + for (const index of args.indexes) { + if (!index.name || index.name === selected.name) continue; + if ( + index.field === args.targetField && + sameEmbeddingVectorIndexFamily(index.name, selected.name) + ) { + retireNames.add(index.name); + } + } + if (args.previousField && args.previousField !== args.targetField) { + const previous = findTargetVectorIndex(args.indexes, args.previousField); + if (previous?.name && previous.name !== selected.name) { + retireNames.add(previous.name); + } + } + if (!retireNames.size) return args.indexes; + for (const name of retireNames) { + try { + await this.deps.deleteVectorIndex(args.schemaName, name); + } catch (err) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Failed to retire superseded vector index '${name}': ${sanitizeErrorMessage(err)}`, + ); + } + } + return this.deps.getVectorIndexes(args.schemaName); } private async supersedeActiveBackfills(configId: string): Promise { diff --git a/modules/embeddings/src/utils/backfillGates.test.ts b/modules/embeddings/src/utils/backfillGates.test.ts index f15631c6f..ca6b2e9e9 100644 --- a/modules/embeddings/src/utils/backfillGates.test.ts +++ b/modules/embeddings/src/utils/backfillGates.test.ts @@ -117,6 +117,21 @@ describe('backfill execution gates', () => { false, ); assert.deepEqual(findTargetVectorIndex([readyIndex], 'embedding'), readyIndex); + assert.equal( + findTargetVectorIndex( + [ + readyIndex, + { + field: 'embedding', + name: 'embedding_vector_v2', + status: VectorIndexStatus.Pending, + queryable: false, + }, + ], + 'embedding', + )?.name, + 'embedding_vector_v2', + ); assert.throws( () => assertBackfillExecutable({ diff --git a/modules/embeddings/src/utils/backfillGates.ts b/modules/embeddings/src/utils/backfillGates.ts index 09c43d336..260ba6236 100644 --- a/modules/embeddings/src/utils/backfillGates.ts +++ b/modules/embeddings/src/utils/backfillGates.ts @@ -4,6 +4,7 @@ import { VectorIndexStatus, } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; +import { selectEmbeddingVectorIndex } from './configChange.js'; export const BACKFILL_GATE_REASONS = [ 'module_disabled', @@ -64,11 +65,7 @@ export function findTargetVectorIndex( indexes: readonly VectorIndexGate[], targetField: string, ): VectorIndexGate | undefined { - return ( - indexes.find( - index => index.field === targetField && index.name === `${targetField}_vector`, - ) ?? indexes.find(index => index.field === targetField) - ); + return selectEmbeddingVectorIndex(indexes, targetField); } export function assertBackfillExecutable(args: { diff --git a/modules/embeddings/src/utils/configChange.test.ts b/modules/embeddings/src/utils/configChange.test.ts index 48f7504eb..b0f2a50d6 100644 --- a/modules/embeddings/src/utils/configChange.test.ts +++ b/modules/embeddings/src/utils/configChange.test.ts @@ -7,7 +7,10 @@ import { hashedEmbeddingSource, isInPlaceDimensionChange, materialChangeWarnings, + nextEmbeddingVectorIndexName, requiresIndexRecreation, + selectEmbeddingVectorIndex, + sameEmbeddingVectorIndexFamily, } from './configChange.js'; const base = { @@ -94,4 +97,36 @@ describe('material embedding config changes', () => { true, ); }); + + it('versions replacement names from live provider-specific indexes', () => { + assert.equal(nextEmbeddingVectorIndexName('embedding', []), 'embedding_vector'); + assert.equal( + nextEmbeddingVectorIndexName('embedding', [ + { field: 'embedding', name: 'embedding_vector' }, + ]), + 'embedding_vector_v2', + ); + assert.equal( + nextEmbeddingVectorIndexName('embedding', [ + { field: 'embedding', name: 'cnd_Article_embedding_vector' }, + { field: 'embedding', name: 'cnd_Article_embedding_vector_v2' }, + ]), + 'cnd_Article_embedding_vector_v3', + ); + assert.equal( + sameEmbeddingVectorIndexFamily('embedding_vector', 'embedding_vector_v2'), + true, + ); + assert.equal( + selectEmbeddingVectorIndex( + [ + { field: 'embedding', name: 'embedding_vector' }, + { field: 'embedding', name: 'embedding_vector_v2' }, + { field: 'title', name: 'title_vector_v9' }, + ], + 'embedding', + )?.name, + 'embedding_vector_v2', + ); + }); }); diff --git a/modules/embeddings/src/utils/configChange.ts b/modules/embeddings/src/utils/configChange.ts index 21d833a56..6235046ec 100644 --- a/modules/embeddings/src/utils/configChange.ts +++ b/modules/embeddings/src/utils/configChange.ts @@ -102,6 +102,70 @@ export function defaultEmbeddingVectorIndexName(field: string): string { return `${field}_vector`; } +export function parseEmbeddingVectorIndexName(name: string): { + base: string; + generation: number; +} { + const match = /^(.*)_v(\d+)$/.exec(name); + if (match) { + return { base: match[1], generation: Number(match[2]) }; + } + return { base: name, generation: 1 }; +} + +export function embeddingVectorIndexGeneration(name?: string): number { + if (typeof name !== 'string' || name.length === 0) return 0; + return parseEmbeddingVectorIndexName(name).generation; +} + +export function sameEmbeddingVectorIndexFamily(left: string, right: string): boolean { + return ( + parseEmbeddingVectorIndexName(left).base === parseEmbeddingVectorIndexName(right).base + ); +} + +export function nextEmbeddingVectorIndexName( + field: string, + indexes: ReadonlyArray<{ field?: string; name?: string }>, +): string { + const names = indexes + .filter( + index => + index.field === field && typeof index.name === 'string' && index.name.length > 0, + ) + .map(index => index.name as string); + if (!names.length) return defaultEmbeddingVectorIndexName(field); + let base = defaultEmbeddingVectorIndexName(field); + let maxGeneration = 0; + for (const name of names) { + const parsed = parseEmbeddingVectorIndexName(name); + if (parsed.generation >= maxGeneration) { + maxGeneration = parsed.generation; + base = parsed.base; + } + } + return `${base}_v${maxGeneration + 1}`; +} + +export function selectEmbeddingVectorIndex( + indexes: readonly T[], + field: string, +): T | undefined { + const matches = indexes.filter(index => index.field === field); + if (!matches.length) return undefined; + const defaultName = defaultEmbeddingVectorIndexName(field); + return matches.reduce((best, current) => { + const bestGeneration = embeddingVectorIndexGeneration(best.name); + const currentGeneration = embeddingVectorIndexGeneration(current.name); + if (currentGeneration !== bestGeneration) { + return currentGeneration > bestGeneration ? current : best; + } + if (current.name === defaultName) return current; + if (best.name === defaultName) return best; + return best; + }); +} + export function materialChangeWarnings( changed: readonly MaterialEmbeddingConfigField[], scheduledBackfill: boolean, From c14713104ac8263a2ee2d81eb0bc3cf3c558ac04 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 6 Sep 2026 22:19:31 +0300 Subject: [PATCH 20/29] fix(embeddings): require matching vector indexes on retry Treat live indexes as ready only when field, dimensions, similarity, and method match, so a failed recreate cannot enable or search the old index. --- .../embeddings/src/api/embeddingsApi.test.ts | 159 ++++++++++++++++++ modules/embeddings/src/api/embeddingsApi.ts | 98 +++++------ .../src/utils/backfillExecution.test.ts | 4 + .../src/utils/backfillGates.test.ts | 44 +++++ modules/embeddings/src/utils/backfillGates.ts | 36 +++- .../embeddings/src/utils/configChange.test.ts | 63 +++++++ modules/embeddings/src/utils/configChange.ts | 41 ++++- .../src/utils/operationalStatus.test.ts | 63 ++++++- .../embeddings/src/utils/operationalStatus.ts | 17 +- 9 files changed, 455 insertions(+), 70 deletions(-) diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts index be9927acc..93569b545 100644 --- a/modules/embeddings/src/api/embeddingsApi.test.ts +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -17,6 +17,7 @@ import { } from './embeddingsApi.js'; import type { Config } from '../config/index.js'; import type { QueueJobCounts } from '../controllers/queue.controller.js'; +import { SearchGateError } from '../utils/operationalStatus.js'; const articleSchema = { name: 'Article', @@ -37,6 +38,8 @@ const readyIndex = { name: 'embedding_vector', queryable: true, status: VectorIndexStatus.Ready, + dimensions: 3, + similarity: VectorSimilarity.Cosine, }; const moduleConfig = { @@ -73,6 +76,9 @@ function createApi(overrides?: { name?: string; queryable?: boolean; status?: string; + dimensions?: number; + similarity?: string; + method?: string; }>; schemas?: Record; declared?: Record; @@ -187,6 +193,9 @@ function createApi(overrides?: { indexes.push({ field: index.field, name, + dimensions: index.dimensions, + similarity: index.similarity, + method: index.method, queryable: overrides?.createdIndexQueryable === true, status: overrides?.createdIndexQueryable === true @@ -827,6 +836,8 @@ describe('typed embeddings API handlers', () => { name: 'embedding_vector_v2', queryable: false, status: VectorIndexStatus.Pending, + dimensions: 3, + similarity: VectorSimilarity.Euclidean, }; const { api, configs, deletedIndexes, createdIndexes, indexes, enqueued } = createApi( { @@ -866,6 +877,154 @@ describe('typed embeddings API handlers', () => { assert.equal(enqueued.length, 0); }); + it('retries a failed similarity recreation without enabling the mismatched live index', async () => { + let attempts = 0; + const createdIndexes: string[] = []; + const { api, configs, deletedIndexes, indexes } = createApi({ + configs: [enabledConfig], + createdIndexes, + createVectorIndex: async (_schema, index) => { + attempts += 1; + if (attempts === 1) { + throw new Error('atlas rejected replacement'); + } + const name = index.name ?? `${index.field}_vector`; + createdIndexes.push(name); + indexes.push({ + field: index.field, + name, + dimensions: index.dimensions, + similarity: index.similarity, + queryable: false, + status: VectorIndexStatus.Pending, + }); + return 'created'; + }, + }); + const euclideanUpsert = { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }; + const first = await api.upsertConfig(euclideanUpsert, { callerModule: 'database' }); + assert.equal(first.config.enabled, false); + assert.deepEqual(createdIndexes, []); + assert.deepEqual(deletedIndexes, []); + + const retry = await api.upsertConfig(euclideanUpsert, { callerModule: 'database' }); + assert.equal(retry.config.enabled, false); + assert.equal(configs[0].enabled, false); + assert.deepEqual(createdIndexes, ['embedding_vector_v2']); + assert.deepEqual(deletedIndexes, []); + assert.equal( + indexes.some(index => index.name === 'embedding_vector' && index.queryable), + true, + ); + assert.equal( + indexes.some( + index => + index.name === 'embedding_vector_v2' && + index.similarity === VectorSimilarity.Euclidean && + index.queryable !== true, + ), + true, + ); + await assert.rejects( + () => + api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1' }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.NOT_FOUND && + /No embedding config found/.test(err.message), + ); + }); + + it('denies activation when the live index contract does not match', async () => { + const { api, configs, deletedIndexes, createdIndexes, indexes } = createApi({ + configs: [ + { ...enabledConfig, similarity: VectorSimilarity.Euclidean, enabled: false }, + ], + capabilities: { + ...readyCapabilities, + indexing: false, + reason: 'indexing unavailable', + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(configs[0].enabled, false); + assert.deepEqual(createdIndexes, []); + assert.deepEqual(deletedIndexes, []); + assert.equal( + indexes.some( + index => + index.name === 'embedding_vector' && + index.queryable && + index.similarity === VectorSimilarity.Cosine, + ), + true, + ); + assert.equal( + saved.warnings.some(warning => /not queryable/.test(warning)), + true, + ); + await assert.rejects( + () => + api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1' }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + (err.code === status.NOT_FOUND || err.code === status.FAILED_PRECONDITION), + ); + }); + + it('does not search through a queryable index that does not match the config contract', async () => { + let searched = false; + const { api } = createApi({ + configs: [ + { ...enabledConfig, similarity: VectorSimilarity.Euclidean, enabled: true }, + ], + vectorSearch: async () => { + searched = true; + return []; + }, + }); + await assert.rejects( + () => + api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1' }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof SearchGateError && + err.reason === 'index_not_queryable' && + api.mapGrpcError(err).code === status.FAILED_PRECONDITION, + ); + assert.equal(searched, false); + }); + it('is idempotent for start, cancel, and resume and does not duplicate active runs', async () => { const { api, runs, enqueued } = createApi({ configs: [enabledConfig] }); const first = await api.startBackfill( diff --git a/modules/embeddings/src/api/embeddingsApi.ts b/modules/embeddings/src/api/embeddingsApi.ts index 40711b1bf..b4c90dd7f 100644 --- a/modules/embeddings/src/api/embeddingsApi.ts +++ b/modules/embeddings/src/api/embeddingsApi.ts @@ -20,9 +20,11 @@ import { } from '../utils/backfillExecution.js'; import { BackfillGateError, + embeddingIndexContractFromConfig, findTargetVectorIndex, grpcErrorFromBackfillGate, isEmbeddingVectorIndexQueryable, + type VectorIndexGate, } from '../utils/backfillGates.js'; import { defaultEmbeddingVectorIndexName, @@ -31,7 +33,6 @@ import { isInPlaceDimensionChange, materialChangeWarnings, nextEmbeddingVectorIndexName, - requiresIndexRecreation, sameEmbeddingVectorIndexFamily, } from '../utils/configChange.js'; import { MAX_QUEUE_BATCH_SIZE } from '../utils/embeddingJobs.js'; @@ -138,11 +139,7 @@ export interface EmbeddingsApiDeps { fields: ConduitModel; }) => Promise; getVectorCapabilities: (schemaName?: string) => Promise; - getVectorIndexes: ( - schemaName: string, - ) => Promise< - Array<{ field?: string; name?: string; queryable?: boolean; status?: string }> - >; + getVectorIndexes: (schemaName: string) => Promise; vectorSearch: (input: { schemaName: string; field: string; @@ -238,16 +235,19 @@ export class EmbeddingsApi { let provisionedIndex = false; let replacementIndexName: string | undefined; const provisionWarnings: string[] = []; + const indexContract = embeddingIndexContractFromConfig(persisted); try { - if (existing && changed.length && requiresIndexRecreation(changed)) { - replacementIndexName = await this.recreateVectorIndex( - existing, - persisted, - indexes, - ); + const matchingIndex = findTargetVectorIndex( + indexes, + persisted.targetField, + indexContract, + ); + const hasFieldIndex = indexes.some(index => index.field === persisted.targetField); + if (!matchingIndex && hasFieldIndex && capabilities.indexing) { + replacementIndexName = await this.recreateVectorIndex(persisted, indexes); indexes = await this.deps.getVectorIndexes(persisted.schemaName); provisionedIndex = true; - } else { + } else if (!matchingIndex) { provisionedIndex = await this.ensureVectorIndex(persisted, indexes, capabilities); if (provisionedIndex) { indexes = await this.deps.getVectorIndexes(persisted.schemaName); @@ -256,6 +256,8 @@ export class EmbeddingsApi { indexes = await this.retireSupersededVectorIndexes({ schemaName: persisted.schemaName, targetField: persisted.targetField, + dimensions: persisted.dimensions, + similarity: persisted.similarity, previousField: existing?.targetField, indexes, }); @@ -273,6 +275,8 @@ export class EmbeddingsApi { targetField: persisted.targetField, enabled, schemaName: persisted.schemaName, + dimensions: persisted.dimensions, + similarity: persisted.similarity, }, ], indexes, @@ -284,7 +288,7 @@ export class EmbeddingsApi { ...provisionWarnings, ]; if ( - !findTargetVectorIndex(indexes, persisted.targetField) && + !findTargetVectorIndex(indexes, persisted.targetField, indexContract) && !capabilities.indexing ) { warnings.push( @@ -312,6 +316,8 @@ export class EmbeddingsApi { enabled, schemaName: persisted.schemaName, targetField: persisted.targetField, + dimensions: persisted.dimensions, + similarity: persisted.similarity, }, indexes, }); @@ -784,15 +790,18 @@ export class EmbeddingsApi { dimensions: number; similarity: string; }, - indexes: Array<{ - field?: string; - name?: string; - queryable?: boolean; - status?: string; - }>, + indexes: VectorIndexGate[], capabilities: VectorCapabilities, ): Promise { - if (findTargetVectorIndex(indexes, next.targetField)) return false; + if ( + findTargetVectorIndex( + indexes, + next.targetField, + embeddingIndexContractFromConfig(next), + ) + ) { + return false; + } if (!capabilities.indexing) return false; try { await this.deps.createVectorIndex(next.schemaName, { @@ -811,21 +820,13 @@ export class EmbeddingsApi { } private async recreateVectorIndex( - _existing: EmbeddingConfigRecord, - next: - | EmbeddingConfigRecord - | { - schemaName: string; - targetField: string; - dimensions: number; - similarity: string; - }, - indexes: Array<{ - field?: string; - name?: string; - queryable?: boolean; - status?: string; - }>, + next: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + }, + indexes: VectorIndexGate[], ): Promise { const replacementName = nextEmbeddingVectorIndexName(next.targetField, indexes); try { @@ -847,17 +848,15 @@ export class EmbeddingsApi { private async retireSupersededVectorIndexes(args: { schemaName: string; targetField: string; + dimensions: number; + similarity: string; previousField?: string; - indexes: Array<{ - field?: string; - name?: string; - queryable?: boolean; - status?: string; - }>; - }): Promise< - Array<{ field?: string; name?: string; queryable?: boolean; status?: string }> - > { - const selected = findTargetVectorIndex(args.indexes, args.targetField); + indexes: VectorIndexGate[]; + }): Promise { + const selected = findTargetVectorIndex(args.indexes, args.targetField, { + dimensions: args.dimensions, + similarity: args.similarity, + }); if (!selected?.name || !isEmbeddingVectorIndexQueryable(selected)) { return args.indexes; } @@ -907,12 +906,7 @@ export class EmbeddingsApi { config: EmbeddingConfigRecord, args: { capabilities: VectorCapabilities; - indexes: Array<{ - field?: string; - name?: string; - queryable?: boolean; - status?: string; - }>; + indexes: VectorIndexGate[]; }, ): Promise { try { diff --git a/modules/embeddings/src/utils/backfillExecution.test.ts b/modules/embeddings/src/utils/backfillExecution.test.ts index 67576bd83..778208ae2 100644 --- a/modules/embeddings/src/utils/backfillExecution.test.ts +++ b/modules/embeddings/src/utils/backfillExecution.test.ts @@ -30,12 +30,16 @@ const config = { enabled: true, schemaName: 'Article', targetField: 'embedding', + dimensions: 3, + similarity: 'cosine', }; const readyIndex = { field: 'embedding', name: 'embedding_vector', status: VectorIndexStatus.Ready, queryable: true, + dimensions: 3, + similarity: 'cosine', }; function memoryStore(initial: PersistedBackfillRun[] = []) { diff --git a/modules/embeddings/src/utils/backfillGates.test.ts b/modules/embeddings/src/utils/backfillGates.test.ts index ca6b2e9e9..1a3cc0093 100644 --- a/modules/embeddings/src/utils/backfillGates.test.ts +++ b/modules/embeddings/src/utils/backfillGates.test.ts @@ -20,6 +20,8 @@ const config = { enabled: true, schemaName: 'Article', targetField: 'embedding', + dimensions: 3, + similarity: 'cosine', }; const readyIndex = { @@ -27,6 +29,8 @@ const readyIndex = { name: 'embedding_vector', status: VectorIndexStatus.Ready, queryable: true, + dimensions: 3, + similarity: 'cosine', }; describe('backfill execution gates', () => { @@ -132,6 +136,44 @@ describe('backfill execution gates', () => { )?.name, 'embedding_vector_v2', ); + assert.equal( + findTargetVectorIndex([readyIndex], 'embedding', { + dimensions: 3, + similarity: 'euclidean', + }), + undefined, + ); + assert.equal( + findTargetVectorIndex( + [ + readyIndex, + { + field: 'embedding', + name: 'embedding_vector_v2', + status: VectorIndexStatus.Pending, + queryable: false, + dimensions: 3, + similarity: 'euclidean', + }, + ], + 'embedding', + { dimensions: 3, similarity: 'euclidean' }, + )?.name, + 'embedding_vector_v2', + ); + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities, + config: { ...config, similarity: 'euclidean' }, + indexes: [readyIndex], + }), + (err: unknown) => + err instanceof BackfillGateError && + err.reason === 'index_not_queryable' && + err.indexStatus === 'missing', + ); assert.throws( () => assertBackfillExecutable({ @@ -144,6 +186,8 @@ describe('backfill execution gates', () => { name: 'embedding_vector', status: VectorIndexStatus.Pending, queryable: false, + dimensions: 3, + similarity: 'cosine', }, ], }), diff --git a/modules/embeddings/src/utils/backfillGates.ts b/modules/embeddings/src/utils/backfillGates.ts index 260ba6236..d565fd92a 100644 --- a/modules/embeddings/src/utils/backfillGates.ts +++ b/modules/embeddings/src/utils/backfillGates.ts @@ -4,7 +4,11 @@ import { VectorIndexStatus, } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; -import { selectEmbeddingVectorIndex } from './configChange.js'; +import { + selectEmbeddingVectorIndex, + type EmbeddingVectorIndexContract, + type EmbeddingVectorIndexShape, +} from './configChange.js'; export const BACKFILL_GATE_REASONS = [ 'module_disabled', @@ -35,15 +39,29 @@ export interface BackfillConfigGate { enabled?: boolean; schemaName?: string; targetField?: string; + dimensions?: number; + similarity?: string; + method?: string; } -export interface VectorIndexGate { - field?: string; - name?: string; +export interface VectorIndexGate extends EmbeddingVectorIndexShape { queryable?: boolean; status?: string; } +export type EmbeddingIndexContractInput = Omit; + +export function embeddingIndexContractFromConfig( + config: BackfillConfigGate, +): EmbeddingIndexContractInput | undefined { + if (typeof config.dimensions !== 'number') return undefined; + return { + dimensions: config.dimensions, + similarity: config.similarity, + method: config.method, + }; +} + export function isEmbeddingVectorIndexQueryable(index?: VectorIndexGate): boolean { if (!index) return false; if (index.queryable === false) return false; @@ -64,8 +82,9 @@ export function isEmbeddingVectorIndexQueryable(index?: VectorIndexGate): boolea export function findTargetVectorIndex( indexes: readonly VectorIndexGate[], targetField: string, + contract?: EmbeddingIndexContractInput, ): VectorIndexGate | undefined { - return selectEmbeddingVectorIndex(indexes, targetField); + return selectEmbeddingVectorIndex(indexes, targetField, contract); } export function assertBackfillExecutable(args: { @@ -117,9 +136,10 @@ export function assertBackfillExecutable(args: { 'Embedding config is missing a target vector field', ); } - const index = findTargetVectorIndex(args.indexes ?? [], targetField); - if (isEmbeddingVectorIndexQueryable(index)) return; - const indexStatus = index?.status ?? 'missing'; + const contract = embeddingIndexContractFromConfig(args.config); + const index = findTargetVectorIndex(args.indexes ?? [], targetField, contract); + if (contract && isEmbeddingVectorIndexQueryable(index)) return; + const indexStatus = contract ? (index?.status ?? 'missing') : 'missing'; throw new BackfillGateError( 'index_not_queryable', `Vector index for field '${targetField}' is not queryable (status: ${indexStatus}). ` + diff --git a/modules/embeddings/src/utils/configChange.test.ts b/modules/embeddings/src/utils/configChange.test.ts index b0f2a50d6..aeb43a9bd 100644 --- a/modules/embeddings/src/utils/configChange.test.ts +++ b/modules/embeddings/src/utils/configChange.test.ts @@ -11,6 +11,7 @@ import { requiresIndexRecreation, selectEmbeddingVectorIndex, sameEmbeddingVectorIndexFamily, + embeddingVectorIndexMatchesContract, } from './configChange.js'; const base = { @@ -129,4 +130,66 @@ describe('material embedding config changes', () => { 'embedding_vector_v2', ); }); + + it('matches live indexes only when field, dimensions, similarity, and method agree', () => { + const cosine = { + field: 'embedding', + name: 'embedding_vector', + dimensions: 3, + similarity: 'cosine', + method: 'hnsw', + }; + const euclidean = { + ...cosine, + name: 'embedding_vector_v2', + similarity: 'euclidean', + }; + const ivf = { ...cosine, method: 'ivfflat' }; + assert.equal( + embeddingVectorIndexMatchesContract(cosine, { + field: 'embedding', + dimensions: 3, + similarity: 'cosine', + }), + true, + ); + assert.equal( + embeddingVectorIndexMatchesContract(cosine, { + field: 'embedding', + dimensions: 3, + similarity: 'euclidean', + }), + false, + ); + assert.equal( + embeddingVectorIndexMatchesContract(cosine, { + field: 'embedding', + dimensions: 8, + similarity: 'cosine', + }), + false, + ); + assert.equal( + embeddingVectorIndexMatchesContract(ivf, { + field: 'embedding', + dimensions: 3, + similarity: 'cosine', + }), + false, + ); + assert.equal( + selectEmbeddingVectorIndex([cosine, euclidean], 'embedding', { + dimensions: 3, + similarity: 'euclidean', + })?.name, + 'embedding_vector_v2', + ); + assert.equal( + selectEmbeddingVectorIndex([cosine], 'embedding', { + dimensions: 3, + similarity: 'euclidean', + }), + undefined, + ); + }); }); diff --git a/modules/embeddings/src/utils/configChange.ts b/modules/embeddings/src/utils/configChange.ts index 6235046ec..0155e4f27 100644 --- a/modules/embeddings/src/utils/configChange.ts +++ b/modules/embeddings/src/utils/configChange.ts @@ -147,11 +147,48 @@ export function nextEmbeddingVectorIndexName( return `${base}_v${maxGeneration + 1}`; } -export function selectEmbeddingVectorIndex( +export interface EmbeddingVectorIndexContract { + field: string; + dimensions: number; + similarity?: string; + method?: string; +} + +export interface EmbeddingVectorIndexShape { + field?: string; + name?: string; + dimensions?: number; + similarity?: string; + method?: string; +} + +const DEFAULT_EMBEDDING_VECTOR_INDEX_METHOD = 'hnsw'; + +export function embeddingVectorIndexMatchesContract( + index: EmbeddingVectorIndexShape, + contract: EmbeddingVectorIndexContract, +): boolean { + if (index.field !== contract.field) return false; + if (typeof index.dimensions !== 'number' || index.dimensions !== contract.dimensions) { + return false; + } + if ((index.similarity ?? '') !== (contract.similarity ?? '')) return false; + return ( + (index.method ?? DEFAULT_EMBEDDING_VECTOR_INDEX_METHOD) === + (contract.method ?? DEFAULT_EMBEDDING_VECTOR_INDEX_METHOD) + ); +} + +export function selectEmbeddingVectorIndex( indexes: readonly T[], field: string, + contract?: Omit, ): T | undefined { - const matches = indexes.filter(index => index.field === field); + const matches = indexes.filter(index => { + if (index.field !== field) return false; + if (!contract) return true; + return embeddingVectorIndexMatchesContract(index, { field, ...contract }); + }); if (!matches.length) return undefined; const defaultName = defaultEmbeddingVectorIndexName(field); return matches.reduce((best, current) => { diff --git a/modules/embeddings/src/utils/operationalStatus.test.ts b/modules/embeddings/src/utils/operationalStatus.test.ts index 2804f8f2f..b47755def 100644 --- a/modules/embeddings/src/utils/operationalStatus.test.ts +++ b/modules/embeddings/src/utils/operationalStatus.test.ts @@ -1,8 +1,9 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { VectorIndexStatus } from '@conduitplatform/grpc-sdk'; +import { GrpcError, VectorIndexStatus } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { + assertConfigActivation, assertSearchExecutable, capabilityWarnings, grpcErrorFromSearchGate, @@ -56,12 +57,20 @@ describe('embeddings operational warnings and search gates', () => { search: true, provider: 'mongodb', }, - config: { _id: 'cfg1', enabled: true, targetField: 'embedding' }, + config: { + _id: 'cfg1', + enabled: true, + targetField: 'embedding', + dimensions: 3, + similarity: 'cosine', + }, indexes: [ { field: 'embedding', status: VectorIndexStatus.Pending, queryable: false, + dimensions: 3, + similarity: 'cosine', }, ], }), @@ -73,6 +82,56 @@ describe('embeddings operational warnings and search gates', () => { ); assert.equal(mapped.code, status.FAILED_PRECONDITION); }); + + it('denies search and activation when a queryable live index does not match the config contract', () => { + const mismatched = { + field: 'embedding', + name: 'embedding_vector', + status: VectorIndexStatus.Ready, + queryable: true, + dimensions: 3, + similarity: 'cosine', + }; + const config = { + _id: 'cfg1', + enabled: true, + schemaName: 'Article', + targetField: 'embedding', + dimensions: 3, + similarity: 'euclidean', + }; + assert.throws( + () => + assertSearchExecutable({ + capabilities: { + supported: true, + search: true, + provider: 'mongodb', + }, + config, + indexes: [mismatched], + }), + (err: unknown) => + err instanceof SearchGateError && err.reason === 'index_not_queryable', + ); + assert.throws( + () => + assertConfigActivation({ + moduleEnabled: true, + capabilities: { + supported: true, + storage: true, + provider: 'mongodb', + }, + config, + indexes: [mismatched], + }), + (err: unknown) => + err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not queryable/.test(err.message), + ); + }); }); describe('typed proto mappers', () => { diff --git a/modules/embeddings/src/utils/operationalStatus.ts b/modules/embeddings/src/utils/operationalStatus.ts index d30fa39bf..6c6c1f031 100644 --- a/modules/embeddings/src/utils/operationalStatus.ts +++ b/modules/embeddings/src/utils/operationalStatus.ts @@ -3,6 +3,7 @@ import { status } from '@grpc/grpc-js'; import { assertBackfillExecutable, BackfillGateError, + embeddingIndexContractFromConfig, findTargetVectorIndex, isEmbeddingVectorIndexQueryable, type BackfillConfigGate, @@ -72,9 +73,10 @@ export function assertSearchExecutable(args: { 'Embedding config is missing a target vector field', ); } - const index = findTargetVectorIndex(args.indexes ?? [], targetField); - if (isEmbeddingVectorIndexQueryable(index)) return; - const indexStatus = index?.status ?? 'missing'; + const contract = embeddingIndexContractFromConfig(args.config); + const index = findTargetVectorIndex(args.indexes ?? [], targetField, contract); + if (contract && isEmbeddingVectorIndexQueryable(index)) return; + const indexStatus = contract ? (index?.status ?? 'missing') : 'missing'; throw new SearchGateError( 'index_not_queryable', `Vector index for field '${targetField}' is not queryable (status: ${indexStatus}). ` + @@ -144,10 +146,13 @@ export function indexReadinessWarnings( for (const config of configs) { const targetField = config.targetField; if (!targetField) continue; - const index = findTargetVectorIndex(indexes, targetField); - if (isEmbeddingVectorIndexQueryable(index)) continue; + const contract = embeddingIndexContractFromConfig(config); + const index = findTargetVectorIndex(indexes, targetField, contract); + if (contract && isEmbeddingVectorIndexQueryable(index)) continue; warnings.push( - `Vector index for field '${targetField}' is not queryable (status: ${index?.status ?? 'missing'})`, + `Vector index for field '${targetField}' is not queryable (status: ${ + contract ? (index?.status ?? 'missing') : 'missing' + })`, ); } return warnings; From 38ca59e6b19e3475bd9610b21d7db1cf3befcb13 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Tue, 8 Sep 2026 15:34:49 +0300 Subject: [PATCH 21/29] fix(ci): restore database bundle smoke and embeddings target discovery Include pgvector in the database bundle install, emit docker target JSON even when GitHub sets GITHUB_OUTPUT, and clear branch-local CodeFactor unused and complexity findings. --- .github/workflows/embeddings-test.yml | 2 +- modules/database/package.bundle-lock.json | 212 ++++++--- modules/database/package.bundle.json | 7 +- modules/database/service-bundle.config.json | 1 + .../database/src/adapters/DatabaseAdapter.ts | 17 +- .../utils/__tests__/mutationEvents.test.ts | 1 - .../utils/validateFieldConstraints.ts | 155 +++---- .../src/adapters/utils/vectorSearchWhere.ts | 42 +- modules/embeddings/Dockerfile | 1 + modules/embeddings/src/api/embeddingsApi.ts | 423 ++++++++++++------ modules/embeddings/src/config/index.ts | 4 +- .../src/controllers/queue.controller.ts | 9 - modules/embeddings/src/providers/index.ts | 84 ++-- .../src/utils/backfillExecution.test.ts | 1 - .../embeddings/src/utils/backfillExecution.ts | 70 ++- modules/embeddings/src/utils/backfillRun.ts | 28 +- modules/embeddings/src/utils/embeddingJobs.ts | 31 +- .../test/deployment-contract.test.mjs | 2 +- scripts/resolve-docker-targets.mjs | 17 +- 19 files changed, 685 insertions(+), 422 deletions(-) diff --git a/.github/workflows/embeddings-test.yml b/.github/workflows/embeddings-test.yml index 598d5539c..bea15e318 100644 --- a/.github/workflows/embeddings-test.yml +++ b/.github/workflows/embeddings-test.yml @@ -119,7 +119,7 @@ jobs: env: CHANGED_FILES: modules/embeddings/src/index.ts run: | - node scripts/resolve-docker-targets.mjs > /tmp/targets.json + env -u GITHUB_OUTPUT node scripts/resolve-docker-targets.mjs > /tmp/targets.json node -e ' const fs = require("fs"); const data = JSON.parse(fs.readFileSync("/tmp/targets.json", "utf8")); diff --git a/modules/database/package.bundle-lock.json b/modules/database/package.bundle-lock.json index 3b64c95e0..85d67f30a 100644 --- a/modules/database/package.bundle-lock.json +++ b/modules/database/package.bundle-lock.json @@ -28,21 +28,22 @@ "mariadb": "^3.5.3", "mongodb": "^7.3.0", "mongodb-schema": "^12.7.0", - "mongoose": "^9.9.3", - "mysql2": "^3.22.5", + "mongoose": "^9.9.4", + "mysql2": "^3.23.1", "nice-grpc": "^2.1.17", "nice-grpc-client-middleware-retry": "^3.1.16", "nice-grpc-common": "^2.0.4", "object-hash": "^3.0.0", "pg": "^8.22.0", "pg-hstore": "^2.3.4", + "pgvector": "^0.3.0", "prom-client": "^15.1.3", "protobufjs": "^8.7.2", "sequelize": "^6.37.8", "sequelize-auto": "^0.8.8", "snappy": "7.4.1", "sqlite3": "^6.0.1", - "uuid": "14.0.1", + "uuid": "14.0.2", "winston": "^3.19.0", "winston-loki": "^6.1.7" }, @@ -51,24 +52,24 @@ } }, "node_modules/@bufbuild/protobuf": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.14.0.tgz", - "integrity": "sha512-C3UGsiCwSprE2NKIIFA3hCDlpXTMCAXRZuEVp88L1GY36Y41+rYL5fryE+nOFhp4p4JPQvdV8PQ4DWgHgeTE+w==", + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.14.1.tgz", + "integrity": "sha512-agRJn3+EJDUe8AvxTx/LnHA/GErvLE62pSaSk7+MwFOtOv8eWBu/qCq2qoZjBjVZ3C2aiFJCveuSs17KMkYGOw==", "license": "(Apache-2.0 AND BSD-3-Clause)" }, "node_modules/@colors/colors": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", - "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.1.tgz", + "integrity": "sha512-dTmUJzXSuayBK+hZydEaXd2mhx61qWQwkwaBBY6LyEOVx/L9aQU5ac8eFNEsd9nrD1+zb9zvDCphLSe8g1F4Qw==", "license": "MIT", "engines": { "node": ">=0.1.90" } }, "node_modules/@dabh/diagnostics": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.8.tgz", - "integrity": "sha512-R4MSXTVnuMzGD7bzHdW2ZhhdPC/igELENcq5IjEverBvq5hn1SXCWcsi6eSsdWP0/Ur+SItRRjAktmdoX/8R/Q==", + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.9.tgz", + "integrity": "sha512-R6siwR65Hm+3yfgP7o8DKhNvputQAwfoz9zTc3kyDudnomj2/BcLmD+uGQQPICjuFUp8ounPBU+jmKsocwVVAg==", "license": "MIT", "dependencies": { "@so-ric/colorspace": "^1.1.6", @@ -108,9 +109,9 @@ } }, "node_modules/@grpc/proto-loader/node_modules/protobufjs": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", - "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -168,9 +169,9 @@ } }, "node_modules/@mongodb-js/saslprep": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.5.0.tgz", - "integrity": "sha512-Hk1SKJCMcCos38+vqDnZzlIo4XRj9yCGzYkjB4LcqpeXRIYfia1UWTz+VrueLxoU+uSRJzgkufxoRZg8gi52YA==", + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.5.2.tgz", + "integrity": "sha512-UBvCBdPHAmiiDNEpD2ORBGzna4qYr06fLELfGDPW3/KZ9uLK+cGT9npAc/x/6/8SLzdbILMuc3HWFQ0FvJMhCw==", "license": "MIT", "dependencies": { "sparse-bitfield": "^3.0.3" @@ -642,12 +643,12 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "26.2.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz", - "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==", + "version": "26.5.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.5.0.tgz", + "integrity": "sha512-dVSGpriSoCgz8WnDNTuSSuSv1PC/ALXihO4ulRZt7Md8k9mlbdin3lGOcDE8SnWOgf513ByWlXd7BK4azmyg/A==", "license": "MIT", "dependencies": { - "undici-types": "~8.3.0" + "undici-types": "~8.9.0" } }, "node_modules/@types/triple-beam": { @@ -956,9 +957,9 @@ } }, "node_modules/bullmq": { - "version": "5.81.3", - "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.81.3.tgz", - "integrity": "sha512-Q7uEH2G92rVjX3Yl2qw3+6VEO15uxehC6DfeTHQBQTehddcRDSDuquD9zuvDjxiCffXsPUaBwrRlvlbVcgKs7g==", + "version": "5.81.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.81.4.tgz", + "integrity": "sha512-n+WHSzz20KooBGoJyASre6oJNz/p5f1IJRRN2ibD+NWPQaNpNQmDrwYuPO+bsXrQeM1MQzUxXGbdjmyOFKP2xQ==", "license": "MIT", "dependencies": { "cron-parser": "4.9.0", @@ -1499,9 +1500,9 @@ } }, "node_modules/fast-jwt": { - "version": "6.3.2", - "resolved": "https://registry.npmjs.org/fast-jwt/-/fast-jwt-6.3.2.tgz", - "integrity": "sha512-JTQImpkXVvj+eq7tJImtsHRt1K6ngloEzIx62Qbf9x4tEM2P2EqGpYJSeUoPf/kMn78rImSHfhf08KShR8PauA==", + "version": "6.3.3", + "resolved": "https://registry.npmjs.org/fast-jwt/-/fast-jwt-6.3.3.tgz", + "integrity": "sha512-pQDXx7IHeZT4jSmpE9o80RrBqfrG4fPrl8anazSM5vErIdK1iCc13z/EWX+H0j7liWSRnwTpHswIKMeLYGAckw==", "license": "Apache-2.0", "dependencies": { "@lukeed/ms": "^2.0.2", @@ -1965,9 +1966,9 @@ "optional": true }, "node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "funding": [ { "type": "github", @@ -2055,6 +2056,15 @@ "node": ">= 12.0.0" } }, + "node_modules/logform/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "license": "MIT", + "engines": { + "node": ">=0.1.90" + } + }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -2071,9 +2081,9 @@ } }, "node_modules/lru.min": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.4.tgz", - "integrity": "sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.5.tgz", + "integrity": "sha512-5J9ysMYUpYIg9RF2vJpy9SinEmSviFSe0GyPpCQ4L5QSkLAgeLXlTAOu2ZwWUU5m+0SBl6gUU1R1ZQB3aKypfA==", "license": "MIT", "engines": { "bun": ">=1.0.0", @@ -2095,9 +2105,9 @@ } }, "node_modules/mariadb": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/mariadb/-/mariadb-3.5.3.tgz", - "integrity": "sha512-i053Kc0MgdUv/hu9mCyq67TYfPXFj3/MV8I7ZW5wvJNixIyXC0VztMPUjIVj/449nQo+BsxFD4Fdk/sA/uqKPQ==", + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/mariadb/-/mariadb-3.5.4.tgz", + "integrity": "sha512-3m0vdfgRkjle5/9hHl4R7rlVvnDA7QfuFSNkdIAQNROWKOS0J3b9IQK9HaZSCGEVKiDUbFXFiINfC+cmR5P/aA==", "license": "LGPL-2.1-or-later", "dependencies": { "@types/geojson": "^7946.0.16", @@ -2272,9 +2282,9 @@ } }, "node_modules/mongodb": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.5.0.tgz", - "integrity": "sha512-5FnrEDLnvp6ycUOGLNLLU33BfCx2qmp2mJjGPDwKLruYsVzXVSK5fsGpoDXvsXJwBfBsD7ebMRdawbDxC2814g==", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.6.0.tgz", + "integrity": "sha512-WbZ6OCjYw2c53LOjfkQa+reXr7kIiOVpXXglnASFuiMtif0BvsMwHe3ClJHLm1/r7wJFwaasfFtD6iYIktB01g==", "license": "Apache-2.0", "dependencies": { "@mongodb-js/saslprep": "^1.4.11", @@ -2331,9 +2341,9 @@ } }, "node_modules/mongodb-ns": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/mongodb-ns/-/mongodb-ns-3.2.1.tgz", - "integrity": "sha512-PhuRl7oAzHbILnn+DoJkHnfduh7VRKMYnkzOv/x32GC8YvnhYXr42rhzxIK/UQvyZcnG968yVvCvjF2xSUP1OQ==", + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/mongodb-ns/-/mongodb-ns-3.2.3.tgz", + "integrity": "sha512-+ECvmAqkRkzbDTC+75EmMYjRS1lVObUX7qtytRwR5xDBoBvC1XBmSnBFgZS/l1piMYKd3bHazd58L8rVjQLnAg==", "license": "Apache-2.0", "optional": true }, @@ -2362,9 +2372,9 @@ } }, "node_modules/mongoose": { - "version": "9.9.3", - "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-9.9.3.tgz", - "integrity": "sha512-9dQaKct05LNgVkunDzFPZsGGBhinobl3Qc5B5KYJkLNG5lBLgqESUEItl1EUIkVaCwZJGBgTuvJiFAjjIATCiw==", + "version": "9.9.5", + "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-9.9.5.tgz", + "integrity": "sha512-t/kUoeDjlHmav7yCaq//YKU3wiIgUPaTj4GrluiTHUw4jQ77/CXtdMDSor4f1u3SKLhIK6NgCF+AzzUqU02Aag==", "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", @@ -2383,6 +2393,52 @@ "url": "https://opencollective.com/mongoose" } }, + "node_modules/mongoose/node_modules/mongodb": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.5.0.tgz", + "integrity": "sha512-5FnrEDLnvp6ycUOGLNLLU33BfCx2qmp2mJjGPDwKLruYsVzXVSK5fsGpoDXvsXJwBfBsD7ebMRdawbDxC2814g==", + "license": "Apache-2.0", + "dependencies": { + "@mongodb-js/saslprep": "^1.4.11", + "bson": "^7.2.0", + "mongodb-connection-string-url": "^7.0.1" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.806.0", + "@mongodb-js/zstd": "^7.0.0", + "gcp-metadata": "^7.0.1", + "kerberos": "^7.0.0", + "mongodb-client-encryption": "^7.2.0", + "snappy": "^7.3.2", + "socks": "^2.8.6" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, "node_modules/mpath": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.9.0.tgz", @@ -2439,9 +2495,9 @@ } }, "node_modules/mysql2": { - "version": "3.24.2", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.24.2.tgz", - "integrity": "sha512-l9kXeKGwd6VCbSjmpO/bLWb+YCYpbvE4whte8ec6InXebvCNyEEydyRCnRU+TSHNqRLJ8B+Tk1uWb+vMCJgJzA==", + "version": "3.24.4", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.24.4.tgz", + "integrity": "sha512-A2olluVlj0mvgyIRRISMEzXc51m+21mRtcMVjJyIpt2GG98+XrC9m9HzsqcMsX2LcnfccJvY5NB22g8fENBnOA==", "license": "MIT", "dependencies": { "aws-ssl-profiles": "^1.1.2", @@ -2537,9 +2593,9 @@ } }, "node_modules/node-abi": { - "version": "3.94.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.94.0.tgz", - "integrity": "sha512-W5ZNO5KRPB5TkYmGVD9F6YqhsglXJzE6etpbmT+f6EQElhiX/UTG551cnsRGvLG3fyZEg9HwaDmNmj5nwJ4z9g==", + "version": "3.96.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.96.0.tgz", + "integrity": "sha512-rebQ/lz7i0EkoLzUVSrKRzA69zMkwLp95kKMWoMDkkM00Suxz0D7zEQPwRml5fQum24mj7bPvmlgLAmu2JCiYg==", "license": "MIT", "dependencies": { "semver": "^7.3.5" @@ -2806,6 +2862,15 @@ "split2": "^4.1.0" } }, + "node_modules/pgvector": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/pgvector/-/pgvector-0.3.0.tgz", + "integrity": "sha512-+t7qcQD2us8fO8YIq/3lA0gUrD+bVO70MG1MhcDcxJz/OlRGGIIHzFq/4x57Vn/LpzX5wFdfOTLQp9QMPd4ljQ==", + "license": "MIT", + "engines": { + "node": ">=22" + } + }, "node_modules/picomatch": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", @@ -2909,6 +2974,7 @@ "version": "15.1.3", "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "deprecated": "prom-client has been replaced by @prometheus-io/client", "license": "Apache-2.0", "dependencies": { "@opentelemetry/api": "^1.4.0", @@ -2919,9 +2985,9 @@ } }, "node_modules/protobufjs": { - "version": "8.7.2", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.7.2.tgz", - "integrity": "sha512-oTVHV+oelUBtiu5iTuTNNZ0eLYsXSMxry4cgr30mayNkgIZL6qZ0IOQVPuSWGcyAaXKl/XgqwWHIC3a0khYVBA==", + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.8.0.tgz", + "integrity": "sha512-N3xhQ5yyBx3vQq4gubBfASzYhJGNzeDbjqBpu61g7UVylsN/qyffU96TKWD3GbbLOKF82VGNRNvv1+BFgE31Eg==", "license": "BSD-3-Clause", "dependencies": { "long": "^5.3.2" @@ -2972,9 +3038,9 @@ } }, "node_modules/qs": { - "version": "6.15.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", - "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", "license": "BSD-3-Clause", "dependencies": { "es-define-property": "^1.0.1", @@ -3715,9 +3781,9 @@ } }, "node_modules/tdigest": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", - "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.3.tgz", + "integrity": "sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==", "license": "MIT", "dependencies": { "bintrees": "1.0.2" @@ -3844,9 +3910,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "6.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.0.tgz", - "integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==", + "version": "6.28.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.1.tgz", + "integrity": "sha512-zWpdTVD54H48CIybL0rWQ3ukpb9d23wM7eH5RtfdmeP70cWHNjtfo7P4vZX+5CoDcO53J4Pu5uXp7lNfjc6DRA==", "license": "MIT", "optional": true, "engines": { @@ -3854,9 +3920,9 @@ } }, "node_modules/undici-types": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", - "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", "license": "MIT" }, "node_modules/universalify": { @@ -3890,9 +3956,9 @@ "license": "MIT" }, "node_modules/uuid": { - "version": "14.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.1.tgz", - "integrity": "sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==", + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.2.tgz", + "integrity": "sha512-xZe/16rV4aa+HGSOCiY2YeLT1OybRLrrkL/Rqaq7p7GMVXjFh+6wN4oMYgjFmnSnhY8t6Xpdl2l9qmnHYuMHwQ==", "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" @@ -3997,9 +4063,9 @@ } }, "node_modules/winston-loki/node_modules/protobufjs": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", - "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { diff --git a/modules/database/package.bundle.json b/modules/database/package.bundle.json index 4ee835133..11646ec2c 100644 --- a/modules/database/package.bundle.json +++ b/modules/database/package.bundle.json @@ -38,7 +38,7 @@ "prom-client": "^15.1.3", "protobufjs": "^8.7.2", "snappy": "7.4.1", - "uuid": "14.0.1", + "uuid": "14.0.2", "winston": "^3.19.0", "winston-loki": "^6.1.7", "bullmq": "^5.79.0", @@ -47,11 +47,12 @@ "mariadb": "^3.5.3", "mongodb": "^7.3.0", "mongodb-schema": "^12.7.0", - "mongoose": "^9.9.3", - "mysql2": "^3.22.5", + "mongoose": "^9.9.4", + "mysql2": "^3.23.1", "object-hash": "^3.0.0", "pg": "^8.22.0", "pg-hstore": "^2.3.4", + "pgvector": "^0.3.0", "sequelize": "^6.37.8", "sequelize-auto": "^0.8.8", "sqlite3": "^6.0.1" diff --git a/modules/database/service-bundle.config.json b/modules/database/service-bundle.config.json index 55a39a87b..4cb26a636 100644 --- a/modules/database/service-bundle.config.json +++ b/modules/database/service-bundle.config.json @@ -17,6 +17,7 @@ "object-hash", "pg", "pg-hstore", + "pgvector", "sequelize", "sequelize-auto", "sqlite3" diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index d8c93bda5..70b08fafa 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -302,22 +302,28 @@ export abstract class DatabaseAdapter { rawQuery: RawMongoQuery | RawSQLQuery, ): Promise; - getVectorCapabilities(_schemaName?: string): Promise { + getVectorCapabilities(schemaName?: string): Promise { + void schemaName; return Promise.resolve(unsupportedVectorCapabilities(this.getDatabaseType())); } - createVectorIndex(_schemaName: string, _index: VectorIndexDefinition): Promise { + createVectorIndex(schemaName: string, index: VectorIndexDefinition): Promise { + void schemaName; + void index; throw new GrpcError( status.UNIMPLEMENTED, `${this.getDatabaseType()} does not support vector indexes`, ); } - getVectorIndexes(_schemaName: string): Promise { + getVectorIndexes(schemaName: string): Promise { + void schemaName; return Promise.resolve([]); } - deleteVectorIndex(_schemaName: string, _indexName: string): Promise { + deleteVectorIndex(schemaName: string, indexName: string): Promise { + void schemaName; + void indexName; throw new GrpcError( status.UNIMPLEMENTED, `${this.getDatabaseType()} does not support vector indexes`, @@ -338,7 +344,8 @@ export abstract class DatabaseAdapter { } } - vectorSearch(_request: VectorSearchInput): Promise { + vectorSearch(request: VectorSearchInput): Promise { + void request; throw new GrpcError( status.UNIMPLEMENTED, `${this.getDatabaseType()} does not support vector search`, diff --git a/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts b/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts index 81a6a5c53..5bdf522d7 100644 --- a/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts +++ b/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts @@ -1,5 +1,4 @@ import { describe, expect, it } from '@jest/globals'; -import { GrpcError } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { buildMutationEventChunks, diff --git a/modules/database/src/adapters/utils/validateFieldConstraints.ts b/modules/database/src/adapters/utils/validateFieldConstraints.ts index a13fa6d1f..855ab5760 100644 --- a/modules/database/src/adapters/utils/validateFieldConstraints.ts +++ b/modules/database/src/adapters/utils/validateFieldConstraints.ts @@ -11,6 +11,77 @@ export function validateFieldConstraints(schema: ConduitDatabaseSchema, db: stri fieldsValidator(schema.name, schema.compiledFields, db); } +function invalidField(message: string): never { + throw new ConduitError('INVALID_ARGUMENTS', 400, message); +} + +function usesSqlRelationBlock(item: unknown): boolean { + return Boolean( + (item && + typeof item === 'object' && + (item as ConduitModelField).hasOwnProperty('type') && + (item as ConduitModelField).type !== 'Relation') || + (item && typeof item === 'object'), + ); +} + +function validateArrayContents( + schemaName: string, + field: string, + items: unknown[], + db: string, + blockRelations: boolean, +) { + if (items.length !== 1) { + invalidField( + `Schema '${schemaName}' array field '${field}' has invalid format (array should contain a single type).`, + ); + } + const nestedBlock = usesSqlRelationBlock(items[0]) ? db === 'sql' : blockRelations; + fieldsValidator(schemaName, items[0] as ConduitModel, db, nestedBlock); +} + +function validateObjectField( + schemaName: string, + field: string, + target: ConduitModelField, + db: string, + blockRelations: boolean, +) { + if (target.unique && !target.required) { + invalidField( + `Schema '${schemaName}' violates unique field '${field}' constraint (field should be 'required').`, + ); + } + if (target.hasOwnProperty('type') && typeof target.type === 'object') { + if (Array.isArray(target.type)) { + validateArrayContents( + schemaName, + field, + target.type as unknown[], + db, + blockRelations, + ); + return; + } + fieldsValidator(schemaName, target.type as ConduitModel, db, blockRelations); + return; + } + if (!target.hasOwnProperty('type') && isObject(target)) { + if (Array.isArray(target)) { + validateArrayContents(schemaName, field, target as unknown[], db, blockRelations); + return; + } + fieldsValidator(schemaName, target as ConduitModel, db, blockRelations); + return; + } + if (target.hasOwnProperty('type') && target.type === 'Relation' && blockRelations) { + invalidField( + `Schema '${schemaName}' violates field '${field}' constraint (relations not allowed in embedded objects).`, + ); + } +} + export function fieldsValidator( schemaName: string, schemaFields: ConduitModel, @@ -19,87 +90,19 @@ export function fieldsValidator( ) { Object.keys(schemaFields).forEach(f => { if (f.includes('.')) { - throw new ConduitError( - 'INVALID_ARGUMENTS', - 400, + invalidField( `Schema '${schemaName}' violates field '${f}' constraint (field names cannot contain '.').`, ); } assertVectorFieldIfPresent(schemaName, f, schemaFields[f]); if (typeof schemaFields[f] === 'object') { - const target: ConduitModelField = schemaFields[f] as ConduitModelField; - const isUnique = !!target.unique; - const isRequired = !!target.required; - if (isUnique && !isRequired) { - throw new ConduitError( - 'INVALID_ARGUMENTS', - 400, - `Schema '${schemaName}' violates unique field '${f}' constraint (field should be 'required').`, - ); - } - - if (target.hasOwnProperty('type') && typeof target.type === 'object') { - if (Array.isArray(target.type)) { - if ((target.type as unknown[]).length !== 1) { - throw new ConduitError( - 'INVALID_ARGUMENTS', - 400, - `Schema '${schemaName}' array field '${f}' has invalid format (array should contain a single type).`, - ); - } - if ( - (target.type[0] && - typeof target.type[0] === 'object' && - target.type[0].hasOwnProperty('type') && - target.type[0].type !== 'Relation') || - (target.type[0] && typeof target.type[0] === 'object') - ) { - fieldsValidator(schemaName, target.type[0] as ConduitModel, db, db === 'sql'); - } else { - fieldsValidator( - schemaName, - target.type[0] as unknown as ConduitModel, - db, - blockRelations, - ); - } - } else { - fieldsValidator(schemaName, target.type as ConduitModel, db, blockRelations); - } - } else if (!target.hasOwnProperty('type') && isObject(target)) { - if (Array.isArray(target)) { - if ((target as unknown[]).length !== 1) { - throw new ConduitError( - 'INVALID_ARGUMENTS', - 400, - `Schema '${schemaName}' array field '${f}' has invalid format (array should contain a single type).`, - ); - } - if ( - (target[0] && - typeof target[0] === 'object' && - target[0].hasOwnProperty('type') && - target[0].type !== 'Relation') || - (target[0] && typeof target[0] === 'object') - ) { - fieldsValidator(schemaName, target[0] as ConduitModel, db, db === 'sql'); - } else { - fieldsValidator(schemaName, target[0] as ConduitModel, db, blockRelations); - } - } else { - fieldsValidator(schemaName, target as ConduitModel, db, blockRelations); - } - } else if ( - target.hasOwnProperty('type') && - target.type === 'Relation' && - blockRelations - ) { - throw new ConduitError( - 'INVALID_ARGUMENTS', - 400, - `Schema '${schemaName}' violates field '${f}' constraint (relations not allowed in embedded objects).`, - ); - } + validateObjectField( + schemaName, + f, + schemaFields[f] as ConduitModelField, + db, + blockRelations, + ); } }); } diff --git a/modules/database/src/adapters/utils/vectorSearchWhere.ts b/modules/database/src/adapters/utils/vectorSearchWhere.ts index e3761a93b..655020d5e 100644 --- a/modules/database/src/adapters/utils/vectorSearchWhere.ts +++ b/modules/database/src/adapters/utils/vectorSearchWhere.ts @@ -40,6 +40,26 @@ function renderComparison( } } +function renderInPredicate( + fieldSql: string, + operand: unknown, + renderer: PostgresWhereRenderer, +): string { + const values = operand as unknown[]; + if (!values.length) return 'FALSE'; + return `${fieldSql} IN (${values.map(item => renderer.escape(item)).join(', ')})`; +} + +function renderNinPredicate( + fieldSql: string, + operand: unknown, + renderer: PostgresWhereRenderer, +): string | undefined { + const values = operand as unknown[]; + if (!values.length) return undefined; + return `${fieldSql} NOT IN (${values.map(item => renderer.escape(item)).join(', ')})`; +} + function renderFieldPredicate( field: string, value: unknown, @@ -61,28 +81,18 @@ function renderFieldPredicate( const clauses: string[] = []; for (const [operator, operand] of Object.entries(value as Record)) { if (operator === '$in') { - const values = operand as unknown[]; - if (!values.length) { - return 'FALSE'; - } - clauses.push( - `${fieldSql} IN (${values.map(item => renderer.escape(item)).join(', ')})`, - ); + const rendered = renderInPredicate(fieldSql, operand, renderer); + if (rendered === 'FALSE') return 'FALSE'; + clauses.push(rendered); continue; } if (operator === '$nin') { - const values = operand as unknown[]; - if (!values.length) { - continue; - } - clauses.push( - `${fieldSql} NOT IN (${values.map(item => renderer.escape(item)).join(', ')})`, - ); + const rendered = renderNinPredicate(fieldSql, operand, renderer); + if (rendered) clauses.push(rendered); continue; } if (operator === '$not') { - const nested = renderFieldPredicate(field, operand, renderer); - clauses.push(`NOT (${nested})`); + clauses.push(`NOT (${renderFieldPredicate(field, operand, renderer)})`); continue; } clauses.push(renderComparison(fieldSql, operator, operand, renderer)); diff --git a/modules/embeddings/Dockerfile b/modules/embeddings/Dockerfile index 3d7742e9f..c0bfd3014 100644 --- a/modules/embeddings/Dockerfile +++ b/modules/embeddings/Dockerfile @@ -1,3 +1,4 @@ +# hadolint ignore=DL3006 FROM conduit-builder WORKDIR /app/modules/embeddings diff --git a/modules/embeddings/src/api/embeddingsApi.ts b/modules/embeddings/src/api/embeddingsApi.ts index b4c90dd7f..a94234314 100644 --- a/modules/embeddings/src/api/embeddingsApi.ts +++ b/modules/embeddings/src/api/embeddingsApi.ts @@ -3,6 +3,7 @@ import { TYPE, VectorCapabilities, VectorSearchResult, + VectorSimilarity, type ConduitModel, type VectorIndexDefinition, } from '@conduitplatform/grpc-sdk'; @@ -34,6 +35,7 @@ import { materialChangeWarnings, nextEmbeddingVectorIndexName, sameEmbeddingVectorIndexFamily, + type MaterialEmbeddingConfigField, } from '../utils/configChange.js'; import { MAX_QUEUE_BATCH_SIZE } from '../utils/embeddingJobs.js'; import { ACTIVE_BACKFILL_STATES } from '../utils/backfillRun.js'; @@ -214,152 +216,46 @@ export class EmbeddingsApi { ); } if (capabilities.storage) { - await this.deps.setSchemaExtension({ - schemaName: persisted.schemaName, - fields: { - [persisted.targetField]: { - type: TYPE.Vector, - dimensions: persisted.dimensions, - similarity: persisted.similarity, - select: false, - }, - [`${persisted.targetField}SourceHash`]: { - type: TYPE.String, - required: false, - select: false, - }, - }, - }); + await this.extendEmbeddingSchema(persisted); } - let persistEnabled = enabled; - let provisionedIndex = false; - let replacementIndexName: string | undefined; - const provisionWarnings: string[] = []; - const indexContract = embeddingIndexContractFromConfig(persisted); - try { - const matchingIndex = findTargetVectorIndex( - indexes, - persisted.targetField, - indexContract, - ); - const hasFieldIndex = indexes.some(index => index.field === persisted.targetField); - if (!matchingIndex && hasFieldIndex && capabilities.indexing) { - replacementIndexName = await this.recreateVectorIndex(persisted, indexes); - indexes = await this.deps.getVectorIndexes(persisted.schemaName); - provisionedIndex = true; - } else if (!matchingIndex) { - provisionedIndex = await this.ensureVectorIndex(persisted, indexes, capabilities); - if (provisionedIndex) { - indexes = await this.deps.getVectorIndexes(persisted.schemaName); - } - } - indexes = await this.retireSupersededVectorIndexes({ - schemaName: persisted.schemaName, - targetField: persisted.targetField, - dimensions: persisted.dimensions, - similarity: persisted.similarity, - previousField: existing?.targetField, + const provisioned = await this.provisionUpsertIndexes({ + persisted, + existing, + capabilities, + indexes, + }); + indexes = provisioned.indexes; + let persistEnabled = enabled && provisioned.persistEnabled; + const warnings = this.upsertConfigWarnings({ + capabilities, + indexes, + persisted, + enabled, + configDefaults, + provisionWarnings: provisioned.provisionWarnings, + }); + if (enabled && persistEnabled) { + const activation = this.deferEnablementIfIndexPending({ + replacementIndexName: provisioned.replacementIndexName, + provisionedIndex: provisioned.provisionedIndex, + enabled, + capabilities, + persisted, indexes, + moduleEnabled: configDefaults.enabled, }); - } catch (err) { - persistEnabled = false; - provisionWarnings.push( - `Config was saved disabled because vector index provisioning failed for '${persisted.targetField}': ${sanitizeErrorMessage(err)}. Repair or create the index and enable the config once Database reports it queryable.`, - ); - } - const warnings = [ - ...capabilityWarnings(capabilities), - ...indexReadinessWarnings( - [ - { - targetField: persisted.targetField, - enabled, - schemaName: persisted.schemaName, - dimensions: persisted.dimensions, - similarity: persisted.similarity, - }, - ], - indexes, - ), - ...providerReadinessWarnings( - configDefaults.providers[persisted.provider] ?? - configDefaults.providers[configDefaults.defaultProvider], - ), - ...provisionWarnings, - ]; - if ( - !findTargetVectorIndex(indexes, persisted.targetField, indexContract) && - !capabilities.indexing - ) { - warnings.push( - `Vector index for field '${persisted.targetField}' was not provisioned automatically because Database indexing is unavailable. Create the index manually and wait until it is queryable before enabling this config.`, - ); - } - if (enabled && persistEnabled) { - try { - if (replacementIndexName) { - const replacement = indexes.find(index => index.name === replacementIndexName); - if (!isEmbeddingVectorIndexQueryable(replacement)) { - throw new BackfillGateError( - 'index_not_queryable', - `Vector index '${replacementIndexName}' is not queryable (status: ${ - replacement?.status ?? 'missing' - }). Wait until the index is ready before enabling this config.`, - replacement?.status ?? 'missing', - ); - } - } - assertConfigActivation({ - moduleEnabled: configDefaults.enabled, - capabilities, - config: { - enabled, - schemaName: persisted.schemaName, - targetField: persisted.targetField, - dimensions: persisted.dimensions, - similarity: persisted.similarity, - }, - indexes, - }); - } catch (err) { - const indexPending = - (err instanceof BackfillGateError && err.reason === 'index_not_queryable') || - (err instanceof GrpcError && - err.code === status.FAILED_PRECONDITION && - /not queryable/.test(err.message)); - if (!indexPending) throw err; - persistEnabled = false; - warnings.push( - provisionedIndex - ? 'Config was saved disabled until the provisioned vector index is queryable. Enable it once Database reports the index ready.' - : 'Config was saved disabled until the vector index is queryable. Enable it once Database reports the index ready.', - ); - } - } - const saved = existing - ? await this.deps.configs.findByIdAndUpdate(existing._id, { - ...persisted, - enabled: persistEnabled, - }) - : await this.deps.configs.create({ ...persisted, enabled: persistEnabled }); - if (!saved) { - throw new GrpcError(status.INTERNAL, 'Failed to persist embedding config'); - } - if (existing && changed.length) { - await this.deps.invalidateHashes( - saved.schemaName, - hashFieldsToInvalidate(existing, saved), - ); - await this.supersedeActiveBackfills(saved._id); - let scheduledBackfill = false; - if (persistEnabled) { - scheduledBackfill = await this.scheduleExplicitBackfill(saved, { - capabilities, - indexes, - }); - } - warnings.push(...materialChangeWarnings(changed, scheduledBackfill)); + persistEnabled = activation.persistEnabled; + warnings.push(...activation.warnings); } + const saved = await this.saveUpsertedConfig({ + existing, + persisted, + persistEnabled, + changed, + capabilities, + indexes, + warnings, + }); await this.deps.onConfigChanged?.(saved.schemaName); return { config: mapEmbeddingConfig(saved), warnings }; } @@ -774,6 +670,249 @@ export class EmbeddingsApi { return config; } + private async extendEmbeddingSchema(persisted: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: VectorSimilarity; + }) { + await this.deps.setSchemaExtension({ + schemaName: persisted.schemaName, + fields: { + [persisted.targetField]: { + type: TYPE.Vector, + dimensions: persisted.dimensions, + similarity: persisted.similarity, + select: false, + }, + [`${persisted.targetField}SourceHash`]: { + type: TYPE.String, + required: false, + select: false, + }, + }, + }); + } + + private async provisionUpsertIndexes(args: { + persisted: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + }; + existing: EmbeddingConfigRecord | null; + capabilities: VectorCapabilities; + indexes: VectorIndexGate[]; + }): Promise<{ + indexes: VectorIndexGate[]; + persistEnabled: boolean; + provisionedIndex: boolean; + replacementIndexName?: string; + provisionWarnings: string[]; + }> { + let { indexes } = args; + let persistEnabled = true; + let provisionedIndex = false; + let replacementIndexName: string | undefined; + const provisionWarnings: string[] = []; + try { + const matchingIndex = findTargetVectorIndex( + indexes, + args.persisted.targetField, + embeddingIndexContractFromConfig(args.persisted), + ); + const hasFieldIndex = indexes.some( + index => index.field === args.persisted.targetField, + ); + if (!matchingIndex && hasFieldIndex && args.capabilities.indexing) { + replacementIndexName = await this.recreateVectorIndex(args.persisted, indexes); + indexes = await this.deps.getVectorIndexes(args.persisted.schemaName); + provisionedIndex = true; + } else if (!matchingIndex) { + provisionedIndex = await this.ensureVectorIndex( + args.persisted, + indexes, + args.capabilities, + ); + if (provisionedIndex) { + indexes = await this.deps.getVectorIndexes(args.persisted.schemaName); + } + } + indexes = await this.retireSupersededVectorIndexes({ + schemaName: args.persisted.schemaName, + targetField: args.persisted.targetField, + dimensions: args.persisted.dimensions, + similarity: args.persisted.similarity, + previousField: args.existing?.targetField, + indexes, + }); + } catch (err) { + persistEnabled = false; + provisionWarnings.push( + `Config was saved disabled because vector index provisioning failed for '${args.persisted.targetField}': ${sanitizeErrorMessage(err)}. Repair or create the index and enable the config once Database reports it queryable.`, + ); + } + return { + indexes, + persistEnabled, + provisionedIndex, + replacementIndexName, + provisionWarnings, + }; + } + + private upsertConfigWarnings(args: { + capabilities: VectorCapabilities; + indexes: VectorIndexGate[]; + persisted: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + provider: string; + }; + enabled: boolean; + configDefaults: Config; + provisionWarnings: string[]; + }): string[] { + const warnings = [ + ...capabilityWarnings(args.capabilities), + ...indexReadinessWarnings( + [ + { + targetField: args.persisted.targetField, + enabled: args.enabled, + schemaName: args.persisted.schemaName, + dimensions: args.persisted.dimensions, + similarity: args.persisted.similarity, + }, + ], + args.indexes, + ), + ...providerReadinessWarnings( + args.configDefaults.providers[args.persisted.provider] ?? + args.configDefaults.providers[args.configDefaults.defaultProvider], + ), + ...args.provisionWarnings, + ]; + if ( + !findTargetVectorIndex( + args.indexes, + args.persisted.targetField, + embeddingIndexContractFromConfig(args.persisted), + ) && + !args.capabilities.indexing + ) { + warnings.push( + `Vector index for field '${args.persisted.targetField}' was not provisioned automatically because Database indexing is unavailable. Create the index manually and wait until it is queryable before enabling this config.`, + ); + } + return warnings; + } + + private deferEnablementIfIndexPending(args: { + replacementIndexName?: string; + provisionedIndex: boolean; + enabled: boolean; + capabilities: VectorCapabilities; + persisted: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + }; + indexes: VectorIndexGate[]; + moduleEnabled: boolean; + }): { persistEnabled: boolean; warnings: string[] } { + try { + if (args.replacementIndexName) { + const replacement = args.indexes.find( + index => index.name === args.replacementIndexName, + ); + if (!isEmbeddingVectorIndexQueryable(replacement)) { + throw new BackfillGateError( + 'index_not_queryable', + `Vector index '${args.replacementIndexName}' is not queryable (status: ${ + replacement?.status ?? 'missing' + }). Wait until the index is ready before enabling this config.`, + replacement?.status ?? 'missing', + ); + } + } + assertConfigActivation({ + moduleEnabled: args.moduleEnabled, + capabilities: args.capabilities, + config: { + enabled: args.enabled, + schemaName: args.persisted.schemaName, + targetField: args.persisted.targetField, + dimensions: args.persisted.dimensions, + similarity: args.persisted.similarity, + }, + indexes: args.indexes, + }); + return { persistEnabled: true, warnings: [] }; + } catch (err) { + const indexPending = + (err instanceof BackfillGateError && err.reason === 'index_not_queryable') || + (err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not queryable/.test(err.message)); + if (!indexPending) throw err; + return { + persistEnabled: false, + warnings: [ + args.provisionedIndex + ? 'Config was saved disabled until the provisioned vector index is queryable. Enable it once Database reports the index ready.' + : 'Config was saved disabled until the vector index is queryable. Enable it once Database reports the index ready.', + ], + }; + } + } + + private async saveUpsertedConfig(args: { + existing: EmbeddingConfigRecord | null; + persisted: Record & { + schemaName: string; + targetField: string; + }; + persistEnabled: boolean; + changed: MaterialEmbeddingConfigField[]; + capabilities: VectorCapabilities; + indexes: VectorIndexGate[]; + warnings: string[]; + }): Promise { + const saved = args.existing + ? await this.deps.configs.findByIdAndUpdate(args.existing._id, { + ...args.persisted, + enabled: args.persistEnabled, + }) + : await this.deps.configs.create({ + ...args.persisted, + enabled: args.persistEnabled, + }); + if (!saved) { + throw new GrpcError(status.INTERNAL, 'Failed to persist embedding config'); + } + if (args.existing && args.changed.length) { + await this.deps.invalidateHashes( + saved.schemaName, + hashFieldsToInvalidate(args.existing, saved), + ); + await this.supersedeActiveBackfills(saved._id); + let scheduledBackfill = false; + if (args.persistEnabled) { + scheduledBackfill = await this.scheduleExplicitBackfill(saved, { + capabilities: args.capabilities, + indexes: args.indexes, + }); + } + args.warnings.push(...materialChangeWarnings(args.changed, scheduledBackfill)); + } + return saved; + } + private async findActiveBackfills(configId: string): Promise { const runs: PersistedBackfillRun[] = []; for (const state of ACTIVE_BACKFILL_STATES) { diff --git a/modules/embeddings/src/config/index.ts b/modules/embeddings/src/config/index.ts index 4caded7bb..67ebf8af5 100644 --- a/modules/embeddings/src/config/index.ts +++ b/modules/embeddings/src/config/index.ts @@ -94,8 +94,8 @@ const AppConfigSchema = { }; const config = convict(AppConfigSchema); -const configProperties = config.getProperties(); -export type Config = typeof configProperties & { +void config; +export type Config = ReturnType & { providers: Record< string, { diff --git a/modules/embeddings/src/controllers/queue.controller.ts b/modules/embeddings/src/controllers/queue.controller.ts index 9d306e609..61b9526c7 100644 --- a/modules/embeddings/src/controllers/queue.controller.ts +++ b/modules/embeddings/src/controllers/queue.controller.ts @@ -81,15 +81,6 @@ export interface QueueControllerDependencies { ) => WorkerLike; } -const EMPTY_COUNTS: QueueJobCounts = { - waiting: 0, - active: 0, - completed: 0, - failed: 0, - delayed: 0, - paused: 0, -}; - export class QueueController { private static _instance: QueueController; private readonly createConnection: () => RedisConnection; diff --git a/modules/embeddings/src/providers/index.ts b/modules/embeddings/src/providers/index.ts index 01eb05862..c0c712dc0 100644 --- a/modules/embeddings/src/providers/index.ts +++ b/modules/embeddings/src/providers/index.ts @@ -35,6 +35,41 @@ export interface EmbeddingProviderDependencies { lookup?: SafeEndpointOptions['lookup']; } +function mapEmbedFetchError(err: unknown): never { + if (err instanceof GrpcError) throw err; + const message = sanitizeErrorMessage(err); + if (/redirect/i.test(message)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embedding provider redirects are not allowed', + ); + } + if (err instanceof Error && err.name === 'TimeoutError') { + throw new GrpcError(status.DEADLINE_EXCEEDED, 'Embedding provider request timed out'); + } + throw new GrpcError(status.UNAVAILABLE, message); +} + +function parseEmbeddingResponse(bodyText: string): number[] { + let body: { data?: { embedding?: number[] }[] }; + try { + body = JSON.parse(bodyText) as { data?: { embedding?: number[] }[] }; + } catch { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding provider response was not valid JSON', + ); + } + const embedding = body.data?.[0]?.embedding; + if (!embedding?.length) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding provider response did not include an embedding', + ); + } + return embedding; +} + export class OpenAICompatibleEmbeddingProvider implements EmbeddingProvider { constructor(private readonly deps: EmbeddingProviderDependencies = {}) {} @@ -45,8 +80,9 @@ export class OpenAICompatibleEmbeddingProvider implements EmbeddingProvider { 'Embedding provider endpoint is not configured', ); } - const maxInput = config.maxInputBytes ?? DEFAULT_MAX_EMBED_INPUT_BYTES; - if (Buffer.byteLength(input) > maxInput) { + if ( + Buffer.byteLength(input) > (config.maxInputBytes ?? DEFAULT_MAX_EMBED_INPUT_BYTES) + ) { throw new GrpcError( status.INVALID_ARGUMENT, 'Embedding input exceeds the allowed size', @@ -56,7 +92,6 @@ export class OpenAICompatibleEmbeddingProvider implements EmbeddingProvider { allowedHosts: config.allowedHosts ?? [], lookup: this.deps.lookup, }); - const timeoutMs = config.timeoutMs ?? DEFAULT_EMBED_TIMEOUT_MS; const fetchImpl = this.deps.fetch ?? fetch; let response: Response; try { @@ -71,24 +106,10 @@ export class OpenAICompatibleEmbeddingProvider implements EmbeddingProvider { model: config.model, }), redirect: 'error', - signal: AbortSignal.timeout(timeoutMs), + signal: AbortSignal.timeout(config.timeoutMs ?? DEFAULT_EMBED_TIMEOUT_MS), }); } catch (err) { - if (err instanceof GrpcError) throw err; - const message = sanitizeErrorMessage(err); - if (/redirect/i.test(message)) { - throw new GrpcError( - status.PERMISSION_DENIED, - 'Embedding provider redirects are not allowed', - ); - } - if (err instanceof Error && err.name === 'TimeoutError') { - throw new GrpcError( - status.DEADLINE_EXCEEDED, - 'Embedding provider request timed out', - ); - } - throw new GrpcError(status.UNAVAILABLE, message); + mapEmbedFetchError(err); } if (!response.ok) { throw new GrpcError( @@ -96,27 +117,12 @@ export class OpenAICompatibleEmbeddingProvider implements EmbeddingProvider { `Embedding provider failed with HTTP ${response.status}`, ); } - const bodyText = await readCappedResponse( - response, - config.maxResponseBytes ?? DEFAULT_MAX_EMBED_RESPONSE_BYTES, + return parseEmbeddingResponse( + await readCappedResponse( + response, + config.maxResponseBytes ?? DEFAULT_MAX_EMBED_RESPONSE_BYTES, + ), ); - let body: { data?: { embedding?: number[] }[] }; - try { - body = JSON.parse(bodyText) as { data?: { embedding?: number[] }[] }; - } catch { - throw new GrpcError( - status.INVALID_ARGUMENT, - 'Embedding provider response was not valid JSON', - ); - } - const embedding = body.data?.[0]?.embedding; - if (!embedding?.length) { - throw new GrpcError( - status.INVALID_ARGUMENT, - 'Embedding provider response did not include an embedding', - ); - } - return embedding; } } diff --git a/modules/embeddings/src/utils/backfillExecution.test.ts b/modules/embeddings/src/utils/backfillExecution.test.ts index 778208ae2..84f034cd8 100644 --- a/modules/embeddings/src/utils/backfillExecution.test.ts +++ b/modules/embeddings/src/utils/backfillExecution.test.ts @@ -351,7 +351,6 @@ describe('backfill cancellation, resume, and counters', () => { { runId: created._id, cursor: null }, deps({ store }), ); - const afterPage = (await store.getRun(created._id))!; const processed = await applyBackfillJobOutcome({ runId: created._id, outcome: 'processed', diff --git a/modules/embeddings/src/utils/backfillExecution.ts b/modules/embeddings/src/utils/backfillExecution.ts index f2b382fd9..3e7cc8093 100644 --- a/modules/embeddings/src/utils/backfillExecution.ts +++ b/modules/embeddings/src/utils/backfillExecution.ts @@ -372,12 +372,11 @@ export async function processBackfillControllerJob( return scanBackfillPage(parsed.data, persisted, deps); } -async function scanBackfillPage( - job: BackfillControllerJobData, +async function startQueuedBackfill( persisted: PersistedBackfillRun, deps: ProcessBackfillDeps, -): Promise<{ action: string; run?: BackfillRunProgress }> { - const now = deps.now ?? new Date(); + now: Date, +): Promise<{ action: string; run?: BackfillRunProgress } | { run: BackfillRunProgress }> { let run: BackfillRunProgress = persisted; if (run.state === 'queued') { const started = startBackfillRun(run, now); @@ -388,15 +387,52 @@ async function scanBackfillPage( if (run.state !== 'running') { return { action: run.state, run }; } + return { run }; +} + +async function rescheduleStaleBackfill( + persisted: PersistedBackfillRun, + run: BackfillRunProgress, + job: BackfillControllerJobData, + deps: ProcessBackfillDeps, +): Promise<{ action: string; run: BackfillRunProgress } | undefined> { const jobCursor = job.cursor ?? null; const runCursor = run.cursor ?? null; - if (jobCursor !== runCursor) { - await deps.enqueueContinuation({ - runId: persisted._id, - cursor: runCursor, - }); - return { action: 'stale', run }; - } + if (jobCursor === runCursor) return undefined; + await deps.enqueueContinuation({ + runId: persisted._id, + cursor: runCursor, + }); + return { action: 'stale', run }; +} + +function pageEmbeddingJobs( + run: BackfillRunProgress, + persistedId: string, + docs: Array<{ _id?: unknown }>, + maxBatchSize: number, +): EmbeddingJobData[] { + return docs + .map(doc => ({ + schemaName: run.schemaName, + documentId: String(doc._id), + ...(run.configId ? { configId: run.configId } : {}), + backfillRunId: persistedId, + })) + .slice(0, Math.min(run.batchSize, maxBatchSize, MAX_QUEUE_BATCH_SIZE)); +} + +async function scanBackfillPage( + job: BackfillControllerJobData, + persisted: PersistedBackfillRun, + deps: ProcessBackfillDeps, +): Promise<{ action: string; run?: BackfillRunProgress }> { + const now = deps.now ?? new Date(); + const started = await startQueuedBackfill(persisted, deps, now); + if ('action' in started) return started; + let run = started.run; + const stale = await rescheduleStaleBackfill(persisted, run, job, deps); + if (stale) return stale; try { if (!run.configId) { @@ -423,14 +459,7 @@ async function scanBackfillPage( await deps.findPage(run.schemaName, pageQuery.page), run.batchSize, ); - const jobs = docs - .map(doc => ({ - schemaName: run.schemaName, - documentId: String(doc._id), - ...(run.configId ? { configId: run.configId } : {}), - backfillRunId: persisted._id, - })) - .slice(0, Math.min(run.batchSize, deps.maxBatchSize, MAX_QUEUE_BATCH_SIZE)); + const jobs = pageEmbeddingJobs(run, persisted._id, docs, deps.maxBatchSize); const queuedDelta = jobs.length ? await deps.enqueueEmbeddingJobs(jobs) : 0; incrementEmbeddingMetric('backfill', queuedDelta); const applied = applyBackfillPage( @@ -456,9 +485,6 @@ async function scanBackfillPage( }); return { action: 'continue', run }; } catch (err) { - if (err instanceof BackfillGateError) { - return failPersistedRun(persisted._id, run, err, deps, now); - } return failPersistedRun(persisted._id, run, err, deps, now); } } diff --git a/modules/embeddings/src/utils/backfillRun.ts b/modules/embeddings/src/utils/backfillRun.ts index d7914511f..9f50dc97f 100644 --- a/modules/embeddings/src/utils/backfillRun.ts +++ b/modules/embeddings/src/utils/backfillRun.ts @@ -461,6 +461,21 @@ function isMembershipOperator( return (BACKFILL_FILTER_MEMBERSHIP_OPERATORS as readonly string[]).includes(operator); } +function isSafeComparisonOperand(operator: string, comparison: unknown): boolean { + if (operator === '$eq' || operator === '$ne') { + return isBackfillFilterScalar(comparison); + } + return typeof comparison === 'number' || typeof comparison === 'string'; +} + +function isSafeMembershipOperand(items: unknown): boolean { + return ( + Array.isArray(items) && + items.length <= MAX_BACKFILL_FILTER_IN_VALUES && + items.every(isBackfillFilterScalar) + ); +} + function isSafeBackfillPredicate(value: unknown, depth: number): boolean { if (depth > MAX_BACKFILL_FILTER_DEPTH) return false; if (isBackfillFilterScalar(value)) return true; @@ -469,20 +484,11 @@ function isSafeBackfillPredicate(value: unknown, depth: number): boolean { if (!operators.length || operators.length > MAX_BACKFILL_FILTER_KEYS) return false; for (const operator of operators) { if (isComparisonOperator(operator)) { - const comparison = value[operator]; - if (operator === '$eq' || operator === '$ne') { - if (!isBackfillFilterScalar(comparison)) return false; - continue; - } - if (typeof comparison !== 'number' && typeof comparison !== 'string') return false; + if (!isSafeComparisonOperand(operator, value[operator])) return false; continue; } if (isMembershipOperator(operator)) { - const items = value[operator]; - if (!Array.isArray(items) || items.length > MAX_BACKFILL_FILTER_IN_VALUES) { - return false; - } - if (!items.every(isBackfillFilterScalar)) return false; + if (!isSafeMembershipOperand(value[operator])) return false; continue; } return false; diff --git a/modules/embeddings/src/utils/embeddingJobs.ts b/modules/embeddings/src/utils/embeddingJobs.ts index 4f542d99c..4c3f740e3 100644 --- a/modules/embeddings/src/utils/embeddingJobs.ts +++ b/modules/embeddings/src/utils/embeddingJobs.ts @@ -64,6 +64,17 @@ export function isDuplicateJobError(err: unknown): boolean { export type ParsedEmbeddingJob = { ok: true; data: EmbeddingJobData } | { ok: false; reason: string }; +function optionalIdentity( + value: unknown, + reason: 'configId' | 'backfillRunId', +): { ok: true; value?: string } | { ok: false; reason: string } { + if (value === undefined) return { ok: true }; + if (typeof value !== 'string' || !IDENTITY.test(value)) { + return { ok: false, reason }; + } + return { ok: true, value }; +} + export function parseEmbeddingJobData( value: unknown, maxBatchIndex?: number, @@ -85,25 +96,17 @@ export function parseEmbeddingJobData( if (typeof record.documentId !== 'string' || !IDENTITY.test(record.documentId)) { return { ok: false, reason: 'documentId' }; } - if ( - record.configId !== undefined && - (typeof record.configId !== 'string' || !IDENTITY.test(record.configId)) - ) { - return { ok: false, reason: 'configId' }; - } - if ( - record.backfillRunId !== undefined && - (typeof record.backfillRunId !== 'string' || !IDENTITY.test(record.backfillRunId)) - ) { - return { ok: false, reason: 'backfillRunId' }; - } + const configId = optionalIdentity(record.configId, 'configId'); + if (!configId.ok) return configId; + const backfillRunId = optionalIdentity(record.backfillRunId, 'backfillRunId'); + if (!backfillRunId.ok) return backfillRunId; return { ok: true, data: { schemaName: record.schemaName, documentId: record.documentId, - ...(record.configId ? { configId: record.configId } : {}), - ...(record.backfillRunId ? { backfillRunId: record.backfillRunId } : {}), + ...(configId.value ? { configId: configId.value } : {}), + ...(backfillRunId.value ? { backfillRunId: backfillRunId.value } : {}), }, }; } diff --git a/modules/embeddings/test/deployment-contract.test.mjs b/modules/embeddings/test/deployment-contract.test.mjs index a63207627..168f997d9 100644 --- a/modules/embeddings/test/deployment-contract.test.mjs +++ b/modules/embeddings/test/deployment-contract.test.mjs @@ -106,7 +106,7 @@ test('PR CI runs compose render and target discovery', () => { /docker compose --profile mongodb --profile embeddings config --services/, ); assert.match(workflow, /docker compose --profile mongodb config --services/); - assert.match(workflow, /node scripts\/resolve-docker-targets\.mjs/); + assert.match(workflow, /env -u GITHUB_OUTPUT node scripts\/resolve-docker-targets\.mjs/); assert.match(workflow, /docker\/\*\*/); assert.match(workflow, /scripts\/resolve-docker-targets\.mjs/); }); diff --git a/scripts/resolve-docker-targets.mjs b/scripts/resolve-docker-targets.mjs index 74dadae83..567fc56a6 100644 --- a/scripts/resolve-docker-targets.mjs +++ b/scripts/resolve-docker-targets.mjs @@ -235,14 +235,19 @@ function resolveTargets({ changedFiles, forceAll } = {}) { function writeOutput(matrix, channel) { const payload = JSON.stringify({ include: matrix }); + const result = { + matrix: payload, + channel, + has_targets: matrix.length > 0, + }; + console.log(JSON.stringify(result, null, 2)); const outputFile = process.env.GITHUB_OUTPUT; - if (outputFile) { - appendFileSync(outputFile, `matrix=${payload}\n`, 'utf8'); - appendFileSync(outputFile, `channel=${channel}\n`, 'utf8'); - appendFileSync(outputFile, `has_targets=${matrix.length > 0}\n`, 'utf8'); - } else { - console.log(JSON.stringify({ matrix: payload, channel, has_targets: matrix.length > 0 }, null, 2)); + if (!outputFile) { + return; } + appendFileSync(outputFile, `matrix=${payload}\n`, 'utf8'); + appendFileSync(outputFile, `channel=${channel}\n`, 'utf8'); + appendFileSync(outputFile, `has_targets=${matrix.length > 0}\n`, 'utf8'); } function isMainModule() { From 736e43154c12e69fbe2fe5a05f982d0523f62911 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Wed, 9 Sep 2026 22:10:34 +0300 Subject: [PATCH 22/29] feat(embeddings): replace gRPC/host settings with a model catalogue Operator settings now own provider models and dimensions, while endpoint SSRF checks derive the HTTPS hostname internally and production GRPC_KEY stays a deployment requirement. --- deploy/embeddings.md | 5 +- modules/embeddings/README.md | 3 +- modules/embeddings/src/Embeddings.ts | 22 +- .../embeddings/src/api/embeddingsApi.test.ts | 4 +- modules/embeddings/src/config/index.ts | 39 ++- .../embeddings/src/providers/index.test.ts | 13 +- modules/embeddings/src/providers/index.ts | 2 - .../src/utils/endpointSecurity.test.ts | 47 ++-- .../embeddings/src/utils/endpointSecurity.ts | 10 +- .../src/utils/operationalStatus.test.ts | 4 +- .../embeddings/src/utils/operationalStatus.ts | 8 +- .../src/utils/productionSecurity.test.ts | 11 +- .../src/utils/productionSecurity.ts | 8 +- .../src/utils/providerConfig.test.ts | 138 +++++++++++ .../embeddings/src/utils/providerConfig.ts | 225 ++++++++++++++++++ .../embeddings/src/utils/redactConfig.test.ts | 11 +- .../test/deployment-contract.test.mjs | 35 +-- 17 files changed, 476 insertions(+), 109 deletions(-) create mode 100644 modules/embeddings/src/utils/providerConfig.test.ts create mode 100644 modules/embeddings/src/utils/providerConfig.ts diff --git a/deploy/embeddings.md b/deploy/embeddings.md index bd5d71886..27a04d4ed 100644 --- a/deploy/embeddings.md +++ b/deploy/embeddings.md @@ -48,8 +48,9 @@ pod with module convict `enabled` still false. true for the target backend (MongoDB Atlas Vector Search or Postgres pgvector). Saving a disabled config may succeed with capability warnings; activation must not. -5. Configure the HTTPS provider (`endpoint`, `apiKey`, `allowedHosts`). Check - `GET /embeddings/status` for provider/index warnings. +5. Configure the HTTPS provider (`endpoint`, `apiKey`, and model catalogue). + `GRPC_KEY` is supplied by the deployment (`NODE_ENV=production`), not by + module settings. Check `GET /embeddings/status` for provider/index warnings. 6. Create an embedding config. The first upsert provisions the vector index when Database indexing is available. The config stays disabled until the index for `targetField` is queryable (`status` ready, not pending/failed). diff --git a/modules/embeddings/README.md b/modules/embeddings/README.md index 455be2acd..2078810fc 100644 --- a/modules/embeddings/README.md +++ b/modules/embeddings/README.md @@ -24,7 +24,8 @@ Enable it and configure an OpenAI-compatible provider: "openai-compatible": { "endpoint": "https://api.openai.com/v1/embeddings", "apiKey": "...", - "allowedHosts": ["api.openai.com"] + "models": [{ "name": "text-embedding-3-small", "dimensions": 1536 }], + "defaultModel": "text-embedding-3-small" } }, "queue": { diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts index ecbeb4e67..06e12c2a6 100644 --- a/modules/embeddings/src/Embeddings.ts +++ b/modules/embeddings/src/Embeddings.ts @@ -35,6 +35,10 @@ import { assertGrpcKeyRequirement, callerModuleName, } from './utils/productionSecurity.js'; +import { + normalizeEmbeddingsConfig, + resolveProviderModelName, +} from './utils/providerConfig.js'; import { sanitizeErrorMessage } from './utils/redactConfig.js'; import { applyBackfillJobOutcome, @@ -123,8 +127,8 @@ export default class EmbeddingsModule extends ManagedModule { } async preConfig(config: Config) { - assertGrpcKeyRequirement(process.env, config); - return config; + assertGrpcKeyRequirement(process.env); + return normalizeEmbeddingsConfig(config); } async onConfig() { @@ -570,19 +574,13 @@ export default class EmbeddingsModule extends ManagedModule { private providerConfig(provider: string, model: string) { const config = this.currentConfig(); - const providers = config.providers as Record>; - const providerConfig = providers[provider] ?? {}; + const providerConfig = config.providers[provider] ?? {}; return { endpoint: typeof providerConfig.endpoint === 'string' ? providerConfig.endpoint : undefined, apiKey: typeof providerConfig.apiKey === 'string' ? providerConfig.apiKey : undefined, - model: String(providerConfig.model ?? model), - allowedHosts: [ - ...new Set( - ((providerConfig.allowedHosts as string[] | undefined) ?? []).filter(Boolean), - ), - ], + model: resolveProviderModelName(providerConfig, model), timeoutMs: config.security.embedTimeoutMs, maxInputBytes: config.security.maxEmbedInputBytes, maxResponseBytes: config.security.maxEmbedResponseBytes, @@ -590,7 +588,9 @@ export default class EmbeddingsModule extends ManagedModule { } private currentConfig() { - return ConfigController.getInstance().config as Config; + return normalizeEmbeddingsConfig(ConfigController.getInstance().config as Config, { + strict: false, + }); } protected registerSchemas(): Promise { diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts index 93569b545..347b75c9c 100644 --- a/modules/embeddings/src/api/embeddingsApi.test.ts +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -49,12 +49,12 @@ const moduleConfig = { 'openai-compatible': { endpoint: 'https://api.openai.com/v1/embeddings', apiKey: 'sk-test', - allowedHosts: ['api.openai.com'], + models: [{ name: 'text-embedding-3-small', dimensions: 1536 }], + defaultModel: 'text-embedding-3-small', }, }, queue: { concurrency: 1, attempts: 3, maxBatchSize: 50 }, security: { - requireGrpcKey: false, sourceFieldAllowlist: [], maxMutationEventIds: 10, embedTimeoutMs: 1000, diff --git a/modules/embeddings/src/config/index.ts b/modules/embeddings/src/config/index.ts index 67ebf8af5..f1443dcc8 100644 --- a/modules/embeddings/src/config/index.ts +++ b/modules/embeddings/src/config/index.ts @@ -25,16 +25,16 @@ const AppConfigSchema = { default: '', sensitive: true, }, - model: { - doc: 'Default provider model name', - format: String, - default: '', - }, - allowedHosts: { - doc: 'Allowed HTTPS hosts for this provider after DNS resolution', + models: { + doc: 'Operator-managed embedding models and output dimensions', format: Array, default: [], }, + defaultModel: { + doc: 'Default model name from the provider catalogue', + format: String, + default: '', + }, }, }, queue: { @@ -60,11 +60,6 @@ const AppConfigSchema = { }, }, security: { - requireGrpcKey: { - doc: 'Require GRPC_KEY. Always enforced when NODE_ENV is production.', - format: 'Boolean', - default: false, - }, sourceFieldAllowlist: { doc: 'Operator-configured source fields allowed even when hidden or sensitive-named. Caller-supplied allowlists are honored only for platform-admin upserts.', format: Array, @@ -95,15 +90,17 @@ const AppConfigSchema = { const config = convict(AppConfigSchema); void config; +export type EmbeddingProviderModel = { + name: string; + dimensions: number; +}; +export type EmbeddingProviderSettings = { + endpoint?: string; + apiKey?: string; + models?: EmbeddingProviderModel[]; + defaultModel?: string; +}; export type Config = ReturnType & { - providers: Record< - string, - { - endpoint?: string; - apiKey?: string; - model?: string; - allowedHosts?: string[]; - } - >; + providers: Record; }; export default AppConfigSchema; diff --git a/modules/embeddings/src/providers/index.test.ts b/modules/embeddings/src/providers/index.test.ts index 33afd2513..788ef9dbf 100644 --- a/modules/embeddings/src/providers/index.test.ts +++ b/modules/embeddings/src/providers/index.test.ts @@ -12,12 +12,11 @@ describe('openai-compatible provider security', () => { }, }); - it('rejects redirects, oversize input, and missing allowlists', async () => { + it('rejects redirects, oversize input, and blocked endpoints', async () => { await assert.rejects( () => provider.embed('hello', { endpoint: 'https://api.openai.com/v1/embeddings', - allowedHosts: ['api.openai.com'], }), err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, ); @@ -25,7 +24,6 @@ describe('openai-compatible provider security', () => { () => provider.embed('x'.repeat(100), { endpoint: 'https://api.openai.com/v1/embeddings', - allowedHosts: ['api.openai.com'], maxInputBytes: 8, }), err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, @@ -33,18 +31,19 @@ describe('openai-compatible provider security', () => { await assert.rejects( () => provider.embed('hello', { - endpoint: 'https://api.openai.com/v1/embeddings', - allowedHosts: [], + endpoint: 'https://127.0.0.1/v1/embeddings', }), err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, ); }); - it('returns embeddings when the allowlisted HTTPS endpoint is safe', async () => { + it('returns embeddings from a public HTTPS endpoint using the selected model', async () => { const safe = new OpenAICompatibleEmbeddingProvider({ lookup: async () => [{ address: '104.18.0.1', family: 4 }], fetch: async (_url, init) => { assert.equal(init?.redirect, 'error'); + const body = JSON.parse(String(init?.body ?? '{}')) as { model?: string }; + assert.equal(body.model, 'text-embedding-3-small'); return new Response(JSON.stringify({ data: [{ embedding: [0.1, 0.2] }] }), { status: 200, }); @@ -52,8 +51,8 @@ describe('openai-compatible provider security', () => { }); const vector = await safe.embed('hello', { endpoint: 'https://api.openai.com/v1/embeddings', - allowedHosts: ['api.openai.com'], apiKey: 'sk-test', + model: 'text-embedding-3-small', }); assert.deepEqual(vector, [0.1, 0.2]); }); diff --git a/modules/embeddings/src/providers/index.ts b/modules/embeddings/src/providers/index.ts index c0c712dc0..45341b1e5 100644 --- a/modules/embeddings/src/providers/index.ts +++ b/modules/embeddings/src/providers/index.ts @@ -15,7 +15,6 @@ export interface EmbeddingProviderConfig { endpoint?: string; apiKey?: string; model?: string; - allowedHosts?: string[]; timeoutMs?: number; maxInputBytes?: number; maxResponseBytes?: number; @@ -89,7 +88,6 @@ export class OpenAICompatibleEmbeddingProvider implements EmbeddingProvider { ); } await assertSafeEmbeddingEndpoint(config.endpoint, { - allowedHosts: config.allowedHosts ?? [], lookup: this.deps.lookup, }); const fetchImpl = this.deps.fetch ?? fetch; diff --git a/modules/embeddings/src/utils/endpointSecurity.test.ts b/modules/embeddings/src/utils/endpointSecurity.test.ts index 9d73e3586..39c55082a 100644 --- a/modules/embeddings/src/utils/endpointSecurity.test.ts +++ b/modules/embeddings/src/utils/endpointSecurity.test.ts @@ -19,40 +19,55 @@ describe('embedding endpoint SSRF controls', () => { assert.equal(isBlockedIp('8.8.8.8'), false); }); - it('requires HTTPS and an allowlisted host', async () => { + it('requires HTTPS and rejects URL credentials', async () => { await assert.rejects( - () => - assertSafeEmbeddingEndpoint('http://api.openai.com/v1/embeddings', { - allowedHosts: ['api.openai.com'], - }), + () => assertSafeEmbeddingEndpoint('http://api.openai.com/v1/embeddings'), err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, ); await assert.rejects( () => - assertSafeEmbeddingEndpoint('https://evil.example/v1/embeddings', { - allowedHosts: ['api.openai.com'], + assertSafeEmbeddingEndpoint('https://user:pass@api.openai.com/v1/embeddings', { + lookup: async () => [{ address: '104.18.0.1', family: 4 }], }), - err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /credentials/.test(err.message), + ); + }); + + it('derives the hostname from the HTTPS endpoint and allows public resolutions', async () => { + const url = await assertSafeEmbeddingEndpoint( + 'https://api.openai.com/v1/embeddings', + { + lookup: async () => [{ address: '104.18.0.1', family: 4 }], + }, + ); + assert.equal(url.hostname, 'api.openai.com'); + const other = await assertSafeEmbeddingEndpoint( + 'https://evil.example/v1/embeddings', + { + lookup: async () => [{ address: '104.18.0.1', family: 4 }], + }, ); + assert.equal(other.hostname, 'evil.example'); }); it('rejects DNS results that resolve to private or metadata addresses', async () => { await assert.rejects( () => assertSafeEmbeddingEndpoint('https://api.openai.com/v1/embeddings', { - allowedHosts: ['api.openai.com'], lookup: async () => [{ address: '169.254.169.254', family: 4 }], }), err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, ); - const url = await assertSafeEmbeddingEndpoint( - 'https://api.openai.com/v1/embeddings', - { - allowedHosts: ['api.openai.com'], - lookup: async () => [{ address: '104.18.0.1', family: 4 }], - }, + await assert.rejects( + () => + assertSafeEmbeddingEndpoint('https://localhost/v1/embeddings', { + lookup: async () => [{ address: '127.0.0.1', family: 4 }], + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, ); - assert.equal(url.hostname, 'api.openai.com'); }); it('caps response payload size', async () => { diff --git a/modules/embeddings/src/utils/endpointSecurity.ts b/modules/embeddings/src/utils/endpointSecurity.ts index 6069453cb..e60bee0ee 100644 --- a/modules/embeddings/src/utils/endpointSecurity.ts +++ b/modules/embeddings/src/utils/endpointSecurity.ts @@ -28,7 +28,6 @@ privateNetworks.addSubnet('fc00::', 7, 'ipv6'); privateNetworks.addSubnet('fe80::', 10, 'ipv6'); export interface SafeEndpointOptions { - allowedHosts: string[]; lookup?: ( hostname: string, options: { all: true; verbatim: true }, @@ -45,7 +44,7 @@ export function isBlockedIp(address: string): boolean { export async function assertSafeEmbeddingEndpoint( endpoint: string, - options: SafeEndpointOptions, + options: SafeEndpointOptions = {}, ): Promise { let url: URL; try { @@ -75,13 +74,6 @@ export async function assertSafeEmbeddingEndpoint( 'Embedding provider endpoint host is not allowed', ); } - const allowed = new Set(options.allowedHosts.map(host => host.toLowerCase())); - if (!allowed.has(hostname)) { - throw new GrpcError( - status.PERMISSION_DENIED, - 'Embedding provider endpoint host is not allowlisted', - ); - } if (isIP(hostname)) { if (isBlockedIp(hostname)) { throw new GrpcError( diff --git a/modules/embeddings/src/utils/operationalStatus.test.ts b/modules/embeddings/src/utils/operationalStatus.test.ts index b47755def..cfab49aab 100644 --- a/modules/embeddings/src/utils/operationalStatus.test.ts +++ b/modules/embeddings/src/utils/operationalStatus.test.ts @@ -27,7 +27,7 @@ describe('embeddings operational warnings and search gates', () => { ...providerReadinessWarnings({ endpoint: '', apiKey: 'sk-secret', - allowedHosts: [], + models: [], }), ]; assert.equal( @@ -35,7 +35,7 @@ describe('embeddings operational warnings and search gates', () => { true, ); assert.equal( - warnings.some(warning => /allowlist is empty/.test(warning)), + warnings.some(warning => /model catalogue is empty/.test(warning)), true, ); assert.equal(warnings.join(' ').includes('sk-secret'), false); diff --git a/modules/embeddings/src/utils/operationalStatus.ts b/modules/embeddings/src/utils/operationalStatus.ts index 6c6c1f031..d7e5260eb 100644 --- a/modules/embeddings/src/utils/operationalStatus.ts +++ b/modules/embeddings/src/utils/operationalStatus.ts @@ -9,6 +9,7 @@ import { type BackfillConfigGate, type VectorIndexGate, } from './backfillGates.js'; +import { providerCatalogueIssues } from './providerConfig.js'; import type { QueueJobCounts } from '../controllers/queue.controller.js'; export const SEARCH_GATE_REASONS = [ @@ -161,7 +162,8 @@ export function indexReadinessWarnings( export function providerReadinessWarnings(provider?: { endpoint?: string; apiKey?: string; - allowedHosts?: string[]; + models?: Array<{ name?: string; dimensions?: number }>; + defaultModel?: string; }): string[] { const warnings: string[] = []; if (!provider?.endpoint) { @@ -170,9 +172,7 @@ export function providerReadinessWarnings(provider?: { if (!provider?.apiKey) { warnings.push('Embedding provider API key is not configured'); } - if (!provider?.allowedHosts?.length) { - warnings.push('Embedding provider host allowlist is empty'); - } + warnings.push(...providerCatalogueIssues(provider)); return warnings; } diff --git a/modules/embeddings/src/utils/productionSecurity.test.ts b/modules/embeddings/src/utils/productionSecurity.test.ts index 61fe05852..640e44efe 100644 --- a/modules/embeddings/src/utils/productionSecurity.test.ts +++ b/modules/embeddings/src/utils/productionSecurity.test.ts @@ -15,15 +15,8 @@ describe('production GRPC_KEY requirement', () => { ); }); - it('does not require GRPC_KEY in non-production unless configured', () => { + it('does not require GRPC_KEY outside production', () => { assert.doesNotThrow(() => assertGrpcKeyRequirement({ NODE_ENV: 'test' })); - assert.throws( - () => - assertGrpcKeyRequirement( - { NODE_ENV: 'development' }, - { security: { requireGrpcKey: true } }, - ), - err => err instanceof GrpcError && err.code === status.FAILED_PRECONDITION, - ); + assert.doesNotThrow(() => assertGrpcKeyRequirement({ NODE_ENV: 'development' })); }); }); diff --git a/modules/embeddings/src/utils/productionSecurity.ts b/modules/embeddings/src/utils/productionSecurity.ts index 3ee7e3f2f..310ec54a1 100644 --- a/modules/embeddings/src/utils/productionSecurity.ts +++ b/modules/embeddings/src/utils/productionSecurity.ts @@ -1,12 +1,8 @@ import { GrpcError } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; -export function assertGrpcKeyRequirement( - env: NodeJS.ProcessEnv = process.env, - config?: { security?: { requireGrpcKey?: boolean } }, -): void { - const required = - env.NODE_ENV === 'production' || config?.security?.requireGrpcKey === true; +export function assertGrpcKeyRequirement(env: NodeJS.ProcessEnv = process.env): void { + const required = env.NODE_ENV === 'production'; if (required && !env.GRPC_KEY) { throw new GrpcError( status.FAILED_PRECONDITION, diff --git a/modules/embeddings/src/utils/providerConfig.test.ts b/modules/embeddings/src/utils/providerConfig.test.ts new file mode 100644 index 000000000..a1ac9b921 --- /dev/null +++ b/modules/embeddings/src/utils/providerConfig.test.ts @@ -0,0 +1,138 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + findProviderModel, + normalizeEmbeddingsConfig, + normalizeProviderSettings, + resolveProviderModelName, +} from './providerConfig.js'; + +describe('provider model catalogue', () => { + it('migrates a singular model setting into a one-item catalogue', () => { + const migrated = normalizeProviderSettings({ + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-test', + model: 'text-embedding-3-small', + dimensions: 1536, + allowedHosts: ['api.openai.com'], + }); + assert.deepEqual(migrated, { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-test', + models: [{ name: 'text-embedding-3-small', dimensions: 1536 }], + defaultModel: 'text-embedding-3-small', + }); + }); + + it('keeps an existing catalogue and drops legacy host and model fields', () => { + const normalized = normalizeProviderSettings({ + endpoint: 'https://api.openai.com/v1/embeddings', + model: 'legacy-model', + dimensions: 512, + allowedHosts: ['api.openai.com'], + models: [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + { name: 'text-embedding-3-large', dimensions: 3072 }, + ], + defaultModel: 'text-embedding-3-large', + }); + assert.deepEqual(normalized.models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + { name: 'text-embedding-3-large', dimensions: 3072 }, + ]); + assert.equal(normalized.defaultModel, 'text-embedding-3-large'); + assert.equal('model' in normalized, false); + assert.equal('allowedHosts' in normalized, false); + assert.equal('dimensions' in normalized, false); + }); + + it('rejects duplicate names, empty names, and non-positive dimensions', () => { + assert.throws( + () => + normalizeProviderSettings({ + models: [ + { name: 'small', dimensions: 1536 }, + { name: 'small', dimensions: 768 }, + ], + }), + err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, + ); + assert.throws( + () => normalizeProviderSettings({ models: [{ name: ' ', dimensions: 1536 }] }), + err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, + ); + assert.throws( + () => normalizeProviderSettings({ models: [{ name: 'small', dimensions: 0 }] }), + err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, + ); + assert.throws( + () => normalizeProviderSettings({ model: 'small' }), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /positive integer/.test(err.message), + ); + }); + + it('requires defaultModel to exist in the catalogue when set', () => { + assert.throws( + () => + normalizeProviderSettings({ + models: [{ name: 'small', dimensions: 1536 }], + defaultModel: 'missing', + }), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /not in the catalogue/.test(err.message), + ); + const normalized = normalizeProviderSettings({ + models: [ + { name: 'small', dimensions: 1536 }, + { name: 'large', dimensions: 3072 }, + ], + }); + assert.equal(normalized.defaultModel, undefined); + assert.equal(resolveProviderModelName(normalized, 'large'), 'large'); + assert.equal(resolveProviderModelName(normalized), 'small'); + assert.equal(findProviderModel(normalized, 'large')?.dimensions, 3072); + assert.equal(findProviderModel(normalized, 'missing'), undefined); + }); + + it('allows an empty catalogue and does not throw on incomplete legacy reads', () => { + assert.deepEqual(normalizeProviderSettings({ endpoint: 'https://api.example/v1' }), { + endpoint: 'https://api.example/v1', + models: [], + }); + assert.deepEqual(normalizeProviderSettings({ model: 'small' }, { strict: false }), { + models: [], + }); + }); + + it('strips requireGrpcKey and returns catalogue-shaped providers', () => { + const normalized = normalizeEmbeddingsConfig({ + enabled: true, + security: { + requireGrpcKey: true, + sourceFieldAllowlist: [], + }, + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + model: 'text-embedding-3-small', + dimensions: 1536, + }, + }, + }); + assert.equal( + 'requireGrpcKey' in (normalized.security as Record), + false, + ); + assert.deepEqual( + (normalized.providers['openai-compatible'] as { models?: unknown }).models, + [{ name: 'text-embedding-3-small', dimensions: 1536 }], + ); + }); +}); diff --git a/modules/embeddings/src/utils/providerConfig.ts b/modules/embeddings/src/utils/providerConfig.ts new file mode 100644 index 000000000..19df43c24 --- /dev/null +++ b/modules/embeddings/src/utils/providerConfig.ts @@ -0,0 +1,225 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import type { + EmbeddingProviderModel, + EmbeddingProviderSettings, +} from '../config/index.js'; + +export type { EmbeddingProviderModel, EmbeddingProviderSettings }; + +export interface LegacyEmbeddingProviderSettings extends EmbeddingProviderSettings { + model?: string; + dimensions?: number; + allowedHosts?: string[]; +} + +function invalidProviderConfig(message: string): GrpcError { + return new GrpcError(status.INVALID_ARGUMENT, message); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function trimName(value: unknown): string { + return typeof value === 'string' ? value.trim() : ''; +} + +function parseDimensions(value: unknown): number | undefined { + if (typeof value !== 'number' || !Number.isInteger(value) || value <= 0) { + return undefined; + } + return value; +} + +function parseModelEntry(value: unknown, index: number): EmbeddingProviderModel { + if (!isRecord(value)) { + throw invalidProviderConfig(`Provider model at index ${index} is invalid`); + } + const name = trimName(value.name); + if (!name) { + throw invalidProviderConfig( + `Provider model at index ${index} must have a non-empty name`, + ); + } + const dimensions = parseDimensions(value.dimensions); + if (dimensions == null) { + throw invalidProviderConfig( + `Provider model '${name}' dimensions must be a positive integer`, + ); + } + return { name, dimensions }; +} + +function parseModelList( + values: unknown[], + options?: { strict?: boolean }, +): EmbeddingProviderModel[] { + const models: EmbeddingProviderModel[] = []; + const names = new Set(); + for (const [index, value] of values.entries()) { + try { + const parsed = parseModelEntry(value, index); + if (names.has(parsed.name)) { + throw invalidProviderConfig(`Provider model '${parsed.name}' is duplicated`); + } + names.add(parsed.name); + models.push(parsed); + } catch (err) { + if (options?.strict !== false) throw err; + } + } + return models; +} + +function migrateLegacyModels( + raw: LegacyEmbeddingProviderSettings, +): EmbeddingProviderModel[] | undefined { + if (Array.isArray(raw.models) && raw.models.length > 0) return undefined; + const name = trimName(raw.model); + if (!name) return []; + const dimensions = parseDimensions(raw.dimensions); + if (dimensions == null) { + throw invalidProviderConfig( + `Provider model '${name}' dimensions must be a positive integer`, + ); + } + return [{ name, dimensions }]; +} + +export function providerCatalogueIssues(provider?: { + models?: Array<{ name?: string; dimensions?: number }>; + defaultModel?: string; +}): string[] { + const issues: string[] = []; + const models = provider?.models; + if (models != null && !Array.isArray(models)) { + return ['Embedding provider model catalogue is invalid']; + } + const list = models ?? []; + const names = new Set(); + for (const [index, model] of list.entries()) { + const name = trimName(model?.name); + const dimensions = parseDimensions(model?.dimensions); + if (!name) { + issues.push(`Provider model at index ${index} must have a non-empty name`); + continue; + } + if (dimensions == null) { + issues.push(`Provider model '${name}' dimensions must be a positive integer`); + } + if (names.has(name)) { + issues.push(`Provider model '${name}' is duplicated`); + } + names.add(name); + } + const defaultModel = trimName(provider?.defaultModel); + if (defaultModel && !names.has(defaultModel)) { + issues.push(`Provider default model '${defaultModel}' is not in the catalogue`); + } + if (!list.length) { + issues.push('Embedding provider model catalogue is empty'); + } + return issues; +} + +export function assertProviderCatalogue( + provider?: Pick, +): void { + const models = provider?.models ?? []; + if (!Array.isArray(models)) { + throw invalidProviderConfig('Provider model catalogue must be an array'); + } + parseModelList(models); + const names = new Set(models.map(model => trimName(model?.name))); + const defaultModel = trimName(provider?.defaultModel); + if (defaultModel && !names.has(defaultModel)) { + throw invalidProviderConfig( + `Provider default model '${defaultModel}' is not in the catalogue`, + ); + } +} + +export function normalizeProviderSettings( + raw: unknown, + options?: { strict?: boolean }, +): EmbeddingProviderSettings { + const source = isRecord(raw) ? (raw as LegacyEmbeddingProviderSettings) : {}; + let models: EmbeddingProviderModel[]; + let migratedFromSingular = false; + try { + const migrated = migrateLegacyModels(source); + migratedFromSingular = migrated != null && migrated.length > 0; + models = + migrated ?? + (Array.isArray(source.models) ? parseModelList(source.models, options) : []); + } catch (err) { + if (options?.strict !== false) throw err; + models = []; + } + const names = new Set(models.map(model => model.name)); + const configuredDefault = trimName(source.defaultModel); + const defaultModel = configuredDefault + ? names.has(configuredDefault) + ? configuredDefault + : '' + : migratedFromSingular + ? (models[0]?.name ?? '') + : ''; + if (configuredDefault && !defaultModel && options?.strict !== false) { + throw invalidProviderConfig( + `Provider default model '${configuredDefault}' is not in the catalogue`, + ); + } + if (options?.strict !== false) { + assertProviderCatalogue({ models, defaultModel }); + } + return { + ...(typeof source.endpoint === 'string' ? { endpoint: source.endpoint } : {}), + ...(typeof source.apiKey === 'string' ? { apiKey: source.apiKey } : {}), + models, + ...(defaultModel ? { defaultModel } : {}), + }; +} + +export function findProviderModel( + provider: EmbeddingProviderSettings | undefined, + name?: string, +): EmbeddingProviderModel | undefined { + const selected = trimName(name); + if (!selected) return undefined; + return provider?.models?.find(model => model.name === selected); +} + +export function resolveProviderModelName( + provider: EmbeddingProviderSettings | undefined, + selected?: string, +): string { + const trimmed = trimName(selected); + if (trimmed) return trimmed; + const defaultModel = trimName(provider?.defaultModel); + if (defaultModel) return defaultModel; + return provider?.models?.[0]?.name ?? ''; +} + +export function normalizeEmbeddingsConfig< + T extends { + providers?: Record; + security?: Record; + }, +>(config: T, options?: { strict?: boolean }): T { + const next = { ...config }; + if (isRecord(next.security)) { + const security = { ...next.security }; + delete (security as { requireGrpcKey?: boolean }).requireGrpcKey; + next.security = security as T['security']; + } + if (isRecord(next.providers)) { + const providers: Record = {}; + for (const [key, value] of Object.entries(next.providers)) { + providers[key] = normalizeProviderSettings(value, options); + } + next.providers = providers as T['providers']; + } + return next; +} diff --git a/modules/embeddings/src/utils/redactConfig.test.ts b/modules/embeddings/src/utils/redactConfig.test.ts index dabc601e5..e0538ca4d 100644 --- a/modules/embeddings/src/utils/redactConfig.test.ts +++ b/modules/embeddings/src/utils/redactConfig.test.ts @@ -25,7 +25,11 @@ describe('provider secret redaction', () => { { enabled: true, providers: { - 'openai-compatible': { endpoint: 'https://api.openai.com', apiKey: 'sk-live' }, + 'openai-compatible': { + endpoint: 'https://api.openai.com', + apiKey: 'sk-live', + models: [{ name: 'text-embedding-3-small', dimensions: 1536 }], + }, }, }, { @@ -33,6 +37,8 @@ describe('provider secret redaction', () => { 'openai-compatible': { apiKey: { format: 'String', default: '', sensitive: true }, endpoint: { format: 'String', default: '' }, + models: { format: Array, default: [] }, + defaultModel: { format: 'String', default: '' }, }, }, }, @@ -42,5 +48,8 @@ describe('provider secret redaction', () => { redacted.providers['openai-compatible'].endpoint, 'https://api.openai.com', ); + assert.deepEqual(redacted.providers['openai-compatible'].models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); }); }); diff --git a/modules/embeddings/test/deployment-contract.test.mjs b/modules/embeddings/test/deployment-contract.test.mjs index 168f997d9..7ab11a4a8 100644 --- a/modules/embeddings/test/deployment-contract.test.mjs +++ b/modules/embeddings/test/deployment-contract.test.mjs @@ -7,8 +7,7 @@ import { } from '../../../scripts/resolve-docker-targets.mjs'; const repo = new URL('../../..', import.meta.url); -const readRepo = relativePath => - readFileSync(new URL(relativePath, repo), 'utf8'); +const readRepo = relativePath => readFileSync(new URL(relativePath, repo), 'utf8'); const runbook = readRepo('deploy/embeddings.md'); const moduleReadme = readRepo('modules/embeddings/README.md'); @@ -58,10 +57,7 @@ test('runbook distinguishes Helm workload install.embeddings.enabled from convic assert.match(runbook, /Helm workload `install\.embeddings\.enabled`/); assert.match(runbook, /Module convict `enabled`/); assert.match(runbook, /install\.embeddings\.enabled=false/); - assert.match( - runbook, - /This is not `install\.embeddings\.enabled`/, - ); + assert.match(runbook, /This is not `install\.embeddings\.enabled`/); assert.doesNotMatch(runbook, /Helm: `install\.embeddings: false`/); assert.match(k8sReadme, /install\.embeddings\.enabled/); assert.match(moduleReadme, /install\.embeddings\.enabled/); @@ -71,7 +67,17 @@ test('rollback retains vector, index, config, and Redis state', () => { assert.match(runbook, /Rollback \*\*retains\*\*/); assert.match(runbook, /vector fields, indexes, `EmbeddingConfig` documents/); assert.match(runbook, /Redis\/BullMQ queue state/); - assert.match(k8sReadme, /retained vector\/index\/config\/\nRedis state|retained vector/); + assert.match( + k8sReadme, + /retained vector\/index\/config\/\nRedis state|retained vector/, + ); +}); + +test('module settings omit gRPC-key and host allowlists in favor of a model catalogue', () => { + assert.doesNotMatch(convictConfig, /requireGrpcKey/); + assert.doesNotMatch(convictConfig, /allowedHosts/); + assert.match(convictConfig, /defaultModel/); + assert.match(convictConfig, /Operator-managed embedding models/); }); test('docs stay default-off and do not claim a published embeddings image', () => { @@ -86,10 +92,7 @@ test('docs stay default-off and do not claim a published embeddings image', () = }); test('compose maps container GRPC_PORT through EMBEDDINGS_GRPC_PORT', () => { - assert.match( - composeSource, - /GRPC_PORT: '\$\{EMBEDDINGS_GRPC_PORT:-55165\}'/, - ); + assert.match(composeSource, /GRPC_PORT: '\$\{EMBEDDINGS_GRPC_PORT:-55165\}'/); assert.match( composeSource, /SERVICE_URL: 'conduit-embeddings:\$\{EMBEDDINGS_GRPC_PORT:-55165\}'/, @@ -106,7 +109,10 @@ test('PR CI runs compose render and target discovery', () => { /docker compose --profile mongodb --profile embeddings config --services/, ); assert.match(workflow, /docker compose --profile mongodb config --services/); - assert.match(workflow, /env -u GITHUB_OUTPUT node scripts\/resolve-docker-targets\.mjs/); + assert.match( + workflow, + /env -u GITHUB_OUTPUT node scripts\/resolve-docker-targets\.mjs/, + ); assert.match(workflow, /docker\/\*\*/); assert.match(workflow, /scripts\/resolve-docker-targets\.mjs/); }); @@ -135,10 +141,7 @@ test('other module rebuilds still select standalone', () => { assert.ok(!chatSelected.includes('embeddings')); const mixed = resolveTargets({ - changedFiles: [ - 'modules/embeddings/src/index.ts', - 'modules/storage/src/Storage.ts', - ], + changedFiles: ['modules/embeddings/src/index.ts', 'modules/storage/src/Storage.ts'], forceAll: false, }).map(entry => entry.target); assert.ok(mixed.includes('embeddings')); From 274bdacf82b54825ba601ede4da4ac8c90aef039 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Wed, 9 Sep 2026 22:23:04 +0300 Subject: [PATCH 23/29] feat(embeddings): enforce catalogue models and extendable schema policy Reject configs unless the provider and model exist in settings. Persist catalogue dimensions and require enabled, extendable schemas. --- .../grpc-sdk/src/modules/embeddings/index.ts | 6 +- modules/embeddings/src/Embeddings.ts | 9 +- modules/embeddings/src/admin/index.ts | 2 +- modules/embeddings/src/admin/routes.ts | 2 +- .../embeddings/src/api/embeddingsApi.test.ts | 261 +++++++++++++++++- modules/embeddings/src/api/embeddingsApi.ts | 60 +++- .../src/utils/providerConfig.test.ts | 38 +++ .../embeddings/src/utils/providerConfig.ts | 51 ++++ .../embeddings/src/utils/schemaPolicy.test.ts | 133 ++++++++- modules/embeddings/src/utils/schemaPolicy.ts | 164 +++++++++++ .../src/utils/validateEmbeddingConfig.test.ts | 68 ++++- .../src/utils/validateEmbeddingConfig.ts | 33 ++- 12 files changed, 775 insertions(+), 52 deletions(-) diff --git a/libraries/grpc-sdk/src/modules/embeddings/index.ts b/libraries/grpc-sdk/src/modules/embeddings/index.ts index 57656817e..13b7567a0 100644 --- a/libraries/grpc-sdk/src/modules/embeddings/index.ts +++ b/libraries/grpc-sdk/src/modules/embeddings/index.ts @@ -10,9 +10,9 @@ export interface EmbeddingConfigInput { schemaName: string; sourceFields: string[]; targetField: string; - provider: string; - model: string; - dimensions: number; + provider?: string; + model?: string; + dimensions?: number; similarity?: string; sourceFieldAllowlist?: string[]; enabled?: boolean; diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts index 06e12c2a6..aac4e07ad 100644 --- a/modules/embeddings/src/Embeddings.ts +++ b/modules/embeddings/src/Embeddings.ts @@ -565,10 +565,15 @@ export default class EmbeddingsModule extends ManagedModule { } private async declaredSchema(schemaName: string) { - return this.database.findOne<{ name: string; ownerModule: string }>( + return this.database.findOne<{ + name: string; + ownerModule: string; + fields?: Record; + extensions?: Array<{ ownerModule: string; fields: Record }>; + }>( '_DeclaredSchema', { name: schemaName }, - { select: 'name ownerModule' }, + { select: 'name ownerModule fields extensions' }, ); } diff --git a/modules/embeddings/src/admin/index.ts b/modules/embeddings/src/admin/index.ts index 3a9f45f93..9a91119f2 100644 --- a/modules/embeddings/src/admin/index.ts +++ b/modules/embeddings/src/admin/index.ts @@ -56,7 +56,7 @@ export class AdminHandlers { targetField: string; provider?: string; model?: string; - dimensions: number; + dimensions?: number; similarity?: string; sourceFieldAllowlist?: string[]; enabled?: boolean; diff --git a/modules/embeddings/src/admin/routes.ts b/modules/embeddings/src/admin/routes.ts index ce7b4c543..d7764c635 100644 --- a/modules/embeddings/src/admin/routes.ts +++ b/modules/embeddings/src/admin/routes.ts @@ -21,7 +21,7 @@ const CONFIG_BODY = { targetField: ConduitString.Required, provider: ConduitString.Optional, model: ConduitString.Optional, - dimensions: ConduitNumber.Required, + dimensions: ConduitNumber.Optional, similarity: ConduitString.Optional, sourceFieldAllowlist: { type: [TYPE.String], required: false }, enabled: ConduitBoolean.Optional, diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts index 347b75c9c..685480b90 100644 --- a/modules/embeddings/src/api/embeddingsApi.test.ts +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -22,7 +22,13 @@ import { SearchGateError } from '../utils/operationalStatus.js'; const articleSchema = { name: 'Article', fields: { title: { type: TYPE.String }, body: { type: TYPE.String } }, - modelOptions: { conduit: { authorization: { enabled: true } } }, + modelOptions: { + conduit: { + cms: { enabled: true }, + permissions: { extendable: true }, + authorization: { enabled: true }, + }, + }, }; const readyCapabilities = { @@ -49,7 +55,11 @@ const moduleConfig = { 'openai-compatible': { endpoint: 'https://api.openai.com/v1/embeddings', apiKey: 'sk-test', - models: [{ name: 'text-embedding-3-small', dimensions: 1536 }], + models: [ + { name: 'text-embedding-3-small', dimensions: 3 }, + { name: 'text-embedding-3-large', dimensions: 3 }, + { name: 'text-embedding-3-wide', dimensions: 8 }, + ], defaultModel: 'text-embedding-3-small', }, }, @@ -81,7 +91,15 @@ function createApi(overrides?: { method?: string; }>; schemas?: Record; - declared?: Record; + declared?: Record< + string, + { + name: string; + ownerModule: string; + fields?: Record; + extensions?: Array<{ ownerModule: string; fields: Record }>; + } + >; embed?: EmbeddingsApiDeps['embed']; vectorSearch?: EmbeddingsApiDeps['vectorSearch']; createVectorIndex?: EmbeddingsApiDeps['createVectorIndex']; @@ -90,6 +108,7 @@ function createApi(overrides?: { invalidated?: string[]; deletedIndexes?: string[]; createdIndexes?: string[]; + schemaExtensions?: Array<{ schemaName: string; fields: Record }>; config?: Config; queue?: { generation: QueueJobCounts; backfill: QueueJobCounts }; }) { @@ -100,6 +119,7 @@ function createApi(overrides?: { const invalidated = overrides?.invalidated ?? []; const deletedIndexes = overrides?.deletedIndexes ?? []; const createdIndexes = overrides?.createdIndexes ?? []; + const schemaExtensions = overrides?.schemaExtensions ?? []; const deps: EmbeddingsApiDeps = { currentConfig: () => overrides?.config ?? moduleConfig, getSchema: async name => { @@ -110,7 +130,10 @@ function createApi(overrides?: { }, declaredSchema: async name => overrides?.declared?.[name] ?? { name, ownerModule: 'database' }, - setSchemaExtension: async () => undefined, + setSchemaExtension: async args => { + schemaExtensions.push(args); + return undefined; + }, getVectorCapabilities: async () => overrides?.capabilities ?? readyCapabilities, getVectorIndexes: async () => indexes, vectorSearch: overrides?.vectorSearch ?? (async () => []), @@ -226,6 +249,7 @@ function createApi(overrides?: { invalidated, deletedIndexes, createdIndexes, + schemaExtensions, indexes, }; } @@ -392,7 +416,7 @@ describe('typed embeddings API handlers', () => { schemaName: 'AccessToken', sourceFields: ['token'], targetField: 'embedding', - model: 'm', + model: 'text-embedding-3-small', dimensions: 3, enabled: false, }, @@ -418,6 +442,7 @@ describe('typed embeddings API handlers', () => { password: { type: TYPE.String }, notes: { type: TYPE.String, select: false }, }, + modelOptions: articleSchema.modelOptions, }; const owner = { callerModule: 'cms-app' }; const declared = { Article: { name: 'Article', ownerModule: 'cms-app' } }; @@ -433,7 +458,7 @@ describe('typed embeddings API handlers', () => { schemaName: 'Article', sourceFields: ['password'], targetField: 'embedding', - model: 'm', + model: 'text-embedding-3-small', dimensions: 3, sourceFieldAllowlist: ['password'], enabled: false, @@ -452,7 +477,7 @@ describe('typed embeddings API handlers', () => { schemaName: 'Article', sourceFields: ['notes'], targetField: 'embedding', - model: 'm', + model: 'text-embedding-3-small', dimensions: 3, sourceFieldAllowlist: ['notes'], enabled: false, @@ -471,7 +496,7 @@ describe('typed embeddings API handlers', () => { schemaName: 'Article', sourceFields: ['password'], targetField: 'embedding', - model: 'm', + model: 'text-embedding-3-small', dimensions: 3, sourceFieldAllowlist: ['password'], enabled: false, @@ -498,7 +523,7 @@ describe('typed embeddings API handlers', () => { schemaName: 'Article', sourceFields: ['notes'], targetField: 'embedding', - model: 'm', + model: 'text-embedding-3-small', dimensions: 3, enabled: false, }, @@ -516,7 +541,7 @@ describe('typed embeddings API handlers', () => { schemaName: 'Article', sourceFields: ['notes'], targetField: 'embedding', - model: 'm', + model: 'text-embedding-3-small', dimensions: 3, sourceFieldAllowlist: ['notes'], enabled: false, @@ -712,7 +737,7 @@ describe('typed embeddings API handlers', () => { sourceFields: ['title'], targetField: 'embedding', provider: 'openai-compatible', - model: 'text-embedding-3-small', + model: 'text-embedding-3-wide', dimensions: 8, enabled: false, }, @@ -1055,4 +1080,218 @@ describe('typed embeddings API handlers', () => { assert.equal(resumedAgain.run.state, 'queued'); assert.equal(enqueued.length >= 2, true); }); + + it('rejects unknown providers, unknown models, and explicit catalogue dimension mismatches', async () => { + const { api } = createApi(); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'missing', + model: 'text-embedding-3-small', + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /not a configured provider/.test(err.message), + ); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'missing-model', + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /not in the catalogue/.test(err.message), + ); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'text-embedding-3-small', + dimensions: 1536, + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /do not match catalogue dimensions/.test(err.message), + ); + }); + + it('derives dimensions from the catalogue when the client omits them', async () => { + const { api, configs } = createApi({ indexes: [readyIndex] }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'text-embedding-3-small', + enabled: false, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.dimensions, 3); + assert.equal(configs[0].dimensions, 3); + assert.equal(saved.config.model, 'text-embedding-3-small'); + }); + + it('rejects CMS-enabled schemas that are not extendable', async () => { + const { api, configs, createdIndexes, schemaExtensions } = createApi({ + schemas: { + Article: { + ...articleSchema, + modelOptions: { + conduit: { + cms: { enabled: true }, + permissions: { extendable: false }, + authorization: { enabled: true }, + }, + }, + }, + }, + }); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'text-embedding-3-small', + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not extendable/.test(err.message), + ); + assert.equal(configs.length, 0); + assert.equal(createdIndexes.length, 0); + assert.equal(schemaExtensions.length, 0); + }); + + it('rejects incompatible field collisions before provisioning extensions or indexes', async () => { + const { api, configs, createdIndexes, schemaExtensions } = createApi({ + schemas: { + Article: { + ...articleSchema, + fields: { + ...articleSchema.fields, + embedding: { type: TYPE.String }, + }, + }, + }, + declared: { + Article: { + name: 'Article', + ownerModule: 'database', + fields: { + title: { type: TYPE.String }, + body: { type: TYPE.String }, + embedding: { type: TYPE.String }, + }, + }, + }, + }); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'text-embedding-3-small', + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.ALREADY_EXISTS && + /not a compatible embeddings extension/.test(err.message), + ); + assert.equal(configs.length, 0); + assert.equal(createdIndexes.length, 0); + assert.equal(schemaExtensions.length, 0); + }); + + it('is idempotent for compatible existing embeddings extensions', async () => { + const compatibleFields = { + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, + embeddingSourceHash: { type: TYPE.String, required: false, select: false }, + otherEmbedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, + }; + const { api, configs, schemaExtensions } = createApi({ + indexes: [readyIndex], + schemas: { + Article: { + ...articleSchema, + fields: { + ...articleSchema.fields, + ...compatibleFields, + }, + }, + }, + declared: { + Article: { + name: 'Article', + ownerModule: 'database', + fields: articleSchema.fields, + extensions: [ + { + ownerModule: 'embeddings', + fields: compatibleFields, + }, + ], + }, + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'text-embedding-3-small', + enabled: false, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(configs.length, 1); + assert.equal(schemaExtensions.length, 1); + assert.equal('otherEmbedding' in schemaExtensions[0].fields, true); + assert.equal('embedding' in schemaExtensions[0].fields, true); + }); }); diff --git a/modules/embeddings/src/api/embeddingsApi.ts b/modules/embeddings/src/api/embeddingsApi.ts index a94234314..ba539eab7 100644 --- a/modules/embeddings/src/api/embeddingsApi.ts +++ b/modules/embeddings/src/api/embeddingsApi.ts @@ -41,11 +41,16 @@ import { MAX_QUEUE_BATCH_SIZE } from '../utils/embeddingJobs.js'; import { ACTIVE_BACKFILL_STATES } from '../utils/backfillRun.js'; import { assertCanManageEmbeddingConfig, + assertEmbeddingExtensionAvailability, assertEmbeddingTargetSchema, + assertSchemaCanReceiveEmbeddings, assertSemanticSearchAccess, canManageEmbeddingConfig, + embeddingSourceHashField, + EMBEDDINGS_OWNER_MODULE, resolveAdminOperatorContext, resolveSourceFieldAllowlist, + type SchemaExtensionInfo, } from '../utils/schemaPolicy.js'; import { clampClientSearchLimit } from '../utils/clientSearchContext.js'; import { validateEmbeddingConfigInput } from '../utils/validateEmbeddingConfig.js'; @@ -75,12 +80,20 @@ import { sanitizeErrorMessage } from '../utils/redactConfig.js'; export interface DeclaredSchemaInfo { name: string; ownerModule?: string; + fields?: Record; + extensions?: SchemaExtensionInfo[]; } export interface SchemaInfo { name: string; fields: Record; - modelOptions?: { conduit?: { authorization?: { enabled?: boolean } } }; + modelOptions?: { + conduit?: { + cms?: { enabled?: boolean }; + permissions?: { extendable?: boolean }; + authorization?: { enabled?: boolean }; + }; + }; } export interface EmbeddingConfigRecord { @@ -179,7 +192,7 @@ export class EmbeddingsApi { targetField: string; provider?: string; model?: string; - dimensions: number; + dimensions?: number; similarity?: string; sourceFieldAllowlist?: string[]; enabled?: boolean; @@ -187,6 +200,8 @@ export class EmbeddingsApi { caller: EmbeddingsApiCaller, ): Promise<{ config: MappedEmbeddingConfig; warnings: string[] }> { const schema = await this.loadTargetSchema(request.schemaName, caller); + assertSchemaCanReceiveEmbeddings(schema); + const declared = await this.deps.declaredSchema(request.schemaName); const configDefaults = this.deps.currentConfig(); const { sourceFieldAllowlist: _allowlist, ...persisted } = validateEmbeddingConfigInput( @@ -198,9 +213,21 @@ export class EmbeddingsApi { platformAdmin: caller.platformAdmin === true, }), }, - { provider: configDefaults.defaultProvider }, + { + provider: configDefaults.defaultProvider, + providers: configDefaults.providers, + }, schema.fields, ); + assertEmbeddingExtensionAvailability({ + schemaName: persisted.schemaName, + targetField: persisted.targetField, + dimensions: persisted.dimensions, + similarity: persisted.similarity, + baseFields: declared?.fields, + compiledFields: schema.fields, + extensions: declared?.extensions, + }); const enabled = request.enabled ?? true; const capabilities = await this.deps.getVectorCapabilities(persisted.schemaName); let indexes = await this.deps.getVectorIndexes(persisted.schemaName); @@ -216,7 +243,7 @@ export class EmbeddingsApi { ); } if (capabilities.storage) { - await this.extendEmbeddingSchema(persisted); + await this.extendEmbeddingSchema(persisted, declared); } const provisioned = await this.provisionUpsertIndexes({ persisted, @@ -670,27 +697,36 @@ export class EmbeddingsApi { return config; } - private async extendEmbeddingSchema(persisted: { - schemaName: string; - targetField: string; - dimensions: number; - similarity: VectorSimilarity; - }) { + private async extendEmbeddingSchema( + persisted: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: VectorSimilarity; + }, + declared?: DeclaredSchemaInfo | null, + ) { + const hashField = embeddingSourceHashField(persisted.targetField); + const existing = + declared?.extensions?.find( + extension => extension.ownerModule === EMBEDDINGS_OWNER_MODULE, + )?.fields ?? {}; await this.deps.setSchemaExtension({ schemaName: persisted.schemaName, fields: { + ...existing, [persisted.targetField]: { type: TYPE.Vector, dimensions: persisted.dimensions, similarity: persisted.similarity, select: false, }, - [`${persisted.targetField}SourceHash`]: { + [hashField]: { type: TYPE.String, required: false, select: false, }, - }, + } as ConduitModel, }); } diff --git a/modules/embeddings/src/utils/providerConfig.test.ts b/modules/embeddings/src/utils/providerConfig.test.ts index a1ac9b921..f706af258 100644 --- a/modules/embeddings/src/utils/providerConfig.test.ts +++ b/modules/embeddings/src/utils/providerConfig.test.ts @@ -7,6 +7,9 @@ import { normalizeEmbeddingsConfig, normalizeProviderSettings, resolveProviderModelName, + assertConfiguredProvider, + resolveCatalogueDimensions, + resolveCatalogueModel, } from './providerConfig.js'; describe('provider model catalogue', () => { @@ -101,6 +104,41 @@ describe('provider model catalogue', () => { assert.equal(findProviderModel(normalized, 'missing'), undefined); }); + it('resolves configured providers and catalogue dimensions, rejecting mismatches', () => { + const providers = { + 'openai-compatible': { + models: [ + { name: 'small', dimensions: 1536 }, + { name: 'large', dimensions: 3072 }, + ], + defaultModel: 'small', + }, + }; + assert.equal( + assertConfiguredProvider(providers, 'openai-compatible').name, + 'openai-compatible', + ); + assert.throws( + () => assertConfiguredProvider(providers, 'missing'), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /not a configured provider/.test(err.message), + ); + assert.equal(resolveCatalogueModel(providers['openai-compatible']).name, 'small'); + assert.equal( + resolveCatalogueDimensions(providers['openai-compatible'].models![1], 0), + 3072, + ); + assert.throws( + () => resolveCatalogueDimensions(providers['openai-compatible'].models![0], 768), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /do not match catalogue dimensions/.test(err.message), + ); + }); + it('allows an empty catalogue and does not throw on incomplete legacy reads', () => { assert.deepEqual(normalizeProviderSettings({ endpoint: 'https://api.example/v1' }), { endpoint: 'https://api.example/v1', diff --git a/modules/embeddings/src/utils/providerConfig.ts b/modules/embeddings/src/utils/providerConfig.ts index 19df43c24..db36129d2 100644 --- a/modules/embeddings/src/utils/providerConfig.ts +++ b/modules/embeddings/src/utils/providerConfig.ts @@ -202,6 +202,57 @@ export function resolveProviderModelName( return provider?.models?.[0]?.name ?? ''; } +export function assertConfiguredProvider( + providers: Record | undefined, + requested?: string, +): { name: string; settings: EmbeddingProviderSettings } { + const name = trimName(requested); + if (!name) { + throw invalidProviderConfig('Embedding provider is required'); + } + const settings = providers?.[name]; + if (!settings) { + throw invalidProviderConfig( + `Embedding provider '${name}' is not a configured provider`, + ); + } + return { name, settings }; +} + +export function resolveCatalogueModel( + provider: EmbeddingProviderSettings | undefined, + requested?: string, +): EmbeddingProviderModel { + const name = resolveProviderModelName(provider, requested); + const model = findProviderModel(provider, name); + if (!model) { + throw invalidProviderConfig( + name + ? `Model '${name}' is not in the catalogue for this provider` + : 'Provider model catalogue has no selectable model', + ); + } + return model; +} + +export function resolveCatalogueDimensions( + model: EmbeddingProviderModel, + requested?: number, +): number { + if (requested == null || requested === 0) { + return model.dimensions; + } + if (!Number.isInteger(requested) || requested <= 0) { + throw invalidProviderConfig('dimensions must be a positive integer'); + } + if (requested !== model.dimensions) { + throw invalidProviderConfig( + `Requested dimensions ${requested} do not match catalogue dimensions ${model.dimensions} for model '${model.name}'`, + ); + } + return model.dimensions; +} + export function normalizeEmbeddingsConfig< T extends { providers?: Record; diff --git a/modules/embeddings/src/utils/schemaPolicy.test.ts b/modules/embeddings/src/utils/schemaPolicy.test.ts index 794ef2ab3..5771828ce 100644 --- a/modules/embeddings/src/utils/schemaPolicy.test.ts +++ b/modules/embeddings/src/utils/schemaPolicy.test.ts @@ -1,10 +1,12 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; +import { GrpcError, TYPE, VectorSimilarity } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { assertCanManageEmbeddingConfig, + assertEmbeddingExtensionAvailability, assertEmbeddingTargetSchema, + assertSchemaCanReceiveEmbeddings, assertSemanticSearchAccess, assertSourceFields, isDeniedEmbeddingSchema, @@ -159,4 +161,133 @@ describe('embedding schema and source policies', () => { err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, ); }); + + it('requires an enabled schema and extendable permissions, not CMS enablement alone', () => { + assert.doesNotThrow(() => + assertSchemaCanReceiveEmbeddings({ + name: 'Article', + modelOptions: { + conduit: { cms: { enabled: true }, permissions: { extendable: true } }, + }, + }), + ); + assert.doesNotThrow(() => + assertSchemaCanReceiveEmbeddings({ + name: 'User', + modelOptions: { conduit: { permissions: { extendable: true } } }, + }), + ); + assert.throws( + () => + assertSchemaCanReceiveEmbeddings({ + name: 'Article', + modelOptions: { + conduit: { cms: { enabled: true }, permissions: { extendable: false } }, + }, + }), + err => + err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not extendable/.test(err.message), + ); + assert.throws( + () => + assertSchemaCanReceiveEmbeddings({ + name: 'Article', + modelOptions: { + conduit: { cms: { enabled: true } }, + }, + }), + err => + err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not extendable/.test(err.message), + ); + assert.throws( + () => + assertSchemaCanReceiveEmbeddings({ + name: 'Article', + modelOptions: { + conduit: { cms: { enabled: false }, permissions: { extendable: true } }, + }, + }), + err => + err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not enabled/.test(err.message), + ); + }); + + it('rejects incompatible vector and hash collisions and allows compatible embeddings extensions', () => { + const proposed = { + schemaName: 'Article', + targetField: 'embedding', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + compiledFields: { + title: { type: TYPE.String }, + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, + embeddingSourceHash: { type: TYPE.String, required: false, select: false }, + }, + extensions: [ + { + ownerModule: 'embeddings', + fields: { + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, + embeddingSourceHash: { type: TYPE.String, required: false, select: false }, + }, + }, + ], + }; + assert.doesNotThrow(() => assertEmbeddingExtensionAvailability(proposed)); + assert.throws( + () => + assertEmbeddingExtensionAvailability({ + ...proposed, + compiledFields: { + title: { type: TYPE.String }, + embedding: { type: TYPE.String }, + }, + extensions: undefined, + }), + err => err instanceof GrpcError && err.code === status.ALREADY_EXISTS, + ); + assert.throws( + () => + assertEmbeddingExtensionAvailability({ + ...proposed, + dimensions: 8, + }), + err => err instanceof GrpcError && err.code === status.ALREADY_EXISTS, + ); + assert.throws( + () => + assertEmbeddingExtensionAvailability({ + ...proposed, + extensions: [ + { + ownerModule: 'chat', + fields: { + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + }, + }, + }, + ], + }), + err => err instanceof GrpcError && err.code === status.ALREADY_EXISTS, + ); + }); }); diff --git a/modules/embeddings/src/utils/schemaPolicy.ts b/modules/embeddings/src/utils/schemaPolicy.ts index cea44a1fe..0d45e1f75 100644 --- a/modules/embeddings/src/utils/schemaPolicy.ts +++ b/modules/embeddings/src/utils/schemaPolicy.ts @@ -70,6 +70,31 @@ export function isDeniedEmbeddingSchema(schema: { return AUTH_SECRET_SCHEMA_NAMES.has(schema.name); } +export function embeddingSourceHashField(targetField: string): string { + return `${targetField}SourceHash`; +} + +export interface EmbeddingSchemaOptions { + conduit?: { + cms?: { enabled?: boolean }; + permissions?: { extendable?: boolean }; + authorization?: { enabled?: boolean }; + }; +} + +export function isCmsEnabled(modelOptions?: EmbeddingSchemaOptions): boolean { + return modelOptions?.conduit?.cms?.enabled === true; +} + +export function isSchemaExtendable(modelOptions?: EmbeddingSchemaOptions): boolean { + return modelOptions?.conduit?.permissions?.extendable === true; +} + +export function isEmbeddingSchemaEnabled(modelOptions?: EmbeddingSchemaOptions): boolean { + if (modelOptions?.conduit?.cms == null) return true; + return isCmsEnabled(modelOptions); +} + export function assertEmbeddingTargetSchema(schema: { name: string; ownerModule?: string; @@ -81,6 +106,24 @@ export function assertEmbeddingTargetSchema(schema: { ); } +export function assertSchemaCanReceiveEmbeddings(schema: { + name: string; + modelOptions?: EmbeddingSchemaOptions; +}): void { + if (!isEmbeddingSchemaEnabled(schema.modelOptions)) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Schema '${schema.name}' is not enabled`, + ); + } + if (!isSchemaExtendable(schema.modelOptions)) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Schema '${schema.name}' is not extendable`, + ); + } +} + export function canManageEmbeddingConfig(args: { callerModule?: string; ownerModule?: string; @@ -196,3 +239,124 @@ export function assertSourceFields(args: { } } } + +export interface SchemaExtensionInfo { + ownerModule: string; + fields: Record; +} + +export interface EmbeddingExtensionField { + type: string; + dimensions?: number; + similarity?: string; + required?: boolean; + select?: boolean; +} + +export function assertEmbeddingExtensionAvailability(args: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + baseFields?: Record; + compiledFields: Record; + extensions?: SchemaExtensionInfo[]; +}): void { + const hashField = embeddingSourceHashField(args.targetField); + const proposed: Record = { + [args.targetField]: { + type: TYPE.Vector, + dimensions: args.dimensions, + similarity: args.similarity, + select: false, + }, + [hashField]: { + type: TYPE.String, + required: false, + select: false, + }, + }; + for (const [name, definition] of Object.entries(proposed)) { + assertExtensionFieldAvailable({ + schemaName: args.schemaName, + fieldName: name, + proposed: definition, + baseFields: args.baseFields, + compiledFields: args.compiledFields, + extensions: args.extensions ?? [], + }); + } +} + +function fieldOwner( + fieldName: string, + extensions: SchemaExtensionInfo[], +): SchemaExtensionInfo | undefined { + return extensions.find(extension => fieldName in (extension.fields ?? {})); +} + +function assertExtensionFieldAvailable(args: { + schemaName: string; + fieldName: string; + proposed: EmbeddingExtensionField; + baseFields?: Record; + compiledFields: Record; + extensions: SchemaExtensionInfo[]; +}): void { + const owned = fieldOwner(args.fieldName, args.extensions); + if (owned && owned.ownerModule !== EMBEDDINGS_OWNER_MODULE) { + throw extensionCollision(args.schemaName, args.fieldName); + } + if (owned?.ownerModule === EMBEDDINGS_OWNER_MODULE) { + if (!isCompatibleEmbeddingField(owned.fields[args.fieldName], args.proposed)) { + throw extensionCollision(args.schemaName, args.fieldName); + } + return; + } + if (args.baseFields && args.fieldName in args.baseFields) { + throw extensionCollision(args.schemaName, args.fieldName); + } + if (args.fieldName in args.compiledFields) { + if (!isCompatibleEmbeddingField(args.compiledFields[args.fieldName], args.proposed)) { + throw extensionCollision(args.schemaName, args.fieldName); + } + } +} + +function extensionCollision(schemaName: string, fieldName: string): GrpcError { + return new GrpcError( + status.ALREADY_EXISTS, + `Field '${fieldName}' already exists on schema '${schemaName}' and is not a compatible embeddings extension`, + ); +} + +function fieldType(field: unknown): string | undefined { + if (typeof field === 'string') return field; + if (!isRecord(field)) return undefined; + if (typeof field.type === 'string') return field.type; + return undefined; +} + +function isCompatibleEmbeddingField( + existing: unknown, + proposed: EmbeddingExtensionField, +): boolean { + const type = fieldType(existing); + if (type !== proposed.type) return false; + if (proposed.type === TYPE.Vector) { + if (!isRecord(existing)) return false; + if (existing.dimensions !== proposed.dimensions) return false; + if ( + typeof existing.similarity === 'string' && + existing.similarity !== proposed.similarity + ) { + return false; + } + return true; + } + if (proposed.type === TYPE.String) { + if (isRecord(existing) && existing.required === true) return false; + return isStringLikeField(existing); + } + return false; +} diff --git a/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts b/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts index c6654f8b7..04e4a9678 100644 --- a/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts +++ b/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts @@ -1,10 +1,22 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { GrpcError, VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; import { validateEmbeddingConfigInput } from './validateEmbeddingConfig.js'; describe('validateEmbeddingConfigInput', () => { - const defaults = { provider: 'openai-compatible' }; + const defaults = { + provider: 'openai-compatible', + providers: { + 'openai-compatible': { + models: [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + { name: 'text-embedding-3-large', dimensions: 3072 }, + ], + defaultModel: 'text-embedding-3-small', + }, + }, + }; const valid = { schemaName: 'Article', sourceFields: ['title', 'body'], @@ -23,13 +35,19 @@ describe('validateEmbeddingConfigInput', () => { ); assert.equal(result.similarity, VectorSimilarity.DotProduct); assert.equal(result.provider, 'openai-compatible'); + assert.equal(result.modelName, 'text-embedding-3-small'); assert.equal(result.dimensions, 1536); assert.deepEqual(result.sourceFieldAllowlist, []); }); - it('defaults omitted similarity to cosine', () => { - const result = validateEmbeddingConfigInput(valid, defaults); + it('defaults omitted similarity to cosine and omitted model to the catalogue default', () => { + const result = validateEmbeddingConfigInput( + { schemaName: 'Article', sourceFields: ['title'], targetField: 'embedding' }, + defaults, + ); assert.equal(result.similarity, VectorSimilarity.Cosine); + assert.equal(result.modelName, 'text-embedding-3-small'); + assert.equal(result.dimensions, 1536); }); it('rejects missing identity fields', () => { @@ -39,15 +57,47 @@ describe('validateEmbeddingConfigInput', () => { ); }); - it('rejects non-positive and non-integer dimensions', () => { + it('rejects unknown providers and models', () => { assert.throws( - () => validateEmbeddingConfigInput({ ...valid, dimensions: 0 }, defaults), - /positive integer/, + () => validateEmbeddingConfigInput({ ...valid, provider: 'missing' }, defaults), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /not a configured provider/.test(err.message), ); assert.throws( - () => validateEmbeddingConfigInput({ ...valid, dimensions: 12.3 }, defaults), - /positive integer/, + () => validateEmbeddingConfigInput({ ...valid, model: 'missing' }, defaults), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /not in the catalogue/.test(err.message), + ); + }); + + it('rejects explicit dimension mismatches and ignores omitted proto dimensions', () => { + assert.throws( + () => + validateEmbeddingConfigInput( + { ...valid, model: 'text-embedding-3-small', dimensions: 768 }, + defaults, + ), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /do not match catalogue dimensions/.test(err.message), + ); + const omitted = validateEmbeddingConfigInput( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'text-embedding-3-large', + dimensions: 0, + }, + defaults, ); + assert.equal(omitted.dimensions, 3072); + assert.equal(omitted.modelName, 'text-embedding-3-large'); }); it('rejects unsupported similarity values', () => { diff --git a/modules/embeddings/src/utils/validateEmbeddingConfig.ts b/modules/embeddings/src/utils/validateEmbeddingConfig.ts index 967e52164..de568f3e2 100644 --- a/modules/embeddings/src/utils/validateEmbeddingConfig.ts +++ b/modules/embeddings/src/utils/validateEmbeddingConfig.ts @@ -1,5 +1,11 @@ import { GrpcError, VectorSimilarity } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; +import type { EmbeddingProviderSettings } from '../config/index.js'; +import { + assertConfiguredProvider, + resolveCatalogueDimensions, + resolveCatalogueModel, +} from './providerConfig.js'; import { assertSourceFields } from './schemaPolicy.js'; export interface EmbeddingConfigInput { @@ -18,17 +24,22 @@ export interface ValidatedEmbeddingConfig { sourceFields: string[]; targetField: string; provider: string; - modelName: string | undefined; + modelName: string; dimensions: number; similarity: VectorSimilarity; sourceFieldAllowlist: string[]; } +export interface EmbeddingConfigDefaults { + provider: string; + providers: Record; +} + const SUPPORTED_SIMILARITY = Object.values(VectorSimilarity); export function validateEmbeddingConfigInput( request: EmbeddingConfigInput, - defaults: { provider: string }, + defaults: EmbeddingConfigDefaults, schemaFields?: Record, ): ValidatedEmbeddingConfig { if (!request.schemaName || !request.targetField || !request.sourceFields?.length) { @@ -43,14 +54,12 @@ export function validateEmbeddingConfigInput( if (!/^[A-Za-z_][A-Za-z0-9_]{0,127}$/.test(request.targetField)) { throw new GrpcError(status.INVALID_ARGUMENT, 'targetField is invalid'); } - const dimensions = request.dimensions; - if ( - typeof dimensions !== 'number' || - !Number.isInteger(dimensions) || - dimensions <= 0 - ) { - throw new GrpcError(status.INVALID_ARGUMENT, 'dimensions must be a positive integer'); - } + const { name: provider, settings: providerSettings } = assertConfiguredProvider( + defaults.providers, + request.provider || defaults.provider, + ); + const model = resolveCatalogueModel(providerSettings, request.model); + const dimensions = resolveCatalogueDimensions(model, request.dimensions); const similarity = request.similarity || VectorSimilarity.Cosine; if (!SUPPORTED_SIMILARITY.includes(similarity as VectorSimilarity)) { throw new GrpcError( @@ -72,8 +81,8 @@ export function validateEmbeddingConfigInput( schemaName: request.schemaName, sourceFields: request.sourceFields, targetField: request.targetField, - provider: request.provider || defaults.provider, - modelName: request.model, + provider, + modelName: model.name, dimensions, similarity: similarity as VectorSimilarity, sourceFieldAllowlist, From 247647ee8d6b859252769cf4d1a03cf1f1e83278 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Wed, 9 Sep 2026 23:05:46 +0300 Subject: [PATCH 24/29] test(embeddings): harden catalogue and settings contracts Cover Core GET/PATCH redaction, default-model selection, derived dimensions, extension collisions, and provider invocation against the operator catalogue shape the Admin UI consumes. --- .../embeddings/src/api/embeddingsApi.test.ts | 66 +++++++++++++++++ .../embeddings/src/providers/index.test.ts | 26 +++++++ .../embeddings/src/utils/mcpToolNames.test.ts | 19 +++++ .../src/utils/operationalStatus.test.ts | 9 +++ .../src/utils/providerConfig.test.ts | 10 +++ .../embeddings/src/utils/redactConfig.test.ts | 71 +++++++++++++++++++ .../embeddings/src/utils/schemaPolicy.test.ts | 12 ++++ .../test/embedding-contract.test.mjs | 28 ++++++++ 8 files changed, 241 insertions(+) diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts index 685480b90..5570c51a1 100644 --- a/modules/embeddings/src/api/embeddingsApi.test.ts +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -619,8 +619,13 @@ describe('typed embeddings API handlers', () => { }); it('runs semantic search with typed hits and fail-closed auth', async () => { + const embedCalls: Array<[string, string, string]> = []; const { api } = createApi({ configs: [enabledConfig], + embed: async (input, provider, model) => { + embedCalls.push([input, provider, model]); + return [0.1, 1.1, 2.1]; + }, vectorSearch: async input => { assert.equal(input.userId, 'user-1'); assert.equal(input.adminOperator, false); @@ -648,6 +653,9 @@ describe('typed embeddings API handlers', () => { { schemaName: 'Article', text: 'hello', userId: 'user-1' }, { callerModule: 'database' }, ); + assert.deepEqual(embedCalls, [ + ['hello', 'openai-compatible', 'text-embedding-3-small'], + ]); assert.equal(result.hits.length, 1); assert.equal(JSON.parse(result.hits[0].document)._id, 'doc1'); assert.equal(result.hits[0].score, 0.91); @@ -1155,6 +1163,64 @@ describe('typed embeddings API handlers', () => { assert.equal(saved.config.model, 'text-embedding-3-small'); }); + it('selects the catalogue default model when upsert omits model', async () => { + const { api, configs } = createApi({ + indexes: [readyIndex], + config: { + ...moduleConfig, + providers: { + 'openai-compatible': { + ...moduleConfig.providers['openai-compatible'], + defaultModel: 'text-embedding-3-large', + }, + }, + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + enabled: false, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.model, 'text-embedding-3-large'); + assert.equal(saved.config.dimensions, 3); + assert.equal(configs[0].modelName, 'text-embedding-3-large'); + }); + + it('reports catalogue readiness on status without leaking provider secrets', async () => { + const { api } = createApi({ + config: { + ...moduleConfig, + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-status', + models: [], + defaultModel: 'missing', + }, + }, + }, + }); + const statusResult = await api.getStatus(); + assert.equal(statusResult.ready, false); + assert.equal( + statusResult.warnings.some(warning => /model catalogue is empty/.test(warning)), + true, + ); + assert.equal(JSON.stringify(statusResult).includes('sk-status'), false); + const capabilities = await api.getCapabilities(); + assert.deepEqual(Object.keys(capabilities.capabilities).sort(), [ + 'indexing', + 'provider', + 'search', + 'storage', + 'supported', + ]); + }); + it('rejects CMS-enabled schemas that are not extendable', async () => { const { api, configs, createdIndexes, schemaExtensions } = createApi({ schemas: { diff --git a/modules/embeddings/src/providers/index.test.ts b/modules/embeddings/src/providers/index.test.ts index 788ef9dbf..7bc18ebe2 100644 --- a/modules/embeddings/src/providers/index.test.ts +++ b/modules/embeddings/src/providers/index.test.ts @@ -42,6 +42,8 @@ describe('openai-compatible provider security', () => { lookup: async () => [{ address: '104.18.0.1', family: 4 }], fetch: async (_url, init) => { assert.equal(init?.redirect, 'error'); + const headers = new Headers(init?.headers); + assert.equal(headers.get('authorization'), 'Bearer sk-test'); const body = JSON.parse(String(init?.body ?? '{}')) as { model?: string }; assert.equal(body.model, 'text-embedding-3-small'); return new Response(JSON.stringify({ data: [{ embedding: [0.1, 0.2] }] }), { @@ -56,4 +58,28 @@ describe('openai-compatible provider security', () => { }); assert.deepEqual(vector, [0.1, 0.2]); }); + + it('maps provider HTTP failures without leaking the API key', async () => { + const failing = new OpenAICompatibleEmbeddingProvider({ + lookup: async () => [{ address: '104.18.0.1', family: 4 }], + fetch: async () => + new Response('invalid apiKey=sk-test', { + status: 401, + headers: { 'content-type': 'text/plain' }, + }), + }); + await assert.rejects( + () => + failing.embed('hello', { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-test', + model: 'text-embedding-3-small', + }), + err => + err instanceof GrpcError && + err.code === status.UNAVAILABLE && + /HTTP 401/.test(err.message) && + !err.message.includes('sk-test'), + ); + }); }); diff --git a/modules/embeddings/src/utils/mcpToolNames.test.ts b/modules/embeddings/src/utils/mcpToolNames.test.ts index 826bfc364..f7f8d75e4 100644 --- a/modules/embeddings/src/utils/mcpToolNames.test.ts +++ b/modules/embeddings/src/utils/mcpToolNames.test.ts @@ -1,8 +1,10 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { ConduitRouteActions } from '@conduitplatform/grpc-sdk'; +import { ConduitNumber, ConduitString } from '@conduitplatform/module-tools'; import { embeddingsMcpToolName, embeddingsPublicPath } from './mcpToolNames.js'; import { + CONFIG_BODY, EMBEDDINGS_ADMIN_ROUTES, EMBEDDINGS_CLIENT_FORBIDDEN_PATHS, EMBEDDINGS_CLIENT_SEARCH_PATH, @@ -70,4 +72,21 @@ describe('embeddings MCP tool names', () => { } assert.equal(EMBEDDINGS_CLIENT_SEARCH_PATH, '/search'); }); + + it('accepts optional catalogue fields on Admin config upsert', () => { + assert.deepEqual(Object.keys(CONFIG_BODY), [ + 'schemaName', + 'sourceFields', + 'targetField', + 'provider', + 'model', + 'dimensions', + 'similarity', + 'sourceFieldAllowlist', + 'enabled', + ]); + assert.equal(CONFIG_BODY.model, ConduitString.Optional); + assert.equal(CONFIG_BODY.dimensions, ConduitNumber.Optional); + assert.equal(CONFIG_BODY.provider, ConduitString.Optional); + }); }); diff --git a/modules/embeddings/src/utils/operationalStatus.test.ts b/modules/embeddings/src/utils/operationalStatus.test.ts index cfab49aab..92a9dd2d0 100644 --- a/modules/embeddings/src/utils/operationalStatus.test.ts +++ b/modules/embeddings/src/utils/operationalStatus.test.ts @@ -39,6 +39,15 @@ describe('embeddings operational warnings and search gates', () => { true, ); assert.equal(warnings.join(' ').includes('sk-secret'), false); + assert.deepEqual( + providerReadinessWarnings({ + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-secret', + models: [{ name: 'small', dimensions: 1536 }], + defaultModel: 'missing', + }), + ["Provider default model 'missing' is not in the catalogue"], + ); assert.equal( isEmbeddingsReady({ moduleEnabled: false, diff --git a/modules/embeddings/src/utils/providerConfig.test.ts b/modules/embeddings/src/utils/providerConfig.test.ts index f706af258..3bae44115 100644 --- a/modules/embeddings/src/utils/providerConfig.test.ts +++ b/modules/embeddings/src/utils/providerConfig.test.ts @@ -102,6 +102,16 @@ describe('provider model catalogue', () => { assert.equal(resolveProviderModelName(normalized), 'small'); assert.equal(findProviderModel(normalized, 'large')?.dimensions, 3072); assert.equal(findProviderModel(normalized, 'missing'), undefined); + const preferred = normalizeProviderSettings({ + models: [ + { name: 'small', dimensions: 1536 }, + { name: 'large', dimensions: 3072 }, + ], + defaultModel: 'large', + }); + assert.equal(resolveProviderModelName(preferred), 'large'); + assert.equal(resolveCatalogueModel(preferred).name, 'large'); + assert.equal(resolveCatalogueDimensions(resolveCatalogueModel(preferred)), 3072); }); it('resolves configured providers and catalogue dimensions, rejecting mismatches', () => { diff --git a/modules/embeddings/src/utils/redactConfig.test.ts b/modules/embeddings/src/utils/redactConfig.test.ts index e0538ca4d..3c82951ba 100644 --- a/modules/embeddings/src/utils/redactConfig.test.ts +++ b/modules/embeddings/src/utils/redactConfig.test.ts @@ -1,8 +1,27 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { redactSensitiveConfig } from '@conduitplatform/module-tools'; +import AppConfigSchema from '../config/index.js'; +import { normalizeEmbeddingsConfig } from './providerConfig.js'; import { redactSecretText } from './redactConfig.js'; +function assertNoLegacySettingsSurface(config: unknown) { + const serialized = JSON.stringify(config); + assert.doesNotMatch(serialized, /requireGrpcKey/); + assert.doesNotMatch(serialized, /allowedHosts/); + const providers = (config as { providers?: Record> }) + .providers; + for (const provider of Object.values(providers ?? {})) { + assert.equal('model' in provider, false); + assert.equal('allowedHosts' in provider, false); + assert.equal('dimensions' in provider, false); + } + const security = (config as { security?: Record }).security; + if (security) { + assert.equal('requireGrpcKey' in security, false); + } +} + describe('provider secret redaction', () => { it('redacts API keys from config objects and error text', () => { assert.equal( @@ -52,4 +71,56 @@ describe('provider secret redaction', () => { { name: 'text-embedding-3-small', dimensions: 1536 }, ]); }); + + it('keeps Core Admin GET/PATCH catalogue shape while redacting keys', () => { + const legacy = { + enabled: true, + defaultProvider: 'openai-compatible', + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-live', + model: 'text-embedding-3-small', + dimensions: 1536, + allowedHosts: ['api.openai.com'], + }, + }, + security: { + requireGrpcKey: true, + sourceFieldAllowlist: ['summary'], + }, + }; + const patched = normalizeEmbeddingsConfig(legacy); + const getResponse = redactSensitiveConfig(patched, AppConfigSchema); + const monoResponse = redactSensitiveConfig(patched); + for (const redacted of [getResponse, monoResponse]) { + const provider = redacted.providers['openai-compatible'] as { + apiKey?: string; + endpoint?: string; + models?: Array<{ name: string; dimensions: number }>; + defaultModel?: string; + }; + assert.equal(provider.apiKey, '[REDACTED]'); + assert.doesNotMatch(JSON.stringify(redacted), /sk-live/); + assert.equal(provider.endpoint, 'https://api.openai.com/v1/embeddings'); + assert.deepEqual(provider.models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + assert.equal(provider.defaultModel, 'text-embedding-3-small'); + assertNoLegacySettingsSurface(redacted); + } + const emptyKey = redactSensitiveConfig( + normalizeEmbeddingsConfig({ + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: '', + models: [{ name: 'text-embedding-3-small', dimensions: 1536 }], + }, + }, + }), + AppConfigSchema, + ); + assert.equal(emptyKey.providers['openai-compatible'].apiKey, ''); + }); }); diff --git a/modules/embeddings/src/utils/schemaPolicy.test.ts b/modules/embeddings/src/utils/schemaPolicy.test.ts index 5771828ce..1a8ab2cbc 100644 --- a/modules/embeddings/src/utils/schemaPolicy.test.ts +++ b/modules/embeddings/src/utils/schemaPolicy.test.ts @@ -289,5 +289,17 @@ describe('embedding schema and source policies', () => { }), err => err instanceof GrpcError && err.code === status.ALREADY_EXISTS, ); + assert.throws( + () => + assertEmbeddingExtensionAvailability({ + ...proposed, + compiledFields: { + title: { type: TYPE.String }, + embeddingSourceHash: { type: TYPE.Number }, + }, + extensions: undefined, + }), + err => err instanceof GrpcError && err.code === status.ALREADY_EXISTS, + ); }); }); diff --git a/modules/embeddings/test/embedding-contract.test.mjs b/modules/embeddings/test/embedding-contract.test.mjs index f51b41a3d..442bdfe8a 100644 --- a/modules/embeddings/test/embedding-contract.test.mjs +++ b/modules/embeddings/test/embedding-contract.test.mjs @@ -11,6 +11,10 @@ const sdkSource = readFileSync( new URL('../../../libraries/grpc-sdk/src/modules/embeddings/index.ts', import.meta.url), 'utf8', ); +const adminRoutesSource = readFileSync( + new URL('../src/admin/routes.ts', import.meta.url), + 'utf8', +); test('embeddings proto exposes typed config, status, backfill, and search RPCs', () => { assert.match( @@ -75,6 +79,30 @@ test('grpc-sdk embeddings client maps typed proto messages instead of JSON-strin assert.match(sdkSource, /resumeBackfill\(/); assert.doesNotMatch(sdkSource, /JSON\.parse\(res\.result\)/); assert.match(sdkSource, /JSON\.parse\(hit\.document\)/); + assert.match( + protoSource, + /message EmbeddingConfig \{\n string id = 1;[\s\S]*string model = 6;[\s\S]*int32 dimensions = 7;/, + ); + assert.doesNotMatch(protoSource, /string modelName/); + assert.doesNotMatch(protoSource, /string _id/); + assert.match( + protoSource, + /message UpsertConfigRequest \{[\s\S]*string model = 5;[\s\S]*int32 dimensions = 6;[\s\S]*optional bool enabled = 9;/, + ); + assert.match( + protoSource, + /message GetStatusResponse \{\n bool enabled = 1;\n bool ready = 2;\n VectorCapabilities capabilities = 3;/, + ); + assert.match( + sdkSource, + /export interface EmbeddingConfigRecord \{\n id: string;[\s\S]*model: string;[\s\S]*dimensions: number;/, + ); + assert.match( + sdkSource, + /export interface EmbeddingConfigInput \{[\s\S]*model\?: string;[\s\S]*dimensions\?: number;/, + ); + assert.match(adminRoutesSource, /model: ConduitString\.Optional/); + assert.match(adminRoutesSource, /dimensions: ConduitNumber\.Optional/); }); test('deployment docs describe provider configuration and rollout workflow', () => { From 09ba8a38ec107f0edea4918f0332270b0b6c21ac Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Wed, 9 Sep 2026 23:19:47 +0300 Subject: [PATCH 25/29] refactor(embeddings): simplify catalogue normalization helpers Flatten default-model selection and drop control-flow try/catch without changing catalogue migration. --- modules/embeddings/README.md | 5 +- .../src/utils/providerConfig.test.ts | 18 ++-- .../embeddings/src/utils/providerConfig.ts | 95 ++++++++----------- 3 files changed, 52 insertions(+), 66 deletions(-) diff --git a/modules/embeddings/README.md b/modules/embeddings/README.md index 2078810fc..f1731704e 100644 --- a/modules/embeddings/README.md +++ b/modules/embeddings/README.md @@ -37,8 +37,9 @@ Enable it and configure an OpenAI-compatible provider: ## Workflow -1. Create an embedding config with `schemaName`, `sourceFields`, `targetField`, - `provider`, `model`, and `dimensions`. The first upsert provisions the vector +1. Create an embedding config with `schemaName`, `sourceFields`, and + `targetField`. `provider`, `model`, and `dimensions` default from the + provider catalogue when omitted. The first upsert provisions the vector index when Database indexing is available. The config stays disabled until Database reports a queryable index. If indexing is unavailable, status returns a manual index lifecycle warning. diff --git a/modules/embeddings/src/utils/providerConfig.test.ts b/modules/embeddings/src/utils/providerConfig.test.ts index 3bae44115..d1903b04e 100644 --- a/modules/embeddings/src/utils/providerConfig.test.ts +++ b/modules/embeddings/src/utils/providerConfig.test.ts @@ -135,13 +135,12 @@ describe('provider model catalogue', () => { err.code === status.INVALID_ARGUMENT && /not a configured provider/.test(err.message), ); - assert.equal(resolveCatalogueModel(providers['openai-compatible']).name, 'small'); - assert.equal( - resolveCatalogueDimensions(providers['openai-compatible'].models![1], 0), - 3072, - ); + const openai = providers['openai-compatible']; + const [small, large] = openai.models; + assert.equal(resolveCatalogueModel(openai).name, 'small'); + assert.equal(resolveCatalogueDimensions(large, 0), 3072); assert.throws( - () => resolveCatalogueDimensions(providers['openai-compatible'].models![0], 768), + () => resolveCatalogueDimensions(small, 768), err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT && @@ -174,12 +173,9 @@ describe('provider model catalogue', () => { }, }, }); - assert.equal( - 'requireGrpcKey' in (normalized.security as Record), - false, - ); + assert.equal('requireGrpcKey' in normalized.security, false); assert.deepEqual( - (normalized.providers['openai-compatible'] as { models?: unknown }).models, + normalizeProviderSettings(normalized.providers['openai-compatible']).models, [{ name: 'text-embedding-3-small', dimensions: 1536 }], ); }); diff --git a/modules/embeddings/src/utils/providerConfig.ts b/modules/embeddings/src/utils/providerConfig.ts index db36129d2..4d0fcab27 100644 --- a/modules/embeddings/src/utils/providerConfig.ts +++ b/modules/embeddings/src/utils/providerConfig.ts @@ -7,12 +7,6 @@ import type { export type { EmbeddingProviderModel, EmbeddingProviderSettings }; -export interface LegacyEmbeddingProviderSettings extends EmbeddingProviderSettings { - model?: string; - dimensions?: number; - allowedHosts?: string[]; -} - function invalidProviderConfig(message: string): GrpcError { return new GrpcError(status.INVALID_ARGUMENT, message); } @@ -57,29 +51,40 @@ function parseModelList( ): EmbeddingProviderModel[] { const models: EmbeddingProviderModel[] = []; const names = new Set(); + const strict = options?.strict !== false; for (const [index, value] of values.entries()) { - try { - const parsed = parseModelEntry(value, index); - if (names.has(parsed.name)) { + const parsed = strict ? parseModelEntry(value, index) : optionalModelEntry(value); + if (!parsed) continue; + if (names.has(parsed.name)) { + if (strict) { throw invalidProviderConfig(`Provider model '${parsed.name}' is duplicated`); } - names.add(parsed.name); - models.push(parsed); - } catch (err) { - if (options?.strict !== false) throw err; + continue; } + names.add(parsed.name); + models.push(parsed); } return models; } +function optionalModelEntry(value: unknown): EmbeddingProviderModel | undefined { + if (!isRecord(value)) return undefined; + const name = trimName(value.name); + const dimensions = parseDimensions(value.dimensions); + if (!name || dimensions == null) return undefined; + return { name, dimensions }; +} + function migrateLegacyModels( - raw: LegacyEmbeddingProviderSettings, + raw: Record, + options?: { strict?: boolean }, ): EmbeddingProviderModel[] | undefined { if (Array.isArray(raw.models) && raw.models.length > 0) return undefined; const name = trimName(raw.model); if (!name) return []; const dimensions = parseDimensions(raw.dimensions); if (dimensions == null) { + if (options?.strict === false) return []; throw invalidProviderConfig( `Provider model '${name}' dimensions must be a positive integer`, ); @@ -123,57 +128,41 @@ export function providerCatalogueIssues(provider?: { return issues; } -export function assertProviderCatalogue( - provider?: Pick, -): void { - const models = provider?.models ?? []; - if (!Array.isArray(models)) { - throw invalidProviderConfig('Provider model catalogue must be an array'); - } - parseModelList(models); - const names = new Set(models.map(model => trimName(model?.name))); - const defaultModel = trimName(provider?.defaultModel); - if (defaultModel && !names.has(defaultModel)) { - throw invalidProviderConfig( - `Provider default model '${defaultModel}' is not in the catalogue`, - ); +function resolveDefaultModelName( + models: EmbeddingProviderModel[], + requested: unknown, + migratedFromSingular: boolean, +): string { + const configured = trimName(requested); + if (configured) { + return models.some(model => model.name === configured) ? configured : ''; } + if (migratedFromSingular) return models[0]?.name ?? ''; + return ''; } export function normalizeProviderSettings( raw: unknown, options?: { strict?: boolean }, ): EmbeddingProviderSettings { - const source = isRecord(raw) ? (raw as LegacyEmbeddingProviderSettings) : {}; - let models: EmbeddingProviderModel[]; - let migratedFromSingular = false; - try { - const migrated = migrateLegacyModels(source); - migratedFromSingular = migrated != null && migrated.length > 0; - models = - migrated ?? - (Array.isArray(source.models) ? parseModelList(source.models, options) : []); - } catch (err) { - if (options?.strict !== false) throw err; - models = []; - } + const source = isRecord(raw) ? raw : {}; + const migrated = migrateLegacyModels(source, options); + const migratedFromSingular = migrated != null && migrated.length > 0; + const models = + migrated ?? + (Array.isArray(source.models) ? parseModelList(source.models, options) : []); const names = new Set(models.map(model => model.name)); const configuredDefault = trimName(source.defaultModel); - const defaultModel = configuredDefault - ? names.has(configuredDefault) - ? configuredDefault - : '' - : migratedFromSingular - ? (models[0]?.name ?? '') - : ''; - if (configuredDefault && !defaultModel && options?.strict !== false) { + if (configuredDefault && !names.has(configuredDefault) && options?.strict !== false) { throw invalidProviderConfig( `Provider default model '${configuredDefault}' is not in the catalogue`, ); } - if (options?.strict !== false) { - assertProviderCatalogue({ models, defaultModel }); - } + const defaultModel = resolveDefaultModelName( + models, + source.defaultModel, + migratedFromSingular, + ); return { ...(typeof source.endpoint === 'string' ? { endpoint: source.endpoint } : {}), ...(typeof source.apiKey === 'string' ? { apiKey: source.apiKey } : {}), @@ -262,7 +251,7 @@ export function normalizeEmbeddingsConfig< const next = { ...config }; if (isRecord(next.security)) { const security = { ...next.security }; - delete (security as { requireGrpcKey?: boolean }).requireGrpcKey; + delete security.requireGrpcKey; next.security = security as T['security']; } if (isRecord(next.providers)) { From 0a6756acda3e8f8d346af9721e5d7480eb7847f5 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Wed, 9 Sep 2026 23:49:56 +0300 Subject: [PATCH 26/29] fix(embeddings): persist catalogue migrations and default empty vector methods Treat empty proto method and missing Mongo indexingMethod as hnsw so live indexes are reused instead of recreated as _vN. Persist migrated catalogue-only provider settings on module config lifecycle so a later unrelated Admin PATCH cannot wipe them. --- libraries/grpc-sdk/package.json | 3 +- libraries/grpc-sdk/src/interfaces/Model.ts | 14 + .../src/interfaces/vectorIndexMethod.test.ts | 22 + .../grpc-sdk/src/modules/database/index.ts | 5 +- libraries/module-tools/package.json | 3 +- libraries/module-tools/src/ManagedModule.ts | 21 +- libraries/module-tools/src/utilities/index.ts | 1 + .../utilities/reconcileModuleConfig.test.ts | 67 ++ .../src/utilities/reconcileModuleConfig.ts | 51 ++ .../src/utilities/redactSensitiveConfig.ts | 37 +- modules/database/package.bundle-lock.json | 26 +- modules/database/src/Database.ts | 6 +- .../__tests__/vectorIndexAdapters.test.ts | 69 +++ .../__tests__/vectorIndexLifecycle.test.ts | 41 ++ .../utils/__tests__/vectorMappings.test.ts | 56 ++ .../adapters/utils/vectorIndexLifecycle.ts | 13 +- .../src/adapters/utils/vectorMappings.ts | 7 +- modules/embeddings/package.bundle-lock.json | 35 +- .../embeddings/src/api/embeddingsApi.test.ts | 33 + modules/embeddings/src/api/embeddingsApi.ts | 3 + .../src/utils/backfillGates.test.ts | 8 + .../embeddings/src/utils/configChange.test.ts | 33 + modules/embeddings/src/utils/configChange.ts | 9 +- .../src/utils/moduleConfigLifecycle.test.ts | 75 +++ .../src/utils/operationalStatus.test.ts | 56 ++ .../embeddings/src/utils/redactConfig.test.ts | 23 +- packages/core/package.bundle-lock.json | 574 +++++++++--------- packages/core/package.bundle.json | 4 +- packages/core/package.json | 3 +- .../tests/embeddingsConfigLifecycle.test.ts | 186 ++++++ 30 files changed, 1144 insertions(+), 340 deletions(-) create mode 100644 libraries/grpc-sdk/src/interfaces/vectorIndexMethod.test.ts create mode 100644 libraries/module-tools/src/utilities/reconcileModuleConfig.test.ts create mode 100644 libraries/module-tools/src/utilities/reconcileModuleConfig.ts create mode 100644 modules/embeddings/src/utils/moduleConfigLifecycle.test.ts create mode 100644 packages/core/tests/embeddingsConfigLifecycle.test.ts diff --git a/libraries/grpc-sdk/package.json b/libraries/grpc-sdk/package.json index 75f534051..9cce9f555 100644 --- a/libraries/grpc-sdk/package.json +++ b/libraries/grpc-sdk/package.json @@ -25,7 +25,8 @@ "prepublish": "npm run build", "prebuild": "npm run protoc", "build": "rimraf dist && tsup", - "protoc": "sh build.sh" + "protoc": "sh build.sh", + "test": "node --experimental-strip-types --test src/interfaces/vectorIndexMethod.test.ts" }, "license": "MIT", "dependencies": { diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 957d3afb3..894ca419e 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -38,6 +38,20 @@ export enum VectorIndexMethod { Flat = 'flat', } +export function defaultVectorIndexMethod(method?: string | null): VectorIndexMethod { + if (method == null || method === '') { + return VectorIndexMethod.HNSW; + } + return method as VectorIndexMethod; +} + +export function vectorIndexMethodsEquivalent( + left?: string | null, + right?: string | null, +): boolean { + return defaultVectorIndexMethod(left) === defaultVectorIndexMethod(right); +} + export enum VectorIndexStatus { Pending = 'pending', Ready = 'ready', diff --git a/libraries/grpc-sdk/src/interfaces/vectorIndexMethod.test.ts b/libraries/grpc-sdk/src/interfaces/vectorIndexMethod.test.ts new file mode 100644 index 000000000..b726f95e2 --- /dev/null +++ b/libraries/grpc-sdk/src/interfaces/vectorIndexMethod.test.ts @@ -0,0 +1,22 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + defaultVectorIndexMethod, + VectorIndexMethod, + vectorIndexMethodsEquivalent, +} from '../../dist/index.esm.js'; + +describe('vector index method defaults', () => { + it('treats empty proto method and missing method as hnsw', () => { + assert.equal(defaultVectorIndexMethod(), VectorIndexMethod.HNSW); + assert.equal(defaultVectorIndexMethod(undefined), VectorIndexMethod.HNSW); + assert.equal(defaultVectorIndexMethod(null), VectorIndexMethod.HNSW); + assert.equal(defaultVectorIndexMethod(''), VectorIndexMethod.HNSW); + assert.equal(defaultVectorIndexMethod('hnsw'), VectorIndexMethod.HNSW); + assert.equal(defaultVectorIndexMethod('ivfflat'), VectorIndexMethod.IVFFlat); + assert.equal(defaultVectorIndexMethod('flat'), VectorIndexMethod.Flat); + assert.equal(vectorIndexMethodsEquivalent('', undefined), true); + assert.equal(vectorIndexMethodsEquivalent('hnsw', ''), true); + assert.equal(vectorIndexMethodsEquivalent('flat', ''), false); + }); +}); diff --git a/libraries/grpc-sdk/src/modules/database/index.ts b/libraries/grpc-sdk/src/modules/database/index.ts index 2b3b42ffb..c26e86a7b 100644 --- a/libraries/grpc-sdk/src/modules/database/index.ts +++ b/libraries/grpc-sdk/src/modules/database/index.ts @@ -20,6 +20,7 @@ import type { VectorSearchInput, VectorSearchResult, } from '../../interfaces/Model.js'; +import { defaultVectorIndexMethod } from '../../interfaces/Model.js'; export type CountDocumentsOptions = AuthzOptions & { readPreference?: string }; @@ -355,7 +356,7 @@ export class DatabaseProvider extends ConduitModule extends ConduitServiceModule { }); } let config = JSON.parse(call.request.newConfig); - config = merge(this.config.getProperties(), config); - config = await this.preConfig(config); const previousConfig = this.config.getProperties(); + config = merge(previousConfig, config); + config = restoreRedactedSecrets(config, previousConfig, this.configSchema); + config = await this.preConfig(config); try { this.config.load(config).validate({ allowed: 'warn', @@ -354,8 +357,20 @@ export abstract class ManagedModule extends ConduitServiceModule { ConfigController.getInstance(); if (config) { + const migrated = await this.preConfig(config); + this.config.load(migrated).validate({ + allowed: 'warn', + }); + const persistable = this.config.getProperties(); + const reconciled = await reconcileStoredModuleConfig({ + stored: config, + migrated: persistable, + configureOverride: next => + this.grpcSdk.config.configure(next, convictConfigParser(configSchema), true), + }); + config = reconciled.config; this.config.load(config); - ConfigController.getInstance().config = config; + ConfigController.getInstance().config = this.config.getProperties(); } if (!config || config.active || !config.hasOwnProperty('active')) await this.onConfig(); diff --git a/libraries/module-tools/src/utilities/index.ts b/libraries/module-tools/src/utilities/index.ts index 3d07b84f1..20d948cf4 100644 --- a/libraries/module-tools/src/utilities/index.ts +++ b/libraries/module-tools/src/utilities/index.ts @@ -5,3 +5,4 @@ export * from './exportHelpers.js'; export * from './conduitPeers.js'; export * from './convictConfigParser.js'; export * from './redactSensitiveConfig.js'; +export * from './reconcileModuleConfig.js'; diff --git a/libraries/module-tools/src/utilities/reconcileModuleConfig.test.ts b/libraries/module-tools/src/utilities/reconcileModuleConfig.test.ts new file mode 100644 index 000000000..fca1b198b --- /dev/null +++ b/libraries/module-tools/src/utilities/reconcileModuleConfig.test.ts @@ -0,0 +1,67 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + containsRedactedMarker, + reconcileStoredModuleConfig, + storedConfigsEquivalent, +} from '../../dist/index.esm.js'; + +describe('stored module config reconciliation', () => { + it('persists migrated config once and skips equivalent follow-ups', async () => { + const stored = { + providers: { + 'openai-compatible': { + model: 'text-embedding-3-small', + dimensions: 1536, + models: [], + }, + }, + }; + const migrated = { + providers: { + 'openai-compatible': { + models: [{ name: 'text-embedding-3-small', dimensions: 1536 }], + defaultModel: 'text-embedding-3-small', + }, + }, + }; + let overrideCalls = 0; + const first = await reconcileStoredModuleConfig({ + stored, + migrated, + configureOverride: async config => { + overrideCalls += 1; + return config; + }, + }); + assert.equal(first.persisted, true); + assert.equal(overrideCalls, 1); + assert.equal(storedConfigsEquivalent(first.config, migrated), true); + + const second = await reconcileStoredModuleConfig({ + stored: first.config, + migrated, + configureOverride: async config => { + overrideCalls += 1; + return config; + }, + }); + assert.equal(second.persisted, false); + assert.equal(overrideCalls, 1); + }); + + it('does not persist redacted secrets', async () => { + let overrideCalls = 0; + const result = await reconcileStoredModuleConfig({ + stored: { apiKey: 'sk-live' }, + migrated: { apiKey: '[REDACTED]' }, + configureOverride: async config => { + overrideCalls += 1; + return config; + }, + }); + assert.equal(result.persisted, false); + assert.equal(overrideCalls, 0); + assert.equal(containsRedactedMarker({ apiKey: '[REDACTED]' }), true); + }); +}); diff --git a/libraries/module-tools/src/utilities/reconcileModuleConfig.ts b/libraries/module-tools/src/utilities/reconcileModuleConfig.ts new file mode 100644 index 000000000..9183e462e --- /dev/null +++ b/libraries/module-tools/src/utilities/reconcileModuleConfig.ts @@ -0,0 +1,51 @@ +const REDACTED_MARKER = '[REDACTED]'; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function sortJson(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(sortJson); + } + if (!isRecord(value)) { + return value; + } + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, nested]) => [key, sortJson(nested)]), + ); +} + +export function stableConfigJson(value: unknown): string { + return JSON.stringify(sortJson(value)); +} + +export function storedConfigsEquivalent(left: unknown, right: unknown): boolean { + return stableConfigJson(left) === stableConfigJson(right); +} + +export function containsRedactedMarker(value: unknown): boolean { + if (value === REDACTED_MARKER) return true; + if (Array.isArray(value)) { + return value.some(containsRedactedMarker); + } + if (!isRecord(value)) return false; + return Object.values(value).some(containsRedactedMarker); +} + +export async function reconcileStoredModuleConfig(args: { + stored: T; + migrated: T; + configureOverride: (config: T) => Promise; +}): Promise<{ config: T; persisted: boolean }> { + if (storedConfigsEquivalent(args.stored, args.migrated)) { + return { config: args.migrated, persisted: false }; + } + if (containsRedactedMarker(args.migrated)) { + return { config: args.migrated, persisted: false }; + } + const persisted = await args.configureOverride(args.migrated); + return { config: persisted, persisted: true }; +} diff --git a/libraries/module-tools/src/utilities/redactSensitiveConfig.ts b/libraries/module-tools/src/utilities/redactSensitiveConfig.ts index b6211eb5a..adb289098 100644 --- a/libraries/module-tools/src/utilities/redactSensitiveConfig.ts +++ b/libraries/module-tools/src/utilities/redactSensitiveConfig.ts @@ -24,6 +24,8 @@ function unwrapSchema(schema: unknown): unknown { return schema; } +const REDACTED_MARKER = '[REDACTED]'; + export function redactSensitiveConfig(config: T, schema?: unknown): T { if (!isRecord(config)) return config; const redacted = Array.isArray(config) ? [...config] : { ...config }; @@ -32,7 +34,7 @@ export function redactSensitiveConfig(config: T, schema?: unknown): T { const childSchema = isRecord(node) ? node[key] : undefined; if (isSensitiveLeaf(childSchema) || WELL_KNOWN_SECRET_KEYS.test(key)) { if (typeof value === 'string' && value.length > 0) { - (redacted as Record)[key] = '[REDACTED]'; + (redacted as Record)[key] = REDACTED_MARKER; } continue; } @@ -45,3 +47,36 @@ export function redactSensitiveConfig(config: T, schema?: unknown): T { } return redacted as T; } + +export function restoreRedactedSecrets(incoming: T, current: T, schema?: unknown): T { + if (Array.isArray(incoming) && Array.isArray(current)) { + return incoming.map((item, index) => + restoreRedactedSecrets(item, current[index], schema), + ) as T; + } + if (!isRecord(incoming) || !isRecord(current)) return incoming; + const restored: Record = { ...incoming }; + const node = unwrapSchema(schema); + for (const [key, value] of Object.entries(restored)) { + const childSchema = isRecord(node) ? node[key] : undefined; + const currentValue = current[key]; + if (isSensitiveLeaf(childSchema) || WELL_KNOWN_SECRET_KEYS.test(key)) { + if ( + value === REDACTED_MARKER && + typeof currentValue === 'string' && + currentValue.length > 0 && + currentValue !== REDACTED_MARKER + ) { + restored[key] = currentValue; + } + continue; + } + if ( + (isRecord(value) || Array.isArray(value)) && + (isRecord(currentValue) || Array.isArray(currentValue)) + ) { + restored[key] = restoreRedactedSecrets(value, currentValue, childSchema); + } + } + return restored as T; +} diff --git a/modules/database/package.bundle-lock.json b/modules/database/package.bundle-lock.json index 85d67f30a..50750c69e 100644 --- a/modules/database/package.bundle-lock.json +++ b/modules/database/package.bundle-lock.json @@ -169,9 +169,9 @@ } }, "node_modules/@mongodb-js/saslprep": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.5.2.tgz", - "integrity": "sha512-UBvCBdPHAmiiDNEpD2ORBGzna4qYr06fLELfGDPW3/KZ9uLK+cGT9npAc/x/6/8SLzdbILMuc3HWFQ0FvJMhCw==", + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.5.4.tgz", + "integrity": "sha512-05UC0jQsjKAOuXQ0H9Ud9vUTJpZIg+n/FinpR30tI5I8pY2inTfPOZ5OF/cg3Ce/N9MoD1xhRCeOsJtuTbFYlw==", "license": "MIT", "dependencies": { "sparse-bitfield": "^3.0.3" @@ -643,12 +643,12 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "26.5.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.5.0.tgz", - "integrity": "sha512-dVSGpriSoCgz8WnDNTuSSuSv1PC/ALXihO4ulRZt7Md8k9mlbdin3lGOcDE8SnWOgf513ByWlXd7BK4azmyg/A==", + "version": "22.20.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.2.tgz", + "integrity": "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==", "license": "MIT", "dependencies": { - "undici-types": "~8.9.0" + "undici-types": "~6.21.0" } }, "node_modules/@types/triple-beam": { @@ -2341,9 +2341,9 @@ } }, "node_modules/mongodb-ns": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/mongodb-ns/-/mongodb-ns-3.2.3.tgz", - "integrity": "sha512-+ECvmAqkRkzbDTC+75EmMYjRS1lVObUX7qtytRwR5xDBoBvC1XBmSnBFgZS/l1piMYKd3bHazd58L8rVjQLnAg==", + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/mongodb-ns/-/mongodb-ns-3.2.5.tgz", + "integrity": "sha512-XGMCU8sDN1OK2Ls38pKDLMLgQd4CYXkaiaHmM/2B67XDUjgrBjAdQpNSmbU5thBGk3IosZHw0oe81rdw9v1rVw==", "license": "Apache-2.0", "optional": true }, @@ -3920,9 +3920,9 @@ } }, "node_modules/undici-types": { - "version": "8.9.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", - "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "license": "MIT" }, "node_modules/universalify": { diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index a70a85ec5..d22fa2cd2 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -7,8 +7,8 @@ import { GrpcResponse, HealthCheckStatus, VectorIndexDefinition, - VectorIndexMethod, VectorSimilarity, + defaultVectorIndexMethod, } from '@conduitplatform/grpc-sdk'; import { AdminHandlers } from './admin/index.js'; import { SchemaAdmin } from './admin/schema.admin.js'; @@ -1091,7 +1091,7 @@ export default class DatabaseModule extends ManagedModule { dimensions: index.dimensions, similarity: index.similarity, name: index.name, - method: index.method, + method: defaultVectorIndexMethod(index.method), filterFields: [...(index.filterFields ?? [])], options: index.options ? JSON.stringify(index.options) : undefined, status: index.status, @@ -1210,7 +1210,7 @@ export default class DatabaseModule extends ManagedModule { dimensions: index.dimensions, similarity: index.similarity as VectorSimilarity, name: index.name, - method: index.method as VectorIndexMethod, + method: defaultVectorIndexMethod(index.method), filterFields: index.filterFields, options: index.options ? JSON.parse(index.options) : undefined, }; diff --git a/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts b/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts index 76f61b393..e48c16439 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts @@ -98,6 +98,75 @@ describe('mongoose vector index lifecycle', () => { expect(createSearchIndex).toHaveBeenCalledTimes(1); }); + it('creates explicit hnsw when method is omitted and reuses missing indexingMethod', async () => { + const createSearchIndex = jest.fn(async () => undefined); + const listSearchIndexes = jest.fn(() => ({ + toArray: async () => [], + })); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + mongoose: { + model: () => ({ + collection: { createSearchIndex, listSearchIndexes }, + }), + }, + }); + + const withoutMethod = { + field: 'embedding', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + filterFields: ['tenantId'], + }; + await adapter.createVectorIndex('Article', withoutMethod); + expect(createSearchIndex).toHaveBeenCalledWith({ + name: 'embedding_vector', + type: 'vectorSearch', + definition: { + fields: [ + { + type: 'vector', + path: 'embedding', + numDimensions: 3, + similarity: VectorSimilarity.Cosine, + indexingMethod: VectorIndexMethod.HNSW, + }, + { type: 'filter', path: '_id' }, + { type: 'filter', path: 'tenantId' }, + ], + }, + }); + + listSearchIndexes.mockImplementation(() => ({ + toArray: async () => [ + { + name: 'embedding_vector', + type: 'vectorSearch', + status: 'READY', + queryable: true, + latestDefinition: { + fields: [ + { + type: 'vector', + path: 'embedding', + numDimensions: 3, + similarity: VectorSimilarity.Cosine, + }, + { type: 'filter', path: '_id' }, + { type: 'filter', path: 'tenantId' }, + ], + }, + }, + ], + })); + await adapter.createVectorIndex('Article', { + ...withoutMethod, + method: '' as VectorIndexMethod, + }); + expect(createSearchIndex).toHaveBeenCalledTimes(1); + }); + it('rejects vector search against a pending Mongo index', async () => { const aggregate = jest.fn(); const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; diff --git a/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts b/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts index 862fba56d..cd885d6b3 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts @@ -188,6 +188,47 @@ describe('vector index lifecycle', () => { ).toThrow(/different definition/); }); + it('reuses Mongo indexes when method is empty, missing, or default hnsw', () => { + const requested = bindVectorIndexToField({ + provider: 'mongodb', + field: vectorField, + index: { + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: '' as VectorIndexMethod, + filterFields: ['tenantId'], + }, + }); + expect(requested.method).toBe(VectorIndexMethod.HNSW); + expect( + planMongoVectorIndexCreate({ + requested, + existing: [ + { + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + filterFields: ['_id', 'tenantId'], + }, + ], + }), + ).toEqual({ action: 'reuse' }); + expect( + planMongoVectorIndexCreate({ + requested: { ...requested, method: VectorIndexMethod.HNSW }, + existing: [ + { + ...requested, + method: '' as VectorIndexMethod, + }, + ], + }), + ).toEqual({ action: 'reuse' }); + }); + it('creates Postgres vector indexes without IF NOT EXISTS and detects mismatches', () => { const sql = renderPostgresCreateVectorIndexSql({ indexName: 'cnd_Article_embedding_vector', diff --git a/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts b/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts index 4ee20e0b7..a153d1780 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts @@ -110,6 +110,7 @@ describe('vector field and index mappings', () => { path: 'embedding', numDimensions: 1536, similarity: VectorSimilarity.Cosine, + indexingMethod: VectorIndexMethod.HNSW, }, { type: 'filter', path: '_id' }, { type: 'filter', path: 'tenantId' }, @@ -196,4 +197,59 @@ describe('vector field and index mappings', () => { options: { ivfflat: { lists: 100 } }, }); }); + + it('treats empty proto method and missing Mongo indexingMethod as hnsw', () => { + expect( + toMongoVectorIndexDefinition({ + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + }).fields[0], + ).toMatchObject({ indexingMethod: VectorIndexMethod.HNSW }); + expect( + toMongoVectorIndexDefinition({ + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: '' as VectorIndexMethod, + }).fields[0], + ).toMatchObject({ indexingMethod: VectorIndexMethod.HNSW }); + + const withoutMethod = fromMongoVectorIndex({ + name: 'embedding_vector', + status: 'READY', + queryable: true, + latestDefinition: { + fields: [ + { + type: 'vector', + path: 'embedding', + numDimensions: 1536, + similarity: VectorSimilarity.Cosine, + }, + ], + }, + }); + expect(withoutMethod.method).toBe(VectorIndexMethod.HNSW); + + const emptyMethod = fromMongoVectorIndex({ + name: 'embedding_vector', + status: 'READY', + queryable: true, + latestDefinition: { + fields: [ + { + type: 'vector', + path: 'embedding', + numDimensions: 1536, + similarity: VectorSimilarity.Cosine, + indexingMethod: '', + }, + ], + }, + }); + expect(emptyMethod.method).toBe(VectorIndexMethod.HNSW); + }); }); diff --git a/modules/database/src/adapters/utils/vectorIndexLifecycle.ts b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts index feaabb945..67bfa8ed4 100644 --- a/modules/database/src/adapters/utils/vectorIndexLifecycle.ts +++ b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts @@ -1,9 +1,10 @@ import { GrpcError, VectorIndexDefinition, - VectorIndexMethod, VectorIndexStatus, VectorSimilarity, + defaultVectorIndexMethod, + vectorIndexMethodsEquivalent, } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { @@ -88,6 +89,7 @@ export function bindVectorIndexToField(args: { args.index.name ?? defaultVectorIndexName(args.index.field, args.physicalTableName), dimensions: args.index.dimensions ?? field?.dimensions, similarity: args.index.similarity ?? field?.similarity, + method: defaultVectorIndexMethod(args.index.method), filterFields: args.provider === 'mongodb' ? mongoVectorFilterFields(args.index.filterFields) @@ -171,9 +173,7 @@ export function vectorIndexesEquivalent( if (left.field !== right.field) return false; if (left.dimensions !== right.dimensions) return false; if (left.similarity !== right.similarity) return false; - const leftMethod = left.method ?? VectorIndexMethod.HNSW; - const rightMethod = right.method ?? VectorIndexMethod.HNSW; - if (leftMethod !== rightMethod) return false; + if (!vectorIndexMethodsEquivalent(left.method, right.method)) return false; if (provider === 'mongodb') { return sameStringSet( mongoVectorFilterFields(left.filterFields), @@ -365,14 +365,13 @@ export function hydratePostgresVectorIndex(args: { declared?: VectorIndexDefinition; }): VectorIndexDefinition { const parsed = parsePostgresVectorIndexDef(args.indexdef); - const method = - (parsed.method as VectorIndexMethod | undefined) ?? args.declared?.method; + const method = defaultVectorIndexMethod(parsed.method ?? args.declared?.method); return { name: args.name, field: parsed.field || args.declared?.field || '', dimensions: args.field?.dimensions ?? args.declared?.dimensions ?? 0, similarity: args.field?.similarity ?? parsed.similarity, - method: method ?? VectorIndexMethod.HNSW, + method, options: mergeVectorIndexOptions(args.declared?.options, parsed.options), status: VectorIndexStatus.Ready, queryable: true, diff --git a/modules/database/src/adapters/utils/vectorMappings.ts b/modules/database/src/adapters/utils/vectorMappings.ts index f7f8473f6..ff707ea64 100644 --- a/modules/database/src/adapters/utils/vectorMappings.ts +++ b/modules/database/src/adapters/utils/vectorMappings.ts @@ -2,6 +2,7 @@ import { VectorIndexDefinition, VectorIndexMethod, VectorSimilarity, + defaultVectorIndexMethod, } from '@conduitplatform/grpc-sdk'; import { isObjectFormVectorField } from './vectorField.js'; import { @@ -106,9 +107,7 @@ export function toMongoVectorIndexDefinition(index: VectorIndexDefinition) { if (index.options?.quantization) { vectorField.quantization = index.options.quantization; } - if (index.method) { - vectorField.indexingMethod = index.method; - } + vectorField.indexingMethod = defaultVectorIndexMethod(index.method); if (index.options?.hnsw) { vectorField.hnswOptions = { ...(index.options.hnsw.maxEdges && { maxEdges: index.options.hnsw.maxEdges }), @@ -146,7 +145,7 @@ export function fromMongoVectorIndex(index: { field: vectorField.path, dimensions: vectorField.numDimensions, similarity: vectorField.similarity, - method: vectorField.indexingMethod, + method: defaultVectorIndexMethod(vectorField.indexingMethod), filterFields: fields .filter(field => field.type === 'filter') .map(field => field.path), diff --git a/modules/embeddings/package.bundle-lock.json b/modules/embeddings/package.bundle-lock.json index d90b14ac4..a6d1c6fc7 100644 --- a/modules/embeddings/package.bundle-lock.json +++ b/modules/embeddings/package.bundle-lock.json @@ -44,18 +44,18 @@ "license": "(Apache-2.0 AND BSD-3-Clause)" }, "node_modules/@colors/colors": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", - "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.1.tgz", + "integrity": "sha512-dTmUJzXSuayBK+hZydEaXd2mhx61qWQwkwaBBY6LyEOVx/L9aQU5ac8eFNEsd9nrD1+zb9zvDCphLSe8g1F4Qw==", "license": "MIT", "engines": { "node": ">=0.1.90" } }, "node_modules/@dabh/diagnostics": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.8.tgz", - "integrity": "sha512-R4MSXTVnuMzGD7bzHdW2ZhhdPC/igELENcq5IjEverBvq5hn1SXCWcsi6eSsdWP0/Ur+SItRRjAktmdoX/8R/Q==", + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.9.tgz", + "integrity": "sha512-R6siwR65Hm+3yfgP7o8DKhNvputQAwfoz9zTc3kyDudnomj2/BcLmD+uGQQPICjuFUp8ounPBU+jmKsocwVVAg==", "license": "MIT", "dependencies": { "@so-ric/colorspace": "^1.1.6", @@ -581,12 +581,12 @@ } }, "node_modules/@types/node": { - "version": "26.4.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.1.tgz", - "integrity": "sha512-k97ENvZWtvA6yqz5/FS6a7duDgOPEeOQOc2iKS/nY6mX6qJUKtLnWzQS+Xj6tXweyj6ZcTAK2Qecetnvi9nCLA==", + "version": "22.20.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.2.tgz", + "integrity": "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==", "license": "MIT", "dependencies": { - "undici-types": "~8.3.0" + "undici-types": "~6.21.0" } }, "node_modules/@types/triple-beam": { @@ -1631,6 +1631,15 @@ "node": ">= 12.0.0" } }, + "node_modules/logform/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "license": "MIT", + "engines": { + "node": ">=0.1.90" + } + }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -2425,9 +2434,9 @@ } }, "node_modules/undici-types": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", - "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "license": "MIT" }, "node_modules/universalify": { diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts index 5570c51a1..5501447c4 100644 --- a/modules/embeddings/src/api/embeddingsApi.test.ts +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -398,6 +398,39 @@ describe('typed embeddings API handlers', () => { ); }); + it('does not recreate _vN indexes when live method is empty or missing', async () => { + const missingMethod = { + ...readyIndex, + method: undefined, + }; + const emptyMethod = { + ...readyIndex, + method: '', + }; + for (const indexes of [[missingMethod], [emptyMethod]]) { + const { api, createdIndexes, deletedIndexes } = createApi({ + configs: [enabledConfig], + indexes, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, true); + assert.deepEqual(createdIndexes, []); + assert.deepEqual(deletedIndexes, []); + } + }); + it('gates system schemas and owner policies on config and backfill', async () => { const { api } = createApi({ declared: { Article: { name: 'Article', ownerModule: 'cms-app' } }, diff --git a/modules/embeddings/src/api/embeddingsApi.ts b/modules/embeddings/src/api/embeddingsApi.ts index ba539eab7..8e9bb64c2 100644 --- a/modules/embeddings/src/api/embeddingsApi.ts +++ b/modules/embeddings/src/api/embeddingsApi.ts @@ -2,6 +2,7 @@ import { GrpcError, TYPE, VectorCapabilities, + VectorIndexMethod, VectorSearchResult, VectorSimilarity, type ConduitModel, @@ -984,6 +985,7 @@ export class EmbeddingsApi { dimensions: next.dimensions, similarity: next.similarity as VectorIndexDefinition['similarity'], name: defaultEmbeddingVectorIndexName(next.targetField), + method: VectorIndexMethod.HNSW, }); } catch (err) { throw new GrpcError( @@ -1010,6 +1012,7 @@ export class EmbeddingsApi { dimensions: next.dimensions, similarity: next.similarity as VectorIndexDefinition['similarity'], name: replacementName, + method: VectorIndexMethod.HNSW, }); } catch (err) { throw new GrpcError( diff --git a/modules/embeddings/src/utils/backfillGates.test.ts b/modules/embeddings/src/utils/backfillGates.test.ts index 1a3cc0093..afe341e21 100644 --- a/modules/embeddings/src/utils/backfillGates.test.ts +++ b/modules/embeddings/src/utils/backfillGates.test.ts @@ -197,6 +197,14 @@ describe('backfill execution gates', () => { err.indexStatus === VectorIndexStatus.Pending && /Wait until the index is ready/.test(err.message), ); + assert.doesNotThrow(() => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities, + config, + indexes: [{ ...readyIndex, method: '' }], + }), + ); const mapped = grpcErrorFromBackfillGate( new BackfillGateError( 'index_not_queryable', diff --git a/modules/embeddings/src/utils/configChange.test.ts b/modules/embeddings/src/utils/configChange.test.ts index aeb43a9bd..62d81036d 100644 --- a/modules/embeddings/src/utils/configChange.test.ts +++ b/modules/embeddings/src/utils/configChange.test.ts @@ -192,4 +192,37 @@ describe('material embedding config changes', () => { undefined, ); }); + + it('treats empty and missing index methods as default hnsw', () => { + const missingMethod = { + field: 'embedding', + name: 'embedding_vector', + dimensions: 3, + similarity: 'cosine', + }; + const emptyMethod = { ...missingMethod, method: '' }; + const hnsw = { ...missingMethod, method: 'hnsw' }; + const contract = { + field: 'embedding', + dimensions: 3, + similarity: 'cosine', + }; + assert.equal(embeddingVectorIndexMatchesContract(missingMethod, contract), true); + assert.equal(embeddingVectorIndexMatchesContract(emptyMethod, contract), true); + assert.equal( + embeddingVectorIndexMatchesContract(hnsw, { ...contract, method: '' }), + true, + ); + assert.equal( + selectEmbeddingVectorIndex([missingMethod], 'embedding', { + dimensions: 3, + similarity: 'cosine', + })?.name, + 'embedding_vector', + ); + assert.equal( + nextEmbeddingVectorIndexName('embedding', [missingMethod]), + 'embedding_vector_v2', + ); + }); }); diff --git a/modules/embeddings/src/utils/configChange.ts b/modules/embeddings/src/utils/configChange.ts index 0155e4f27..da4763386 100644 --- a/modules/embeddings/src/utils/configChange.ts +++ b/modules/embeddings/src/utils/configChange.ts @@ -1,3 +1,5 @@ +import { vectorIndexMethodsEquivalent } from '@conduitplatform/grpc-sdk'; + export const MATERIAL_EMBEDDING_CONFIG_FIELDS = [ 'provider', 'modelName', @@ -162,8 +164,6 @@ export interface EmbeddingVectorIndexShape { method?: string; } -const DEFAULT_EMBEDDING_VECTOR_INDEX_METHOD = 'hnsw'; - export function embeddingVectorIndexMatchesContract( index: EmbeddingVectorIndexShape, contract: EmbeddingVectorIndexContract, @@ -173,10 +173,7 @@ export function embeddingVectorIndexMatchesContract( return false; } if ((index.similarity ?? '') !== (contract.similarity ?? '')) return false; - return ( - (index.method ?? DEFAULT_EMBEDDING_VECTOR_INDEX_METHOD) === - (contract.method ?? DEFAULT_EMBEDDING_VECTOR_INDEX_METHOD) - ); + return vectorIndexMethodsEquivalent(index.method, contract.method); } export function selectEmbeddingVectorIndex( diff --git a/modules/embeddings/src/utils/moduleConfigLifecycle.test.ts b/modules/embeddings/src/utils/moduleConfigLifecycle.test.ts new file mode 100644 index 000000000..b2bbfb9de --- /dev/null +++ b/modules/embeddings/src/utils/moduleConfigLifecycle.test.ts @@ -0,0 +1,75 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import convict from 'convict'; +import { + merge, + reconcileStoredModuleConfig, + restoreRedactedSecrets, +} from '@conduitplatform/module-tools'; +import AppConfigSchema, { type Config } from '../config/index.js'; +import { normalizeEmbeddingsConfig } from './providerConfig.js'; + +describe('embeddings module config lifecycle', () => { + it('migrates stored legacy provider settings and keeps them across setConfig', async () => { + const schema = convict(AppConfigSchema); + const stored = normalizeEmbeddingsConfig({ + ...schema.getProperties(), + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-live', + model: 'text-embedding-3-small', + dimensions: 1536, + models: [], + allowedHosts: ['api.openai.com'], + }, + }, + security: { + ...schema.getProperties().security, + requireGrpcKey: true, + }, + } as unknown as Config); + schema.load(stored).validate({ allowed: 'warn' }); + let persistCalls = 0; + const reconciled = await reconcileStoredModuleConfig({ + stored: { + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-live', + model: 'text-embedding-3-small', + dimensions: 1536, + models: [], + allowedHosts: ['api.openai.com'], + }, + }, + security: { requireGrpcKey: true, sourceFieldAllowlist: [] }, + } as unknown as Config, + migrated: schema.getProperties() as Config, + configureOverride: async config => { + persistCalls += 1; + return config; + }, + }); + assert.equal(persistCalls, 1); + schema.load(reconciled.config); + + const previous = schema.getProperties() as Config; + let next = merge(previous, { enabled: true } as Config); + next = restoreRedactedSecrets(next, previous, AppConfigSchema); + next = normalizeEmbeddingsConfig(next); + schema.load(next).validate({ allowed: 'warn' }); + const patched = schema.getProperties(); + const provider = patched.providers['openai-compatible']; + assert.equal(patched.enabled, true); + assert.equal(provider.apiKey, 'sk-live'); + assert.deepEqual(provider.models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + assert.equal(provider.defaultModel, 'text-embedding-3-small'); + assert.equal('model' in provider, false); + assert.equal('dimensions' in provider, false); + assert.equal('allowedHosts' in provider, false); + assert.equal('requireGrpcKey' in patched.security, false); + }); +}); diff --git a/modules/embeddings/src/utils/operationalStatus.test.ts b/modules/embeddings/src/utils/operationalStatus.test.ts index 92a9dd2d0..850c6ce4a 100644 --- a/modules/embeddings/src/utils/operationalStatus.test.ts +++ b/modules/embeddings/src/utils/operationalStatus.test.ts @@ -92,6 +92,62 @@ describe('embeddings operational warnings and search gates', () => { assert.equal(mapped.code, status.FAILED_PRECONDITION); }); + it('allows search when the live index method is empty or omitted', () => { + assert.doesNotThrow(() => + assertSearchExecutable({ + capabilities: { + supported: true, + search: true, + provider: 'mongodb', + }, + config: { + _id: 'cfg1', + enabled: true, + targetField: 'embedding', + dimensions: 3, + similarity: 'cosine', + }, + indexes: [ + { + field: 'embedding', + name: 'embedding_vector', + status: VectorIndexStatus.Ready, + queryable: true, + dimensions: 3, + similarity: 'cosine', + }, + ], + }), + ); + assert.doesNotThrow(() => + assertSearchExecutable({ + capabilities: { + supported: true, + search: true, + provider: 'mongodb', + }, + config: { + _id: 'cfg1', + enabled: true, + targetField: 'embedding', + dimensions: 3, + similarity: 'cosine', + }, + indexes: [ + { + field: 'embedding', + name: 'embedding_vector', + status: VectorIndexStatus.Ready, + queryable: true, + dimensions: 3, + similarity: 'cosine', + method: '', + }, + ], + }), + ); + }); + it('denies search and activation when a queryable live index does not match the config contract', () => { const mismatched = { field: 'embedding', diff --git a/modules/embeddings/src/utils/redactConfig.test.ts b/modules/embeddings/src/utils/redactConfig.test.ts index 3c82951ba..19f5ac57d 100644 --- a/modules/embeddings/src/utils/redactConfig.test.ts +++ b/modules/embeddings/src/utils/redactConfig.test.ts @@ -1,6 +1,9 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { redactSensitiveConfig } from '@conduitplatform/module-tools'; +import { + redactSensitiveConfig, + restoreRedactedSecrets, +} from '@conduitplatform/module-tools'; import AppConfigSchema from '../config/index.js'; import { normalizeEmbeddingsConfig } from './providerConfig.js'; import { redactSecretText } from './redactConfig.js'; @@ -123,4 +126,22 @@ describe('provider secret redaction', () => { ); assert.equal(emptyKey.providers['openai-compatible'].apiKey, ''); }); + + it('restores redacted API keys from the currently stored config', () => { + const current = normalizeEmbeddingsConfig({ + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-live', + models: [{ name: 'text-embedding-3-small', dimensions: 1536 }], + }, + }, + }); + const incoming = redactSensitiveConfig(current, AppConfigSchema); + const restored = restoreRedactedSecrets(incoming, current, AppConfigSchema); + assert.equal(restored.providers['openai-compatible'].apiKey, 'sk-live'); + assert.deepEqual(restored.providers['openai-compatible'].models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + }); }); diff --git a/packages/core/package.bundle-lock.json b/packages/core/package.bundle-lock.json index 98788f7ac..eb6a81c0d 100644 --- a/packages/core/package.bundle-lock.json +++ b/packages/core/package.bundle-lock.json @@ -16,7 +16,7 @@ "@grpc/proto-loader": "^0.8.1", "@modelcontextprotocol/sdk": "^1.29.0", "@scalar/api-reference": "^1.60.0", - "@scalar/express-api-reference": "^0.10.14", + "@scalar/express-api-reference": "^0.10.16", "@sesamecare-oss/redlock": "^1.4.0", "@socket.io/redis-streams-adapter": "^0.3.1", "abort-controller-x": "^0.5.0", @@ -58,7 +58,7 @@ "socket.io-adapter": "2.5.8", "swagger-ui-express": "5.0.1", "thirty-two": "1.0.2", - "uuid": "^14.0.1", + "uuid": "^14.0.2", "winston": "^3.19.0", "winston-loki": "^6.1.7", "zod": "^4.4.3" @@ -465,9 +465,9 @@ } }, "node_modules/@bufbuild/protobuf": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.14.0.tgz", - "integrity": "sha512-C3UGsiCwSprE2NKIIFA3hCDlpXTMCAXRZuEVp88L1GY36Y41+rYL5fryE+nOFhp4p4JPQvdV8PQ4DWgHgeTE+w==", + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.14.1.tgz", + "integrity": "sha512-agRJn3+EJDUe8AvxTx/LnHA/GErvLE62pSaSk7+MwFOtOv8eWBu/qCq2qoZjBjVZ3C2aiFJCveuSs17KMkYGOw==", "license": "(Apache-2.0 AND BSD-3-Clause)" }, "node_modules/@codemirror/autocomplete": { @@ -604,18 +604,18 @@ } }, "node_modules/@codemirror/state": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.7.1.tgz", - "integrity": "sha512-9QzNDgE4EYDnAHfrTlR2lwiPciiOymLtwKK+8yHQzCc7GXhAP9xdEbEJFy2IWB1j9UGUl9BsgMmTo/ImA02T7A==", + "version": "6.7.4", + "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.7.4.tgz", + "integrity": "sha512-QhQIVRY+xHZDxwOSFrJ1eUMapJBUID3IdeAjf7dHO7zBUzSkyooHiodnalz5MG3iHzwixKMlAAyn7244y537EA==", "license": "MIT", "dependencies": { "@marijn/find-cluster-break": "^1.0.0" } }, "node_modules/@codemirror/view": { - "version": "6.43.9", - "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.43.9.tgz", - "integrity": "sha512-sTuUzTpPMFebRhg6dawChoKKgndIwfjmJgKVxBefPElcU2NwQ6AFroupk0SFqEerQyZOGRfDNnSN8Dw/lMAsXw==", + "version": "6.43.11", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.43.11.tgz", + "integrity": "sha512-2+esucbQX6wB2JYi1eDvdCPFTA31BN8oSy6xCmk3G6CloV11yOvEjYk+gH7kLrP0MuHG94E8WDhjs5oMiu3+Wg==", "license": "MIT", "dependencies": { "@codemirror/state": "^6.7.0", @@ -625,18 +625,18 @@ } }, "node_modules/@colors/colors": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", - "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.1.tgz", + "integrity": "sha512-dTmUJzXSuayBK+hZydEaXd2mhx61qWQwkwaBBY6LyEOVx/L9aQU5ac8eFNEsd9nrD1+zb9zvDCphLSe8g1F4Qw==", "license": "MIT", "engines": { "node": ">=0.1.90" } }, "node_modules/@dabh/diagnostics": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.8.tgz", - "integrity": "sha512-R4MSXTVnuMzGD7bzHdW2ZhhdPC/igELENcq5IjEverBvq5hn1SXCWcsi6eSsdWP0/Ur+SItRRjAktmdoX/8R/Q==", + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.9.tgz", + "integrity": "sha512-R6siwR65Hm+3yfgP7o8DKhNvputQAwfoz9zTc3kyDudnomj2/BcLmD+uGQQPICjuFUp8ounPBU+jmKsocwVVAg==", "license": "MIT", "dependencies": { "@so-ric/colorspace": "^1.1.6", @@ -719,12 +719,12 @@ } }, "node_modules/@graphql-tools/merge": { - "version": "9.2.3", - "resolved": "https://registry.npmjs.org/@graphql-tools/merge/-/merge-9.2.3.tgz", - "integrity": "sha512-cKRoXqJGy2zSRBLvotQpkACbXlHAb0yuHLN0l0ypKGCuL3NnF0zofYalkBJqiBxxxpK0lr3s9uowKFHCKi1/ZQ==", + "version": "9.2.4", + "resolved": "https://registry.npmjs.org/@graphql-tools/merge/-/merge-9.2.4.tgz", + "integrity": "sha512-vV+8uWNWn0+OsqT0r22lZuoT0cTe6fBqBtpLHre2rriLjI/ZTrsOHmabLPTUOyzgFnRrS2PzFSYGL3zKL1Wj4Q==", "license": "MIT", "dependencies": { - "@graphql-tools/utils": "^12.0.0", + "@graphql-tools/utils": "^12.0.1", "tslib": "^2.4.0" }, "engines": { @@ -735,13 +735,13 @@ } }, "node_modules/@graphql-tools/schema": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/@graphql-tools/schema/-/schema-10.1.0.tgz", - "integrity": "sha512-wao48XQnfY631s3jXoNrhEHvCI8mlKXmIuWrR7F6zAdv92VuSOfHoq9P9KL2EnUMgBUnaStnByOx9Mn6RieWDg==", + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/@graphql-tools/schema/-/schema-10.1.1.tgz", + "integrity": "sha512-24jJghRxEW+SG1lbJ45Zg9HEZ8ZKS923ihbFjOicspFLpryxJkWp6gZJ7D8tCVaSAhli4x1+1wDPAfhT8mLqxg==", "license": "MIT", "dependencies": { - "@graphql-tools/merge": "^9.2.3", - "@graphql-tools/utils": "^12.0.0", + "@graphql-tools/merge": "^9.2.4", + "@graphql-tools/utils": "^12.0.1", "tslib": "^2.4.0" }, "engines": { @@ -752,9 +752,9 @@ } }, "node_modules/@graphql-tools/utils": { - "version": "12.0.0", - "resolved": "https://registry.npmjs.org/@graphql-tools/utils/-/utils-12.0.0.tgz", - "integrity": "sha512-aMdIo/l+8j4lhamWAf+MWHr+lOW8zArSBKXspqtKHgt5I2JF9LS55KDLUe/L9AiJOV/O6qJJ60hgYydbnJHbxg==", + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/@graphql-tools/utils/-/utils-12.0.1.tgz", + "integrity": "sha512-8YC6jn4xYDS6YTY6xDAgQAs/nGgvzRaIGHS8GeSfVItQzNK7Ms24CQtE3EJY+amvR+tBnhRSX0N83aGj+V/RIg==", "license": "MIT", "dependencies": { "@graphql-typed-document-node/core": "^3.1.1", @@ -816,9 +816,9 @@ "license": "Apache-2.0" }, "node_modules/@grpc/proto-loader/node_modules/protobufjs": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", - "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -878,18 +878,18 @@ } }, "node_modules/@internationalized/date": { - "version": "3.12.3", - "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.12.3.tgz", - "integrity": "sha512-fuLX+3ZKLsxI73y8b01EG/WjHb6gE6weCqlfawPO27kBWGMh9G1yH6Csv1uU7/cac9H2GHmOMt6CjmuQ1aia4Q==", + "version": "3.12.4", + "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.12.4.tgz", + "integrity": "sha512-M1dEn4c1U1HsSlaVR8upZtSqvXrTkHDfv18H01uCSJyjVLDxnBR38v/fMxecmlwXKR4i9HeZcmgQAPE6A+aGJQ==", "license": "Apache-2.0", "dependencies": { "@swc/helpers": "^0.5.0" } }, "node_modules/@internationalized/number": { - "version": "3.6.7", - "resolved": "https://registry.npmjs.org/@internationalized/number/-/number-3.6.7.tgz", - "integrity": "sha512-3ji1fcrT+FPAK86UqEhB/psHixYo6niWPJtt7+qRaYFynt/BaJG8GhAPimtWUpEiVSTq8ZM8L5psMxGquiB/Vg==", + "version": "3.6.8", + "resolved": "https://registry.npmjs.org/@internationalized/number/-/number-3.6.8.tgz", + "integrity": "sha512-8UmMFia46DUt+k97zKd9fKWXcWHR+k8ae3eYzILETuT2KbIvLyOfac7zesw+sJdRAAZ7Q9pM1Mk22aXp2LD0Ig==", "license": "Apache-2.0", "dependencies": { "@swc/helpers": "^0.5.0" @@ -902,9 +902,9 @@ "license": "MIT" }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", "license": "MIT" }, "node_modules/@js-sdsl/ordered-map": { @@ -1433,24 +1433,24 @@ } }, "node_modules/@scalar/agent-chat": { - "version": "0.12.28", - "resolved": "https://registry.npmjs.org/@scalar/agent-chat/-/agent-chat-0.12.28.tgz", - "integrity": "sha512-N1ZEIKHrOhbB6mSUuaeMbNEK8G0lhXu8aHj8RqPMjKAkm0IbZVGCEkFgRCzaqDMcdsXKqQKygzx2a5pV5gX6Ow==", + "version": "0.12.30", + "resolved": "https://registry.npmjs.org/@scalar/agent-chat/-/agent-chat-0.12.30.tgz", + "integrity": "sha512-1+09CTY/eVLg6ygdrrKyNkhWXnbHcVGrr4Jq4HYr7XogrkIsTCuHCBcpPbYRWOFJ2zRpToPFu5M3R7eLE810zA==", "license": "MIT", "dependencies": { "@ai-sdk/vue": "3.0.33", - "@scalar/api-client": "3.16.3", - "@scalar/components": "0.28.1", - "@scalar/helpers": "0.11.1", + "@scalar/api-client": "3.18.0", + "@scalar/components": "0.29.1", + "@scalar/helpers": "0.11.3", "@scalar/icons": "0.7.6", - "@scalar/json-magic": "0.13.2", + "@scalar/json-magic": "0.13.4", "@scalar/openapi-types": "0.9.5", - "@scalar/schemas": "0.8.3", - "@scalar/themes": "0.17.3", - "@scalar/types": "0.18.2", + "@scalar/schemas": "0.9.0", + "@scalar/themes": "0.17.4", + "@scalar/types": "0.19.0", "@scalar/use-toasts": "0.10.5", "@scalar/validation": "0.6.3", - "@scalar/workspace-store": "0.58.1", + "@scalar/workspace-store": "0.60.0", "@vueuse/core": "13.9.0", "ai": "6.0.33", "js-base64": "^3.9.2", @@ -1463,28 +1463,28 @@ } }, "node_modules/@scalar/api-client": { - "version": "3.16.3", - "resolved": "https://registry.npmjs.org/@scalar/api-client/-/api-client-3.16.3.tgz", - "integrity": "sha512-4F0aZtdnCWZN5EvJQTAlXPzVetniu9pQqPeukck/AUNtvsk59CdNHjd8DjAB3+SS3wv3gfNj0C+obhzjUbCNZg==", + "version": "3.18.0", + "resolved": "https://registry.npmjs.org/@scalar/api-client/-/api-client-3.18.0.tgz", + "integrity": "sha512-FlseC6xWfx0ganh4s0IAOelJ5kF+aJ07BJfjUriwKDdK8Y4oAFkRlneKQNoT/S+4mrNCXdQ/T5VPEjYFlx4Zxw==", "license": "MIT", "dependencies": { "@headlessui/tailwindcss": "^0.2.2", "@headlessui/vue": "1.7.23", - "@scalar/blocks": "0.1.14", - "@scalar/components": "0.28.1", - "@scalar/helpers": "0.11.1", + "@scalar/blocks": "0.1.16", + "@scalar/components": "0.29.1", + "@scalar/helpers": "0.11.3", "@scalar/icons": "0.7.6", - "@scalar/oas-utils": "0.19.14", + "@scalar/oas-utils": "0.19.16", "@scalar/openapi-types": "0.9.5", - "@scalar/sidebar": "0.10.1", - "@scalar/snippetz": "0.9.28", - "@scalar/themes": "0.17.3", + "@scalar/sidebar": "0.11.1", + "@scalar/snippetz": "0.9.30", + "@scalar/themes": "0.17.4", "@scalar/typebox": "^0.1.3", - "@scalar/types": "0.18.2", + "@scalar/types": "0.19.0", "@scalar/use-codemirror": "0.14.15", - "@scalar/use-hooks": "0.4.10", + "@scalar/use-hooks": "0.4.11", "@scalar/use-toasts": "0.10.5", - "@scalar/workspace-store": "0.58.1", + "@scalar/workspace-store": "0.60.0", "@vueuse/core": "13.9.0", "@vueuse/integrations": "13.9.0", "focus-trap": "^7.8.0", @@ -1497,36 +1497,36 @@ "set-cookie-parser": "3.1.0", "vue": "^3.5.40", "yaml": "^2.9.0", - "zod": "^4.3.5" + "zod": "^4.4.3" }, "engines": { "node": ">=22" } }, "node_modules/@scalar/api-reference": { - "version": "1.66.1", - "resolved": "https://registry.npmjs.org/@scalar/api-reference/-/api-reference-1.66.1.tgz", - "integrity": "sha512-+iHSJX8HPUyDGinNiLRV8qgFb+fbNwAjGDWzl8Wg/VNEcbmA40t1ZuL/WoWNqphI2QaPfjiCLtmsIhanWAed3w==", + "version": "1.68.0", + "resolved": "https://registry.npmjs.org/@scalar/api-reference/-/api-reference-1.68.0.tgz", + "integrity": "sha512-rY43w3REwCxp+rDDx/0CncZxmlzISnGTK9zZ8moq0Ij2vRHhLQCJ0/BXut9pBAupVrOZF7MoqKXcG+gISgTu5g==", "license": "MIT", "dependencies": { "@headlessui/vue": "1.7.23", - "@scalar/agent-chat": "0.12.28", - "@scalar/api-client": "3.16.3", - "@scalar/blocks": "0.1.14", + "@scalar/agent-chat": "0.12.30", + "@scalar/api-client": "3.18.0", + "@scalar/blocks": "0.1.16", "@scalar/code-highlight": "0.4.5", - "@scalar/components": "0.28.1", - "@scalar/helpers": "0.11.1", + "@scalar/components": "0.29.1", + "@scalar/helpers": "0.11.3", "@scalar/icons": "0.7.6", - "@scalar/oas-utils": "0.19.14", - "@scalar/schemas": "0.8.3", - "@scalar/sidebar": "0.10.1", - "@scalar/snippetz": "0.9.28", - "@scalar/themes": "0.17.3", - "@scalar/types": "0.18.2", - "@scalar/use-hooks": "0.4.10", + "@scalar/oas-utils": "0.19.16", + "@scalar/schemas": "0.9.0", + "@scalar/sidebar": "0.11.1", + "@scalar/snippetz": "0.9.30", + "@scalar/themes": "0.17.4", + "@scalar/types": "0.19.0", + "@scalar/use-hooks": "0.4.11", "@scalar/use-toasts": "0.10.5", "@scalar/validation": "0.6.3", - "@scalar/workspace-store": "0.58.1", + "@scalar/workspace-store": "0.60.0", "@unhead/vue": "^2.1.4", "@vueuse/core": "13.9.0", "fuse.js": "^7.5.0", @@ -1540,30 +1540,30 @@ } }, "node_modules/@scalar/asyncapi-upgrader": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/@scalar/asyncapi-upgrader/-/asyncapi-upgrader-0.1.7.tgz", - "integrity": "sha512-RU3CrNV77hWiZ9Ik0GJHDf/bEg8C0iF9Rg5b6GDJz3F9Y7ZyBJcgqq++do9GUzPTKzftfOORfQ9180s09XQp7Q==", + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/@scalar/asyncapi-upgrader/-/asyncapi-upgrader-0.1.9.tgz", + "integrity": "sha512-+kK4dp1J8GvOcTU2OCMFwj62VMP3Y0lppjQ2mdfYsczhTTP9saf12vkRzuHQ+ILkLJJXo98km5zj7pEVMdOSPA==", "license": "MIT", "dependencies": { - "@scalar/helpers": "0.11.1" + "@scalar/helpers": "0.11.3" }, "engines": { "node": ">=22" } }, "node_modules/@scalar/blocks": { - "version": "0.1.14", - "resolved": "https://registry.npmjs.org/@scalar/blocks/-/blocks-0.1.14.tgz", - "integrity": "sha512-4goVCRnz8QWCzQIuMV58GUSoj+WJNZBSGS5L6n3kc8TAKY6qiTOf+Unc5oy2DgwHCJJ6AO531hnGDAsMx7sNaQ==", + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@scalar/blocks/-/blocks-0.1.16.tgz", + "integrity": "sha512-k72Dxwj2Bh9jhJkIyTlrpja6QRnvnT5Cb+LH++BGMV70lSu22cyzJBmhtKORTfQVb671s2I7WLUBU6zirxkhsA==", "license": "MIT", "dependencies": { - "@scalar/components": "0.28.1", - "@scalar/helpers": "0.11.1", + "@scalar/components": "0.29.1", + "@scalar/helpers": "0.11.3", "@scalar/icons": "0.7.6", - "@scalar/snippetz": "0.9.28", - "@scalar/themes": "0.17.3", - "@scalar/types": "0.18.2", - "@scalar/workspace-store": "0.58.1", + "@scalar/snippetz": "0.9.30", + "@scalar/themes": "0.17.4", + "@scalar/types": "0.19.0", + "@scalar/workspace-store": "0.60.0", "@types/har-format": "^1.2.16", "js-base64": "^3.9.2", "vue": "^3.5.40" @@ -1573,13 +1573,13 @@ } }, "node_modules/@scalar/client-side-rendering": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@scalar/client-side-rendering/-/client-side-rendering-0.3.9.tgz", - "integrity": "sha512-Gg+VLhreiWHmHN0i9uatEoIxzsr0FaJoq0x+PkypmFgZFniD477DVCAnmWqx+KXlpFg0VtDanDHARIJXZ1fIeQ==", + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@scalar/client-side-rendering/-/client-side-rendering-0.4.0.tgz", + "integrity": "sha512-a8OUod1LZbNqkPv73ijRPL3MH2/3E92I5Awa9mGXtCaIVavY3JcF7+qZW4GOtffihTfuD/CxhZ+CnQhCZds2oQ==", "license": "MIT", "dependencies": { - "@scalar/schemas": "0.8.3", - "@scalar/types": "0.18.2", + "@scalar/schemas": "0.9.0", + "@scalar/types": "0.19.0", "@scalar/validation": "0.6.3" }, "engines": { @@ -1613,9 +1613,9 @@ } }, "node_modules/@scalar/components": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@scalar/components/-/components-0.28.1.tgz", - "integrity": "sha512-mYI2WwvVM6a9E/o6vrft9FKoLP7Chcit5kOc+Iz+MV6xQ63Cjx2e/dACagEXE9O5i3b4+8+8C00iy4p1TS6P0A==", + "version": "0.29.1", + "resolved": "https://registry.npmjs.org/@scalar/components/-/components-0.29.1.tgz", + "integrity": "sha512-mxlJ/3Pv1YqqaXSBU2SLAiTZSGdam3ZyoYW7CAITSoVlruDBIEfSm6a8eABpM9mWSsoNQhmscLEq2QSC0szUPA==", "license": "MIT", "dependencies": { "@floating-ui/utils": "0.2.10", @@ -1623,10 +1623,10 @@ "@headlessui/tailwindcss": "^0.2.2", "@headlessui/vue": "1.7.23", "@scalar/code-highlight": "0.4.5", - "@scalar/helpers": "0.11.1", + "@scalar/helpers": "0.11.3", "@scalar/icons": "0.7.6", - "@scalar/themes": "0.17.3", - "@scalar/use-hooks": "0.4.10", + "@scalar/themes": "0.17.4", + "@scalar/use-hooks": "0.4.11", "@vueuse/core": "13.9.0", "cva": "1.0.0-beta.4", "radix-vue": "^1.9.17", @@ -1638,21 +1638,21 @@ } }, "node_modules/@scalar/express-api-reference": { - "version": "0.10.16", - "resolved": "https://registry.npmjs.org/@scalar/express-api-reference/-/express-api-reference-0.10.16.tgz", - "integrity": "sha512-Jh7gDxGjJZjkJK3nmGVmLL7Ti6EP69Q2KEOnSCI/mtQxjRuavZt3XnYobubkoHUg9aUErvEpDcPH+ANNa4pggg==", + "version": "0.10.18", + "resolved": "https://registry.npmjs.org/@scalar/express-api-reference/-/express-api-reference-0.10.18.tgz", + "integrity": "sha512-PvDEMUNwfMnn0ak7L+rfbN6YwSY4UtM6SaOZPVjvGvHqxt7GrMV8rdax/Vu9sOLK1pL5F92K/UVmu0K9eMEcdQ==", "license": "MIT", "dependencies": { - "@scalar/client-side-rendering": "0.3.9" + "@scalar/client-side-rendering": "0.4.0" }, "engines": { "node": ">=22" } }, "node_modules/@scalar/helpers": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/@scalar/helpers/-/helpers-0.11.1.tgz", - "integrity": "sha512-Knwbe0IYqFk0PPDoOKLasqglBHfyf9/zwWWqFsSNi/AtdjM29wSZXN6p8DFid6iB5B9epYH9YiSgJ6tpD00TEw==", + "version": "0.11.3", + "resolved": "https://registry.npmjs.org/@scalar/helpers/-/helpers-0.11.3.tgz", + "integrity": "sha512-4zPzuNTXObDUtZS93xzAoK83ddHDgBGifv/vKFe6bHqEl5i+IMLTTtEHOaaOZK4dusPEXcXsU5TBSaY/I6Mi+A==", "license": "MIT", "engines": { "node": ">=22" @@ -1674,12 +1674,12 @@ } }, "node_modules/@scalar/json-magic": { - "version": "0.13.2", - "resolved": "https://registry.npmjs.org/@scalar/json-magic/-/json-magic-0.13.2.tgz", - "integrity": "sha512-T8rQw5u7+MSTDpUcd5ShX1taOUxpZMv2b/P6xsahdlv/u68VX/Bq/+uzuAf2xW8IIOy7BEP4MBggle/vMDgAXw==", + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/@scalar/json-magic/-/json-magic-0.13.4.tgz", + "integrity": "sha512-pOZdlzkgLB+/4OlIlzMToV/cr4vsvWy/MtbtJoRcNHIzDnT8sfNtv/cH8MR2pNDg/29sPpTV1HaE4SZ8b9jUOQ==", "license": "MIT", "dependencies": { - "@scalar/helpers": "0.11.1", + "@scalar/helpers": "0.11.3", "pathe": "^2.0.3", "yaml": "^2.9.0" }, @@ -1688,15 +1688,15 @@ } }, "node_modules/@scalar/oas-utils": { - "version": "0.19.14", - "resolved": "https://registry.npmjs.org/@scalar/oas-utils/-/oas-utils-0.19.14.tgz", - "integrity": "sha512-rnVIOK6+oHTc4SeXQBkEj+Kb2xB/VUJxckKGNdHnHHlsLjpN4VXhwUBldYAvV9dA/AENfeMj5ys6GubP0RyNwQ==", + "version": "0.19.16", + "resolved": "https://registry.npmjs.org/@scalar/oas-utils/-/oas-utils-0.19.16.tgz", + "integrity": "sha512-0u0/vd62lEektF9u6d7ywAYwamkrG1xTfxMf5gOkRGTVZJ7jV+J9LoSfUv+NCR3mmQpeGyKSaiD/3/Psa4OwRA==", "license": "MIT", "dependencies": { - "@scalar/helpers": "0.11.1", - "@scalar/themes": "0.17.3", - "@scalar/types": "0.18.2", - "@scalar/workspace-store": "0.58.1", + "@scalar/helpers": "0.11.3", + "@scalar/themes": "0.17.4", + "@scalar/types": "0.19.0", + "@scalar/workspace-store": "0.60.0", "flatted": "^3.4.0", "vue": "^3.5.40", "yaml": "^2.9.0" @@ -1727,12 +1727,12 @@ } }, "node_modules/@scalar/schemas": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/@scalar/schemas/-/schemas-0.8.3.tgz", - "integrity": "sha512-cTjgiJxXFXMqXlFZafXOyTOKm1lUfbDTbe9La+dPcQPe6q0zXAiVtys0IKILQWNrjXPidY0eaB9Va/rd16bfxw==", + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@scalar/schemas/-/schemas-0.9.0.tgz", + "integrity": "sha512-yYRlIWzw+7HuIX4z7rk7tg4y1nERwvvWKbolZOm7LveSTrppllGKyjtnIqS5uXsmJddERxuurSgDW224gGJlFQ==", "license": "MIT", "dependencies": { - "@scalar/helpers": "0.11.1", + "@scalar/helpers": "0.11.3", "@scalar/validation": "0.6.3" }, "engines": { @@ -1740,17 +1740,17 @@ } }, "node_modules/@scalar/sidebar": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/@scalar/sidebar/-/sidebar-0.10.1.tgz", - "integrity": "sha512-RbDJD22tAMGquDh7ItUeVujSvQxLoc7Eo95gPhHaokEYJprBs/B30Es3J1qhNZQ6IVZ0kcxCjuD4haI1h/UmrQ==", + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/@scalar/sidebar/-/sidebar-0.11.1.tgz", + "integrity": "sha512-xXgB0WYWG4WJTFL93WkvoAVdD1H4k+A2n2jgwa/PrUM6RO8TTh+7BLcekpwvPLvXd5muWxzWDTcR413saocphw==", "license": "MIT", "dependencies": { - "@scalar/components": "0.28.1", - "@scalar/helpers": "0.11.1", + "@scalar/components": "0.29.1", + "@scalar/helpers": "0.11.3", "@scalar/icons": "0.7.6", - "@scalar/themes": "0.17.3", - "@scalar/use-hooks": "0.4.10", - "@scalar/workspace-store": "0.58.1", + "@scalar/themes": "0.17.4", + "@scalar/use-hooks": "0.4.11", + "@scalar/workspace-store": "0.60.0", "vue": "^3.5.40" }, "engines": { @@ -1758,13 +1758,13 @@ } }, "node_modules/@scalar/snippetz": { - "version": "0.9.28", - "resolved": "https://registry.npmjs.org/@scalar/snippetz/-/snippetz-0.9.28.tgz", - "integrity": "sha512-xpzQ5NgJDfV5Y5Xmpo2lDZclbXuIRolIm6qAaShNVBvO3q2GYtxJ9RSsrMFTpuk1NIqcQ4WKVAVRllRhYuzlWg==", + "version": "0.9.30", + "resolved": "https://registry.npmjs.org/@scalar/snippetz/-/snippetz-0.9.30.tgz", + "integrity": "sha512-mDluVSGZet1Go8NgJK9s9Z8zNKqePG7zNn8PMCphAzwXNomvMy6j8WRuLDln+Dz33jILfiKlUtv4cnLkmzB+7g==", "license": "MIT", "dependencies": { - "@scalar/helpers": "0.11.1", - "@scalar/types": "0.18.2", + "@scalar/helpers": "0.11.3", + "@scalar/types": "0.19.0", "js-base64": "^3.9.2", "stringify-object": "^6.0.0" }, @@ -1773,9 +1773,9 @@ } }, "node_modules/@scalar/themes": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@scalar/themes/-/themes-0.17.3.tgz", - "integrity": "sha512-QJPHeGCg0hF30IGPjX2nMcMOvBYyd62d5vey1mIC17CLhOe5tLVTn9D6G175D+jPVb0WhVPcSbaax6KxSZTUEQ==", + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@scalar/themes/-/themes-0.17.4.tgz", + "integrity": "sha512-tSCtGLb0noijR8GzgH6H/tlbSuZoLupe66ta6I9FzZxdvgj4SdPM+2Q7E8k1uwPRfrE5NvMyp61s1BsA3DTFsg==", "license": "MIT", "dependencies": { "nanoid": "^5.1.6" @@ -1791,15 +1791,15 @@ "license": "MIT" }, "node_modules/@scalar/types": { - "version": "0.18.2", - "resolved": "https://registry.npmjs.org/@scalar/types/-/types-0.18.2.tgz", - "integrity": "sha512-q7fGMn0IygdLbYk9W4quM0w1caHDfL9FIxsYXNsgIep6uuO0T/t4BOStyf0qyzQ3pjt0B7rWFxO//EM9I7F/Tw==", + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/@scalar/types/-/types-0.19.0.tgz", + "integrity": "sha512-EKeoWgUlP+uepbM/zEHKbsdpBNyOmSrw6DdU/WC0p53gz5uRzI7d/IEecIP9SQMUkResPR9DmWeWfFQftG7vqg==", "license": "MIT", "dependencies": { - "@scalar/helpers": "0.11.1", + "@scalar/helpers": "0.11.3", "nanoid": "^5.1.6", "type-fest": "^5.8.0", - "zod": "^4.3.5" + "zod": "^4.4.3" }, "engines": { "node": ">=22" @@ -1832,11 +1832,12 @@ } }, "node_modules/@scalar/use-hooks": { - "version": "0.4.10", - "resolved": "https://registry.npmjs.org/@scalar/use-hooks/-/use-hooks-0.4.10.tgz", - "integrity": "sha512-YDIohEujqRmPCLpRlE+NTzjKPjeMJZtI4oJcZ5uH2vA1gR8wiaEStK8djBsldOFIn+LmhHagl+HHn6NX054f7Q==", + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/@scalar/use-hooks/-/use-hooks-0.4.11.tgz", + "integrity": "sha512-wCUn9WWKv4abiFZOcjBp8nIcyZ30t7QNPgHbcYu5MXgLgPclYwKb/A+TZiyp63TORWJOK8sp1RzrNGw2CT+m/Q==", "license": "MIT", "dependencies": { + "@scalar/helpers": "0.11.3", "@scalar/use-toasts": "0.10.5", "@scalar/validation": "0.6.3", "@vueuse/core": "13.9.0", @@ -1871,19 +1872,19 @@ } }, "node_modules/@scalar/workspace-store": { - "version": "0.58.1", - "resolved": "https://registry.npmjs.org/@scalar/workspace-store/-/workspace-store-0.58.1.tgz", - "integrity": "sha512-aKBwM7Tp+VzdxqVC971c8uxM8w9cw+RS7JY5V/VHj29HyNwKQTSJX6+qf8gma46bgUK0W15Ra/T8rN1mAR+EIw==", + "version": "0.60.0", + "resolved": "https://registry.npmjs.org/@scalar/workspace-store/-/workspace-store-0.60.0.tgz", + "integrity": "sha512-O3Zp6Olq7+L2Yp6xd7Z9sIQ4VG5SwR2oHkiGTagZBSrqEuvuce5Z93q3NVPux6j3cID/geW0vC0sPJ5LxNTnBA==", "license": "MIT", "dependencies": { - "@scalar/asyncapi-upgrader": "0.1.7", - "@scalar/helpers": "0.11.1", - "@scalar/json-magic": "0.13.2", + "@scalar/asyncapi-upgrader": "0.1.9", + "@scalar/helpers": "0.11.3", + "@scalar/json-magic": "0.13.4", "@scalar/openapi-upgrader": "0.2.15", - "@scalar/schemas": "0.8.3", - "@scalar/snippetz": "0.9.28", + "@scalar/schemas": "0.9.0", + "@scalar/snippetz": "0.9.30", "@scalar/typebox": "0.1.3", - "@scalar/types": "0.18.2", + "@scalar/types": "0.19.0", "@scalar/validation": "0.6.3", "js-base64": "^3.9.2", "type-fest": "^5.8.0", @@ -1978,9 +1979,9 @@ } }, "node_modules/@tanstack/virtual-core": { - "version": "3.17.8", - "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.17.8.tgz", - "integrity": "sha512-BfEvehNpOT75r5Ksc5xW6NZuXujTfb7nlSEyVu4XHG3gdxNg1KqXruWbDewXOUaUYIo4oRbSfkjIajz4MAT8tA==", + "version": "3.17.9", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.17.9.tgz", + "integrity": "sha512-M8Bzy7CCMvUjRiuoHVH9mRjyUVczRs8v8RcUEphLFGc445ZP4KQ15Y1sLqhQqJi/HgGJrxfCHnEnIX0x9mVrBg==", "license": "MIT", "funding": { "type": "github", @@ -1988,12 +1989,12 @@ } }, "node_modules/@tanstack/vue-virtual": { - "version": "3.13.36", - "resolved": "https://registry.npmjs.org/@tanstack/vue-virtual/-/vue-virtual-3.13.36.tgz", - "integrity": "sha512-gKpExv4RbB9luVG+SucTXoqPZv/gzu/Yvz6BNO+8kpNxJ2x+I/ulryzl5W9BRciahZGp5Tls3Dp5XP1ztVGbMw==", + "version": "3.13.37", + "resolved": "https://registry.npmjs.org/@tanstack/vue-virtual/-/vue-virtual-3.13.37.tgz", + "integrity": "sha512-1QNT8EXVKUx537/qvn/tFGs48eITBrrTWcyaMVjD4SCpij7E1LksbXZQLqkXBoJ9lfOVely/8Fgzqe/olQ+drw==", "license": "MIT", "dependencies": { - "@tanstack/virtual-core": "3.17.8" + "@tanstack/virtual-core": "3.17.9" }, "funding": { "type": "github", @@ -2058,9 +2059,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "24.13.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", - "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "version": "24.13.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.4.tgz", + "integrity": "sha512-YJ7EqCstVTzIr0fMr7qul/977en+pQHrfmuKIo6Zr9i75Be21dr3MovcfvGtyvi2HAUrRerWps5sMO9I7WaxDw==", "license": "MIT", "dependencies": { "undici-types": "~7.18.0" @@ -2094,9 +2095,9 @@ } }, "node_modules/@ungap/structured-clone": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.3.tgz", - "integrity": "sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.4.0.tgz", + "integrity": "sha512-1mEZtMKPM09vDmQt5y7YvmN2+DFTP7Tg0EWXdic8/C6VRnpb33e4ghisCIE3WZjsE2N8mf+QV1Zqh7ZFYLWInQ==", "license": "ISC" }, "node_modules/@unhead/vue": { @@ -2125,13 +2126,13 @@ } }, "node_modules/@vue/compiler-core": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.41.tgz", - "integrity": "sha512-q0Xtv/F9w2YO/7htQhtiL+Ev2WCJbe5N2hc+XfgyKkEKqWpSxknmT8QOuGdEKNdjPq0c3F7rNpFkTo3Kfrm7pg==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.42.tgz", + "integrity": "sha512-2Ye1ilMtKXxl8qZUrQ5j0CdgenFp/HFQmta6rfRyfEsTG69L6Wk+tWuNoHYHMx9E8tF2Slvdg1FuwDvAXdy1LQ==", "license": "MIT", "dependencies": { "@babel/parser": "^7.29.8", - "@vue/shared": "3.5.41", + "@vue/shared": "3.5.42", "entities": "^7.0.1", "estree-walker": "^2.0.2", "source-map-js": "^1.2.1" @@ -2150,26 +2151,26 @@ } }, "node_modules/@vue/compiler-dom": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.41.tgz", - "integrity": "sha512-oKacVfNglLvGjnS6BXOlGL7EyG2h8X03pqXCjzotRZUaXGjbrTJUnVAQjrCqUnS+lyu31nwQjZY/d817GmCnfw==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.42.tgz", + "integrity": "sha512-qbhQZEFmycr+ni/qyuccS4sucNN7VAbDfbkvNxWOX2VfgFm90MNs3/UhRNKoPMEIVn0F8gdlYjLPvqxHwHeQOA==", "license": "MIT", "dependencies": { - "@vue/compiler-core": "3.5.41", - "@vue/shared": "3.5.41" + "@vue/compiler-core": "3.5.42", + "@vue/shared": "3.5.42" } }, "node_modules/@vue/compiler-sfc": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.41.tgz", - "integrity": "sha512-XJhip7R2wy6vX3knCxdZN4KracFaZUef58s1KYewqluedHIJaPIVfXoYT7MF1F8nCvv6k8bWWxDC8opMkg1VTQ==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.42.tgz", + "integrity": "sha512-fkCAFB4okcAANGMThboWnScp/gzWjU0ZSkVnjTIiplmMDq2uq0tIB3j+xVu4rhv5rvOgBySCysudmbMd6xRRqw==", "license": "MIT", "dependencies": { "@babel/parser": "^7.29.8", - "@vue/compiler-core": "3.5.41", - "@vue/compiler-dom": "3.5.41", - "@vue/compiler-ssr": "3.5.41", - "@vue/shared": "3.5.41", + "@vue/compiler-core": "3.5.42", + "@vue/compiler-dom": "3.5.42", + "@vue/compiler-ssr": "3.5.42", + "@vue/shared": "3.5.42", "estree-walker": "^2.0.2", "magic-string": "^0.30.21", "postcss": "^8.5.19", @@ -2177,61 +2178,61 @@ } }, "node_modules/@vue/compiler-ssr": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.41.tgz", - "integrity": "sha512-U3v5OejKEGqOI0Wy0+Sz7hGuIFZHA4LSXzrNM3IMIeDyJEBBfTpX26n3SDgToRpP2bLc9FfI2j/kSgcJ8Emq5A==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.42.tgz", + "integrity": "sha512-xmLk3wLkbizPAiLyomjgFFosf2ys9b5Ghb+oh/k2tnvipNz8OFrQOiTcWCzyK7MpBp9KkyGtfvgfLUivbmuGYA==", "license": "MIT", "dependencies": { - "@vue/compiler-dom": "3.5.41", - "@vue/shared": "3.5.41" + "@vue/compiler-dom": "3.5.42", + "@vue/shared": "3.5.42" } }, "node_modules/@vue/reactivity": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.41.tgz", - "integrity": "sha512-rznsqKM0np0x18EjzF8x88MpEhdNsffbvFbckLL5+oUKz1BxAImEmO7J1ArRYSyo6aQaVoBDp7jEkT91OOxydA==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.42.tgz", + "integrity": "sha512-TzNNfKpb7hDxbQltwAut8VDQA5YP+BuRlxntHUuRjyKwlMvmAPbs3unhCvieijifY6vFfVBwsS7wG/C7uq+bEQ==", "license": "MIT", "dependencies": { - "@vue/shared": "3.5.41" + "@vue/shared": "3.5.42" } }, "node_modules/@vue/runtime-core": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.41.tgz", - "integrity": "sha512-Vcry58hiAKwGen9Z1jUZE0feFsNArPCMOImYI8el48A9Idf6DuQYD0U05zZIF2Iad1hGhPSvcbBbAOhNr55fhg==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.42.tgz", + "integrity": "sha512-9uACtuHs7vJGkm5Bp3xu4xRDLFTIYy5DgxpToVjqGIAhAEKwQfsaLvKINhM6nFVp6bZPRFGdDqd1g52MqKsotA==", "license": "MIT", "dependencies": { - "@vue/reactivity": "3.5.41", - "@vue/shared": "3.5.41" + "@vue/reactivity": "3.5.42", + "@vue/shared": "3.5.42" } }, "node_modules/@vue/runtime-dom": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.41.tgz", - "integrity": "sha512-3vVBahVBS9+U6cmXBLyb8nE6/yYo4J/CGI9eVFs3KiMc0YHuudwKyShTD65jtJy/L9PUUxNAFu4cj4LiJ0UFbw==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.42.tgz", + "integrity": "sha512-rsCmhiWLaRxGltLwhlCWyYkFn7WAbKRh0q17eZ1A6Dq6eqc2ACQ61IIryxz0LrsvCzHSilLA9JHovVwM8CNE2g==", "license": "MIT", "dependencies": { - "@vue/reactivity": "3.5.41", - "@vue/runtime-core": "3.5.41", - "@vue/shared": "3.5.41", + "@vue/reactivity": "3.5.42", + "@vue/runtime-core": "3.5.42", + "@vue/shared": "3.5.42", "csstype": "^3.2.3" } }, "node_modules/@vue/server-renderer": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.41.tgz", - "integrity": "sha512-n6hx/pNFfbD6SuyeuMVkvqox8bwf/ET9JlA/kAz/imw8sw++wkqKe2mHX5KutjPpbKE4Z56yTHszoOjGMI9igQ==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.42.tgz", + "integrity": "sha512-2++5dUyYS4gvo7xQXSECUDhB7TS0aOl5SeVfC5qSq1Jgfhjvegw1zqhwTIR3imZ+QYPJQw9gfcFvXGAjGZ7ajQ==", "license": "MIT", "dependencies": { - "@vue/compiler-ssr": "3.5.41", - "@vue/runtime-dom": "3.5.41", - "@vue/shared": "3.5.41" + "@vue/compiler-ssr": "3.5.42", + "@vue/runtime-dom": "3.5.42", + "@vue/shared": "3.5.42" } }, "node_modules/@vue/shared": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.41.tgz", - "integrity": "sha512-IOnwSCma8j+9xJT6b8H0dEYidC80NsYmNMlZxRsukYcSoGaDBohog5hDxzeUXdFeGWFA++vWvxqOmrr96VlqMA==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.42.tgz", + "integrity": "sha512-2rPxex1jQf4jvl9MOHl6YaXCPcrNqz/FstMOEh3QWY+/OME9nQTvl9WYeCwhW7AFjaR0SnngZGlp/wkR6rkI6g==", "license": "MIT" }, "node_modules/@vueuse/core": { @@ -3233,16 +3234,15 @@ } }, "node_modules/engine.io": { - "version": "6.6.9", - "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.6.9.tgz", - "integrity": "sha512-clKkw4C7nJ22mGgoVcCg6V/W/TxdNyIOTr89k2ONZu81qqkddPFDF0LXcbAwhzPD8DjkiRCjzuiO6Y+fkpD4vg==", + "version": "6.6.10", + "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.6.10.tgz", + "integrity": "sha512-9/lX2bdlizlCXMHRMOIm03VBQHQYC7VvydcxtTAUJRxNW1QzM/2PMFSmr6h/lCiMHcyCP6abK+t9Q+j4vekk8Q==", "license": "MIT", "dependencies": { "@types/cors": "^2.8.12", "@types/node": ">=10.0.0", "@types/ws": "^8.5.12", "accepts": "~1.3.4", - "base64id": "2.0.0", "cookie": "~0.7.2", "cors": "~2.8.5", "debug": "~4.4.1", @@ -3466,9 +3466,9 @@ } }, "node_modules/express-rate-limit": { - "version": "8.6.2", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.6.2.tgz", - "integrity": "sha512-YH4ru+eOJxQABscKFfRCy9R7x9QFGdezclVMwwgFFndzS2Xnm0uo6B0ABZsLhcpeptGv2qvuJVWlQr9gQZoC3A==", + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.7.0.tgz", + "integrity": "sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g==", "license": "MIT", "dependencies": { "debug": "^4.4.3", @@ -3563,9 +3563,9 @@ "license": "MIT" }, "node_modules/fast-jwt": { - "version": "6.3.2", - "resolved": "https://registry.npmjs.org/fast-jwt/-/fast-jwt-6.3.2.tgz", - "integrity": "sha512-JTQImpkXVvj+eq7tJImtsHRt1K6ngloEzIx62Qbf9x4tEM2P2EqGpYJSeUoPf/kMn78rImSHfhf08KShR8PauA==", + "version": "6.3.3", + "resolved": "https://registry.npmjs.org/fast-jwt/-/fast-jwt-6.3.3.tgz", + "integrity": "sha512-pQDXx7IHeZT4jSmpE9o80RrBqfrG4fPrl8anazSM5vErIdK1iCc13z/EWX+H0j7liWSRnwTpHswIKMeLYGAckw==", "license": "Apache-2.0", "dependencies": { "@lukeed/ms": "^2.0.2", @@ -3579,9 +3579,9 @@ } }, "node_modules/fast-uri": { - "version": "3.1.6", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz", - "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz", + "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==", "funding": [ { "type": "github", @@ -3897,19 +3897,19 @@ } }, "node_modules/graphql-tools": { - "version": "9.0.34", - "resolved": "https://registry.npmjs.org/graphql-tools/-/graphql-tools-9.0.34.tgz", - "integrity": "sha512-pccboGsOGmF5falh1aKJTX0u7Apot8m66dsIU7aUaVQ8RWpoXqT37j6b1O9ToA0oMuSuS+qLt8LokTmsEsP1Gw==", + "version": "9.0.35", + "resolved": "https://registry.npmjs.org/graphql-tools/-/graphql-tools-9.0.35.tgz", + "integrity": "sha512-bmHHVqGIqkRCb95ShdNS4oMR3bayUz+Q26pVt5JMVG8IpUSMnrjL3uqbNtaTMXn3wLnkaARE039W7hwXpIOE9Q==", "license": "MIT", "dependencies": { - "@graphql-tools/schema": "^10.1.0", + "@graphql-tools/schema": "^10.1.1", "tslib": "^2.4.0" }, "engines": { "node": ">=16.0.0" }, "optionalDependencies": { - "@apollo/client": "~4.2.10" + "@apollo/client": "~4.2.12" }, "peerDependencies": { "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" @@ -4300,9 +4300,9 @@ } }, "node_modules/hono": { - "version": "4.13.4", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.4.tgz", - "integrity": "sha512-AGEwKIyRMHRv1t8Wjwa3LHxQ61X5CqrdFT+4BRNTpqS5aJNnpl5WLjADb7vFlJzI/8uK7T5QLVApCMQKNa3LgQ==", + "version": "4.13.7", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.7.tgz", + "integrity": "sha512-c8/gF9ac8Y78/agExVocyLevgR+JlpNB444Py0FSX8pJoPdYUfUzRcXtYEYGwt6l19qIlVZPN5Mfsw9jFShmQQ==", "license": "MIT", "engines": { "node": ">=16.9.0" @@ -4427,9 +4427,9 @@ } }, "node_modules/ip-address": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz", - "integrity": "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==", + "version": "10.7.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.0.tgz", + "integrity": "sha512-BGFsyJd5mpXp3rK6jIdADLNgpJUK1jnjzvYF8lK+VyDab9JAmqN0YOKDdP17HlgKb2+ehPgDc8EtnRLbGCAMhA==", "license": "MIT", "engines": { "node": ">= 12" @@ -4575,9 +4575,9 @@ "license": "ISC" }, "node_modules/jose": { - "version": "6.2.10", - "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.10.tgz", - "integrity": "sha512-iiW7J9qRFlGxvCOIBDBDxFePQSn7ZMAnrYGhrrOo6siO/MIqwfyilLR27pkfDgUk+raLuzADS8A3S/KLBisc0g==", + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/panva" @@ -4763,6 +4763,15 @@ "node": ">= 12.0.0" } }, + "node_modules/logform/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "license": "MIT", + "engines": { + "node": ">=0.1.90" + } + }, "node_modules/loglevel": { "version": "1.9.2", "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz", @@ -6053,9 +6062,9 @@ } }, "node_modules/postcss": { - "version": "8.5.26", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", - "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", "funding": [ { "type": "opencollective", @@ -6072,7 +6081,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.17", + "nanoid": "^3.3.18", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -6099,9 +6108,9 @@ } }, "node_modules/pretty-ms": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-9.3.0.tgz", - "integrity": "sha512-gjVS5hOP+M3wMm5nmNOucbIrqudzs9v/57bWRHQWLYklXqoXKrVfYW2W9+glfGsqtPgpiz5WwyEEB+ksXIx3gQ==", + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-9.3.1.tgz", + "integrity": "sha512-HzMy3Geq23nVALD/M2LliU+F+M+gVNsvkQWWqeBZ8HDiCgzo6YPJ/Omrmtq24EFrIsk0a3EkQGEd7bDOo+IhGA==", "license": "MIT", "dependencies": { "parse-ms": "^4.0.0" @@ -6117,6 +6126,7 @@ "version": "15.1.3", "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "deprecated": "prom-client has been replaced by @prometheus-io/client", "license": "Apache-2.0", "dependencies": { "@opentelemetry/api": "^1.4.0", @@ -6137,9 +6147,9 @@ } }, "node_modules/protobufjs": { - "version": "8.7.2", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.7.2.tgz", - "integrity": "sha512-oTVHV+oelUBtiu5iTuTNNZ0eLYsXSMxry4cgr30mayNkgIZL6qZ0IOQVPuSWGcyAaXKl/XgqwWHIC3a0khYVBA==", + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.8.0.tgz", + "integrity": "sha512-N3xhQ5yyBx3vQq4gubBfASzYhJGNzeDbjqBpu61g7UVylsN/qyffU96TKWD3GbbLOKF82VGNRNvv1+BFgE31Eg==", "license": "BSD-3-Clause", "dependencies": { "long": "^5.3.2" @@ -6177,9 +6187,9 @@ } }, "node_modules/qs": { - "version": "6.15.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", - "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", "license": "BSD-3-Clause", "dependencies": { "es-define-property": "^1.0.1", @@ -7155,9 +7165,9 @@ } }, "node_modules/swagger-ui-dist": { - "version": "5.32.14", - "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-5.32.14.tgz", - "integrity": "sha512-nOA2pSQhcmODMUQZpJHYKNuwniDUqcOWGNaSCOoZv12FdOSJ9JxV95HtyRGNMqEBj6h6lCNTy20TgZDYTSuUIg==", + "version": "5.32.15", + "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-5.32.15.tgz", + "integrity": "sha512-TSFER+rFQlf1nzk6WvKkMaHTxAPQ3eAAxigFThnxQedSREanfZgSbJFayZVs/ULnSbNdrJOb99vLD6xpb3R3eg==", "license": "Apache-2.0", "dependencies": { "@scarf/scarf": "=1.4.0" @@ -7223,9 +7233,9 @@ "peer": true }, "node_modules/tdigest": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", - "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.3.tgz", + "integrity": "sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==", "license": "MIT", "dependencies": { "bintrees": "1.0.2" @@ -7339,9 +7349,9 @@ "license": "0BSD" }, "node_modules/type-fest": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.8.0.tgz", - "integrity": "sha512-YGYEVz3Fm5iy/AybuA0oyNFq7H4CgQNfRp/qfe8nurE1kuCeNm3/vfm9X4Mtl+qLyaKJUh5xrFZwogr41SMjYA==", + "version": "5.9.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.9.0.tgz", + "integrity": "sha512-yANm3Jr3GiJ1qgJlxGAVxTOIcEOk1rhQHamlXtnrCK7EHP4HeM9OGxtMg/W7HFdrVzw/ZWJKGVIJusVH85sLtw==", "license": "(MIT OR CC0-1.0)", "dependencies": { "tagged-tag": "^1.0.0" @@ -7612,16 +7622,16 @@ } }, "node_modules/vue": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.41.tgz", - "integrity": "sha512-2laE0p+aK+/AOPG/XL/WepOs/GlK755LJ1XECi9kDUrz1FKNw8rb2Xzlw9JS1rqEV55nb0ttsKxVlTCcd+R5cg==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.42.tgz", + "integrity": "sha512-4RyHQTbQvOPs3MfvUO1Sg0YRrKNnA0mAVtvpd12Tg1fKDN7OHBUl1IqSn8zGJjK9nI3NkNp8cgTpVrSZC5TTcA==", "license": "MIT", "dependencies": { - "@vue/compiler-dom": "3.5.41", - "@vue/compiler-sfc": "3.5.41", - "@vue/runtime-dom": "3.5.41", - "@vue/server-renderer": "3.5.41", - "@vue/shared": "3.5.41" + "@vue/compiler-dom": "3.5.42", + "@vue/compiler-sfc": "3.5.42", + "@vue/runtime-dom": "3.5.42", + "@vue/server-renderer": "3.5.42", + "@vue/shared": "3.5.42" }, "peerDependencies": { "typescript": "*" @@ -7756,9 +7766,9 @@ "license": "Apache-2.0" }, "node_modules/winston-loki/node_modules/protobufjs": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", - "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -7897,9 +7907,9 @@ } }, "node_modules/zod": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", - "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.0.tgz", + "integrity": "sha512-iIvwyDnebKYpww2ta0DjaNOL8RnVLmPMhgWyOzlW9y0EIIxv5gl+H6Y0ONpt83HqjGqkk78uWp6xWtpxzZdPbw==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/packages/core/package.bundle.json b/packages/core/package.bundle.json index 4288fc87e..6a49d8a0b 100644 --- a/packages/core/package.bundle.json +++ b/packages/core/package.bundle.json @@ -28,7 +28,7 @@ "prom-client": "^15.1.3", "protobufjs": "^8.7.2", "snappy": "7.4.1", - "uuid": "^14.0.1", + "uuid": "^14.0.2", "winston": "^3.19.0", "winston-loki": "^6.1.7", "@apollo/cache-control-types": "^1.0.3", @@ -36,7 +36,7 @@ "@as-integrations/express5": "^1.1.2", "@modelcontextprotocol/sdk": "^1.29.0", "@scalar/api-reference": "^1.60.0", - "@scalar/express-api-reference": "^0.10.14", + "@scalar/express-api-reference": "^0.10.16", "@socket.io/redis-streams-adapter": "^0.3.1", "bcrypt": "^6.0.0", "body-parser": "^2.3.0", diff --git a/packages/core/package.json b/packages/core/package.json index 669147c01..1d115d7e0 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -19,7 +19,8 @@ "prebuild:bundle": "pnpm --filter @conduitplatform/service-bundle run build", "build:bundle": "rimraf bundle && node ../../libraries/service-bundle/dist/cli.js generate-manifest && tsup && node ../../libraries/service-bundle/dist/cli.js copy-assets && node ../../libraries/service-bundle/dist/cli.js generate-lockfile", "prepare": "npm run build", - "prepublish": "npm run build" + "prepublish": "npm run build", + "test": "node --experimental-strip-types --test tests/*.test.ts" }, "license": "ISC", "dependencies": { diff --git a/packages/core/tests/embeddingsConfigLifecycle.test.ts b/packages/core/tests/embeddingsConfigLifecycle.test.ts new file mode 100644 index 000000000..8fc390219 --- /dev/null +++ b/packages/core/tests/embeddingsConfigLifecycle.test.ts @@ -0,0 +1,186 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import convict from 'convict'; +import { merge as lodashMerge } from 'lodash-es'; +import { + merge, + reconcileStoredModuleConfig, + redactSensitiveConfig, + restoreRedactedSecrets, +} from '@conduitplatform/module-tools'; +import { getModuleConfigRoute } from '../dist/admin/routes/GetModuleConfig.route.js'; +import { setModuleConfigRoute } from '../dist/admin/routes/SetModuleConfig.route.js'; +import AppConfigSchema, { + type Config, +} from '../../../modules/embeddings/dist/config/index.js'; +import { normalizeEmbeddingsConfig } from '../../../modules/embeddings/dist/utils/providerConfig.js'; + +const MODULE_NAME = 'embeddings'; +const STORE_KEY = `moduleConfigs.${MODULE_NAME}`; + +const legacyStored = { + enabled: false, + defaultProvider: 'openai-compatible', + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-live', + model: 'text-embedding-3-small', + dimensions: 1536, + models: [], + allowedHosts: ['api.openai.com'], + }, + }, + queue: { + concurrency: 2, + attempts: 3, + maxBatchSize: 500, + drainTimeoutMs: 15 * 60 * 1000, + }, + security: { + requireGrpcKey: true, + sourceFieldAllowlist: [], + maxMutationEventIds: 500, + embedTimeoutMs: 10_000, + maxEmbedInputBytes: 32 * 1024, + maxEmbedResponseBytes: 1024 * 1024, + }, +}; + +function providerOf(config: Record) { + const providers = config.providers as Record>; + return providers['openai-compatible']; +} + +describe('embeddings Admin GET/PATCH config lifecycle', () => { + it('persists catalogue migration so an unrelated PATCH cannot wipe it', async () => { + const store = new Map(); + store.set(STORE_KEY, JSON.stringify(legacyStored)); + const schema = convict(AppConfigSchema); + let configureCalls = 0; + + const local = normalizeEmbeddingsConfig(schema.getProperties() as Config); + const existing = JSON.parse(store.get(STORE_KEY)!) as Config; + const merged = lodashMerge({}, local, existing) as Config; + store.set(STORE_KEY, JSON.stringify(merged)); + + const migrated = normalizeEmbeddingsConfig(merged); + schema.load(migrated).validate({ allowed: 'warn' }); + const persistable = schema.getProperties() as Config; + const reconciled = await reconcileStoredModuleConfig({ + stored: merged, + migrated: persistable, + configureOverride: async next => { + configureCalls += 1; + store.set(STORE_KEY, JSON.stringify(next)); + return next; + }, + }); + schema.load(reconciled.config); + assert.equal(configureCalls, 1); + + const second = await reconcileStoredModuleConfig({ + stored: JSON.parse(store.get(STORE_KEY)!) as Config, + migrated: schema.getProperties() as Config, + configureOverride: async next => { + configureCalls += 1; + store.set(STORE_KEY, JSON.stringify(next)); + return next; + }, + }); + assert.equal(second.persisted, false); + assert.equal(configureCalls, 1); + + const storedAfterLifecycle = JSON.parse(store.get(STORE_KEY)!) as Record< + string, + unknown + >; + const storedProvider = providerOf(storedAfterLifecycle); + assert.deepEqual(storedProvider.models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + assert.equal(storedProvider.defaultModel, 'text-embedding-3-small'); + assert.equal(storedProvider.apiKey, 'sk-live'); + assert.equal('model' in storedProvider, false); + assert.equal('dimensions' in storedProvider, false); + assert.equal('allowedHosts' in storedProvider, false); + const storedSecurity = storedAfterLifecycle.security as Record; + assert.equal('requireGrpcKey' in storedSecurity, false); + + const grpcSdk = { + state: { + getKey: async (key: string) => store.get(key) ?? null, + }, + getModuleClient: () => ({ + setConfig: async ({ newConfig }: { newConfig: string }) => { + const previous = schema.getProperties() as Config; + let next = merge(previous, JSON.parse(newConfig) as Config); + next = restoreRedactedSecrets(next, previous, AppConfigSchema); + next = normalizeEmbeddingsConfig(next); + schema.load(next).validate({ allowed: 'warn' }); + return { updatedConfig: JSON.stringify(schema.getProperties()) }; + }, + }), + }; + const configManager = { + set: async (_name: string, config: unknown) => { + store.set(STORE_KEY, JSON.stringify(config)); + return config; + }, + }; + + const getRoute = getModuleConfigRoute(grpcSdk as never, MODULE_NAME, AppConfigSchema); + const getResponse = await getRoute.executeRequest({} as never); + const getProvider = providerOf(getResponse.config); + assert.equal(getProvider.apiKey, '[REDACTED]'); + assert.deepEqual(getProvider.models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + assert.equal(getProvider.defaultModel, 'text-embedding-3-small'); + assert.equal('model' in getProvider, false); + assert.equal('dimensions' in getProvider, false); + assert.equal('allowedHosts' in getProvider, false); + assert.doesNotMatch(JSON.stringify(getResponse), /sk-live/); + assert.equal( + redactSensitiveConfig(getResponse.config, AppConfigSchema).providers[ + 'openai-compatible' + ].apiKey, + '[REDACTED]', + ); + + const patchRoute = setModuleConfigRoute( + grpcSdk as never, + configManager, + MODULE_NAME, + AppConfigSchema, + ); + const patchResponse = await patchRoute.executeRequest({ + params: { config: { enabled: true } }, + } as never); + assert.equal(patchResponse.config.enabled, true); + assert.equal( + patchResponse.config.providers['openai-compatible'].apiKey, + '[REDACTED]', + ); + assert.deepEqual(patchResponse.config.providers['openai-compatible'].models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + + const storedAfterPatch = JSON.parse(store.get(STORE_KEY)!) as Record; + const patchedProvider = providerOf(storedAfterPatch); + assert.equal(storedAfterPatch.enabled, true); + assert.deepEqual(patchedProvider.models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + assert.equal(patchedProvider.defaultModel, 'text-embedding-3-small'); + assert.equal(patchedProvider.apiKey, 'sk-live'); + assert.equal('model' in patchedProvider, false); + assert.equal('dimensions' in patchedProvider, false); + + const getAfterPatch = await getRoute.executeRequest({} as never); + assert.deepEqual(providerOf(getAfterPatch.config).models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + assert.equal(providerOf(getAfterPatch.config).apiKey, '[REDACTED]'); + }); +}); From e656c4fec61fe5d1acc29110c11b7ee5090bfcef Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Thu, 10 Sep 2026 00:47:52 +0300 Subject: [PATCH 27/29] fix(embeddings): deny internal schemas as embedding targets Admin, middleware, Client, and Database-owned system schemas can be extendable in the CMS list; they still must not be embedding sources. --- modules/database/src/Database.ts | 10 +-- .../database/src/adapters/DatabaseAdapter.ts | 10 +-- .../models/__tests__/systemSchemas.test.ts | 32 ++++++++ modules/database/src/models/systemSchemas.ts | 20 +++++ .../embeddings/src/api/embeddingsApi.test.ts | 25 +++++- .../embeddings/src/utils/schemaPolicy.test.ts | 78 +++++++++++++++++++ modules/embeddings/src/utils/schemaPolicy.ts | 39 ++++++++-- 7 files changed, 194 insertions(+), 20 deletions(-) create mode 100644 modules/database/src/models/__tests__/systemSchemas.test.ts create mode 100644 modules/database/src/models/systemSchemas.ts diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index d22fa2cd2..37933941c 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -15,6 +15,7 @@ import { SchemaAdmin } from './admin/schema.admin.js'; import { CustomEndpointsAdmin } from './admin/customEndpoints/customEndpoints.admin.js'; import { DatabaseRoutes } from './routes/index.js'; import * as models from './models/index.js'; +import { DATABASE_SYSTEM_SCHEMAS } from './models/systemSchemas.js'; import { ColumnExistenceRequest, ColumnExistenceResponse, @@ -166,10 +167,9 @@ export default class DatabaseModule extends ManagedModule { const isReplica = this.grpcSdk.isAvailable('database'); await this._activeAdapter.registerSystemSchema(models.DeclaredSchema, isReplica); await this._activeAdapter.registerSystemSchema(models.MigratedSchemas, isReplica); - let modelPromises = Object.values(models).flatMap((model: ConduitSchema) => { - if (['_DeclaredSchema', 'MigratedSchemas'].includes(model.name)) return []; - return this._activeAdapter.registerSystemSchema(model, isReplica); - }); + let modelPromises = DATABASE_SYSTEM_SCHEMAS.filter( + model => !['_DeclaredSchema', 'MigratedSchemas'].includes(model.name), + ).map(model => this._activeAdapter.registerSystemSchema(model, isReplica)); await Promise.all(modelPromises); await this._activeAdapter.retrieveForeignSchemas(); await this._activeAdapter.recoverSchemasFromDatabase(); @@ -177,7 +177,7 @@ export default class DatabaseModule extends ManagedModule { if (!isReplica) { await runMigrations(this._activeAdapter); } - modelPromises = Object.values(models).flatMap((model: ConduitSchema) => { + modelPromises = DATABASE_SYSTEM_SCHEMAS.map(model => { return this._activeAdapter.registerSystemSchema(model, isReplica).then(() => { if (this._activeAdapter.getDatabaseType() !== 'MongoDB' && !isReplica) { return this._activeAdapter.syncSchema(model.name); diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index 70b08fafa..5a538634f 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -27,7 +27,7 @@ import { stitchSchema, validateExtensionFields } from './utils/extensions.js'; import { status } from '@grpc/grpc-js'; import { isEqual, isNil } from 'lodash-es'; import ObjectHash from 'object-hash'; -import * as systemModels from '../models/index.js'; +import { DATABASE_SYSTEM_SCHEMA_NAME_SET } from '../models/systemSchemas.js'; export abstract class DatabaseAdapter { registeredSchemas: Map; @@ -462,13 +462,7 @@ export abstract class DatabaseAdapter { ); models = models // do not recover system schemas as they have already been - .filter((model: _ConduitSchema) => { - let isSystemModel = false; - Object.values(systemModels).forEach((systemModel: ConduitSchema) => { - systemModel.name === model.name && (isSystemModel = true); - }); - return !isSystemModel; - }) + .filter((model: _ConduitSchema) => !DATABASE_SYSTEM_SCHEMA_NAME_SET.has(model.name)) .map((model: _ConduitSchema) => { const schema = new ConduitSchema( model.name, diff --git a/modules/database/src/models/__tests__/systemSchemas.test.ts b/modules/database/src/models/__tests__/systemSchemas.test.ts new file mode 100644 index 000000000..6389ea67c --- /dev/null +++ b/modules/database/src/models/__tests__/systemSchemas.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from '@jest/globals'; +import * as models from '../index.js'; +import { + DATABASE_SYSTEM_SCHEMA_NAMES, + DATABASE_SYSTEM_SCHEMAS, +} from '../systemSchemas.js'; + +describe('database system schema registry', () => { + it('lists the Database-owned schemas registered as system schemas', () => { + expect([...DATABASE_SYSTEM_SCHEMA_NAMES]).toEqual([ + '_DeclaredSchema', + 'MigratedSchemas', + 'CustomEndpoints', + '_PendingSchemas', + 'Views', + ]); + expect(DATABASE_SYSTEM_SCHEMAS.map(schema => schema.name)).toEqual([ + ...DATABASE_SYSTEM_SCHEMA_NAMES, + ]); + }); + + it('stays aligned with the models barrel used for registration', () => { + const barrelNames = Object.values(models) + .filter( + (value): value is { name: string } => + typeof value === 'object' && value !== null && 'name' in value, + ) + .map(schema => schema.name) + .sort(); + expect(barrelNames).toEqual([...DATABASE_SYSTEM_SCHEMA_NAMES].sort()); + }); +}); diff --git a/modules/database/src/models/systemSchemas.ts b/modules/database/src/models/systemSchemas.ts new file mode 100644 index 000000000..d36bd0f0e --- /dev/null +++ b/modules/database/src/models/systemSchemas.ts @@ -0,0 +1,20 @@ +import { CustomEndpoints } from './CustomEndpoints.schema.js'; +import { DeclaredSchema } from './DeclaredSchema.schema.js'; +import { MigratedSchemas } from './MigratedSchemas.schema.js'; +import { PendingSchemas } from './PendingSchemas.schema.js'; +import { Views } from './Views.schema.js'; + +export const DATABASE_SYSTEM_SCHEMAS = [ + DeclaredSchema, + MigratedSchemas, + CustomEndpoints, + PendingSchemas, + Views, +] as const; + +export const DATABASE_SYSTEM_SCHEMA_NAMES: readonly string[] = + DATABASE_SYSTEM_SCHEMAS.map(schema => schema.name); + +export const DATABASE_SYSTEM_SCHEMA_NAME_SET = new Set( + DATABASE_SYSTEM_SCHEMA_NAMES, +); diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts index 5501447c4..dd5996107 100644 --- a/modules/embeddings/src/api/embeddingsApi.test.ts +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -433,13 +433,21 @@ describe('typed embeddings API handlers', () => { it('gates system schemas and owner policies on config and backfill', async () => { const { api } = createApi({ - declared: { Article: { name: 'Article', ownerModule: 'cms-app' } }, + declared: { + Article: { name: 'Article', ownerModule: 'cms-app' }, + Admin: { name: 'Admin', ownerModule: 'core' }, + }, schemas: { Article: articleSchema, AccessToken: { name: 'AccessToken', fields: { token: { type: TYPE.String } }, }, + Admin: { + name: 'Admin', + fields: { username: { type: TYPE.String } }, + modelOptions: articleSchema.modelOptions, + }, }, }); await assert.rejects( @@ -457,6 +465,21 @@ describe('typed embeddings API handlers', () => { ), (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, ); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Admin', + sourceFields: ['username'], + targetField: 'embedding', + model: 'text-embedding-3-small', + dimensions: 3, + enabled: false, + }, + { platformAdmin: true }, + ), + (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); await assert.rejects( () => api.startBackfill({ schemaName: 'Article' }, { callerModule: 'chat' }), (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, diff --git a/modules/embeddings/src/utils/schemaPolicy.test.ts b/modules/embeddings/src/utils/schemaPolicy.test.ts index 1a8ab2cbc..34106d9eb 100644 --- a/modules/embeddings/src/utils/schemaPolicy.test.ts +++ b/modules/embeddings/src/utils/schemaPolicy.test.ts @@ -9,12 +9,18 @@ import { assertSchemaCanReceiveEmbeddings, assertSemanticSearchAccess, assertSourceFields, + DATABASE_SYSTEM_SCHEMA_NAMES, isDeniedEmbeddingSchema, + PLATFORM_INTERNAL_SCHEMA_NAMES, resolveAdminOperatorContext, resolveSourceFieldAllowlist, } from './schemaPolicy.js'; describe('embedding schema and source policies', () => { + const extendableEnabled = { + conduit: { cms: { enabled: true }, permissions: { extendable: true } }, + }; + it('denies system, auth-secret, and embeddings-owned schemas', () => { assert.equal(isDeniedEmbeddingSchema({ name: 'EmbeddingConfig' }), true); assert.equal(isDeniedEmbeddingSchema({ name: 'BackfillRun' }), true); @@ -27,6 +33,10 @@ describe('embedding schema and source policies', () => { isDeniedEmbeddingSchema({ name: 'Views', ownerModule: 'database' }), true, ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'CustomEndpoints', ownerModule: 'database' }), + true, + ); assert.equal( isDeniedEmbeddingSchema({ name: 'AccessToken', ownerModule: 'authentication' }), true, @@ -43,6 +53,10 @@ describe('embedding schema and source policies', () => { isDeniedEmbeddingSchema({ name: 'User', ownerModule: 'authentication' }), false, ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'Team', ownerModule: 'authentication' }), + false, + ); assert.equal( isDeniedEmbeddingSchema({ name: 'File', ownerModule: 'storage' }), false, @@ -53,6 +67,70 @@ describe('embedding schema and source policies', () => { ); }); + it('denies platform internal schemas even when they are enabled and extendable', () => { + assert.deepEqual( + [...DATABASE_SYSTEM_SCHEMA_NAMES], + [ + '_DeclaredSchema', + 'MigratedSchemas', + 'CustomEndpoints', + '_PendingSchemas', + 'Views', + ], + ); + for (const name of ['Admin', 'AdminMiddleware', 'AppMiddleware', 'Client'] as const) { + assert.equal(PLATFORM_INTERNAL_SCHEMA_NAMES.has(name), true); + } + const internals = [ + { name: 'Admin', ownerModule: 'core' }, + { name: 'AdminMiddleware', ownerModule: 'core' }, + { name: 'AppMiddleware', ownerModule: 'router' }, + { name: 'Client', ownerModule: 'router' }, + { name: 'Config', ownerModule: 'core' }, + { name: 'CustomEndpoints', ownerModule: 'database' }, + { name: 'ActorIndex', ownerModule: 'authorization' }, + ]; + for (const schema of internals) { + assert.equal(isDeniedEmbeddingSchema(schema), true); + assert.throws( + () => + assertSchemaCanReceiveEmbeddings({ + ...schema, + modelOptions: extendableEnabled, + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + } + assert.equal( + isDeniedEmbeddingSchema({ name: 'FutureCoreDoc', ownerModule: 'core' }), + true, + ); + }); + + it('allows enabled and extendable owner-controlled business schemas', () => { + assert.doesNotThrow(() => + assertSchemaCanReceiveEmbeddings({ + name: 'Article', + ownerModule: 'cms-app', + modelOptions: extendableEnabled, + }), + ); + assert.doesNotThrow(() => + assertSchemaCanReceiveEmbeddings({ + name: 'User', + ownerModule: 'authentication', + modelOptions: { conduit: { permissions: { extendable: true } } }, + }), + ); + assert.doesNotThrow(() => + assertSchemaCanReceiveEmbeddings({ + name: 'Team', + ownerModule: 'authentication', + modelOptions: extendableEnabled, + }), + ); + }); + it('restricts config and backfill to the schema owner or platform admin', () => { assert.doesNotThrow(() => assertCanManageEmbeddingConfig({ diff --git a/modules/embeddings/src/utils/schemaPolicy.ts b/modules/embeddings/src/utils/schemaPolicy.ts index 0d45e1f75..97cd96705 100644 --- a/modules/embeddings/src/utils/schemaPolicy.ts +++ b/modules/embeddings/src/utils/schemaPolicy.ts @@ -23,12 +23,34 @@ export const AUTH_SECRET_SCHEMA_NAMES = new Set([ 'AdminApiToken', ]); -export const SYSTEM_SCHEMA_NAMES = new Set([ - 'Views', - 'Config', +export const DATABASE_SYSTEM_SCHEMA_NAMES = new Set([ + '_DeclaredSchema', 'MigratedSchemas', - 'PendingSchemas', 'CustomEndpoints', + '_PendingSchemas', + 'Views', +]); + +export const PLATFORM_INTERNAL_SCHEMA_NAMES = new Set([ + 'Admin', + 'AdminMiddleware', + 'AdminApiToken', + 'AdminTwoFactorSecret', + 'Config', + 'Client', + 'AppMiddleware', + 'ResourceDefinition', + 'Relationship', + 'ObjectIndex', + 'Permission', + 'ActorIndex', +]); + +export const INTERNAL_OWNER_MODULES = new Set(['core', 'router', 'authorization']); + +export const SYSTEM_SCHEMA_NAMES = new Set([ + ...DATABASE_SYSTEM_SCHEMA_NAMES, + ...PLATFORM_INTERNAL_SCHEMA_NAMES, ]); const SENSITIVE_FIELD_NAME = @@ -55,8 +77,10 @@ export function isHiddenField(field: unknown): boolean { } /** - * Explicit denylist for embeddings sources. Owner-controlled business schemas - * (including authentication User/Team) are not denied by ownerModule alone. + * Explicit denylist for embeddings sources. Platform internals (Database system + * schemas, core/router/authorization) are denied even when extendable. + * Owner-controlled business schemas (including authentication User/Team) are + * not denied by ownerModule alone. */ export function isDeniedEmbeddingSchema(schema: { name: string; @@ -67,6 +91,7 @@ export function isDeniedEmbeddingSchema(schema: { if (EMBEDDING_OWNED_SCHEMA_NAMES.has(schema.name)) return true; if (schema.name.startsWith('_')) return true; if (SYSTEM_SCHEMA_NAMES.has(schema.name)) return true; + if (schema.ownerModule && INTERNAL_OWNER_MODULES.has(schema.ownerModule)) return true; return AUTH_SECRET_SCHEMA_NAMES.has(schema.name); } @@ -108,8 +133,10 @@ export function assertEmbeddingTargetSchema(schema: { export function assertSchemaCanReceiveEmbeddings(schema: { name: string; + ownerModule?: string; modelOptions?: EmbeddingSchemaOptions; }): void { + assertEmbeddingTargetSchema(schema); if (!isEmbeddingSchemaEnabled(schema.modelOptions)) { throw new GrpcError( status.FAILED_PRECONDITION, From 1ac7e239cfbab85d603110afe8fde21644de67a5 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Thu, 10 Sep 2026 01:04:34 +0300 Subject: [PATCH 28/29] refactor: collapse duplicate catalogue and system-schema helpers Share source-hash naming, redaction detection, and catalogue strictness so the latest migration and schema-policy work matches surrounding style. --- .../src/utilities/reconcileModuleConfig.ts | 11 +----- .../src/utilities/redactSensitiveConfig.ts | 9 +++++ modules/database/src/Database.ts | 4 ++- .../database/src/adapters/DatabaseAdapter.ts | 2 +- .../models/__tests__/systemSchemas.test.ts | 12 +++---- modules/database/src/models/systemSchemas.ts | 7 ++-- modules/embeddings/src/Embeddings.ts | 18 ++++------ modules/embeddings/src/api/embeddingsApi.ts | 13 +++---- modules/embeddings/src/utils/configChange.ts | 15 +++++--- .../embeddings/src/utils/processEmbedding.ts | 10 +++--- .../embeddings/src/utils/providerConfig.ts | 20 +++++++---- modules/embeddings/src/utils/schemaPolicy.ts | 34 +++++++------------ 12 files changed, 72 insertions(+), 83 deletions(-) diff --git a/libraries/module-tools/src/utilities/reconcileModuleConfig.ts b/libraries/module-tools/src/utilities/reconcileModuleConfig.ts index 9183e462e..18455360f 100644 --- a/libraries/module-tools/src/utilities/reconcileModuleConfig.ts +++ b/libraries/module-tools/src/utilities/reconcileModuleConfig.ts @@ -1,4 +1,4 @@ -const REDACTED_MARKER = '[REDACTED]'; +import { containsRedactedMarker } from './redactSensitiveConfig.js'; function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); @@ -26,15 +26,6 @@ export function storedConfigsEquivalent(left: unknown, right: unknown): boolean return stableConfigJson(left) === stableConfigJson(right); } -export function containsRedactedMarker(value: unknown): boolean { - if (value === REDACTED_MARKER) return true; - if (Array.isArray(value)) { - return value.some(containsRedactedMarker); - } - if (!isRecord(value)) return false; - return Object.values(value).some(containsRedactedMarker); -} - export async function reconcileStoredModuleConfig(args: { stored: T; migrated: T; diff --git a/libraries/module-tools/src/utilities/redactSensitiveConfig.ts b/libraries/module-tools/src/utilities/redactSensitiveConfig.ts index adb289098..88023d818 100644 --- a/libraries/module-tools/src/utilities/redactSensitiveConfig.ts +++ b/libraries/module-tools/src/utilities/redactSensitiveConfig.ts @@ -26,6 +26,15 @@ function unwrapSchema(schema: unknown): unknown { const REDACTED_MARKER = '[REDACTED]'; +export function containsRedactedMarker(value: unknown): boolean { + if (value === REDACTED_MARKER) return true; + if (Array.isArray(value)) { + return value.some(containsRedactedMarker); + } + if (!isRecord(value)) return false; + return Object.values(value).some(containsRedactedMarker); +} + export function redactSensitiveConfig(config: T, schema?: unknown): T { if (!isRecord(config)) return config; const redacted = Array.isArray(config) ? [...config] : { ...config }; diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index 37933941c..ed46fa5bd 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -168,7 +168,9 @@ export default class DatabaseModule extends ManagedModule { await this._activeAdapter.registerSystemSchema(models.DeclaredSchema, isReplica); await this._activeAdapter.registerSystemSchema(models.MigratedSchemas, isReplica); let modelPromises = DATABASE_SYSTEM_SCHEMAS.filter( - model => !['_DeclaredSchema', 'MigratedSchemas'].includes(model.name), + model => + model.name !== models.DeclaredSchema.name && + model.name !== models.MigratedSchemas.name, ).map(model => this._activeAdapter.registerSystemSchema(model, isReplica)); await Promise.all(modelPromises); await this._activeAdapter.retrieveForeignSchemas(); diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index 5a538634f..18f27662c 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -461,7 +461,7 @@ export abstract class DatabaseAdapter { { readPreference: 'primary' }, ); models = models - // do not recover system schemas as they have already been + // do not recover system schemas; they are already registered .filter((model: _ConduitSchema) => !DATABASE_SYSTEM_SCHEMA_NAME_SET.has(model.name)) .map((model: _ConduitSchema) => { const schema = new ConduitSchema( diff --git a/modules/database/src/models/__tests__/systemSchemas.test.ts b/modules/database/src/models/__tests__/systemSchemas.test.ts index 6389ea67c..aba6cfa04 100644 --- a/modules/database/src/models/__tests__/systemSchemas.test.ts +++ b/modules/database/src/models/__tests__/systemSchemas.test.ts @@ -1,22 +1,22 @@ import { describe, expect, it } from '@jest/globals'; import * as models from '../index.js'; import { - DATABASE_SYSTEM_SCHEMA_NAMES, + DATABASE_SYSTEM_SCHEMA_NAME_SET, DATABASE_SYSTEM_SCHEMAS, } from '../systemSchemas.js'; describe('database system schema registry', () => { + const names = DATABASE_SYSTEM_SCHEMAS.map(schema => schema.name); + it('lists the Database-owned schemas registered as system schemas', () => { - expect([...DATABASE_SYSTEM_SCHEMA_NAMES]).toEqual([ + expect(names).toEqual([ '_DeclaredSchema', 'MigratedSchemas', 'CustomEndpoints', '_PendingSchemas', 'Views', ]); - expect(DATABASE_SYSTEM_SCHEMAS.map(schema => schema.name)).toEqual([ - ...DATABASE_SYSTEM_SCHEMA_NAMES, - ]); + expect([...DATABASE_SYSTEM_SCHEMA_NAME_SET]).toEqual(names); }); it('stays aligned with the models barrel used for registration', () => { @@ -27,6 +27,6 @@ describe('database system schema registry', () => { ) .map(schema => schema.name) .sort(); - expect(barrelNames).toEqual([...DATABASE_SYSTEM_SCHEMA_NAMES].sort()); + expect(barrelNames).toEqual([...names].sort()); }); }); diff --git a/modules/database/src/models/systemSchemas.ts b/modules/database/src/models/systemSchemas.ts index d36bd0f0e..0d46cba6a 100644 --- a/modules/database/src/models/systemSchemas.ts +++ b/modules/database/src/models/systemSchemas.ts @@ -12,9 +12,6 @@ export const DATABASE_SYSTEM_SCHEMAS = [ Views, ] as const; -export const DATABASE_SYSTEM_SCHEMA_NAMES: readonly string[] = - DATABASE_SYSTEM_SCHEMAS.map(schema => schema.name); - -export const DATABASE_SYSTEM_SCHEMA_NAME_SET = new Set( - DATABASE_SYSTEM_SCHEMA_NAMES, +export const DATABASE_SYSTEM_SCHEMA_NAME_SET = new Set( + DATABASE_SYSTEM_SCHEMAS.map(schema => schema.name), ); diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts index aac4e07ad..fd995b6bc 100644 --- a/modules/embeddings/src/Embeddings.ts +++ b/modules/embeddings/src/Embeddings.ts @@ -25,6 +25,7 @@ import { import { buildEmbeddingDocumentSelect, generateEmbeddingsForDocument, + sourceHashField, } from './utils/processEmbedding.js'; import { MAX_QUEUE_BATCH_SIZE, @@ -50,7 +51,7 @@ import { import { toBackfillCountUpdateQuery } from './utils/backfillRun.js'; import { incrementEmbeddingMetric } from './utils/embeddingMetrics.js'; import metricsSchema from './metrics/index.js'; -import { EmbeddingsApi } from './api/embeddingsApi.js'; +import { EmbeddingsApi, type DeclaredSchemaInfo } from './api/embeddingsApi.js'; import { AdminHandlers } from './admin/index.js'; import { EmbeddingsRoutes } from './routes/index.js'; import { @@ -504,7 +505,7 @@ export default class EmbeddingsModule extends ManagedModule { ...new Set( matching.flatMap(config => [ ...config.sourceFields, - `${config.targetField}SourceHash`, + sourceHashField(config.targetField), ]), ), ]; @@ -565,12 +566,7 @@ export default class EmbeddingsModule extends ManagedModule { } private async declaredSchema(schemaName: string) { - return this.database.findOne<{ - name: string; - ownerModule: string; - fields?: Record; - extensions?: Array<{ ownerModule: string; fields: Record }>; - }>( + return this.database.findOne( '_DeclaredSchema', { name: schemaName }, { select: 'name ownerModule fields extensions' }, @@ -581,10 +577,8 @@ export default class EmbeddingsModule extends ManagedModule { const config = this.currentConfig(); const providerConfig = config.providers[provider] ?? {}; return { - endpoint: - typeof providerConfig.endpoint === 'string' ? providerConfig.endpoint : undefined, - apiKey: - typeof providerConfig.apiKey === 'string' ? providerConfig.apiKey : undefined, + endpoint: providerConfig.endpoint, + apiKey: providerConfig.apiKey, model: resolveProviderModelName(providerConfig, model), timeoutMs: config.security.embedTimeoutMs, maxInputBytes: config.security.maxEmbedInputBytes, diff --git a/modules/embeddings/src/api/embeddingsApi.ts b/modules/embeddings/src/api/embeddingsApi.ts index 8e9bb64c2..1e2e1d8c3 100644 --- a/modules/embeddings/src/api/embeddingsApi.ts +++ b/modules/embeddings/src/api/embeddingsApi.ts @@ -36,6 +36,7 @@ import { materialChangeWarnings, nextEmbeddingVectorIndexName, sameEmbeddingVectorIndexFamily, + sourceHashField, type MaterialEmbeddingConfigField, } from '../utils/configChange.js'; import { MAX_QUEUE_BATCH_SIZE } from '../utils/embeddingJobs.js'; @@ -47,10 +48,10 @@ import { assertSchemaCanReceiveEmbeddings, assertSemanticSearchAccess, canManageEmbeddingConfig, - embeddingSourceHashField, EMBEDDINGS_OWNER_MODULE, resolveAdminOperatorContext, resolveSourceFieldAllowlist, + type EmbeddingSchemaOptions, type SchemaExtensionInfo, } from '../utils/schemaPolicy.js'; import { clampClientSearchLimit } from '../utils/clientSearchContext.js'; @@ -88,13 +89,7 @@ export interface DeclaredSchemaInfo { export interface SchemaInfo { name: string; fields: Record; - modelOptions?: { - conduit?: { - cms?: { enabled?: boolean }; - permissions?: { extendable?: boolean }; - authorization?: { enabled?: boolean }; - }; - }; + modelOptions?: EmbeddingSchemaOptions; } export interface EmbeddingConfigRecord { @@ -707,7 +702,7 @@ export class EmbeddingsApi { }, declared?: DeclaredSchemaInfo | null, ) { - const hashField = embeddingSourceHashField(persisted.targetField); + const hashField = sourceHashField(persisted.targetField); const existing = declared?.extensions?.find( extension => extension.ownerModule === EMBEDDINGS_OWNER_MODULE, diff --git a/modules/embeddings/src/utils/configChange.ts b/modules/embeddings/src/utils/configChange.ts index da4763386..cf4580735 100644 --- a/modules/embeddings/src/utils/configChange.ts +++ b/modules/embeddings/src/utils/configChange.ts @@ -89,15 +89,20 @@ export function isInPlaceDimensionChange( ); } +export function sourceHashField(targetField: string): string { + return `${targetField}SourceHash`; +} + export function hashFieldsToInvalidate( existing: MaterialEmbeddingConfig, next: MaterialEmbeddingConfig, ): string[] { - const fields = new Set([ - `${existing.targetField}SourceHash`, - `${next.targetField}SourceHash`, - ]); - return [...fields]; + return [ + ...new Set([ + sourceHashField(existing.targetField), + sourceHashField(next.targetField), + ]), + ]; } export function defaultEmbeddingVectorIndexName(field: string): string { diff --git a/modules/embeddings/src/utils/processEmbedding.ts b/modules/embeddings/src/utils/processEmbedding.ts index 8e0929e3d..103c3ced1 100644 --- a/modules/embeddings/src/utils/processEmbedding.ts +++ b/modules/embeddings/src/utils/processEmbedding.ts @@ -1,4 +1,6 @@ -import { hashedEmbeddingSource } from './configChange.js'; +import { hashedEmbeddingSource, sourceHashField } from './configChange.js'; + +export { sourceHashField }; export interface EmbeddingConfigLike { sourceFields: string[]; @@ -22,15 +24,11 @@ export function buildEmbeddingDocumentSelect( for (const field of config.sourceFields) { fields.add(`+${field}`); } - fields.add(`+${config.targetField}SourceHash`); + fields.add(`+${sourceHashField(config.targetField)}`); } return [...fields].join(' '); } -export function sourceHashField(targetField: string): string { - return `${targetField}SourceHash`; -} - export function buildEmbeddingInput( doc: Record, sourceFields: string[], diff --git a/modules/embeddings/src/utils/providerConfig.ts b/modules/embeddings/src/utils/providerConfig.ts index 4d0fcab27..21cf19e29 100644 --- a/modules/embeddings/src/utils/providerConfig.ts +++ b/modules/embeddings/src/utils/providerConfig.ts @@ -7,10 +7,16 @@ import type { export type { EmbeddingProviderModel, EmbeddingProviderSettings }; +type CatalogueOptions = { strict?: boolean }; + function invalidProviderConfig(message: string): GrpcError { return new GrpcError(status.INVALID_ARGUMENT, message); } +function isStrict(options?: CatalogueOptions): boolean { + return options?.strict !== false; +} + function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } @@ -47,11 +53,11 @@ function parseModelEntry(value: unknown, index: number): EmbeddingProviderModel function parseModelList( values: unknown[], - options?: { strict?: boolean }, + options?: CatalogueOptions, ): EmbeddingProviderModel[] { const models: EmbeddingProviderModel[] = []; const names = new Set(); - const strict = options?.strict !== false; + const strict = isStrict(options); for (const [index, value] of values.entries()) { const parsed = strict ? parseModelEntry(value, index) : optionalModelEntry(value); if (!parsed) continue; @@ -77,14 +83,14 @@ function optionalModelEntry(value: unknown): EmbeddingProviderModel | undefined function migrateLegacyModels( raw: Record, - options?: { strict?: boolean }, + options?: CatalogueOptions, ): EmbeddingProviderModel[] | undefined { if (Array.isArray(raw.models) && raw.models.length > 0) return undefined; const name = trimName(raw.model); if (!name) return []; const dimensions = parseDimensions(raw.dimensions); if (dimensions == null) { - if (options?.strict === false) return []; + if (!isStrict(options)) return []; throw invalidProviderConfig( `Provider model '${name}' dimensions must be a positive integer`, ); @@ -143,7 +149,7 @@ function resolveDefaultModelName( export function normalizeProviderSettings( raw: unknown, - options?: { strict?: boolean }, + options?: CatalogueOptions, ): EmbeddingProviderSettings { const source = isRecord(raw) ? raw : {}; const migrated = migrateLegacyModels(source, options); @@ -153,7 +159,7 @@ export function normalizeProviderSettings( (Array.isArray(source.models) ? parseModelList(source.models, options) : []); const names = new Set(models.map(model => model.name)); const configuredDefault = trimName(source.defaultModel); - if (configuredDefault && !names.has(configuredDefault) && options?.strict !== false) { + if (configuredDefault && !names.has(configuredDefault) && isStrict(options)) { throw invalidProviderConfig( `Provider default model '${configuredDefault}' is not in the catalogue`, ); @@ -247,7 +253,7 @@ export function normalizeEmbeddingsConfig< providers?: Record; security?: Record; }, ->(config: T, options?: { strict?: boolean }): T { +>(config: T, options?: CatalogueOptions): T { const next = { ...config }; if (isRecord(next.security)) { const security = { ...next.security }; diff --git a/modules/embeddings/src/utils/schemaPolicy.ts b/modules/embeddings/src/utils/schemaPolicy.ts index 97cd96705..e215c9df9 100644 --- a/modules/embeddings/src/utils/schemaPolicy.ts +++ b/modules/embeddings/src/utils/schemaPolicy.ts @@ -1,6 +1,7 @@ import { GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { BACKFILL_RUN_SCHEMA } from './backfillRun.js'; +import { sourceHashField } from './configChange.js'; export const EMBEDDING_CONFIG_SCHEMA = 'EmbeddingConfig'; export { BACKFILL_RUN_SCHEMA }; @@ -95,10 +96,6 @@ export function isDeniedEmbeddingSchema(schema: { return AUTH_SECRET_SCHEMA_NAMES.has(schema.name); } -export function embeddingSourceHashField(targetField: string): string { - return `${targetField}SourceHash`; -} - export interface EmbeddingSchemaOptions { conduit?: { cms?: { enabled?: boolean }; @@ -107,17 +104,13 @@ export interface EmbeddingSchemaOptions { }; } -export function isCmsEnabled(modelOptions?: EmbeddingSchemaOptions): boolean { - return modelOptions?.conduit?.cms?.enabled === true; -} - -export function isSchemaExtendable(modelOptions?: EmbeddingSchemaOptions): boolean { +function isSchemaExtendable(modelOptions?: EmbeddingSchemaOptions): boolean { return modelOptions?.conduit?.permissions?.extendable === true; } -export function isEmbeddingSchemaEnabled(modelOptions?: EmbeddingSchemaOptions): boolean { +function isEmbeddingSchemaEnabled(modelOptions?: EmbeddingSchemaOptions): boolean { if (modelOptions?.conduit?.cms == null) return true; - return isCmsEnabled(modelOptions); + return modelOptions.conduit.cms.enabled === true; } export function assertEmbeddingTargetSchema(schema: { @@ -289,7 +282,7 @@ export function assertEmbeddingExtensionAvailability(args: { compiledFields: Record; extensions?: SchemaExtensionInfo[]; }): void { - const hashField = embeddingSourceHashField(args.targetField); + const hashField = sourceHashField(args.targetField); const proposed: Record = { [args.targetField]: { type: TYPE.Vector, @@ -319,7 +312,7 @@ function fieldOwner( fieldName: string, extensions: SchemaExtensionInfo[], ): SchemaExtensionInfo | undefined { - return extensions.find(extension => fieldName in (extension.fields ?? {})); + return extensions.find(extension => fieldName in extension.fields); } function assertExtensionFieldAvailable(args: { @@ -331,10 +324,10 @@ function assertExtensionFieldAvailable(args: { extensions: SchemaExtensionInfo[]; }): void { const owned = fieldOwner(args.fieldName, args.extensions); - if (owned && owned.ownerModule !== EMBEDDINGS_OWNER_MODULE) { - throw extensionCollision(args.schemaName, args.fieldName); - } - if (owned?.ownerModule === EMBEDDINGS_OWNER_MODULE) { + if (owned) { + if (owned.ownerModule !== EMBEDDINGS_OWNER_MODULE) { + throw extensionCollision(args.schemaName, args.fieldName); + } if (!isCompatibleEmbeddingField(owned.fields[args.fieldName], args.proposed)) { throw extensionCollision(args.schemaName, args.fieldName); } @@ -343,10 +336,9 @@ function assertExtensionFieldAvailable(args: { if (args.baseFields && args.fieldName in args.baseFields) { throw extensionCollision(args.schemaName, args.fieldName); } - if (args.fieldName in args.compiledFields) { - if (!isCompatibleEmbeddingField(args.compiledFields[args.fieldName], args.proposed)) { - throw extensionCollision(args.schemaName, args.fieldName); - } + if (!(args.fieldName in args.compiledFields)) return; + if (!isCompatibleEmbeddingField(args.compiledFields[args.fieldName], args.proposed)) { + throw extensionCollision(args.schemaName, args.fieldName); } } From 51620ba61016aa4a07000f1df6c86abcb1f85a08 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Wed, 16 Sep 2026 14:51:49 +0300 Subject: [PATCH 29/29] fix(embeddings): align ts-proto with the workspace lockfile The embeddings importer still pinned ts-proto 2.12.1 after the main rebase, so frozen CI installs failed before any tests ran. --- modules/embeddings/package.json | 16 ++++++++-------- pnpm-lock.yaml | 32 ++++++++++---------------------- 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/modules/embeddings/package.json b/modules/embeddings/package.json index c76d3f0cb..6d2141157 100644 --- a/modules/embeddings/package.json +++ b/modules/embeddings/package.json @@ -28,25 +28,25 @@ "build:docker": "docker build -t ghcr.io/conduitplatform/embeddings:latest -f ./Dockerfile ../../ && docker push ghcr.io/conduitplatform/embeddings:latest" }, "dependencies": { - "@bufbuild/protobuf": "^2.10.2", + "@bufbuild/protobuf": "^2.12.0", "@conduitplatform/grpc-sdk": "workspace:*", "@conduitplatform/module-tools": "workspace:*", - "@grpc/grpc-js": "^1.14.3", - "@grpc/proto-loader": "^0.8.0", - "bullmq": "^5.21.2", + "@grpc/grpc-js": "^1.14.4", + "@grpc/proto-loader": "^0.8.1", + "bullmq": "^5.79.0", "convict": "^6.2.5", - "ioredis": "^5.10.1", + "ioredis": "^5.11.1", "lodash-es": "^4.18.1" }, "devDependencies": { "@conduitplatform/service-bundle": "workspace:*", "@types/convict": "^6.1.6", "@types/lodash-es": "^4.17.12", - "@types/node": "24.9.1", + "@types/node": "24.13.4", "copyfiles": "^2.4.1", "rimraf": "^6.1.3", - "ts-proto": "^2.11.6", + "ts-proto": "^2.12.3", "tsup": "^8.5.1", - "typescript": "~6.0.2" + "typescript": "~6.0.3" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fdfa38163..be1bde5f1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -942,8 +942,8 @@ importers: modules/embeddings: dependencies: '@bufbuild/protobuf': - specifier: ^2.10.2 - version: 2.12.0 + specifier: ^2.12.0 + version: 2.14.1 '@conduitplatform/grpc-sdk': specifier: workspace:* version: link:../../libraries/grpc-sdk @@ -951,13 +951,13 @@ importers: specifier: workspace:* version: link:../../libraries/module-tools '@grpc/grpc-js': - specifier: ^1.14.3 + specifier: ^1.14.4 version: 1.14.4 '@grpc/proto-loader': - specifier: ^0.8.0 + specifier: ^0.8.1 version: 0.8.1 bullmq: - specifier: ^5.21.2 + specifier: ^5.79.0 version: 5.79.0 convict: specifier: ^6.2.5 @@ -979,8 +979,8 @@ importers: specifier: ^4.17.12 version: 4.17.12 '@types/node': - specifier: 24.9.1 - version: 24.9.1 + specifier: 24.13.4 + version: 24.13.4 copyfiles: specifier: ^2.4.1 version: 2.4.1 @@ -988,13 +988,13 @@ importers: specifier: ^6.1.3 version: 6.1.3 ts-proto: - specifier: ^2.11.6 - version: 2.12.1 + specifier: ^2.12.3 + version: 2.12.3 tsup: specifier: ^8.5.1 version: 8.5.1(jiti@2.6.1)(postcss@8.5.15)(typescript@6.0.3)(yaml@2.9.0) typescript: - specifier: ~6.0.2 + specifier: ~6.0.3 version: 6.0.3 modules/functions: @@ -3439,9 +3439,6 @@ packages: '@types/node@24.13.4': resolution: {integrity: sha512-YJ7EqCstVTzIr0fMr7qul/977en+pQHrfmuKIo6Zr9i75Be21dr3MovcfvGtyvi2HAUrRerWps5sMO9I7WaxDw==} - '@types/node@24.9.1': - resolution: {integrity: sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg==} - '@types/nodemailer-mailgun-transport@1.4.6': resolution: {integrity: sha512-6qhtDo+1ZLtrmmpQN7O9e3NLK5ggnTS2Oca+22SvmhwChNKxDZErecTlF6qTOLnNW/CCcHmDaSmG2MXUeP1w9g==} @@ -8145,9 +8142,6 @@ packages: underscore@1.13.8: resolution: {integrity: sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==} - undici-types@7.16.0: - resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} - undici-types@7.18.2: resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} @@ -11116,10 +11110,6 @@ snapshots: dependencies: undici-types: 7.18.2 - '@types/node@24.9.1': - dependencies: - undici-types: 7.16.0 - '@types/nodemailer-mailgun-transport@1.4.6': dependencies: '@types/nodemailer': 8.0.1 @@ -16440,8 +16430,6 @@ snapshots: underscore@1.13.8: {} - undici-types@7.16.0: {} - undici-types@7.18.2: {} undici@6.27.0: