diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b17b13166..55304fdc3 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -95,6 +95,7 @@ You may find additional scripts in the `scripts` section of the `package.json` f - `authorization`: resource authorization based on Google Zanzibar - `chat`: chat room functionality - `database`: database engine (MongoDB, PostgreSQL), CMS, CRUD/functional endpoint generation + - `embeddings`: opt-in text-to-vector generation and semantic search (disabled by default; not in standalone v1) - `email`: email sending with templates support - `forms`: form generation and submission - `push-notifications`: provides support for push notifications diff --git a/.github/workflows/embeddings-test.yml b/.github/workflows/embeddings-test.yml new file mode 100644 index 000000000..bea15e318 --- /dev/null +++ b/.github/workflows/embeddings-test.yml @@ -0,0 +1,137 @@ +name: Embeddings offline tests + +on: + workflow_dispatch: + pull_request: + paths: + - 'modules/embeddings/**' + - 'modules/database/**' + - 'libraries/hermes/**' + - 'libraries/grpc-sdk/**' + - 'libraries/module-tools/**' + - 'packages/core/**' + - 'docker/**' + - 'deploy/**' + - 'scripts/resolve-docker-targets.mjs' + - '.github/workflows/embeddings-test.yml' + push: + branches: + - main + paths: + - 'modules/embeddings/**' + - 'modules/database/**' + - 'libraries/hermes/**' + - 'libraries/grpc-sdk/**' + - 'libraries/module-tools/**' + - 'packages/core/**' + - 'docker/**' + - 'deploy/**' + - 'scripts/resolve-docker-targets.mjs' + - '.github/workflows/embeddings-test.yml' + +permissions: + contents: read + pull-requests: read + +jobs: + test: + runs-on: ubuntu-24.04 + name: Embeddings unit and contract tests + steps: + - name: Checkout + uses: actions/checkout@v7 + + - uses: pnpm/action-setup@v6 + with: + version: 11.5.0 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + + - name: Install Protoc + uses: arduino/setup-protoc@v3 + with: + version: '29.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: pnpm install --frozen-lockfile --ignore-scripts + + - name: Build embeddings, database, hermes, grpc-sdk, module-tools, and Core tsc + run: > + pnpm exec turbo run build + --filter=@conduitplatform/embeddings... + --filter=@conduitplatform/database... + --filter=@conduitplatform/hermes... + --filter=@conduitplatform/grpc-sdk... + --filter=@conduitplatform/module-tools... + --filter=@conduitplatform/core + + # Core `build:bundle` still aliases @conduitplatform/node-2fa to its + # TypeScript source because the library's CommonJS dist breaks ESM + # bundling. That pre-existing packaging issue is out of this embeddings + # offline gate; this job only typechecks Core (`tsc`), it does not run + # authentication/Core service-bundle verification. + + - name: Run embeddings offline tests + run: pnpm --filter @conduitplatform/embeddings test + + - name: Run database vector and mutation offline tests + run: pnpm --filter @conduitplatform/database test --testPathIgnorePatterns=integration + + - name: Run hermes vector offline tests + run: pnpm --filter @conduitplatform/hermes test + + deploy-contracts: + runs-on: ubuntu-24.04 + name: Compose render and target discovery + steps: + - name: Checkout + uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + + - name: Render compose with embeddings profile + working-directory: docker + env: + GRPC_KEY: ci-nonempty-grpc-key + run: | + docker compose --profile mongodb --profile embeddings config --services > /tmp/compose-embeddings-services.txt + grep -qx 'embeddings' /tmp/compose-embeddings-services.txt + docker compose --profile mongodb --profile embeddings config > /tmp/compose-embeddings.yml + grep -q 'ci-nonempty-grpc-key' /tmp/compose-embeddings.yml + grep -q '55165' /tmp/compose-embeddings.yml + + - name: Render compose without embeddings profile + working-directory: docker + run: | + docker compose --profile mongodb config --services > /tmp/compose-default-services.txt + if grep -qx 'embeddings' /tmp/compose-default-services.txt; then + echo 'embeddings service must stay profile-gated' >&2 + exit 1 + fi + + - name: Discover docker targets for embeddings-only changes + env: + CHANGED_FILES: modules/embeddings/src/index.ts + run: | + env -u GITHUB_OUTPUT node scripts/resolve-docker-targets.mjs > /tmp/targets.json + node -e ' + const fs = require("fs"); + const data = JSON.parse(fs.readFileSync("/tmp/targets.json", "utf8")); + const matrix = JSON.parse(data.matrix); + const targets = matrix.include.map((row) => row.target); + if (!targets.includes("embeddings")) { + throw new Error("expected embeddings target, got " + targets.join(",")); + } + if (targets.includes("conduit-standalone")) { + throw new Error("embeddings-only changes must not select standalone: " + targets.join(",")); + } + ' + + - name: Run deployment contract tests + run: node --test modules/embeddings/test/deployment-contract.test.mjs diff --git a/.github/workflows/service-bundle-verify.yml b/.github/workflows/service-bundle-verify.yml index 9117de15c..83175c7e2 100644 --- a/.github/workflows/service-bundle-verify.yml +++ b/.github/workflows/service-bundle-verify.yml @@ -11,6 +11,7 @@ on: - 'modules/authorization/**' - 'modules/communications/**' - 'modules/database/**' + - 'modules/embeddings/**' - 'modules/router/**' - 'packages/core/**' - 'libraries/service-bundle/**' @@ -39,6 +40,7 @@ on: - 'modules/authorization/**' - 'modules/communications/**' - 'modules/database/**' + - 'modules/embeddings/**' - 'modules/router/**' - 'packages/core/**' - 'libraries/service-bundle/**' @@ -67,7 +69,7 @@ jobs: strategy: fail-fast: false matrix: - service: [chat, functions, storage, authentication, authorization, communications, database, router, core] + service: [chat, functions, storage, authentication, authorization, communications, database, embeddings, router, core] name: Verify ${{ matrix.service }} bundle steps: - name: Checkout diff --git a/Dockerfile b/Dockerfile index de7013193..9ed902748 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,8 @@ RUN pnpm install --frozen-lockfile --ignore-scripts && \ # Compile first, then bundle. Router/authentication turbo branches must not skip # build:bundle or COPY --from=conduit-base .../bundle fails in image CI. +# Standalone v1 (empty BUILDING_SERVICE) bundles core + database/router/authentication/ +# authorization/communications/storage/chat only. Embeddings is a separate image. RUN pnpm --filter @conduitplatform/service-bundle run build && \ if [ -z "$BUILDING_SERVICE" ] ; then npx turbo run build ; \ elif [ "$BUILDING_SERVICE" = "conduit" ] ; then npx turbo run build --filter=@conduitplatform/core --filter=@conduitplatform/hermes \ diff --git a/README.md b/README.md index 8cf88427c..960924966 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Can't find what you're interested in? Shoot us a message [on Discord](https://di - [Chat](https://getconduit.dev/docs/modules/chat) - Build realtime chat applications. - [Database](https://getconduit.dev/docs/modules/database) - Create schemas with auto-generated CRUD and Query-based functional endpoints. Supports MongoDB and PostgreSQL. - [Email](https://getconduit.dev/docs/modules/email) - Send emails using multiple supported providers. +- [Embeddings](modules/embeddings) - Opt-in text-to-vector generation and semantic search. Disabled by default; not included in standalone v1. See the [rollout runbook](deploy/embeddings.md). - [Forms](https://getconduit.dev/docs/modules/forms) - Submit forms and have responses forwarded to an email address. - [PushNotifications](https://getconduit.dev/docs/modules/push-notifications) - Send push notifications to your users. - [Router](https://getconduit.dev/docs/modules/router) - Seamlessly expose REST, GraphQL and WebSockets APIs with auto-generated endpoint documentation. diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 7d8ede759..c858071dd 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -29,3 +29,8 @@ To run the microservices version: - Open the admin panel in your browser at [http://localhost:8080](http://localhost:8080) - Open the router in your browser at [http://localhost:8081](http://localhost:8081) - You can inject `--profile {profile_name}` command on compose to configure more services + (`mongodb` / `postgres` for the database engine, `embeddings` for the embeddings module). + Embeddings stays omitted until you pass `--profile embeddings` **and** export a + non-empty `GRPC_KEY`. The embeddings image is not published until a compatible + release tag exists. Helm workload `install.embeddings.enabled` is separate from + module convict `enabled`. See the [embeddings rollout runbook](../embeddings.md). diff --git a/deploy/embeddings.md b/deploy/embeddings.md new file mode 100644 index 000000000..27a04d4ed --- /dev/null +++ b/deploy/embeddings.md @@ -0,0 +1,91 @@ +# Embeddings rollout and rollback + +Embeddings is a separate, disabled-by-default module image. It is not part of +standalone v1. No embeddings image is published until a compatible release tag +exists; do not enable the compose profile or Helm workload against `latest` +until that tag is published. Live MongoDB Atlas, pgvector, Redis, and provider +suites are **not** covered by CI; operators must complete the capability and +index readiness checks below before enabling generation or search. + +Production containers set `NODE_ENV=production` and **require a non-empty +`GRPC_KEY`**. Two independent enablement flags exist: + +- Helm workload `install.embeddings.enabled` (charts repo, default `false`) + only deploys or removes the embeddings process. It does not start workers. +- Module convict `enabled` (Core config, default `false`) turns on embedding + generation workers, mutation subscriptions, and search. Keep this `false` + until peer health and vector capabilities are confirmed. + +## Compose (opt-in) + +```bash +# A non-empty GRPC_KEY is required; empty values fail in production. +export GRPC_KEY='replace-with-a-non-empty-key' +docker compose --profile mongodb --profile embeddings up +``` + +- gRPC: `${EMBEDDINGS_GRPC_PORT:-55165}` (container `GRPC_PORT` uses the same value) +- Metrics: `9192` (Prometheus scrapes `conduit-embeddings:9192`) +- Image name after a compatible release: `docker.io/conduitplatform/embeddings:`. + Compose interpolates `${IMAGE_TAG}`; that tag is not published by this change. + +Helm workload `install.embeddings.enabled` is documented in the charts +repository and remains `false` by default. Setting it to `true` deploys the +pod with module convict `enabled` still false. + +## Rollout order + +1. Publish compatible Core, Database, grpc-sdk, **and** the embeddings image + tag you will run. Do not start the workload before that tag exists. +2. Deploy the embeddings **workload** with convict `enabled: false` + (`install.embeddings.enabled=true` in Helm, or the compose embeddings + profile with a non-empty `GRPC_KEY`). Confirm the process is serving and + waiting on / registered with Core. Health stays serving while workers are + disabled so operators can configure the module. +3. Confirm `GRPC_KEY` is set and gRPC peer health is good. +4. Call `GET /embeddings/capabilities` (or gRPC `getCapabilities`) and verify + Database `getVectorCapabilities`: storage, indexing, and search must be + true for the target backend (MongoDB Atlas Vector Search or Postgres + pgvector). Saving a disabled config may succeed with capability warnings; + activation must not. +5. Configure the HTTPS provider (`endpoint`, `apiKey`, and model catalogue). + `GRPC_KEY` is supplied by the deployment (`NODE_ENV=production`), not by + module settings. Check `GET /embeddings/status` for provider/index warnings. +6. Create an embedding config. The first upsert provisions the vector index + when Database indexing is available. The config stays disabled until the + index for `targetField` is queryable (`status` ready, not pending/failed). + If indexing is unavailable, status reports a manual lifecycle warning and + the operator must create the index before enabling. +7. Enable the config only after index readiness. Start a **bounded** backfill + (`onlyMissing` recommended). Watch `GET /embeddings/backfills/:id` and + `GET /embeddings/status` queue counts. Do not scan collections in the + request thread; backfills are queued. +8. Run a scoped canary semantic search (`POST /embeddings/search` as an + operator, or client search with authenticated user/scope). Confirm + fail-closed behavior on authorization-enabled schemas. +9. Enable workers/search for normal traffic (module convict `enabled: true` + through Core config). This is not `install.embeddings.enabled`. + +## Rollback + +1. Disable workers and embedding configs (module convict `enabled: false` + and per-config `enabled: false`). Generation and search stop; existing + vectors remain. +2. Scale down or stop the embeddings **workload**: + - Compose: omit `--profile embeddings` / `docker compose stop embeddings` + - Helm: `install.embeddings.enabled=false` (charts repo). This is the + workload flag, not module convict `enabled`. +3. Roll back the embeddings image and/or chart to the previous **published** + version, if any. +4. Rollback **retains** vector fields, indexes, `EmbeddingConfig` documents, + `BackfillRun` records, and Redis/BullMQ queue state. Data and index + removal is a separate explicit operator action. + +## Residual validation + +Offline CI covers unit/contract tests, bundle smoke (`Waiting for Core`), +image target discovery, and compose rendering. It does not prove Atlas, +pgvector, Redis queue behavior, a live provider, or a published embeddings +image. Repeat capability and index readiness checks in the target +environment before activation. The remaining release prerequisite is +publishing the first compatible embeddings image tag. diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index dee03b64c..72584d306 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -6,3 +6,11 @@ We've included some basic instructions to get you started on a local k8s cluster Current setup includes: - [Minikube](minikube.md) - [AKS](aks.md) + +Embeddings is not part of the standalone image. Helm workload +`install.embeddings.enabled` (charts repo, default `false`) deploys the +process; module convict `enabled` is a separate Core config switch. For a +disabled-by-default embeddings rollout, capability/index readiness, and +rollback (`install.embeddings.enabled=false`, retained vector/index/config/ +Redis state), see [embeddings.md](../embeddings.md). No embeddings image is +published until a compatible release tag exists. diff --git a/docker-bake.hcl b/docker-bake.hcl index 7360ed224..e28038b3f 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -159,6 +159,24 @@ target "database" { } } +target "conduit-base-bundle-embeddings" { + inherits = ["conduit-base"] + args = { + BUILDING_SERVICE = "modules/embeddings" + BUILD_BUNDLE = "1" + } +} + +target "embeddings" { + inherits = ["_runtime"] + context = "modules/embeddings" + dockerfile = "Dockerfile" + contexts = { + conduit-base = "target:conduit-base-bundle-embeddings" + conduit-builder = "target:conduit-builder" + } +} + target "functions" { inherits = ["_runtime"] context = "modules/functions" @@ -230,6 +248,7 @@ group "all" { "chat", "communications", "database", + "embeddings", "functions", "router", "storage", diff --git a/docker/.env b/docker/.env index 66dad89bd..10f1e39d6 100644 --- a/docker/.env +++ b/docker/.env @@ -10,6 +10,7 @@ AUTHN_GRPC_PORT="55162" AUTHZ_GRPC_PORT="55169" CHAT_GRPC_PORT="55163" COMMS_GRPC_PORT="55164" +EMBEDDINGS_GRPC_PORT="55165" STORAGE_GRPC_PORT="55168" @@ -30,6 +31,8 @@ DB_CONN_URI="mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin #DB_CONN_URI="postgres://conduit:pass@conduit-postgres:5432/conduit" # profile: postgres # Security +# Leave GRPC_KEY empty for default profiles. Enabling '--profile embeddings' +# requires exporting a non-empty GRPC_KEY; production images refuse empty values. CORE_MASTER_KEY="M4ST3RK3Y" GRPC_KEY="" diff --git a/docker/docker-compose.standalone.yml b/docker/docker-compose.standalone.yml index 8cd0b0e9b..aaf8e5db8 100644 --- a/docker/docker-compose.standalone.yml +++ b/docker/docker-compose.standalone.yml @@ -2,6 +2,10 @@ # This compose file deploys a "standalone" version of conduit with most modules # packaged in a single image. Loki and Prometheus are not deployed, since # metrics and logs can be viewed directly from the Docker daemon. +# Embeddings is not included in standalone v1. After a compatible embeddings +# image is published, use docker-compose.yml with a non-empty GRPC_KEY: +# export GRPC_KEY='replace-with-a-non-empty-key' +# docker compose --profile embeddings up #------------------------------------------------------------------------------------------- version: '3.9' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 83b3e9b55..102f68d58 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,9 +11,14 @@ # Otherwise, simply update the env values with any available port. # 3. Specify at least '--profile mongodb' or '--profile postgres' # If you're going to use PostgreSQL, swap out 'DB_CONN_URI' in the '.env' file +# 4. Embeddings is opt-in and disabled by default. Enabling the profile requires +# exporting a non-empty GRPC_KEY (production images refuse to start without it). +# The embeddings image tag is not published until a compatible release exists. # # Examples: # docker compose --profile mongodb up +# export GRPC_KEY='replace-with-a-non-empty-key' +# docker compose --profile mongodb --profile embeddings up # ---------------------------------------------------------------------------------------------- version: '3.9' @@ -226,6 +231,32 @@ services: extra_hosts: - host.docker.internal:host-gateway + embeddings: + container_name: 'conduit-embeddings' + image: 'docker.io/conduitplatform/embeddings:${IMAGE_TAG}' + restart: unless-stopped + profiles: ['embeddings'] + depends_on: + - core + - database + - prometheus + - loki + ports: + - '${EMBEDDINGS_GRPC_PORT:-55165}:${EMBEDDINGS_GRPC_PORT:-55165}' + environment: + CONDUIT_SERVER: 'conduit:${CORE_GRPC_PORT:-55152}' + SERVICE_URL: 'conduit-embeddings:${EMBEDDINGS_GRPC_PORT:-55165}' + GRPC_PORT: '${EMBEDDINGS_GRPC_PORT:-55165}' + METRICS_PORT: '9192' + LOKI_URL: 'http://conduit-loki:3100' + GRPC_KEY: '${GRPC_KEY}' + networks: + default: + aliases: + - conduit-embeddings + extra_hosts: + - host.docker.internal:host-gateway + storage: container_name: 'conduit-storage' image: 'docker.io/conduitplatform/storage:${IMAGE_TAG}' diff --git a/docker/prometheus.cfg.yml b/docker/prometheus.cfg.yml index 286b307b1..3d3dc687e 100644 --- a/docker/prometheus.cfg.yml +++ b/docker/prometheus.cfg.yml @@ -38,6 +38,9 @@ scrape_configs: - labels: module: 'Communications' targets: ['conduit-communications:9096'] + - labels: + module: 'Embeddings' + targets: ['conduit-embeddings:9192'] - labels: module: 'Storage' targets: ['conduit-storage:9190'] diff --git a/libraries/grpc-sdk/package.json b/libraries/grpc-sdk/package.json index 75f534051..9cce9f555 100644 --- a/libraries/grpc-sdk/package.json +++ b/libraries/grpc-sdk/package.json @@ -25,7 +25,8 @@ "prepublish": "npm run build", "prebuild": "npm run protoc", "build": "rimraf dist && tsup", - "protoc": "sh build.sh" + "protoc": "sh build.sh", + "test": "node --experimental-strip-types --test src/interfaces/vectorIndexMethod.test.ts" }, "license": "MIT", "dependencies": { diff --git a/libraries/grpc-sdk/src/index.ts b/libraries/grpc-sdk/src/index.ts index 7ccac264f..d43ebdffa 100644 --- a/libraries/grpc-sdk/src/index.ts +++ b/libraries/grpc-sdk/src/index.ts @@ -7,6 +7,7 @@ import { Config, Core, DatabaseProvider, + EmbeddingsProvider, Email, PushNotifications, Router, @@ -66,6 +67,7 @@ class ConduitGrpcSdk { private readonly _availableModules: any = { router: Router, database: DatabaseProvider, + embeddings: EmbeddingsProvider, storage: Storage, email: Email, pushNotifications: PushNotifications, @@ -158,12 +160,10 @@ class ConduitGrpcSdk { } private _redisDetails?: - | RedisOptions - | { nodes: { host: string; port: number }[]; options: ClusterOptions }; + RedisOptions | { nodes: { host: string; port: number }[]; options: ClusterOptions }; get redisDetails(): - | RedisOptions - | { nodes: { host: string; port: number }[]; options: ClusterOptions } { + RedisOptions | { nodes: { host: string; port: number }[]; options: ClusterOptions } { if (this._redisDetails) { return this._redisDetails; } else { @@ -223,6 +223,15 @@ class ConduitGrpcSdk { return this.database; } + get embeddings(): EmbeddingsProvider | null { + if (this._modules['embeddings']) { + return this._modules['embeddings'] as EmbeddingsProvider; + } else { + ConduitGrpcSdk.Logger.warn('Embeddings provider not up yet!'); + return null; + } + } + get storage(): Storage | null { if (this._modules['storage']) { return this._modules['storage'] as Storage; @@ -812,5 +821,41 @@ export * from './classes/index.js'; export * from './modules/index.js'; export * from './constants/index.js'; export * from './types/index.js'; -export * from './protoUtils/index.js'; +export * from './protoUtils/authentication.js'; +export * from './protoUtils/authorization.js'; +export * from './protoUtils/chat.js'; +export * from './protoUtils/communications.js'; +export * from './protoUtils/core.js'; +export * from './protoUtils/database.js'; +export * from './protoUtils/grpc_health_check.js'; +export * from './protoUtils/module.js'; +export * from './protoUtils/router.js'; +export * from './protoUtils/storage.js'; +// Embeddings proto VectorCapabilities/QueueCounts collide with the public SDK types. +export { + BackfillMutationResponse, + BackfillRun, + CancelBackfillRequest, + DeleteEmbeddingConfigRequest, + DeleteEmbeddingConfigResponse, + EmbeddingConfig, + EmbeddingsProviderDefinition, + GetBackfillRequest, + GetCapabilitiesRequest, + GetCapabilitiesResponse, + GetConfigsRequest, + GetConfigsResponse, + GetStatusRequest, + GetStatusResponse, + ListBackfillsRequest, + ListBackfillsResponse, + ResumeBackfillRequest, + SemanticSearchHit, + SemanticSearchRequest, + SemanticSearchResponse, + StartBackfillRequest, + StartBackfillResponse, + UpsertConfigRequest, + UpsertConfigResponse, +} from './protoUtils/embeddings.js'; export * from '@grpc/grpc-js'; diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 6b7319a9b..894ca419e 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -1,3 +1,5 @@ +import type { Indexable } from './Indexable.js'; + export enum TYPE { String = 'String', Number = 'Number', @@ -6,6 +8,7 @@ export enum TYPE { ObjectId = 'ObjectId', JSON = 'JSON', Relation = 'Relation', + Vector = 'Vector', } export enum SQLDataType { @@ -20,8 +23,43 @@ export enum SQLDataType { TIME = 'TIME', DATETIME = 'DATETIME', TIMESTAMP = 'TIMESTAMP', + VECTOR = 'VECTOR', +} + +export enum VectorSimilarity { + Cosine = 'cosine', + Euclidean = 'euclidean', + DotProduct = 'dotProduct', +} + +export enum VectorIndexMethod { + HNSW = 'hnsw', + IVFFlat = 'ivfflat', + Flat = 'flat', +} + +export function defaultVectorIndexMethod(method?: string | null): VectorIndexMethod { + if (method == null || method === '') { + return VectorIndexMethod.HNSW; + } + return method as VectorIndexMethod; } +export function vectorIndexMethodsEquivalent( + left?: string | null, + right?: string | null, +): boolean { + return defaultVectorIndexMethod(left) === defaultVectorIndexMethod(right); +} + +export enum VectorIndexStatus { + Pending = 'pending', + Ready = 'ready', + Failed = 'failed', +} + +export type VectorSearchProvider = 'mongodb' | 'postgres'; + export enum MongoIndexType { Ascending = 1, Descending = -1, @@ -71,9 +109,7 @@ export interface ConduitArrayValidation { } export type ConduitValidationRules = - | ConduitStringValidation - | ConduitNumberValidation - | ConduitArrayValidation; + ConduitStringValidation | ConduitNumberValidation | ConduitArrayValidation; type BaseConduitModelField = { type?: TYPE | TYPE[] | ConduitModel | ArrayConduitModel[]; @@ -109,6 +145,14 @@ export type ConduitModelFieldJSON = BasicConduitModelField & { type: TYPE.JSON | TYPE.JSON[]; }; +export type ConduitModelFieldVector = BasicConduitModelField & { + type: TYPE.Vector; + dimensions: number; + similarity?: VectorSimilarity; + provider?: string; + model?: string; +}; + export type ConduitModelFieldEnum = BasicConduitModelField & { type: ExcludeJSONRelation | ExcludeJSONRelation[]; enum: any; @@ -127,6 +171,7 @@ export type allowedTypes = | ConduitModelField | ConduitModelFieldEnum | ConduitModelFieldJSON + | ConduitModelFieldVector | ConduitModelFieldRelation; type embeddableArray = @@ -195,6 +240,7 @@ export interface ConduitSchemaOptions { }; /** Includes readonly/const-asserted index arrays (e.g. `as const` in schema definitions). */ indexes?: ReadonlyArray; + vectorIndexes?: ReadonlyArray; } export interface SchemaFieldIndex { @@ -247,3 +293,69 @@ export interface PostgresIndexOptions { [opt: string]: any; }; } + +export interface VectorIndexDefinition { + name?: string; + field: string; + dimensions: number; + similarity: VectorSimilarity; + method?: VectorIndexMethod; + filterFields?: string[]; + status?: VectorIndexStatus; + queryable?: boolean; + options?: { + numCandidates?: number; + quantization?: 'none' | 'scalar' | 'binary'; + hnsw?: { + maxEdges?: number; + numEdgeCandidates?: number; + m?: number; + efConstruction?: number; + }; + ivfflat?: { + lists?: number; + probes?: number; + }; + storedSource?: boolean | { include?: string[]; exclude?: string[] }; + }; +} + +export interface VectorCapabilities { + supported: boolean; + storage: boolean; + indexing: boolean; + search: boolean; + provider: 'mongodb' | 'postgres' | 'unsupported'; + reason?: string; +} + +export interface VectorSearchInput { + schemaName: string; + field: string; + vector: number[]; + indexName?: string; + filter?: Indexable; + limit?: number; + numCandidates?: number; + select?: string; + userId?: string; + scope?: string; + adminOperator?: boolean; +} + +export interface VectorSearchResult { + document: T; + /** + * Provider-neutral, higher-is-better similarity. + * + * Cosine is normalized so identical vectors score `1` (Postgres cosine + * distance is converted with `1 - distance`). Euclidean and inner-product + * scores are also higher-is-better ranking values, but they are **not** + * comparable across Mongo Atlas Vector Search and pgvector. + */ + score: number; + /** Raw backend distance or provider score before Conduit normalization. */ + distance?: number; + metric?: VectorSimilarity; + provider?: VectorSearchProvider; +} diff --git a/libraries/grpc-sdk/src/interfaces/vectorIndexMethod.test.ts b/libraries/grpc-sdk/src/interfaces/vectorIndexMethod.test.ts new file mode 100644 index 000000000..b726f95e2 --- /dev/null +++ b/libraries/grpc-sdk/src/interfaces/vectorIndexMethod.test.ts @@ -0,0 +1,22 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + defaultVectorIndexMethod, + VectorIndexMethod, + vectorIndexMethodsEquivalent, +} from '../../dist/index.esm.js'; + +describe('vector index method defaults', () => { + it('treats empty proto method and missing method as hnsw', () => { + assert.equal(defaultVectorIndexMethod(), VectorIndexMethod.HNSW); + assert.equal(defaultVectorIndexMethod(undefined), VectorIndexMethod.HNSW); + assert.equal(defaultVectorIndexMethod(null), VectorIndexMethod.HNSW); + assert.equal(defaultVectorIndexMethod(''), VectorIndexMethod.HNSW); + assert.equal(defaultVectorIndexMethod('hnsw'), VectorIndexMethod.HNSW); + assert.equal(defaultVectorIndexMethod('ivfflat'), VectorIndexMethod.IVFFlat); + assert.equal(defaultVectorIndexMethod('flat'), VectorIndexMethod.Flat); + assert.equal(vectorIndexMethodsEquivalent('', undefined), true); + assert.equal(vectorIndexMethodsEquivalent('hnsw', ''), true); + assert.equal(vectorIndexMethodsEquivalent('flat', ''), false); + }); +}); diff --git a/libraries/grpc-sdk/src/modules/database/index.ts b/libraries/grpc-sdk/src/modules/database/index.ts index 218407713..c26e86a7b 100644 --- a/libraries/grpc-sdk/src/modules/database/index.ts +++ b/libraries/grpc-sdk/src/modules/database/index.ts @@ -14,6 +14,13 @@ import { Query } from '../../types/db.js'; import type { FindOneOptions, FindManyOptions } from './types.js'; export type { FindOneOptions, FindManyOptions } from './types.js'; import { AuthzOptions, PopulateAuthzOptions } from '../../types/options.js'; +import type { + VectorCapabilities, + VectorIndexDefinition, + VectorSearchInput, + VectorSearchResult, +} from '../../interfaces/Model.js'; +import { defaultVectorIndexMethod } from '../../interfaces/Model.js'; export type CountDocumentsOptions = AuthzOptions & { readPreference?: string }; @@ -187,8 +194,11 @@ export class DatabaseProvider extends ConduitModule { return JSON.parse(res.result); }); @@ -206,8 +216,10 @@ export class DatabaseProvider extends ConduitModule { return JSON.parse(res.result); }); @@ -225,8 +237,10 @@ export class DatabaseProvider extends ConduitModule { return JSON.parse(res.result); }); @@ -244,8 +258,10 @@ export class DatabaseProvider extends ConduitModule { return JSON.parse(res.result); }); @@ -263,8 +279,10 @@ export class DatabaseProvider extends ConduitModule { return JSON.parse(res.result); }); @@ -319,6 +337,72 @@ export class DatabaseProvider extends ConduitModule { + return this.client!.getVectorCapabilities({ schemaName }).then(res => ({ + supported: res.supported, + storage: res.storage, + indexing: res.indexing, + search: res.search, + provider: res.provider as VectorCapabilities['provider'], + reason: res.reason, + })); + } + + createVectorIndex(schemaName: string, index: VectorIndexDefinition): Promise { + return this.client!.createVectorIndex({ + schemaName, + index: { + field: index.field, + dimensions: index.dimensions, + similarity: index.similarity, + name: index.name, + method: defaultVectorIndexMethod(index.method), + filterFields: [...(index.filterFields ?? [])], + options: index.options ? JSON.stringify(index.options) : undefined, + }, + }).then(res => JSON.parse(res.result)); + } + + getVectorIndexes(schemaName: string): Promise { + return this.client!.getVectorIndexes({ schemaName }).then(res => + res.indexes.map(index => ({ + field: index.field, + dimensions: index.dimensions, + similarity: index.similarity as VectorIndexDefinition['similarity'], + name: index.name, + method: defaultVectorIndexMethod(index.method), + filterFields: index.filterFields, + options: index.options ? JSON.parse(index.options) : undefined, + status: index.status as VectorIndexDefinition['status'], + queryable: index.queryable, + })), + ); + } + + deleteVectorIndex(schemaName: string, indexName: string): Promise { + return this.client!.deleteVectorIndex({ schemaName, indexName }).then(res => + JSON.parse(res.result), + ); + } + + vectorSearch( + request: VectorSearchInput, + ): Promise[]> { + return this.client!.vectorSearch({ + schemaName: request.schemaName, + field: request.field, + vector: request.vector, + indexName: request.indexName, + filter: request.filter ? JSON.stringify(request.filter) : undefined, + limit: request.limit, + numCandidates: request.numCandidates, + select: request.select, + userId: request.userId, + scope: request.scope, + adminOperator: request.adminOperator, + }).then(res => JSON.parse(res.result)); + } + createView( schemaName: string, viewName: string, diff --git a/libraries/grpc-sdk/src/modules/database/types.ts b/libraries/grpc-sdk/src/modules/database/types.ts index 7d39f70d3..672e3674d 100644 --- a/libraries/grpc-sdk/src/modules/database/types.ts +++ b/libraries/grpc-sdk/src/modules/database/types.ts @@ -4,6 +4,8 @@ export type FindOneOptions = { userId?: string; scope?: string; readPreference?: string; + embeddingsJob?: boolean; + embeddingsAllowedFields?: string[]; }; export type FindManyOptions = { diff --git a/libraries/grpc-sdk/src/modules/embeddings/index.ts b/libraries/grpc-sdk/src/modules/embeddings/index.ts new file mode 100644 index 000000000..13b7567a0 --- /dev/null +++ b/libraries/grpc-sdk/src/modules/embeddings/index.ts @@ -0,0 +1,262 @@ +import { ConduitModule } from '../../classes/index.js'; +import { EmbeddingsProviderDefinition } from '../../protoUtils/embeddings.js'; +import type { + Indexable, + VectorCapabilities, + VectorSearchResult, +} from '../../interfaces/index.js'; + +export interface EmbeddingConfigInput { + schemaName: string; + sourceFields: string[]; + targetField: string; + provider?: string; + model?: string; + dimensions?: number; + similarity?: string; + sourceFieldAllowlist?: string[]; + enabled?: boolean; +} + +export interface EmbeddingConfigRecord { + id: string; + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + model: string; + dimensions: number; + similarity: string; + enabled: boolean; + createdAt?: string; + updatedAt?: string; +} + +export interface QueueCounts { + waiting: number; + active: number; + completed: number; + failed: number; + delayed: number; + paused: number; +} + +export interface EmbeddingsStatus { + enabled: boolean; + ready: boolean; + capabilities: VectorCapabilities; + generationQueue: QueueCounts; + backfillQueue: QueueCounts; + warnings: string[]; +} + +export interface BackfillRunRecord { + id: string; + schemaName: string; + configId?: string; + state: string; + cursor?: string; + batchSize: number; + onlyMissing: boolean; + filter?: Indexable; + scannedCount: number; + queuedCount: number; + processedCount: number; + failedCount: number; + startedAt?: string; + finishedAt?: string; + error?: string; + createdAt?: string; + updatedAt?: string; +} + +export interface StartBackfillInput { + schemaName: string; + batchSize?: number; + configId?: string; + onlyMissing?: boolean; + filter?: Indexable; +} + +export interface SemanticSearchInput { + schemaName: string; + text: string; + targetField?: string; + filter?: Indexable; + limit?: number; + userId?: string; + scope?: string; + adminOperator?: boolean; +} + +function parseOptionalJson(value?: string): Indexable | undefined { + if (!value) return undefined; + return JSON.parse(value) as Indexable; +} + +function mapBackfillRun(run: { + id: string; + schemaName: string; + configId?: string; + state: string; + cursor?: string; + batchSize: number; + onlyMissing: boolean; + filter?: string; + scannedCount: number; + queuedCount: number; + processedCount: number; + failedCount: number; + startedAt?: string; + finishedAt?: string; + error?: string; + createdAt?: string; + updatedAt?: string; +}): BackfillRunRecord { + return { + ...run, + filter: parseOptionalJson(run.filter), + }; +} + +function mapCapabilities(capabilities: { + supported: boolean; + storage: boolean; + indexing: boolean; + search: boolean; + provider: string; + reason?: string; +}): VectorCapabilities { + return { + supported: capabilities.supported, + storage: capabilities.storage, + indexing: capabilities.indexing, + search: capabilities.search, + provider: capabilities.provider as VectorCapabilities['provider'], + reason: capabilities.reason, + }; +} + +export class EmbeddingsProvider extends ConduitModule< + typeof EmbeddingsProviderDefinition +> { + constructor( + private readonly moduleName: string, + url: string, + grpcToken?: string, + ) { + super(moduleName, 'embeddings', url, grpcToken); + this.initializeClient(EmbeddingsProviderDefinition); + } + + upsertConfig( + config: EmbeddingConfigInput, + ): Promise<{ config: EmbeddingConfigRecord; warnings: string[] }> { + return this.client!.upsertConfig(config).then(res => ({ + config: res.config!, + warnings: res.warnings, + })); + } + + getConfigs(query?: { + schemaName?: string; + id?: string; + }): Promise { + return this.client!.getConfigs(query ?? {}).then(res => res.configs); + } + + deleteConfig(query: { + id?: string; + schemaName?: string; + targetField?: string; + }): Promise { + return this.client!.deleteConfig(query).then(res => res.config!); + } + + getCapabilities(schemaName?: string): Promise<{ + capabilities: VectorCapabilities; + warnings: string[]; + }> { + return this.client!.getCapabilities({ schemaName }).then(res => ({ + capabilities: mapCapabilities(res.capabilities!), + warnings: res.warnings, + })); + } + + getStatus(schemaName?: string): Promise { + return this.client!.getStatus({ schemaName }).then(res => ({ + enabled: res.enabled, + ready: res.ready, + capabilities: mapCapabilities(res.capabilities!), + generationQueue: res.generationQueue!, + backfillQueue: res.backfillQueue!, + warnings: res.warnings, + })); + } + + startBackfill(input: StartBackfillInput): Promise<{ + queued: number; + runs: BackfillRunRecord[]; + warnings: string[]; + }> { + return this.client!.startBackfill({ + schemaName: input.schemaName, + batchSize: input.batchSize, + configId: input.configId, + onlyMissing: input.onlyMissing, + filter: input.filter ? JSON.stringify(input.filter) : undefined, + }).then(res => ({ + queued: res.queued, + runs: res.runs.map(mapBackfillRun), + warnings: res.warnings, + })); + } + + getBackfill(id: string): Promise { + return this.client!.getBackfill({ id }).then(mapBackfillRun); + } + + listBackfills(query?: { + schemaName?: string; + state?: string; + configId?: string; + skip?: number; + limit?: number; + }): Promise<{ runs: BackfillRunRecord[]; count: number }> { + return this.client!.listBackfills(query ?? {}).then(res => ({ + runs: res.runs.map(mapBackfillRun), + count: res.count, + })); + } + + cancelBackfill(id: string): Promise { + return this.client!.cancelBackfill({ id }).then(res => mapBackfillRun(res.run!)); + } + + resumeBackfill(id: string): Promise { + return this.client!.resumeBackfill({ id }).then(res => mapBackfillRun(res.run!)); + } + + semanticSearch( + input: SemanticSearchInput, + ): Promise[]> { + return this.client!.semanticSearch({ + schemaName: input.schemaName, + text: input.text, + targetField: input.targetField, + filter: input.filter ? JSON.stringify(input.filter) : undefined, + limit: input.limit, + userId: input.userId, + scope: input.scope, + adminOperator: input.adminOperator, + }).then(res => + res.hits.map(hit => ({ + document: JSON.parse(hit.document) as T, + score: hit.score, + distance: hit.distance, + metric: hit.metric as VectorSearchResult['metric'], + provider: hit.provider as VectorSearchResult['provider'], + })), + ); + } +} diff --git a/libraries/grpc-sdk/src/modules/index.ts b/libraries/grpc-sdk/src/modules/index.ts index bfd9e6ed9..a31c9b5fd 100644 --- a/libraries/grpc-sdk/src/modules/index.ts +++ b/libraries/grpc-sdk/src/modules/index.ts @@ -2,6 +2,7 @@ export * from './storage/index.js'; export * from './router/index.js'; export * from './email/index.js'; export * from './database/index.js'; +export * from './embeddings/index.js'; export * from './config/index.js'; export * from './core/index.js'; export * from './admin/index.js'; diff --git a/libraries/grpc-sdk/src/types/db.ts b/libraries/grpc-sdk/src/types/db.ts index dd0396c58..55d4f5117 100644 --- a/libraries/grpc-sdk/src/types/db.ts +++ b/libraries/grpc-sdk/src/types/db.ts @@ -37,9 +37,18 @@ type setQuery = { $set: simpleQuery; }; +type numericDocumentKeys = { + [K in keyof T]-?: NonNullable extends number ? K : never; +}[keyof T]; + +type incQuery = { + $inc: { [K in numericDocumentKeys]?: number }; +}; + export type Query = | simpleQuery | pushQuery | setQuery + | incQuery // | arrayQuery | conditionalQuery; diff --git a/libraries/grpc-sdk/src/types/options.ts b/libraries/grpc-sdk/src/types/options.ts index 122083286..90e8228b3 100644 --- a/libraries/grpc-sdk/src/types/options.ts +++ b/libraries/grpc-sdk/src/types/options.ts @@ -5,4 +5,14 @@ export type AuthzOptions = { export type PopulateAuthzOptions = { populate?: string | string[]; + /** + * When true, Database skips publishing the mutation event. + * Existing callers omit this and keep the default publish behavior. + */ + suppressEvent?: boolean; + /** + * When true, Database verifies the caller is the embeddings module + * and restricts the write to embeddings-owned vector/hash fields. + */ + embeddingsJob?: boolean; } & AuthzOptions; diff --git a/libraries/hermes/README.mdx b/libraries/hermes/README.mdx index c43299e68..ea2a72656 100644 --- a/libraries/hermes/README.mdx +++ b/libraries/hermes/README.mdx @@ -12,3 +12,5 @@ It is utilized by [Admin](../../packages/admin) and [Router](../../modules/route - WebSockets (via Socket.io) - Support for middleware - Auto-generated API documentation + +`TYPE.Vector` is documented and validated as an array of finite numbers (`[Number]` in GraphQL, OpenAPI `array` of `number`). It is never emitted as a referenced `Vector` schema or GraphQL type. Dimension constraints are applied in Zod/OpenAPI when `dimensions` is present; GraphQL cannot express array length. diff --git a/libraries/hermes/package.json b/libraries/hermes/package.json index 743552033..45d49f4c1 100644 --- a/libraries/hermes/package.json +++ b/libraries/hermes/package.json @@ -11,6 +11,7 @@ "scripts": { "prepublish": "npm run build", "build": "rimraf dist && tsc", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/classes/vectorParser.test.js dist-test/Rest/vectorOpenApi.test.js dist-test/GraphQl/vectorGraphQl.test.js dist-test/MCP/vectorMcp.test.js", "publish": "npm publish", "postbuild": "copyfiles -u 1 src/*.proto src/**/*.json ./dist/" }, diff --git a/libraries/hermes/src/GraphQl/GraphQlParser.ts b/libraries/hermes/src/GraphQl/GraphQlParser.ts index 2ba7c0782..138a1f615 100644 --- a/libraries/hermes/src/GraphQl/GraphQlParser.ts +++ b/libraries/hermes/src/GraphQl/GraphQlParser.ts @@ -74,6 +74,8 @@ export class GraphQlParser extends ConduitParser return 'ID'; case 'JSON': return 'JSONObject'; + case 'Vector': + return '[Number]'; default: this.requestedTypes.add(conduitType); return conduitType; diff --git a/libraries/hermes/src/GraphQl/utils/SimpleTypeParamUtils.ts b/libraries/hermes/src/GraphQl/utils/SimpleTypeParamUtils.ts index 14d4f8ddf..5bf02869f 100644 --- a/libraries/hermes/src/GraphQl/utils/SimpleTypeParamUtils.ts +++ b/libraries/hermes/src/GraphQl/utils/SimpleTypeParamUtils.ts @@ -1,4 +1,4 @@ -import { ConduitModel, Indexable } from '@conduitplatform/grpc-sdk'; +import { ConduitModel, Indexable, TYPE } from '@conduitplatform/grpc-sdk'; const GQL_PRIMITIVES = ['Number', 'Boolean', 'Date', 'String']; @@ -7,6 +7,8 @@ function extractParam(param: string, required: boolean = false) { return 'ID' + (required ? '!' : ''); } else if (param === 'JSON') { return 'JSONObject' + (required ? '!' : ''); + } else if (param === TYPE.Vector || param === 'Vector') { + return '[Number]' + (required ? '!' : ''); } else { return param + (required ? '!' : ''); } @@ -17,7 +19,9 @@ function extractArrayParam( required: boolean = false, originalParam?: any, ) { - if (GQL_PRIMITIVES.indexOf(param) !== -1) { + if (param === TYPE.Vector || param === 'Vector') { + return '[[Number]]' + (required ? '!' : ''); + } else if (GQL_PRIMITIVES.indexOf(param) !== -1) { return `[${param}]` + (required ? '!' : ''); } else if (param === 'ObjectId') { return '[ID]' + (required ? '!' : ''); diff --git a/libraries/hermes/src/GraphQl/vectorGraphQl.test.ts b/libraries/hermes/src/GraphQl/vectorGraphQl.test.ts new file mode 100644 index 000000000..78746e5da --- /dev/null +++ b/libraries/hermes/src/GraphQl/vectorGraphQl.test.ts @@ -0,0 +1,46 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { TYPE } from '@conduitplatform/grpc-sdk'; +import { GraphQlParser } from './GraphQlParser.js'; +import { processParams } from './utils/SimpleTypeParamUtils.js'; + +describe('GraphQL Vector mapping', () => { + it('renders Vector fields as [Number] and does not request a Vector type', () => { + const parser = new GraphQlParser(); + const result = parser.extractTypes( + 'Doc', + { + title: TYPE.String, + count: TYPE.Number, + embedding: { type: TYPE.Vector, dimensions: 8 }, + owner: { type: TYPE.Relation, model: 'User' }, + payload: TYPE.JSON, + }, + false, + ); + + assert.equal(parser.requestedTypes.has('Vector'), false); + assert.equal(parser.requestedTypes.has('User'), true); + assert.match(result.typeString, /embedding: \[Number]/); + assert.match(result.typeString, /title: String/); + assert.match(result.typeString, /count: Number/); + assert.match(result.typeString, /payload: JSONObject/); + assert.equal(/\btype Vector\b/.test(result.typeString), false); + assert.equal(/: Vector\b/.test(result.typeString), false); + }); + + it('maps simple Vector parameters to [Number] rather than a Vector named type', () => { + const params = processParams( + { + q: TYPE.String, + embedding: { type: TYPE.Vector, dimensions: 3, required: true }, + ids: [TYPE.ObjectId], + }, + '', + ); + assert.match(params, /q:String/); + assert.match(params, /embedding:\[Number]!/); + assert.match(params, /ids:\[ID]/); + assert.equal(params.includes('Vector'), false); + }); +}); diff --git a/libraries/hermes/src/MCP/constants.ts b/libraries/hermes/src/MCP/constants.ts index 01977b0c5..3a7b8c62b 100644 --- a/libraries/hermes/src/MCP/constants.ts +++ b/libraries/hermes/src/MCP/constants.ts @@ -39,6 +39,7 @@ Module discovery and activation: Common modules: - database: schemas, documents, custom endpoints, indexes +- embeddings: embedding configuration, backfills, semantic search - authentication: users, teams, OAuth services - storage: file storage configuration - authorization: relations, resources, permission checks (RBAC/ReBAC) diff --git a/libraries/hermes/src/MCP/vectorMcp.test.ts b/libraries/hermes/src/MCP/vectorMcp.test.ts new file mode 100644 index 000000000..c0066ab3a --- /dev/null +++ b/libraries/hermes/src/MCP/vectorMcp.test.ts @@ -0,0 +1,51 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { z } from 'zod'; +import { + ConduitRouteActions, + ConduitRouteReturnDefinition, + TYPE, +} from '@conduitplatform/grpc-sdk'; +import { ConduitRoute } from '../classes/index.js'; +import { ConduitRouter } from '../Router.js'; +import { RouteToToolConverter } from './RouteToTool.js'; + +describe('MCP Vector tool schemas', () => { + it('validates Vector body params as finite numeric arrays without a Vector named type', () => { + const converter = new RouteToToolConverter({} as ConduitRouter); + const route = new ConduitRoute( + { + path: '/search', + action: ConduitRouteActions.POST, + bodyParams: { + query: { type: TYPE.String, required: true }, + embedding: { type: TYPE.Vector, dimensions: 3, required: true }, + owner: { type: TYPE.Relation, model: 'User' }, + }, + }, + new ConduitRouteReturnDefinition('Search', { hits: TYPE.JSON }), + async () => ({}), + ); + + const tool = converter.convertRouteToTool(route); + const schema = z.object(tool.inputSchema); + const ok = schema.safeParse({ + query: 'hello', + embedding: [0.1, 0.2, 0.3], + owner: 'user-1', + }); + assert.equal(ok.success, true); + assert.equal( + schema.safeParse({ query: 'hello', embedding: [0.1, 0.2] }).success, + false, + ); + assert.equal( + schema.safeParse({ + query: 'hello', + embedding: [0.1, 0.2, Number.POSITIVE_INFINITY], + }).success, + false, + ); + assert.equal(JSON.stringify(tool.inputSchema).includes('Vector'), false); + }); +}); diff --git a/libraries/hermes/src/Rest/SimpleTypeParamUtils.ts b/libraries/hermes/src/Rest/SimpleTypeParamUtils.ts index eb98fde75..fd58bdc8f 100644 --- a/libraries/hermes/src/Rest/SimpleTypeParamUtils.ts +++ b/libraries/hermes/src/Rest/SimpleTypeParamUtils.ts @@ -1,4 +1,5 @@ import { ConduitModel, ConduitValidationRules, TYPE } from '@conduitplatform/grpc-sdk'; +import { ParserUtils } from '../classes/index.js'; export function applyOpenApiFieldValidation( res: Record, @@ -24,6 +25,7 @@ function extractParam( param: string, required: boolean = false, validate?: ConduitValidationRules, + sourceField?: unknown, ) { const res: Record = { type: 'string' }; switch (param) { @@ -38,6 +40,10 @@ function extractParam( case TYPE.Relation: res.type = 'string'; break; + case TYPE.Vector: + res.type = 'array'; + res.items = { type: 'number' }; + break; case 'String': case 'Number': case 'Boolean': @@ -45,6 +51,9 @@ function extractParam( break; } applyOpenApiFieldValidation(res, validate); + if (param === TYPE.Vector) { + ParserUtils.applyVectorOpenApiConstraints(res, sourceField); + } return res; } @@ -67,13 +76,13 @@ export function processSwaggerParams(paramObj: any) { let params: Record = {}; if (typeof paramObj === 'string') { - params = extractParam(paramObj); + params = extractParam(paramObj, false, undefined, paramObj); } else if (Array.isArray(paramObj)) { const elementZero = paramObj[0]; if (typeof elementZero === 'string') { params = { type: 'array', - items: { ...extractParam(elementZero, false) }, + items: { ...extractParam(elementZero, false, undefined, elementZero) }, minItems: 0, }; } else { @@ -84,7 +93,12 @@ export function processSwaggerParams(paramObj: any) { params = { type: 'array', items: { - ...extractParam(typeZero! as string, typeZeroRequired, itemValidate), + ...extractParam( + typeZero! as string, + typeZeroRequired, + itemValidate, + elementZero, + ), }, minItems: typeZeroRequired ? 1 : 0, }; @@ -94,7 +108,7 @@ export function processSwaggerParams(paramObj: any) { const typeZeroRequired = (paramObj as ConduitModel).required; const validate = (paramObj as { validate?: ConduitValidationRules }).validate; if (typeof typeZero === 'string') { - params = extractParam(typeZero, typeZeroRequired, validate); + params = extractParam(typeZero, typeZeroRequired, validate, paramObj); } else if (Array.isArray(typeZero)) { const elementZero = typeZero[0]; if (typeof elementZero === 'string') { @@ -107,7 +121,12 @@ export function processSwaggerParams(paramObj: any) { params = { type: 'array', items: { - ...extractParam(typeZeroTwo! as string, typeZeroTwoRequired, itemValidate), + ...extractParam( + typeZeroTwo! as string, + typeZeroTwoRequired, + itemValidate, + elementZero, + ), }, minItems: typeZeroTwoRequired ? 1 : 0, }; diff --git a/libraries/hermes/src/Rest/SwaggerParser.ts b/libraries/hermes/src/Rest/SwaggerParser.ts index 9464555d7..7005fcc13 100644 --- a/libraries/hermes/src/Rest/SwaggerParser.ts +++ b/libraries/hermes/src/Rest/SwaggerParser.ts @@ -5,7 +5,7 @@ import { TYPE, UntypedArray, } from '@conduitplatform/grpc-sdk'; -import { ConduitParser } from '../classes/index.js'; +import { ConduitParser, ParserUtils } from '../classes/index.js'; import { applyOpenApiFieldValidation } from './SimpleTypeParamUtils.js'; export interface ParseResult { @@ -65,6 +65,9 @@ export class SwaggerParser extends ConduitParser $ref?: string; format?: string; properties?: object; + items?: { type?: string }; + minItems?: number; + maxItems?: number; } = {}; switch (conduitType) { case TYPE.JSON: @@ -79,6 +82,10 @@ export class SwaggerParser extends ConduitParser case TYPE.Relation: res.type = 'string'; break; + case TYPE.Vector: + res.type = 'array'; + res.items = { type: 'number' }; + break; case 'String': case 'Number': case 'Boolean': @@ -129,6 +136,10 @@ export class SwaggerParser extends ConduitParser processingObject as unknown as Record, v as any, ); + ParserUtils.applyVectorOpenApiConstraints( + processingObject as unknown as Record, + sourceField, + ); } else { if (!processingObject.properties) { processingObject.properties = {}; @@ -140,6 +151,10 @@ export class SwaggerParser extends ConduitParser processingObject.properties[name] as Record, v as any, ); + ParserUtils.applyVectorOpenApiConstraints( + processingObject.properties[name] as Record, + sourceField, + ); } this.addFieldToRequired(processingObject, name, isRequired); } diff --git a/libraries/hermes/src/Rest/vectorOpenApi.test.ts b/libraries/hermes/src/Rest/vectorOpenApi.test.ts new file mode 100644 index 000000000..ca7fa50e7 --- /dev/null +++ b/libraries/hermes/src/Rest/vectorOpenApi.test.ts @@ -0,0 +1,73 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { TYPE } from '@conduitplatform/grpc-sdk'; +import { SwaggerParser } from './SwaggerParser.js'; +import { processSwaggerParams } from './SimpleTypeParamUtils.js'; + +function assertNoPhantomVector(value: unknown) { + const serialized = JSON.stringify(value); + assert.equal(serialized.includes('#/components/schemas/Vector'), false); + assert.equal(serialized.includes('"Vector"'), false); +} + +describe('OpenAPI Vector mapping', () => { + it('emits a numeric array with dimensions and never a Vector $ref', () => { + const parser = new SwaggerParser(); + const result = parser.extractTypes( + 'Doc', + { + title: TYPE.String, + embedding: { type: TYPE.Vector, dimensions: 8, required: true }, + owner: { type: TYPE.Relation, model: 'User' }, + payload: TYPE.JSON, + }, + false, + ); + + assert.equal(parser.requestedTypes.has('Vector'), false); + assert.equal(parser.requestedTypes.has('User'), true); + assertNoPhantomVector(result); + assert.deepEqual( + (result as { properties: Record }).properties.embedding, + { + type: 'array', + items: { type: 'number' }, + minItems: 8, + maxItems: 8, + }, + ); + assert.equal( + ( + (result as { properties: Record }).properties + .title as { type?: string } + ).type, + 'string', + ); + assert.equal( + ( + (result as { properties: Record }).properties + .payload as { type?: string } + ).type, + 'object', + ); + }); + + it('maps simple Vector params to numeric arrays with dimensions where present', () => { + assert.deepEqual(processSwaggerParams(TYPE.Vector), { + type: 'array', + items: { type: 'number' }, + }); + assert.deepEqual( + processSwaggerParams({ type: TYPE.Vector, dimensions: 4, required: true }), + { + type: 'array', + items: { type: 'number' }, + minItems: 4, + maxItems: 4, + }, + ); + assert.deepEqual(processSwaggerParams(TYPE.Number), { type: 'number' }); + assert.deepEqual(processSwaggerParams(TYPE.JSON), { type: 'object' }); + assertNoPhantomVector(processSwaggerParams({ type: TYPE.Vector, dimensions: 2 })); + }); +}); diff --git a/libraries/hermes/src/classes/ConduitParser.ts b/libraries/hermes/src/classes/ConduitParser.ts index a847953dd..500475634 100644 --- a/libraries/hermes/src/classes/ConduitParser.ts +++ b/libraries/hermes/src/classes/ConduitParser.ts @@ -9,7 +9,7 @@ import { } from '@conduitplatform/grpc-sdk'; import { ParserUtils } from './ParserUtils.js'; -const baseTypes = ['String', 'Number', 'Boolean', 'Date', 'ObjectId', 'JSON']; +const baseTypes = ['String', 'Number', 'Boolean', 'Date', 'ObjectId', 'JSON', 'Vector']; export abstract class ConduitParser { result!: ParseResult; diff --git a/libraries/hermes/src/classes/ParserUtils.ts b/libraries/hermes/src/classes/ParserUtils.ts index b0e217435..c9143065b 100644 --- a/libraries/hermes/src/classes/ParserUtils.ts +++ b/libraries/hermes/src/classes/ParserUtils.ts @@ -60,6 +60,52 @@ export class ParserUtils { return baseType === TYPE.Relation || baseType === 'Relation'; } + /** + * True for TYPE.Vector / 'Vector'. Never treat this as a named schema/reference. + */ + static isVectorTypeName(value: unknown): boolean { + return value === TYPE.Vector || value === 'Vector'; + } + + /** + * Check if a field is a Vector type (shorthand or object form). + */ + static isVectorType(field: unknown): boolean { + return ParserUtils.isVectorTypeName(ParserUtils.getBaseType(field)); + } + + /** + * Positive integer dimensions from a Vector field (or a raw dimensions value). + */ + static getVectorDimensions(fieldOrDimensions: unknown): number | undefined { + let value: unknown = fieldOrDimensions; + if ( + typeof fieldOrDimensions === 'object' && + fieldOrDimensions !== null && + 'dimensions' in fieldOrDimensions + ) { + value = (fieldOrDimensions as { dimensions?: unknown }).dimensions; + } + if (typeof value === 'number' && Number.isInteger(value) && value > 0) { + return value; + } + return undefined; + } + + /** + * OpenAPI/Swagger: constrain a numeric array to the Vector field's dimensions. + */ + static applyVectorOpenApiConstraints( + schema: Record, + sourceField?: unknown, + ): void { + if (schema.type !== 'array') return; + const dimensions = ParserUtils.getVectorDimensions(sourceField); + if (dimensions === undefined) return; + schema.minItems = dimensions; + schema.maxItems = dimensions; + } + /** * Get the model name for a Relation field */ diff --git a/libraries/hermes/src/classes/ZodParser.ts b/libraries/hermes/src/classes/ZodParser.ts index 4f553a5e9..7037065eb 100644 --- a/libraries/hermes/src/classes/ZodParser.ts +++ b/libraries/hermes/src/classes/ZodParser.ts @@ -165,7 +165,17 @@ export class ZodParser { ); } - private getZodType(conduitType: TYPE): z.ZodTypeAny { + private vectorZodType(sourceField?: unknown): z.ZodTypeAny { + const item = this.useCoercion ? z.coerce.number().finite() : z.number().finite(); + let arrayType = z.array(item); + const dimensions = ParserUtils.getVectorDimensions(sourceField); + if (dimensions !== undefined) { + arrayType = arrayType.length(dimensions); + } + return arrayType; + } + + private getZodType(conduitType: TYPE, sourceField?: unknown): z.ZodTypeAny { switch (conduitType) { case TYPE.String: return z.string(); @@ -181,6 +191,8 @@ export class ZodParser { return this.conduitJsonZodType(); case TYPE.Relation: return z.string(); + case TYPE.Vector: + return this.vectorZodType(sourceField); default: return z.any(); } @@ -194,7 +206,7 @@ export class ZodParser { if (typeof fields === 'string') { const t = fields as TYPE; - let zodType = this.getZodType(t); + let zodType = this.getZodType(t, fields); if (t === TYPE.JSON) { zodType = this.finalizeJsonRouteParam(zodType, true); } @@ -218,7 +230,7 @@ export class ZodParser { if (typeof field === 'string') { const t = field as TYPE; - let zodType = this.getZodType(t); + let zodType = this.getZodType(t, field); if (t === TYPE.JSON) { zodType = this.finalizeJsonRouteParam(zodType, !isRequired); } else if (!isRequired) { @@ -252,13 +264,13 @@ export class ZodParser { let itemType: z.ZodTypeAny; if (typeof firstItem === 'string') { - itemType = this.getZodType(firstItem as TYPE); + itemType = this.getZodType(firstItem as TYPE, firstItem); } else if (typeof firstItem === 'object' && firstItem !== null) { if (firstItem.type) { if (firstItem.type === TYPE.Relation) { itemType = z.string(); } else if (typeof firstItem.type === 'string') { - itemType = this.getZodType(firstItem.type as TYPE); + itemType = this.getZodType(firstItem.type as TYPE, firstItem); } else { const nestedResult = this.extractTypesInternal( fieldName, @@ -316,7 +328,7 @@ export class ZodParser { zodType = z.string(); } else if (typeof field.type === 'string') { const t = field.type as TYPE; - zodType = this.getZodType(t); + zodType = this.getZodType(t, field); if (t === TYPE.JSON) { zodType = this.finalizeJsonRouteParam(zodType, !isRequired); } diff --git a/libraries/hermes/src/classes/vectorParser.test.ts b/libraries/hermes/src/classes/vectorParser.test.ts new file mode 100644 index 000000000..988647980 --- /dev/null +++ b/libraries/hermes/src/classes/vectorParser.test.ts @@ -0,0 +1,83 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { TYPE } from '@conduitplatform/grpc-sdk'; +import { ParserUtils } from './ParserUtils.js'; +import { ZodParser } from './ZodParser.js'; + +const vectorField = { + type: TYPE.Vector, + dimensions: 3, + required: true, +}; + +describe('ParserUtils Vector helpers', () => { + it('recognizes Vector shorthand and object form without treating it as a relation', () => { + assert.equal(ParserUtils.isVectorTypeName(TYPE.Vector), true); + assert.equal(ParserUtils.isVectorTypeName('Vector'), true); + assert.equal(ParserUtils.isVectorType(TYPE.Vector), true); + assert.equal(ParserUtils.isVectorType(vectorField), true); + assert.equal(ParserUtils.isVectorType(TYPE.JSON), false); + assert.equal(ParserUtils.isRelationType(vectorField), false); + assert.equal(ParserUtils.getVectorDimensions(vectorField), 3); + assert.equal(ParserUtils.getVectorDimensions({ type: TYPE.Vector }), undefined); + assert.equal(ParserUtils.getVectorDimensions({ dimensions: 1.5 }), undefined); + }); +}); + +describe('ZodParser Vector validation', () => { + const parser = new ZodParser(); + + it('accepts finite numeric arrays of the declared dimensions', () => { + const schema = parser.buildZodSchema({ + title: TYPE.String, + count: TYPE.Number, + embedding: vectorField, + }); + const parsed = schema.parse({ + title: 'doc', + count: 2, + embedding: [0.1, 0.2, 0.3], + }); + assert.deepEqual(parsed.embedding, [0.1, 0.2, 0.3]); + assert.equal(parsed.title, 'doc'); + assert.equal(parsed.count, 2); + }); + + it('rejects non-finite values, wrong length, and shorthand still as a numeric array', () => { + const schema = parser.buildZodSchema({ + embedding: vectorField, + raw: TYPE.Vector, + }); + assert.equal(schema.safeParse({ embedding: [0.1, 0.2] }).success, false); + assert.equal(schema.safeParse({ embedding: [0.1, 0.2, Number.NaN] }).success, false); + assert.equal( + schema.safeParse({ embedding: [0.1, 0.2, Number.POSITIVE_INFINITY] }).success, + false, + ); + const shorthand = schema.safeParse({ embedding: [1, 2, 3], raw: [1, 2, 3, 4] }); + assert.equal(shorthand.success, true); + assert.equal( + schema.safeParse({ embedding: [1, 2, 3], raw: 'Vector' }).success, + false, + ); + }); + + it('preserves existing non-vector types including JSON and Relation', () => { + const schema = parser.buildZodSchema({ + name: TYPE.String, + active: TYPE.Boolean, + created: TYPE.Date, + owner: { type: TYPE.Relation, model: 'User', required: true }, + meta: TYPE.JSON, + }); + const parsed = schema.parse({ + name: 'ok', + active: true, + created: '2026-01-01T00:00:00.000Z', + owner: '507f1f77bcf86cd799439011', + meta: { a: 1 }, + }); + assert.equal(parsed.owner, '507f1f77bcf86cd799439011'); + assert.deepEqual(parsed.meta, { a: 1 }); + }); +}); diff --git a/libraries/hermes/tsconfig.json b/libraries/hermes/tsconfig.json index cd32b1ca8..b15700386 100644 --- a/libraries/hermes/tsconfig.json +++ b/libraries/hermes/tsconfig.json @@ -65,5 +65,6 @@ /* Advanced Options */ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ - } + }, + "exclude": ["node_modules", "dist", "dist-test", "**/*.test.ts"] } diff --git a/libraries/hermes/tsconfig.test.json b/libraries/hermes/tsconfig.test.json new file mode 100644 index 000000000..99f8d1331 --- /dev/null +++ b/libraries/hermes/tsconfig.test.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "removeComments": false, + "types": ["node"] + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "dist-test"] +} diff --git a/libraries/module-tools/package.json b/libraries/module-tools/package.json index 7ff7081f9..67348ef2b 100644 --- a/libraries/module-tools/package.json +++ b/libraries/module-tools/package.json @@ -23,7 +23,8 @@ "type": "module", "scripts": { "prepublish": "npm run build", - "build": "rimraf dist && tsup" + "build": "rimraf dist && tsup", + "test": "node --experimental-strip-types --test src/utilities/reconcileModuleConfig.test.ts" }, "license": "MIT", "dependencies": { diff --git a/libraries/module-tools/src/ManagedModule.ts b/libraries/module-tools/src/ManagedModule.ts index d48171087..da8bfd050 100644 --- a/libraries/module-tools/src/ManagedModule.ts +++ b/libraries/module-tools/src/ManagedModule.ts @@ -13,6 +13,8 @@ import { import { initializeSdk, merge, readConduitPeersManifest } from './utilities/index.js'; import type { ConduitPeersManifest } from './utilities/conduitPeers.js'; import { convictConfigParser } from './utilities/convictConfigParser.js'; +import { reconcileStoredModuleConfig } from './utilities/reconcileModuleConfig.js'; +import { restoreRedactedSecrets } from './utilities/redactSensitiveConfig.js'; import { RoutingManager } from './routing/index.js'; import { RoutingController } from './routing/RoutingController.js'; @@ -268,9 +270,10 @@ export abstract class ManagedModule extends ConduitServiceModule { }); } let config = JSON.parse(call.request.newConfig); - config = merge(this.config.getProperties(), config); - config = await this.preConfig(config); const previousConfig = this.config.getProperties(); + config = merge(previousConfig, config); + config = restoreRedactedSecrets(config, previousConfig, this.configSchema); + config = await this.preConfig(config); try { this.config.load(config).validate({ allowed: 'warn', @@ -354,8 +357,20 @@ export abstract class ManagedModule extends ConduitServiceModule { ConfigController.getInstance(); if (config) { + const migrated = await this.preConfig(config); + this.config.load(migrated).validate({ + allowed: 'warn', + }); + const persistable = this.config.getProperties(); + const reconciled = await reconcileStoredModuleConfig({ + stored: config, + migrated: persistable, + configureOverride: next => + this.grpcSdk.config.configure(next, convictConfigParser(configSchema), true), + }); + config = reconciled.config; this.config.load(config); - ConfigController.getInstance().config = config; + ConfigController.getInstance().config = this.config.getProperties(); } if (!config || config.active || !config.hasOwnProperty('active')) await this.onConfig(); diff --git a/libraries/module-tools/src/helpers/wrapGrpcFunctions.ts b/libraries/module-tools/src/helpers/wrapGrpcFunctions.ts index 579f83ce6..711dbde04 100644 --- a/libraries/module-tools/src/helpers/wrapGrpcFunctions.ts +++ b/libraries/module-tools/src/helpers/wrapGrpcFunctions.ts @@ -1,6 +1,6 @@ import { createVerifier } from 'fast-jwt'; import { status } from '@grpc/grpc-js'; -import { ConduitGrpcSdk, GrpcCallback } from '@conduitplatform/grpc-sdk'; +import { ConduitGrpcSdk, GrpcCallback, GrpcError } from '@conduitplatform/grpc-sdk'; interface JWT { moduleName: string; @@ -43,11 +43,19 @@ export function wrapGrpcFunctions( try { invoked = functions[name](call, callback); } catch (error) { - return throwError(callback, (error as Error).message); + return throwError( + callback, + (error as Error).message, + error instanceof GrpcError ? error.code : status.INTERNAL, + ); } if (typeof invoked?.then === 'function') { invoked.then().catch((error: Error) => { - return throwError(callback, error.message); + return throwError( + callback, + error.message, + error instanceof GrpcError ? error.code : status.INTERNAL, + ); }); } }; diff --git a/libraries/module-tools/src/utilities/index.ts b/libraries/module-tools/src/utilities/index.ts index c43b3e854..20d948cf4 100644 --- a/libraries/module-tools/src/utilities/index.ts +++ b/libraries/module-tools/src/utilities/index.ts @@ -4,3 +4,5 @@ export * from './merge.js'; export * from './exportHelpers.js'; export * from './conduitPeers.js'; export * from './convictConfigParser.js'; +export * from './redactSensitiveConfig.js'; +export * from './reconcileModuleConfig.js'; diff --git a/libraries/module-tools/src/utilities/reconcileModuleConfig.test.ts b/libraries/module-tools/src/utilities/reconcileModuleConfig.test.ts new file mode 100644 index 000000000..fca1b198b --- /dev/null +++ b/libraries/module-tools/src/utilities/reconcileModuleConfig.test.ts @@ -0,0 +1,67 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + containsRedactedMarker, + reconcileStoredModuleConfig, + storedConfigsEquivalent, +} from '../../dist/index.esm.js'; + +describe('stored module config reconciliation', () => { + it('persists migrated config once and skips equivalent follow-ups', async () => { + const stored = { + providers: { + 'openai-compatible': { + model: 'text-embedding-3-small', + dimensions: 1536, + models: [], + }, + }, + }; + const migrated = { + providers: { + 'openai-compatible': { + models: [{ name: 'text-embedding-3-small', dimensions: 1536 }], + defaultModel: 'text-embedding-3-small', + }, + }, + }; + let overrideCalls = 0; + const first = await reconcileStoredModuleConfig({ + stored, + migrated, + configureOverride: async config => { + overrideCalls += 1; + return config; + }, + }); + assert.equal(first.persisted, true); + assert.equal(overrideCalls, 1); + assert.equal(storedConfigsEquivalent(first.config, migrated), true); + + const second = await reconcileStoredModuleConfig({ + stored: first.config, + migrated, + configureOverride: async config => { + overrideCalls += 1; + return config; + }, + }); + assert.equal(second.persisted, false); + assert.equal(overrideCalls, 1); + }); + + it('does not persist redacted secrets', async () => { + let overrideCalls = 0; + const result = await reconcileStoredModuleConfig({ + stored: { apiKey: 'sk-live' }, + migrated: { apiKey: '[REDACTED]' }, + configureOverride: async config => { + overrideCalls += 1; + return config; + }, + }); + assert.equal(result.persisted, false); + assert.equal(overrideCalls, 0); + assert.equal(containsRedactedMarker({ apiKey: '[REDACTED]' }), true); + }); +}); diff --git a/libraries/module-tools/src/utilities/reconcileModuleConfig.ts b/libraries/module-tools/src/utilities/reconcileModuleConfig.ts new file mode 100644 index 000000000..18455360f --- /dev/null +++ b/libraries/module-tools/src/utilities/reconcileModuleConfig.ts @@ -0,0 +1,42 @@ +import { containsRedactedMarker } from './redactSensitiveConfig.js'; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function sortJson(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(sortJson); + } + if (!isRecord(value)) { + return value; + } + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, nested]) => [key, sortJson(nested)]), + ); +} + +export function stableConfigJson(value: unknown): string { + return JSON.stringify(sortJson(value)); +} + +export function storedConfigsEquivalent(left: unknown, right: unknown): boolean { + return stableConfigJson(left) === stableConfigJson(right); +} + +export async function reconcileStoredModuleConfig(args: { + stored: T; + migrated: T; + configureOverride: (config: T) => Promise; +}): Promise<{ config: T; persisted: boolean }> { + if (storedConfigsEquivalent(args.stored, args.migrated)) { + return { config: args.migrated, persisted: false }; + } + if (containsRedactedMarker(args.migrated)) { + return { config: args.migrated, persisted: false }; + } + const persisted = await args.configureOverride(args.migrated); + return { config: persisted, persisted: true }; +} diff --git a/libraries/module-tools/src/utilities/redactSensitiveConfig.ts b/libraries/module-tools/src/utilities/redactSensitiveConfig.ts new file mode 100644 index 000000000..88023d818 --- /dev/null +++ b/libraries/module-tools/src/utilities/redactSensitiveConfig.ts @@ -0,0 +1,91 @@ +const WELL_KNOWN_SECRET_KEYS = + /^(apiKey|api_key|password|secret|privateKey|private_key)$/i; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isSchemaLeaf(value: unknown): value is Record { + return ( + isRecord(value) && + ('default' in value || 'format' in value || 'type' in value) && + !isRecord(value._cvtProperties) + ); +} + +function isSensitiveLeaf(value: unknown): boolean { + return isSchemaLeaf(value) && value.sensitive === true; +} + +function unwrapSchema(schema: unknown): unknown { + if (isRecord(schema) && isRecord(schema._cvtProperties)) { + return schema._cvtProperties; + } + return schema; +} + +const REDACTED_MARKER = '[REDACTED]'; + +export function containsRedactedMarker(value: unknown): boolean { + if (value === REDACTED_MARKER) return true; + if (Array.isArray(value)) { + return value.some(containsRedactedMarker); + } + if (!isRecord(value)) return false; + return Object.values(value).some(containsRedactedMarker); +} + +export function redactSensitiveConfig(config: T, schema?: unknown): T { + if (!isRecord(config)) return config; + const redacted = Array.isArray(config) ? [...config] : { ...config }; + const node = unwrapSchema(schema); + for (const [key, value] of Object.entries(redacted as Record)) { + const childSchema = isRecord(node) ? node[key] : undefined; + if (isSensitiveLeaf(childSchema) || WELL_KNOWN_SECRET_KEYS.test(key)) { + if (typeof value === 'string' && value.length > 0) { + (redacted as Record)[key] = REDACTED_MARKER; + } + continue; + } + if (isRecord(value) || Array.isArray(value)) { + (redacted as Record)[key] = redactSensitiveConfig( + value, + childSchema, + ); + } + } + return redacted as T; +} + +export function restoreRedactedSecrets(incoming: T, current: T, schema?: unknown): T { + if (Array.isArray(incoming) && Array.isArray(current)) { + return incoming.map((item, index) => + restoreRedactedSecrets(item, current[index], schema), + ) as T; + } + if (!isRecord(incoming) || !isRecord(current)) return incoming; + const restored: Record = { ...incoming }; + const node = unwrapSchema(schema); + for (const [key, value] of Object.entries(restored)) { + const childSchema = isRecord(node) ? node[key] : undefined; + const currentValue = current[key]; + if (isSensitiveLeaf(childSchema) || WELL_KNOWN_SECRET_KEYS.test(key)) { + if ( + value === REDACTED_MARKER && + typeof currentValue === 'string' && + currentValue.length > 0 && + currentValue !== REDACTED_MARKER + ) { + restored[key] = currentValue; + } + continue; + } + if ( + (isRecord(value) || Array.isArray(value)) && + (isRecord(currentValue) || Array.isArray(currentValue)) + ) { + restored[key] = restoreRedactedSecrets(value, currentValue, childSchema); + } + } + return restored as T; +} diff --git a/modules/database/README.md b/modules/database/README.md new file mode 100644 index 000000000..950630943 --- /dev/null +++ b/modules/database/README.md @@ -0,0 +1,49 @@ +# Database Module + +## Vector Search + +Conduit supports provider-neutral vector storage and search through `TYPE.Vector`, +vector index contracts, and the Database gRPC/admin vector APIs. + +### Schema Field + +```ts +{ + embedding: { + type: TYPE.Vector, + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + select: false, + }, +} +``` + +### Capabilities + +Use `getVectorCapabilities` before creating indexes or running searches. Capability +responses distinguish storage support from index/search support: + +- MongoDB stores vectors as numeric arrays and uses MongoDB Search/Vector Search + indexes when `createSearchIndex` and `listSearchIndexes` are available. +- PostgreSQL uses `pgvector`; the adapter attempts `CREATE EXTENSION IF NOT EXISTS + vector` during startup and reports missing privileges or extension support through + capabilities. +- Other Sequelize dialects report vector search as unsupported. + +### Rollout + +1. Add a `TYPE.Vector` field to the schema or via the Embeddings module schema + extension. +2. Call `getVectorCapabilities` and verify `indexing` and `search` are true. +3. Create a vector index with the field, dimensions, similarity, and optional + filter fields. +4. Backfill embeddings. +5. Run `vectorSearch` with a query vector. + +The Embeddings module is a separate opt-in image (not standalone v1). See +[deploy/embeddings.md](../../deploy/embeddings.md) before enabling generation +or search. Live Atlas/pgvector validation is an operator runbook step, not CI. + +CMS create/update bodies omit `TYPE.Vector` fields, `*SourceHash` fields, and +any `select: false` field so clients cannot write managed embeddings. Read/return +projections still include those schema fields. diff --git a/modules/database/package.bundle-lock.json b/modules/database/package.bundle-lock.json index 3b64c95e0..50750c69e 100644 --- a/modules/database/package.bundle-lock.json +++ b/modules/database/package.bundle-lock.json @@ -28,21 +28,22 @@ "mariadb": "^3.5.3", "mongodb": "^7.3.0", "mongodb-schema": "^12.7.0", - "mongoose": "^9.9.3", - "mysql2": "^3.22.5", + "mongoose": "^9.9.4", + "mysql2": "^3.23.1", "nice-grpc": "^2.1.17", "nice-grpc-client-middleware-retry": "^3.1.16", "nice-grpc-common": "^2.0.4", "object-hash": "^3.0.0", "pg": "^8.22.0", "pg-hstore": "^2.3.4", + "pgvector": "^0.3.0", "prom-client": "^15.1.3", "protobufjs": "^8.7.2", "sequelize": "^6.37.8", "sequelize-auto": "^0.8.8", "snappy": "7.4.1", "sqlite3": "^6.0.1", - "uuid": "14.0.1", + "uuid": "14.0.2", "winston": "^3.19.0", "winston-loki": "^6.1.7" }, @@ -51,24 +52,24 @@ } }, "node_modules/@bufbuild/protobuf": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.14.0.tgz", - "integrity": "sha512-C3UGsiCwSprE2NKIIFA3hCDlpXTMCAXRZuEVp88L1GY36Y41+rYL5fryE+nOFhp4p4JPQvdV8PQ4DWgHgeTE+w==", + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.14.1.tgz", + "integrity": "sha512-agRJn3+EJDUe8AvxTx/LnHA/GErvLE62pSaSk7+MwFOtOv8eWBu/qCq2qoZjBjVZ3C2aiFJCveuSs17KMkYGOw==", "license": "(Apache-2.0 AND BSD-3-Clause)" }, "node_modules/@colors/colors": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", - "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.1.tgz", + "integrity": "sha512-dTmUJzXSuayBK+hZydEaXd2mhx61qWQwkwaBBY6LyEOVx/L9aQU5ac8eFNEsd9nrD1+zb9zvDCphLSe8g1F4Qw==", "license": "MIT", "engines": { "node": ">=0.1.90" } }, "node_modules/@dabh/diagnostics": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.8.tgz", - "integrity": "sha512-R4MSXTVnuMzGD7bzHdW2ZhhdPC/igELENcq5IjEverBvq5hn1SXCWcsi6eSsdWP0/Ur+SItRRjAktmdoX/8R/Q==", + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.9.tgz", + "integrity": "sha512-R6siwR65Hm+3yfgP7o8DKhNvputQAwfoz9zTc3kyDudnomj2/BcLmD+uGQQPICjuFUp8ounPBU+jmKsocwVVAg==", "license": "MIT", "dependencies": { "@so-ric/colorspace": "^1.1.6", @@ -108,9 +109,9 @@ } }, "node_modules/@grpc/proto-loader/node_modules/protobufjs": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", - "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -168,9 +169,9 @@ } }, "node_modules/@mongodb-js/saslprep": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.5.0.tgz", - "integrity": "sha512-Hk1SKJCMcCos38+vqDnZzlIo4XRj9yCGzYkjB4LcqpeXRIYfia1UWTz+VrueLxoU+uSRJzgkufxoRZg8gi52YA==", + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.5.4.tgz", + "integrity": "sha512-05UC0jQsjKAOuXQ0H9Ud9vUTJpZIg+n/FinpR30tI5I8pY2inTfPOZ5OF/cg3Ce/N9MoD1xhRCeOsJtuTbFYlw==", "license": "MIT", "dependencies": { "sparse-bitfield": "^3.0.3" @@ -642,12 +643,12 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "26.2.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz", - "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==", + "version": "22.20.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.2.tgz", + "integrity": "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==", "license": "MIT", "dependencies": { - "undici-types": "~8.3.0" + "undici-types": "~6.21.0" } }, "node_modules/@types/triple-beam": { @@ -956,9 +957,9 @@ } }, "node_modules/bullmq": { - "version": "5.81.3", - "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.81.3.tgz", - "integrity": "sha512-Q7uEH2G92rVjX3Yl2qw3+6VEO15uxehC6DfeTHQBQTehddcRDSDuquD9zuvDjxiCffXsPUaBwrRlvlbVcgKs7g==", + "version": "5.81.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.81.4.tgz", + "integrity": "sha512-n+WHSzz20KooBGoJyASre6oJNz/p5f1IJRRN2ibD+NWPQaNpNQmDrwYuPO+bsXrQeM1MQzUxXGbdjmyOFKP2xQ==", "license": "MIT", "dependencies": { "cron-parser": "4.9.0", @@ -1499,9 +1500,9 @@ } }, "node_modules/fast-jwt": { - "version": "6.3.2", - "resolved": "https://registry.npmjs.org/fast-jwt/-/fast-jwt-6.3.2.tgz", - "integrity": "sha512-JTQImpkXVvj+eq7tJImtsHRt1K6ngloEzIx62Qbf9x4tEM2P2EqGpYJSeUoPf/kMn78rImSHfhf08KShR8PauA==", + "version": "6.3.3", + "resolved": "https://registry.npmjs.org/fast-jwt/-/fast-jwt-6.3.3.tgz", + "integrity": "sha512-pQDXx7IHeZT4jSmpE9o80RrBqfrG4fPrl8anazSM5vErIdK1iCc13z/EWX+H0j7liWSRnwTpHswIKMeLYGAckw==", "license": "Apache-2.0", "dependencies": { "@lukeed/ms": "^2.0.2", @@ -1965,9 +1966,9 @@ "optional": true }, "node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "funding": [ { "type": "github", @@ -2055,6 +2056,15 @@ "node": ">= 12.0.0" } }, + "node_modules/logform/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "license": "MIT", + "engines": { + "node": ">=0.1.90" + } + }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -2071,9 +2081,9 @@ } }, "node_modules/lru.min": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.4.tgz", - "integrity": "sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.5.tgz", + "integrity": "sha512-5J9ysMYUpYIg9RF2vJpy9SinEmSviFSe0GyPpCQ4L5QSkLAgeLXlTAOu2ZwWUU5m+0SBl6gUU1R1ZQB3aKypfA==", "license": "MIT", "engines": { "bun": ">=1.0.0", @@ -2095,9 +2105,9 @@ } }, "node_modules/mariadb": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/mariadb/-/mariadb-3.5.3.tgz", - "integrity": "sha512-i053Kc0MgdUv/hu9mCyq67TYfPXFj3/MV8I7ZW5wvJNixIyXC0VztMPUjIVj/449nQo+BsxFD4Fdk/sA/uqKPQ==", + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/mariadb/-/mariadb-3.5.4.tgz", + "integrity": "sha512-3m0vdfgRkjle5/9hHl4R7rlVvnDA7QfuFSNkdIAQNROWKOS0J3b9IQK9HaZSCGEVKiDUbFXFiINfC+cmR5P/aA==", "license": "LGPL-2.1-or-later", "dependencies": { "@types/geojson": "^7946.0.16", @@ -2272,9 +2282,9 @@ } }, "node_modules/mongodb": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.5.0.tgz", - "integrity": "sha512-5FnrEDLnvp6ycUOGLNLLU33BfCx2qmp2mJjGPDwKLruYsVzXVSK5fsGpoDXvsXJwBfBsD7ebMRdawbDxC2814g==", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.6.0.tgz", + "integrity": "sha512-WbZ6OCjYw2c53LOjfkQa+reXr7kIiOVpXXglnASFuiMtif0BvsMwHe3ClJHLm1/r7wJFwaasfFtD6iYIktB01g==", "license": "Apache-2.0", "dependencies": { "@mongodb-js/saslprep": "^1.4.11", @@ -2331,9 +2341,9 @@ } }, "node_modules/mongodb-ns": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/mongodb-ns/-/mongodb-ns-3.2.1.tgz", - "integrity": "sha512-PhuRl7oAzHbILnn+DoJkHnfduh7VRKMYnkzOv/x32GC8YvnhYXr42rhzxIK/UQvyZcnG968yVvCvjF2xSUP1OQ==", + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/mongodb-ns/-/mongodb-ns-3.2.5.tgz", + "integrity": "sha512-XGMCU8sDN1OK2Ls38pKDLMLgQd4CYXkaiaHmM/2B67XDUjgrBjAdQpNSmbU5thBGk3IosZHw0oe81rdw9v1rVw==", "license": "Apache-2.0", "optional": true }, @@ -2362,9 +2372,9 @@ } }, "node_modules/mongoose": { - "version": "9.9.3", - "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-9.9.3.tgz", - "integrity": "sha512-9dQaKct05LNgVkunDzFPZsGGBhinobl3Qc5B5KYJkLNG5lBLgqESUEItl1EUIkVaCwZJGBgTuvJiFAjjIATCiw==", + "version": "9.9.5", + "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-9.9.5.tgz", + "integrity": "sha512-t/kUoeDjlHmav7yCaq//YKU3wiIgUPaTj4GrluiTHUw4jQ77/CXtdMDSor4f1u3SKLhIK6NgCF+AzzUqU02Aag==", "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", @@ -2383,6 +2393,52 @@ "url": "https://opencollective.com/mongoose" } }, + "node_modules/mongoose/node_modules/mongodb": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.5.0.tgz", + "integrity": "sha512-5FnrEDLnvp6ycUOGLNLLU33BfCx2qmp2mJjGPDwKLruYsVzXVSK5fsGpoDXvsXJwBfBsD7ebMRdawbDxC2814g==", + "license": "Apache-2.0", + "dependencies": { + "@mongodb-js/saslprep": "^1.4.11", + "bson": "^7.2.0", + "mongodb-connection-string-url": "^7.0.1" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.806.0", + "@mongodb-js/zstd": "^7.0.0", + "gcp-metadata": "^7.0.1", + "kerberos": "^7.0.0", + "mongodb-client-encryption": "^7.2.0", + "snappy": "^7.3.2", + "socks": "^2.8.6" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, "node_modules/mpath": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.9.0.tgz", @@ -2439,9 +2495,9 @@ } }, "node_modules/mysql2": { - "version": "3.24.2", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.24.2.tgz", - "integrity": "sha512-l9kXeKGwd6VCbSjmpO/bLWb+YCYpbvE4whte8ec6InXebvCNyEEydyRCnRU+TSHNqRLJ8B+Tk1uWb+vMCJgJzA==", + "version": "3.24.4", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.24.4.tgz", + "integrity": "sha512-A2olluVlj0mvgyIRRISMEzXc51m+21mRtcMVjJyIpt2GG98+XrC9m9HzsqcMsX2LcnfccJvY5NB22g8fENBnOA==", "license": "MIT", "dependencies": { "aws-ssl-profiles": "^1.1.2", @@ -2537,9 +2593,9 @@ } }, "node_modules/node-abi": { - "version": "3.94.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.94.0.tgz", - "integrity": "sha512-W5ZNO5KRPB5TkYmGVD9F6YqhsglXJzE6etpbmT+f6EQElhiX/UTG551cnsRGvLG3fyZEg9HwaDmNmj5nwJ4z9g==", + "version": "3.96.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.96.0.tgz", + "integrity": "sha512-rebQ/lz7i0EkoLzUVSrKRzA69zMkwLp95kKMWoMDkkM00Suxz0D7zEQPwRml5fQum24mj7bPvmlgLAmu2JCiYg==", "license": "MIT", "dependencies": { "semver": "^7.3.5" @@ -2806,6 +2862,15 @@ "split2": "^4.1.0" } }, + "node_modules/pgvector": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/pgvector/-/pgvector-0.3.0.tgz", + "integrity": "sha512-+t7qcQD2us8fO8YIq/3lA0gUrD+bVO70MG1MhcDcxJz/OlRGGIIHzFq/4x57Vn/LpzX5wFdfOTLQp9QMPd4ljQ==", + "license": "MIT", + "engines": { + "node": ">=22" + } + }, "node_modules/picomatch": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", @@ -2909,6 +2974,7 @@ "version": "15.1.3", "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "deprecated": "prom-client has been replaced by @prometheus-io/client", "license": "Apache-2.0", "dependencies": { "@opentelemetry/api": "^1.4.0", @@ -2919,9 +2985,9 @@ } }, "node_modules/protobufjs": { - "version": "8.7.2", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.7.2.tgz", - "integrity": "sha512-oTVHV+oelUBtiu5iTuTNNZ0eLYsXSMxry4cgr30mayNkgIZL6qZ0IOQVPuSWGcyAaXKl/XgqwWHIC3a0khYVBA==", + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.8.0.tgz", + "integrity": "sha512-N3xhQ5yyBx3vQq4gubBfASzYhJGNzeDbjqBpu61g7UVylsN/qyffU96TKWD3GbbLOKF82VGNRNvv1+BFgE31Eg==", "license": "BSD-3-Clause", "dependencies": { "long": "^5.3.2" @@ -2972,9 +3038,9 @@ } }, "node_modules/qs": { - "version": "6.15.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", - "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", "license": "BSD-3-Clause", "dependencies": { "es-define-property": "^1.0.1", @@ -3715,9 +3781,9 @@ } }, "node_modules/tdigest": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", - "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.3.tgz", + "integrity": "sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==", "license": "MIT", "dependencies": { "bintrees": "1.0.2" @@ -3844,9 +3910,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "6.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.0.tgz", - "integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==", + "version": "6.28.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.1.tgz", + "integrity": "sha512-zWpdTVD54H48CIybL0rWQ3ukpb9d23wM7eH5RtfdmeP70cWHNjtfo7P4vZX+5CoDcO53J4Pu5uXp7lNfjc6DRA==", "license": "MIT", "optional": true, "engines": { @@ -3854,9 +3920,9 @@ } }, "node_modules/undici-types": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", - "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "license": "MIT" }, "node_modules/universalify": { @@ -3890,9 +3956,9 @@ "license": "MIT" }, "node_modules/uuid": { - "version": "14.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.1.tgz", - "integrity": "sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==", + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.2.tgz", + "integrity": "sha512-xZe/16rV4aa+HGSOCiY2YeLT1OybRLrrkL/Rqaq7p7GMVXjFh+6wN4oMYgjFmnSnhY8t6Xpdl2l9qmnHYuMHwQ==", "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" @@ -3997,9 +4063,9 @@ } }, "node_modules/winston-loki/node_modules/protobufjs": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", - "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { diff --git a/modules/database/package.bundle.json b/modules/database/package.bundle.json index 4ee835133..11646ec2c 100644 --- a/modules/database/package.bundle.json +++ b/modules/database/package.bundle.json @@ -38,7 +38,7 @@ "prom-client": "^15.1.3", "protobufjs": "^8.7.2", "snappy": "7.4.1", - "uuid": "14.0.1", + "uuid": "14.0.2", "winston": "^3.19.0", "winston-loki": "^6.1.7", "bullmq": "^5.79.0", @@ -47,11 +47,12 @@ "mariadb": "^3.5.3", "mongodb": "^7.3.0", "mongodb-schema": "^12.7.0", - "mongoose": "^9.9.3", - "mysql2": "^3.22.5", + "mongoose": "^9.9.4", + "mysql2": "^3.23.1", "object-hash": "^3.0.0", "pg": "^8.22.0", "pg-hstore": "^2.3.4", + "pgvector": "^0.3.0", "sequelize": "^6.37.8", "sequelize-auto": "^0.8.8", "sqlite3": "^6.0.1" diff --git a/modules/database/package.json b/modules/database/package.json index 4a17ae844..d7f771863 100644 --- a/modules/database/package.json +++ b/modules/database/package.json @@ -54,6 +54,7 @@ "object-hash": "^3.0.0", "pg": "^8.22.0", "pg-hstore": "^2.3.4", + "pgvector": "^0.3.0", "sequelize": "^6.37.8", "sequelize-auto": "^0.8.8", "sqlite3": "^6.0.1" diff --git a/modules/database/service-bundle.config.json b/modules/database/service-bundle.config.json index 55a39a87b..4cb26a636 100644 --- a/modules/database/service-bundle.config.json +++ b/modules/database/service-bundle.config.json @@ -17,6 +17,7 @@ "object-hash", "pg", "pg-hstore", + "pgvector", "sequelize", "sequelize-auto", "sqlite3" diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index 57286cb1e..ed46fa5bd 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -6,12 +6,16 @@ import { GrpcRequest, GrpcResponse, HealthCheckStatus, + VectorIndexDefinition, + VectorSimilarity, + defaultVectorIndexMethod, } from '@conduitplatform/grpc-sdk'; import { AdminHandlers } from './admin/index.js'; import { SchemaAdmin } from './admin/schema.admin.js'; import { CustomEndpointsAdmin } from './admin/customEndpoints/customEndpoints.admin.js'; import { DatabaseRoutes } from './routes/index.js'; import * as models from './models/index.js'; +import { DATABASE_SYSTEM_SCHEMAS } from './models/systemSchemas.js'; import { ColumnExistenceRequest, ColumnExistenceResponse, @@ -31,6 +35,14 @@ import { Schema as SchemaDto, UpdateManyRequest, UpdateRequest, + DeleteVectorIndexRequest, + VectorCapabilitiesRequest, + VectorCapabilitiesResponse, + VectorIndex, + VectorIndexListRequest, + VectorIndexListResponse, + VectorIndexRequest, + VectorSearchRequest, } from './protoTypes/database.js'; import { CreateSchemaExtensionRequest, @@ -42,7 +54,13 @@ import { MongooseAdapter } from './adapters/mongoose-adapter/index.js'; import { MongooseSchema } from './adapters/mongoose-adapter/MongooseSchema.js'; import { SequelizeSchema } from './adapters/sequelize-adapter/SequelizeSchema.js'; import { ConduitDatabaseSchema, IView, Schema } from './interfaces/index.js'; -import { canCreate, canDelete, canModify } from './permissions/index.js'; +import { + canCreate, + canDelete, + canModify, + vectorIndexDeleteMutationData, + vectorIndexMutationData, +} from './permissions/index.js'; import { runMigrations } from './migrations/index.js'; import { SchemaController } from './controllers/cms/schema.controller.js'; import { CustomEndpointController } from './controllers/customEndpoints/customEndpoint.controller.js'; @@ -61,6 +79,19 @@ import { type ImportResult, } from '@conduitplatform/module-tools'; import { QueueController } from './controllers/queue.controller.js'; +import { + buildMutationEventChunks, + collectBoundedMutationIds, + mutationEventChannel, + shouldPublishMutationEvent, + grpcStatusFromError, + callerModuleName, + resolveAdminOperatorContext, + assertVectorSearchAccess, + assertEmbeddingsJobCaller, + assertEmbeddingsJobRead, + assertEmbeddingsJobWrite, +} from './adapters/utils/index.js'; import AppConfigSchema, { Config } from './config/index.js'; import { Empty } from './protoTypes/google/protobuf/empty.js'; import { fileURLToPath } from 'node:url'; @@ -97,6 +128,11 @@ export default class DatabaseModule extends ManagedModule { migrate: this.migrate.bind(this), getDatabaseType: this.getDatabaseType.bind(this), generateId: this.generateId.bind(this), + getVectorCapabilities: this.getVectorCapabilities.bind(this), + createVectorIndex: this.createVectorIndex.bind(this), + getVectorIndexes: this.getVectorIndexes.bind(this), + deleteVectorIndex: this.deleteVectorIndex.bind(this), + vectorSearch: this.vectorSearch.bind(this), }, }; protected metricsSchema = metricsSchema; @@ -131,10 +167,11 @@ export default class DatabaseModule extends ManagedModule { const isReplica = this.grpcSdk.isAvailable('database'); await this._activeAdapter.registerSystemSchema(models.DeclaredSchema, isReplica); await this._activeAdapter.registerSystemSchema(models.MigratedSchemas, isReplica); - let modelPromises = Object.values(models).flatMap((model: ConduitSchema) => { - if (['_DeclaredSchema', 'MigratedSchemas'].includes(model.name)) return []; - return this._activeAdapter.registerSystemSchema(model, isReplica); - }); + let modelPromises = DATABASE_SYSTEM_SCHEMAS.filter( + model => + model.name !== models.DeclaredSchema.name && + model.name !== models.MigratedSchemas.name, + ).map(model => this._activeAdapter.registerSystemSchema(model, isReplica)); await Promise.all(modelPromises); await this._activeAdapter.retrieveForeignSchemas(); await this._activeAdapter.recoverSchemasFromDatabase(); @@ -142,7 +179,7 @@ export default class DatabaseModule extends ManagedModule { if (!isReplica) { await runMigrations(this._activeAdapter); } - modelPromises = Object.values(models).flatMap((model: ConduitSchema) => { + modelPromises = DATABASE_SYSTEM_SCHEMAS.map(model => { return this._activeAdapter.registerSystemSchema(model, isReplica).then(() => { if (this._activeAdapter.getDatabaseType() !== 'MongoDB' && !isReplica) { return this._activeAdapter.syncSchema(model.name); @@ -490,6 +527,15 @@ export default class DatabaseModule extends ManagedModule { ) { try { const schemaAdapter = this._activeAdapter.getSchemaModel(call.request.schemaName); + if (call.request.embeddingsJob) { + assertEmbeddingsJobCaller(callerModuleName(call.metadata)); + assertEmbeddingsJobRead({ + query: call.request.query, + select: call.request.select, + allowedFields: call.request.embeddingsAllowedFields, + schema: schemaAdapter.model.originalSchema as ConduitDatabaseSchema, + }); + } const doc = await schemaAdapter.model.findOne(call.request.query, { select: call.request.select, populate: call.request.populate, @@ -499,10 +545,7 @@ export default class DatabaseModule extends ManagedModule { }); callback(null, { result: JSON.stringify(doc) }); } catch (err) { - callback({ - code: status.INTERNAL, - message: (err as Error).message, - }); + callback(grpcStatusFromError(err)); } } @@ -561,7 +604,10 @@ export default class DatabaseModule extends ManagedModule { }); const docString = JSON.stringify(doc); - this.grpcSdk.bus?.publish(`${this.name}:create:${schemaName}`, docString); + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'create', schemaName), + docString, + ); callback(null, { result: docString }); } catch (err) { @@ -593,7 +639,10 @@ export default class DatabaseModule extends ManagedModule { }); const docsString = JSON.stringify(docs); - this.grpcSdk.bus?.publish(`${this.name}:createMany:${schemaName}`, docsString); + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'createMany', schemaName), + docsString, + ); callback(null, { result: docsString }); } catch (err) { @@ -608,13 +657,19 @@ export default class DatabaseModule extends ManagedModule { call: GrpcRequest, callback: GrpcResponse, ) { - const moduleName = call.metadata!.get('module-name')![0] as string; + const moduleName = callerModuleName(call.metadata); const { schemaName } = call.request; try { const schemaAdapter = this._activeAdapter.getSchemaModel(schemaName); - if ( + if (call.request.embeddingsJob) { + assertEmbeddingsJobCaller(moduleName); + assertEmbeddingsJobWrite({ + document: call.request.query, + schema: schemaAdapter.model.originalSchema as ConduitDatabaseSchema, + }); + } else if ( !(await canModify( - moduleName, + moduleName ?? '', schemaAdapter.model, JSON.parse(call.request.query), )) @@ -636,14 +691,16 @@ export default class DatabaseModule extends ManagedModule { ); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:update:${schemaName}`, resultString); + if (shouldPublishMutationEvent(call.request.suppressEvent)) { + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'update', schemaName), + resultString, + ); + } callback(null, { result: resultString }); } catch (err) { - callback({ - code: status.INTERNAL, - message: (err as Error).message, - }); + callback(grpcStatusFromError(err)); } } @@ -673,7 +730,12 @@ export default class DatabaseModule extends ManagedModule { ); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:update:${schemaName}`, resultString); + if (shouldPublishMutationEvent(call.request.suppressEvent)) { + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'update', schemaName), + resultString, + ); + } callback(null, { result: resultString }); } catch (err) { @@ -710,7 +772,12 @@ export default class DatabaseModule extends ManagedModule { ); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:update:${schemaName}`, resultString); + if (shouldPublishMutationEvent(call.request.suppressEvent)) { + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'update', schemaName), + resultString, + ); + } callback(null, { result: resultString }); } catch (err) { @@ -753,7 +820,12 @@ export default class DatabaseModule extends ManagedModule { ); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:updateMany:${schemaName}`, resultString); + if (shouldPublishMutationEvent(call.request.suppressEvent)) { + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'update', schemaName), + resultString, + ); + } callback(null, { result: resultString }); } catch (err) { @@ -785,6 +857,12 @@ export default class DatabaseModule extends ManagedModule { }); } + const ids = shouldPublishMutationEvent(call.request.suppressEvent) + ? await this.collectMutationIds(schemaAdapter.model, call.request.filterQuery, { + userId: call.request.userId, + scope: call.request.scope, + }) + : []; const result = await schemaAdapter.model.updateMany( call.request.filterQuery, call.request.query, @@ -796,14 +874,18 @@ export default class DatabaseModule extends ManagedModule { ); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:updateMany:${schemaName}`, resultString); + if (shouldPublishMutationEvent(call.request.suppressEvent)) { + for (const payload of buildMutationEventChunks(ids)) { + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'updateMany', schemaName), + payload, + ); + } + } callback(null, { result: resultString }); } catch (err) { - callback({ - code: status.INTERNAL, - message: (err as Error).message, - }); + callback(grpcStatusFromError(err)); } } @@ -828,7 +910,10 @@ export default class DatabaseModule extends ManagedModule { }); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:delete:${schemaName}`, resultString); + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'delete', schemaName), + resultString, + ); callback(null, { result: resultString }); } catch (err) { @@ -860,7 +945,10 @@ export default class DatabaseModule extends ManagedModule { }); const resultString = JSON.stringify(result); - this.grpcSdk.bus?.publish(`${this.name}:delete:${schemaName}`, resultString); + this.grpcSdk.bus?.publish( + mutationEventChannel(this.name, 'delete', schemaName), + resultString, + ); callback(null, { result: resultString }); } catch (err) { @@ -943,6 +1031,147 @@ export default class DatabaseModule extends ManagedModule { callback(null, { result: exist }); } + async getVectorCapabilities( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this._activeAdapter.getVectorCapabilities( + call.request.schemaName, + ); + callback(null, result); + } catch (err) { + callback({ code: status.INTERNAL, message: (err as Error).message }); + } + } + + async createVectorIndex( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + if (!call.request.index) { + return callback({ + code: status.INVALID_ARGUMENT, + message: 'Vector index definition is required', + }); + } + const moduleName = call.metadata!.get('module-name')![0] as string; + const schemaAdapter = this._activeAdapter.getSchemaModel(call.request.schemaName); + const index = this.parseVectorIndex(call.request.index); + if ( + !(await canModify( + moduleName, + schemaAdapter.model, + vectorIndexMutationData(index.field), + )) + ) { + return callback({ + code: status.PERMISSION_DENIED, + message: `Module ${moduleName} is not authorized to create vector indexes for ${call.request.schemaName}!`, + }); + } + const result = await this._activeAdapter.createVectorIndex( + call.request.schemaName, + index, + ); + callback(null, { result: JSON.stringify(result) }); + } catch (err) { + callback(grpcStatusFromError(err)); + } + } + + async getVectorIndexes( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const indexes = await this._activeAdapter.getVectorIndexes(call.request.schemaName); + callback(null, { + indexes: indexes.map(index => ({ + field: index.field, + dimensions: index.dimensions, + similarity: index.similarity, + name: index.name, + method: defaultVectorIndexMethod(index.method), + filterFields: [...(index.filterFields ?? [])], + options: index.options ? JSON.stringify(index.options) : undefined, + status: index.status, + queryable: index.queryable, + })), + }); + } catch (err) { + callback(grpcStatusFromError(err)); + } + } + + async deleteVectorIndex( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const moduleName = call.metadata!.get('module-name')![0] as string; + const schemaAdapter = this._activeAdapter.getSchemaModel(call.request.schemaName); + const liveIndexes = await this._activeAdapter.getVectorIndexes( + call.request.schemaName, + ); + if ( + !(await canModify( + moduleName, + schemaAdapter.model, + vectorIndexDeleteMutationData(liveIndexes, call.request.indexName), + )) + ) { + return callback({ + code: status.PERMISSION_DENIED, + message: `Module ${moduleName} is not authorized to delete vector indexes for ${call.request.schemaName}!`, + }); + } + const result = await this._activeAdapter.deleteVectorIndex( + call.request.schemaName, + call.request.indexName, + ); + callback(null, { result: JSON.stringify(result) }); + } catch (err) { + callback(grpcStatusFromError(err)); + } + } + + async vectorSearch( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const schemaAdapter = this._activeAdapter.getSchemaModel(call.request.schemaName); + const adminOperator = resolveAdminOperatorContext({ + requested: call.request.adminOperator, + callerModule: callerModuleName(call.metadata), + }); + assertVectorSearchAccess({ + authzEnabled: !!schemaAdapter.model.authzEnabled, + userId: call.request.userId, + scope: call.request.scope, + adminOperator, + }); + const result = await this._activeAdapter.vectorSearch({ + schemaName: call.request.schemaName, + field: call.request.field, + vector: call.request.vector, + indexName: call.request.indexName, + filter: call.request.filter ? JSON.parse(call.request.filter) : undefined, + limit: call.request.limit, + numCandidates: call.request.numCandidates, + select: call.request.select, + userId: call.request.userId, + scope: call.request.scope, + adminOperator, + }); + callback(null, { result: JSON.stringify(result) }); + } catch (err) { + callback(grpcStatusFromError(err)); + } + } + async migrate(call: GrpcRequest, callback: GrpcResponse) { if (this._activeAdapter.getDatabaseType() !== 'MongoDB') { const schemaName = call.request.schemaName; @@ -977,6 +1206,18 @@ export default class DatabaseModule extends ManagedModule { callback(null, { result }); } + private parseVectorIndex(index: VectorIndex): VectorIndexDefinition { + return { + field: index.field, + dimensions: index.dimensions, + similarity: index.similarity as VectorSimilarity, + name: index.name, + method: defaultVectorIndexMethod(index.method), + filterFields: index.filterFields, + options: index.options ? JSON.parse(index.options) : undefined, + }; + } + private registerInstanceSyncEvents() { this.grpcSdk.bus?.subscribe('database:request:schemas', () => { this._activeAdapter.registeredSchemas.forEach(schema => { @@ -1046,4 +1287,22 @@ export default class DatabaseModule extends ManagedModule { ); } } + + private async collectMutationIds( + model: MongooseSchema | SequelizeSchema, + filterQuery: string, + options: { userId?: string; scope?: string }, + ): Promise { + return collectBoundedMutationIds({ + findPage: (skip, limit) => + model.findMany(filterQuery, { + select: '_id', + skip, + limit, + sort: { _id: 1 }, + userId: options.userId, + scope: options.scope, + }), + }); + } } diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index 7e5f898c9..18f27662c 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -8,9 +8,15 @@ import { RawMongoQuery, RawSQLQuery, TYPE, + VectorCapabilities, + VectorIndexDefinition, + VectorSearchInput, + VectorSearchResult, } from '@conduitplatform/grpc-sdk'; import { ConfigController } from '@conduitplatform/module-tools'; import type { Config } from '../config/index.js'; +import { unsupportedVectorCapabilities } from './utils/vectorCapabilities.js'; +import { declaredVectorIndexes } from './utils/vectorSearchQuery.js'; import { _ConduitSchema, ConduitDatabaseSchema, @@ -21,7 +27,7 @@ import { stitchSchema, validateExtensionFields } from './utils/extensions.js'; import { status } from '@grpc/grpc-js'; import { isEqual, isNil } from 'lodash-es'; import ObjectHash from 'object-hash'; -import * as systemModels from '../models/index.js'; +import { DATABASE_SYSTEM_SCHEMA_NAME_SET } from '../models/systemSchemas.js'; export abstract class DatabaseAdapter { registeredSchemas: Map; @@ -296,6 +302,56 @@ export abstract class DatabaseAdapter { rawQuery: RawMongoQuery | RawSQLQuery, ): Promise; + getVectorCapabilities(schemaName?: string): Promise { + void schemaName; + return Promise.resolve(unsupportedVectorCapabilities(this.getDatabaseType())); + } + + createVectorIndex(schemaName: string, index: VectorIndexDefinition): Promise { + void schemaName; + void index; + throw new GrpcError( + status.UNIMPLEMENTED, + `${this.getDatabaseType()} does not support vector indexes`, + ); + } + + getVectorIndexes(schemaName: string): Promise { + void schemaName; + return Promise.resolve([]); + } + + deleteVectorIndex(schemaName: string, indexName: string): Promise { + void schemaName; + void indexName; + throw new GrpcError( + status.UNIMPLEMENTED, + `${this.getDatabaseType()} does not support vector indexes`, + ); + } + + protected async applyDeclaredVectorIndexes( + schemaName: string, + isInstanceSync: boolean, + ): Promise { + if (isInstanceSync) return; + const declared = declaredVectorIndexes(this.models[schemaName]?.originalSchema ?? {}); + if (!declared.length) return; + const capabilities = await this.getVectorCapabilities(schemaName); + if (!capabilities.indexing) return; + for (const index of declared) { + await this.createVectorIndex(schemaName, index); + } + } + + vectorSearch(request: VectorSearchInput): Promise { + void request; + throw new GrpcError( + status.UNIMPLEMENTED, + `${this.getDatabaseType()} does not support vector search`, + ); + } + abstract syncSchema(name: string): Promise; fixDatabaseSchemaOwnership(schema: ConduitSchema) { @@ -405,14 +461,8 @@ export abstract class DatabaseAdapter { { readPreference: 'primary' }, ); models = models - // do not recover system schemas as they have already been - .filter((model: _ConduitSchema) => { - let isSystemModel = false; - Object.values(systemModels).forEach((systemModel: ConduitSchema) => { - systemModel.name === model.name && (isSystemModel = true); - }); - return !isSystemModel; - }) + // do not recover system schemas; they are already registered + .filter((model: _ConduitSchema) => !DATABASE_SYSTEM_SCHEMA_NAME_SET.has(model.name)) .map((model: _ConduitSchema) => { const schema = new ConduitSchema( model.name, diff --git a/modules/database/src/adapters/SchemaAdapter.ts b/modules/database/src/adapters/SchemaAdapter.ts index e3a8559b1..4c6f5b390 100644 --- a/modules/database/src/adapters/SchemaAdapter.ts +++ b/modules/database/src/adapters/SchemaAdapter.ts @@ -218,6 +218,33 @@ export abstract class SchemaAdapter { } } + async lookupAuthorizedCandidateIds( + operation: string, + candidateIds: Array, + userId?: string, + scope?: string, + ): Promise { + const ids = candidateIds.map(id => String(id)).filter(Boolean); + if (!ids.length) return []; + if (!this.authzEnabled || (isNil(userId) && isNil(scope))) { + return ids; + } + const view = await this.permissionCheck(operation, userId, scope); + if (!view) return ids; + const query = + this.adapter.getDatabaseType() === 'MongoDB' + ? { _id: { $in: ids } } + : { _id: { [Op.in]: ids } }; + const docs = await this.runAuthorizedViewQuery(operation, userId, scope, view, v => + v.findMany(query, { + select: '_id', + userId: undefined, + scope: undefined, + }), + ); + return (docs ?? []).map((doc: { _id?: unknown }) => String(doc._id)); + } + async getPaginatedAuthorizedQuery( operation: string, query: Indexable, diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index 81a995253..fe9dab7ba 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -8,6 +8,8 @@ import { } from '@conduitplatform/grpc-sdk'; import { cloneDeep, isArray, isNil, isObject } from 'lodash-es'; import { checkIfMongoOptions } from './utils.js'; +import { applyMongoVectorField } from '../utils/vectorMappings.js'; +import { isVectorTypeName } from '../utils/vectorField.js'; import * as deepdash from 'deepdash-es/standalone'; @@ -87,6 +89,10 @@ function convert(value: any, key: any, parentValue: any) { parentValue[key].type = Schema.Types.Mixed; } + if (isVectorTypeName(parentValue[key]?.type)) { + parentValue[key] = applyMongoVectorField(parentValue[key]); + } + if (!isNil(parentValue[key]) && parentValue[key] === 'JSON') { parentValue[key] = Schema.Types.Mixed; } diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 7d5362f57..47c3b1a4b 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -9,9 +9,27 @@ import { ModelOptionsIndexes, MongoIndexType, RawMongoQuery, + VectorCapabilities, + VectorIndexDefinition, + VectorSearchInput, + VectorSearchResult, } from '@conduitplatform/grpc-sdk'; import { DatabaseAdapter } from '../DatabaseAdapter.js'; -import { validateFieldChanges, validateFieldConstraints } from '../utils/index.js'; +import { + validateFieldChanges, + validateFieldConstraints, + mongoVectorCapabilities, + fromMongoVectorIndex, + toMongoVectorIndexDefinition, + assertVectorSearchAccess, + completeVectorSearch, + declaredVectorIndexes, + mergeVectorIndexes, + planMongoVectorSearch, + bindVectorIndexToField, + planMongoVectorIndexCreate, + assertMongoVectorSearchIndexDropTarget, +} from '../utils/index.js'; import pluralize from '../../utils/pluralize.js'; import { mongoSchemaConverter } from '../../introspection/mongoose/utils.js'; import { status } from '@grpc/grpc-js'; @@ -720,6 +738,146 @@ export class MongooseAdapter extends DatabaseAdapter { return 'Indexes deleted'; } + async getVectorCapabilities(schemaName?: string): Promise { + const modelName = schemaName ?? Object.keys(this.models)[0]; + if (!modelName || !this.models[modelName]) { + return mongoVectorCapabilities({ hasSchema: false }); + } + + try { + const collection: any = this.mongoose.model(modelName).collection; + if (typeof collection.listSearchIndexes !== 'function') { + return mongoVectorCapabilities({ + hasSchema: true, + searchIndexCommandsAvailable: false, + }); + } + await collection.listSearchIndexes().toArray(); + return mongoVectorCapabilities({ hasSchema: true }); + } catch (err) { + return mongoVectorCapabilities({ + hasSchema: true, + probeError: (err as Error).message, + }); + } + } + + async createVectorIndex( + schemaName: string, + index: VectorIndexDefinition, + ): Promise { + if (!this.models[schemaName]) + throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const schema = this.models[schemaName].originalSchema; + const field = schema.compiledFields?.[index.field] ?? schema.fields?.[index.field]; + const bound = bindVectorIndexToField({ + provider: 'mongodb', + index, + field, + }); + const collection: any = this.mongoose.model(schemaName).collection; + if (typeof collection.createSearchIndex !== 'function') { + throw new GrpcError( + status.FAILED_PRECONDITION, + 'MongoDB Vector Search index commands are not available for this deployment', + ); + } + const existing = await this.getVectorIndexes(schemaName); + const plan = planMongoVectorIndexCreate({ requested: bound, existing }); + if (plan.action === 'reuse') return 'Vector index created!'; + await collection.createSearchIndex({ + name: bound.name, + type: 'vectorSearch', + definition: toMongoVectorIndexDefinition(bound), + }); + return 'Vector index created!'; + } + + async getVectorIndexes(schemaName: string): Promise { + if (!this.models[schemaName]) + throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const collection: any = this.mongoose.model(schemaName).collection; + if (typeof collection.listSearchIndexes !== 'function') return []; + const indexes = await collection.listSearchIndexes().toArray(); + return indexes + .filter((index: any) => index.type === 'vectorSearch') + .map((index: any) => fromMongoVectorIndex(index)); + } + + async deleteVectorIndex(schemaName: string, indexName: string): Promise { + if (!this.models[schemaName]) + throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const collection: any = this.mongoose.model(schemaName).collection; + if ( + typeof collection.dropSearchIndex !== 'function' || + typeof collection.listSearchIndexes !== 'function' + ) { + throw new GrpcError( + status.FAILED_PRECONDITION, + 'MongoDB Vector Search index commands are not available for this deployment', + ); + } + const indexes = await collection.listSearchIndexes().toArray(); + const existing = (indexes as Array<{ name?: string; type?: string }>).find( + index => index.name === indexName, + ); + assertMongoVectorSearchIndexDropTarget({ indexName, existing }); + await collection.dropSearchIndex(indexName); + return 'Vector index deleted'; + } + + async vectorSearch(request: VectorSearchInput): Promise { + if (!this.models[request.schemaName]) + throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const model = this.models[request.schemaName]; + const schemaField = + model.originalSchema.compiledFields?.[request.field] ?? + model.originalSchema.fields?.[request.field]; + if (schemaField?.type !== 'Vector') { + throw new GrpcError(status.INVALID_ARGUMENT, 'Requested field is not a vector'); + } + if (request.vector.length !== schemaField.dimensions) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Vector dimensions mismatch: expected ${schemaField.dimensions}`, + ); + } + + assertVectorSearchAccess({ + authzEnabled: !!model.authzEnabled, + userId: request.userId, + scope: request.scope, + adminOperator: request.adminOperator, + }); + + const schemaFields = (model.originalSchema.compiledFields ?? + model.originalSchema.fields) as Record; + const liveIndexes = await this.getVectorIndexes(request.schemaName); + const planned = planMongoVectorSearch({ + request, + indexes: mergeVectorIndexes( + declaredVectorIndexes(model.originalSchema), + liveIndexes, + ), + schemaFields, + }); + return completeVectorSearch({ + emptyResult: planned.emptyResult, + limit: planned.limits.limit, + authzEnabled: !!model.authzEnabled, + adminOperator: request.adminOperator, + provider: 'mongodb', + metric: schemaField.similarity, + fetchCandidates: async () => + this.mongoose + .model(request.schemaName) + .collection.aggregate(planned.pipeline) + .toArray(), + lookupAuthorizedIds: ids => + model.lookupAuthorizedCandidateIds('read', ids, request.userId, request.scope), + }); + } + async execRawQuery(schemaName: string, rawQuery: RawMongoQuery) { let collection = this.models[schemaName]?.model.collection; if (!collection) { @@ -847,6 +1005,7 @@ export class MongooseAdapter extends DatabaseAdapter { if (!isInstanceSync) { await this.createMongooseFieldIndexes(schema.name); } + await this.applyDeclaredVectorIndexes(schema.name, isInstanceSync); return this.models[schema.name]; } diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 30df75904..5c3a95048 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -10,6 +10,10 @@ import { PostgresIndexType, RawSQLQuery, UntypedArray, + VectorCapabilities, + VectorIndexDefinition, + VectorSearchInput, + VectorSearchResult, } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { SequelizeAuto } from 'sequelize-auto'; @@ -29,6 +33,24 @@ import { import { sqlSchemaConverter } from './sql-adapter/SqlSchemaConverter.js'; import { pgSchemaConverter } from './postgres-adapter/PgSchemaConverter.js'; import { isEqual, isNil } from 'lodash-es'; +import { + assertVectorSearchAccess, + bindVectorIndexToField, + completeVectorSearch, + declaredVectorIndexes, + fromPostgresVectorIndex, + mergeVectorIndexes, + pgVectorOperator, + planPostgresVectorIndexCreate, + planPostgresVectorSearch, + postgresIndexMethodSql, + postgresVectorCapabilities, + parsePostgresVectorIndexDef, + resolveVectorFieldFromSchema, + sqlFallbackVectorCapabilities, + assertPostgresVectorIndexDropTarget, + type PostgresCatalogIndex, +} from '../utils/index.js'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; @@ -282,6 +304,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter await this.compareAndStoreMigratedSchema(schema); await this.saveSchemaToDatabase(schema); } + await this.applyDeclaredVectorIndexes(schema.name, isInstanceSync); return this.models[schema.name]; } @@ -422,6 +445,152 @@ export abstract class SequelizeAdapter extends DatabaseAdapter return 'Indexes deleted'; } + async getVectorCapabilities(schemaName?: string): Promise { + if (this.sequelize.getDialect() !== 'postgres') { + return sqlFallbackVectorCapabilities(this.sequelize.getDialect()); + } + try { + await this.sequelize.query("SELECT 'vector'::regtype"); + return postgresVectorCapabilities({ pgvectorAvailable: true }); + } catch (err) { + return postgresVectorCapabilities({ + pgvectorAvailable: false, + error: (err as Error).message, + schemaName, + }); + } + } + + async createVectorIndex( + schemaName: string, + index: VectorIndexDefinition, + ): Promise { + this.ensurePostgresVectorSupport(schemaName); + const schema = this.models[schemaName].originalSchema; + const field = (schema.compiledFields?.[index.field] ?? + schema.fields?.[index.field]) as unknown; + const tableName = this.getPhysicalTableName(schemaName); + const bound = bindVectorIndexToField({ + provider: 'postgres', + index, + field, + physicalTableName: tableName, + }); + const existing = await this.findPostgresCatalogIndex(bound.name!); + const method = postgresIndexMethodSql(bound.method); + const operator = pgVectorOperator(bound.similarity); + const withOptions = + method === 'ivfflat' + ? this.renderWithOptions({ lists: bound.options?.ivfflat?.lists }) + : this.renderWithOptions({ + m: bound.options?.hnsw?.m, + ef_construction: bound.options?.hnsw?.efConstruction, + }); + const plan = planPostgresVectorIndexCreate({ + indexName: bound.name!, + tableName, + field: bound.field, + method, + operator, + withOptions, + existing, + quoteIdentifier: identifier => this.quoteIdentifier(identifier), + }); + if (plan.action === 'reuse') return 'Vector index created!'; + await this.sequelize.query(plan.sql); + return 'Vector index created!'; + } + + async getVectorIndexes(schemaName: string): Promise { + this.ensurePostgresVectorSupport(schemaName); + const tableName = this.getPhysicalTableName(schemaName); + const rows = await this.listPostgresCatalogIndexes(tableName); + const schema = this.models[schemaName]?.originalSchema; + const schemaFields = (schema?.compiledFields ?? schema?.fields) as + Record | undefined; + const declared = declaredVectorIndexes(schema ?? {}); + return rows + .filter(row => /USING (hnsw|ivfflat)/i.test(row.indexdef)) + .map(row => { + const parsed = parsePostgresVectorIndexDef(row.indexdef); + const field = resolveVectorFieldFromSchema(schemaFields, parsed.field); + const matchingDeclared = declared.find( + item => item.name === row.indexname || item.field === parsed.field, + ); + return fromPostgresVectorIndex( + row.indexname, + row.indexdef, + field, + matchingDeclared, + ); + }); + } + + async deleteVectorIndex(schemaName: string, indexName: string): Promise { + this.ensurePostgresVectorSupport(schemaName); + const tableName = this.getPhysicalTableName(schemaName); + const existing = await this.findPostgresCatalogIndex(indexName); + assertPostgresVectorIndexDropTarget({ + indexName, + tableName, + existing, + }); + await this.sequelize.query(`DROP INDEX ${this.quoteIdentifier(indexName)}`); + return 'Vector index deleted'; + } + + async vectorSearch(request: VectorSearchInput): Promise { + this.ensurePostgresVectorSupport(request.schemaName); + const schema = this.models[request.schemaName]; + const schemaFields = (schema.originalSchema.compiledFields ?? + schema.originalSchema.fields) as Record; + const field = resolveVectorFieldFromSchema(schemaFields, request.field); + if (!field) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Requested field is not a vector'); + } + if (request.vector.length !== field.dimensions) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Vector dimensions mismatch: expected ${field.dimensions}`, + ); + } + assertVectorSearchAccess({ + authzEnabled: !!schema.authzEnabled, + userId: request.userId, + scope: request.scope, + adminOperator: request.adminOperator, + }); + const liveIndexes = await this.getVectorIndexes(request.schemaName); + const planned = planPostgresVectorSearch({ + request, + indexes: mergeVectorIndexes( + declaredVectorIndexes(schema.originalSchema), + liveIndexes, + ), + schemaFields, + tableName: this.getPhysicalTableName(request.schemaName), + similarity: field.similarity, + renderer: { + quoteIdentifier: identifier => this.quoteIdentifier(identifier), + escape: value => this.sequelize.escape(value as string | number), + }, + }); + return completeVectorSearch({ + emptyResult: planned.emptyResult, + limit: planned.limits.limit, + authzEnabled: !!schema.authzEnabled, + adminOperator: request.adminOperator, + provider: 'postgres', + metric: field.similarity, + fetchCandidates: async () => { + const rows = await this.sequelize.query(planned.sql); + return (rows[0] as Indexable[]) ?? []; + }, + lookupAuthorizedIds: ids => + schema.lookupAuthorizedCandidateIds('read', ids, request.userId, request.scope), + }); + } + async execRawQuery(schemaName: string, rawQuery: RawSQLQuery) { return await this.sequelize .query(rawQuery.query, rawQuery.options) @@ -464,6 +633,61 @@ export abstract class SequelizeAdapter extends DatabaseAdapter protected abstract hasLegacyCollections(): Promise; + private ensurePostgresVectorSupport(schemaName: string) { + if (this.sequelize.getDialect() !== 'postgres') { + throw new GrpcError( + status.UNIMPLEMENTED, + `${this.sequelize.getDialect()} does not support vector search`, + ); + } + if (!this.models[schemaName]) { + throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + } + } + + private async listPostgresCatalogIndexes( + tableName?: string, + ): Promise { + const tableFilter = tableName + ? ` AND tablename = ${this.sequelize.escape(tableName)}` + : ''; + const rows = await this.sequelize.query( + `SELECT indexname, tablename, indexdef FROM pg_indexes WHERE schemaname = current_schema()${tableFilter}`, + ); + return ((rows[0] as PostgresCatalogIndex[]) ?? []).map(row => ({ + indexname: row.indexname, + tablename: row.tablename, + indexdef: row.indexdef, + })); + } + + private async findPostgresCatalogIndex( + indexName: string, + ): Promise { + const rows = await this.sequelize.query( + `SELECT indexname, tablename, indexdef FROM pg_indexes WHERE schemaname = current_schema() AND indexname = ${this.sequelize.escape( + indexName, + )}`, + ); + return ((rows[0] as PostgresCatalogIndex[]) ?? [])[0]; + } + + private getPhysicalTableName(schemaName: string) { + return this.models[schemaName].originalSchema.collectionName || `cnd_${schemaName}`; + } + + private quoteIdentifier(identifier: string) { + return `"${identifier.replace(/"/g, '""')}"`; + } + + private renderWithOptions(options: Record) { + const entries = Object.entries(options).filter((entry): entry is [string, number] => + Number.isFinite(entry[1]), + ); + if (!entries.length) return ''; + return ` WITH (${entries.map(([key, value]) => `${key} = ${value}`).join(', ')})`; + } + private checkAndConvertIndexes( schemaName: string, indexes: ModelOptionsIndexes[], diff --git a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts index 8c023fd7a..6504e65ba 100644 --- a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts @@ -18,6 +18,8 @@ import { extractRelations, RelationType, } from '../utils/extractors/index.js'; +import { vectorFieldStorageMapping } from '../../utils/vectorMappings.js'; +import { isVectorTypeName } from '../../utils/vectorField.js'; /** * This function should take as an input a JSON schema and convert it to the sequelize equivalent @@ -94,6 +96,8 @@ function extractType(type: string, sqlType?: SQLDataType) { } case 'JSON': return DataTypes.JSONB; + case 'Vector': + return (DataTypes as any).VECTOR; case 'Relation': case 'ObjectId': return DataTypes.UUID; @@ -152,6 +156,12 @@ function extractObjectType(objectField: Indexable): res.type = extractArrayType(objectField.type).type; } else { res.type = extractType(objectField.type, objectField.sqlType); + if (isVectorTypeName(objectField.type)) { + const mapping = vectorFieldStorageMapping('postgres', { + dimensions: objectField.dimensions, + }); + res.type = res.type(mapping.dimensions); + } } if (objectField.hasOwnProperty('default')) { res.defaultValue = checkDefaultValue(objectField.type, objectField.default); diff --git a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/index.ts index a3aa471f7..9d09b79f0 100644 --- a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/index.ts @@ -1,12 +1,23 @@ import { SequelizeAdapter } from '../index.js'; +import pgvector from 'pgvector/sequelize'; +import { Sequelize } from 'sequelize'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; export class PostgresAdapter extends SequelizeAdapter { constructor(connectionUri: string) { + pgvector.registerTypes(Sequelize); super(connectionUri); } + protected async ensureConnected() { + await super.ensureConnected(); + await this.sequelize.query('CREATE EXTENSION IF NOT EXISTS vector').catch(() => { + // Capability probing reports missing privileges/extension support later. Keeping + // startup alive lets non-vector schemas continue to work on restricted Postgres. + }); + } + protected async hasLegacyCollections() { const res = await this.sequelize .query( diff --git a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts index fc869ca37..7f418d43a 100644 --- a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts @@ -92,6 +92,8 @@ function extractType(type: string, sqlType?: SQLDataType) { } case 'JSON': return DataTypes.JSON; + case 'Vector': + return DataTypes.JSON; case 'Relation': case 'ObjectId': return DataTypes.UUID; diff --git a/modules/database/src/adapters/sequelize-adapter/utils/sqlTypeMap.ts b/modules/database/src/adapters/sequelize-adapter/utils/sqlTypeMap.ts index 9def07866..80179feb7 100644 --- a/modules/database/src/adapters/sequelize-adapter/utils/sqlTypeMap.ts +++ b/modules/database/src/adapters/sequelize-adapter/utils/sqlTypeMap.ts @@ -12,4 +12,5 @@ export const sqlDataTypeMap = new Map([ [SQLDataType.TIME, 'Date'], [SQLDataType.DATETIME, 'Date'], [SQLDataType.TIMESTAMP, 'Date'], + [SQLDataType.VECTOR, 'Vector'], ]); diff --git a/modules/database/src/adapters/utils/__tests__/embeddingsJobContext.test.ts b/modules/database/src/adapters/utils/__tests__/embeddingsJobContext.test.ts new file mode 100644 index 000000000..2c5e1b576 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/embeddingsJobContext.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it } from '@jest/globals'; +import { GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertEmbeddingsJobCaller, + assertEmbeddingsJobRead, + assertEmbeddingsJobWrite, +} from '../embeddingsJobContext.js'; +import { canModify } from '../../../permissions/index.js'; + +const articleSchema = { + name: 'Article', + compiledFields: { + title: { type: TYPE.String }, + body: { type: TYPE.String }, + views: { type: TYPE.Number }, + embedding: { type: TYPE.Vector, dimensions: 2 }, + embeddingSourceHash: { type: TYPE.String, select: false }, + }, + extensions: [ + { + ownerModule: 'embeddings', + fields: { + embedding: { type: TYPE.Vector, dimensions: 2 }, + embeddingSourceHash: { type: TYPE.String, select: false }, + }, + createdAt: new Date(), + updatedAt: new Date(), + }, + ], +}; + +describe('embeddings job context', () => { + it('rejects callers that are not the embeddings module', () => { + try { + assertEmbeddingsJobCaller('database'); + throw new Error('expected failure'); + } catch (err) { + expect((err as GrpcError).code).toBe(status.PERMISSION_DENIED); + } + expect(() => assertEmbeddingsJobCaller('embeddings')).not.toThrow(); + }); + + it('allows configured source and hash reads by id only', () => { + expect(() => + assertEmbeddingsJobRead({ + query: { _id: 'doc-1' }, + select: '+title +body +embeddingSourceHash', + allowedFields: ['title', 'body', 'embeddingSourceHash'], + schema: articleSchema, + }), + ).not.toThrow(); + expect(() => + assertEmbeddingsJobRead({ + query: JSON.stringify({ id: 'doc-1' }), + select: '+title', + allowedFields: ['title'], + schema: articleSchema, + }), + ).not.toThrow(); + }); + + it('rejects embeddingsJob reads that use operator objects instead of scalar ids', () => { + const denied = (query: unknown) => { + expect(() => + assertEmbeddingsJobRead({ + query, + select: '+title', + allowedFields: ['title'], + schema: articleSchema, + }), + ).toThrow(GrpcError); + }; + denied({ _id: { $gt: '' } }); + denied({ _id: { $ne: null } }); + denied({ _id: { $in: ['doc-1'] } }); + denied({ _id: 123 }); + denied({ _id: '' }); + denied({ id: { $regex: '.*' } }); + }); + + it('rejects collection scans, extra selected fields, and non-string sources', () => { + expect(() => + assertEmbeddingsJobRead({ + query: { title: 'x' }, + select: '+title', + allowedFields: ['title'], + schema: articleSchema, + }), + ).toThrow(GrpcError); + expect(() => + assertEmbeddingsJobRead({ + query: { _id: 'doc-1' }, + select: '+title +password', + allowedFields: ['title', 'embeddingSourceHash'], + schema: articleSchema, + }), + ).toThrow(GrpcError); + expect(() => + assertEmbeddingsJobRead({ + query: { _id: 'doc-1' }, + select: '+views', + allowedFields: ['views'], + schema: articleSchema, + }), + ).toThrow(GrpcError); + }); + + it('allows embeddings-owned vector and hash writes and rejects other fields', () => { + expect(() => + assertEmbeddingsJobWrite({ + document: { embedding: [0.1, 0.2], embeddingSourceHash: 'abc' }, + schema: articleSchema, + }), + ).not.toThrow(); + expect(() => + assertEmbeddingsJobWrite({ + document: { $set: { embedding: [0.1, 0.2] } }, + schema: articleSchema, + }), + ).not.toThrow(); + try { + assertEmbeddingsJobWrite({ + document: { title: 'nope' }, + schema: articleSchema, + }); + throw new Error('expected failure'); + } catch (err) { + expect((err as GrpcError).code).toBe(status.PERMISSION_DENIED); + } + expect(() => + assertEmbeddingsJobWrite({ + document: { $unset: { title: 1 } }, + schema: articleSchema, + }), + ).toThrow(GrpcError); + }); + + it('does not change global canModify behavior', async () => { + const schema = { + originalSchema: { + name: 'Article', + ownerModule: 'database', + modelOptions: { conduit: { permissions: { canModify: 'Nothing' } } }, + extensions: articleSchema.extensions, + }, + }; + await expect(canModify('embeddings', schema as never, { title: 'x' })).resolves.toBe( + false, + ); + await expect(canModify('database', schema as never, { title: 'x' })).resolves.toBe( + true, + ); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/grpcStatus.test.ts b/modules/database/src/adapters/utils/__tests__/grpcStatus.test.ts new file mode 100644 index 000000000..84795bd4a --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/grpcStatus.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from '@jest/globals'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { grpcStatusFromError } from '../grpcStatus.js'; + +describe('grpcStatusFromError', () => { + it('preserves typed GrpcError codes instead of collapsing them to INTERNAL', () => { + expect( + grpcStatusFromError(new GrpcError(status.INVALID_ARGUMENT, 'bad filter')), + ).toEqual({ + code: status.INVALID_ARGUMENT, + message: 'bad filter', + }); + expect( + grpcStatusFromError(new GrpcError(status.PERMISSION_DENIED, 'no subject')), + ).toEqual({ + code: status.PERMISSION_DENIED, + message: 'no subject', + }); + }); + + it('maps unknown errors to INTERNAL', () => { + expect(grpcStatusFromError(new Error('boom'))).toEqual({ + code: status.INTERNAL, + message: 'boom', + }); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts b/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts new file mode 100644 index 000000000..5bdf522d7 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/mutationEvents.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from '@jest/globals'; +import { status } from '@grpc/grpc-js'; +import { + buildMutationEventChunks, + collectBoundedMutationIds, + collectDocumentIds, + MAX_MUTATION_EVENT_COLLECT_IDS, + mutationEventChannel, + mutationIdCollectionExhaustedError, + shouldPublishMutationEvent, +} from '../mutationEvents.js'; + +describe('mutation event helpers', () => { + it('publishes events unless suppressEvent is explicitly true', () => { + expect(shouldPublishMutationEvent()).toBe(true); + expect(shouldPublishMutationEvent(false)).toBe(true); + expect(shouldPublishMutationEvent(true)).toBe(false); + }); + + it('maps updateOne onto the update channel instead of updateMany', () => { + expect(mutationEventChannel('database', 'update', 'Article')).toBe( + 'database:update:Article', + ); + expect(mutationEventChannel('database', 'updateMany', 'Article')).toBe( + 'database:updateMany:Article', + ); + }); + + it('collects document ids from create and bulk payloads', () => { + expect(collectDocumentIds({ _id: 'a' })).toEqual(['a']); + expect(collectDocumentIds([{ _id: 'a' }, { _id: 'b' }, { _id: 'a' }])).toEqual([ + 'a', + 'b', + ]); + }); + + it('does not treat a Mongo updateMany result as document ids', () => { + expect( + collectDocumentIds({ + acknowledged: true, + matchedCount: 3, + modifiedCount: 3, + upsertedCount: 0, + upsertedId: null, + }), + ).toEqual([]); + }); + + it('publishes affected ids in bounded chunks without altering caller-supplied ids', () => { + const ids = ['1', '2', '3', '4', '5']; + const chunks = buildMutationEventChunks(ids, 2); + expect(chunks).toEqual([ + JSON.stringify([{ _id: '1' }, { _id: '2' }]), + JSON.stringify([{ _id: '3' }, { _id: '4' }]), + JSON.stringify([{ _id: '5' }]), + ]); + expect(ids).toEqual(['1', '2', '3', '4', '5']); + }); + + it('pages and caps updateMany mutation id collection instead of materializing unlimited ids', async () => { + const docs = Array.from({ length: 7 }, (_, index) => ({ _id: String(index) })); + const seen: Array<{ skip: number; limit: number }> = []; + const ids = await collectBoundedMutationIds({ + findPage: async (skip, limit) => { + seen.push({ skip, limit }); + return docs.slice(skip, skip + limit); + }, + cap: 10, + pageSize: 3, + }); + expect(ids).toEqual(['0', '1', '2', '3', '4', '5', '6']); + expect(seen).toEqual([ + { skip: 0, limit: 3 }, + { skip: 3, limit: 3 }, + { skip: 6, limit: 3 }, + ]); + + await expect( + collectBoundedMutationIds({ + findPage: async (skip, limit) => + Array.from({ length: limit }, (_, index) => ({ + _id: String(skip + index), + })), + cap: 4, + pageSize: 3, + }), + ).rejects.toMatchObject({ + code: status.RESOURCE_EXHAUSTED, + message: mutationIdCollectionExhaustedError(4).message, + }); + expect(mutationIdCollectionExhaustedError().code).toBe(status.RESOURCE_EXHAUSTED); + expect(MAX_MUTATION_EVENT_COLLECT_IDS).toBe(10_000); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorCapabilities.test.ts b/modules/database/src/adapters/utils/__tests__/vectorCapabilities.test.ts new file mode 100644 index 000000000..13ac55579 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorCapabilities.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from '@jest/globals'; +import { + mongoVectorCapabilities, + postgresVectorCapabilities, + sqlFallbackVectorCapabilities, + unsupportedVectorCapabilities, +} from '../vectorCapabilities.js'; + +describe('vector capability contracts', () => { + it('reports MongoDB storage even when search indexes cannot be probed', () => { + expect(mongoVectorCapabilities({ hasSchema: false })).toMatchObject({ + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + }); + expect( + mongoVectorCapabilities({ + hasSchema: true, + searchIndexCommandsAvailable: false, + }), + ).toMatchObject({ + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + }); + expect(mongoVectorCapabilities({ hasSchema: true })).toEqual({ + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'mongodb', + }); + }); + + it('keeps Postgres as a vector provider while distinguishing missing pgvector', () => { + expect(postgresVectorCapabilities({ pgvectorAvailable: true })).toEqual({ + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'postgres', + }); + expect( + postgresVectorCapabilities({ + pgvectorAvailable: false, + error: 'type "vector" does not exist', + }), + ).toMatchObject({ + supported: true, + storage: false, + indexing: false, + search: false, + provider: 'postgres', + }); + }); + + it('describes JSON storage fallback for non-Postgres SQL without claiming search', () => { + expect(sqlFallbackVectorCapabilities('mysql')).toEqual({ + supported: false, + storage: true, + indexing: false, + search: false, + provider: 'unsupported', + reason: + 'mysql does not support Conduit vector search; Vector fields can be stored as JSON', + }); + expect(sqlFallbackVectorCapabilities('sqlite')).toMatchObject({ + supported: false, + storage: true, + search: false, + provider: 'unsupported', + }); + }); + + it('keeps unknown adapters unsupported with no storage fallback', () => { + expect(unsupportedVectorCapabilities('custom')).toEqual({ + supported: false, + storage: false, + indexing: false, + search: false, + provider: 'unsupported', + reason: 'custom does not support Conduit vector search', + }); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorField.test.ts b/modules/database/src/adapters/utils/__tests__/vectorField.test.ts new file mode 100644 index 000000000..815456293 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorField.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it } from '@jest/globals'; +import { + ConduitError, + GrpcError, + TYPE, + VectorIndexMethod, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertObjectFormVectorField, + assertSafeVectorFieldChange, + assertSupportedVectorIndexMethod, + assertVectorFieldIfPresent, + assertVectorIndexContract, + assertVectorIndexMatchesField, + isVectorShorthand, + parseVectorSimilarity, + SUPPORTED_VECTOR_INDEX_METHODS, +} from '../vectorField.js'; +import { fieldsValidator, validateFieldChanges } from '../index.js'; +import { ConduitDatabaseSchema } from '../../../interfaces/index.js'; + +describe('vector field contracts', () => { + const validField = { + type: TYPE.Vector, + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + }; + + it('rejects shorthand Vector definitions', () => { + expect(isVectorShorthand(TYPE.Vector)).toBe(true); + expect(isVectorShorthand('Vector')).toBe(true); + expect(isVectorShorthand(['Vector'])).toBe(true); + expect(() => assertVectorFieldIfPresent('Docs', 'embedding', 'Vector')).toThrow( + ConduitError, + ); + expect(() => fieldsValidator('Docs', { embedding: 'Vector' }, 'mongodb')).toThrow( + /object form/, + ); + }); + + it('requires a positive integer dimensions value', () => { + expect(() => + assertObjectFormVectorField('Docs', 'embedding', { + type: TYPE.Vector, + dimensions: 0, + }), + ).toThrow(/positive integer/); + expect(() => + assertObjectFormVectorField('Docs', 'embedding', { + type: TYPE.Vector, + dimensions: 1.5, + }), + ).toThrow(/positive integer/); + expect(() => + assertObjectFormVectorField('Docs', 'embedding', { + type: TYPE.Vector, + dimensions: -8, + }), + ).toThrow(/positive integer/); + }); + + it('rejects unsupported similarity values and accepts the enum', () => { + expect(() => + assertObjectFormVectorField('Docs', 'embedding', { + type: TYPE.Vector, + dimensions: 8, + similarity: 'manhattan', + }), + ).toThrow(/unsupported similarity/i); + expect(assertObjectFormVectorField('Docs', 'embedding', validField)).toEqual( + validField, + ); + expect(parseVectorSimilarity(undefined)).toBe(VectorSimilarity.Cosine); + expect(parseVectorSimilarity(VectorSimilarity.DotProduct)).toBe( + VectorSimilarity.DotProduct, + ); + }); + + it('rejects unsafe in-place dimension changes and allows same-dimension updates', () => { + expect(() => + assertSafeVectorFieldChange('embedding', validField, { + ...validField, + dimensions: 768, + }), + ).toThrow(/dimensions/); + expect(() => + assertSafeVectorFieldChange('embedding', validField, { + ...validField, + similarity: VectorSimilarity.Euclidean, + }), + ).not.toThrow(); + + const oldSchema = { + compiledFields: { embedding: validField, title: TYPE.String }, + } as unknown as ConduitDatabaseSchema; + const newSchema = { + compiledFields: { + embedding: { ...validField, dimensions: 3072 }, + title: TYPE.String, + }, + } as unknown as ConduitDatabaseSchema; + expect(() => validateFieldChanges(oldSchema, newSchema)).toThrow(ConduitError); + }); + + it('validates provider-specific index methods', () => { + expect(SUPPORTED_VECTOR_INDEX_METHODS.mongodb).toEqual([ + VectorIndexMethod.HNSW, + VectorIndexMethod.Flat, + ]); + expect(SUPPORTED_VECTOR_INDEX_METHODS.postgres).toEqual([ + VectorIndexMethod.HNSW, + VectorIndexMethod.IVFFlat, + ]); + expect(() => + assertSupportedVectorIndexMethod('mongodb', VectorIndexMethod.IVFFlat), + ).toThrow(GrpcError); + expect(() => + assertSupportedVectorIndexMethod('postgres', VectorIndexMethod.Flat), + ).toThrow(GrpcError); + expect(() => + assertSupportedVectorIndexMethod('postgres', VectorIndexMethod.HNSW), + ).not.toThrow(); + }); + + it('rejects index contracts that do not match the schema field', () => { + expect(() => + assertVectorIndexMatchesField( + { type: TYPE.String }, + { + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + }, + ), + ).toThrow(/not a vector/); + expect(() => + assertVectorIndexMatchesField(validField, { + field: 'embedding', + dimensions: 768, + similarity: VectorSimilarity.Cosine, + }), + ).toThrow(/dimensions mismatch/); + expect(() => + assertVectorIndexContract('mongodb', { + field: 'embedding', + dimensions: 1536, + similarity: 'manhattan' as VectorSimilarity, + method: VectorIndexMethod.HNSW, + }), + ).toThrow(GrpcError); + try { + assertVectorIndexContract('mongodb', { + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.IVFFlat, + }); + throw new Error('expected method rejection'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.INVALID_ARGUMENT); + } + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts b/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts new file mode 100644 index 000000000..e48c16439 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts @@ -0,0 +1,348 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { + GrpcError, + TYPE, + VectorIndexMethod, + VectorIndexStatus, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { DatabaseAdapter } from '../../DatabaseAdapter.js'; +import { MongooseAdapter } from '../../mongoose-adapter/index.js'; +import { SequelizeAdapter } from '../../sequelize-adapter/index.js'; + +const schemaFields = { + _id: { type: TYPE.ObjectId }, + title: { type: TYPE.String }, + tenantId: { type: TYPE.String }, + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, +}; + +const declaredIndex = { + field: 'embedding', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.HNSW, + filterFields: ['tenantId'], +}; + +function articleModel() { + return { + originalSchema: { + name: 'Article', + fields: schemaFields, + compiledFields: schemaFields, + collectionName: 'cnd_Article', + modelOptions: { vectorIndexes: [declaredIndex] }, + }, + }; +} + +describe('mongoose vector index lifecycle', () => { + it('creates search indexes with _id filters and default names, reusing matches', async () => { + const createSearchIndex = jest.fn(async () => undefined); + const listSearchIndexes = jest.fn(() => ({ + toArray: async () => [], + })); + const createIndex = jest.fn(async () => 'title_1'); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + mongoose: { + model: () => ({ + collection: { createSearchIndex, listSearchIndexes, createIndex }, + }), + }, + }); + + await adapter.createVectorIndex('Article', declaredIndex); + expect(createSearchIndex).toHaveBeenCalledWith({ + name: 'embedding_vector', + type: 'vectorSearch', + definition: { + fields: [ + { + type: 'vector', + path: 'embedding', + numDimensions: 3, + similarity: VectorSimilarity.Cosine, + indexingMethod: VectorIndexMethod.HNSW, + }, + { type: 'filter', path: '_id' }, + { type: 'filter', path: 'tenantId' }, + ], + }, + }); + + listSearchIndexes.mockImplementation(() => ({ + toArray: async () => [ + { + name: 'embedding_vector', + type: 'vectorSearch', + status: 'READY', + queryable: true, + latestDefinition: createSearchIndex.mock.calls[0][0].definition, + }, + ], + })); + await adapter.createVectorIndex('Article', declaredIndex); + expect(createSearchIndex).toHaveBeenCalledTimes(1); + + await adapter.createIndexes('Article', [{ fields: ['title'] }], 'database'); + expect(createIndex).toHaveBeenCalled(); + expect(createSearchIndex).toHaveBeenCalledTimes(1); + }); + + it('creates explicit hnsw when method is omitted and reuses missing indexingMethod', async () => { + const createSearchIndex = jest.fn(async () => undefined); + const listSearchIndexes = jest.fn(() => ({ + toArray: async () => [], + })); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + mongoose: { + model: () => ({ + collection: { createSearchIndex, listSearchIndexes }, + }), + }, + }); + + const withoutMethod = { + field: 'embedding', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + filterFields: ['tenantId'], + }; + await adapter.createVectorIndex('Article', withoutMethod); + expect(createSearchIndex).toHaveBeenCalledWith({ + name: 'embedding_vector', + type: 'vectorSearch', + definition: { + fields: [ + { + type: 'vector', + path: 'embedding', + numDimensions: 3, + similarity: VectorSimilarity.Cosine, + indexingMethod: VectorIndexMethod.HNSW, + }, + { type: 'filter', path: '_id' }, + { type: 'filter', path: 'tenantId' }, + ], + }, + }); + + listSearchIndexes.mockImplementation(() => ({ + toArray: async () => [ + { + name: 'embedding_vector', + type: 'vectorSearch', + status: 'READY', + queryable: true, + latestDefinition: { + fields: [ + { + type: 'vector', + path: 'embedding', + numDimensions: 3, + similarity: VectorSimilarity.Cosine, + }, + { type: 'filter', path: '_id' }, + { type: 'filter', path: 'tenantId' }, + ], + }, + }, + ], + })); + await adapter.createVectorIndex('Article', { + ...withoutMethod, + method: '' as VectorIndexMethod, + }); + expect(createSearchIndex).toHaveBeenCalledTimes(1); + }); + + it('rejects vector search against a pending Mongo index', async () => { + const aggregate = jest.fn(); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + mongoose: { model: () => ({ collection: { aggregate } }) }, + getVectorIndexes: async () => [ + { + name: 'embedding_vector', + field: 'embedding', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + filterFields: ['_id', 'tenantId'], + status: VectorIndexStatus.Pending, + queryable: false, + }, + ], + }); + try { + await adapter.vectorSearch({ + schemaName: 'Article', + field: 'embedding', + vector: [0.1, 0.2, 0.3], + limit: 2, + }); + throw new Error('expected not-ready error'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.FAILED_PRECONDITION); + expect((err as GrpcError).message).toMatch(/not queryable/); + } + expect(aggregate).not.toHaveBeenCalled(); + }); + + it('drops Mongo search indexes only after verifying type vectorSearch', async () => { + const dropSearchIndex = jest.fn(async () => undefined); + const listSearchIndexes = jest.fn(() => ({ + toArray: async () => [{ name: 'article_text', type: 'search' }], + })); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + mongoose: { + model: () => ({ + collection: { dropSearchIndex, listSearchIndexes }, + }), + }, + }); + await expect(adapter.deleteVectorIndex('Article', 'article_text')).rejects.toThrow( + /is not a vectorSearch index/, + ); + expect(dropSearchIndex).not.toHaveBeenCalled(); + + listSearchIndexes.mockImplementation(() => ({ + toArray: async () => [{ name: 'embedding_vector', type: 'vectorSearch' }], + })); + await expect(adapter.deleteVectorIndex('Article', 'embedding_vector')).resolves.toBe( + 'Vector index deleted', + ); + expect(dropSearchIndex).toHaveBeenCalledWith('embedding_vector'); + }); +}); + +describe('postgres vector index lifecycle', () => { + it('creates vector indexes without IF NOT EXISTS and restores catalog options', async () => { + const query = jest.fn(async (sql: string) => { + if (sql.includes('pg_indexes') && sql.includes('indexname =')) return [[]]; + if (sql.includes('pg_indexes')) { + return [ + [ + { + indexname: 'cnd_Article_embedding_vector', + tablename: 'cnd_Article', + indexdef: + 'CREATE INDEX cnd_Article_embedding_vector ON cnd_Article USING hnsw ("embedding" vector_cosine_ops) WITH (m=16, ef_construction=64)', + }, + ], + ]; + } + return [[]]; + }); + const adapter = Object.create(SequelizeAdapter.prototype) as SequelizeAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + sequelize: { + getDialect: () => 'postgres', + escape: (value: unknown) => `'${value}'`, + query, + }, + }); + + await adapter.createVectorIndex('Article', { + ...declaredIndex, + options: { hnsw: { m: 16, efConstruction: 64 } }, + }); + const createSql = query.mock.calls.find(call => + String(call[0]).startsWith('CREATE INDEX'), + )?.[0] as string; + expect(createSql).toContain('CREATE INDEX "cnd_Article_embedding_vector"'); + expect(createSql).not.toMatch(/IF NOT EXISTS/i); + + const indexes = await adapter.getVectorIndexes('Article'); + expect(indexes[0]).toMatchObject({ + dimensions: 3, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.HNSW, + status: VectorIndexStatus.Ready, + queryable: true, + options: { hnsw: { m: 16, efConstruction: 64 } }, + }); + }); + + it('does not drop a vector index that belongs to another table', async () => { + const query = jest.fn(async () => [ + [ + { + indexname: 'embedding_vector', + tablename: 'cnd_Other', + indexdef: + 'CREATE INDEX embedding_vector ON cnd_Other USING hnsw ("embedding" vector_cosine_ops)', + }, + ], + ]); + const adapter = Object.create(SequelizeAdapter.prototype) as SequelizeAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + sequelize: { + getDialect: () => 'postgres', + escape: (value: unknown) => `'${value}'`, + query, + }, + }); + await expect( + adapter.deleteVectorIndex('Article', 'embedding_vector'), + ).rejects.toThrow(/was not found on table/); + expect(query.mock.calls.some(call => String(call[0]).startsWith('DROP INDEX'))).toBe( + false, + ); + }); +}); + +describe('declared vectorIndexes application', () => { + it('applies modelOptions.vectorIndexes through createVectorIndex, not regular indexes', async () => { + const createVectorIndex = jest.fn(async () => 'Vector index created!'); + const createIndexes = jest.fn(async () => 'Indexes created!'); + const adapter = Object.create(DatabaseAdapter.prototype) as DatabaseAdapter; + Object.assign(adapter, { + models: { Article: articleModel() }, + createVectorIndex, + createIndexes, + getVectorCapabilities: async () => ({ + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'mongodb', + }), + }); + + await (adapter as any).applyDeclaredVectorIndexes('Article', false); + expect(createVectorIndex).toHaveBeenCalledWith('Article', declaredIndex); + expect(createIndexes).not.toHaveBeenCalled(); + + await (adapter as any).applyDeclaredVectorIndexes('Article', true); + expect(createVectorIndex).toHaveBeenCalledTimes(1); + + Object.assign(adapter, { + getVectorCapabilities: async () => ({ + supported: false, + storage: true, + indexing: false, + search: false, + provider: 'unsupported', + }), + }); + await (adapter as any).applyDeclaredVectorIndexes('Article', false); + expect(createVectorIndex).toHaveBeenCalledTimes(1); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts b/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts new file mode 100644 index 000000000..cd885d6b3 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorIndexLifecycle.test.ts @@ -0,0 +1,388 @@ +import { describe, expect, it } from '@jest/globals'; +import { + GrpcError, + TYPE, + VectorIndexMethod, + VectorIndexStatus, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertMongoVectorSearchIndexDropTarget, + assertPostgresVectorIndexDropTarget, + assertVectorIndexQueryable, + bindVectorIndexToField, + defaultVectorIndexName, + hydratePostgresVectorIndex, + isVectorIndexQueryable, + mongoSearchIndexReadiness, + mongoVectorFilterFields, + planMongoVectorIndexCreate, + planPostgresVectorIndexCreate, + postgresVectorIndexDefinitionMatches, + renderPostgresCreateVectorIndexSql, + selectLiveVectorIndexForField, +} from '../vectorIndexLifecycle.js'; + +const vectorField = { + type: TYPE.Vector, + dimensions: 1536, + similarity: VectorSimilarity.Cosine, +}; + +const quote = (identifier: string) => `"${identifier}"`; + +describe('vector index lifecycle', () => { + it('uses consistent default names and always includes Mongo _id filter fields', () => { + expect(defaultVectorIndexName('embedding')).toBe('embedding_vector'); + expect(defaultVectorIndexName('embedding', 'cnd_Article')).toBe( + 'cnd_Article_embedding_vector', + ); + expect(mongoVectorFilterFields(['tenantId'])).toEqual(['_id', 'tenantId']); + expect(mongoVectorFilterFields(['_id', 'tenantId'])).toEqual(['_id', 'tenantId']); + expect(mongoVectorFilterFields()).toEqual(['_id']); + }); + + it('selects the highest generation live index for a field', () => { + expect( + selectLiveVectorIndexForField( + [ + { field: 'embedding', name: 'embedding_vector' }, + { field: 'embedding', name: 'embedding_vector_v2' }, + { field: 'title', name: 'title_vector_v4' }, + ], + 'embedding', + )?.name, + ).toBe('embedding_vector_v2'); + expect( + selectLiveVectorIndexForField( + [ + { field: 'embedding', name: 'cnd_Article_embedding_vector' }, + { field: 'embedding', name: 'embedding_vector' }, + ], + 'embedding', + )?.name, + ).toBe('embedding_vector'); + }); + + it('binds declared indexes to field dimensions/similarity and rejects mismatches', () => { + const bound = bindVectorIndexToField({ + provider: 'mongodb', + field: vectorField, + index: { + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + filterFields: ['tenantId'], + }, + }); + expect(bound.name).toBe('embedding_vector'); + expect(bound.filterFields).toEqual(['_id', 'tenantId']); + + expect(() => + bindVectorIndexToField({ + provider: 'mongodb', + field: vectorField, + index: { + field: 'embedding', + dimensions: 768, + similarity: VectorSimilarity.Cosine, + }, + }), + ).toThrow(/dimensions mismatch/); + expect(() => + bindVectorIndexToField({ + provider: 'postgres', + field: vectorField, + index: { + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.Flat, + }, + }), + ).toThrow(/Unsupported vector index method/); + }); + + it('maps Atlas search index status to ready/pending/failed queryability', () => { + expect(mongoSearchIndexReadiness({ status: 'READY' })).toEqual({ + status: VectorIndexStatus.Ready, + queryable: true, + }); + expect(mongoSearchIndexReadiness({ status: 'PENDING' })).toEqual({ + status: VectorIndexStatus.Pending, + queryable: false, + }); + expect(mongoSearchIndexReadiness({ status: 'FAILED' })).toEqual({ + status: VectorIndexStatus.Failed, + queryable: false, + }); + expect(mongoSearchIndexReadiness({ status: 'STALE', queryable: true })).toEqual({ + status: VectorIndexStatus.Ready, + queryable: true, + }); + }); + + it('fails clearly when a vector index is missing or not queryable', () => { + try { + assertVectorIndexQueryable(undefined, { field: 'embedding' }); + throw new Error('expected missing index error'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.FAILED_PRECONDITION); + expect((err as GrpcError).message).toMatch(/No vector index is available/); + } + try { + assertVectorIndexQueryable( + { + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + status: VectorIndexStatus.Pending, + queryable: false, + }, + { field: 'embedding' }, + ); + throw new Error('expected not-ready error'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.FAILED_PRECONDITION); + expect((err as GrpcError).message).toMatch(/not queryable \(status: pending\)/); + } + expect( + isVectorIndexQueryable({ + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + }), + ).toBe(false); + }); + + it('reuses matching Mongo indexes and rejects silent definition changes', () => { + const requested = bindVectorIndexToField({ + provider: 'mongodb', + field: vectorField, + index: { + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.HNSW, + filterFields: ['tenantId'], + }, + }); + expect( + planMongoVectorIndexCreate({ + requested, + existing: [requested], + }), + ).toEqual({ action: 'reuse' }); + expect(planMongoVectorIndexCreate({ requested, existing: [] }).action).toBe('create'); + expect(() => + planMongoVectorIndexCreate({ + requested, + existing: [{ ...requested, dimensions: 768 }], + }), + ).toThrow(/different definition/); + }); + + it('reuses Mongo indexes when method is empty, missing, or default hnsw', () => { + const requested = bindVectorIndexToField({ + provider: 'mongodb', + field: vectorField, + index: { + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: '' as VectorIndexMethod, + filterFields: ['tenantId'], + }, + }); + expect(requested.method).toBe(VectorIndexMethod.HNSW); + expect( + planMongoVectorIndexCreate({ + requested, + existing: [ + { + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + filterFields: ['_id', 'tenantId'], + }, + ], + }), + ).toEqual({ action: 'reuse' }); + expect( + planMongoVectorIndexCreate({ + requested: { ...requested, method: VectorIndexMethod.HNSW }, + existing: [ + { + ...requested, + method: '' as VectorIndexMethod, + }, + ], + }), + ).toEqual({ action: 'reuse' }); + }); + + it('creates Postgres vector indexes without IF NOT EXISTS and detects mismatches', () => { + const sql = renderPostgresCreateVectorIndexSql({ + indexName: 'cnd_Article_embedding_vector', + tableName: 'cnd_Article', + field: 'embedding', + method: 'hnsw', + operator: 'vector_cosine_ops', + withOptions: ' WITH (m = 16, ef_construction = 64)', + quoteIdentifier: quote, + }); + expect(sql).toContain('CREATE INDEX "cnd_Article_embedding_vector"'); + expect(sql).not.toMatch(/IF NOT EXISTS/i); + + expect( + planPostgresVectorIndexCreate({ + indexName: 'cnd_Article_embedding_vector', + tableName: 'cnd_Article', + field: 'embedding', + method: 'hnsw', + operator: 'vector_cosine_ops', + withOptions: '', + quoteIdentifier: quote, + }).action, + ).toBe('create'); + + expect( + planPostgresVectorIndexCreate({ + indexName: 'cnd_Article_embedding_vector', + tableName: 'cnd_Article', + field: 'embedding', + method: 'hnsw', + operator: 'vector_cosine_ops', + withOptions: '', + existing: { + indexname: 'cnd_Article_embedding_vector', + tablename: 'cnd_Article', + indexdef: + 'CREATE INDEX cnd_Article_embedding_vector ON cnd_Article USING hnsw ("embedding" vector_cosine_ops) WITH (m=16, ef_construction=64)', + }, + quoteIdentifier: quote, + }), + ).toEqual({ action: 'reuse' }); + + expect(() => + planPostgresVectorIndexCreate({ + indexName: 'cnd_Article_embedding_vector', + tableName: 'cnd_Article', + field: 'embedding', + method: 'hnsw', + operator: 'vector_cosine_ops', + withOptions: '', + existing: { + indexname: 'cnd_Article_embedding_vector', + tablename: 'cnd_Article', + indexdef: + 'CREATE INDEX cnd_Article_embedding_vector ON cnd_Article USING ivfflat ("embedding" vector_cosine_ops) WITH (lists=100)', + }, + quoteIdentifier: quote, + }), + ).toThrow(/different definition/); + }); + + it('scopes Postgres vector-index deletion to the requested table', () => { + expect(() => + assertPostgresVectorIndexDropTarget({ + indexName: 'embedding_vector', + tableName: 'cnd_Article', + existing: { + indexname: 'embedding_vector', + tablename: 'cnd_Other', + indexdef: + 'CREATE INDEX embedding_vector ON cnd_Other USING hnsw ("embedding" vector_cosine_ops)', + }, + }), + ).toThrow(/was not found on table/); + expect(() => + assertPostgresVectorIndexDropTarget({ + indexName: 'title_idx', + tableName: 'cnd_Article', + existing: { + indexname: 'title_idx', + tablename: 'cnd_Article', + indexdef: 'CREATE INDEX title_idx ON cnd_Article USING btree (title)', + }, + }), + ).toThrow(/is not a vector index/); + expect( + assertPostgresVectorIndexDropTarget({ + indexName: 'cnd_Article_embedding_vector', + tableName: 'cnd_Article', + existing: { + indexname: 'cnd_Article_embedding_vector', + tablename: 'cnd_Article', + indexdef: + 'CREATE INDEX cnd_Article_embedding_vector ON cnd_Article USING hnsw ("embedding" vector_cosine_ops)', + }, + }).indexname, + ).toBe('cnd_Article_embedding_vector'); + }); + + it('refuses to drop Mongo search indexes that are not vectorSearch', () => { + expect(() => + assertMongoVectorSearchIndexDropTarget({ + indexName: 'article_text', + }), + ).toThrow(/was not found/); + expect(() => + assertMongoVectorSearchIndexDropTarget({ + indexName: 'article_text', + existing: { name: 'article_text', type: 'search' }, + }), + ).toThrow(/is not a vectorSearch index/); + expect( + assertMongoVectorSearchIndexDropTarget({ + indexName: 'embedding_vector', + existing: { name: 'embedding_vector', type: 'vectorSearch' }, + }), + ).toEqual({ name: 'embedding_vector', type: 'vectorSearch' }); + }); + + it('restores Postgres dimensions and WITH options during catalog read-back', () => { + const hydrated = hydratePostgresVectorIndex({ + name: 'cnd_Article_embedding_vector', + indexdef: + 'CREATE INDEX cnd_Article_embedding_vector ON cnd_Article USING hnsw ("embedding" vector_cosine_ops) WITH (m=16, ef_construction=64)', + field: vectorField, + declared: { + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + options: { hnsw: { m: 16 } }, + }, + }); + expect(hydrated).toMatchObject({ + name: 'cnd_Article_embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.HNSW, + status: VectorIndexStatus.Ready, + queryable: true, + options: { hnsw: { m: 16, efConstruction: 64 } }, + }); + expect( + postgresVectorIndexDefinitionMatches( + 'CREATE INDEX cnd_Article_embedding_vector ON cnd_Article USING ivfflat ("embedding" vector_l2_ops) WITH (lists=100)', + { + tableName: 'cnd_Article', + field: 'embedding', + method: 'ivfflat', + operator: 'vector_l2_ops', + options: { ivfflat: { lists: 100 } }, + }, + ), + ).toBe(true); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts b/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts new file mode 100644 index 000000000..a153d1780 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorMappings.test.ts @@ -0,0 +1,255 @@ +import { describe, expect, it } from '@jest/globals'; +import { + ConduitSchema, + TYPE, + VectorIndexMethod, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { DataTypes } from 'sequelize'; +import 'pgvector/sequelize'; +import { schemaConverter } from '../../mongoose-adapter/SchemaConverter.js'; +import { pgSchemaConverter } from '../../sequelize-adapter/postgres-adapter/PgSchemaConverter.js'; +import { sqlSchemaConverter } from '../../sequelize-adapter/sql-adapter/SqlSchemaConverter.js'; +import { + applyMongoVectorField, + fromMongoVectorIndex, + fromPostgresVectorIndex, + mongoVectorStorageType, + pgVectorDistanceOperator, + pgVectorOperator, + postgresIndexMethodSql, + toMongoVectorIndexDefinition, + vectorFieldStorageMapping, +} from '../vectorMappings.js'; + +describe('vector field and index mappings', () => { + const vectorField = { + type: TYPE.Vector, + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + select: false, + }; + + it('maps provider-neutral storage contracts', () => { + expect(vectorFieldStorageMapping('mongodb', vectorField)).toEqual({ + backend: 'mongodb', + storage: 'numberArray', + dimensions: 1536, + searchSupported: true, + }); + expect(vectorFieldStorageMapping('postgres', vectorField)).toEqual({ + backend: 'postgres', + storage: 'pgvector', + dimensions: 1536, + searchSupported: true, + }); + expect(vectorFieldStorageMapping('sql', vectorField)).toEqual({ + backend: 'sql', + storage: 'json', + dimensions: 1536, + searchSupported: false, + }); + expect(mongoVectorStorageType()).toEqual([Number]); + expect(applyMongoVectorField(vectorField).type).toEqual([Number]); + }); + + it('converts Mongo schema Vector fields to a number array without a live database', () => { + const converted = schemaConverter( + new ConduitSchema('Article', { + title: TYPE.String, + embedding: vectorField, + }), + ); + expect(converted.fields.embedding).toMatchObject({ + type: [Number], + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + select: false, + }); + }); + + it('converts Postgres schema Vector fields to pgvector with dimensions', () => { + const [converted] = pgSchemaConverter( + new ConduitSchema('Article', { + title: { type: TYPE.String }, + embedding: vectorField, + }), + ); + const columnType = converted.fields.embedding.type as { + key?: string; + _dimensions?: number; + toSql?: () => string; + }; + expect(columnType.key).toBe('vector'); + expect(columnType._dimensions).toBe(1536); + expect(columnType.toSql?.()).toBe('VECTOR(1536)'); + }); + + it('converts non-Postgres SQL Vector fields to JSON storage', () => { + const [converted] = sqlSchemaConverter( + new ConduitSchema('Article', { + title: { type: TYPE.String }, + embedding: vectorField, + }), + ); + expect(converted.fields.embedding.type).toBe(DataTypes.JSON); + }); + + it('always includes _id as a Mongo vector index filter field', () => { + expect( + toMongoVectorIndexDefinition({ + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + filterFields: ['tenantId'], + }).fields, + ).toEqual([ + { + type: 'vector', + path: 'embedding', + numDimensions: 1536, + similarity: VectorSimilarity.Cosine, + indexingMethod: VectorIndexMethod.HNSW, + }, + { type: 'filter', path: '_id' }, + { type: 'filter', path: 'tenantId' }, + ]); + }); + + it('round-trips Mongo vector index definitions', () => { + const definition = toMongoVectorIndexDefinition({ + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.HNSW, + filterFields: ['_id', 'tenantId'], + options: { quantization: 'scalar', hnsw: { maxEdges: 16 } }, + }); + expect(definition).toEqual({ + fields: [ + { + type: 'vector', + path: 'embedding', + numDimensions: 1536, + similarity: VectorSimilarity.Cosine, + quantization: 'scalar', + indexingMethod: VectorIndexMethod.HNSW, + hnswOptions: { maxEdges: 16 }, + }, + { type: 'filter', path: '_id' }, + { type: 'filter', path: 'tenantId' }, + ], + }); + expect( + fromMongoVectorIndex({ + name: 'embedding_vector', + status: 'READY', + queryable: true, + latestDefinition: definition, + }), + ).toMatchObject({ + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: VectorIndexMethod.HNSW, + filterFields: ['_id', 'tenantId'], + status: 'ready', + queryable: true, + }); + }); + + it('maps Postgres similarity operators and parses index definitions', () => { + expect(pgVectorOperator(VectorSimilarity.Cosine)).toBe('vector_cosine_ops'); + expect(pgVectorOperator(VectorSimilarity.Euclidean)).toBe('vector_l2_ops'); + expect(pgVectorOperator(VectorSimilarity.DotProduct)).toBe('vector_ip_ops'); + expect(pgVectorDistanceOperator(VectorSimilarity.Cosine)).toBe('<=>'); + expect(pgVectorDistanceOperator(VectorSimilarity.Euclidean)).toBe('<->'); + expect(pgVectorDistanceOperator(VectorSimilarity.DotProduct)).toBe('<#>'); + expect(postgresIndexMethodSql(VectorIndexMethod.IVFFlat)).toBe('ivfflat'); + expect(postgresIndexMethodSql()).toBe('hnsw'); + + const parsed = fromPostgresVectorIndex( + 'cnd_article_embedding_vector', + 'CREATE INDEX cnd_article_embedding_vector ON cnd_article USING hnsw ("embedding" vector_cosine_ops)', + ); + expect(parsed).toMatchObject({ + name: 'cnd_article_embedding_vector', + field: 'embedding', + dimensions: 0, + similarity: VectorSimilarity.Cosine, + method: 'hnsw', + }); + expect( + fromPostgresVectorIndex( + 'cnd_article_embedding_vector', + 'CREATE INDEX cnd_article_embedding_vector ON cnd_article USING ivfflat (embedding vector_l2_ops) WITH (lists=100)', + { dimensions: 1536, similarity: VectorSimilarity.Euclidean }, + ), + ).toMatchObject({ + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Euclidean, + method: 'ivfflat', + queryable: true, + options: { ivfflat: { lists: 100 } }, + }); + }); + + it('treats empty proto method and missing Mongo indexingMethod as hnsw', () => { + expect( + toMongoVectorIndexDefinition({ + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + }).fields[0], + ).toMatchObject({ indexingMethod: VectorIndexMethod.HNSW }); + expect( + toMongoVectorIndexDefinition({ + name: 'embedding_vector', + field: 'embedding', + dimensions: 1536, + similarity: VectorSimilarity.Cosine, + method: '' as VectorIndexMethod, + }).fields[0], + ).toMatchObject({ indexingMethod: VectorIndexMethod.HNSW }); + + const withoutMethod = fromMongoVectorIndex({ + name: 'embedding_vector', + status: 'READY', + queryable: true, + latestDefinition: { + fields: [ + { + type: 'vector', + path: 'embedding', + numDimensions: 1536, + similarity: VectorSimilarity.Cosine, + }, + ], + }, + }); + expect(withoutMethod.method).toBe(VectorIndexMethod.HNSW); + + const emptyMethod = fromMongoVectorIndex({ + name: 'embedding_vector', + status: 'READY', + queryable: true, + latestDefinition: { + fields: [ + { + type: 'vector', + path: 'embedding', + numDimensions: 1536, + similarity: VectorSimilarity.Cosine, + indexingMethod: '', + }, + ], + }, + }); + expect(emptyMethod.method).toBe(VectorIndexMethod.HNSW); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorProjection.test.ts b/modules/database/src/adapters/utils/__tests__/vectorProjection.test.ts new file mode 100644 index 000000000..84426350c --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorProjection.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from '@jest/globals'; +import { TYPE } from '@conduitplatform/grpc-sdk'; +import { mongoVectorProjection, postgresVectorSelectList } from '../vectorProjection.js'; + +const schemaFields = { + _id: { type: TYPE.ObjectId }, + title: { type: TYPE.String }, + embedding: { type: TYPE.Vector, dimensions: 8, select: false }, + sourceHash: { type: TYPE.String, select: false }, +}; + +describe('vector search projections', () => { + it('keeps select:false fields out of Mongo include and exclude projections', () => { + expect(mongoVectorProjection(schemaFields, 'title embedding')).toEqual({ + _id: 1, + _score: 1, + title: 1, + }); + expect(mongoVectorProjection(schemaFields, '-title')).toEqual({ + embedding: 0, + sourceHash: 0, + title: 0, + }); + expect(mongoVectorProjection(schemaFields)).toEqual({ + embedding: 0, + sourceHash: 0, + }); + }); + + it('keeps select:false fields out of Postgres select lists', () => { + const quote = (identifier: string) => `"${identifier}"`; + expect(postgresVectorSelectList(schemaFields, 'title embedding', quote)).toBe( + '"title", "_id"', + ); + expect(postgresVectorSelectList(schemaFields, undefined, quote)).toBe( + '"_id", "title"', + ); + expect(postgresVectorSelectList(schemaFields, '-title', quote)).toBe('"_id"'); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorScore.test.ts b/modules/database/src/adapters/utils/__tests__/vectorScore.test.ts new file mode 100644 index 000000000..d9e2ecca7 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorScore.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from '@jest/globals'; +import { VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { normalizeVectorSearchScore, toVectorSearchResult } from '../vectorScore.js'; + +describe('vector search score contract', () => { + it('keeps Mongo cosine scores as higher-is-better without claiming a raw distance', () => { + const normalized = normalizeVectorSearchScore({ + provider: 'mongodb', + metric: VectorSimilarity.Cosine, + raw: 0.91, + }); + expect(normalized).toEqual({ + score: 0.91, + metric: VectorSimilarity.Cosine, + provider: 'mongodb', + comparable: true, + }); + expect(toVectorSearchResult({ _id: 'a' }, normalized)).toEqual({ + document: { _id: 'a' }, + score: 0.91, + metric: VectorSimilarity.Cosine, + provider: 'mongodb', + }); + }); + + it('converts Postgres cosine distance with 1 - distance', () => { + const identical = normalizeVectorSearchScore({ + provider: 'postgres', + metric: VectorSimilarity.Cosine, + raw: 0, + }); + expect(identical.score).toBe(1); + expect(identical.distance).toBe(0); + expect(identical.comparable).toBe(true); + + const orthogonal = normalizeVectorSearchScore({ + provider: 'postgres', + metric: 'cosine', + raw: 1, + }); + expect(orthogonal.score).toBe(0); + expect(orthogonal.distance).toBe(1); + }); + + it('does not treat Euclidean or inner-product scores as cross-provider equivalent', () => { + const mongoEuclidean = normalizeVectorSearchScore({ + provider: 'mongodb', + metric: VectorSimilarity.Euclidean, + raw: 0.4, + }); + const postgresEuclidean = normalizeVectorSearchScore({ + provider: 'postgres', + metric: VectorSimilarity.Euclidean, + raw: 0.6, + }); + const mongoDot = normalizeVectorSearchScore({ + provider: 'mongodb', + metric: VectorSimilarity.DotProduct, + raw: 12, + }); + const postgresDot = normalizeVectorSearchScore({ + provider: 'postgres', + metric: VectorSimilarity.DotProduct, + raw: -12, + }); + + expect(mongoEuclidean).toMatchObject({ + score: 0.4, + comparable: false, + provider: 'mongodb', + }); + expect(postgresEuclidean).toMatchObject({ + score: -0.6, + distance: 0.6, + comparable: false, + provider: 'postgres', + }); + expect(mongoDot.comparable).toBe(false); + expect(postgresDot).toMatchObject({ + score: 12, + distance: -12, + comparable: false, + }); + expect(mongoEuclidean.score).not.toBe(postgresEuclidean.score); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchAdapters.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchAdapters.test.ts new file mode 100644 index 000000000..22a989c11 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchAdapters.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { TYPE, VectorIndexStatus, VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { MongooseAdapter } from '../../mongoose-adapter/index.js'; +import { SequelizeAdapter } from '../../sequelize-adapter/index.js'; + +const schemaFields = { + _id: { type: TYPE.ObjectId }, + title: { type: TYPE.String }, + tenantId: { type: TYPE.String }, + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, +}; + +function articleModel(overrides?: { + authzEnabled?: boolean; + lookupAuthorizedCandidateIds?: (ids: string[]) => Promise; + getAuthorizedQuery?: (...args: unknown[]) => Promise; +}) { + const resolveIds = + overrides?.lookupAuthorizedCandidateIds ?? (async (ids: string[]) => ids); + return { + authzEnabled: overrides?.authzEnabled ?? true, + originalSchema: { + name: 'Article', + fields: schemaFields, + compiledFields: schemaFields, + collectionName: 'cnd_Article', + modelOptions: { + conduit: { authorization: { enabled: overrides?.authzEnabled ?? true } }, + vectorIndexes: [ + { + name: 'embedding_vector', + field: 'embedding', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + filterFields: ['_id', 'tenantId'], + status: VectorIndexStatus.Ready, + queryable: true, + }, + ], + }, + }, + lookupAuthorizedCandidateIds: jest.fn(async (_operation: string, ids: string[]) => + resolveIds(ids), + ), + getAuthorizedQuery: jest.fn( + overrides?.getAuthorizedQuery ?? (async () => ({ _id: { $in: ['all-docs'] } })), + ), + }; +} + +const searchRequest = { + schemaName: 'Article', + field: 'embedding', + vector: [0.1, 0.2, 0.3], + filter: { tenantId: 'org-1' }, + limit: 2, + numCandidates: 4, + select: 'title embedding', + userId: 'user-1', +}; + +describe('mongoose vector search adapter', () => { + it('runs ANN with indexed prefilters then authorizes only candidate ids', async () => { + const model = articleModel({ + lookupAuthorizedCandidateIds: async ids => ids.filter(id => id !== 'denied'), + }); + const aggregate = jest.fn(() => ({ + toArray: async () => [ + { _id: 'a', title: 'one', _score: 0.9 }, + { _id: 'denied', title: 'secret', _score: 0.8 }, + { _id: 'b', title: 'two', _score: 0.7 }, + ], + })); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + Object.assign(adapter, { + models: { Article: model }, + mongoose: { + model: () => ({ collection: { aggregate } }), + }, + getVectorIndexes: async () => model.originalSchema.modelOptions.vectorIndexes, + }); + + const results = await adapter.vectorSearch(searchRequest); + + expect(model.getAuthorizedQuery).not.toHaveBeenCalled(); + expect(model.lookupAuthorizedCandidateIds).toHaveBeenCalledWith( + 'read', + ['a', 'denied', 'b'], + 'user-1', + undefined, + ); + expect(aggregate.mock.calls[0][0][0].$vectorSearch).toMatchObject({ + filter: { tenantId: 'org-1' }, + limit: 4, + numCandidates: 4, + }); + expect(aggregate.mock.calls[0][0][0].$vectorSearch.filter).not.toHaveProperty('_id'); + expect(aggregate.mock.calls[0][0][2].$project).toEqual({ + _id: 1, + _score: 1, + title: 1, + }); + expect(results.map(result => result.document._id)).toEqual(['a', 'b']); + expect(results[0]).toMatchObject({ + score: 0.9, + provider: 'mongodb', + metric: VectorSimilarity.Cosine, + }); + expect(results[0].document).not.toHaveProperty('embedding'); + }); + + it('does not query Mongo when an empty $in filter matches no rows', async () => { + const model = articleModel(); + const aggregate = jest.fn(); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + Object.assign(adapter, { + models: { Article: model }, + mongoose: { model: () => ({ collection: { aggregate } }) }, + getVectorIndexes: async () => model.originalSchema.modelOptions.vectorIndexes, + }); + const results = await adapter.vectorSearch({ + ...searchRequest, + filter: { tenantId: { $in: [] } }, + }); + expect(results).toEqual([]); + expect(aggregate).not.toHaveBeenCalled(); + expect(model.lookupAuthorizedCandidateIds).not.toHaveBeenCalled(); + }); +}); + +describe('postgres vector search adapter', () => { + it('keeps user filters in SQL, hides select:false columns, and authorizes candidates only', async () => { + const model = articleModel({ + lookupAuthorizedCandidateIds: async ids => ids.filter(id => id !== 'denied'), + }); + const query = jest.fn(async () => [ + [ + { _id: 'a', title: 'one', _score: 0.2 }, + { _id: 'denied', title: 'secret', _score: 0.3 }, + { _id: 'b', title: 'two', _score: 0.5 }, + ], + ]); + const adapter = Object.create(SequelizeAdapter.prototype) as SequelizeAdapter; + Object.assign(adapter, { + models: { Article: model }, + sequelize: { + getDialect: () => 'postgres', + escape: (value: unknown) => + typeof value === 'string' ? `'${value}'` : String(value), + query, + }, + getVectorIndexes: async () => model.originalSchema.modelOptions.vectorIndexes, + }); + + const results = await adapter.vectorSearch(searchRequest); + const sql = query.mock.calls[0][0] as string; + + expect(model.getAuthorizedQuery).not.toHaveBeenCalled(); + expect(sql).toContain('WHERE "tenantId" = \'org-1\''); + expect(sql).toContain('LIMIT 4'); + expect(sql).toMatch(/^SELECT "title", "_id",/); + expect(sql).not.toContain('all-docs'); + expect(results).toEqual([ + { + document: { _id: 'a', title: 'one' }, + score: 0.8, + distance: 0.2, + metric: VectorSimilarity.Cosine, + provider: 'postgres', + }, + { + document: { _id: 'b', title: 'two' }, + score: 0.5, + distance: 0.5, + metric: VectorSimilarity.Cosine, + provider: 'postgres', + }, + ]); + }); + + it('does not query Postgres when empty $in matches no rows', async () => { + const model = articleModel(); + const query = jest.fn(); + const adapter = Object.create(SequelizeAdapter.prototype) as SequelizeAdapter; + Object.assign(adapter, { + models: { Article: model }, + sequelize: { + getDialect: () => 'postgres', + escape: (value: unknown) => `'${value}'`, + query, + }, + getVectorIndexes: async () => [], + }); + const results = await adapter.vectorSearch({ + ...searchRequest, + filter: { tenantId: { $in: [] } }, + }); + expect(results).toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchAuth.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchAuth.test.ts new file mode 100644 index 000000000..1e165469c --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchAuth.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + applyBoundedVectorAuthorization, + assertVectorSearchAccess, + authorizeBoundedVectorCandidates, + resolveAdminOperatorContext, +} from '../vectorSearchAuth.js'; + +describe('vector search authorization', () => { + it('allows unscoped search when authorization is disabled', () => { + expect(() => + assertVectorSearchAccess({ + authzEnabled: false, + }), + ).not.toThrow(); + }); + + it('fails closed on authorization-enabled schemas without subject, scope, or admin operator', () => { + try { + assertVectorSearchAccess({ authzEnabled: true }); + throw new Error('expected failure'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.PERMISSION_DENIED); + } + }); + + it('allows a subject or scope on authorization-enabled schemas', () => { + expect(() => + assertVectorSearchAccess({ authzEnabled: true, userId: 'user-1' }), + ).not.toThrow(); + expect(() => + assertVectorSearchAccess({ authzEnabled: true, scope: 'Team:org' }), + ).not.toThrow(); + }); + + it('allows an explicit admin operator context', () => { + expect(() => + assertVectorSearchAccess({ authzEnabled: true, adminOperator: true }), + ).not.toThrow(); + }); + + it('only honors adminOperator from verified platform operator modules', () => { + expect( + resolveAdminOperatorContext({ requested: true, callerModule: 'database' }), + ).toBe(true); + expect(resolveAdminOperatorContext({ requested: true, callerModule: 'core' })).toBe( + true, + ); + expect( + resolveAdminOperatorContext({ requested: true, callerModule: 'embeddings' }), + ).toBe(true); + expect(resolveAdminOperatorContext({ requested: false, callerModule: 'chat' })).toBe( + false, + ); + try { + resolveAdminOperatorContext({ requested: true, callerModule: 'chat' }); + throw new Error('expected failure'); + } catch (err) { + expect((err as GrpcError).code).toBe(status.PERMISSION_DENIED); + } + }); + + it('authorizes only the bounded candidate ids instead of materializing every authorized document', async () => { + const lookupAuthorizedIds = jest.fn(async (ids: string[]) => + ids.filter(id => id === 'keep'), + ); + const authorized = await authorizeBoundedVectorCandidates({ + authzEnabled: true, + candidateIds: ['keep', 'drop'], + lookupAuthorizedIds, + }); + expect(lookupAuthorizedIds).toHaveBeenCalledWith(['keep', 'drop']); + expect([...authorized]).toEqual(['keep']); + expect( + applyBoundedVectorAuthorization( + [{ _id: 'keep' }, { _id: 'drop' }, { _id: 'also-keep' }], + new Set(['keep', 'also-keep']), + 1, + ), + ).toEqual([{ _id: 'keep' }]); + }); + + it('skips authorization lookup for admin operators while still capping the result limit', async () => { + const lookupAuthorizedIds = jest.fn(async (ids: string[]) => ids); + const authorized = await authorizeBoundedVectorCandidates({ + authzEnabled: true, + adminOperator: true, + candidateIds: ['a', 'b'], + lookupAuthorizedIds, + }); + expect(lookupAuthorizedIds).not.toHaveBeenCalled(); + expect(authorized.size).toBe(2); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchFilter.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchFilter.test.ts new file mode 100644 index 000000000..9215285c6 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchFilter.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from '@jest/globals'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { validateVectorSearchFilter } from '../vectorSearchFilter.js'; + +function expectInvalid(run: () => unknown) { + try { + run(); + throw new Error('expected INVALID_ARGUMENT'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.INVALID_ARGUMENT); + } +} + +describe('vector search filter validation', () => { + const mongoFields = ['_id', 'tenantId', 'status']; + const postgresFields = ['_id', 'tenantId', 'status', 'score']; + + it('accepts Atlas-legal indexed Mongo prefilters', () => { + const validated = validateVectorSearchFilter( + { + tenantId: 'org-1', + status: { $in: ['active', 'draft'] }, + $or: [{ score: { $gte: 1 } }, { score: { $lte: 0 } }], + }, + { provider: 'mongodb', allowedFilterFields: [...mongoFields, 'score'] }, + ); + expect(validated.emptyResult).toBe(false); + expect(validated.filter.tenantId).toBe('org-1'); + }); + + it('rejects Mongo filters that are not declared indexed filter fields', () => { + expectInvalid(() => + validateVectorSearchFilter( + { authorId: 'user-1' }, + { provider: 'mongodb', allowedFilterFields: mongoFields }, + ), + ); + }); + + it('rejects Mongo filters when no indexed filter fields are declared', () => { + expectInvalid(() => + validateVectorSearchFilter({ tenantId: 'org-1' }, { provider: 'mongodb' }), + ); + }); + + it('rejects unsupported Mongo operators that Atlas vector search cannot prefilter', () => { + expectInvalid(() => + validateVectorSearchFilter( + { tenantId: { $regex: '^org' } }, + { provider: 'mongodb', allowedFilterFields: mongoFields }, + ), + ); + expectInvalid(() => + validateVectorSearchFilter( + { tenantId: { $exists: true } }, + { provider: 'mongodb', allowedFilterFields: mongoFields }, + ), + ); + expectInvalid(() => + validateVectorSearchFilter( + { tenantId: { $elemMatch: { a: 1 } } }, + { provider: 'mongodb', allowedFilterFields: mongoFields }, + ), + ); + }); + + it('never silently drops unsupported Postgres filters', () => { + expectInvalid(() => + validateVectorSearchFilter( + { tenantId: { $regex: '^org' } }, + { provider: 'postgres', allowedFilterFields: postgresFields }, + ), + ); + expectInvalid(() => + validateVectorSearchFilter( + { unknownColumn: 'x' }, + { provider: 'postgres', allowedFilterFields: postgresFields }, + ), + ); + }); + + it('treats empty $in as a no-row filter', () => { + expect( + validateVectorSearchFilter( + { status: { $in: [] } }, + { provider: 'mongodb', allowedFilterFields: mongoFields }, + ).emptyResult, + ).toBe(true); + expect( + validateVectorSearchFilter( + { $and: [{ tenantId: 'org-1' }, { status: { $in: [] } }] }, + { provider: 'postgres', allowedFilterFields: postgresFields }, + ).emptyResult, + ).toBe(true); + expect( + validateVectorSearchFilter( + { $or: [{ status: { $in: [] } }, { tenantId: { $in: [] } }] }, + { provider: 'postgres', allowedFilterFields: postgresFields }, + ).emptyResult, + ).toBe(true); + }); + + it('does not treat negated or $nor empty $in as a no-row filter', () => { + expect( + validateVectorSearchFilter( + { status: { $not: { $in: [] } } }, + { provider: 'mongodb', allowedFilterFields: mongoFields }, + ).emptyResult, + ).toBe(false); + expect( + validateVectorSearchFilter( + { $nor: [{ status: { $in: [] } }] }, + { provider: 'postgres', allowedFilterFields: postgresFields }, + ).emptyResult, + ).toBe(false); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchLimits.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchLimits.test.ts new file mode 100644 index 000000000..b31398d11 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchLimits.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from '@jest/globals'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + clampVectorSearchLimits, + VECTOR_SEARCH_MAX_CANDIDATES, + VECTOR_SEARCH_MAX_LIMIT, +} from '../vectorSearchLimits.js'; + +describe('vector search limits', () => { + it('defaults limit and requires candidates to cover the requested limit', () => { + expect(clampVectorSearchLimits({})).toEqual({ limit: 10, numCandidates: 100 }); + expect(clampVectorSearchLimits({ limit: 5 })).toEqual({ + limit: 5, + numCandidates: 100, + }); + expect(clampVectorSearchLimits({ limit: 25, numCandidates: 250 })).toEqual({ + limit: 25, + numCandidates: 250, + }); + }); + + it('clamps oversized limit and candidate values', () => { + expect(clampVectorSearchLimits({ limit: 50_000, numCandidates: 80_000 })).toEqual({ + limit: VECTOR_SEARCH_MAX_LIMIT, + numCandidates: VECTOR_SEARCH_MAX_CANDIDATES, + }); + }); + + it('rejects non-positive values and candidates below the requested limit', () => { + try { + clampVectorSearchLimits({ limit: 0 }); + throw new Error('expected failure'); + } catch (err) { + expect((err as GrpcError).code).toBe(status.INVALID_ARGUMENT); + } + try { + clampVectorSearchLimits({ limit: 20, numCandidates: 5 }); + throw new Error('expected failure'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).message).toMatch(/numCandidates must be at least/); + } + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts new file mode 100644 index 000000000..4fccc30bb --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchQuery.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { + GrpcError, + TYPE, + VectorIndexStatus, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + completeVectorSearch, + mergeVectorIndexes, + planMongoVectorSearch, + planPostgresVectorSearch, +} from '../vectorSearchQuery.js'; + +const schemaFields = { + _id: { type: TYPE.ObjectId }, + title: { type: TYPE.String }, + tenantId: { type: TYPE.String }, + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, +}; + +const indexes = [ + { + name: 'embedding_vector', + field: 'embedding', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + filterFields: ['_id', 'tenantId'], + status: VectorIndexStatus.Ready, + queryable: true, + }, +]; + +const request = { + schemaName: 'Article', + field: 'embedding', + vector: [0.1, 0.2, 0.3], + filter: { tenantId: 'org-1' }, + limit: 2, + numCandidates: 5, + select: 'title embedding', +}; + +describe('vector search query planning', () => { + it('builds a Mongo Atlas pipeline with indexed prefilters and hidden-field projection', () => { + const planned = planMongoVectorSearch({ request, indexes, schemaFields }); + expect(planned.emptyResult).toBe(false); + expect(planned.limits).toEqual({ limit: 2, numCandidates: 5 }); + expect(planned.pipeline[0]).toEqual({ + $vectorSearch: { + index: 'embedding_vector', + path: 'embedding', + queryVector: request.vector, + numCandidates: 5, + limit: 5, + filter: { tenantId: 'org-1' }, + }, + }); + expect(planned.pipeline[2]).toEqual({ + $project: { _id: 1, _score: 1, title: 1 }, + }); + }); + + it('selects the highest generation live index when a named request is not provided', () => { + const planned = planMongoVectorSearch({ + request, + indexes: [ + indexes[0], + { + ...indexes[0], + name: 'embedding_vector_v2', + }, + ], + schemaFields, + }); + expect(planned.pipeline[0]).toEqual({ + $vectorSearch: { + index: 'embedding_vector_v2', + path: 'embedding', + queryVector: request.vector, + numCandidates: 5, + limit: 5, + filter: { tenantId: 'org-1' }, + }, + }); + expect(() => + planMongoVectorSearch({ + request, + indexes: [ + indexes[0], + { + ...indexes[0], + name: 'embedding_vector_v2', + status: VectorIndexStatus.Pending, + queryable: false, + }, + ], + schemaFields, + }), + ).toThrow(/not queryable/); + }); + + it('short-circuits empty Mongo $in without emitting a pipeline', () => { + const planned = planMongoVectorSearch({ + request: { ...request, filter: { tenantId: { $in: [] } } }, + indexes, + schemaFields, + }); + expect(planned.emptyResult).toBe(true); + expect(planned.pipeline).toEqual([]); + }); + + it('renders Postgres SQL that keeps filters, hides select:false columns, and fetches candidates', () => { + const planned = planPostgresVectorSearch({ + request, + indexes, + schemaFields, + tableName: 'cnd_Article', + similarity: VectorSimilarity.Cosine, + renderer: { + quoteIdentifier: identifier => `"${identifier}"`, + escape: value => (typeof value === 'string' ? `'${value}'` : String(value)), + }, + }); + expect(planned.emptyResult).toBe(false); + expect(planned.sql).toContain('WHERE "tenantId" = \'org-1\''); + expect(planned.sql).toContain('LIMIT 5'); + expect(planned.sql).toMatch(/^SELECT "title", "_id",/); + expect(planned.sql).toContain('<=>'); + }); + + it('fails clearly when the selected vector index is not queryable', () => { + expect(() => + planMongoVectorSearch({ + request, + indexes: [ + { + ...indexes[0], + status: VectorIndexStatus.Pending, + queryable: false, + }, + ], + schemaFields, + }), + ).toThrow(GrpcError); + try { + planMongoVectorSearch({ + request, + indexes: [ + { + ...indexes[0], + status: VectorIndexStatus.Failed, + queryable: false, + }, + ], + schemaFields, + }); + throw new Error('expected failed index error'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.FAILED_PRECONDITION); + expect((err as GrpcError).message).toMatch(/not queryable/); + } + }); + + it('does not treat declared-only modelOptions vector indexes as live or queryable', () => { + const declaredOnly = mergeVectorIndexes(indexes, []); + expect(declaredOnly).toEqual([]); + expect(() => + planMongoVectorSearch({ + request, + indexes: declaredOnly, + schemaFields, + }), + ).toThrow(GrpcError); + const livePending = mergeVectorIndexes(indexes, [ + { + ...indexes[0], + status: VectorIndexStatus.Pending, + queryable: false, + }, + ]); + expect(livePending[0]).toMatchObject({ + status: VectorIndexStatus.Pending, + queryable: false, + }); + expect(() => + planMongoVectorSearch({ + request, + indexes: livePending, + schemaFields, + }), + ).toThrow(/not queryable/); + }); +}); + +describe('bounded vector search completion', () => { + it('authorizes only ANN candidate ids and returns normalized higher-is-better scores', async () => { + const lookupAuthorizedIds = jest.fn(async (ids: string[]) => + ids.filter(id => id !== 'denied'), + ); + const results = await completeVectorSearch({ + emptyResult: false, + limit: 2, + authzEnabled: true, + provider: 'postgres', + metric: VectorSimilarity.Cosine, + fetchCandidates: async () => [ + { _id: 'a', title: 'one', _score: 0.1 }, + { _id: 'denied', title: 'secret', _score: 0.2 }, + { _id: 'b', title: 'two', _score: 0.4 }, + { _id: 'c', title: 'three', _score: 0.5 }, + ], + lookupAuthorizedIds, + }); + expect(lookupAuthorizedIds).toHaveBeenCalledWith(['a', 'denied', 'b', 'c']); + expect(results).toEqual([ + { + document: { _id: 'a', title: 'one' }, + score: 0.9, + distance: 0.1, + metric: VectorSimilarity.Cosine, + provider: 'postgres', + }, + { + document: { _id: 'b', title: 'two' }, + score: 0.6, + distance: 0.4, + metric: VectorSimilarity.Cosine, + provider: 'postgres', + }, + ]); + }); + + it('does not call authorization lookup for admin operator or empty $in results', async () => { + const lookupAuthorizedIds = jest.fn(async (ids: string[]) => ids); + await completeVectorSearch({ + emptyResult: true, + limit: 2, + authzEnabled: true, + provider: 'mongodb', + metric: VectorSimilarity.Cosine, + fetchCandidates: async () => [{ _id: 'a', _score: 1 }], + lookupAuthorizedIds, + }); + expect(lookupAuthorizedIds).not.toHaveBeenCalled(); + + const results = await completeVectorSearch({ + emptyResult: false, + limit: 1, + authzEnabled: true, + adminOperator: true, + provider: 'mongodb', + metric: VectorSimilarity.Cosine, + fetchCandidates: async () => [ + { _id: 'a', _score: 0.9 }, + { _id: 'b', _score: 0.8 }, + ], + lookupAuthorizedIds, + }); + expect(lookupAuthorizedIds).not.toHaveBeenCalled(); + expect(results).toHaveLength(1); + expect(results[0]).toMatchObject({ + score: 0.9, + provider: 'mongodb', + metric: VectorSimilarity.Cosine, + }); + expect(results[0].distance).toBeUndefined(); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/vectorSearchWhere.test.ts b/modules/database/src/adapters/utils/__tests__/vectorSearchWhere.test.ts new file mode 100644 index 000000000..7a2a7c14e --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/vectorSearchWhere.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from '@jest/globals'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { renderPostgresVectorWhere } from '../vectorSearchWhere.js'; + +const renderer = { + quoteIdentifier: (identifier: string) => `"${identifier.replace(/"/g, '""')}"`, + escape: (value: unknown) => + typeof value === 'string' ? `'${value.replace(/'/g, "''")}'` : String(value), +}; + +describe('postgres vector search where rendering', () => { + it('renders supported comparison and membership filters', () => { + expect( + renderPostgresVectorWhere( + { + tenantId: 'org-1', + status: { $in: ['active', 'draft'] }, + score: { $gte: 1, $lt: 10 }, + }, + renderer, + ), + ).toBe( + ` WHERE "tenantId" = 'org-1' AND "status" IN ('active', 'draft') AND ("score" >= 1 AND "score" < 10)`, + ); + }); + + it('returns no rows for empty $in instead of dropping the predicate', () => { + expect(renderPostgresVectorWhere({ status: { $in: [] } }, renderer)).toBe( + ' WHERE FALSE', + ); + }); + + it('fails instead of silently dropping unsupported operators', () => { + try { + renderPostgresVectorWhere({ tenantId: { $regex: '^org' } }, renderer); + throw new Error('expected failure'); + } catch (err) { + expect(err).toBeInstanceOf(GrpcError); + expect((err as GrpcError).code).toBe(status.INVALID_ARGUMENT); + } + }); +}); diff --git a/modules/database/src/adapters/utils/embeddingsJobContext.ts b/modules/database/src/adapters/utils/embeddingsJobContext.ts new file mode 100644 index 000000000..f13fee743 --- /dev/null +++ b/modules/database/src/adapters/utils/embeddingsJobContext.ts @@ -0,0 +1,205 @@ +import { GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import type { DeclaredSchemaExtension } from '../../interfaces/DeclaredSchemaExtension.js'; + +export const EMBEDDINGS_MODULE_NAME = 'embeddings'; + +export interface EmbeddingsJobSchema { + name: string; + fields?: Record; + compiledFields?: Record; + extensions?: DeclaredSchemaExtension[]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isStringLikeField(field: unknown): boolean { + if (field === TYPE.String || field === 'String') return true; + if (Array.isArray(field) && field.length === 1) return isStringLikeField(field[0]); + if (!isRecord(field)) return false; + if (field.type === TYPE.String || field.type === 'String') return true; + return Array.isArray(field.type) && isStringLikeField(field.type); +} + +function isVectorField(field: unknown): boolean { + if (field === TYPE.Vector || field === 'Vector') return true; + return isRecord(field) && (field.type === TYPE.Vector || field.type === 'Vector'); +} + +function schemaFields(schema: EmbeddingsJobSchema): Record { + return schema.compiledFields ?? schema.fields ?? {}; +} + +export function parseSelectFields(select?: string): string[] { + if (!select) return []; + return select + .split(/\s+/) + .map(part => part.trim()) + .filter(Boolean) + .map(part => part.replace(/^[+-]/, '')); +} + +export const EMBEDDINGS_JOB_DOCUMENT_ID = /^[A-Za-z0-9._-]{1,128}$/; + +export function isScalarDocumentId(value: unknown): value is string { + return typeof value === 'string' && EMBEDDINGS_JOB_DOCUMENT_ID.test(value); +} + +export function isIdOnlyQuery(query: unknown): boolean { + let parsed = query; + if (typeof query === 'string') { + try { + parsed = JSON.parse(query); + } catch { + return false; + } + } + if (!isRecord(parsed)) return false; + const keys = Object.keys(parsed); + if (keys.length !== 1) return false; + const key = keys[0]; + if (key !== '_id' && key !== 'id') return false; + return isScalarDocumentId(parsed[key]); +} + +export function assertEmbeddingsJobCaller(moduleName?: string): void { + if (moduleName === EMBEDDINGS_MODULE_NAME) return; + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embeddings job context is only available to the embeddings module', + ); +} + +export function assertEmbeddingsJobRead(args: { + query: unknown; + select?: string; + allowedFields?: string[]; + schema: EmbeddingsJobSchema; +}): void { + if (!isIdOnlyQuery(args.query)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embeddings jobs may only read documents by id', + ); + } + const allowed = new Set( + (args.allowedFields ?? []).filter(field => typeof field === 'string' && field.length), + ); + if (!allowed.size) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embeddings jobs require an explicit allowed field list', + ); + } + const fields = schemaFields(args.schema); + for (const field of allowed) { + if (field === '_id' || field === 'id') continue; + if (!(field in fields)) { + throw new GrpcError( + status.PERMISSION_DENIED, + `Embeddings jobs cannot read unknown field '${field}'`, + ); + } + const definition = fields[field]; + const hashField = field.endsWith('SourceHash'); + if (!hashField && !isStringLikeField(definition)) { + throw new GrpcError( + status.PERMISSION_DENIED, + `Embeddings jobs cannot read non-string field '${field}'`, + ); + } + } + const selected = parseSelectFields(args.select); + if (!selected.length) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embeddings jobs must select configured source and hash fields', + ); + } + for (const field of selected) { + if (field === '_id' || field === 'id') continue; + if (!allowed.has(field)) { + throw new GrpcError( + status.PERMISSION_DENIED, + `Embeddings jobs cannot select field '${field}'`, + ); + } + } +} + +export function embeddingsOwnedWriteFields(schema: EmbeddingsJobSchema): Set { + const owned = new Set(); + for (const extension of schema.extensions ?? []) { + if (extension.ownerModule !== EMBEDDINGS_MODULE_NAME) continue; + for (const [name, definition] of Object.entries(extension.fields ?? {})) { + if (isVectorField(definition) || name.endsWith('SourceHash')) { + owned.add(name); + } + } + } + return owned; +} + +export function updateDocumentFields(document: unknown): string[] { + let parsed = document; + if (typeof document === 'string') { + try { + parsed = JSON.parse(document); + } catch { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embeddings job write is not valid JSON', + ); + } + } + if (!isRecord(parsed)) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Embeddings job writes must be objects'); + } + if ('$set' in parsed) { + const keys = Object.keys(parsed); + if (keys.some(key => key !== '$set')) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embeddings jobs may only use $set when sending update operators', + ); + } + if (!isRecord(parsed.$set)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embeddings job $set must be an object', + ); + } + return Object.keys(parsed.$set); + } + if (Object.keys(parsed).some(key => key.startsWith('$'))) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embeddings jobs cannot use update operators other than $set', + ); + } + return Object.keys(parsed); +} + +export function assertEmbeddingsJobWrite(args: { + document: unknown; + schema: EmbeddingsJobSchema; +}): void { + const owned = embeddingsOwnedWriteFields(args.schema); + const fields = updateDocumentFields(args.document); + if (!fields.length) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embeddings job writes must include fields', + ); + } + for (const field of fields) { + if (!owned.has(field)) { + throw new GrpcError( + status.PERMISSION_DENIED, + `Embeddings jobs cannot write field '${field}'`, + ); + } + } +} diff --git a/modules/database/src/adapters/utils/grpcStatus.ts b/modules/database/src/adapters/utils/grpcStatus.ts new file mode 100644 index 000000000..6efdf522d --- /dev/null +++ b/modules/database/src/adapters/utils/grpcStatus.ts @@ -0,0 +1,21 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export function grpcStatusFromError(err: unknown): { code: status; message: string } { + if (err instanceof GrpcError) { + return { code: err.code, message: err.message }; + } + return { + code: status.INTERNAL, + message: err instanceof Error ? err.message : String(err), + }; +} + +export function callerModuleName(metadata?: { + get(key: string): Array; +}): string | undefined { + const value = metadata?.get('module-name')?.[0]; + if (typeof value === 'string' && value.length > 0) return value; + if (Buffer.isBuffer(value) && value.length > 0) return value.toString(); + return undefined; +} diff --git a/modules/database/src/adapters/utils/index.ts b/modules/database/src/adapters/utils/index.ts index ec8998dd4..a514cd7ad 100644 --- a/modules/database/src/adapters/utils/index.ts +++ b/modules/database/src/adapters/utils/index.ts @@ -2,3 +2,17 @@ export * from './validateFieldChanges.js'; export * from './validateFieldConstraints.js'; export * from './database-transform-utils.js'; export * from './extensions.js'; +export * from './vectorField.js'; +export * from './vectorCapabilities.js'; +export * from './vectorMappings.js'; +export * from './mutationEvents.js'; +export * from './grpcStatus.js'; +export * from './vectorSearchAuth.js'; +export * from './embeddingsJobContext.js'; +export * from './vectorScore.js'; +export * from './vectorSearchLimits.js'; +export * from './vectorSearchFilter.js'; +export * from './vectorSearchWhere.js'; +export * from './vectorSearchQuery.js'; +export * from './vectorProjection.js'; +export * from './vectorIndexLifecycle.js'; diff --git a/modules/database/src/adapters/utils/mutationEvents.ts b/modules/database/src/adapters/utils/mutationEvents.ts new file mode 100644 index 000000000..7caf4856b --- /dev/null +++ b/modules/database/src/adapters/utils/mutationEvents.ts @@ -0,0 +1,123 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export const MUTATION_EVENT_ID_CHUNK_SIZE = 500; +export const MUTATION_EVENT_ID_PAGE_SIZE = MUTATION_EVENT_ID_CHUNK_SIZE; +export const MAX_MUTATION_EVENT_COLLECT_IDS = 10_000; + +export type MutationOperation = + 'create' | 'createMany' | 'update' | 'updateMany' | 'delete'; + +export function shouldPublishMutationEvent(suppressEvent?: boolean): boolean { + return suppressEvent !== true; +} + +export function mutationEventChannel( + moduleName: string, + operation: MutationOperation, + schemaName: string, +): string { + return `${moduleName}:${operation}:${schemaName}`; +} + +export function chunkItems( + items: T[], + size: number = MUTATION_EVENT_ID_CHUNK_SIZE, +): T[][] { + const chunkSize = size > 0 ? size : MUTATION_EVENT_ID_CHUNK_SIZE; + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += chunkSize) { + chunks.push(items.slice(i, i + chunkSize)); + } + return chunks; +} + +export function collectDocumentIds(docs: unknown): string[] { + if (isMongoBulkWriteResult(docs)) return []; + const values = Array.isArray(docs) ? docs : docs == null ? [] : [docs]; + const ids = new Set(); + for (const value of values) { + const id = extractId(value); + if (id) ids.add(id); + } + return [...ids]; +} + +export function mutationIdCollectionExhaustedError( + cap: number = MAX_MUTATION_EVENT_COLLECT_IDS, +): GrpcError { + return new GrpcError( + status.RESOURCE_EXHAUSTED, + `updateMany matched more than ${cap} documents; refuse unbounded mutation event collection`, + ); +} + +export async function collectBoundedMutationIds(args: { + findPage: (skip: number, limit: number) => Promise; + cap?: number; + pageSize?: number; +}): Promise { + const cap = args.cap ?? MAX_MUTATION_EVENT_COLLECT_IDS; + const pageSize = args.pageSize ?? MUTATION_EVENT_ID_PAGE_SIZE; + if (!Number.isInteger(cap) || cap < 1 || !Number.isInteger(pageSize) || pageSize < 1) { + throw mutationIdCollectionExhaustedError(cap); + } + const ids: string[] = []; + let skip = 0; + while (ids.length <= cap) { + const remainingWithOverflowProbe = cap - ids.length + 1; + const limit = Math.min(pageSize, remainingWithOverflowProbe); + const page = await args.findPage(skip, limit); + const pageLength = Array.isArray(page) ? page.length : page == null ? 0 : 1; + if (!pageLength) break; + skip += pageLength; + const pageIds = collectDocumentIds(page); + if (ids.length + pageIds.length > cap) { + throw mutationIdCollectionExhaustedError(cap); + } + ids.push(...pageIds); + if (pageLength < limit) break; + } + return ids; +} + +export function toIdEventPayload(ids: string[]): { _id: string }[] { + return ids.map(_id => ({ _id })); +} + +export function buildMutationEventChunks( + ids: string[], + chunkSize: number = MUTATION_EVENT_ID_CHUNK_SIZE, +): string[] { + return chunkItems( + ids.filter(id => id.length > 0), + chunkSize, + ).map(chunk => JSON.stringify(toIdEventPayload(chunk))); +} + +function extractId(value: unknown): string | undefined { + if (value == null) return undefined; + if (typeof value === 'string' || typeof value === 'number') { + const id = String(value); + return id.length ? id : undefined; + } + if (typeof value !== 'object') return undefined; + const record = value as Record; + if (record._id !== undefined) return extractId(record._id); + if (record.id !== undefined) return extractId(record.id); + return undefined; +} + +function isMongoBulkWriteResult(payload: unknown): boolean { + if (payload == null || typeof payload !== 'object' || Array.isArray(payload)) { + return false; + } + const record = payload as Record; + if (record._id !== undefined) return false; + return ( + typeof record.matchedCount === 'number' || + typeof record.modifiedCount === 'number' || + typeof record.nModified === 'number' || + typeof record.n === 'number' + ); +} diff --git a/modules/database/src/adapters/utils/validateFieldChanges.ts b/modules/database/src/adapters/utils/validateFieldChanges.ts index 187ad21be..92052979e 100644 --- a/modules/database/src/adapters/utils/validateFieldChanges.ts +++ b/modules/database/src/adapters/utils/validateFieldChanges.ts @@ -2,6 +2,7 @@ import { ConduitError, Indexable } from '@conduitplatform/grpc-sdk'; import { ConduitDatabaseSchema, Fields } from '../../interfaces/index.js'; import { isArray, isEqual, isNil, isString } from 'lodash-es'; import { DataTypes } from 'sequelize'; +import { assertSafeVectorFieldChange } from './vectorField.js'; /* * Validates schema compiled fields for type changes. @@ -29,6 +30,7 @@ function validateSchemaFields(oldSchemaFields: Indexable, newSchemaFields: Index if (isNil(newSchemaFields)) return; const newType = newSchemaFields[key]?.type ?? null; if (!newType) return; + assertSafeVectorFieldChange(key, oldSchemaFields[key], newSchemaFields[key]); if (oldType === DataTypes.JSONB && newType === 'JSON') return; if (isArray(oldType) && isArray(newType)) { if (typeof oldType[0] === 'object') { diff --git a/modules/database/src/adapters/utils/validateFieldConstraints.ts b/modules/database/src/adapters/utils/validateFieldConstraints.ts index b04f2593a..855ab5760 100644 --- a/modules/database/src/adapters/utils/validateFieldConstraints.ts +++ b/modules/database/src/adapters/utils/validateFieldConstraints.ts @@ -1,6 +1,7 @@ import { ConduitError, ConduitModel, ConduitModelField } from '@conduitplatform/grpc-sdk'; import { ConduitDatabaseSchema } from '../../interfaces/index.js'; import { isObject } from 'lodash-es'; +import { assertVectorFieldIfPresent } from './vectorField.js'; /* * Validates schema field constraints. @@ -10,6 +11,77 @@ export function validateFieldConstraints(schema: ConduitDatabaseSchema, db: stri fieldsValidator(schema.name, schema.compiledFields, db); } +function invalidField(message: string): never { + throw new ConduitError('INVALID_ARGUMENTS', 400, message); +} + +function usesSqlRelationBlock(item: unknown): boolean { + return Boolean( + (item && + typeof item === 'object' && + (item as ConduitModelField).hasOwnProperty('type') && + (item as ConduitModelField).type !== 'Relation') || + (item && typeof item === 'object'), + ); +} + +function validateArrayContents( + schemaName: string, + field: string, + items: unknown[], + db: string, + blockRelations: boolean, +) { + if (items.length !== 1) { + invalidField( + `Schema '${schemaName}' array field '${field}' has invalid format (array should contain a single type).`, + ); + } + const nestedBlock = usesSqlRelationBlock(items[0]) ? db === 'sql' : blockRelations; + fieldsValidator(schemaName, items[0] as ConduitModel, db, nestedBlock); +} + +function validateObjectField( + schemaName: string, + field: string, + target: ConduitModelField, + db: string, + blockRelations: boolean, +) { + if (target.unique && !target.required) { + invalidField( + `Schema '${schemaName}' violates unique field '${field}' constraint (field should be 'required').`, + ); + } + if (target.hasOwnProperty('type') && typeof target.type === 'object') { + if (Array.isArray(target.type)) { + validateArrayContents( + schemaName, + field, + target.type as unknown[], + db, + blockRelations, + ); + return; + } + fieldsValidator(schemaName, target.type as ConduitModel, db, blockRelations); + return; + } + if (!target.hasOwnProperty('type') && isObject(target)) { + if (Array.isArray(target)) { + validateArrayContents(schemaName, field, target as unknown[], db, blockRelations); + return; + } + fieldsValidator(schemaName, target as ConduitModel, db, blockRelations); + return; + } + if (target.hasOwnProperty('type') && target.type === 'Relation' && blockRelations) { + invalidField( + `Schema '${schemaName}' violates field '${field}' constraint (relations not allowed in embedded objects).`, + ); + } +} + export function fieldsValidator( schemaName: string, schemaFields: ConduitModel, @@ -18,86 +90,19 @@ export function fieldsValidator( ) { Object.keys(schemaFields).forEach(f => { if (f.includes('.')) { - throw new ConduitError( - 'INVALID_ARGUMENTS', - 400, + invalidField( `Schema '${schemaName}' violates field '${f}' constraint (field names cannot contain '.').`, ); } + assertVectorFieldIfPresent(schemaName, f, schemaFields[f]); if (typeof schemaFields[f] === 'object') { - const target: ConduitModelField = schemaFields[f] as ConduitModelField; - const isUnique = !!target.unique; - const isRequired = !!target.required; - if (isUnique && !isRequired) { - throw new ConduitError( - 'INVALID_ARGUMENTS', - 400, - `Schema '${schemaName}' violates unique field '${f}' constraint (field should be 'required').`, - ); - } - - if (target.hasOwnProperty('type') && typeof target.type === 'object') { - if (Array.isArray(target.type)) { - if ((target.type as unknown[]).length !== 1) { - throw new ConduitError( - 'INVALID_ARGUMENTS', - 400, - `Schema '${schemaName}' array field '${f}' has invalid format (array should contain a single type).`, - ); - } - if ( - (target.type[0] && - typeof target.type[0] === 'object' && - target.type[0].hasOwnProperty('type') && - target.type[0].type !== 'Relation') || - (target.type[0] && typeof target.type[0] === 'object') - ) { - fieldsValidator(schemaName, target.type[0] as ConduitModel, db, db === 'sql'); - } else { - fieldsValidator( - schemaName, - target.type[0] as unknown as ConduitModel, - db, - blockRelations, - ); - } - } else { - fieldsValidator(schemaName, target.type as ConduitModel, db, blockRelations); - } - } else if (!target.hasOwnProperty('type') && isObject(target)) { - if (Array.isArray(target)) { - if ((target as unknown[]).length !== 1) { - throw new ConduitError( - 'INVALID_ARGUMENTS', - 400, - `Schema '${schemaName}' array field '${f}' has invalid format (array should contain a single type).`, - ); - } - if ( - (target[0] && - typeof target[0] === 'object' && - target[0].hasOwnProperty('type') && - target[0].type !== 'Relation') || - (target[0] && typeof target[0] === 'object') - ) { - fieldsValidator(schemaName, target[0] as ConduitModel, db, db === 'sql'); - } else { - fieldsValidator(schemaName, target[0] as ConduitModel, db, blockRelations); - } - } else { - fieldsValidator(schemaName, target as ConduitModel, db, blockRelations); - } - } else if ( - target.hasOwnProperty('type') && - target.type === 'Relation' && - blockRelations - ) { - throw new ConduitError( - 'INVALID_ARGUMENTS', - 400, - `Schema '${schemaName}' violates field '${f}' constraint (relations not allowed in embedded objects).`, - ); - } + validateObjectField( + schemaName, + f, + schemaFields[f] as ConduitModelField, + db, + blockRelations, + ); } }); } diff --git a/modules/database/src/adapters/utils/vectorCapabilities.ts b/modules/database/src/adapters/utils/vectorCapabilities.ts new file mode 100644 index 000000000..ba612b339 --- /dev/null +++ b/modules/database/src/adapters/utils/vectorCapabilities.ts @@ -0,0 +1,96 @@ +import { VectorCapabilities } from '@conduitplatform/grpc-sdk'; + +export function mongoVectorCapabilities(input: { + hasSchema: boolean; + searchIndexCommandsAvailable?: boolean; + probeError?: string; +}): VectorCapabilities { + if (!input.hasSchema) { + return { + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + reason: 'No schema is available to probe MongoDB Vector Search support', + }; + } + if (input.searchIndexCommandsAvailable === false) { + return { + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + reason: 'MongoDB driver does not expose search index commands', + }; + } + if (input.probeError) { + return { + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + reason: input.probeError, + }; + } + return { + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'mongodb', + }; +} + +export function postgresVectorCapabilities(input: { + pgvectorAvailable: boolean; + error?: string; + schemaName?: string; +}): VectorCapabilities { + if (!input.pgvectorAvailable) { + const detail = input.error ?? 'pgvector is not available'; + return { + supported: true, + storage: false, + indexing: false, + search: false, + provider: 'postgres', + reason: input.schemaName + ? `Schema ${input.schemaName} cannot use pgvector: ${detail}` + : detail, + }; + } + return { + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'postgres', + }; +} + +export function sqlFallbackVectorCapabilities(dialect: string): VectorCapabilities { + return { + supported: false, + storage: true, + indexing: false, + search: false, + provider: 'unsupported', + reason: + `${dialect} does not support Conduit vector search; ` + + 'Vector fields can be stored as JSON', + }; +} + +export function unsupportedVectorCapabilities(databaseType: string): VectorCapabilities { + return { + supported: false, + storage: false, + indexing: false, + search: false, + provider: 'unsupported', + reason: `${databaseType} does not support Conduit vector search`, + }; +} diff --git a/modules/database/src/adapters/utils/vectorField.ts b/modules/database/src/adapters/utils/vectorField.ts new file mode 100644 index 000000000..cf15c7d42 --- /dev/null +++ b/modules/database/src/adapters/utils/vectorField.ts @@ -0,0 +1,209 @@ +import { + ConduitError, + GrpcError, + TYPE, + VectorIndexDefinition, + VectorIndexMethod, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export type VectorIndexProvider = 'mongodb' | 'postgres'; + +export const SUPPORTED_VECTOR_INDEX_METHODS: Record< + VectorIndexProvider, + readonly VectorIndexMethod[] +> = { + mongodb: [VectorIndexMethod.HNSW, VectorIndexMethod.Flat], + postgres: [VectorIndexMethod.HNSW, VectorIndexMethod.IVFFlat], +}; + +export interface ParsedVectorField { + type: TYPE.Vector; + dimensions: number; + similarity?: VectorSimilarity; +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function isVectorTypeName(value: unknown): value is TYPE.Vector { + return value === TYPE.Vector || value === 'Vector'; +} + +export function isVectorShorthand(field: unknown): boolean { + if (isVectorTypeName(field)) return true; + return Array.isArray(field) && field.length === 1 && isVectorTypeName(field[0]); +} + +export function isObjectFormVectorField(field: unknown): field is ParsedVectorField { + return isPlainObject(field) && isVectorTypeName(field.type); +} + +export function assertObjectFormVectorField( + schemaName: string, + fieldName: string, + field: unknown, +): ParsedVectorField { + if (isVectorShorthand(field)) { + throw new ConduitError( + 'INVALID_ARGUMENTS', + 400, + `Schema '${schemaName}' vector field '${fieldName}' must use object form ` + + `'{ type: "Vector", dimensions, similarity }'. Shorthand 'Vector' is not allowed.`, + ); + } + if (!isObjectFormVectorField(field)) { + throw new ConduitError( + 'INVALID_ARGUMENTS', + 400, + `Schema '${schemaName}' field '${fieldName}' is not a Vector field.`, + ); + } + if (!Number.isInteger(field.dimensions) || field.dimensions <= 0) { + throw new ConduitError( + 'INVALID_ARGUMENTS', + 400, + `Schema '${schemaName}' vector field '${fieldName}' requires a positive integer 'dimensions' value.`, + ); + } + if (field.similarity !== undefined && !isSupportedVectorSimilarity(field.similarity)) { + throw new ConduitError( + 'INVALID_ARGUMENTS', + 400, + `Schema '${schemaName}' vector field '${fieldName}' has unsupported similarity ` + + `'${String(field.similarity)}'. Supported values: ${Object.values(VectorSimilarity).join(', ')}.`, + ); + } + return { + type: TYPE.Vector, + dimensions: field.dimensions, + similarity: field.similarity, + }; +} + +export function assertVectorFieldIfPresent( + schemaName: string, + fieldName: string, + field: unknown, +): ParsedVectorField | undefined { + if (isVectorShorthand(field) || isObjectFormVectorField(field)) { + return assertObjectFormVectorField(schemaName, fieldName, field); + } + return undefined; +} + +export function isSupportedVectorSimilarity(value: unknown): value is VectorSimilarity { + return Object.values(VectorSimilarity).includes(value as VectorSimilarity); +} + +export function parseVectorSimilarity( + value: unknown, + fallback: VectorSimilarity = VectorSimilarity.Cosine, +): VectorSimilarity { + if (value === undefined || value === null || value === '') { + return fallback; + } + if (!isSupportedVectorSimilarity(value)) { + throw new ConduitError( + 'INVALID_ARGUMENTS', + 400, + `Unsupported similarity '${String(value)}'. Supported values: ${Object.values(VectorSimilarity).join(', ')}.`, + ); + } + return value; +} + +export function assertSafeVectorFieldChange( + fieldName: string, + oldField: unknown, + newField: unknown, +): void { + if (!isObjectFormVectorField(oldField) && !isVectorShorthand(oldField)) { + return; + } + if (newField == null) return; + const newType = isPlainObject(newField) ? newField.type : newField; + if (!isVectorTypeName(newType) && !isObjectFormVectorField(newField)) { + return; + } + if (isVectorShorthand(newField) || !isObjectFormVectorField(newField)) { + throw ConduitError.forbidden( + `Vector field '${fieldName}' must keep object form '{ type: "Vector", dimensions, similarity }'.`, + ); + } + if (!isObjectFormVectorField(oldField)) return; + if ( + Number.isInteger(oldField.dimensions) && + Number.isInteger(newField.dimensions) && + oldField.dimensions !== newField.dimensions + ) { + throw ConduitError.forbidden( + `Changing vector field '${fieldName}' dimensions from ${oldField.dimensions} to ${newField.dimensions} is not allowed.`, + ); + } +} + +export function assertSupportedVectorIndexMethod( + provider: VectorIndexProvider, + method?: string, +): void { + if (method === undefined || method === '') return; + const supported = SUPPORTED_VECTOR_INDEX_METHODS[provider]; + if (!supported.includes(method as VectorIndexMethod)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Unsupported vector index method '${method}' for ${provider}. ` + + `Supported methods: ${supported.join(', ')}.`, + ); + } +} + +export function assertVectorIndexMatchesField( + field: unknown, + index: VectorIndexDefinition, +): void { + if (!isObjectFormVectorField(field)) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Vector index field is not a vector'); + } + if (field.dimensions !== index.dimensions) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Vector index dimensions mismatch: field ${field.dimensions}, index ${index.dimensions}`, + ); + } + if ( + index.similarity !== undefined && + field.similarity !== undefined && + field.similarity !== index.similarity + ) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Vector index similarity mismatch: field ${field.similarity}, index ${index.similarity}`, + ); + } +} + +export function assertVectorIndexContract( + provider: VectorIndexProvider, + index: VectorIndexDefinition, +): void { + if (!index.field || typeof index.field !== 'string') { + throw new GrpcError(status.INVALID_ARGUMENT, 'Vector index field is required'); + } + if (!Number.isInteger(index.dimensions) || index.dimensions <= 0) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Vector index dimensions must be a positive integer', + ); + } + if (!isSupportedVectorSimilarity(index.similarity)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Unsupported vector index similarity '${String(index.similarity)}'. ` + + `Supported values: ${Object.values(VectorSimilarity).join(', ')}.`, + ); + } + assertSupportedVectorIndexMethod(provider, index.method); +} diff --git a/modules/database/src/adapters/utils/vectorIndexLifecycle.ts b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts new file mode 100644 index 000000000..67bfa8ed4 --- /dev/null +++ b/modules/database/src/adapters/utils/vectorIndexLifecycle.ts @@ -0,0 +1,464 @@ +import { + GrpcError, + VectorIndexDefinition, + VectorIndexStatus, + VectorSimilarity, + defaultVectorIndexMethod, + vectorIndexMethodsEquivalent, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertVectorIndexContract, + assertVectorIndexMatchesField, + isObjectFormVectorField, + VectorIndexProvider, +} from './vectorField.js'; + +export const MONGO_VECTOR_ID_FILTER_FIELD = '_id'; + +export interface PostgresCatalogIndex { + indexname: string; + tablename: string; + indexdef: string; +} + +export interface ParsedPostgresVectorIndex { + name?: string; + tableName?: string; + field: string; + method?: string; + similarity: VectorSimilarity; + options?: VectorIndexDefinition['options']; +} + +export type PostgresVectorIndexCreatePlan = + { action: 'create'; sql: string } | { action: 'reuse' }; + +export function defaultVectorIndexName( + field: string, + physicalTableName?: string, +): string { + return physicalTableName ? `${physicalTableName}_${field}_vector` : `${field}_vector`; +} + +export function vectorIndexGeneration(name?: string): number { + if (typeof name !== 'string' || name.length === 0) return 0; + const match = /_v(\d+)$/.exec(name); + if (match) return Number(match[1]); + return 1; +} + +export function selectLiveVectorIndexForField< + T extends { field?: string; name?: string }, +>(indexes: readonly T[], field: string): T | undefined { + const matches = indexes.filter(index => index.field === field); + if (!matches.length) return undefined; + const defaultName = defaultVectorIndexName(field); + return matches.reduce((best, current) => { + const bestGeneration = vectorIndexGeneration(best.name); + const currentGeneration = vectorIndexGeneration(current.name); + if (currentGeneration !== bestGeneration) { + return currentGeneration > bestGeneration ? current : best; + } + if (current.name === defaultName) return current; + if (best.name === defaultName) return best; + return best; + }); +} + +export function mongoVectorFilterFields(filterFields?: readonly string[]): string[] { + const fields: string[] = []; + for (const field of [MONGO_VECTOR_ID_FILTER_FIELD, ...(filterFields ?? [])]) { + if (!fields.includes(field)) { + fields.push(field); + } + } + return fields; +} + +export function bindVectorIndexToField(args: { + provider: VectorIndexProvider; + index: VectorIndexDefinition; + field: unknown; + physicalTableName?: string; +}): VectorIndexDefinition { + const field = isObjectFormVectorField(args.field) ? args.field : undefined; + const bound: VectorIndexDefinition = { + ...args.index, + name: + args.index.name ?? defaultVectorIndexName(args.index.field, args.physicalTableName), + dimensions: args.index.dimensions ?? field?.dimensions, + similarity: args.index.similarity ?? field?.similarity, + method: defaultVectorIndexMethod(args.index.method), + filterFields: + args.provider === 'mongodb' + ? mongoVectorFilterFields(args.index.filterFields) + : args.index.filterFields, + }; + assertVectorIndexContract(args.provider, bound); + assertVectorIndexMatchesField(args.field, bound); + return bound; +} + +export function mongoSearchIndexReadiness(index: { + status?: string; + queryable?: boolean; +}): { status: VectorIndexStatus; queryable: boolean } { + const raw = (index.status ?? '').toUpperCase(); + switch (raw) { + case 'READY': + return { status: VectorIndexStatus.Ready, queryable: true }; + case 'STALE': + return { + status: VectorIndexStatus.Ready, + queryable: index.queryable !== false, + }; + case 'FAILED': + case 'DOES_NOT_EXIST': + return { status: VectorIndexStatus.Failed, queryable: false }; + case 'PENDING': + case 'BUILDING': + case 'DELETING': + case '': + return { + status: VectorIndexStatus.Pending, + queryable: index.queryable === true, + }; + default: + if (index.queryable === true) { + return { status: VectorIndexStatus.Ready, queryable: true }; + } + return { status: VectorIndexStatus.Pending, queryable: false }; + } +} + +export function isVectorIndexQueryable(index?: VectorIndexDefinition): boolean { + if (!index) return false; + if (index.queryable === false) return false; + if (index.status === VectorIndexStatus.Failed) return false; + if (index.status === VectorIndexStatus.Pending && index.queryable !== true) { + return false; + } + if (index.queryable === true) return true; + return index.status === VectorIndexStatus.Ready; +} + +export function assertVectorIndexQueryable( + index: VectorIndexDefinition | undefined, + request: { field: string; indexName?: string }, +): asserts index is VectorIndexDefinition { + if (!index) { + const named = request.indexName ? ` (index '${request.indexName}')` : ''; + throw new GrpcError( + status.FAILED_PRECONDITION, + `No vector index is available for field '${request.field}'${named}. ` + + 'Create the index and wait until it is ready before searching.', + ); + } + if (isVectorIndexQueryable(index)) return; + const indexName = index.name ?? defaultVectorIndexName(index.field); + const statusLabel = index.status ?? 'unknown'; + throw new GrpcError( + status.FAILED_PRECONDITION, + `Vector index '${indexName}' is not queryable (status: ${statusLabel}). ` + + 'Wait until the index is ready before searching.', + ); +} + +export function vectorIndexesEquivalent( + left: VectorIndexDefinition, + right: VectorIndexDefinition, + provider: VectorIndexProvider, +): boolean { + if (left.field !== right.field) return false; + if (left.dimensions !== right.dimensions) return false; + if (left.similarity !== right.similarity) return false; + if (!vectorIndexMethodsEquivalent(left.method, right.method)) return false; + if (provider === 'mongodb') { + return sameStringSet( + mongoVectorFilterFields(left.filterFields), + mongoVectorFilterFields(right.filterFields), + ); + } + return true; +} + +export function planMongoVectorIndexCreate(args: { + requested: VectorIndexDefinition; + existing: VectorIndexDefinition[]; +}): { action: 'create' } | { action: 'reuse' } { + const existing = args.existing.find(index => index.name === args.requested.name); + if (!existing) return { action: 'create' }; + if (vectorIndexesEquivalent(args.requested, existing, 'mongodb')) { + return { action: 'reuse' }; + } + throw new GrpcError( + status.FAILED_PRECONDITION, + `Vector index '${args.requested.name}' already exists with a different definition. ` + + 'Drop it before recreating.', + ); +} + +function postgresSimilarityFromOperator(operator?: string): VectorSimilarity { + if (operator === 'l2' || operator === 'vector_l2_ops') { + return VectorSimilarity.Euclidean; + } + if (operator === 'ip' || operator === 'vector_ip_ops') { + return VectorSimilarity.DotProduct; + } + return VectorSimilarity.Cosine; +} + +export function parsePostgresVectorIndexDef(indexdef: string): ParsedPostgresVectorIndex { + const tableMatch = /ON\s+(?:(?:"[^"]+"|\w+)\.)?(?:"([^"]+)"|(\w+))/i.exec(indexdef); + const method = /USING\s+(\w+)/i.exec(indexdef)?.[1]?.toLowerCase(); + const fieldMatch = /\((?:"([^"]+)"|(\w+))\s+vector_/i.exec(indexdef); + const operator = /vector_(l2|cosine|ip)_ops/i.exec(indexdef)?.[1]; + const similarity = postgresSimilarityFromOperator(operator); + return { + tableName: tableMatch?.[1] ?? tableMatch?.[2], + field: fieldMatch?.[1] ?? fieldMatch?.[2] ?? '', + method, + similarity, + options: parsePostgresIndexOptions(indexdef, method), + }; +} + +export function postgresVectorIndexDefinitionMatches( + indexdef: string, + expected: { + tableName: string; + field: string; + method: string; + operator: string; + options?: VectorIndexDefinition['options']; + }, +): boolean { + const parsed = parsePostgresVectorIndexDef(indexdef); + if (parsed.tableName && parsed.tableName !== expected.tableName) return false; + if (parsed.field !== expected.field) return false; + if ((parsed.method ?? '').toLowerCase() !== expected.method.toLowerCase()) { + return false; + } + if (parsed.similarity !== postgresSimilarityFromOperator(expected.operator)) { + return false; + } + return postgresRequestedOptionsMatch(expected.options, parsed.options, expected.method); +} + +export function planPostgresVectorIndexCreate(args: { + indexName: string; + tableName: string; + field: string; + method: string; + operator: string; + withOptions: string; + existing?: PostgresCatalogIndex; + quoteIdentifier: (identifier: string) => string; +}): PostgresVectorIndexCreatePlan { + if (!args.existing) { + return { + action: 'create', + sql: renderPostgresCreateVectorIndexSql(args), + }; + } + if (args.existing.tablename !== args.tableName) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Vector index name '${args.indexName}' already exists on table '${args.existing.tablename}'.`, + ); + } + if (!isPostgresVectorIndexDef(args.existing.indexdef)) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Index '${args.indexName}' exists on '${args.tableName}' but is not a vector index.`, + ); + } + if ( + !postgresVectorIndexDefinitionMatches(args.existing.indexdef, { + tableName: args.tableName, + field: args.field, + method: args.method, + operator: args.operator, + options: withOptionsToDefinition(args.method, args.withOptions), + }) + ) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Vector index '${args.indexName}' already exists on '${args.tableName}' with a different definition. ` + + 'Drop it before recreating.', + ); + } + return { action: 'reuse' }; +} + +export function renderPostgresCreateVectorIndexSql(args: { + indexName: string; + tableName: string; + field: string; + method: string; + operator: string; + withOptions: string; + quoteIdentifier: (identifier: string) => string; +}): string { + return ( + `CREATE INDEX ${args.quoteIdentifier(args.indexName)} ON ${args.quoteIdentifier( + args.tableName, + )} USING ${args.method} (${args.quoteIdentifier(args.field)} ${args.operator})` + + args.withOptions + ); +} + +export function assertPostgresVectorIndexDropTarget(args: { + indexName: string; + tableName: string; + existing?: PostgresCatalogIndex; +}): PostgresCatalogIndex { + if (!args.existing || args.existing.tablename !== args.tableName) { + throw new GrpcError( + status.NOT_FOUND, + `Vector index '${args.indexName}' was not found on table '${args.tableName}'.`, + ); + } + if (!isPostgresVectorIndexDef(args.existing.indexdef)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Index '${args.indexName}' on '${args.tableName}' is not a vector index.`, + ); + } + return args.existing; +} + +export function isMongoVectorSearchIndex(index?: { + name?: string; + type?: string; +}): boolean { + return index?.type === 'vectorSearch'; +} + +export function assertMongoVectorSearchIndexDropTarget(args: { + indexName: string; + existing?: { name?: string; type?: string }; +}): { name: string; type: 'vectorSearch' } { + if (!args.existing) { + throw new GrpcError( + status.NOT_FOUND, + `Vector search index '${args.indexName}' was not found.`, + ); + } + if (!isMongoVectorSearchIndex(args.existing)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Search index '${args.indexName}' is not a vectorSearch index.`, + ); + } + return { + name: args.existing.name ?? args.indexName, + type: 'vectorSearch', + }; +} + +export function hydratePostgresVectorIndex(args: { + name: string; + indexdef: string; + field?: { dimensions?: number; similarity?: VectorSimilarity }; + declared?: VectorIndexDefinition; +}): VectorIndexDefinition { + const parsed = parsePostgresVectorIndexDef(args.indexdef); + const method = defaultVectorIndexMethod(parsed.method ?? args.declared?.method); + return { + name: args.name, + field: parsed.field || args.declared?.field || '', + dimensions: args.field?.dimensions ?? args.declared?.dimensions ?? 0, + similarity: args.field?.similarity ?? parsed.similarity, + method, + options: mergeVectorIndexOptions(args.declared?.options, parsed.options), + status: VectorIndexStatus.Ready, + queryable: true, + }; +} + +export function isPostgresVectorIndexDef(indexdef: string): boolean { + return /USING\s+(hnsw|ivfflat)\b/i.test(indexdef); +} + +function sameStringSet(left: string[], right: string[]): boolean { + if (left.length !== right.length) return false; + const rightSet = new Set(right); + return left.every(value => rightSet.has(value)); +} + +function parsePostgresIndexOptions( + indexdef: string, + method?: string, +): VectorIndexDefinition['options'] | undefined { + const match = /WITH\s*\(([^)]*)\)/i.exec(indexdef); + if (!match) return undefined; + const values: Record = {}; + for (const part of match[1].split(',')) { + const [rawKey, rawValue] = part.split('=').map(item => item.trim()); + if (!rawKey || rawValue === undefined) continue; + const value = Number(rawValue.replace(/^['"]|['"]$/g, '')); + if (!Number.isFinite(value)) continue; + values[rawKey.toLowerCase()] = value; + } + if (method === 'ivfflat') { + return values.lists !== undefined ? { ivfflat: { lists: values.lists } } : undefined; + } + const hnsw: NonNullable['hnsw'] = {}; + if (values.m !== undefined) hnsw.m = values.m; + if (values.ef_construction !== undefined) { + hnsw.efConstruction = values.ef_construction; + } + return Object.keys(hnsw).length ? { hnsw } : undefined; +} + +function withOptionsToDefinition( + method: string, + withOptions: string, +): VectorIndexDefinition['options'] | undefined { + if (!withOptions.trim()) return undefined; + return parsePostgresIndexOptions(` ${withOptions}`, method.toLowerCase()); +} + +function postgresRequestedOptionsMatch( + requested: VectorIndexDefinition['options'] | undefined, + actual: VectorIndexDefinition['options'] | undefined, + method: string, +): boolean { + if (method === 'ivfflat') { + if (requested?.ivfflat?.lists === undefined) return true; + return requested.ivfflat.lists === actual?.ivfflat?.lists; + } + if (requested?.hnsw?.m === undefined && requested?.hnsw?.efConstruction === undefined) { + return true; + } + if (requested?.hnsw?.m !== undefined && requested.hnsw.m !== actual?.hnsw?.m) { + return false; + } + if ( + requested?.hnsw?.efConstruction !== undefined && + requested.hnsw.efConstruction !== actual?.hnsw?.efConstruction + ) { + return false; + } + return true; +} + +function mergeVectorIndexOptions( + declared?: VectorIndexDefinition['options'], + catalog?: VectorIndexDefinition['options'], +): VectorIndexDefinition['options'] | undefined { + if (!declared && !catalog) return undefined; + const hnsw = { ...declared?.hnsw, ...catalog?.hnsw }; + const ivfflat = { ...declared?.ivfflat, ...catalog?.ivfflat }; + const merged: VectorIndexDefinition['options'] = { + ...declared, + ...catalog, + }; + if (Object.keys(hnsw).length) merged.hnsw = hnsw; + else delete merged.hnsw; + if (Object.keys(ivfflat).length) merged.ivfflat = ivfflat; + else delete merged.ivfflat; + return Object.values(merged).some(value => value !== undefined) ? merged : undefined; +} diff --git a/modules/database/src/adapters/utils/vectorMappings.ts b/modules/database/src/adapters/utils/vectorMappings.ts new file mode 100644 index 000000000..ff707ea64 --- /dev/null +++ b/modules/database/src/adapters/utils/vectorMappings.ts @@ -0,0 +1,184 @@ +import { + VectorIndexDefinition, + VectorIndexMethod, + VectorSimilarity, + defaultVectorIndexMethod, +} from '@conduitplatform/grpc-sdk'; +import { isObjectFormVectorField } from './vectorField.js'; +import { + hydratePostgresVectorIndex, + mongoSearchIndexReadiness, + mongoVectorFilterFields, +} from './vectorIndexLifecycle.js'; + +export type VectorStorageBackend = 'mongodb' | 'postgres' | 'sql'; + +export type VectorFieldStorageMapping = + | { + backend: 'mongodb'; + storage: 'numberArray'; + dimensions: number; + searchSupported: true; + } + | { + backend: 'postgres'; + storage: 'pgvector'; + dimensions: number; + searchSupported: true; + } + | { + backend: 'sql'; + storage: 'json'; + dimensions: number; + searchSupported: false; + }; + +export function mongoVectorStorageType() { + return [Number]; +} + +export function vectorFieldStorageMapping( + backend: VectorStorageBackend, + field: { dimensions: number }, +): VectorFieldStorageMapping { + switch (backend) { + case 'mongodb': + return { + backend: 'mongodb', + storage: 'numberArray', + dimensions: field.dimensions, + searchSupported: true, + }; + case 'postgres': + return { + backend: 'postgres', + storage: 'pgvector', + dimensions: field.dimensions, + searchSupported: true, + }; + case 'sql': + return { + backend: 'sql', + storage: 'json', + dimensions: field.dimensions, + searchSupported: false, + }; + default: { + const exhaustive: never = backend; + throw new Error(`Unsupported vector storage backend: ${String(exhaustive)}`); + } + } +} + +export function applyMongoVectorField>(field: T): T { + return { + ...field, + type: mongoVectorStorageType(), + }; +} + +export function pgVectorOperator(similarity: string) { + if (similarity === VectorSimilarity.Euclidean || similarity === 'euclidean') { + return 'vector_l2_ops'; + } + if (similarity === VectorSimilarity.DotProduct || similarity === 'dotProduct') { + return 'vector_ip_ops'; + } + return 'vector_cosine_ops'; +} + +export function pgVectorDistanceOperator(similarity: string) { + if (similarity === VectorSimilarity.Euclidean || similarity === 'euclidean') { + return '<->'; + } + if (similarity === VectorSimilarity.DotProduct || similarity === 'dotProduct') { + return '<#>'; + } + return '<=>'; +} + +export function toMongoVectorIndexDefinition(index: VectorIndexDefinition) { + const vectorField: Record = { + type: 'vector', + path: index.field, + numDimensions: index.dimensions, + similarity: index.similarity, + }; + if (index.options?.quantization) { + vectorField.quantization = index.options.quantization; + } + vectorField.indexingMethod = defaultVectorIndexMethod(index.method); + if (index.options?.hnsw) { + vectorField.hnswOptions = { + ...(index.options.hnsw.maxEdges && { maxEdges: index.options.hnsw.maxEdges }), + ...(index.options.hnsw.numEdgeCandidates && { + numEdgeCandidates: index.options.hnsw.numEdgeCandidates, + }), + }; + } + return { + fields: [ + vectorField, + ...mongoVectorFilterFields(index.filterFields).map((path: string) => ({ + type: 'filter', + path, + })), + ], + ...(index.options?.storedSource !== undefined && { + storedSource: index.options.storedSource, + }), + }; +} + +export function fromMongoVectorIndex(index: { + name?: string; + status?: string; + queryable?: boolean; + latestDefinition?: { fields?: Array> }; + definition?: { fields?: Array> }; +}): VectorIndexDefinition { + const fields = index.latestDefinition?.fields ?? index.definition?.fields ?? []; + const vectorField = fields.find(field => field.type === 'vector') ?? {}; + const readiness = mongoSearchIndexReadiness(index); + return { + name: index.name, + field: vectorField.path, + dimensions: vectorField.numDimensions, + similarity: vectorField.similarity, + method: defaultVectorIndexMethod(vectorField.indexingMethod), + filterFields: fields + .filter(field => field.type === 'filter') + .map(field => field.path), + status: readiness.status, + queryable: readiness.queryable, + }; +} + +export function fromPostgresVectorIndex( + name: string, + definition: string, + field?: { dimensions?: number; similarity?: VectorSimilarity }, + declared?: VectorIndexDefinition, +): VectorIndexDefinition { + return hydratePostgresVectorIndex({ + name, + indexdef: definition, + field, + declared, + }); +} + +export function postgresIndexMethodSql(method?: VectorIndexMethod | string) { + return method === VectorIndexMethod.IVFFlat || method === 'ivfflat' + ? 'ivfflat' + : 'hnsw'; +} + +export function resolveVectorFieldFromSchema( + schemaFields: Record | undefined, + fieldName: string, +) { + const field = schemaFields?.[fieldName]; + if (!isObjectFormVectorField(field)) return undefined; + return field; +} diff --git a/modules/database/src/adapters/utils/vectorProjection.ts b/modules/database/src/adapters/utils/vectorProjection.ts new file mode 100644 index 000000000..758381636 --- /dev/null +++ b/modules/database/src/adapters/utils/vectorProjection.ts @@ -0,0 +1,73 @@ +export function hiddenSelectFalseFields( + schemaFields: Record, +): Set { + return new Set( + Object.entries(schemaFields) + .filter(([, field]) => { + return ( + typeof field === 'object' && + field !== null && + (field as { select?: boolean }).select === false + ); + }) + .map(([field]) => field), + ); +} + +export function mongoVectorProjection( + schemaFields: Record, + select?: string, +): Record { + const hiddenFields = hiddenSelectFalseFields(schemaFields); + const tokens = select?.split(' ').filter(Boolean) ?? []; + const includeTokens = tokens.filter(token => !token.startsWith('-')); + if (includeTokens.length) { + const projection: Record = { _id: 1, _score: 1 }; + for (const token of includeTokens) { + if (token !== '_id' && !hiddenFields.has(token)) { + projection[token] = 1; + } + } + return projection; + } + const projection: Record = {}; + for (const field of hiddenFields) { + projection[field] = 0; + } + for (const token of tokens + .filter(token => token.startsWith('-')) + .map(token => token.slice(1))) { + if (token !== '_id') { + projection[token] = 0; + } + } + return projection; +} + +export function postgresVectorSelectList( + schemaFields: Record, + select: string | undefined, + quoteIdentifier: (identifier: string) => string, +): string { + const hiddenFields = hiddenSelectFalseFields(schemaFields); + const availableFields = Object.keys(schemaFields).filter( + field => !hiddenFields.has(field), + ); + if (!availableFields.includes('_id')) { + availableFields.unshift('_id'); + } + const tokens = select?.split(' ').filter(Boolean) ?? []; + const includeTokens = tokens.filter( + token => !token.startsWith('-') && !hiddenFields.has(token), + ); + const selected = new Set(includeTokens.length ? includeTokens : availableFields); + tokens + .filter(token => token.startsWith('-')) + .map(token => token.slice(1)) + .forEach(field => { + if (field !== '_id') selected.delete(field); + }); + hiddenFields.forEach(field => selected.delete(field)); + selected.add('_id'); + return [...selected].map(field => quoteIdentifier(field)).join(', '); +} diff --git a/modules/database/src/adapters/utils/vectorScore.ts b/modules/database/src/adapters/utils/vectorScore.ts new file mode 100644 index 000000000..64248e942 --- /dev/null +++ b/modules/database/src/adapters/utils/vectorScore.ts @@ -0,0 +1,114 @@ +import { + VectorSearchProvider, + VectorSearchResult, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import type { Indexable } from '@conduitplatform/grpc-sdk'; +import { parseVectorSimilarity } from './vectorField.js'; + +export interface NormalizedVectorScore { + score: number; + distance?: number; + metric: VectorSimilarity; + provider: VectorSearchProvider; + comparable: boolean; +} + +function finiteNumber(value: unknown, fallback = 0): number { + const numeric = typeof value === 'number' ? value : Number(value); + return Number.isFinite(numeric) ? numeric : fallback; +} + +/** + * Convert a backend raw score/distance into the documented result contract. + * + * Mongo Atlas `vectorSearchScore` is already higher-is-better. + * Postgres `<=>` is cosine *distance* (lower-is-better) and is inverted with + * `1 - distance`. Euclidean and inner-product values are higher-is-better + * rankings only and must not be treated as comparable across providers. + */ +export function normalizeVectorSearchScore(args: { + provider: VectorSearchProvider; + metric: VectorSimilarity | string | undefined; + raw: unknown; +}): NormalizedVectorScore { + const raw = finiteNumber(args.raw); + const metric = parseVectorSimilarity(args.metric); + switch (metric) { + case VectorSimilarity.Cosine: { + if (args.provider === 'postgres') { + return { + score: 1 - raw, + distance: raw, + metric, + provider: args.provider, + comparable: true, + }; + } + return { + score: raw, + metric, + provider: args.provider, + comparable: true, + }; + } + case VectorSimilarity.Euclidean: { + if (args.provider === 'postgres') { + return { + score: -raw, + distance: raw, + metric, + provider: args.provider, + comparable: false, + }; + } + return { + score: raw, + metric, + provider: args.provider, + comparable: false, + }; + } + case VectorSimilarity.DotProduct: { + if (args.provider === 'postgres') { + // pgvector `<#>` stores the negative inner product. + return { + score: -raw, + distance: raw, + metric, + provider: args.provider, + comparable: false, + }; + } + return { + score: raw, + metric, + provider: args.provider, + comparable: false, + }; + } + default: { + const exhaustive: never = metric; + throw new Error(`Unsupported vector similarity '${String(exhaustive)}'`); + } + } +} + +export function stripVectorScoreField(document: T): T { + const next = { ...document }; + delete next._score; + return next; +} + +export function toVectorSearchResult( + document: T, + normalized: NormalizedVectorScore, +): VectorSearchResult { + return { + document, + score: normalized.score, + ...(normalized.distance !== undefined ? { distance: normalized.distance } : {}), + metric: normalized.metric, + provider: normalized.provider, + }; +} diff --git a/modules/database/src/adapters/utils/vectorSearchAuth.ts b/modules/database/src/adapters/utils/vectorSearchAuth.ts new file mode 100644 index 000000000..2457b908c --- /dev/null +++ b/modules/database/src/adapters/utils/vectorSearchAuth.ts @@ -0,0 +1,63 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export const VECTOR_SEARCH_OPERATOR_MODULES = ['database', 'core', 'embeddings'] as const; + +export function resolveAdminOperatorContext(args: { + requested?: boolean; + callerModule?: string; + operatorModules?: readonly string[]; +}): boolean { + if (!args.requested) return false; + const operators = args.operatorModules ?? VECTOR_SEARCH_OPERATOR_MODULES; + if (!args.callerModule || !operators.includes(args.callerModule)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Admin operator context is not allowed for this caller', + ); + } + return true; +} + +export function assertVectorSearchAccess(args: { + authzEnabled: boolean; + userId?: string; + scope?: string; + adminOperator?: boolean; +}): void { + if (!args.authzEnabled) return; + if (args.userId || args.scope || args.adminOperator) return; + throw new GrpcError( + status.PERMISSION_DENIED, + 'Vector search on authorization-enabled schemas requires a subject, scope, or admin operator context', + ); +} + +export async function authorizeBoundedVectorCandidates(args: { + authzEnabled: boolean; + adminOperator?: boolean; + candidateIds: Array; + lookupAuthorizedIds: (ids: string[]) => Promise; +}): Promise> { + const ids = args.candidateIds.map(id => String(id)).filter(Boolean); + if (!ids.length) return new Set(); + if (!args.authzEnabled || args.adminOperator) { + return new Set(ids); + } + const authorized = await args.lookupAuthorizedIds(ids); + return new Set(authorized.map(id => String(id))); +} + +export function applyBoundedVectorAuthorization< + T extends { _id?: unknown; id?: unknown }, +>(documents: T[], authorizedIds: Set, limit: number): T[] { + const next: T[] = []; + for (const document of documents) { + const id = document._id ?? document.id; + if (id === undefined || id === null) continue; + if (!authorizedIds.has(String(id))) continue; + next.push(document); + if (next.length >= limit) break; + } + return next; +} diff --git a/modules/database/src/adapters/utils/vectorSearchFilter.ts b/modules/database/src/adapters/utils/vectorSearchFilter.ts new file mode 100644 index 000000000..f1f12fcce --- /dev/null +++ b/modules/database/src/adapters/utils/vectorSearchFilter.ts @@ -0,0 +1,195 @@ +import { GrpcError, Indexable } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export type VectorFilterProvider = 'mongodb' | 'postgres'; + +const COMPARISON_OPERATORS = new Set(['$eq', '$ne', '$gt', '$gte', '$lt', '$lte']); +const MEMBERSHIP_OPERATORS = new Set(['$in', '$nin']); +const LOGICAL_OPERATORS = new Set(['$and', '$or', '$nor']); +const MONGO_FIELD_PATTERN = /^[A-Za-z_][A-Za-z0-9_.]*$/; +const POSTGRES_FIELD_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; +const RESERVED_FIELD_NAMES = new Set(['__proto__', 'prototype', 'constructor']); + +export interface ValidatedVectorSearchFilter { + filter: Indexable; + emptyResult: boolean; +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isScalar(value: unknown): boolean { + return ( + value === null || + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' + ); +} + +function invalid(message: string): never { + throw new GrpcError(status.INVALID_ARGUMENT, message); +} + +function assertAllowedField( + field: string, + provider: VectorFilterProvider, + allowedFields?: readonly string[], +): void { + const pattern = provider === 'postgres' ? POSTGRES_FIELD_PATTERN : MONGO_FIELD_PATTERN; + if (!pattern.test(field) || field.startsWith('$') || RESERVED_FIELD_NAMES.has(field)) { + invalid(`Unsupported vector search filter field '${field}'`); + } + if (allowedFields && !allowedFields.includes(field)) { + invalid( + provider === 'mongodb' + ? `Vector search filter field '${field}' is not an indexed filter field. ` + + `Allowed fields: ${allowedFields.length ? allowedFields.join(', ') : '(none)'}` + : `Vector search filter field '${field}' is not a schema field. ` + + `Allowed fields: ${allowedFields.join(', ')}`, + ); + } +} + +function validateMembership(value: unknown, operator: string): { empty: boolean } { + if (!Array.isArray(value)) { + invalid(`Vector search operator ${operator} requires an array`); + } + for (const item of value) { + if (!isScalar(item)) { + invalid(`Vector search operator ${operator} only accepts scalar values`); + } + } + if (operator === '$in' && value.length === 0) { + return { empty: true }; + } + return { empty: false }; +} + +function validateComparisonValue(value: unknown, operator: string): void { + if (operator === '$eq' || operator === '$ne') { + if (!isScalar(value)) { + invalid(`Vector search operator ${operator} only accepts scalar values`); + } + return; + } + if (typeof value !== 'number' && typeof value !== 'string') { + invalid(`Vector search operator ${operator} only accepts number or string values`); + } +} + +function validateFieldPredicate( + field: string, + value: unknown, + provider: VectorFilterProvider, + allowedFields?: readonly string[], +): { empty: boolean } { + assertAllowedField(field, provider, allowedFields); + if (isScalar(value)) { + return { empty: false }; + } + if (!isPlainObject(value)) { + invalid(`Unsupported vector search filter value for '${field}'`); + } + const keys = Object.keys(value); + if (!keys.length) { + invalid(`Unsupported vector search filter value for '${field}'`); + } + let empty = false; + for (const operator of keys) { + if (operator === '$not') { + const nested = value[operator]; + if (!isPlainObject(nested)) { + invalid(`Vector search operator $not requires a comparison object`); + } + // Negating a match-nothing predicate matches everything. + validateFieldPredicate(field, nested, provider, allowedFields); + continue; + } + if (COMPARISON_OPERATORS.has(operator)) { + validateComparisonValue(value[operator], operator); + continue; + } + if (MEMBERSHIP_OPERATORS.has(operator)) { + const result = validateMembership(value[operator], operator); + if (result.empty) empty = true; + continue; + } + invalid(`Unsupported vector search filter operator '${operator}'`); + } + return { empty }; +} + +function validateLogical( + operator: string, + value: unknown, + provider: VectorFilterProvider, + allowedFields?: readonly string[], +): { empty: boolean } { + if (!Array.isArray(value) || value.length === 0) { + invalid(`Vector search operator ${operator} requires a non-empty array`); + } + const branchEmpty = value.map(branch => + validateFilterNode(branch, provider, allowedFields), + ); + if (operator === '$and') { + return { empty: branchEmpty.some(branch => branch.empty) }; + } + if (operator === '$or') { + return { empty: branchEmpty.every(branch => branch.empty) }; + } + return { empty: false }; +} + +function validateFilterNode( + node: unknown, + provider: VectorFilterProvider, + allowedFields?: readonly string[], +): { empty: boolean } { + if (!isPlainObject(node)) { + invalid('Vector search filter must be an object'); + } + const keys = Object.keys(node); + if (!keys.length) { + return { empty: false }; + } + let emptyAnd = false; + const orEmpty: boolean[] = []; + for (const key of keys) { + if (LOGICAL_OPERATORS.has(key)) { + const result = validateLogical(key, node[key], provider, allowedFields); + if (key === '$and' && result.empty) emptyAnd = true; + if (key === '$or') orEmpty.push(result.empty); + continue; + } + const result = validateFieldPredicate(key, node[key], provider, allowedFields); + if (result.empty) emptyAnd = true; + } + if (emptyAnd) return { empty: true }; + if (orEmpty.length && orEmpty.every(Boolean) && keys.every(key => key === '$or')) { + return { empty: true }; + } + return { empty: false }; +} + +export function validateVectorSearchFilter( + filter: Indexable | undefined, + options: { + provider: VectorFilterProvider; + allowedFilterFields?: readonly string[]; + }, +): ValidatedVectorSearchFilter { + if (filter === undefined || filter === null) { + return { filter: {}, emptyResult: false }; + } + if (!isPlainObject(filter)) { + invalid('Vector search filter must be an object'); + } + const allowedFields = + options.provider === 'mongodb' + ? (options.allowedFilterFields ?? []) + : options.allowedFilterFields; + const emptyResult = validateFilterNode(filter, options.provider, allowedFields).empty; + return { filter, emptyResult }; +} diff --git a/modules/database/src/adapters/utils/vectorSearchLimits.ts b/modules/database/src/adapters/utils/vectorSearchLimits.ts new file mode 100644 index 000000000..3be08185d --- /dev/null +++ b/modules/database/src/adapters/utils/vectorSearchLimits.ts @@ -0,0 +1,44 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export const VECTOR_SEARCH_DEFAULT_LIMIT = 10; +export const VECTOR_SEARCH_MAX_LIMIT = 1000; +export const VECTOR_SEARCH_MAX_CANDIDATES = 10_000; + +export interface VectorSearchLimits { + limit: number; + numCandidates: number; +} + +function parsePositiveInt(value: unknown, field: string): number | undefined { + if (value === undefined || value === null || value === '') return undefined; + const numeric = typeof value === 'number' ? value : Number(value); + if (!Number.isFinite(numeric) || !Number.isInteger(numeric) || numeric < 1) { + throw new GrpcError(status.INVALID_ARGUMENT, `${field} must be a positive integer`); + } + return numeric; +} + +export function clampVectorSearchLimits(input: { + limit?: number; + numCandidates?: number; +}): VectorSearchLimits { + const parsedLimit = parsePositiveInt(input.limit, 'limit'); + const parsedCandidates = parsePositiveInt(input.numCandidates, 'numCandidates'); + const limit = Math.min( + parsedLimit ?? VECTOR_SEARCH_DEFAULT_LIMIT, + VECTOR_SEARCH_MAX_LIMIT, + ); + const defaultCandidates = Math.max(limit * 10, 100); + const numCandidates = Math.min( + parsedCandidates ?? defaultCandidates, + VECTOR_SEARCH_MAX_CANDIDATES, + ); + if (numCandidates < limit) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `numCandidates must be at least the requested limit (${limit})`, + ); + } + return { limit, numCandidates }; +} diff --git a/modules/database/src/adapters/utils/vectorSearchQuery.ts b/modules/database/src/adapters/utils/vectorSearchQuery.ts new file mode 100644 index 000000000..cd9d3ed08 --- /dev/null +++ b/modules/database/src/adapters/utils/vectorSearchQuery.ts @@ -0,0 +1,225 @@ +import { + Indexable, + VectorIndexDefinition, + VectorSearchInput, + VectorSearchProvider, + VectorSearchResult, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { clampVectorSearchLimits } from './vectorSearchLimits.js'; +import { validateVectorSearchFilter } from './vectorSearchFilter.js'; +import { renderPostgresVectorWhere, PostgresWhereRenderer } from './vectorSearchWhere.js'; +import { pgVectorDistanceOperator } from './vectorMappings.js'; +import { mongoVectorProjection, postgresVectorSelectList } from './vectorProjection.js'; +import { + applyBoundedVectorAuthorization, + authorizeBoundedVectorCandidates, +} from './vectorSearchAuth.js'; +import { + normalizeVectorSearchScore, + stripVectorScoreField, + toVectorSearchResult, +} from './vectorScore.js'; +import { parseVectorSimilarity } from './vectorField.js'; +import { + assertVectorIndexQueryable, + defaultVectorIndexName, + selectLiveVectorIndexForField, +} from './vectorIndexLifecycle.js'; + +export interface PlannedMongoVectorSearch { + emptyResult: boolean; + limits: { limit: number; numCandidates: number }; + index?: VectorIndexDefinition; + pipeline: Indexable[]; +} + +export interface PlannedPostgresVectorSearch { + emptyResult: boolean; + limits: { limit: number; numCandidates: number }; + index?: VectorIndexDefinition; + sql: string; + distanceOperator: string; +} + +export function schemaFieldNames(schemaFields: Record): string[] { + const fields = Object.keys(schemaFields); + if (!fields.includes('_id')) { + fields.unshift('_id'); + } + return fields; +} + +export function declaredVectorIndexes(schema: { + modelOptions?: { vectorIndexes?: ReadonlyArray }; +}): VectorIndexDefinition[] { + return [...(schema.modelOptions?.vectorIndexes ?? [])]; +} + +export function findVectorIndexForSearch( + indexes: VectorIndexDefinition[], + request: { indexName?: string; field: string }, +): VectorIndexDefinition | undefined { + if (request.indexName) { + return indexes.find(item => item.name === request.indexName); + } + return selectLiveVectorIndexForField(indexes, request.field); +} + +export function mergeVectorIndexes( + declared: VectorIndexDefinition[], + live: VectorIndexDefinition[], +): VectorIndexDefinition[] { + if (!live.length) return []; + if (!declared.length) return live; + const declaredByKey = new Map(); + for (const index of declared) { + declaredByKey.set(index.name ?? defaultVectorIndexName(index.field), index); + } + return live.map(liveIndex => { + const key = liveIndex.name ?? defaultVectorIndexName(liveIndex.field); + const declaredIndex = + declaredByKey.get(key) ?? declared.find(item => item.field === liveIndex.field); + if (!declaredIndex) return liveIndex; + return { + ...declaredIndex, + ...liveIndex, + status: liveIndex.status, + queryable: liveIndex.queryable, + }; + }); +} + +export function buildMongoVectorSearchPipeline(args: { + indexName: string; + field: string; + vector: number[]; + numCandidates: number; + limit: number; + filter?: Indexable; + projection: Indexable; +}): Indexable[] { + const vectorStage: Indexable = { + index: args.indexName, + path: args.field, + queryVector: args.vector, + numCandidates: args.numCandidates, + limit: args.limit, + }; + if (args.filter && Object.keys(args.filter).length > 0) { + vectorStage.filter = args.filter; + } + return [ + { $vectorSearch: vectorStage }, + { $addFields: { _score: { $meta: 'vectorSearchScore' } } }, + { $project: args.projection }, + ]; +} + +export function planMongoVectorSearch(args: { + request: VectorSearchInput; + indexes: VectorIndexDefinition[]; + schemaFields: Record; +}): PlannedMongoVectorSearch { + const limits = clampVectorSearchLimits({ + limit: args.request.limit, + numCandidates: args.request.numCandidates, + }); + const index = findVectorIndexForSearch(args.indexes, args.request); + const validated = validateVectorSearchFilter(args.request.filter, { + provider: 'mongodb', + allowedFilterFields: index?.filterFields ?? [], + }); + if (validated.emptyResult) { + return { emptyResult: true, limits, index, pipeline: [] }; + } + assertVectorIndexQueryable(index, args.request); + return { + emptyResult: false, + limits, + index, + pipeline: buildMongoVectorSearchPipeline({ + indexName: + args.request.indexName ?? + index.name ?? + defaultVectorIndexName(args.request.field), + field: args.request.field, + vector: args.request.vector, + numCandidates: limits.numCandidates, + limit: limits.numCandidates, + filter: validated.filter, + projection: mongoVectorProjection(args.schemaFields, args.request.select), + }), + }; +} + +export function planPostgresVectorSearch(args: { + request: VectorSearchInput; + indexes: VectorIndexDefinition[]; + schemaFields: Record; + tableName: string; + similarity: VectorSimilarity | string | undefined; + renderer: PostgresWhereRenderer; +}): PlannedPostgresVectorSearch { + const limits = clampVectorSearchLimits({ + limit: args.request.limit, + numCandidates: args.request.numCandidates, + }); + const index = findVectorIndexForSearch(args.indexes, args.request); + const validated = validateVectorSearchFilter(args.request.filter, { + provider: 'postgres', + allowedFilterFields: schemaFieldNames(args.schemaFields), + }); + const distanceOperator = pgVectorDistanceOperator( + parseVectorSimilarity(args.similarity), + ); + if (validated.emptyResult) { + return { emptyResult: true, limits, index, sql: '', distanceOperator }; + } + assertVectorIndexQueryable(index, args.request); + const where = renderPostgresVectorWhere(validated.filter, args.renderer); + const selectedColumns = postgresVectorSelectList( + args.schemaFields, + args.request.select, + args.renderer.quoteIdentifier, + ); + const vectorLiteral = args.renderer.escape(`[${args.request.vector.join(',')}]`); + const fieldSql = args.renderer.quoteIdentifier(args.request.field); + const sql = + `SELECT ${selectedColumns}, (${fieldSql} ${distanceOperator} ${vectorLiteral}::vector) AS _score ` + + `FROM ${args.renderer.quoteIdentifier(args.tableName)}${where} ` + + `ORDER BY ${fieldSql} ${distanceOperator} ${vectorLiteral}::vector ` + + `LIMIT ${limits.numCandidates}`; + return { emptyResult: false, limits, index, sql, distanceOperator }; +} + +export async function completeVectorSearch(args: { + emptyResult: boolean; + limit: number; + authzEnabled: boolean; + adminOperator?: boolean; + provider: VectorSearchProvider; + metric: VectorSimilarity | string | undefined; + fetchCandidates: () => Promise; + lookupAuthorizedIds: (ids: string[]) => Promise; +}): Promise[]> { + if (args.emptyResult) return []; + const documents = await args.fetchCandidates(); + const authorizedIds = await authorizeBoundedVectorCandidates({ + authzEnabled: args.authzEnabled, + adminOperator: args.adminOperator, + candidateIds: documents + .map(document => document._id ?? document.id) + .filter((id): id is string | { toString(): string } => id != null), + lookupAuthorizedIds: args.lookupAuthorizedIds, + }); + const allowed = applyBoundedVectorAuthorization(documents, authorizedIds, args.limit); + return allowed.map(document => { + const normalized = normalizeVectorSearchScore({ + provider: args.provider, + metric: args.metric, + raw: document._score, + }); + return toVectorSearchResult(stripVectorScoreField(document), normalized); + }); +} diff --git a/modules/database/src/adapters/utils/vectorSearchWhere.ts b/modules/database/src/adapters/utils/vectorSearchWhere.ts new file mode 100644 index 000000000..655020d5e --- /dev/null +++ b/modules/database/src/adapters/utils/vectorSearchWhere.ts @@ -0,0 +1,144 @@ +import { GrpcError, Indexable } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export interface PostgresWhereRenderer { + quoteIdentifier: (identifier: string) => string; + escape: (value: unknown) => string; +} + +function invalid(message: string): never { + throw new GrpcError(status.INVALID_ARGUMENT, message); +} + +function renderComparison( + fieldSql: string, + operator: string, + value: unknown, + renderer: PostgresWhereRenderer, +): string { + switch (operator) { + case '$eq': + return value === null + ? `${fieldSql} IS NULL` + : `${fieldSql} = ${renderer.escape(value)}`; + case '$ne': + return value === null + ? `${fieldSql} IS NOT NULL` + : `${fieldSql} <> ${renderer.escape(value)}`; + case '$gt': + return `${fieldSql} > ${renderer.escape(value)}`; + case '$gte': + return `${fieldSql} >= ${renderer.escape(value)}`; + case '$lt': + return `${fieldSql} < ${renderer.escape(value)}`; + case '$lte': + return `${fieldSql} <= ${renderer.escape(value)}`; + default: { + const exhaustive: never = operator as never; + invalid(`Unsupported vector search filter operator '${String(exhaustive)}'`); + } + } +} + +function renderInPredicate( + fieldSql: string, + operand: unknown, + renderer: PostgresWhereRenderer, +): string { + const values = operand as unknown[]; + if (!values.length) return 'FALSE'; + return `${fieldSql} IN (${values.map(item => renderer.escape(item)).join(', ')})`; +} + +function renderNinPredicate( + fieldSql: string, + operand: unknown, + renderer: PostgresWhereRenderer, +): string | undefined { + const values = operand as unknown[]; + if (!values.length) return undefined; + return `${fieldSql} NOT IN (${values.map(item => renderer.escape(item)).join(', ')})`; +} + +function renderFieldPredicate( + field: string, + value: unknown, + renderer: PostgresWhereRenderer, +): string { + const fieldSql = renderer.quoteIdentifier(field); + if (value === null) { + return `${fieldSql} IS NULL`; + } + if (typeof value === 'boolean') { + return `${fieldSql} = ${value ? 'TRUE' : 'FALSE'}`; + } + if (typeof value === 'string' || typeof value === 'number') { + return `${fieldSql} = ${renderer.escape(value)}`; + } + if (!value || typeof value !== 'object' || Array.isArray(value)) { + invalid('Unsupported vector search filter shape'); + } + const clauses: string[] = []; + for (const [operator, operand] of Object.entries(value as Record)) { + if (operator === '$in') { + const rendered = renderInPredicate(fieldSql, operand, renderer); + if (rendered === 'FALSE') return 'FALSE'; + clauses.push(rendered); + continue; + } + if (operator === '$nin') { + const rendered = renderNinPredicate(fieldSql, operand, renderer); + if (rendered) clauses.push(rendered); + continue; + } + if (operator === '$not') { + clauses.push(`NOT (${renderFieldPredicate(field, operand, renderer)})`); + continue; + } + clauses.push(renderComparison(fieldSql, operator, operand, renderer)); + } + if (!clauses.length) { + return 'TRUE'; + } + return clauses.length === 1 ? clauses[0] : `(${clauses.join(' AND ')})`; +} + +function renderNode(node: Indexable, renderer: PostgresWhereRenderer): string { + const clauses: string[] = []; + for (const [key, value] of Object.entries(node)) { + if (key === '$and' && Array.isArray(value)) { + const nested = value + .map(item => renderNode(item as Indexable, renderer)) + .filter(Boolean); + if (nested.length) clauses.push(`(${nested.join(' AND ')})`); + continue; + } + if (key === '$or' && Array.isArray(value)) { + const nested = value.map(item => renderNode(item as Indexable, renderer)); + clauses.push(`(${nested.join(' OR ')})`); + continue; + } + if (key === '$nor' && Array.isArray(value)) { + const nested = value.map(item => renderNode(item as Indexable, renderer)); + clauses.push(`NOT (${nested.join(' OR ')})`); + continue; + } + clauses.push(renderFieldPredicate(key, value, renderer)); + } + return clauses.filter(Boolean).join(' AND '); +} + +export function renderPostgresVectorWhere( + filter: Indexable | undefined, + renderer: PostgresWhereRenderer, +): string { + if (!filter || !Object.keys(filter).length) { + return ''; + } + const body = renderNode(filter, renderer); + if (!body) return ''; + if (body === 'FALSE') { + return ' WHERE FALSE'; + } + return ` WHERE ${body}`; +} diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index a4c1be91f..5a4605e9f 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -677,6 +677,91 @@ export class AdminHandlers { new ConduitRouteReturnDefinition('deleteIndexes', 'String'), this.schemaAdmin.deleteIndexes.bind(this.schemaAdmin), ); + this.routingManager.route( + { + path: '/vector/capabilities', + action: ConduitRouteActions.GET, + description: `Returns vector storage, index, and search capabilities for the active database.`, + queryParams: { + schemaName: ConduitString.Optional, + }, + }, + new ConduitRouteReturnDefinition('getVectorCapabilities', { + supported: { type: TYPE.Boolean, required: true }, + storage: { type: TYPE.Boolean, required: true }, + indexing: { type: TYPE.Boolean, required: true }, + search: { type: TYPE.Boolean, required: true }, + provider: ConduitString.Required, + reason: ConduitString.Optional, + }), + this.schemaAdmin.getVectorCapabilities.bind(this.schemaAdmin), + ); + this.routingManager.route( + { + path: '/schemas/:id/vector-indexes', + action: ConduitRouteActions.POST, + description: `Creates a vector index for a schema.`, + urlParams: { + id: { type: TYPE.String, required: true }, + }, + bodyParams: { + index: ConduitJson.Required, + }, + } as any, + new ConduitRouteReturnDefinition('createVectorIndex', 'String'), + this.schemaAdmin.createVectorIndex.bind(this.schemaAdmin), + ); + this.routingManager.route( + { + path: '/schemas/:id/vector-indexes', + action: ConduitRouteActions.GET, + description: `Returns vector indexes of a schema.`, + urlParams: { + id: { type: TYPE.String, required: true }, + }, + }, + new ConduitRouteReturnDefinition('getVectorIndexes', { + indexes: [ConduitJson.Required], + }), + this.schemaAdmin.getVectorIndexes.bind(this.schemaAdmin), + ); + this.routingManager.route( + { + path: '/schemas/:id/vector-indexes/:indexName', + action: ConduitRouteActions.DELETE, + description: `Deletes a vector index of a schema.`, + urlParams: { + id: { type: TYPE.String, required: true }, + indexName: ConduitString.Required, + }, + }, + new ConduitRouteReturnDefinition('deleteVectorIndex', 'String'), + this.schemaAdmin.deleteVectorIndex.bind(this.schemaAdmin), + ); + this.routingManager.route( + { + path: '/schemas/:schemaName/vector-search', + action: ConduitRouteActions.POST, + description: `Runs vector search for a schema.`, + urlParams: { + schemaName: ConduitString.Required, + }, + bodyParams: { + field: ConduitString.Required, + vector: [ConduitNumber.Required], + indexName: ConduitString.Optional, + filter: ConduitJson.Optional, + limit: ConduitNumber.Optional, + numCandidates: ConduitNumber.Optional, + select: ConduitString.Optional, + scope: ConduitString.Optional, + }, + } as any, + new ConduitRouteReturnDefinition('vectorSearch', { + results: [ConduitJson.Required], + }), + this.schemaAdmin.vectorSearch.bind(this.schemaAdmin), + ); this.routingManager.route( { path: '/database-type', diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index 9c149d6db..9e8ae74cc 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -684,6 +684,63 @@ export class SchemaAdmin { return this.database.getDatabaseType(); } + async getVectorCapabilities( + call: ParsedRouterRequest, + ): Promise { + return this.database.getVectorCapabilities(call.request.params.schemaName); + } + + async createVectorIndex(call: ParsedRouterRequest): Promise { + const { id, index } = call.request.params; + const requestedSchema = await this.database + .getSchemaModel('_DeclaredSchema') + .model.findOne({ _id: id }); + if (isNil(requestedSchema)) { + throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); + } + return this.database.createVectorIndex(requestedSchema.name, index); + } + + async getVectorIndexes(call: ParsedRouterRequest): Promise { + const id = call.request.params.id; + const requestedSchema = await this.database + .getSchemaModel('_DeclaredSchema') + .model.findOne({ _id: id }); + if (isNil(requestedSchema)) { + throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); + } + return { indexes: await this.database.getVectorIndexes(requestedSchema.name) }; + } + + async deleteVectorIndex(call: ParsedRouterRequest): Promise { + const { id, indexName } = call.request.params; + const requestedSchema = await this.database + .getSchemaModel('_DeclaredSchema') + .model.findOne({ _id: id }); + if (isNil(requestedSchema)) { + throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); + } + return this.database.deleteVectorIndex(requestedSchema.name, indexName); + } + + async vectorSearch(call: ParsedRouterRequest): Promise { + const { schemaName } = call.request.params; + const results = await this.database.vectorSearch({ + schemaName, + field: call.request.params.field, + vector: call.request.params.vector, + indexName: call.request.params.indexName, + filter: call.request.params.filter, + limit: call.request.params.limit, + numCandidates: call.request.params.numCandidates, + select: call.request.params.select, + userId: call.request.context.user?._id, + scope: call.request.params.scope, + adminOperator: true, + }); + return { results }; + } + async createIndexes(call: ParsedRouterRequest): Promise { const { id, indexes } = call.request.params; const requestedSchema = await this.database diff --git a/modules/database/src/controllers/cms/__tests__/assignableFields.test.ts b/modules/database/src/controllers/cms/__tests__/assignableFields.test.ts new file mode 100644 index 000000000..e2df454c8 --- /dev/null +++ b/modules/database/src/controllers/cms/__tests__/assignableFields.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { + ConduitRouteActions, + ConduitSchema, + TYPE, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { CmsHandlers } from '../../../handlers/cms/crud.handler.js'; +import { getAssignableCmsFields, getOps, isCmsWriteOmittedField } from '../utils.js'; + +const fields = { + _id: { type: TYPE.ObjectId }, + title: { type: TYPE.String, required: true }, + body: TYPE.String, + owner: { type: TYPE.Relation, model: 'User' }, + meta: TYPE.JSON, + secret: { type: TYPE.String, select: false }, + embedding: { + type: TYPE.Vector, + dimensions: 8, + similarity: VectorSimilarity.Cosine, + select: false, + }, + embeddingSourceHash: { type: TYPE.String, select: false }, + createdAt: TYPE.Date, + updatedAt: TYPE.Date, +}; + +function enabledCmsSchema() { + return new ConduitSchema('Article', fields, { + conduit: { + cms: { + enabled: true, + crudOperations: { + create: { enabled: true, authenticated: false }, + read: { enabled: true, authenticated: false }, + update: { enabled: true, authenticated: false }, + delete: { enabled: true, authenticated: false }, + }, + }, + authorization: { enabled: false }, + }, + }); +} + +describe('CMS assignable fields', () => { + it('omits vector, hash, select:false, and system fields while keeping writable types', () => { + const assignable = getAssignableCmsFields(fields); + expect(Object.keys(assignable).sort()).toEqual(['body', 'meta', 'owner', 'title']); + expect(isCmsWriteOmittedField('embedding', fields.embedding)).toBe(true); + expect( + isCmsWriteOmittedField('embeddingSourceHash', fields.embeddingSourceHash), + ).toBe(true); + expect(isCmsWriteOmittedField('secret', fields.secret)).toBe(true); + expect( + isCmsWriteOmittedField('embedding', { + type: TYPE.Vector, + dimensions: 2, + }), + ).toBe(true); + expect(isCmsWriteOmittedField('title', fields.title)).toBe(false); + }); + + it('recursively omits nested Vector, select:false, and source-hash fields from embedded objects', () => { + const nested = { + title: { type: TYPE.String, required: true }, + profile: { + type: { + bio: TYPE.String, + embedding: { + type: TYPE.Vector, + dimensions: 4, + similarity: VectorSimilarity.Cosine, + }, + embeddingSourceHash: { type: TYPE.String, select: false }, + secret: { type: TYPE.String, select: false }, + }, + }, + items: [ + { + name: TYPE.String, + embedding: { + type: TYPE.Vector, + dimensions: 2, + similarity: VectorSimilarity.Cosine, + }, + hidden: { type: TYPE.String, select: false }, + }, + ], + }; + const assignable = getAssignableCmsFields(nested); + expect(Object.keys(assignable).sort()).toEqual(['items', 'profile', 'title']); + expect(assignable.profile).toMatchObject({ + type: { bio: TYPE.String }, + }); + expect( + (assignable.profile as { type: Record }).type, + ).not.toHaveProperty('embedding'); + expect( + (assignable.profile as { type: Record }).type, + ).not.toHaveProperty('embeddingSourceHash'); + expect( + (assignable.profile as { type: Record }).type, + ).not.toHaveProperty('secret'); + expect(assignable.items).toEqual([{ name: TYPE.String }]); + }); + + it('keeps vector fields on CMS return projections but not create/update bodies', () => { + const handlers = { + getDocuments: jest.fn(), + getDocumentById: jest.fn(), + createDocument: jest.fn(), + createManyDocuments: jest.fn(), + updateManyDocuments: jest.fn(), + patchManyDocuments: jest.fn(), + updateDocument: jest.fn(), + patchDocument: jest.fn(), + deleteDocument: jest.fn(), + } as unknown as CmsHandlers; + + const routes = getOps('Article', enabledCmsSchema(), handlers); + const create = routes.find( + route => + route.input.action === ConduitRouteActions.POST && + route.input.path === '/Article', + ); + const update = routes.find( + route => + route.input.action === ConduitRouteActions.UPDATE && + route.input.path === '/Article/:id', + ); + const getById = routes.find( + route => + route.input.action === ConduitRouteActions.GET && + route.input.path === '/Article/:id', + ); + + expect(create?.input.bodyParams).toMatchObject({ + title: { type: TYPE.String, required: true }, + body: TYPE.String, + owner: { type: TYPE.Relation, model: 'User' }, + meta: TYPE.JSON, + }); + expect(Object.keys(create?.input.bodyParams ?? {}).sort()).toEqual([ + 'body', + 'meta', + 'owner', + 'title', + ]); + expect(create?.input.bodyParams).not.toHaveProperty('embedding'); + expect(create?.input.bodyParams).not.toHaveProperty('embeddingSourceHash'); + expect(create?.input.bodyParams).not.toHaveProperty('secret'); + expect(update?.input.bodyParams).not.toHaveProperty('embedding'); + expect(getById?.returnType.fields).toMatchObject({ + title: { type: TYPE.String, required: true }, + embedding: { + type: TYPE.Vector, + dimensions: 8, + select: false, + }, + embeddingSourceHash: { type: TYPE.String, select: false }, + }); + expect(fields.title.required).toBe(true); + }); +}); diff --git a/modules/database/src/controllers/cms/utils.ts b/modules/database/src/controllers/cms/utils.ts index 2efa4106f..27162718f 100644 --- a/modules/database/src/controllers/cms/utils.ts +++ b/modules/database/src/controllers/cms/utils.ts @@ -57,6 +57,112 @@ export function compareFunction(schemaA: ConduitModel, schemaB: ConduitModel): n } } +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isVectorField(field: unknown): boolean { + if (field === TYPE.Vector || field === 'Vector') return true; + if (Array.isArray(field) && field.length > 0) return isVectorField(field[0]); + if (!isPlainObject(field)) return false; + return isVectorField(field.type); +} + +function isHiddenSelectField(field: unknown): boolean { + return isPlainObject(field) && field.select === false; +} + +function isManagedHashField(name: string): boolean { + return name.endsWith('SourceHash'); +} + +export function isCmsWriteOmittedField(name: string, field: unknown): boolean { + if (name === '_id' || name === 'createdAt' || name === 'updatedAt') return true; + if (isManagedHashField(name)) return true; + if (isHiddenSelectField(field)) return true; + return isVectorField(field); +} + +export function getAssignableCmsFields(sourceFields: ConduitModel): ConduitModel { + return stripAssignableModel(sourceFields); +} + +const FIELD_DESCRIPTOR_KEYS = new Set([ + 'type', + 'sqlType', + 'default', + 'description', + 'required', + 'select', + 'unique', + 'index', + 'enum', + 'model', + 'validate', + 'dimensions', + 'similarity', + 'provider', +]); + +function isNestedConduitModel(field: unknown): field is ConduitModel { + if (!isPlainObject(field)) return false; + if (isVectorField(field) || isHiddenSelectField(field)) return false; + if ('type' in field || 'enum' in field || 'model' in field) return false; + return Object.keys(field).some(key => !FIELD_DESCRIPTOR_KEYS.has(key)); +} + +function stripAssignableModel(source: ConduitModel): ConduitModel { + const assignable: ConduitModel = {}; + for (const [name, field] of Object.entries(source)) { + if (isCmsWriteOmittedField(name, field)) continue; + assignable[name] = cloneAssignableValue(field); + } + return assignable; +} + +function cloneAssignableArrayItem(item: unknown): unknown { + if (Array.isArray(item)) { + return item.map(cloneAssignableArrayItem); + } + if (!isPlainObject(item)) return item; + if (isVectorField(item)) { + return { ...item }; + } + if (isNestedConduitModel(item)) { + return stripAssignableModel(item); + } + return cloneAssignableValue(item); +} + +function cloneAssignableValue(field: unknown): ConduitModel[string] { + if (Array.isArray(field)) { + return field.map(cloneAssignableArrayItem) as ConduitModel[string]; + } + if (!isPlainObject(field)) { + return field as ConduitModel[string]; + } + if (isNestedConduitModel(field)) { + return stripAssignableModel(field) as ConduitModel[string]; + } + const cloned: Record = { ...field }; + if (Array.isArray(cloned.type)) { + cloned.type = cloned.type.map(cloneAssignableArrayItem); + } else if (isPlainObject(cloned.type) && !isVectorField(cloned)) { + if (isNestedConduitModel(cloned.type) || isPlainObject(cloned.type)) { + cloned.type = stripAssignableModel(cloned.type as ConduitModel); + } + } + return cloned as ConduitModel[string]; +} + +function cloneConduitModel(fields: ConduitModel): ConduitModel { + const cloned: ConduitModel = {}; + for (const [name, field] of Object.entries(fields)) { + cloned[name] = cloneAssignableValue(field); + } + return cloned; +} + function removeRequiredFields(fields: ConduitModel) { for (const field in fields) { const modelField = fields[field] as ConduitModelField; @@ -133,10 +239,7 @@ export function getOps( const sourceFields = (actualSchema as unknown as { compiledFields?: ConduitModel }).compiledFields ?? actualSchema.fields; - const assignableFields: ConduitModel = Object.assign({}, sourceFields); - delete assignableFields._id; - delete assignableFields.createdAt; - delete assignableFields.updatedAt; + const assignableFields: ConduitModel = getAssignableCmsFields(sourceFields); if (createIsEnabled) { let route = new RouteBuilder() .path(`/${schemaName}`) @@ -210,7 +313,7 @@ export function getOps( docs: { type: [ { - ...removeRequiredFields(Object.assign({}, assignableFields)), + ...removeRequiredFields(cloneConduitModel(assignableFields)), _id: { type: 'String', unique: true }, } as unknown as ArrayConduitModel, ], @@ -254,7 +357,7 @@ export function getOps( }) .bodyParams( removeRequiredFields( - Object.assign({}, assignableFields), + cloneConduitModel(assignableFields), ) as unknown as ConduitModel, ) .return(`patch${schemaName}`, actualSchema.fields) diff --git a/modules/database/src/database.proto b/modules/database/src/database.proto index 7ab8c2e91..276d2d232 100644 --- a/modules/database/src/database.proto +++ b/modules/database/src/database.proto @@ -45,6 +45,9 @@ message FindOneRequest { optional string userId = 5; optional string scope = 6; optional string readPreference = 7; + // When true, Database verifies the caller is embeddings and restricts the read. + optional bool embeddingsJob = 8; + repeated string embeddingsAllowedFields = 9; } message FindRequest { @@ -123,6 +126,10 @@ message UpdateRequest { repeated string populate = 4; optional string userId = 5; optional string scope = 6; + // When true, skip bus publication. Existing callers omit this and keep publishing. + optional bool suppressEvent = 7; + // When true, Database verifies the caller is embeddings and restricts the write. + optional bool embeddingsJob = 8; } message UpdateManyRequest { @@ -132,6 +139,8 @@ message UpdateManyRequest { repeated string populate = 4; optional string userId = 5; optional string scope = 6; + // When true, skip bus publication. Existing callers omit this and keep publishing. + optional bool suppressEvent = 7; } message DropCollectionRequest { @@ -156,6 +165,64 @@ message GetDatabaseTypeResponse { string result = 1; } +message VectorCapabilitiesRequest { + optional string schemaName = 1; +} + +message VectorCapabilitiesResponse { + bool supported = 1; + bool storage = 2; + bool indexing = 3; + bool search = 4; + string provider = 5; + optional string reason = 6; +} + +message VectorIndex { + string field = 1; + int32 dimensions = 2; + string similarity = 3; + optional string name = 4; + optional string method = 5; + repeated string filterFields = 6; + optional string options = 7; + optional string status = 8; + optional bool queryable = 9; +} + +message VectorIndexRequest { + string schemaName = 1; + VectorIndex index = 2; +} + +message VectorIndexListRequest { + string schemaName = 1; +} + +message VectorIndexListResponse { + repeated VectorIndex indexes = 1; +} + +message DeleteVectorIndexRequest { + string schemaName = 1; + string indexName = 2; +} + +message VectorSearchRequest { + string schemaName = 1; + string field = 2; + repeated double vector = 3; + optional string indexName = 4; + optional string filter = 5; + optional int32 limit = 6; + optional int32 numCandidates = 7; + optional string select = 8; + optional string userId = 9; + optional string scope = 10; + // Honored only when the verified caller is a platform operator module. + optional bool adminOperator = 11; +} + service DatabaseProvider { rpc CreateSchemaFromAdapter(CreateSchemaRequest) returns (Schema); rpc GetSchema(GetSchemaRequest) returns (Schema); @@ -182,4 +249,9 @@ service DatabaseProvider { rpc createView(CreateViewRequest) returns (google.protobuf.Empty); rpc deleteView(DeleteViewRequest) returns (google.protobuf.Empty); rpc columnExistence(ColumnExistenceRequest) returns (ColumnExistenceResponse); + rpc getVectorCapabilities(VectorCapabilitiesRequest) returns (VectorCapabilitiesResponse); + rpc createVectorIndex(VectorIndexRequest) returns (QueryResponse); + rpc getVectorIndexes(VectorIndexListRequest) returns (VectorIndexListResponse); + rpc deleteVectorIndex(DeleteVectorIndexRequest) returns (QueryResponse); + rpc vectorSearch(VectorSearchRequest) returns (QueryResponse); } diff --git a/modules/database/src/interfaces/SchemaFieldTypes.ts b/modules/database/src/interfaces/SchemaFieldTypes.ts index 7df176006..6b961e92b 100644 --- a/modules/database/src/interfaces/SchemaFieldTypes.ts +++ b/modules/database/src/interfaces/SchemaFieldTypes.ts @@ -23,7 +23,7 @@ export const SchemaField = { type: 'String', required: true, description: - 'Field type. One of: String, Number, Boolean, Date, ObjectId, JSON, Relation', + 'Field type. One of: String, Number, Boolean, Date, ObjectId, JSON, Relation, Vector', }, required: ConduitBoolean.Optional, unique: ConduitBoolean.Optional, @@ -35,6 +35,16 @@ export const SchemaField = { required: false, description: 'Required when type is "Relation". The name of the related schema.', }, + dimensions: { + type: 'Number', + required: false, + description: 'Required when type is "Vector". The number of embedding dimensions.', + }, + similarity: { + type: 'String', + required: false, + description: 'Optional for Vector fields. One of: cosine, euclidean, dotProduct.', + }, }; /** @@ -46,23 +56,26 @@ export const SchemaField = { */ export const SchemaFieldsDescription = `Object mapping field names to field definitions. -**Field Types:** String, Number, Boolean, Date, ObjectId, JSON, Relation +**Field Types:** String, Number, Boolean, Date, ObjectId, JSON, Relation, Vector **Definition Formats:** - Shorthand: \`{ fieldName: "String" }\` - Object: \`{ fieldName: { type: "String", required: true } }\` - Array: \`{ fieldName: ["String"] }\` or \`{ fieldName: [{ type: "String" }] }\` - Relation: \`{ fieldName: { type: "Relation", model: "SchemaName" } }\` +- Vector: \`{ fieldName: { type: "Vector", dimensions: 1536, similarity: "cosine", select: false } }\` (object form only; shorthand \`"Vector"\` is rejected) - Nested: \`{ fieldName: { nestedField: { type: "String" } } }\` **Field Properties:** -- \`type\` (required): String | Number | Boolean | Date | ObjectId | JSON | Relation +- \`type\` (required): String | Number | Boolean | Date | ObjectId | JSON | Relation | Vector - \`required\` (optional): boolean - Whether the field is required - \`unique\` (optional): boolean - Whether values must be unique (requires required: true) - \`select\` (optional): boolean - Whether to include in query results by default - \`default\` (optional): string - Default value for the field - \`description\` (optional): string - Field description - \`model\` (required for Relation): string - Name of the related schema +- \`dimensions\` (required for Vector): positive integer - Embedding vector dimensions +- \`similarity\` (optional for Vector): cosine | euclidean | dotProduct **Example:** \`\`\`json @@ -71,6 +84,7 @@ export const SchemaFieldsDescription = `Object mapping field names to field defi "price": { "type": "Number", "required": true }, "description": "String", "category": { "type": "Relation", "model": "Category" }, + "embedding": { "type": "Vector", "dimensions": 1536, "similarity": "cosine", "select": false }, "tags": ["String"], "metadata": { "key": "String", "value": "String" } } diff --git a/modules/database/src/models/__tests__/systemSchemas.test.ts b/modules/database/src/models/__tests__/systemSchemas.test.ts new file mode 100644 index 000000000..aba6cfa04 --- /dev/null +++ b/modules/database/src/models/__tests__/systemSchemas.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from '@jest/globals'; +import * as models from '../index.js'; +import { + DATABASE_SYSTEM_SCHEMA_NAME_SET, + DATABASE_SYSTEM_SCHEMAS, +} from '../systemSchemas.js'; + +describe('database system schema registry', () => { + const names = DATABASE_SYSTEM_SCHEMAS.map(schema => schema.name); + + it('lists the Database-owned schemas registered as system schemas', () => { + expect(names).toEqual([ + '_DeclaredSchema', + 'MigratedSchemas', + 'CustomEndpoints', + '_PendingSchemas', + 'Views', + ]); + expect([...DATABASE_SYSTEM_SCHEMA_NAME_SET]).toEqual(names); + }); + + it('stays aligned with the models barrel used for registration', () => { + const barrelNames = Object.values(models) + .filter( + (value): value is { name: string } => + typeof value === 'object' && value !== null && 'name' in value, + ) + .map(schema => schema.name) + .sort(); + expect(barrelNames).toEqual([...names].sort()); + }); +}); diff --git a/modules/database/src/models/systemSchemas.ts b/modules/database/src/models/systemSchemas.ts new file mode 100644 index 000000000..0d46cba6a --- /dev/null +++ b/modules/database/src/models/systemSchemas.ts @@ -0,0 +1,17 @@ +import { CustomEndpoints } from './CustomEndpoints.schema.js'; +import { DeclaredSchema } from './DeclaredSchema.schema.js'; +import { MigratedSchemas } from './MigratedSchemas.schema.js'; +import { PendingSchemas } from './PendingSchemas.schema.js'; +import { Views } from './Views.schema.js'; + +export const DATABASE_SYSTEM_SCHEMAS = [ + DeclaredSchema, + MigratedSchemas, + CustomEndpoints, + PendingSchemas, + Views, +] as const; + +export const DATABASE_SYSTEM_SCHEMA_NAME_SET = new Set( + DATABASE_SYSTEM_SCHEMAS.map(schema => schema.name), +); diff --git a/modules/database/src/permissions/__tests__/canModify.vectorIndex.test.ts b/modules/database/src/permissions/__tests__/canModify.vectorIndex.test.ts new file mode 100644 index 000000000..f2891b90d --- /dev/null +++ b/modules/database/src/permissions/__tests__/canModify.vectorIndex.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from '@jest/globals'; +import { TYPE } from '@conduitplatform/grpc-sdk'; +import { + canModify, + vectorIndexDeleteMutationData, + vectorIndexMutationData, +} from '../index.js'; + +const embeddingExtension = { + ownerModule: 'embeddings', + fields: { + embedding: { type: TYPE.Vector, dimensions: 3 }, + embeddingSourceHash: { type: TYPE.String, select: false }, + }, + createdAt: new Date(), + updatedAt: new Date(), +}; + +function schema(args: { + ownerModule: string; + canModify: 'Everything' | 'Nothing' | 'ExtensionOnly'; + name?: string; +}) { + return { + originalSchema: { + name: args.name ?? 'User', + ownerModule: args.ownerModule, + modelOptions: { conduit: { permissions: { canModify: args.canModify } } }, + extensions: [embeddingExtension], + }, + }; +} + +describe('createVectorIndex canModify field evaluation', () => { + it('allows embeddings to index its own extension field on ExtensionOnly schemas', async () => { + const user = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + await expect( + canModify('embeddings', user as never, vectorIndexMutationData('embedding')), + ).resolves.toBe(true); + }); + + it('denies embeddings indexing unrelated fields on ExtensionOnly schemas', async () => { + const user = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + await expect( + canModify('embeddings', user as never, vectorIndexMutationData('email')), + ).resolves.toBe(false); + await expect( + canModify('chat', user as never, vectorIndexMutationData('embedding')), + ).resolves.toBe(false); + }); + + it('preserves owner and Everything authorization without field data', async () => { + const owned = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + const open = schema({ + ownerModule: 'database', + canModify: 'Everything', + name: 'Article', + }); + await expect(canModify('authentication', owned as never)).resolves.toBe(true); + await expect( + canModify('authentication', owned as never, vectorIndexMutationData('email')), + ).resolves.toBe(true); + await expect(canModify('embeddings', open as never)).resolves.toBe(true); + await expect( + canModify('chat', open as never, vectorIndexMutationData('title')), + ).resolves.toBe(true); + }); + + it('denies ExtensionOnly callers when the vector field is missing', async () => { + const user = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + await expect(canModify('embeddings', user as never)).resolves.toBe(false); + await expect( + canModify('embeddings', user as never, vectorIndexMutationData('')), + ).resolves.toBe(false); + }); +}); + +describe('deleteVectorIndex canModify field evaluation', () => { + const liveIndexes = [ + { name: 'embedding_vector', field: 'embedding' }, + { name: 'email_1', field: 'email' }, + ]; + + it('allows embeddings to delete its own live extension index on ExtensionOnly schemas', async () => { + const user = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + await expect( + canModify( + 'embeddings', + user as never, + vectorIndexDeleteMutationData(liveIndexes, 'embedding_vector'), + ), + ).resolves.toBe(true); + }); + + it('denies unknown and non-vector index names on ExtensionOnly schemas', async () => { + const user = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + await expect( + canModify( + 'embeddings', + user as never, + vectorIndexDeleteMutationData(liveIndexes, 'missing_vector'), + ), + ).resolves.toBe(false); + await expect( + canModify( + 'embeddings', + user as never, + vectorIndexDeleteMutationData([], 'embedding_vector'), + ), + ).resolves.toBe(false); + await expect( + canModify( + 'embeddings', + user as never, + vectorIndexDeleteMutationData([{ name: 'title_idx' }], 'title_idx'), + ), + ).resolves.toBe(false); + }); + + it('denies unowned live vector indexes on ExtensionOnly schemas', async () => { + const user = schema({ ownerModule: 'authentication', canModify: 'ExtensionOnly' }); + await expect( + canModify( + 'embeddings', + user as never, + vectorIndexDeleteMutationData(liveIndexes, 'email_1'), + ), + ).resolves.toBe(false); + await expect( + canModify( + 'chat', + user as never, + vectorIndexDeleteMutationData(liveIndexes, 'embedding_vector'), + ), + ).resolves.toBe(false); + }); +}); diff --git a/modules/database/src/permissions/index.ts b/modules/database/src/permissions/index.ts index 2c5452902..8e7058e6b 100644 --- a/modules/database/src/permissions/index.ts +++ b/modules/database/src/permissions/index.ts @@ -17,6 +17,20 @@ export async function canCreate(moduleName: string, schema: Schema) { ); } +export function vectorIndexMutationData(field?: string): Indexable | undefined { + if (typeof field !== 'string' || field.length === 0) return undefined; + return { [field]: true }; +} + +export function vectorIndexDeleteMutationData( + liveIndexes: ReadonlyArray<{ name?: string; field?: string }>, + indexName: string, +) { + if (typeof indexName !== 'string' || indexName.length === 0) return undefined; + const live = liveIndexes.find(index => index.name === indexName); + return vectorIndexMutationData(live?.field); +} + export async function canModify(moduleName: string, schema: Schema, data?: Indexable) { if (moduleName === 'database' && schema.originalSchema.name === '_DeclaredSchema') return true; diff --git a/modules/embeddings/Dockerfile b/modules/embeddings/Dockerfile new file mode 100644 index 000000000..c0bfd3014 --- /dev/null +++ b/modules/embeddings/Dockerfile @@ -0,0 +1,19 @@ +# hadolint ignore=DL3006 +FROM conduit-builder + +WORKDIR /app/modules/embeddings + +COPY --from=conduit-base /app/modules/embeddings/bundle /app/modules/embeddings/bundle +COPY --from=conduit-base /app/modules/embeddings/package.bundle.json /app/modules/embeddings/package.json +COPY --from=conduit-base /app/modules/embeddings/package.bundle-lock.json /app/modules/embeddings/package-lock.json + +RUN npm ci --omit=dev --ignore-scripts && npm cache clean --force + +ENV NODE_ENV=production +ENV CONDUIT_SERVER=conduit_server +ENV SERVICE_URL=0.0.0.0:5000 +ENV GRPC_PORT=5000 + +EXPOSE 5000 + +CMD ["node", "bundle/index.js"] diff --git a/modules/embeddings/README.md b/modules/embeddings/README.md new file mode 100644 index 000000000..f1731704e --- /dev/null +++ b/modules/embeddings/README.md @@ -0,0 +1,92 @@ +# Embeddings Module + +The Embeddings module owns text-to-vector generation, embedding configuration, +backfills, and semantic search by text. The Database module remains responsible +for vector storage, index creation, and vector-in/vector-out search. + +## Configuration + +The module convict `enabled` setting is `false` by default. Production +deployments require a non-empty `GRPC_KEY` (`NODE_ENV=production` in the +image). The module is **not** included in the standalone image for the first +production release; run it as a separate opt-in compose profile or Helm +workload after a compatible image tag is published. Helm +`install.embeddings.enabled` only deploys the process; it does not set convict +`enabled`. + +Enable it and configure an OpenAI-compatible provider: + +```json +{ + "enabled": true, + "defaultProvider": "openai-compatible", + "providers": { + "openai-compatible": { + "endpoint": "https://api.openai.com/v1/embeddings", + "apiKey": "...", + "models": [{ "name": "text-embedding-3-small", "dimensions": 1536 }], + "defaultModel": "text-embedding-3-small" + } + }, + "queue": { + "concurrency": 2, + "attempts": 3 + } +} +``` + +## Workflow + +1. Create an embedding config with `schemaName`, `sourceFields`, and + `targetField`. `provider`, `model`, and `dimensions` default from the + provider catalogue when omitted. The first upsert provisions the vector + index when Database indexing is available. The config stays disabled until + Database reports a queryable index. If indexing is unavailable, status + returns a manual index lifecycle warning. +2. The module adds a vector schema extension for the target field and a source + hash field used to skip unchanged documents. +3. Start a backfill, or rely on database create/update events to enqueue + incremental embedding jobs. Backfills persist `BackfillRun` state and can be + canceled or resumed from the stored cursor. +4. Use `semanticSearch` to generate a query embedding and delegate search to the + Database module. + +Provider output dimensions must match the configured vector dimensions. Mismatches +fail before vectors are written or searched. + +## Admin and MCP + +Operator-only Admin routes are registered under `/embeddings/*` and become MCP +tools through Hermes: + +- `GET /embeddings/configs` +- `POST /embeddings/configs` +- `GET /embeddings/capabilities` +- `GET /embeddings/status` +- `POST /embeddings/backfills` +- `POST /embeddings/backfills/:id/cancel` +- `POST /embeddings/backfills/:id/resume` +- `POST /embeddings/search` + +Config and backfill APIs are never exposed as client routes. Client +`POST /embeddings/search` accepts text only and takes user/scope from the +authenticated router context. + +## Packaging + +- Bake target: `embeddings` (BullMQ is an extra bundle dependency). The image + is not published until a compatible release; do not pull + `docker.io/conduitplatform/embeddings:latest` until that tag exists. +- Compose: export a non-empty `GRPC_KEY`, then + `docker compose --profile embeddings up` (gRPC + `${EMBEDDINGS_GRPC_PORT:-55165}`, metrics `9192`). +- Standalone v1 does not ship embeddings. +- Helm `install.embeddings.enabled` (charts repo) deploys the workload only. + Module convict `enabled` (default false) is a separate Core config switch + for workers and search. + +Operator rollout, capability/index readiness, and rollback: +[deploy/embeddings.md](../../deploy/embeddings.md). + +Live Atlas/pgvector/provider behavior is not covered by CI. Repeat the +capability and index checks in the target environment before activation. diff --git a/modules/embeddings/build.sh b/modules/embeddings/build.sh new file mode 100644 index 000000000..41bfc3986 --- /dev/null +++ b/modules/embeddings/build.sh @@ -0,0 +1,19 @@ +rm -rf ./src/protoTypes +mkdir ./src/protoTypes + +cp ./src/*.proto ./src/protoTypes + +cd ./src/protoTypes || exit + +echo "Generating typescript code" +protoc \ + --plugin=protoc-gen-ts_proto=../../node_modules/.bin/protoc-gen-ts_proto \ + --ts_proto_opt=esModuleInterop=true \ + --ts_proto_opt=outputServices=generic-definitions,useExactTypes=false \ + --ts_proto_out=./ \ + --ts_proto_opt=importSuffix=.js \ + --ts_proto_opt=snakeToCamel=false \ + ./*.proto + +echo "Cleaning up folders" +rm -rf ./*.proto diff --git a/modules/embeddings/package.bundle-lock.json b/modules/embeddings/package.bundle-lock.json new file mode 100644 index 000000000..a6d1c6fc7 --- /dev/null +++ b/modules/embeddings/package.bundle-lock.json @@ -0,0 +1,2638 @@ +{ + "name": "@conduitplatform/embeddings", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@conduitplatform/embeddings", + "version": "1.0.0", + "dependencies": { + "@bufbuild/protobuf": "^2.10.2", + "@grpc/grpc-js": "^1.14.3", + "@grpc/proto-loader": "^0.8.0", + "@sesamecare-oss/redlock": "^1.4.0", + "abort-controller-x": "^0.5.0", + "axios": "1.18.0", + "bullmq": "^5.21.2", + "convict": "^6.2.5", + "escape-string-regexp": "1.0.5", + "express": "^5.2.1", + "fast-jwt": "^6.2.4", + "fs-extra": "^11.3.5", + "ioredis": "^5.10.1", + "lodash": "^4.18.1", + "lodash-es": "^4.18.1", + "nice-grpc": "^2.1.17", + "nice-grpc-client-middleware-retry": "^3.1.16", + "nice-grpc-common": "^2.0.4", + "prom-client": "^15.1.3", + "protobufjs": "^8.7.2", + "snappy": "7.4.1", + "uuid": "14.0.2", + "winston": "^3.19.0", + "winston-loki": "^6.1.7" + }, + "engines": { + "node": ">=24" + } + }, + "node_modules/@bufbuild/protobuf": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.14.1.tgz", + "integrity": "sha512-agRJn3+EJDUe8AvxTx/LnHA/GErvLE62pSaSk7+MwFOtOv8eWBu/qCq2qoZjBjVZ3C2aiFJCveuSs17KMkYGOw==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, + "node_modules/@colors/colors": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.1.tgz", + "integrity": "sha512-dTmUJzXSuayBK+hZydEaXd2mhx61qWQwkwaBBY6LyEOVx/L9aQU5ac8eFNEsd9nrD1+zb9zvDCphLSe8g1F4Qw==", + "license": "MIT", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@dabh/diagnostics": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.9.tgz", + "integrity": "sha512-R6siwR65Hm+3yfgP7o8DKhNvputQAwfoz9zTc3kyDudnomj2/BcLmD+uGQQPICjuFUp8ounPBU+jmKsocwVVAg==", + "license": "MIT", + "dependencies": { + "@so-ric/colorspace": "^1.1.6", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.4.tgz", + "integrity": "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@grpc/proto-loader/node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "license": "MIT" + }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/@lukeed/ms": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@lukeed/ms/-/ms-2.0.2.tgz", + "integrity": "sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.4.tgz", + "integrity": "sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.4.tgz", + "integrity": "sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.4.tgz", + "integrity": "sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.4.tgz", + "integrity": "sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.4.tgz", + "integrity": "sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.4.tgz", + "integrity": "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@napi-rs/snappy-android-arm-eabi": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.4.1.tgz", + "integrity": "sha512-7siGMYnpi4pjI07XXoIgXlBkIbI/XsmXYMi+dSPEZz/7V/f/4MvK9OGGKT4cgHLiszgxkTSTZzL+AjCF7OACtQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-android-arm64": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.4.1.tgz", + "integrity": "sha512-8tcrG2V3LSzCS3OdppuvalNbsgsvyr8hIkUliPBaFzjOiAjRpCOyS05s7KZe7YdkJyAht/OCTjIV1XqKdAUJXw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-darwin-arm64": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.4.1.tgz", + "integrity": "sha512-imzAEKEySv3dmzMFCnVj0SeaVbaZptlVGjQXLuIQ1n157/DpqJYuG4Yj1pPIRh3S/VzniC3ewPmYQWfxWepRsA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-darwin-x64": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.4.1.tgz", + "integrity": "sha512-Q9LDalgq5uqpd1JwpWxKb34rmheOmk7BhlwUMadaECAZU/v5HwArAZjsWomRXiYm4tfXFnQeL3ItFGW0Irv6+A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-freebsd-x64": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.4.1.tgz", + "integrity": "sha512-O1B/ynTjOOlnr6dnVHdj/xrYcPcWLjZPC3dNWyOy7xGbkjQHHYiTt0EXDxmfANALt9YEa1rSfzICMmKcyFCtnQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm-gnueabihf": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.4.1.tgz", + "integrity": "sha512-8vY+IGu1qm2EEuqEvanMh73edIFlRD19HeXMSEZZLOzJeIhlOcMGCAqKApPJSgdwMf7nxNmnBYyXnH+QM1ryQA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm64-gnu": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.4.1.tgz", + "integrity": "sha512-FRjLCPfbtmr4B4OwR0gIEAKnnaczyLKL7QtG3iAn5+n4g284Fur1RoIGTCloq7zGKAuPbKBRhrARCOrXr7nx3A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm64-musl": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-musl/-/snappy-linux-arm64-musl-7.4.1.tgz", + "integrity": "sha512-seZSp/mCSOMZ/ve29s3bAhnLTPpwUZmf9N7+HCCndVckbS6lzRfasstn4RDWP2fVn0MoGbCDFAHADrGP55dzPQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-ppc64-gnu": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-ppc64-gnu/-/snappy-linux-ppc64-gnu-7.4.1.tgz", + "integrity": "sha512-vBguLu+Dc3J7pwn6QXYDK1iPgGSoqfSig7O1FQMg7v9iUxexR7IAna7vKMTprMHP8FDM9szxcYOzEMrNGmcdaA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-riscv64-gnu": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-riscv64-gnu/-/snappy-linux-riscv64-gnu-7.4.1.tgz", + "integrity": "sha512-UmgLvreYF6NWV/cBOwZp25dOm5uxiOTC8Rj+vFdOAx8whi/jiwui/pbxjjdUfwBQutzxFk3nMOSC5Jq6TgAZTA==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-s390x-gnu": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-s390x-gnu/-/snappy-linux-s390x-gnu-7.4.1.tgz", + "integrity": "sha512-mXTqUnLUMZeYiSMT6wRiDEmqLDm9VJXJhhtKExqK2sYiTPRGQfn1NR/qzv192LlOsXikIRvY6jWMq4Ron0LA1Q==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-x64-gnu": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.4.1.tgz", + "integrity": "sha512-D+2LJTgAAv10SRZcX9rdmdx/rgqU+Lp1oJvZAMkiL7fhkFeHmKRHND8DOkMyAdcVi2ES7EpZnUJBXdcYqxWxjQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-x64-musl": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-musl/-/snappy-linux-x64-musl-7.4.1.tgz", + "integrity": "sha512-2DgFW7mQ9cy0EtiddIOuEk9rPUCnt8xv875AYtEjjPrQArXbgVmB/Lt6ngbTlckv5Xgb0Cx4lt7R7ahWXcTbxw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-openharmony-arm64": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-openharmony-arm64/-/snappy-openharmony-arm64-7.4.1.tgz", + "integrity": "sha512-sF1LJRTvZn2A3AyrZUEr6zpNEQW5mm9GO09clzd0nQJlNKoPbWaLusv+nhZYTBxEVLFApISp1ggL8yPNkIpjvA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-win32-arm64-msvc": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.4.1.tgz", + "integrity": "sha512-sqZbbT1yKlKp1LPO4Yqkja6He0txXOvWKANKZJM98vvHqQikLayq4X6ogovzVXOFYzmpkHFdT10NfGwh4hZbog==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-win32-ia32-msvc": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.4.1.tgz", + "integrity": "sha512-RrgbOcrH52k+dFfuey2wm98OxNmNayI4qyGZ1J36pGWMEOemft+g10zsVzHkN86J5D1RWcD6z25SzB87eW3SEQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-win32-x64-msvc": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.4.1.tgz", + "integrity": "sha512-n8OpxSCvNr1QPmXquDkMatYQaG80pjXrmgPuwvbXbGTr8BbZay6wZWlAE0OW32mEBrGWGqLUs4uheIJdF1cLVA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@sesamecare-oss/redlock": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@sesamecare-oss/redlock/-/redlock-1.4.0.tgz", + "integrity": "sha512-2z589R+yxKLN4CgKxP1oN4dsg6Y548SE4bVYam/R0kHk7Q9VrQ9l66q+k1ehhSLLY4or9hcchuF9/MhuuZdjJg==", + "license": "UNLICENSED", + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "ioredis": ">=5" + } + }, + "node_modules/@so-ric/colorspace": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@so-ric/colorspace/-/colorspace-1.1.6.tgz", + "integrity": "sha512-/KiKkpHNOBgkFJwu9sh48LkHSMYGyuTcSFK/qMBdnOAlrRJzRSXAOFB5qwzaVQuDl8wAvHVMkaASQDReTahxuw==", + "license": "MIT", + "dependencies": { + "color": "^5.0.2", + "text-hex": "1.0.x" + } + }, + "node_modules/@types/node": { + "version": "22.20.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.2.tgz", + "integrity": "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/triple-beam": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", + "license": "MIT" + }, + "node_modules/abort-controller-x": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/abort-controller-x/-/abort-controller-x-0.5.0.tgz", + "integrity": "sha512-yTt9CI0x+nRfX6BFMenEGP8ooPvErGH6AbFz20C2IeOLIlDsrw/VHpgne3GsCEuTA410IiFiaLVFKmgM4bKEPQ==", + "license": "MIT" + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/ansi-styles/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/ansi-styles/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "license": "MIT", + "dependencies": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/async-exit-hook": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/async-exit-hook/-/async-exit-hook-2.0.1.tgz", + "integrity": "sha512-NW2cX8m1Q7KPA7a5M2ULQeZ2wR5qI5PAbw5L0UOMxdioVk9PMZ0h1TmyZEkPYrCvYjDlFICusOu1dlEKAAeXBw==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.18.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.18.0.tgz", + "integrity": "sha512-E32NzpYKp++W7XRe52rHiXV2ehxmh3wbdgO7MHeFM+vqxLBYHzt0ElkiImtOBxtOmyp0yoC8C6uESVV84Y2/hw==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.16.0", + "form-data": "^4.0.5", + "https-proxy-agent": "^5.0.1", + "proxy-from-env": "^2.1.0" + } + }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, + "node_modules/bn.js": { + "version": "4.12.5", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.5.tgz", + "integrity": "sha512-3aRg6/JxfffFD+OlOjOFR3Vo79l39ooBTFucxx+MT3dhCtzn3EmiUPQo+6/OZuI2jbXi3YKgmiTFBgChQMwIRQ==", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/btoa": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", + "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==", + "license": "(MIT OR Apache-2.0)", + "bin": { + "btoa": "bin/btoa.js" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/bullmq": { + "version": "5.81.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.81.4.tgz", + "integrity": "sha512-n+WHSzz20KooBGoJyASre6oJNz/p5f1IJRRN2ibD+NWPQaNpNQmDrwYuPO+bsXrQeM1MQzUxXGbdjmyOFKP2xQ==", + "license": "MIT", + "dependencies": { + "cron-parser": "4.9.0", + "ioredis": "5.11.1", + "msgpackr": "2.0.5", + "node-abort-controller": "3.1.1", + "semver": "7.8.5", + "tslib": "2.8.1" + }, + "engines": { + "node": ">=12.22.0" + }, + "peerDependencies": { + "redis": ">=5.0.0" + }, + "peerDependenciesMeta": { + "redis": { + "optional": true + } + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/color/-/color-5.0.3.tgz", + "integrity": "sha512-ezmVcLR3xAVp8kYOm4GS45ZLLgIE6SPAFoduLr6hTDajwb3KZ2F46gulK3XpcwRFb5KKGCSezCBAY4Dw4HsyXA==", + "license": "MIT", + "dependencies": { + "color-convert": "^3.1.3", + "color-string": "^2.1.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/color-convert": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-3.1.3.tgz", + "integrity": "sha512-fasDH2ont2GqF5HpyO4w0+BcewlhHEZOFn9c1ckZdHpJ56Qb7MHhH/IcJZbBGgvdtwdwNbLvxiBEdg336iA9Sg==", + "license": "MIT", + "dependencies": { + "color-name": "^2.0.0" + }, + "engines": { + "node": ">=14.6" + } + }, + "node_modules/color-name": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-2.1.1.tgz", + "integrity": "sha512-p2FdgwVx1a9yWBHP2wI0VgShkDpgN4kZISkxdNipGBJWpa5G6b04OINlVWCyJj0JmfvcPrgqt95E9k8yvaOJFg==", + "license": "MIT", + "engines": { + "node": ">=12.20" + } + }, + "node_modules/color-string": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-2.1.4.tgz", + "integrity": "sha512-Bb6Cq8oq0IjDOe8wJmi4JeNn763Xs9cfrBcaylK1tPypWzyoy2G3l90v9k64kjphl/ZJjPIShFztenRomi8WTg==", + "license": "MIT", + "dependencies": { + "color-name": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/convict": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/convict/-/convict-6.2.5.tgz", + "integrity": "sha512-JtXpxqDqJ8P0UwEHwhxLzCIXQy97vlYBZR222Sbzb1q1Erex9ASrztJ29SyhWFQjod1AeFBaPzEEC8YvtZMIYg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.clonedeep": "^4.5.0", + "yargs-parser": "^20.2.7" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "deprecated": "v4 is no longer maintained, upgrade to v5", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/fast-jwt": { + "version": "6.3.3", + "resolved": "https://registry.npmjs.org/fast-jwt/-/fast-jwt-6.3.3.tgz", + "integrity": "sha512-pQDXx7IHeZT4jSmpE9o80RrBqfrG4fPrl8anazSM5vErIdK1iCc13z/EWX+H0j7liWSRnwTpHswIKMeLYGAckw==", + "license": "Apache-2.0", + "dependencies": { + "@lukeed/ms": "^2.0.2", + "asn1.js": "^5.4.1", + "ecdsa-sig-formatter": "^1.0.11", + "mnemonist": "^0.40.0", + "safe-regex2": "^5.1.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", + "license": "MIT" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==", + "license": "MIT" + }, + "node_modules/follow-redirects": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.4", + "mime-types": "^2.1.35" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/form-data/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/fs-extra": { + "version": "11.4.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.0.tgz", + "integrity": "sha512-EQsFzMUJkCKGr1ePqlYADkIUmHW1s3ZXr5Yqy6wbGrfUCphpl2maM/kyOIRA2HpP3AaFQTZXD4ldjek+nccddA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jsonfile": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", + "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", + "license": "MIT" + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, + "node_modules/lodash-es": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", + "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", + "license": "MIT" + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==", + "license": "MIT" + }, + "node_modules/logform": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.7.0.tgz", + "integrity": "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ==", + "license": "MIT", + "dependencies": { + "@colors/colors": "1.6.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/logform/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "license": "MIT", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", + "license": "ISC" + }, + "node_modules/mnemonist": { + "version": "0.40.4", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.40.4.tgz", + "integrity": "sha512-ZAv+KNavneRVzu4tUeOgzkScI3W5BGwZ3rkxIpKtzzVgfTtWQFN1CgX0U72cyvyh3iTuHL3SiSmrQxTlryEIcw==", + "license": "MIT", + "dependencies": { + "obliterator": "^2.0.4" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/msgpackr": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-2.0.5.tgz", + "integrity": "sha512-cef05H/dSYpLpqp3sj/qyZh5vhUYCalnaLO7j1yOmpsR0y/XwLVtK7r5gn+U/F7CTEfMowcGhlUQJDLcLf7jcA==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.4" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.4.tgz", + "integrity": "sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.4" + } + }, + "node_modules/negotiator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/negotiator/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/nice-grpc": { + "version": "2.1.17", + "resolved": "https://registry.npmjs.org/nice-grpc/-/nice-grpc-2.1.17.tgz", + "integrity": "sha512-pu9xYPlWSeqoYQOCqb2ftQqZi/P2DaI70PSuwnxvoyIRiilaOVtktyPe6LmuXegWB4Ic1sPuDGCnCCASbwzK0w==", + "license": "MIT", + "dependencies": { + "@grpc/grpc-js": "^1.14.0", + "abort-controller-x": "^0.5.0", + "nice-grpc-common": "^2.0.4" + } + }, + "node_modules/nice-grpc-client-middleware-retry": { + "version": "3.1.16", + "resolved": "https://registry.npmjs.org/nice-grpc-client-middleware-retry/-/nice-grpc-client-middleware-retry-3.1.16.tgz", + "integrity": "sha512-8EbOCUZZ1Uq1pCfTD8dB2zMU1AYrilRvm9al5VSUi444eqhPhxElnI7iqP6ZY/ixb1NGsvMpfbU5FW/3L7DdFw==", + "license": "MIT", + "dependencies": { + "abort-controller-x": "^0.5.0", + "nice-grpc-common": "^2.0.4" + } + }, + "node_modules/nice-grpc-common": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/nice-grpc-common/-/nice-grpc-common-2.0.4.tgz", + "integrity": "sha512-gOEXlD6ShXZMZ8k+49wm/bA2j1+3IKbEFV9WYREJcvZV4kM5L1KkILYTX9VYBzZF2OuYCe1wp8c82bQkvR0fHw==", + "license": "MIT", + "dependencies": { + "ts-error": "^1.0.6" + } + }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", + "license": "MIT" + }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/obliterator": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.5.tgz", + "integrity": "sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==", + "license": "MIT" + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "license": "MIT", + "dependencies": { + "fn.name": "1.x.x" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "deprecated": "prom-client has been replaced by @prometheus-io/client", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, + "node_modules/protobufjs": { + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.8.0.tgz", + "integrity": "sha512-N3xhQ5yyBx3vQq4gubBfASzYhJGNzeDbjqBpu61g7UVylsN/qyffU96TKWD3GbbLOKF82VGNRNvv1+BFgE31Eg==", + "license": "BSD-3-Clause", + "dependencies": { + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/proxy-from-env": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", + "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/qs": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ret": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.5.0.tgz", + "integrity": "sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-regex2": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/safe-regex2/-/safe-regex2-5.1.1.tgz", + "integrity": "sha512-mOSBvHGDZMuIEZMdOz/aCEYDCv0E7nfcNsIhUF+/P+xC7Hyf3FkvymqgPbg9D1EdSGu+uKbJgy09K/RKKc7kJA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "ret": "~0.5.0" + }, + "bin": { + "safe-regex2": "bin/safe-regex2.js" + } + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/snappy": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/snappy/-/snappy-7.4.1.tgz", + "integrity": "sha512-Em7vzkTe3d2K4BHhEoKqU3Xz6cZnVuAmx66/iMlly7KZDBIviIRcNRejsvWCQg+/snpalbh1mWZfIKzCfFyiHg==", + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/snappy-android-arm-eabi": "7.4.1", + "@napi-rs/snappy-android-arm64": "7.4.1", + "@napi-rs/snappy-darwin-arm64": "7.4.1", + "@napi-rs/snappy-darwin-x64": "7.4.1", + "@napi-rs/snappy-freebsd-x64": "7.4.1", + "@napi-rs/snappy-linux-arm-gnueabihf": "7.4.1", + "@napi-rs/snappy-linux-arm64-gnu": "7.4.1", + "@napi-rs/snappy-linux-arm64-musl": "7.4.1", + "@napi-rs/snappy-linux-ppc64-gnu": "7.4.1", + "@napi-rs/snappy-linux-riscv64-gnu": "7.4.1", + "@napi-rs/snappy-linux-s390x-gnu": "7.4.1", + "@napi-rs/snappy-linux-x64-gnu": "7.4.1", + "@napi-rs/snappy-linux-x64-musl": "7.4.1", + "@napi-rs/snappy-openharmony-arm64": "7.4.1", + "@napi-rs/snappy-win32-arm64-msvc": "7.4.1", + "@napi-rs/snappy-win32-ia32-msvc": "7.4.1", + "@napi-rs/snappy-win32-x64-msvc": "7.4.1" + } + }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tdigest": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.3.tgz", + "integrity": "sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + }, + "node_modules/text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", + "license": "MIT" + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/triple-beam": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/ts-error": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/ts-error/-/ts-error-1.0.6.tgz", + "integrity": "sha512-tLJxacIQUM82IR7JO1UUkKlYuUTmoY9HBJAmNWFzheSlDS5SPMcNIepejHJa4BpPQLAcbRhRf3GDJzyj6rbKvA==", + "license": "MIT" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/url-polyfill": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/url-polyfill/-/url-polyfill-1.1.14.tgz", + "integrity": "sha512-p4f3TTAG6ADVF3mwbXw7hGw+QJyw5CnNGvYh5fCuQQZIiuKUswqcznyV3pGDP9j0TSmC4UvRKm8kl1QsX1diiQ==", + "license": "MIT" + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/uuid": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.2.tgz", + "integrity": "sha512-xZe/16rV4aa+HGSOCiY2YeLT1OybRLrrkL/Rqaq7p7GMVXjFh+6wN4oMYgjFmnSnhY8t6Xpdl2l9qmnHYuMHwQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/winston": { + "version": "3.19.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz", + "integrity": "sha512-LZNJgPzfKR+/J3cHkxcpHKpKKvGfDZVPS4hfJCc4cCG0CgYzvlD6yE/S3CIL/Yt91ak327YCpiF/0MyeZHEHKA==", + "license": "MIT", + "dependencies": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.8", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.7.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.9.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-loki": { + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/winston-loki/-/winston-loki-6.1.7.tgz", + "integrity": "sha512-QxwyQJezn3ZmDx1AsngecxwnZ8SoVuJ8hW4LtK5CA+9eyswgH4+AE93fY7t/Ub6s/zu4KDP1qB9Lxb8tO8vvAw==", + "license": "MIT", + "dependencies": { + "async-exit-hook": "2.0.1", + "btoa": "^1.2.1", + "protobufjs": "^7.2.4", + "url-polyfill": "^1.1.12", + "winston-transport": "^4.3.0" + }, + "optionalDependencies": { + "snappy": "^7.2.2" + } + }, + "node_modules/winston-loki/node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/winston-transport": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.9.0.tgz", + "integrity": "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A==", + "license": "MIT", + "dependencies": { + "logform": "^2.7.0", + "readable-stream": "^3.6.2", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + } + } +} diff --git a/modules/embeddings/package.bundle.json b/modules/embeddings/package.bundle.json new file mode 100644 index 000000000..d3bc4bdeb --- /dev/null +++ b/modules/embeddings/package.bundle.json @@ -0,0 +1,49 @@ +{ + "name": "@conduitplatform/embeddings", + "version": "1.0.0", + "private": true, + "type": "module", + "main": "bundle/index.js", + "engines": { + "node": ">=24" + }, + "conduit": { + "peers": { + "await": [ + "database" + ], + "watch": [ + { + "module": "database", + "edge": "rising" + } + ] + } + }, + "dependencies": { + "@bufbuild/protobuf": "^2.10.2", + "@grpc/grpc-js": "^1.14.3", + "@grpc/proto-loader": "^0.8.0", + "@sesamecare-oss/redlock": "^1.4.0", + "abort-controller-x": "^0.5.0", + "axios": "1.18.0", + "convict": "^6.2.5", + "escape-string-regexp": "1.0.5", + "express": "^5.2.1", + "fast-jwt": "^6.2.4", + "fs-extra": "^11.3.5", + "ioredis": "^5.10.1", + "lodash": "^4.18.1", + "lodash-es": "^4.18.1", + "nice-grpc": "^2.1.17", + "nice-grpc-client-middleware-retry": "^3.1.16", + "nice-grpc-common": "^2.0.4", + "prom-client": "^15.1.3", + "protobufjs": "^8.7.2", + "snappy": "7.4.1", + "uuid": "14.0.2", + "winston": "^3.19.0", + "winston-loki": "^6.1.7", + "bullmq": "^5.21.2" + } +} diff --git a/modules/embeddings/package.json b/modules/embeddings/package.json new file mode 100644 index 000000000..6d2141157 --- /dev/null +++ b/modules/embeddings/package.json @@ -0,0 +1,52 @@ +{ + "name": "@conduitplatform/embeddings", + "version": "1.0.0", + "description": "", + "private": true, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "type": "module", + "engines": { + "node": ">=24" + }, + "conduit": { + "peers": { + "await": ["database"], + "watch": [{ "module": "database", "edge": "rising" }] + } + }, + "scripts": { + "start": "node dist/index.js", + "start:bundle": "node bundle/index.js", + "prebuild": "npm run generateTypes", + "build": "rimraf dist && tsc", + "postbuild": "copyfiles -u 1 src/**/*.proto src/*.proto ./dist/", + "prebuild:bundle": "pnpm --filter @conduitplatform/service-bundle run build", + "build:bundle": "rimraf bundle && node ../../libraries/service-bundle/dist/cli.js generate-manifest && tsup && node ../../libraries/service-bundle/dist/cli.js copy-assets && node ../../libraries/service-bundle/dist/cli.js generate-lockfile", + "generateTypes": "sh build.sh", + "test": "npx tsc -p tsconfig.test.json && node --test dist-test/utils/*.test.js dist-test/controllers/*.test.js dist-test/providers/*.test.js dist-test/api/*.test.js test/embedding-contract.test.mjs test/deployment-contract.test.mjs", + "build:docker": "docker build -t ghcr.io/conduitplatform/embeddings:latest -f ./Dockerfile ../../ && docker push ghcr.io/conduitplatform/embeddings:latest" + }, + "dependencies": { + "@bufbuild/protobuf": "^2.12.0", + "@conduitplatform/grpc-sdk": "workspace:*", + "@conduitplatform/module-tools": "workspace:*", + "@grpc/grpc-js": "^1.14.4", + "@grpc/proto-loader": "^0.8.1", + "bullmq": "^5.79.0", + "convict": "^6.2.5", + "ioredis": "^5.11.1", + "lodash-es": "^4.18.1" + }, + "devDependencies": { + "@conduitplatform/service-bundle": "workspace:*", + "@types/convict": "^6.1.6", + "@types/lodash-es": "^4.17.12", + "@types/node": "24.13.4", + "copyfiles": "^2.4.1", + "rimraf": "^6.1.3", + "ts-proto": "^2.12.3", + "tsup": "^8.5.1", + "typescript": "~6.0.3" + } +} diff --git a/modules/embeddings/service-bundle.config.json b/modules/embeddings/service-bundle.config.json new file mode 100644 index 000000000..18feb0810 --- /dev/null +++ b/modules/embeddings/service-bundle.config.json @@ -0,0 +1,5 @@ +{ + "extraDependencies": [ + "bullmq" + ] +} diff --git a/modules/embeddings/src/Embeddings.ts b/modules/embeddings/src/Embeddings.ts new file mode 100644 index 000000000..fd995b6bc --- /dev/null +++ b/modules/embeddings/src/Embeddings.ts @@ -0,0 +1,609 @@ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + ConduitGrpcSdk, + DatabaseProvider, + GrpcRequest, + GrpcResponse, + HealthCheckStatus, +} from '@conduitplatform/grpc-sdk'; +import { + ConfigController, + ConduitActiveSchema, + ManagedModule, +} from '@conduitplatform/module-tools'; +import AppConfigSchema, { Config } from './config/index.js'; +import * as models from './models/index.js'; +import { BackfillRun, EmbeddingConfig } from './models/index.js'; +import { QueueController } from './controllers/queue.controller.js'; +import { getProvider, hashEmbeddingInput } from './providers/index.js'; +import { + embeddingOwnedFields, + isEmbeddingOwnedMutation, + parseBoundedMutationEvent, +} from './utils/mutationEvents.js'; +import { + buildEmbeddingDocumentSelect, + generateEmbeddingsForDocument, + sourceHashField, +} from './utils/processEmbedding.js'; +import { + MAX_QUEUE_BATCH_SIZE, + parseEmbeddingJobData, + type EmbeddingJobData, +} from './utils/embeddingJobs.js'; +import { + assertGrpcKeyRequirement, + callerModuleName, +} from './utils/productionSecurity.js'; +import { + normalizeEmbeddingsConfig, + resolveProviderModelName, +} from './utils/providerConfig.js'; +import { sanitizeErrorMessage } from './utils/redactConfig.js'; +import { + applyBackfillJobOutcome, + backfillRunFromDocument, + persistableBackfillRun, + processBackfillControllerJob, + type BackfillControllerJobData, +} from './utils/backfillExecution.js'; +import { toBackfillCountUpdateQuery } from './utils/backfillRun.js'; +import { incrementEmbeddingMetric } from './utils/embeddingMetrics.js'; +import metricsSchema from './metrics/index.js'; +import { EmbeddingsApi, type DeclaredSchemaInfo } from './api/embeddingsApi.js'; +import { AdminHandlers } from './admin/index.js'; +import { EmbeddingsRoutes } from './routes/index.js'; +import { + CancelBackfillRequest, + DeleteEmbeddingConfigRequest, + DeleteEmbeddingConfigResponse, + GetBackfillRequest, + GetCapabilitiesRequest, + GetCapabilitiesResponse, + GetConfigsRequest, + GetConfigsResponse, + GetStatusRequest, + GetStatusResponse, + ListBackfillsRequest, + ListBackfillsResponse, + ResumeBackfillRequest, + SemanticSearchRequest, + SemanticSearchResponse, + StartBackfillRequest, + StartBackfillResponse, + UpsertConfigRequest, + UpsertConfigResponse, + BackfillMutationResponse, + BackfillRun as BackfillRunMessage, +} from './protoTypes/embeddings.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +export default class EmbeddingsModule extends ManagedModule { + configSchema = AppConfigSchema; + protected metricsSchema = metricsSchema; + service = { + protoPath: path.resolve(__dirname, 'embeddings.proto'), + protoDescription: 'embeddings.EmbeddingsProvider', + functions: { + upsertConfig: this.upsertConfig.bind(this), + getConfigs: this.getConfigs.bind(this), + deleteConfig: this.deleteConfig.bind(this), + getCapabilities: this.getCapabilities.bind(this), + getStatus: this.getStatus.bind(this), + startBackfill: this.startBackfill.bind(this), + getBackfill: this.getBackfill.bind(this), + listBackfills: this.listBackfills.bind(this), + cancelBackfill: this.cancelBackfill.bind(this), + resumeBackfill: this.resumeBackfill.bind(this), + semanticSearch: this.semanticSearch.bind(this), + }, + }; + + private database: DatabaseProvider; + private queueController: QueueController; + private subscribedSchemas = new Map(); + private api: EmbeddingsApi; + private adminRouter?: AdminHandlers; + private clientRouter?: EmbeddingsRoutes; + private routerWatchDispose?: () => void; + + constructor(peerManifestRoot?: string) { + super('embeddings', peerManifestRoot); + this.updateHealth(HealthCheckStatus.UNKNOWN, true); + } + + async onServerStart() { + assertGrpcKeyRequirement(process.env); + await this.awaitPeersFromManifest(); + this.database = this.grpcSdk.database!; + await this.registerSchemas(); + this.queueController = QueueController.getInstance(this.grpcSdk); + this.api = this.createApi(); + this.adminRouter = new AdminHandlers(this.grpcServer, this.grpcSdk, this.api); + await this.configureRuntime(); + this.updateHealth(HealthCheckStatus.SERVING); + } + + async preConfig(config: Config) { + assertGrpcKeyRequirement(process.env); + return normalizeEmbeddingsConfig(config); + } + + async onConfig() { + if (!this.database) return; + await this.configureRuntime(); + } + + async onRegister() { + this.routerWatchDispose = this.grpcSdk.watchPeer( + 'router', + serving => { + if (serving) void this.ensureClientRoutes(); + }, + { edge: 'rising', syncInitialState: true }, + ); + } + + async upsertConfig( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.upsertConfig(call.request, { + callerModule: callerModuleName(call.metadata), + }); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async getConfigs( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.getConfigs(call.request, { + callerModule: callerModuleName(call.metadata), + }); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async deleteConfig( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.deleteConfig(call.request, { + callerModule: callerModuleName(call.metadata), + }); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async getCapabilities( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.getCapabilities(call.request.schemaName); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async getStatus( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.getStatus(call.request.schemaName); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async startBackfill( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.startBackfill(call.request, { + callerModule: callerModuleName(call.metadata), + }); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async getBackfill( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.getBackfill(call.request.id, { + callerModule: callerModuleName(call.metadata), + }); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async listBackfills( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.listBackfills(call.request, { + callerModule: callerModuleName(call.metadata), + }); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async cancelBackfill( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.cancelBackfill(call.request.id, { + callerModule: callerModuleName(call.metadata), + }); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async resumeBackfill( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.resumeBackfill(call.request.id, { + callerModule: callerModuleName(call.metadata), + }); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + async semanticSearch( + call: GrpcRequest, + callback: GrpcResponse, + ) { + try { + const result = await this.api.semanticSearch(call.request, { + callerModule: callerModuleName(call.metadata), + }); + callback(null, result); + } catch (err) { + callback(this.api.mapGrpcError(err)); + } + } + + private async ensureClientRoutes() { + if (!this.api || !this.grpcSdk.router) return; + this.clientRouter ??= new EmbeddingsRoutes(this.grpcServer, this.grpcSdk, this.api); + await this.clientRouter.registerRoutes(); + } + + private createApi() { + return new EmbeddingsApi({ + currentConfig: () => this.currentConfig(), + getSchema: schemaName => this.database.getSchema(schemaName), + declaredSchema: schemaName => this.declaredSchema(schemaName), + setSchemaExtension: extension => this.database.setSchemaExtension(extension), + getVectorCapabilities: schemaName => + this.database.getVectorCapabilities(schemaName), + getVectorIndexes: schemaName => this.database.getVectorIndexes(schemaName), + createVectorIndex: (schemaName, index) => + this.database.createVectorIndex(schemaName, index), + deleteVectorIndex: (schemaName, indexName) => + this.database.deleteVectorIndex(schemaName, indexName), + invalidateHashes: async (schemaName, hashFields) => { + for (const field of hashFields) { + await this.database.updateMany( + schemaName, + {}, + { [field]: null }, + { suppressEvent: true }, + ); + } + }, + vectorSearch: input => this.database.vectorSearch(input), + configs: { + findMany: query => EmbeddingConfig.getInstance().findMany(query), + findOne: query => EmbeddingConfig.getInstance().findOne(query), + create: doc => EmbeddingConfig.getInstance().create(doc), + findByIdAndUpdate: (id, doc) => + EmbeddingConfig.getInstance().findByIdAndUpdate(id, doc), + deleteOne: query => EmbeddingConfig.getInstance().deleteOne(query), + }, + backfills: { + findMany: (query, options) => BackfillRun.getInstance().findMany(query, options), + findOne: query => BackfillRun.getInstance().findOne(query), + countDocuments: query => BackfillRun.getInstance().countDocuments(query), + create: doc => BackfillRun.getInstance().create(doc), + findByIdAndUpdate: (id, doc) => + BackfillRun.getInstance().findByIdAndUpdate(id, doc), + }, + getQueueStatus: () => this.queueController.getQueueStatus(), + enqueueBackfill: job => this.queueController.addBackfillControllerJob(job), + embed: (input, provider, model) => + getProvider(provider).embed(input, this.providerConfig(provider, model)), + onConfigChanged: async schemaName => { + const enabled = await EmbeddingConfig.getInstance().findMany({ + schemaName, + enabled: true, + }); + if (this.currentConfig().enabled && enabled.length) { + this.subscribeToSchema(schemaName); + } else { + this.unsubscribeFromSchema(schemaName); + } + }, + }); + } + + private async configureRuntime() { + const config = this.currentConfig(); + this.queueController ??= QueueController.getInstance(this.grpcSdk); + if (!config.enabled) { + await this.queueController.closeWorker(); + this.unsubscribeAll(); + return; + } + this.queueController.setBackfillJobOutcomeHandler((runId, outcome) => + this.recordBackfillJobOutcome(runId, outcome), + ); + await this.queueController.ensureWorker( + data => this.processEmbeddingJob(data), + config.queue.concurrency, + ); + await this.queueController.ensureBackfillWorker( + data => this.processBackfillJob(data), + 1, + ); + const configs = await EmbeddingConfig.getInstance().findMany({ enabled: true }); + const enabledSchemas = new Set(configs.map(item => item.schemaName)); + for (const schemaName of this.subscribedSchemas.keys()) { + if (!enabledSchemas.has(schemaName)) { + this.unsubscribeFromSchema(schemaName); + } + } + configs.forEach(item => this.subscribeToSchema(item.schemaName)); + } + + private subscribeToSchema(schemaName: string) { + if (this.subscribedSchemas.has(schemaName)) return; + const events = ['create', 'update', 'createMany', 'updateMany'] as const; + const ids = events.map(event => { + const id = `embeddings:${schemaName}:${event}`; + this.grpcSdk.bus?.subscribe( + `database:${event}:${schemaName}`, + message => this.enqueueMutation(schemaName, message), + id, + ); + return id; + }); + this.subscribedSchemas.set(schemaName, ids); + } + + private unsubscribeFromSchema(schemaName: string) { + const ids = this.subscribedSchemas.get(schemaName); + if (!ids) return; + ids.forEach(id => this.grpcSdk.bus?.unsubscribe(id)); + this.subscribedSchemas.delete(schemaName); + } + + private unsubscribeAll() { + [...this.subscribedSchemas.keys()].forEach(schemaName => + this.unsubscribeFromSchema(schemaName), + ); + } + + private enqueueMutation(schemaName: string, message: string) { + this.enqueueMutationAsync(schemaName, message).catch(err => + ConduitGrpcSdk.Logger.error(sanitizeErrorMessage(err)), + ); + } + + private async enqueueMutationAsync(schemaName: string, message: string) { + const parsed = parseBoundedMutationEvent( + message, + this.currentConfig().security.maxMutationEventIds, + ); + if (!parsed.ok) { + incrementEmbeddingMetric('malformedEvents'); + return; + } + if (!parsed.event.ids.length) return; + const configs = await EmbeddingConfig.getInstance().findMany({ + schemaName, + enabled: true, + }); + if (!configs.length) return; + if (isEmbeddingOwnedMutation(parsed.event.payload, embeddingOwnedFields(configs))) { + return; + } + const attempts = this.currentConfig().queue.attempts; + await this.queueController.addBulkEmbeddingJobs( + parsed.event.ids.map(documentId => ({ schemaName, documentId })), + attempts, + ); + } + + private async processBackfillJob(data: BackfillControllerJobData) { + await processBackfillControllerJob(data, { + maxBatchSize: this.currentConfig().queue.maxBatchSize ?? MAX_QUEUE_BATCH_SIZE, + drainTimeoutMs: this.currentConfig().queue.drainTimeoutMs, + moduleEnabled: this.currentConfig().enabled, + getRun: async id => { + const doc = await BackfillRun.getInstance().findOne({ _id: id }); + return doc ? backfillRunFromDocument(doc) : null; + }, + saveRun: (id, run) => + BackfillRun.getInstance() + .findByIdAndUpdate(id, persistableBackfillRun(run)) + .then(() => undefined), + findPage: (schemaName, page) => + this.database.findMany<{ _id?: unknown }>(schemaName, page.query, { + sort: page.sort, + limit: page.limit, + select: '_id', + }), + enqueueEmbeddingJobs: jobs => + this.queueController.addBulkEmbeddingJobs( + jobs, + this.currentConfig().queue.attempts, + ), + enqueueContinuation: job => this.queueController.addBackfillControllerJob(job), + getCapabilities: schemaName => this.database.getVectorCapabilities(schemaName), + getConfig: id => EmbeddingConfig.getInstance().findOne({ _id: id }), + getIndexes: schemaName => this.database.getVectorIndexes(schemaName), + }); + } + + private async processEmbeddingJob(data: EmbeddingJobData) { + const parsed = parseEmbeddingJobData(data); + if (!parsed.ok) { + incrementEmbeddingMetric('malformedJobs'); + return; + } + const configs = ( + parsed.data.configId + ? [await EmbeddingConfig.getInstance().findOne({ _id: parsed.data.configId })] + : await EmbeddingConfig.getInstance().findMany({ + schemaName: parsed.data.schemaName, + enabled: true, + }) + ).filter((config): config is EmbeddingConfig => Boolean(config)); + const matching = configs.filter( + config => config.enabled && config.schemaName === parsed.data.schemaName, + ); + if (!matching.length) { + await this.recordBackfillJobOutcome(parsed.data.backfillRunId, 'processed'); + return; + } + const allowedFields = [ + ...new Set( + matching.flatMap(config => [ + ...config.sourceFields, + sourceHashField(config.targetField), + ]), + ), + ]; + const doc = await this.database.findOne>( + parsed.data.schemaName, + { _id: parsed.data.documentId }, + { + select: buildEmbeddingDocumentSelect(matching), + embeddingsJob: true, + embeddingsAllowedFields: allowedFields, + }, + ); + if (!doc) { + await this.recordBackfillJobOutcome(parsed.data.backfillRunId, 'processed'); + return; + } + const result = await generateEmbeddingsForDocument({ + doc, + configs: matching, + hashInput: hashEmbeddingInput, + embed: (input, config) => + getProvider(config.provider).embed( + input, + this.providerConfig(config.provider, config.modelName ?? ''), + ), + update: (fields, options) => + this.database.findByIdAndUpdate( + parsed.data.schemaName, + parsed.data.documentId, + fields, + { + ...options, + embeddingsJob: true, + }, + ), + }); + incrementEmbeddingMetric('generated', result.generated); + incrementEmbeddingMetric('skipped', result.skipped); + await this.recordBackfillJobOutcome(parsed.data.backfillRunId, 'processed'); + } + + private async recordBackfillJobOutcome( + runId: string | undefined, + outcome: 'processed' | 'failed', + ) { + if (!runId) return; + await applyBackfillJobOutcome({ + runId, + outcome, + incrementCounts: async (id, patch) => { + const updated = await BackfillRun.getInstance().findByIdAndUpdate( + id, + toBackfillCountUpdateQuery(patch), + ); + return updated ? backfillRunFromDocument(updated) : null; + }, + }); + } + + private async declaredSchema(schemaName: string) { + return this.database.findOne( + '_DeclaredSchema', + { name: schemaName }, + { select: 'name ownerModule fields extensions' }, + ); + } + + private providerConfig(provider: string, model: string) { + const config = this.currentConfig(); + const providerConfig = config.providers[provider] ?? {}; + return { + endpoint: providerConfig.endpoint, + apiKey: providerConfig.apiKey, + model: resolveProviderModelName(providerConfig, model), + timeoutMs: config.security.embedTimeoutMs, + maxInputBytes: config.security.maxEmbedInputBytes, + maxResponseBytes: config.security.maxEmbedResponseBytes, + }; + } + + private currentConfig() { + return normalizeEmbeddingsConfig(ConfigController.getInstance().config as Config, { + strict: false, + }); + } + + protected registerSchemas(): Promise { + const promises = Object.values(models).map(model => { + const modelInstance = model.getInstance(this.database); + if ( + Object.keys((modelInstance as ConduitActiveSchema).fields) + .length !== 0 + ) { + return this.database + .createSchemaFromAdapter(modelInstance) + .then(() => this.database.migrate(modelInstance.name)); + } + }); + return Promise.all(promises); + } +} diff --git a/modules/embeddings/src/admin/index.ts b/modules/embeddings/src/admin/index.ts new file mode 100644 index 000000000..9a91119f2 --- /dev/null +++ b/modules/embeddings/src/admin/index.ts @@ -0,0 +1,334 @@ +import { + ConduitGrpcSdk, + ConduitRouteActions, + ConduitRouteReturnDefinition, + ParsedRouterRequest, + TYPE, + UnparsedRouterResponse, +} from '@conduitplatform/grpc-sdk'; +import { + ConduitBoolean, + ConduitJson, + ConduitNumber, + ConduitString, + GrpcServer, + RoutingManager, +} from '@conduitplatform/module-tools'; +import { EmbeddingsApi } from '../api/embeddingsApi.js'; +import { CONFIG_BODY, EMBEDDINGS_ADMIN_ROUTES } from './routes.js'; + +const ADMIN_CALLER = { platformAdmin: true as const }; + +export class AdminHandlers { + private readonly routingManager: RoutingManager; + + constructor( + private readonly server: GrpcServer, + private readonly grpcSdk: ConduitGrpcSdk, + private readonly api: EmbeddingsApi, + ) { + this.routingManager = new RoutingManager(this.grpcSdk.admin, this.server); + this.registerAdminRoutes(); + } + + async listConfigs(call: ParsedRouterRequest): Promise { + return this.api.getConfigs( + { + schemaName: call.request.params.schemaName, + id: call.request.params.id, + }, + ADMIN_CALLER, + ); + } + + async getConfig(call: ParsedRouterRequest): Promise { + const result = await this.api.getConfigs( + { id: call.request.params.id }, + ADMIN_CALLER, + ); + return result.configs[0]; + } + + async upsertConfig(call: ParsedRouterRequest): Promise { + const params = call.request.params as { + schemaName: string; + sourceFields: string[]; + targetField: string; + provider?: string; + model?: string; + dimensions?: number; + similarity?: string; + sourceFieldAllowlist?: string[]; + enabled?: boolean; + }; + return this.api.upsertConfig(params, ADMIN_CALLER); + } + + async deleteConfig(call: ParsedRouterRequest): Promise { + return this.api.deleteConfig({ id: call.request.params.id }, ADMIN_CALLER); + } + + async getCapabilities(call: ParsedRouterRequest): Promise { + return this.api.getCapabilities(call.request.params.schemaName); + } + + async getStatus(call: ParsedRouterRequest): Promise { + return this.api.getStatus(call.request.params.schemaName); + } + + async listBackfills(call: ParsedRouterRequest): Promise { + return this.api.listBackfills( + { + schemaName: call.request.params.schemaName, + state: call.request.params.state, + configId: call.request.params.configId, + skip: call.request.params.skip, + limit: call.request.params.limit, + }, + ADMIN_CALLER, + ); + } + + async startBackfill(call: ParsedRouterRequest): Promise { + const filter = call.request.params.filter; + return this.api.startBackfill( + { + schemaName: call.request.params.schemaName, + batchSize: call.request.params.batchSize, + configId: call.request.params.configId, + onlyMissing: call.request.params.onlyMissing, + filter: + filter == null + ? undefined + : typeof filter === 'string' + ? filter + : JSON.stringify(filter), + }, + ADMIN_CALLER, + ); + } + + async getBackfill(call: ParsedRouterRequest): Promise { + return this.api.getBackfill(call.request.params.id, ADMIN_CALLER); + } + + async cancelBackfill(call: ParsedRouterRequest): Promise { + return this.api.cancelBackfill(call.request.params.id, ADMIN_CALLER); + } + + async resumeBackfill(call: ParsedRouterRequest): Promise { + return this.api.resumeBackfill(call.request.params.id, ADMIN_CALLER); + } + + async semanticSearch(call: ParsedRouterRequest): Promise { + const filter = call.request.params.filter; + const result = await this.api.semanticSearch( + { + schemaName: call.request.params.schemaName, + text: call.request.params.text, + targetField: call.request.params.targetField, + limit: call.request.params.limit, + filter: + filter == null + ? undefined + : typeof filter === 'string' + ? filter + : JSON.stringify(filter), + adminOperator: true, + }, + ADMIN_CALLER, + ); + return { + hits: result.hits.map(hit => ({ + ...hit, + document: JSON.parse(hit.document), + })), + }; + } + + private registerAdminRoutes() { + this.routingManager.clear(); + const descriptions = new Map( + EMBEDDINGS_ADMIN_ROUTES.map(route => [ + `${route.action}:${route.path}`, + route.description, + ]), + ); + this.routingManager.route( + { + path: '/configs', + action: ConduitRouteActions.GET, + description: descriptions.get(`${ConduitRouteActions.GET}:/configs`), + queryParams: { + schemaName: ConduitString.Optional, + id: ConduitString.Optional, + }, + }, + new ConduitRouteReturnDefinition('GetEmbeddingConfigs', { + configs: [ConduitJson.Required], + }), + this.listConfigs.bind(this), + ); + this.routingManager.route( + { + path: '/configs', + action: ConduitRouteActions.POST, + description: descriptions.get(`${ConduitRouteActions.POST}:/configs`), + bodyParams: CONFIG_BODY as never, + }, + new ConduitRouteReturnDefinition('UpsertEmbeddingConfig', { + config: ConduitJson.Required, + warnings: [ConduitString.Required], + }), + this.upsertConfig.bind(this), + ); + this.routingManager.route( + { + path: '/configs/:id', + action: ConduitRouteActions.GET, + description: descriptions.get(`${ConduitRouteActions.GET}:/configs/:id`), + urlParams: { id: ConduitString.Required }, + }, + new ConduitRouteReturnDefinition('GetEmbeddingConfig', TYPE.JSON), + this.getConfig.bind(this), + ); + this.routingManager.route( + { + path: '/configs/:id', + action: ConduitRouteActions.DELETE, + description: descriptions.get(`${ConduitRouteActions.DELETE}:/configs/:id`), + urlParams: { id: ConduitString.Required }, + }, + new ConduitRouteReturnDefinition('DeleteEmbeddingConfig', { + config: ConduitJson.Required, + }), + this.deleteConfig.bind(this), + ); + this.routingManager.route( + { + path: '/capabilities', + action: ConduitRouteActions.GET, + description: descriptions.get(`${ConduitRouteActions.GET}:/capabilities`), + queryParams: { schemaName: ConduitString.Optional }, + }, + new ConduitRouteReturnDefinition('GetEmbeddingCapabilities', { + capabilities: ConduitJson.Required, + warnings: [ConduitString.Required], + }), + this.getCapabilities.bind(this), + ); + this.routingManager.route( + { + path: '/status', + action: ConduitRouteActions.GET, + description: descriptions.get(`${ConduitRouteActions.GET}:/status`), + queryParams: { schemaName: ConduitString.Optional }, + }, + new ConduitRouteReturnDefinition('GetEmbeddingStatus', { + enabled: ConduitBoolean.Required, + ready: ConduitBoolean.Required, + capabilities: ConduitJson.Required, + generationQueue: ConduitJson.Required, + backfillQueue: ConduitJson.Required, + warnings: [ConduitString.Required], + }), + this.getStatus.bind(this), + ); + this.routingManager.route( + { + path: '/backfills', + action: ConduitRouteActions.GET, + description: descriptions.get(`${ConduitRouteActions.GET}:/backfills`), + queryParams: { + schemaName: ConduitString.Optional, + state: ConduitString.Optional, + configId: ConduitString.Optional, + skip: ConduitNumber.Optional, + limit: ConduitNumber.Optional, + }, + }, + new ConduitRouteReturnDefinition('ListEmbeddingBackfills', { + runs: [ConduitJson.Required], + count: ConduitNumber.Required, + }), + this.listBackfills.bind(this), + ); + this.routingManager.route( + { + path: '/backfills', + action: ConduitRouteActions.POST, + description: descriptions.get(`${ConduitRouteActions.POST}:/backfills`), + bodyParams: { + schemaName: ConduitString.Required, + batchSize: ConduitNumber.Optional, + configId: ConduitString.Optional, + onlyMissing: ConduitBoolean.Optional, + filter: ConduitJson.Optional, + }, + }, + new ConduitRouteReturnDefinition('StartEmbeddingBackfill', { + queued: ConduitNumber.Required, + runs: [ConduitJson.Required], + warnings: [ConduitString.Required], + }), + this.startBackfill.bind(this), + ); + this.routingManager.route( + { + path: '/backfills/:id', + action: ConduitRouteActions.GET, + description: descriptions.get(`${ConduitRouteActions.GET}:/backfills/:id`), + urlParams: { id: ConduitString.Required }, + }, + new ConduitRouteReturnDefinition('GetEmbeddingBackfill', TYPE.JSON), + this.getBackfill.bind(this), + ); + this.routingManager.route( + { + path: '/backfills/:id/cancel', + action: ConduitRouteActions.POST, + description: descriptions.get( + `${ConduitRouteActions.POST}:/backfills/:id/cancel`, + ), + urlParams: { id: ConduitString.Required }, + }, + new ConduitRouteReturnDefinition('CancelEmbeddingBackfill', { + run: ConduitJson.Required, + }), + this.cancelBackfill.bind(this), + ); + this.routingManager.route( + { + path: '/backfills/:id/resume', + action: ConduitRouteActions.POST, + description: descriptions.get( + `${ConduitRouteActions.POST}:/backfills/:id/resume`, + ), + urlParams: { id: ConduitString.Required }, + }, + new ConduitRouteReturnDefinition('ResumeEmbeddingBackfill', { + run: ConduitJson.Required, + }), + this.resumeBackfill.bind(this), + ); + this.routingManager.route( + { + path: '/search', + action: ConduitRouteActions.POST, + description: descriptions.get(`${ConduitRouteActions.POST}:/search`), + bodyParams: { + schemaName: ConduitString.Required, + text: ConduitString.Required, + targetField: ConduitString.Optional, + filter: ConduitJson.Optional, + limit: ConduitNumber.Optional, + }, + }, + new ConduitRouteReturnDefinition('AdminSemanticSearch', { + hits: [ConduitJson.Required], + }), + this.semanticSearch.bind(this), + ); + void this.routingManager.registerRoutes(); + } +} diff --git a/modules/embeddings/src/admin/routes.ts b/modules/embeddings/src/admin/routes.ts new file mode 100644 index 000000000..d7764c635 --- /dev/null +++ b/modules/embeddings/src/admin/routes.ts @@ -0,0 +1,117 @@ +import { ConduitRouteActions, TYPE } from '@conduitplatform/grpc-sdk'; +import { + ConduitBoolean, + ConduitNumber, + ConduitString, +} from '@conduitplatform/module-tools'; +import { embeddingsMcpToolName, embeddingsPublicPath } from '../utils/mcpToolNames.js'; + +export interface EmbeddingsAdminRouteContract { + path: string; + action: ConduitRouteActions; + description: string; + publicPath: string; + mcpName: string; + clientExposed: false; +} + +const CONFIG_BODY = { + schemaName: ConduitString.Required, + sourceFields: { type: [TYPE.String], required: true }, + targetField: ConduitString.Required, + provider: ConduitString.Optional, + model: ConduitString.Optional, + dimensions: ConduitNumber.Optional, + similarity: ConduitString.Optional, + sourceFieldAllowlist: { type: [TYPE.String], required: false }, + enabled: ConduitBoolean.Optional, +}; + +function contract( + path: string, + action: ConduitRouteActions, + description: string, +): EmbeddingsAdminRouteContract { + const publicPath = embeddingsPublicPath(path); + return { + path, + action, + description, + publicPath, + mcpName: embeddingsMcpToolName(action, publicPath), + clientExposed: false, + }; +} + +export const EMBEDDINGS_ADMIN_ROUTES: EmbeddingsAdminRouteContract[] = [ + contract( + '/configs', + ConduitRouteActions.GET, + 'Lists embedding configurations. Operator-only. Filter by schemaName or id. Never expose this as a client route.', + ), + contract( + '/configs', + ConduitRouteActions.POST, + 'Creates or updates an embedding config for a schema. Operator-only. The first upsert provisions the vector index when Database indexing is available. Saving enabled=true while the index is pending, or if provisioning fails, stores the config disabled until it is queryable. If indexing is unavailable, status reports a manual index lifecycle warning. Caller-supplied sourceFieldAllowlist is honored only for platform-admin upserts; schema-owner gRPC callers use operator config allowlists.', + ), + contract( + '/configs/:id', + ConduitRouteActions.GET, + 'Returns one embedding configuration by id. Operator-only.', + ), + contract( + '/configs/:id', + ConduitRouteActions.DELETE, + 'Deletes an embedding configuration by id. Operator-only. Does not drop vector fields or indexes.', + ), + contract( + '/capabilities', + ConduitRouteActions.GET, + 'Returns Database vector storage, index, and search capabilities plus readiness warnings. Operator-only.', + ), + contract( + '/status', + ConduitRouteActions.GET, + 'Returns embeddings module readiness, provider/index warnings, and generation/backfill queue counts. Operator-only.', + ), + contract( + '/backfills', + ConduitRouteActions.GET, + 'Lists persisted BackfillRun records with scanned/queued/processed/failed counts, cursor, onlyMissing, and state. Operator-only.', + ), + contract( + '/backfills', + ConduitRouteActions.POST, + 'Starts a queued cursor-based backfill. Operator-only. Supports onlyMissing, configId, and bounded batchSize. Never scans in the request thread.', + ), + contract( + '/backfills/:id', + ConduitRouteActions.GET, + 'Returns one persisted BackfillRun including counts, cursor, onlyMissing, and sanitized error. Operator-only.', + ), + contract( + '/backfills/:id/cancel', + ConduitRouteActions.POST, + 'Cancels a queued or running BackfillRun. Operator-only.', + ), + contract( + '/backfills/:id/resume', + ConduitRouteActions.POST, + 'Resumes a failed or canceled BackfillRun from its persisted cursor. Operator-only. Re-checks capability and index readiness.', + ), + contract( + '/search', + ConduitRouteActions.POST, + 'Runs operator semantic search by text. Generates a query embedding and delegates vector search to Database. Operator-only; does not accept raw vectors.', + ), +]; + +export const EMBEDDINGS_CLIENT_SEARCH_PATH = '/search'; +export const EMBEDDINGS_CLIENT_FORBIDDEN_PATHS = [ + '/configs', + '/backfills', + '/capabilities', + '/status', +]; + +export { CONFIG_BODY }; diff --git a/modules/embeddings/src/api/embeddingsApi.test.ts b/modules/embeddings/src/api/embeddingsApi.test.ts new file mode 100644 index 000000000..dd5996107 --- /dev/null +++ b/modules/embeddings/src/api/embeddingsApi.test.ts @@ -0,0 +1,1419 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + GrpcError, + TYPE, + VectorCapabilities, + VectorIndexStatus, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + EmbeddingsApi, + type EmbeddingsApiDeps, + type EmbeddingConfigRecord, + type BackfillRunRecord, + type SchemaInfo, +} from './embeddingsApi.js'; +import type { Config } from '../config/index.js'; +import type { QueueJobCounts } from '../controllers/queue.controller.js'; +import { SearchGateError } from '../utils/operationalStatus.js'; + +const articleSchema = { + name: 'Article', + fields: { title: { type: TYPE.String }, body: { type: TYPE.String } }, + modelOptions: { + conduit: { + cms: { enabled: true }, + permissions: { extendable: true }, + authorization: { enabled: true }, + }, + }, +}; + +const readyCapabilities = { + supported: true, + storage: true, + indexing: true, + search: true, + provider: 'mongodb' as const, +}; + +const readyIndex = { + field: 'embedding', + name: 'embedding_vector', + queryable: true, + status: VectorIndexStatus.Ready, + dimensions: 3, + similarity: VectorSimilarity.Cosine, +}; + +const moduleConfig = { + enabled: true, + defaultProvider: 'openai-compatible', + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-test', + models: [ + { name: 'text-embedding-3-small', dimensions: 3 }, + { name: 'text-embedding-3-large', dimensions: 3 }, + { name: 'text-embedding-3-wide', dimensions: 8 }, + ], + defaultModel: 'text-embedding-3-small', + }, + }, + queue: { concurrency: 1, attempts: 3, maxBatchSize: 50 }, + security: { + sourceFieldAllowlist: [], + maxMutationEventIds: 10, + embedTimeoutMs: 1000, + maxEmbedInputBytes: 1024, + maxEmbedResponseBytes: 1024, + }, +} as Config; + +function emptyCounts(): QueueJobCounts { + return { waiting: 0, active: 0, completed: 0, failed: 0, delayed: 0, paused: 0 }; +} + +function createApi(overrides?: { + configs?: EmbeddingConfigRecord[]; + runs?: BackfillRunRecord[]; + capabilities?: VectorCapabilities; + indexes?: Array<{ + field?: string; + name?: string; + queryable?: boolean; + status?: string; + dimensions?: number; + similarity?: string; + method?: string; + }>; + schemas?: Record; + declared?: Record< + string, + { + name: string; + ownerModule: string; + fields?: Record; + extensions?: Array<{ ownerModule: string; fields: Record }>; + } + >; + embed?: EmbeddingsApiDeps['embed']; + vectorSearch?: EmbeddingsApiDeps['vectorSearch']; + createVectorIndex?: EmbeddingsApiDeps['createVectorIndex']; + createdIndexQueryable?: boolean; + enqueue?: string[]; + invalidated?: string[]; + deletedIndexes?: string[]; + createdIndexes?: string[]; + schemaExtensions?: Array<{ schemaName: string; fields: Record }>; + config?: Config; + queue?: { generation: QueueJobCounts; backfill: QueueJobCounts }; +}) { + const configs = [...(overrides?.configs ?? [])]; + const runs = [...(overrides?.runs ?? [])]; + const indexes = [...(overrides?.indexes ?? [readyIndex])]; + const enqueued = overrides?.enqueue ?? []; + const invalidated = overrides?.invalidated ?? []; + const deletedIndexes = overrides?.deletedIndexes ?? []; + const createdIndexes = overrides?.createdIndexes ?? []; + const schemaExtensions = overrides?.schemaExtensions ?? []; + const deps: EmbeddingsApiDeps = { + currentConfig: () => overrides?.config ?? moduleConfig, + getSchema: async name => { + const schema = + overrides?.schemas?.[name] ?? (name === 'Article' ? articleSchema : undefined); + if (!schema) throw new GrpcError(status.NOT_FOUND, `Schema ${name} not found`); + return schema; + }, + declaredSchema: async name => + overrides?.declared?.[name] ?? { name, ownerModule: 'database' }, + setSchemaExtension: async args => { + schemaExtensions.push(args); + return undefined; + }, + getVectorCapabilities: async () => overrides?.capabilities ?? readyCapabilities, + getVectorIndexes: async () => indexes, + vectorSearch: overrides?.vectorSearch ?? (async () => []), + configs: { + findMany: async query => + configs.filter(config => + Object.entries(query).every(([key, value]) => (config as never)[key] === value), + ), + findOne: async query => + configs.find(config => + Object.entries(query).every(([key, value]) => (config as never)[key] === value), + ) ?? null, + create: async doc => { + const created = { + _id: `cfg${configs.length + 1}`, + ...doc, + } as EmbeddingConfigRecord; + configs.push(created); + return created; + }, + findByIdAndUpdate: async (id, doc) => { + const index = configs.findIndex(config => config._id === id); + if (index < 0) return null; + configs[index] = { ...configs[index], ...doc }; + return configs[index]; + }, + deleteOne: async query => { + const index = configs.findIndex(config => + Object.entries(query).every(([key, value]) => (config as never)[key] === value), + ); + if (index >= 0) configs.splice(index, 1); + }, + }, + backfills: { + findMany: async (query, options) => { + const matched = runs.filter(run => + Object.entries(query).every(([key, value]) => (run as never)[key] === value), + ); + const skip = options?.skip ?? 0; + const limit = options?.limit ?? matched.length; + return matched.slice(skip, skip + limit); + }, + findOne: async query => + runs.find(run => + Object.entries(query).every(([key, value]) => (run as never)[key] === value), + ) ?? null, + countDocuments: async query => + runs.filter(run => + Object.entries(query).every(([key, value]) => (run as never)[key] === value), + ).length, + create: async doc => { + const created = { + _id: `run${runs.length + 1}`, + ...doc, + } as BackfillRunRecord; + runs.push(created); + return { _id: created._id }; + }, + findByIdAndUpdate: async (id, doc) => { + const index = runs.findIndex(run => run._id === id); + if (index < 0) return null; + runs[index] = { ...runs[index], ...doc } as BackfillRunRecord; + return runs[index]; + }, + }, + getQueueStatus: async () => + overrides?.queue ?? { + generation: { ...emptyCounts(), waiting: 2 }, + backfill: emptyCounts(), + }, + enqueueBackfill: async job => { + enqueued.push(job.runId); + }, + createVectorIndex: + overrides?.createVectorIndex ?? + (async (_schema, index) => { + const name = index.name ?? `${index.field}_vector`; + createdIndexes.push(name); + if (!indexes.some(item => item.name === name)) { + indexes.push({ + field: index.field, + name, + dimensions: index.dimensions, + similarity: index.similarity, + method: index.method, + queryable: overrides?.createdIndexQueryable === true, + status: + overrides?.createdIndexQueryable === true + ? VectorIndexStatus.Ready + : VectorIndexStatus.Pending, + }); + } + return 'created'; + }), + deleteVectorIndex: async (_schema, indexName) => { + deletedIndexes.push(indexName); + const index = indexes.findIndex(item => item.name === indexName); + if (index >= 0) indexes.splice(index, 1); + return 'deleted'; + }, + invalidateHashes: async (schemaName, hashFields) => { + invalidated.push(...hashFields.map(field => `${schemaName}.${field}`)); + }, + embed: + overrides?.embed ?? + (async () => Array.from({ length: 3 }, (_, index) => index + 0.1)), + }; + return { + api: new EmbeddingsApi(deps), + configs, + runs, + enqueued, + invalidated, + deletedIndexes, + createdIndexes, + schemaExtensions, + indexes, + }; +} + +const enabledConfig: EmbeddingConfigRecord = { + _id: 'cfg1', + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + modelName: 'text-embedding-3-small', + dimensions: 3, + similarity: 'cosine', + enabled: true, +}; + +describe('typed embeddings API handlers', () => { + it('upserts a typed config and provisions a missing vector index on first save', async () => { + const { api, configs, createdIndexes } = createApi({ indexes: [] }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(saved.config.model, 'text-embedding-3-small'); + assert.equal(typeof saved.config.id, 'string'); + assert.equal(createdIndexes.includes('embedding_vector'), true); + assert.equal( + saved.warnings.some(warning => /not queryable/.test(warning)), + true, + ); + assert.equal( + saved.warnings.some(warning => + /saved disabled until the provisioned vector index/.test(warning), + ), + true, + ); + assert.equal(configs.length, 1); + assert.equal(configs[0].enabled, false); + }); + + it('saves the config disabled when vector index provisioning fails', async () => { + const { api, configs, createdIndexes } = createApi({ + indexes: [], + createVectorIndex: async () => { + throw new Error('atlas search index rejected'); + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(configs.length, 1); + assert.equal(configs[0].enabled, false); + assert.equal(createdIndexes.length, 0); + assert.equal( + saved.warnings.some( + warning => + /saved disabled because vector index provisioning failed/.test(warning) && + /atlas search index rejected/.test(warning) && + /queryable/.test(warning), + ), + true, + ); + }); + + it('saves the updated config disabled when index recreation fails', async () => { + const { api, configs, deletedIndexes } = createApi({ + configs: [enabledConfig], + createVectorIndex: async () => { + throw new Error('recreate rejected'); + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(configs[0].enabled, false); + assert.deepEqual(deletedIndexes, []); + assert.equal( + saved.warnings.some( + warning => + /saved disabled because vector index provisioning failed/.test(warning) && + /recreate rejected/.test(warning), + ), + true, + ); + }); + + it('reports a manual index lifecycle when Database indexing is unavailable', async () => { + const { api, createdIndexes } = createApi({ + indexes: [], + capabilities: { + supported: true, + storage: true, + indexing: false, + search: false, + provider: 'mongodb', + reason: 'indexing unavailable', + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(createdIndexes.length, 0); + assert.equal( + saved.warnings.some(warning => /Create the index manually/.test(warning)), + true, + ); + }); + + it('does not recreate _vN indexes when live method is empty or missing', async () => { + const missingMethod = { + ...readyIndex, + method: undefined, + }; + const emptyMethod = { + ...readyIndex, + method: '', + }; + for (const indexes of [[missingMethod], [emptyMethod]]) { + const { api, createdIndexes, deletedIndexes } = createApi({ + configs: [enabledConfig], + indexes, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, true); + assert.deepEqual(createdIndexes, []); + assert.deepEqual(deletedIndexes, []); + } + }); + + it('gates system schemas and owner policies on config and backfill', async () => { + const { api } = createApi({ + declared: { + Article: { name: 'Article', ownerModule: 'cms-app' }, + Admin: { name: 'Admin', ownerModule: 'core' }, + }, + schemas: { + Article: articleSchema, + AccessToken: { + name: 'AccessToken', + fields: { token: { type: TYPE.String } }, + }, + Admin: { + name: 'Admin', + fields: { username: { type: TYPE.String } }, + modelOptions: articleSchema.modelOptions, + }, + }, + }); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'AccessToken', + sourceFields: ['token'], + targetField: 'embedding', + model: 'text-embedding-3-small', + dimensions: 3, + enabled: false, + }, + { platformAdmin: true }, + ), + (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Admin', + sourceFields: ['username'], + targetField: 'embedding', + model: 'text-embedding-3-small', + dimensions: 3, + enabled: false, + }, + { platformAdmin: true }, + ), + (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + await assert.rejects( + () => api.startBackfill({ schemaName: 'Article' }, { callerModule: 'chat' }), + (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + await assert.rejects( + () => api.getConfigs({}, { callerModule: 'chat' }), + (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + }); + + it('ignores caller-supplied sourceFieldAllowlist for schema-owner gRPC callers', async () => { + const sensitiveSchema = { + name: 'Article', + fields: { + title: { type: TYPE.String }, + password: { type: TYPE.String }, + notes: { type: TYPE.String, select: false }, + }, + modelOptions: articleSchema.modelOptions, + }; + const owner = { callerModule: 'cms-app' }; + const declared = { Article: { name: 'Article', ownerModule: 'cms-app' } }; + const { api: ownerApi } = createApi({ + schemas: { Article: sensitiveSchema }, + declared, + indexes: [readyIndex], + }); + await assert.rejects( + () => + ownerApi.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['password'], + targetField: 'embedding', + model: 'text-embedding-3-small', + dimensions: 3, + sourceFieldAllowlist: ['password'], + enabled: false, + }, + owner, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /sensitive/.test(err.message), + ); + await assert.rejects( + () => + ownerApi.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['notes'], + targetField: 'embedding', + model: 'text-embedding-3-small', + dimensions: 3, + sourceFieldAllowlist: ['notes'], + enabled: false, + }, + owner, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /hidden/.test(err.message), + ); + await assert.rejects( + () => + ownerApi.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['password'], + targetField: 'embedding', + model: 'text-embedding-3-small', + dimensions: 3, + sourceFieldAllowlist: ['password'], + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /sensitive/.test(err.message), + ); + + const { api: operatorAllowlistApi } = createApi({ + schemas: { Article: sensitiveSchema }, + declared, + indexes: [readyIndex], + config: { + ...moduleConfig, + security: { ...moduleConfig.security, sourceFieldAllowlist: ['notes'] }, + } as Config, + }); + const operatorSaved = await operatorAllowlistApi.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['notes'], + targetField: 'embedding', + model: 'text-embedding-3-small', + dimensions: 3, + enabled: false, + }, + owner, + ); + assert.equal(operatorSaved.config.enabled, false); + + const { api: adminApi } = createApi({ + schemas: { Article: sensitiveSchema }, + declared, + indexes: [readyIndex], + }); + const adminSaved = await adminApi.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['notes'], + targetField: 'embedding', + model: 'text-embedding-3-small', + dimensions: 3, + sourceFieldAllowlist: ['notes'], + enabled: false, + }, + { platformAdmin: true }, + ); + assert.equal(adminSaved.config.enabled, false); + }); + + it('starts, lists, cancels, and resumes persisted backfill runs with onlyMissing', async () => { + const { api, runs, enqueued } = createApi({ configs: [enabledConfig] }); + const started = await api.startBackfill( + { schemaName: 'Article', onlyMissing: true, batchSize: 10 }, + { callerModule: 'database' }, + ); + assert.equal(started.queued, 1); + assert.equal(started.runs[0].onlyMissing, true); + assert.equal(started.runs[0].state, 'queued'); + assert.equal(enqueued.length, 1); + const listed = await api.listBackfills( + { schemaName: 'Article' }, + { callerModule: 'database' }, + ); + assert.equal(listed.count, 1); + const gotten = await api.getBackfill(started.runs[0].id, { + callerModule: 'database', + }); + assert.equal(gotten.queuedCount, 0); + const canceled = await api.cancelBackfill(started.runs[0].id, { + callerModule: 'database', + }); + assert.equal(canceled.run.state, 'canceled'); + const resumed = await api.resumeBackfill(started.runs[0].id, { + callerModule: 'database', + }); + assert.equal(resumed.run.state, 'queued'); + assert.equal(runs[0].state, 'queued'); + await assert.rejects( + () => api.cancelBackfill('missing', { platformAdmin: true }), + (err: unknown) => err instanceof GrpcError && err.code === status.NOT_FOUND, + ); + }); + + it('returns typed status, queue counts, and capability warnings', async () => { + const { api } = createApi({ + capabilities: { + supported: false, + storage: false, + indexing: false, + search: false, + provider: 'unsupported', + reason: 'mysql is storage-only', + }, + config: { ...moduleConfig, enabled: false }, + queue: { + generation: { ...emptyCounts(), waiting: 4, failed: 1 }, + backfill: { ...emptyCounts(), active: 1 }, + }, + }); + const statusResult = await api.getStatus(); + assert.equal(statusResult.enabled, false); + assert.equal(statusResult.ready, false); + assert.equal(statusResult.generationQueue.waiting, 4); + assert.equal(statusResult.backfillQueue.active, 1); + assert.equal( + statusResult.warnings.some(warning => /disabled/.test(warning)), + true, + ); + assert.equal( + statusResult.warnings.some(warning => /mysql is storage-only/.test(warning)), + true, + ); + const capabilities = await api.getCapabilities('Article'); + assert.equal(capabilities.capabilities.search, false); + }); + + it('runs semantic search with typed hits and fail-closed auth', async () => { + const embedCalls: Array<[string, string, string]> = []; + const { api } = createApi({ + configs: [enabledConfig], + embed: async (input, provider, model) => { + embedCalls.push([input, provider, model]); + return [0.1, 1.1, 2.1]; + }, + vectorSearch: async input => { + assert.equal(input.userId, 'user-1'); + assert.equal(input.adminOperator, false); + assert.deepEqual(input.vector, [0.1, 1.1, 2.1]); + return [ + { + document: { _id: 'doc1', title: 'Hello' }, + score: 0.91, + distance: 0.09, + metric: VectorSimilarity.Cosine, + provider: 'mongodb', + }, + ]; + }, + }); + await assert.rejects( + () => + api.semanticSearch( + { schemaName: 'Article', text: 'hello' }, + { callerModule: 'database' }, + ), + (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + const result = await api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1' }, + { callerModule: 'database' }, + ); + assert.deepEqual(embedCalls, [ + ['hello', 'openai-compatible', 'text-embedding-3-small'], + ]); + assert.equal(result.hits.length, 1); + assert.equal(JSON.parse(result.hits[0].document)._id, 'doc1'); + assert.equal(result.hits[0].score, 0.91); + const mapped = api.mapGrpcError( + new GrpcError(status.FAILED_PRECONDITION, 'index not ready'), + ); + assert.equal(mapped.code, status.FAILED_PRECONDITION); + }); + + it('caps client semantic-search limit below admin and gRPC callers', async () => { + const seen: number[] = []; + const { api } = createApi({ + configs: [enabledConfig], + vectorSearch: async input => { + seen.push(input.limit ?? -1); + return []; + }, + }); + await api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1', limit: 1000 }, + { callerModule: 'router' }, + ); + await api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1', limit: 1000 }, + { callerModule: 'database' }, + ); + await api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1', limit: 1000 }, + { platformAdmin: true }, + ); + assert.deepEqual(seen, [50, 1000, 1000]); + }); + + it('maps provider dimension mismatches and illegal backfill transitions to typed statuses', async () => { + const { api } = createApi({ + configs: [enabledConfig], + embed: async () => [1, 2], + }); + await assert.rejects( + () => + api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1' }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && err.code === status.FAILED_PRECONDITION, + ); + const started = await createApi({ configs: [enabledConfig] }).api.startBackfill( + { schemaName: 'Article' }, + { callerModule: 'database' }, + ); + const { api: resumeApi } = createApi({ + configs: [enabledConfig], + runs: [ + { + _id: started.runs[0].id, + schemaName: 'Article', + configId: 'cfg1', + state: 'completed', + batchSize: 10, + onlyMissing: false, + scannedCount: 1, + queuedCount: 1, + processedCount: 1, + failedCount: 0, + }, + ], + }); + await assert.rejects( + () => resumeApi.resumeBackfill(started.runs[0].id, { callerModule: 'database' }), + (err: unknown) => + err instanceof GrpcError && err.code === status.FAILED_PRECONDITION, + ); + }); + + it('invalidates hashes, recreates the index, and schedules a backfill on material config changes', async () => { + const { api, invalidated, deletedIndexes, createdIndexes, enqueued, runs } = + createApi({ + configs: [enabledConfig], + createdIndexQueryable: true, + }); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-wide', + dimensions: 8, + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /dimensions/.test(err.message), + ); + + const updated = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title', 'body'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-large', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(updated.config.model, 'text-embedding-3-large'); + assert.deepEqual(invalidated, ['Article.embeddingSourceHash']); + assert.deepEqual(createdIndexes, ['embedding_vector_v2']); + assert.deepEqual(deletedIndexes, ['embedding_vector']); + assert.equal(enqueued.length, 1); + assert.equal(runs[0].state, 'queued'); + assert.equal(runs[0].onlyMissing, false); + assert.equal( + updated.warnings.some(warning => /explicit backfill was scheduled/.test(warning)), + true, + ); + }); + + it('keeps the previous index when replacement provisioning fails', async () => { + const { api, configs, deletedIndexes, createdIndexes, indexes } = createApi({ + configs: [enabledConfig], + createVectorIndex: async () => { + throw new Error('atlas rejected replacement'); + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(configs[0].enabled, false); + assert.deepEqual(createdIndexes, []); + assert.deepEqual(deletedIndexes, []); + assert.equal( + indexes.some(index => index.name === 'embedding_vector' && index.queryable), + true, + ); + assert.equal( + saved.warnings.some( + warning => + /saved disabled because vector index provisioning failed/.test(warning) && + /atlas rejected replacement/.test(warning), + ), + true, + ); + }); + + it('keeps the previous index while a versioned replacement is not queryable', async () => { + const { api, configs, deletedIndexes, createdIndexes, indexes, enqueued } = createApi( + { + configs: [enabledConfig], + }, + ); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(configs[0].enabled, false); + assert.deepEqual(createdIndexes, ['embedding_vector_v2']); + assert.deepEqual(deletedIndexes, []); + assert.equal( + indexes.some(index => index.name === 'embedding_vector' && index.queryable), + true, + ); + assert.equal( + indexes.some( + index => index.name === 'embedding_vector_v2' && index.queryable !== true, + ), + true, + ); + assert.equal(enqueued.length, 0); + assert.equal( + saved.warnings.some(warning => + /saved disabled until the provisioned vector index/.test(warning), + ), + true, + ); + }); + + it('retires the previous index after a pending replacement becomes queryable', async () => { + const pendingReplacement = { + field: 'embedding', + name: 'embedding_vector_v2', + queryable: false, + status: VectorIndexStatus.Pending, + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + }; + const { api, configs, deletedIndexes, createdIndexes, indexes, enqueued } = createApi( + { + configs: [ + { ...enabledConfig, similarity: VectorSimilarity.Euclidean, enabled: false }, + ], + indexes: [readyIndex, pendingReplacement], + }, + ); + pendingReplacement.queryable = true; + pendingReplacement.status = VectorIndexStatus.Ready; + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, true); + assert.equal(configs[0].enabled, true); + assert.deepEqual(createdIndexes, []); + assert.deepEqual(deletedIndexes, ['embedding_vector']); + assert.equal( + indexes.some(index => index.name === 'embedding_vector'), + false, + ); + assert.equal( + indexes.some(index => index.name === 'embedding_vector_v2' && index.queryable), + true, + ); + assert.equal(enqueued.length, 0); + }); + + it('retries a failed similarity recreation without enabling the mismatched live index', async () => { + let attempts = 0; + const createdIndexes: string[] = []; + const { api, configs, deletedIndexes, indexes } = createApi({ + configs: [enabledConfig], + createdIndexes, + createVectorIndex: async (_schema, index) => { + attempts += 1; + if (attempts === 1) { + throw new Error('atlas rejected replacement'); + } + const name = index.name ?? `${index.field}_vector`; + createdIndexes.push(name); + indexes.push({ + field: index.field, + name, + dimensions: index.dimensions, + similarity: index.similarity, + queryable: false, + status: VectorIndexStatus.Pending, + }); + return 'created'; + }, + }); + const euclideanUpsert = { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }; + const first = await api.upsertConfig(euclideanUpsert, { callerModule: 'database' }); + assert.equal(first.config.enabled, false); + assert.deepEqual(createdIndexes, []); + assert.deepEqual(deletedIndexes, []); + + const retry = await api.upsertConfig(euclideanUpsert, { callerModule: 'database' }); + assert.equal(retry.config.enabled, false); + assert.equal(configs[0].enabled, false); + assert.deepEqual(createdIndexes, ['embedding_vector_v2']); + assert.deepEqual(deletedIndexes, []); + assert.equal( + indexes.some(index => index.name === 'embedding_vector' && index.queryable), + true, + ); + assert.equal( + indexes.some( + index => + index.name === 'embedding_vector_v2' && + index.similarity === VectorSimilarity.Euclidean && + index.queryable !== true, + ), + true, + ); + await assert.rejects( + () => + api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1' }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.NOT_FOUND && + /No embedding config found/.test(err.message), + ); + }); + + it('denies activation when the live index contract does not match', async () => { + const { api, configs, deletedIndexes, createdIndexes, indexes } = createApi({ + configs: [ + { ...enabledConfig, similarity: VectorSimilarity.Euclidean, enabled: false }, + ], + capabilities: { + ...readyCapabilities, + indexing: false, + reason: 'indexing unavailable', + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 3, + similarity: VectorSimilarity.Euclidean, + enabled: true, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(configs[0].enabled, false); + assert.deepEqual(createdIndexes, []); + assert.deepEqual(deletedIndexes, []); + assert.equal( + indexes.some( + index => + index.name === 'embedding_vector' && + index.queryable && + index.similarity === VectorSimilarity.Cosine, + ), + true, + ); + assert.equal( + saved.warnings.some(warning => /not queryable/.test(warning)), + true, + ); + await assert.rejects( + () => + api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1' }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + (err.code === status.NOT_FOUND || err.code === status.FAILED_PRECONDITION), + ); + }); + + it('does not search through a queryable index that does not match the config contract', async () => { + let searched = false; + const { api } = createApi({ + configs: [ + { ...enabledConfig, similarity: VectorSimilarity.Euclidean, enabled: true }, + ], + vectorSearch: async () => { + searched = true; + return []; + }, + }); + await assert.rejects( + () => + api.semanticSearch( + { schemaName: 'Article', text: 'hello', userId: 'user-1' }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof SearchGateError && + err.reason === 'index_not_queryable' && + api.mapGrpcError(err).code === status.FAILED_PRECONDITION, + ); + assert.equal(searched, false); + }); + + it('is idempotent for start, cancel, and resume and does not duplicate active runs', async () => { + const { api, runs, enqueued } = createApi({ configs: [enabledConfig] }); + const first = await api.startBackfill( + { schemaName: 'Article', configId: 'cfg1' }, + { callerModule: 'database' }, + ); + const second = await api.startBackfill( + { schemaName: 'Article', configId: 'cfg1' }, + { callerModule: 'database' }, + ); + assert.equal(first.runs[0].id, second.runs[0].id); + assert.equal(runs.filter(run => run.state === 'queued').length, 1); + const canceled = await api.cancelBackfill(first.runs[0].id, { + callerModule: 'database', + }); + assert.equal(canceled.run.state, 'canceled'); + const canceledAgain = await api.cancelBackfill(first.runs[0].id, { + callerModule: 'database', + }); + assert.equal(canceledAgain.run.state, 'canceled'); + const resumed = await api.resumeBackfill(first.runs[0].id, { + callerModule: 'database', + }); + assert.equal(resumed.run.state, 'queued'); + const resumedAgain = await api.resumeBackfill(first.runs[0].id, { + callerModule: 'database', + }); + assert.equal(resumedAgain.run.state, 'queued'); + assert.equal(enqueued.length >= 2, true); + }); + + it('rejects unknown providers, unknown models, and explicit catalogue dimension mismatches', async () => { + const { api } = createApi(); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'missing', + model: 'text-embedding-3-small', + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /not a configured provider/.test(err.message), + ); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'missing-model', + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /not in the catalogue/.test(err.message), + ); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'text-embedding-3-small', + dimensions: 1536, + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /do not match catalogue dimensions/.test(err.message), + ); + }); + + it('derives dimensions from the catalogue when the client omits them', async () => { + const { api, configs } = createApi({ indexes: [readyIndex] }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'text-embedding-3-small', + enabled: false, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.dimensions, 3); + assert.equal(configs[0].dimensions, 3); + assert.equal(saved.config.model, 'text-embedding-3-small'); + }); + + it('selects the catalogue default model when upsert omits model', async () => { + const { api, configs } = createApi({ + indexes: [readyIndex], + config: { + ...moduleConfig, + providers: { + 'openai-compatible': { + ...moduleConfig.providers['openai-compatible'], + defaultModel: 'text-embedding-3-large', + }, + }, + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + enabled: false, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.model, 'text-embedding-3-large'); + assert.equal(saved.config.dimensions, 3); + assert.equal(configs[0].modelName, 'text-embedding-3-large'); + }); + + it('reports catalogue readiness on status without leaking provider secrets', async () => { + const { api } = createApi({ + config: { + ...moduleConfig, + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-status', + models: [], + defaultModel: 'missing', + }, + }, + }, + }); + const statusResult = await api.getStatus(); + assert.equal(statusResult.ready, false); + assert.equal( + statusResult.warnings.some(warning => /model catalogue is empty/.test(warning)), + true, + ); + assert.equal(JSON.stringify(statusResult).includes('sk-status'), false); + const capabilities = await api.getCapabilities(); + assert.deepEqual(Object.keys(capabilities.capabilities).sort(), [ + 'indexing', + 'provider', + 'search', + 'storage', + 'supported', + ]); + }); + + it('rejects CMS-enabled schemas that are not extendable', async () => { + const { api, configs, createdIndexes, schemaExtensions } = createApi({ + schemas: { + Article: { + ...articleSchema, + modelOptions: { + conduit: { + cms: { enabled: true }, + permissions: { extendable: false }, + authorization: { enabled: true }, + }, + }, + }, + }, + }); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'text-embedding-3-small', + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not extendable/.test(err.message), + ); + assert.equal(configs.length, 0); + assert.equal(createdIndexes.length, 0); + assert.equal(schemaExtensions.length, 0); + }); + + it('rejects incompatible field collisions before provisioning extensions or indexes', async () => { + const { api, configs, createdIndexes, schemaExtensions } = createApi({ + schemas: { + Article: { + ...articleSchema, + fields: { + ...articleSchema.fields, + embedding: { type: TYPE.String }, + }, + }, + }, + declared: { + Article: { + name: 'Article', + ownerModule: 'database', + fields: { + title: { type: TYPE.String }, + body: { type: TYPE.String }, + embedding: { type: TYPE.String }, + }, + }, + }, + }); + await assert.rejects( + () => + api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'text-embedding-3-small', + enabled: false, + }, + { callerModule: 'database' }, + ), + (err: unknown) => + err instanceof GrpcError && + err.code === status.ALREADY_EXISTS && + /not a compatible embeddings extension/.test(err.message), + ); + assert.equal(configs.length, 0); + assert.equal(createdIndexes.length, 0); + assert.equal(schemaExtensions.length, 0); + }); + + it('is idempotent for compatible existing embeddings extensions', async () => { + const compatibleFields = { + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, + embeddingSourceHash: { type: TYPE.String, required: false, select: false }, + otherEmbedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, + }; + const { api, configs, schemaExtensions } = createApi({ + indexes: [readyIndex], + schemas: { + Article: { + ...articleSchema, + fields: { + ...articleSchema.fields, + ...compatibleFields, + }, + }, + }, + declared: { + Article: { + name: 'Article', + ownerModule: 'database', + fields: articleSchema.fields, + extensions: [ + { + ownerModule: 'embeddings', + fields: compatibleFields, + }, + ], + }, + }, + }); + const saved = await api.upsertConfig( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'text-embedding-3-small', + enabled: false, + }, + { callerModule: 'database' }, + ); + assert.equal(saved.config.enabled, false); + assert.equal(configs.length, 1); + assert.equal(schemaExtensions.length, 1); + assert.equal('otherEmbedding' in schemaExtensions[0].fields, true); + assert.equal('embedding' in schemaExtensions[0].fields, true); + }); +}); diff --git a/modules/embeddings/src/api/embeddingsApi.ts b/modules/embeddings/src/api/embeddingsApi.ts new file mode 100644 index 000000000..1e2e1d8c3 --- /dev/null +++ b/modules/embeddings/src/api/embeddingsApi.ts @@ -0,0 +1,1116 @@ +import { + GrpcError, + TYPE, + VectorCapabilities, + VectorIndexMethod, + VectorSearchResult, + VectorSimilarity, + type ConduitModel, + type VectorIndexDefinition, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { Config } from '../config/index.js'; +import { QueueJobCounts } from '../controllers/queue.controller.js'; +import { + cancelBackfillExecution, + persistableBackfillRun, + persistableNewBackfillRun, + queueBackfillRuns, + resumeBackfillExecution, + type BackfillControllerJobData, + type PersistedBackfillRun, +} from '../utils/backfillExecution.js'; +import { + BackfillGateError, + embeddingIndexContractFromConfig, + findTargetVectorIndex, + grpcErrorFromBackfillGate, + isEmbeddingVectorIndexQueryable, + type VectorIndexGate, +} from '../utils/backfillGates.js'; +import { + defaultEmbeddingVectorIndexName, + diffMaterialEmbeddingConfig, + hashFieldsToInvalidate, + isInPlaceDimensionChange, + materialChangeWarnings, + nextEmbeddingVectorIndexName, + sameEmbeddingVectorIndexFamily, + sourceHashField, + type MaterialEmbeddingConfigField, +} from '../utils/configChange.js'; +import { MAX_QUEUE_BATCH_SIZE } from '../utils/embeddingJobs.js'; +import { ACTIVE_BACKFILL_STATES } from '../utils/backfillRun.js'; +import { + assertCanManageEmbeddingConfig, + assertEmbeddingExtensionAvailability, + assertEmbeddingTargetSchema, + assertSchemaCanReceiveEmbeddings, + assertSemanticSearchAccess, + canManageEmbeddingConfig, + EMBEDDINGS_OWNER_MODULE, + resolveAdminOperatorContext, + resolveSourceFieldAllowlist, + type EmbeddingSchemaOptions, + type SchemaExtensionInfo, +} from '../utils/schemaPolicy.js'; +import { clampClientSearchLimit } from '../utils/clientSearchContext.js'; +import { validateEmbeddingConfigInput } from '../utils/validateEmbeddingConfig.js'; +import { + assertConfigActivation, + assertSearchExecutable, + capabilityWarnings, + emptyQueueCounts, + indexReadinessWarnings, + isEmbeddingsReady, + providerReadinessWarnings, + SearchGateError, + grpcErrorFromSearchGate, +} from '../utils/operationalStatus.js'; +import { + mapBackfillRun, + mapCapabilities, + mapEmbeddingConfig, + mapQueueCounts, + mapSearchHits, + parseJsonObject, + type MappedBackfillRun, + type MappedEmbeddingConfig, +} from '../utils/protoMappers.js'; +import { sanitizeErrorMessage } from '../utils/redactConfig.js'; + +export interface DeclaredSchemaInfo { + name: string; + ownerModule?: string; + fields?: Record; + extensions?: SchemaExtensionInfo[]; +} + +export interface SchemaInfo { + name: string; + fields: Record; + modelOptions?: EmbeddingSchemaOptions; +} + +export interface EmbeddingConfigRecord { + _id: string; + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + modelName: string; + dimensions: number; + similarity: string; + enabled: boolean; + createdAt?: Date | string; + updatedAt?: Date | string; +} + +export interface BackfillRunRecord extends PersistedBackfillRun { + createdAt?: Date | string; + updatedAt?: Date | string; +} + +export interface ConfigStore { + findMany: (query: Record) => Promise; + findOne: (query: Record) => Promise; + create: (doc: Record) => Promise; + findByIdAndUpdate: ( + id: string, + doc: Record, + ) => Promise; + deleteOne: (query: Record) => Promise; +} + +export interface BackfillStore { + findMany: ( + query: Record, + options?: { skip?: number; limit?: number; sort?: Record }, + ) => Promise; + findOne: (query: Record) => Promise; + countDocuments: (query: Record) => Promise; + create: (doc: Record) => Promise<{ _id: string }>; + findByIdAndUpdate: ( + id: string, + doc: Record, + ) => Promise; +} + +export interface EmbeddingsApiCaller { + callerModule?: string; + platformAdmin?: boolean; +} + +export interface EmbeddingsApiDeps { + currentConfig: () => Config; + getSchema: (schemaName: string) => Promise; + declaredSchema: (schemaName: string) => Promise; + setSchemaExtension: (args: { + schemaName: string; + fields: ConduitModel; + }) => Promise; + getVectorCapabilities: (schemaName?: string) => Promise; + getVectorIndexes: (schemaName: string) => Promise; + vectorSearch: (input: { + schemaName: string; + field: string; + vector: number[]; + filter?: Record; + limit?: number; + userId?: string; + scope?: string; + adminOperator?: boolean; + }) => Promise; + configs: ConfigStore; + backfills: BackfillStore; + getQueueStatus: () => Promise<{ generation: QueueJobCounts; backfill: QueueJobCounts }>; + enqueueBackfill: (job: BackfillControllerJobData) => Promise; + createVectorIndex: ( + schemaName: string, + index: VectorIndexDefinition, + ) => Promise; + deleteVectorIndex: (schemaName: string, indexName: string) => Promise; + invalidateHashes: (schemaName: string, hashFields: string[]) => Promise; + embed: (input: string, provider: string, model: string) => Promise; + onConfigChanged?: (schemaName: string) => Promise | void; +} + +const DEFAULT_LIST_LIMIT = 25; +const MAX_LIST_LIMIT = 100; + +export class EmbeddingsApi { + constructor(private readonly deps: EmbeddingsApiDeps) {} + + async upsertConfig( + request: { + schemaName: string; + sourceFields: string[]; + targetField: string; + provider?: string; + model?: string; + dimensions?: number; + similarity?: string; + sourceFieldAllowlist?: string[]; + enabled?: boolean; + }, + caller: EmbeddingsApiCaller, + ): Promise<{ config: MappedEmbeddingConfig; warnings: string[] }> { + const schema = await this.loadTargetSchema(request.schemaName, caller); + assertSchemaCanReceiveEmbeddings(schema); + const declared = await this.deps.declaredSchema(request.schemaName); + const configDefaults = this.deps.currentConfig(); + const { sourceFieldAllowlist: _allowlist, ...persisted } = + validateEmbeddingConfigInput( + { + ...request, + sourceFieldAllowlist: resolveSourceFieldAllowlist({ + operatorAllowlist: configDefaults.security.sourceFieldAllowlist, + requestAllowlist: request.sourceFieldAllowlist, + platformAdmin: caller.platformAdmin === true, + }), + }, + { + provider: configDefaults.defaultProvider, + providers: configDefaults.providers, + }, + schema.fields, + ); + assertEmbeddingExtensionAvailability({ + schemaName: persisted.schemaName, + targetField: persisted.targetField, + dimensions: persisted.dimensions, + similarity: persisted.similarity, + baseFields: declared?.fields, + compiledFields: schema.fields, + extensions: declared?.extensions, + }); + const enabled = request.enabled ?? true; + const capabilities = await this.deps.getVectorCapabilities(persisted.schemaName); + let indexes = await this.deps.getVectorIndexes(persisted.schemaName); + const existing = await this.deps.configs.findOne({ + schemaName: persisted.schemaName, + targetField: persisted.targetField, + }); + const changed = existing ? diffMaterialEmbeddingConfig(existing, persisted) : []; + if (existing && isInPlaceDimensionChange(existing, persisted)) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Changing vector field '${existing.targetField}' dimensions from ${existing.dimensions} to ${persisted.dimensions} is not allowed. Create a new targetField and run an explicit backfill.`, + ); + } + if (capabilities.storage) { + await this.extendEmbeddingSchema(persisted, declared); + } + const provisioned = await this.provisionUpsertIndexes({ + persisted, + existing, + capabilities, + indexes, + }); + indexes = provisioned.indexes; + let persistEnabled = enabled && provisioned.persistEnabled; + const warnings = this.upsertConfigWarnings({ + capabilities, + indexes, + persisted, + enabled, + configDefaults, + provisionWarnings: provisioned.provisionWarnings, + }); + if (enabled && persistEnabled) { + const activation = this.deferEnablementIfIndexPending({ + replacementIndexName: provisioned.replacementIndexName, + provisionedIndex: provisioned.provisionedIndex, + enabled, + capabilities, + persisted, + indexes, + moduleEnabled: configDefaults.enabled, + }); + persistEnabled = activation.persistEnabled; + warnings.push(...activation.warnings); + } + const saved = await this.saveUpsertedConfig({ + existing, + persisted, + persistEnabled, + changed, + capabilities, + indexes, + warnings, + }); + await this.deps.onConfigChanged?.(saved.schemaName); + return { config: mapEmbeddingConfig(saved), warnings }; + } + + async getConfigs( + request: { schemaName?: string; id?: string }, + caller: EmbeddingsApiCaller, + ): Promise<{ configs: MappedEmbeddingConfig[] }> { + if (request.id) { + const config = await this.requireConfig({ id: request.id }); + await this.loadTargetSchema(config.schemaName, caller); + return { configs: [mapEmbeddingConfig(config)] }; + } + if (request.schemaName) { + await this.loadTargetSchema(request.schemaName, caller); + const configs = await this.deps.configs.findMany({ + schemaName: request.schemaName, + }); + return { configs: configs.map(mapEmbeddingConfig) }; + } + this.assertOperatorListAccess(caller); + const configs = await this.deps.configs.findMany({}); + return { configs: configs.map(mapEmbeddingConfig) }; + } + + async deleteConfig( + request: { id?: string; schemaName?: string; targetField?: string }, + caller: EmbeddingsApiCaller, + ): Promise<{ config: MappedEmbeddingConfig }> { + const existing = await this.requireConfig(request); + await this.loadTargetSchema(existing.schemaName, caller); + await this.deps.configs.deleteOne({ _id: existing._id }); + await this.deps.onConfigChanged?.(existing.schemaName); + return { config: mapEmbeddingConfig(existing) }; + } + + async getCapabilities(schemaName?: string): Promise<{ + capabilities: ReturnType; + warnings: string[]; + }> { + const capabilities = await this.deps.getVectorCapabilities(schemaName); + return { + capabilities: mapCapabilities(capabilities), + warnings: capabilityWarnings(capabilities), + }; + } + + async getStatus(schemaName?: string): Promise<{ + enabled: boolean; + ready: boolean; + capabilities: ReturnType; + generationQueue: ReturnType; + backfillQueue: ReturnType; + warnings: string[]; + }> { + const config = this.deps.currentConfig(); + const capabilities = await this.deps.getVectorCapabilities(schemaName); + const queue = await this.deps.getQueueStatus().catch(() => ({ + generation: emptyQueueCounts(), + backfill: emptyQueueCounts(), + })); + const warnings = [ + ...(config.enabled ? [] : ['Embeddings module is disabled']), + ...capabilityWarnings(capabilities), + ...providerReadinessWarnings( + config.providers[config.defaultProvider] ?? Object.values(config.providers)[0], + ), + ]; + if (schemaName) { + const configs = await this.deps.configs.findMany({ schemaName, enabled: true }); + const indexes = await this.deps.getVectorIndexes(schemaName); + warnings.push(...indexReadinessWarnings(configs, indexes)); + } + return { + enabled: config.enabled, + ready: isEmbeddingsReady({ moduleEnabled: config.enabled, warnings }), + capabilities: mapCapabilities(capabilities), + generationQueue: mapQueueCounts(queue.generation), + backfillQueue: mapQueueCounts(queue.backfill), + warnings, + }; + } + + async startBackfill( + request: { + schemaName: string; + batchSize?: number; + configId?: string; + onlyMissing?: boolean; + filter?: string; + }, + caller: EmbeddingsApiCaller, + ): Promise<{ queued: number; runs: MappedBackfillRun[]; warnings: string[] }> { + await this.loadTargetSchema(request.schemaName, caller); + const filter = this.parseOptionalFilter(request.filter); + const [configs, capabilities, indexes] = await Promise.all([ + this.deps.configs.findMany({ + schemaName: request.schemaName, + ...(request.configId ? { _id: request.configId } : { enabled: true }), + }), + this.deps.getVectorCapabilities(request.schemaName), + this.deps.getVectorIndexes(request.schemaName), + ]); + const queued = await queueBackfillRuns( + { + schemaName: request.schemaName, + batchSize: request.batchSize, + configId: request.configId, + onlyMissing: request.onlyMissing, + filter, + maxBatchSize: + this.deps.currentConfig().queue.maxBatchSize ?? MAX_QUEUE_BATCH_SIZE, + }, + { + moduleEnabled: this.deps.currentConfig().enabled, + capabilities, + configs, + indexes, + createRun: async run => + this.deps.backfills.create(persistableNewBackfillRun(run)), + saveRun: async (id, run) => { + await this.deps.backfills.findByIdAndUpdate(id, persistableBackfillRun(run)); + }, + findActiveRuns: configId => this.findActiveBackfills(configId), + enqueueController: job => this.deps.enqueueBackfill(job), + }, + ); + const runs = await Promise.all( + queued.runs.map(async item => { + const persisted = await this.deps.backfills.findOne({ _id: item.id }); + if (!persisted) { + throw new GrpcError(status.INTERNAL, 'Failed to load queued backfill run'); + } + return mapBackfillRun(persisted); + }), + ); + return { + queued: queued.queued, + runs, + warnings: [ + ...capabilityWarnings(capabilities), + ...indexReadinessWarnings(configs, indexes), + ], + }; + } + + async getBackfill(id: string, caller: EmbeddingsApiCaller): Promise { + const run = await this.requireBackfill(id); + await this.loadTargetSchema(run.schemaName, caller); + return mapBackfillRun(run); + } + + async listBackfills( + request: { + schemaName?: string; + state?: string; + configId?: string; + skip?: number; + limit?: number; + }, + caller: EmbeddingsApiCaller, + ): Promise<{ runs: MappedBackfillRun[]; count: number }> { + if (request.schemaName) { + await this.loadTargetSchema(request.schemaName, caller); + } else { + this.assertOperatorListAccess(caller); + } + const query: Record = {}; + if (request.schemaName) query.schemaName = request.schemaName; + if (request.state) query.state = request.state; + if (request.configId) query.configId = request.configId; + const skip = Math.max(0, request.skip ?? 0); + const limit = Math.min( + MAX_LIST_LIMIT, + Math.max(1, request.limit ?? DEFAULT_LIST_LIMIT), + ); + const [runs, count] = await Promise.all([ + this.deps.backfills.findMany(query, { skip, limit, sort: { createdAt: -1 } }), + this.deps.backfills.countDocuments(query), + ]); + return { runs: runs.map(mapBackfillRun), count }; + } + + async cancelBackfill( + id: string, + caller: EmbeddingsApiCaller, + ): Promise<{ run: MappedBackfillRun }> { + const existing = await this.requireBackfill(id); + await this.loadTargetSchema(existing.schemaName, caller); + const result = await cancelBackfillExecution({ + run: existing, + saveRun: async (runId, run) => { + await this.deps.backfills.findByIdAndUpdate(runId, persistableBackfillRun(run)); + }, + }); + if (!result.ok) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Backfill run '${id}' cannot be canceled from state '${existing.state}'`, + ); + } + return { run: mapBackfillRun({ ...existing, ...result.run }) }; + } + + async resumeBackfill( + id: string, + caller: EmbeddingsApiCaller, + ): Promise<{ run: MappedBackfillRun }> { + const existing = await this.requireBackfill(id); + await this.loadTargetSchema(existing.schemaName, caller); + const config = existing.configId + ? await this.deps.configs.findOne({ _id: existing.configId }) + : await this.deps.configs.findOne({ + schemaName: existing.schemaName, + enabled: true, + }); + const [capabilities, indexes] = await Promise.all([ + this.deps.getVectorCapabilities(existing.schemaName), + this.deps.getVectorIndexes(existing.schemaName), + ]); + assertConfigActivation({ + moduleEnabled: this.deps.currentConfig().enabled, + capabilities, + config: config ?? null, + indexes, + }); + const result = await resumeBackfillExecution({ + run: existing, + saveRun: async (runId, run) => { + await this.deps.backfills.findByIdAndUpdate(runId, persistableBackfillRun(run)); + }, + enqueueController: job => this.deps.enqueueBackfill(job), + }); + if (!result.ok) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Backfill run '${id}' cannot be resumed from state '${existing.state}'`, + ); + } + return { run: mapBackfillRun({ ...existing, ...result.run }) }; + } + + async semanticSearch( + request: { + schemaName: string; + text: string; + targetField?: string; + filter?: string; + limit?: number; + userId?: string; + scope?: string; + adminOperator?: boolean; + }, + caller: EmbeddingsApiCaller, + ): Promise<{ hits: ReturnType }> { + if (typeof request.text !== 'string' || request.text.trim().length === 0) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Search text is required'); + } + const adminOperator = caller.platformAdmin + ? true + : resolveAdminOperatorContext({ + requested: request.adminOperator, + callerModule: caller.callerModule, + }); + const schema = await this.deps.getSchema(request.schemaName); + const declared = await this.deps.declaredSchema(request.schemaName); + assertEmbeddingTargetSchema({ + name: schema.name, + ownerModule: declared?.ownerModule, + }); + if (schema.modelOptions?.conduit?.authorization?.enabled) { + assertSemanticSearchAccess({ + userId: request.userId, + scope: request.scope, + adminOperator, + }); + } + const config = await this.resolveEnabledConfig( + request.schemaName, + request.targetField, + ); + const [capabilities, indexes] = await Promise.all([ + this.deps.getVectorCapabilities(request.schemaName), + this.deps.getVectorIndexes(request.schemaName), + ]); + assertSearchExecutable({ + capabilities, + config, + indexes, + }); + const vector = await this.deps.embed(request.text, config.provider, config.modelName); + if (vector.length !== config.dimensions) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Embedding provider returned ${vector.length} dimensions; expected ${config.dimensions}`, + ); + } + const results = await this.deps.vectorSearch({ + schemaName: request.schemaName, + field: config.targetField, + vector, + filter: this.parseOptionalFilter(request.filter), + limit: this.clampSemanticSearchLimit(request.limit, caller), + userId: request.userId, + scope: request.scope, + adminOperator, + }); + return { hits: mapSearchHits(results) }; + } + + mapGrpcError(err: unknown): { code: number; message: string } { + if (err instanceof BackfillGateError) { + const mapped = grpcErrorFromBackfillGate(err); + return { code: mapped.code, message: sanitizeErrorMessage(mapped) }; + } + if (err instanceof SearchGateError) { + const mapped = grpcErrorFromSearchGate(err); + return { code: mapped.code, message: sanitizeErrorMessage(mapped) }; + } + if (err instanceof GrpcError) { + return { code: err.code, message: sanitizeErrorMessage(err) }; + } + return { code: status.INTERNAL, message: sanitizeErrorMessage(err) }; + } + + private clampSemanticSearchLimit( + limit: number | undefined, + caller: EmbeddingsApiCaller, + ): number | undefined { + if (caller.platformAdmin || caller.callerModule !== 'router') return limit; + return clampClientSearchLimit(limit); + } + + private parseOptionalFilter(filter?: string): Record | undefined { + try { + return parseJsonObject(filter, 'filter'); + } catch { + throw new GrpcError(status.INVALID_ARGUMENT, 'filter must be a JSON object'); + } + } + + private async loadTargetSchema(schemaName: string, caller: EmbeddingsApiCaller) { + const schema = await this.deps.getSchema(schemaName); + const declared = await this.deps.declaredSchema(schemaName); + assertEmbeddingTargetSchema({ + name: schema.name, + ownerModule: declared?.ownerModule, + }); + if (!caller.platformAdmin) { + assertCanManageEmbeddingConfig({ + callerModule: caller.callerModule, + ownerModule: declared?.ownerModule, + schemaName: schema.name, + }); + } + return schema; + } + + private assertOperatorListAccess(caller: EmbeddingsApiCaller) { + if (caller.platformAdmin) return; + if (canManageEmbeddingConfig({ callerModule: caller.callerModule })) return; + throw new GrpcError( + status.PERMISSION_DENIED, + 'Listing embedding resources requires the schema owner or a platform operator', + ); + } + + private async requireConfig(request: { + id?: string; + schemaName?: string; + targetField?: string; + }): Promise { + const query: Record = {}; + if (request.id) query._id = request.id; + else if (request.schemaName && request.targetField) { + query.schemaName = request.schemaName; + query.targetField = request.targetField; + } else { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'id or schemaName and targetField are required', + ); + } + const existing = await this.deps.configs.findOne(query); + if (!existing) { + throw new GrpcError(status.NOT_FOUND, 'Embedding config not found'); + } + return existing; + } + + private async requireBackfill(id: string): Promise { + if (!id) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Backfill id is required'); + } + const run = await this.deps.backfills.findOne({ _id: id }); + if (!run) { + throw new GrpcError(status.NOT_FOUND, 'Backfill run not found'); + } + return run; + } + + private async resolveEnabledConfig(schemaName: string, targetField?: string) { + const query: Record = { schemaName, enabled: true }; + if (targetField) query.targetField = targetField; + const config = await this.deps.configs.findOne(query); + if (!config) { + throw new GrpcError( + status.NOT_FOUND, + 'No embedding config found for semantic search', + ); + } + return config; + } + + private async extendEmbeddingSchema( + persisted: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: VectorSimilarity; + }, + declared?: DeclaredSchemaInfo | null, + ) { + const hashField = sourceHashField(persisted.targetField); + const existing = + declared?.extensions?.find( + extension => extension.ownerModule === EMBEDDINGS_OWNER_MODULE, + )?.fields ?? {}; + await this.deps.setSchemaExtension({ + schemaName: persisted.schemaName, + fields: { + ...existing, + [persisted.targetField]: { + type: TYPE.Vector, + dimensions: persisted.dimensions, + similarity: persisted.similarity, + select: false, + }, + [hashField]: { + type: TYPE.String, + required: false, + select: false, + }, + } as ConduitModel, + }); + } + + private async provisionUpsertIndexes(args: { + persisted: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + }; + existing: EmbeddingConfigRecord | null; + capabilities: VectorCapabilities; + indexes: VectorIndexGate[]; + }): Promise<{ + indexes: VectorIndexGate[]; + persistEnabled: boolean; + provisionedIndex: boolean; + replacementIndexName?: string; + provisionWarnings: string[]; + }> { + let { indexes } = args; + let persistEnabled = true; + let provisionedIndex = false; + let replacementIndexName: string | undefined; + const provisionWarnings: string[] = []; + try { + const matchingIndex = findTargetVectorIndex( + indexes, + args.persisted.targetField, + embeddingIndexContractFromConfig(args.persisted), + ); + const hasFieldIndex = indexes.some( + index => index.field === args.persisted.targetField, + ); + if (!matchingIndex && hasFieldIndex && args.capabilities.indexing) { + replacementIndexName = await this.recreateVectorIndex(args.persisted, indexes); + indexes = await this.deps.getVectorIndexes(args.persisted.schemaName); + provisionedIndex = true; + } else if (!matchingIndex) { + provisionedIndex = await this.ensureVectorIndex( + args.persisted, + indexes, + args.capabilities, + ); + if (provisionedIndex) { + indexes = await this.deps.getVectorIndexes(args.persisted.schemaName); + } + } + indexes = await this.retireSupersededVectorIndexes({ + schemaName: args.persisted.schemaName, + targetField: args.persisted.targetField, + dimensions: args.persisted.dimensions, + similarity: args.persisted.similarity, + previousField: args.existing?.targetField, + indexes, + }); + } catch (err) { + persistEnabled = false; + provisionWarnings.push( + `Config was saved disabled because vector index provisioning failed for '${args.persisted.targetField}': ${sanitizeErrorMessage(err)}. Repair or create the index and enable the config once Database reports it queryable.`, + ); + } + return { + indexes, + persistEnabled, + provisionedIndex, + replacementIndexName, + provisionWarnings, + }; + } + + private upsertConfigWarnings(args: { + capabilities: VectorCapabilities; + indexes: VectorIndexGate[]; + persisted: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + provider: string; + }; + enabled: boolean; + configDefaults: Config; + provisionWarnings: string[]; + }): string[] { + const warnings = [ + ...capabilityWarnings(args.capabilities), + ...indexReadinessWarnings( + [ + { + targetField: args.persisted.targetField, + enabled: args.enabled, + schemaName: args.persisted.schemaName, + dimensions: args.persisted.dimensions, + similarity: args.persisted.similarity, + }, + ], + args.indexes, + ), + ...providerReadinessWarnings( + args.configDefaults.providers[args.persisted.provider] ?? + args.configDefaults.providers[args.configDefaults.defaultProvider], + ), + ...args.provisionWarnings, + ]; + if ( + !findTargetVectorIndex( + args.indexes, + args.persisted.targetField, + embeddingIndexContractFromConfig(args.persisted), + ) && + !args.capabilities.indexing + ) { + warnings.push( + `Vector index for field '${args.persisted.targetField}' was not provisioned automatically because Database indexing is unavailable. Create the index manually and wait until it is queryable before enabling this config.`, + ); + } + return warnings; + } + + private deferEnablementIfIndexPending(args: { + replacementIndexName?: string; + provisionedIndex: boolean; + enabled: boolean; + capabilities: VectorCapabilities; + persisted: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + }; + indexes: VectorIndexGate[]; + moduleEnabled: boolean; + }): { persistEnabled: boolean; warnings: string[] } { + try { + if (args.replacementIndexName) { + const replacement = args.indexes.find( + index => index.name === args.replacementIndexName, + ); + if (!isEmbeddingVectorIndexQueryable(replacement)) { + throw new BackfillGateError( + 'index_not_queryable', + `Vector index '${args.replacementIndexName}' is not queryable (status: ${ + replacement?.status ?? 'missing' + }). Wait until the index is ready before enabling this config.`, + replacement?.status ?? 'missing', + ); + } + } + assertConfigActivation({ + moduleEnabled: args.moduleEnabled, + capabilities: args.capabilities, + config: { + enabled: args.enabled, + schemaName: args.persisted.schemaName, + targetField: args.persisted.targetField, + dimensions: args.persisted.dimensions, + similarity: args.persisted.similarity, + }, + indexes: args.indexes, + }); + return { persistEnabled: true, warnings: [] }; + } catch (err) { + const indexPending = + (err instanceof BackfillGateError && err.reason === 'index_not_queryable') || + (err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not queryable/.test(err.message)); + if (!indexPending) throw err; + return { + persistEnabled: false, + warnings: [ + args.provisionedIndex + ? 'Config was saved disabled until the provisioned vector index is queryable. Enable it once Database reports the index ready.' + : 'Config was saved disabled until the vector index is queryable. Enable it once Database reports the index ready.', + ], + }; + } + } + + private async saveUpsertedConfig(args: { + existing: EmbeddingConfigRecord | null; + persisted: Record & { + schemaName: string; + targetField: string; + }; + persistEnabled: boolean; + changed: MaterialEmbeddingConfigField[]; + capabilities: VectorCapabilities; + indexes: VectorIndexGate[]; + warnings: string[]; + }): Promise { + const saved = args.existing + ? await this.deps.configs.findByIdAndUpdate(args.existing._id, { + ...args.persisted, + enabled: args.persistEnabled, + }) + : await this.deps.configs.create({ + ...args.persisted, + enabled: args.persistEnabled, + }); + if (!saved) { + throw new GrpcError(status.INTERNAL, 'Failed to persist embedding config'); + } + if (args.existing && args.changed.length) { + await this.deps.invalidateHashes( + saved.schemaName, + hashFieldsToInvalidate(args.existing, saved), + ); + await this.supersedeActiveBackfills(saved._id); + let scheduledBackfill = false; + if (args.persistEnabled) { + scheduledBackfill = await this.scheduleExplicitBackfill(saved, { + capabilities: args.capabilities, + indexes: args.indexes, + }); + } + args.warnings.push(...materialChangeWarnings(args.changed, scheduledBackfill)); + } + return saved; + } + + private async findActiveBackfills(configId: string): Promise { + const runs: PersistedBackfillRun[] = []; + for (const state of ACTIVE_BACKFILL_STATES) { + const found = await this.deps.backfills.findMany({ configId, state }); + runs.push(...found); + } + return runs; + } + + private async ensureVectorIndex( + next: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + }, + indexes: VectorIndexGate[], + capabilities: VectorCapabilities, + ): Promise { + if ( + findTargetVectorIndex( + indexes, + next.targetField, + embeddingIndexContractFromConfig(next), + ) + ) { + return false; + } + if (!capabilities.indexing) return false; + try { + await this.deps.createVectorIndex(next.schemaName, { + field: next.targetField, + dimensions: next.dimensions, + similarity: next.similarity as VectorIndexDefinition['similarity'], + name: defaultEmbeddingVectorIndexName(next.targetField), + method: VectorIndexMethod.HNSW, + }); + } catch (err) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Failed to provision vector index for '${next.targetField}': ${sanitizeErrorMessage(err)}`, + ); + } + return true; + } + + private async recreateVectorIndex( + next: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + }, + indexes: VectorIndexGate[], + ): Promise { + const replacementName = nextEmbeddingVectorIndexName(next.targetField, indexes); + try { + await this.deps.createVectorIndex(next.schemaName, { + field: next.targetField, + dimensions: next.dimensions, + similarity: next.similarity as VectorIndexDefinition['similarity'], + name: replacementName, + method: VectorIndexMethod.HNSW, + }); + } catch (err) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Failed to provision replacement vector index for '${next.targetField}': ${sanitizeErrorMessage(err)}`, + ); + } + return replacementName; + } + + private async retireSupersededVectorIndexes(args: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + previousField?: string; + indexes: VectorIndexGate[]; + }): Promise { + const selected = findTargetVectorIndex(args.indexes, args.targetField, { + dimensions: args.dimensions, + similarity: args.similarity, + }); + if (!selected?.name || !isEmbeddingVectorIndexQueryable(selected)) { + return args.indexes; + } + const retireNames = new Set(); + for (const index of args.indexes) { + if (!index.name || index.name === selected.name) continue; + if ( + index.field === args.targetField && + sameEmbeddingVectorIndexFamily(index.name, selected.name) + ) { + retireNames.add(index.name); + } + } + if (args.previousField && args.previousField !== args.targetField) { + const previous = findTargetVectorIndex(args.indexes, args.previousField); + if (previous?.name && previous.name !== selected.name) { + retireNames.add(previous.name); + } + } + if (!retireNames.size) return args.indexes; + for (const name of retireNames) { + try { + await this.deps.deleteVectorIndex(args.schemaName, name); + } catch (err) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Failed to retire superseded vector index '${name}': ${sanitizeErrorMessage(err)}`, + ); + } + } + return this.deps.getVectorIndexes(args.schemaName); + } + + private async supersedeActiveBackfills(configId: string): Promise { + const active = await this.findActiveBackfills(configId); + for (const run of active) { + await cancelBackfillExecution({ + run, + saveRun: async (id, next) => { + await this.deps.backfills.findByIdAndUpdate(id, persistableBackfillRun(next)); + }, + }); + } + } + + private async scheduleExplicitBackfill( + config: EmbeddingConfigRecord, + args: { + capabilities: VectorCapabilities; + indexes: VectorIndexGate[]; + }, + ): Promise { + try { + const queued = await queueBackfillRuns( + { + schemaName: config.schemaName, + configId: config._id, + onlyMissing: false, + maxBatchSize: + this.deps.currentConfig().queue.maxBatchSize ?? MAX_QUEUE_BATCH_SIZE, + }, + { + moduleEnabled: this.deps.currentConfig().enabled, + capabilities: args.capabilities, + configs: [config], + indexes: args.indexes, + createRun: async run => + this.deps.backfills.create(persistableNewBackfillRun(run)), + saveRun: async (id, run) => { + await this.deps.backfills.findByIdAndUpdate(id, persistableBackfillRun(run)); + }, + findActiveRuns: configId => this.findActiveBackfills(configId), + enqueueController: job => this.deps.enqueueBackfill(job), + }, + ); + return queued.queued > 0; + } catch (err) { + if (err instanceof BackfillGateError) { + return false; + } + throw err; + } + } +} diff --git a/modules/embeddings/src/config/index.ts b/modules/embeddings/src/config/index.ts new file mode 100644 index 000000000..f1443dcc8 --- /dev/null +++ b/modules/embeddings/src/config/index.ts @@ -0,0 +1,106 @@ +import convict from 'convict'; + +const AppConfigSchema = { + doc: 'Embeddings module configuration', + enabled: { + doc: 'Enable embedding generation workers and event subscriptions', + format: 'Boolean', + default: false, + }, + defaultProvider: { + doc: 'Default embedding provider', + format: 'String', + default: 'openai-compatible', + }, + providers: { + 'openai-compatible': { + endpoint: { + doc: 'HTTPS embedding provider endpoint', + format: String, + default: '', + }, + apiKey: { + doc: 'Provider API key', + format: String, + default: '', + sensitive: true, + }, + models: { + doc: 'Operator-managed embedding models and output dimensions', + format: Array, + default: [], + }, + defaultModel: { + doc: 'Default model name from the provider catalogue', + format: String, + default: '', + }, + }, + }, + queue: { + concurrency: { + doc: 'Embedding generation worker concurrency', + format: 'Number', + default: 2, + }, + attempts: { + doc: 'Embedding generation retry attempts', + format: 'Number', + default: 3, + }, + maxBatchSize: { + doc: 'Maximum jobs accepted from a single enqueue or backfill request', + format: 'Number', + default: 500, + }, + drainTimeoutMs: { + doc: 'Maximum time a backfill may wait for generation jobs during drain before failing', + format: 'Number', + default: 15 * 60 * 1000, + }, + }, + security: { + sourceFieldAllowlist: { + doc: 'Operator-configured source fields allowed even when hidden or sensitive-named. Caller-supplied allowlists are honored only for platform-admin upserts.', + format: Array, + default: [], + }, + maxMutationEventIds: { + doc: 'Maximum document ids accepted from a single mutation bus payload', + format: 'Number', + default: 500, + }, + embedTimeoutMs: { + doc: 'Provider request timeout in milliseconds', + format: 'Number', + default: 10_000, + }, + maxEmbedInputBytes: { + doc: 'Maximum embedding input payload size in bytes', + format: 'Number', + default: 32 * 1024, + }, + maxEmbedResponseBytes: { + doc: 'Maximum embedding provider response size in bytes', + format: 'Number', + default: 1024 * 1024, + }, + }, +}; + +const config = convict(AppConfigSchema); +void config; +export type EmbeddingProviderModel = { + name: string; + dimensions: number; +}; +export type EmbeddingProviderSettings = { + endpoint?: string; + apiKey?: string; + models?: EmbeddingProviderModel[]; + defaultModel?: string; +}; +export type Config = ReturnType & { + providers: Record; +}; +export default AppConfigSchema; diff --git a/modules/embeddings/src/controllers/queue.controller.test.ts b/modules/embeddings/src/controllers/queue.controller.test.ts new file mode 100644 index 000000000..c6ab25f58 --- /dev/null +++ b/modules/embeddings/src/controllers/queue.controller.test.ts @@ -0,0 +1,297 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { QueueController } from './queue.controller.js'; +import { embeddingJobId } from '../utils/embeddingJobs.js'; +import { EMBEDDING_METRICS } from '../utils/embeddingMetrics.js'; + +type StoredJob = { + name: string; + data: Record; + opts?: { jobId?: string; delay?: number; attempts?: number }; + state?: string; +}; + +class FakeQueue { + jobs: StoredJob[] = []; + closed = false; + + async add(name: string, data: Record, opts?: StoredJob['opts']) { + if (opts?.jobId && this.jobs.some(job => job.opts?.jobId === opts.jobId)) { + throw new Error(`Job ${opts.jobId} already exists`); + } + this.jobs.push({ name, data, opts, state: 'waiting' }); + } + + async addBulk(jobs: StoredJob[]) { + for (const job of jobs) { + await this.add(job.name, job.data, job.opts); + } + } + + async getJob(jobId: string) { + const job = this.jobs.find(stored => stored.opts?.jobId === jobId); + if (!job) return undefined; + return { + getState: async () => job.state ?? 'waiting', + remove: async () => { + this.jobs = this.jobs.filter(stored => stored !== job); + }, + }; + } + + markState(jobId: string, state: string) { + const job = this.jobs.find(stored => stored.opts?.jobId === jobId); + if (job) job.state = state; + } + + async getJobCounts() { + return { + waiting: this.jobs.length, + active: 0, + completed: 0, + failed: 0, + delayed: this.jobs.filter(job => (job.opts?.delay ?? 0) > 0).length, + paused: 0, + }; + } + + async close() { + this.closed = true; + } +} + +class FakeWorker { + static instances: FakeWorker[] = []; + closed = false; + concurrency: number; + name: string; + handlers: Record void> = {}; + + constructor( + name: string, + _processor: (job: { data: unknown }) => Promise, + opts: { concurrency: number }, + ) { + this.name = name; + this.concurrency = opts.concurrency; + FakeWorker.instances.push(this); + } + + on(event: string, handler: (...args: unknown[]) => void) { + this.handlers[event] = handler; + return this; + } + + async close() { + this.closed = true; + } +} + +function createController( + queue: FakeQueue = new FakeQueue(), + backfillQueue: FakeQueue = new FakeQueue(), +) { + return { + queue, + backfillQueue, + controller: new QueueController(fakeSdk(), { + Queue: class { + constructor(name: string) { + return name.includes('backfill') ? backfillQueue : queue; + } + } as never, + Worker: FakeWorker as never, + }), + }; +} + +function fakeSdk() { + return { + redisManager: { + getClient: () => ({ quit: async () => 'OK' }), + }, + } as unknown as ConduitGrpcSdk; +} + +function withMetrics() { + const seen: Array<{ name: string; amount?: number; labels?: unknown }> = []; + const previous = ConduitGrpcSdk.Metrics; + ConduitGrpcSdk.Metrics = { + increment(name: string, amount?: number, labels?: unknown) { + seen.push({ name, amount, labels }); + }, + } as never; + return { + seen, + restore() { + ConduitGrpcSdk.Metrics = previous; + }, + }; +} + +describe('embedding queue worker lifecycle', () => { + it('keeps a single worker, recreates on concurrency change, and closes idempotently', async () => { + FakeWorker.instances = []; + const { controller } = createController(); + + await controller.ensureWorker(async () => undefined, 2); + await controller.ensureWorker(async () => undefined, 2); + assert.equal(FakeWorker.instances.length, 1); + assert.equal(controller.hasWorker, true); + assert.equal(controller.currentConcurrency, 2); + + await controller.ensureWorker(async () => undefined, 4); + assert.equal(FakeWorker.instances.length, 2); + assert.equal(FakeWorker.instances[0].closed, true); + assert.equal(FakeWorker.instances[1].closed, false); + assert.equal(controller.currentConcurrency, 4); + + await controller.closeWorker(); + await controller.closeWorker(); + assert.equal(FakeWorker.instances[1].closed, true); + assert.equal(controller.hasWorker, false); + }); + + it('does not recreate the backfill worker when generation concurrency changes', async () => { + FakeWorker.instances = []; + const { controller } = createController(); + await controller.ensureWorker(async () => undefined, 2); + await controller.ensureBackfillWorker(async () => undefined, 1); + assert.equal(controller.hasBackfillWorker, true); + await controller.ensureWorker(async () => undefined, 3); + const backfill = FakeWorker.instances.find( + worker => worker.name === 'embeddings-backfill-queue', + ); + assert.equal(backfill?.closed, false); + assert.equal(controller.hasBackfillWorker, true); + }); + + it('deduplicates queued jobs by identity', async () => { + FakeWorker.instances = []; + const { queue, controller } = createController(); + const job = { schemaName: 'Article', documentId: 'a' }; + await controller.addEmbeddingJob(job, 3); + await controller.addEmbeddingJob(job, 3); + await controller.addBulkEmbeddingJobs( + [job, { schemaName: 'Article', documentId: 'b' }, job], + 3, + ); + assert.deepEqual( + queue.jobs.map(stored => stored.opts?.jobId), + [embeddingJobId(job), embeddingJobId({ schemaName: 'Article', documentId: 'b' })], + ); + }); + + it('re-enqueues the same identity after a retained completed or failed job', async () => { + FakeWorker.instances = []; + const { queue, controller } = createController(); + const job = { schemaName: 'Article', documentId: 'a' }; + assert.equal(await controller.addEmbeddingJob(job, 3), 1); + queue.markState(embeddingJobId(job), 'completed'); + assert.equal(await controller.addEmbeddingJob(job, 3), 1); + assert.equal(queue.jobs.length, 1); + assert.equal(queue.jobs[0].state, 'waiting'); + queue.markState(embeddingJobId(job), 'failed'); + assert.equal( + await controller.addBulkEmbeddingJobs( + [job, { schemaName: 'Article', documentId: 'b' }], + 3, + ), + 2, + ); + assert.deepEqual( + queue.jobs.map(stored => stored.opts?.jobId), + [embeddingJobId(job), embeddingJobId({ schemaName: 'Article', documentId: 'b' })], + ); + }); + + it('skips malformed queue payloads instead of throwing', async () => { + FakeWorker.instances = []; + const { queue, controller } = createController(); + await controller.addEmbeddingJob( + { schemaName: '../nope', documentId: 'a' } as never, + 3, + ); + await controller.addBulkEmbeddingJobs( + [ + { schemaName: 'Article', documentId: 'ok' }, + { schemaName: 'Article', documentId: '' } as never, + ], + 3, + ); + assert.deepEqual( + queue.jobs.map(stored => stored.opts?.jobId), + [embeddingJobId({ schemaName: 'Article', documentId: 'ok' })], + ); + }); +}); + +describe('embedding queue status and backfill jobs', () => { + it('reports generation and backfill counts separately', async () => { + const { controller, queue, backfillQueue } = createController(); + await controller.addEmbeddingJob({ schemaName: 'Article', documentId: 'a' }, 3); + await controller.addBackfillControllerJob({ runId: 'run1', cursor: null }); + const status = await controller.getQueueStatus(); + assert.equal(status.generation.waiting, queue.jobs.length); + assert.equal(status.backfill.waiting, backfillQueue.jobs.length); + assert.equal((await controller.getJobCounts('generation')).waiting, 1); + assert.equal((await controller.getJobCounts('backfill')).waiting, 1); + }); + + it('enqueues lightweight backfill controller jobs with cursor identity', async () => { + const { backfillQueue, controller } = createController(); + await controller.addBackfillControllerJob({ runId: 'run1', cursor: null }); + await controller.addBackfillControllerJob({ runId: 'run1', cursor: null }); + await controller.addBackfillControllerJob({ runId: 'run1', cursor: 'b' }); + await controller.addBackfillControllerJob({ runId: 'run1', drain: true }); + assert.deepEqual( + backfillQueue.jobs.map(job => job.opts?.jobId), + ['backfill:run1:start', 'backfill:run1:b', undefined], + ); + assert.equal( + backfillQueue.jobs.some(job => job.opts?.delay === 1000), + true, + ); + }); + + it('does not let a completed backfill page job block a later scan of the same cursor', async () => { + const { backfillQueue, controller } = createController(); + await controller.addBackfillControllerJob({ runId: 'run1', cursor: 'b' }); + backfillQueue.markState('backfill:run1:b', 'completed'); + await controller.addBackfillControllerJob({ runId: 'run1', cursor: 'b' }); + assert.deepEqual( + backfillQueue.jobs.map(job => job.opts?.jobId), + ['backfill:run1:b'], + ); + assert.equal(backfillQueue.jobs[0].state, 'waiting'); + }); + + it('increments retried then failed metrics without job payload labels', async () => { + FakeWorker.instances = []; + const metrics = withMetrics(); + try { + const { controller } = createController(); + await controller.ensureWorker(async () => undefined, 1); + const worker = FakeWorker.instances[0]; + const job = { + data: { schemaName: 'Article', documentId: 'a', backfillRunId: 'run1' }, + attemptsMade: 1, + opts: { attempts: 3 }, + }; + worker.handlers.failed?.(job, new Error('provider timeout apiKey=sk-secret')); + job.attemptsMade = 3; + worker.handlers.failed?.(job, new Error('provider timeout')); + assert.deepEqual( + metrics.seen.map(item => item.name), + [EMBEDDING_METRICS.retried, EMBEDDING_METRICS.failed], + ); + assert.equal( + metrics.seen.every(item => item.labels === undefined), + true, + ); + } finally { + metrics.restore(); + } + }); +}); diff --git a/modules/embeddings/src/controllers/queue.controller.ts b/modules/embeddings/src/controllers/queue.controller.ts new file mode 100644 index 000000000..61b9526c7 --- /dev/null +++ b/modules/embeddings/src/controllers/queue.controller.ts @@ -0,0 +1,441 @@ +import { Queue, Worker } from 'bullmq'; +import { Cluster, Redis } from 'ioredis'; +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { + EmbeddingJobData, + dedupeEmbeddingJobs, + embeddingJobId, + isDuplicateJobError, + isInFlightQueueJobState, + parseEmbeddingJobData, + shouldReplaceRetainedQueueJob, +} from '../utils/embeddingJobs.js'; +import { + BackfillControllerJobData, + parseBackfillControllerJob, + BACKFILL_DRAIN_DELAY_MS, +} from '../utils/backfillExecution.js'; +import { incrementEmbeddingMetric } from '../utils/embeddingMetrics.js'; +import { sanitizeErrorMessage } from '../utils/redactConfig.js'; + +export type { EmbeddingJobData } from '../utils/embeddingJobs.js'; +export type { BackfillControllerJobData } from '../utils/backfillExecution.js'; + +type RedisConnection = Redis | Cluster; + +export interface QueueJobCounts { + waiting: number; + active: number; + completed: number; + failed: number; + delayed: number; + paused: number; +} + +export interface EmbeddingQueueStatus { + generation: QueueJobCounts; + backfill: QueueJobCounts; +} + +type QueueJobHandle = { + getState: () => Promise; + remove: () => Promise; +}; + +type QueueLike = { + add: ( + name: string, + data: Record, + opts?: Record, + ) => Promise; + addBulk: ( + jobs: Array<{ + name: string; + data: Record; + opts?: Record; + }>, + ) => Promise; + close: () => Promise; + getJob?: (jobId: string) => Promise; + getJobCounts: () => Promise & Record>; +}; + +type WorkerJob = { + data?: unknown; + attemptsMade?: number; + opts?: { attempts?: number }; +}; + +type WorkerLike = { + on: (event: string, handler: (...args: unknown[]) => void) => unknown; + close: () => Promise; +}; + +export interface QueueControllerDependencies { + createConnection?: () => RedisConnection; + Queue?: new (name: string, opts: { connection: RedisConnection }) => QueueLike; + Worker?: new ( + name: string, + processor: (job: WorkerJob) => Promise, + opts: { connection: RedisConnection; concurrency: number } & Record, + ) => WorkerLike; +} + +export class QueueController { + private static _instance: QueueController; + private readonly createConnection: () => RedisConnection; + private readonly QueueImpl: NonNullable; + private readonly WorkerImpl: NonNullable; + private readonly queueConnection: RedisConnection; + private readonly embeddingQueue: QueueLike; + private readonly backfillQueue: QueueLike; + private worker?: WorkerLike; + private workerConnection?: RedisConnection; + private workerConcurrency?: number; + private closingWorker = false; + private backfillWorker?: WorkerLike; + private backfillWorkerConnection?: RedisConnection; + private backfillWorkerConcurrency?: number; + private closingBackfillWorker = false; + private onBackfillJobOutcome?: ( + runId: string, + outcome: 'processed' | 'failed', + ) => Promise; + + constructor( + private readonly grpcSdk: ConduitGrpcSdk, + deps: QueueControllerDependencies = {}, + ) { + this.createConnection = + deps.createConnection ?? (() => this.grpcSdk.redisManager.getClient()); + this.QueueImpl = + deps.Queue ?? + (Queue as unknown as NonNullable); + this.WorkerImpl = + deps.Worker ?? + (Worker as unknown as NonNullable); + this.queueConnection = this.createConnection(); + this.embeddingQueue = new this.QueueImpl('embeddings-generation-queue', { + connection: this.queueConnection, + }); + this.backfillQueue = new this.QueueImpl('embeddings-backfill-queue', { + connection: this.queueConnection, + }); + } + + static getInstance(grpcSdk?: ConduitGrpcSdk, deps?: QueueControllerDependencies) { + if (QueueController._instance) return QueueController._instance; + if (!grpcSdk) throw new Error('No grpcSdk instance provided!'); + return (QueueController._instance = new QueueController(grpcSdk, deps)); + } + + static resetInstance() { + QueueController._instance = undefined as unknown as QueueController; + } + + get hasWorker() { + return this.worker !== undefined; + } + + get hasBackfillWorker() { + return this.backfillWorker !== undefined; + } + + get currentConcurrency() { + return this.workerConcurrency; + } + + get currentBackfillConcurrency() { + return this.backfillWorkerConcurrency; + } + + setBackfillJobOutcomeHandler( + handler?: (runId: string, outcome: 'processed' | 'failed') => Promise, + ) { + this.onBackfillJobOutcome = handler; + } + + async ensureWorker( + processor: (data: EmbeddingJobData) => Promise, + concurrency: number, + ) { + if (this.worker && this.workerConcurrency === concurrency) { + return this.worker; + } + await this.closeGenerationWorker(); + this.workerConnection = this.createConnection(); + const worker = new this.WorkerImpl( + 'embeddings-generation-queue', + job => { + const parsed = parseEmbeddingJobData(job.data); + if (!parsed.ok) { + incrementEmbeddingMetric('malformedJobs'); + return Promise.resolve(); + } + return processor(parsed.data); + }, + { + concurrency, + connection: this.workerConnection, + removeOnComplete: { age: 3600, count: 1000 }, + removeOnFail: { age: 24 * 3600 }, + }, + ); + worker.on('failed', (job, error) => + this.handleGenerationFailure(job as WorkerJob | undefined, error), + ); + worker.on('error', error => ConduitGrpcSdk.Logger.error(sanitizeErrorMessage(error))); + this.worker = worker; + this.workerConcurrency = concurrency; + return worker; + } + + async ensureBackfillWorker( + processor: (data: BackfillControllerJobData) => Promise, + concurrency: number, + ) { + if (this.backfillWorker && this.backfillWorkerConcurrency === concurrency) { + return this.backfillWorker; + } + await this.closeBackfillWorker(); + this.backfillWorkerConnection = this.createConnection(); + const worker = new this.WorkerImpl( + 'embeddings-backfill-queue', + job => { + const parsed = parseBackfillControllerJob(job.data); + if (!parsed.ok) { + incrementEmbeddingMetric('malformedJobs'); + return Promise.resolve(); + } + return processor(parsed.data); + }, + { + concurrency, + connection: this.backfillWorkerConnection, + removeOnComplete: { age: 3600, count: 1000 }, + removeOnFail: { age: 24 * 3600 }, + }, + ); + worker.on('failed', (_job, error) => + ConduitGrpcSdk.Logger.error(sanitizeErrorMessage(error)), + ); + worker.on('error', error => ConduitGrpcSdk.Logger.error(sanitizeErrorMessage(error))); + this.backfillWorker = worker; + this.backfillWorkerConcurrency = concurrency; + return worker; + } + + async closeWorker() { + await Promise.all([this.closeGenerationWorker(), this.closeBackfillWorker()]); + } + + async close() { + await this.closeWorker(); + await this.embeddingQueue.close(); + await this.backfillQueue.close(); + await this.queueConnection.quit(); + } + + async getJobCounts( + queue: 'generation' | 'backfill' = 'generation', + ): Promise { + const counts = + queue === 'backfill' + ? await this.backfillQueue.getJobCounts() + : await this.embeddingQueue.getJobCounts(); + return normalizeJobCounts(counts); + } + + async getQueueStatus(): Promise { + const [generation, backfill] = await Promise.all([ + this.getJobCounts('generation'), + this.getJobCounts('backfill'), + ]); + return { generation, backfill }; + } + + async addEmbeddingJob(data: EmbeddingJobData, attempts: number) { + const parsed = parseEmbeddingJobData(data); + if (!parsed.ok) { + incrementEmbeddingMetric('malformedJobs'); + return 0; + } + const jobId = embeddingJobId(parsed.data); + const decision = await resolveExistingQueueJob(this.embeddingQueue, jobId); + if (decision === 'skip') return 0; + try { + await this.embeddingQueue.add( + jobId, + { ...parsed.data }, + { + jobId, + attempts, + backoff: { type: 'exponential', delay: 1000 }, + }, + ); + return 1; + } catch (err) { + if (!isDuplicateJobError(err)) throw err; + return 0; + } + } + + async addBulkEmbeddingJobs(data: EmbeddingJobData[], attempts: number) { + const jobs: EmbeddingJobData[] = []; + for (const [index, item] of data.entries()) { + const parsed = parseEmbeddingJobData(item, index); + if (!parsed.ok) { + incrementEmbeddingMetric('malformedJobs'); + continue; + } + jobs.push(parsed.data); + } + const unique = dedupeEmbeddingJobs(jobs); + if (!unique.length) return 0; + const enqueueable: EmbeddingJobData[] = []; + for (const job of unique) { + const decision = await resolveExistingQueueJob( + this.embeddingQueue, + embeddingJobId(job), + ); + if (decision === 'skip') continue; + enqueueable.push(job); + } + if (!enqueueable.length) return 0; + try { + await this.embeddingQueue.addBulk( + enqueueable.map(job => ({ + name: embeddingJobId(job), + data: { ...job }, + opts: { + jobId: embeddingJobId(job), + attempts, + backoff: { type: 'exponential', delay: 1000 }, + }, + })), + ); + return enqueueable.length; + } catch (err) { + if (!isDuplicateJobError(err)) throw err; + const added = await Promise.all( + enqueueable.map(job => this.addEmbeddingJob(job, attempts)), + ); + let queued = 0; + for (const count of added) queued += count; + return queued; + } + } + + async addBackfillControllerJob( + data: BackfillControllerJobData, + opts?: { delay?: number }, + ) { + const parsed = parseBackfillControllerJob(data); + if (!parsed.ok) { + incrementEmbeddingMetric('malformedJobs'); + return; + } + const delay = + parsed.data.drain === true ? (opts?.delay ?? BACKFILL_DRAIN_DELAY_MS) : opts?.delay; + const jobId = parsed.data.drain + ? undefined + : `backfill:${parsed.data.runId}:${parsed.data.cursor ?? 'start'}`; + if (jobId) { + const decision = await resolveExistingQueueJob(this.backfillQueue, jobId); + if (decision === 'skip') return; + } + try { + await this.backfillQueue.add( + 'backfill-page', + { ...parsed.data }, + { + ...(jobId ? { jobId } : {}), + attempts: 3, + backoff: { type: 'exponential', delay: 1000 }, + ...(delay ? { delay } : {}), + }, + ); + } catch (err) { + if (!isDuplicateJobError(err)) throw err; + } + } + + private async closeGenerationWorker() { + if (this.closingWorker || !this.worker) return; + this.closingWorker = true; + const worker = this.worker; + const connection = this.workerConnection; + this.worker = undefined; + this.workerConnection = undefined; + this.workerConcurrency = undefined; + try { + await worker.close(); + await connection?.quit(); + } finally { + this.closingWorker = false; + } + } + + private async closeBackfillWorker() { + if (this.closingBackfillWorker || !this.backfillWorker) return; + this.closingBackfillWorker = true; + const worker = this.backfillWorker; + const connection = this.backfillWorkerConnection; + this.backfillWorker = undefined; + this.backfillWorkerConnection = undefined; + this.backfillWorkerConcurrency = undefined; + try { + await worker.close(); + await connection?.quit(); + } finally { + this.closingBackfillWorker = false; + } + } + + private handleGenerationFailure(job: WorkerJob | undefined, error: unknown) { + ConduitGrpcSdk.Logger.error(sanitizeErrorMessage(error)); + const attempts = job?.opts?.attempts ?? 1; + const made = job?.attemptsMade ?? 1; + if (made < attempts) { + incrementEmbeddingMetric('retried'); + return; + } + incrementEmbeddingMetric('failed'); + const parsed = parseEmbeddingJobData(job?.data); + if (!parsed.ok || !parsed.data.backfillRunId || !this.onBackfillJobOutcome) return; + this.onBackfillJobOutcome(parsed.data.backfillRunId, 'failed').catch(err => + ConduitGrpcSdk.Logger.error(sanitizeErrorMessage(err)), + ); + } +} + +function normalizeJobCounts( + counts: Partial & Record, +): QueueJobCounts { + return { + waiting: counts.waiting ?? 0, + active: counts.active ?? 0, + completed: counts.completed ?? 0, + failed: counts.failed ?? 0, + delayed: counts.delayed ?? 0, + paused: counts.paused ?? 0, + }; +} + +async function resolveExistingQueueJob( + queue: QueueLike, + jobId: string, +): Promise<'enqueue' | 'skip'> { + if (!queue.getJob) return 'enqueue'; + const existing = await queue.getJob(jobId); + if (!existing) return 'enqueue'; + const state = await existing.getState(); + if (isInFlightQueueJobState(state)) return 'skip'; + if (!shouldReplaceRetainedQueueJob(state)) return 'skip'; + try { + await existing.remove(); + } catch { + return 'skip'; + } + return 'enqueue'; +} diff --git a/modules/embeddings/src/embeddings.proto b/modules/embeddings/src/embeddings.proto new file mode 100644 index 000000000..177cdb926 --- /dev/null +++ b/modules/embeddings/src/embeddings.proto @@ -0,0 +1,193 @@ +syntax = 'proto3'; +package embeddings; + +message EmbeddingConfig { + string id = 1; + string schemaName = 2; + repeated string sourceFields = 3; + string targetField = 4; + string provider = 5; + string model = 6; + int32 dimensions = 7; + string similarity = 8; + bool enabled = 9; + optional string createdAt = 10; + optional string updatedAt = 11; +} + +message UpsertConfigRequest { + string schemaName = 1; + repeated string sourceFields = 2; + string targetField = 3; + string provider = 4; + string model = 5; + int32 dimensions = 6; + optional string similarity = 7; + repeated string sourceFieldAllowlist = 8; + optional bool enabled = 9; +} + +message UpsertConfigResponse { + EmbeddingConfig config = 1; + repeated string warnings = 2; +} + +message GetConfigsRequest { + optional string schemaName = 1; + optional string id = 2; +} + +message GetConfigsResponse { + repeated EmbeddingConfig configs = 1; +} + +message DeleteEmbeddingConfigRequest { + optional string id = 1; + optional string schemaName = 2; + optional string targetField = 3; +} + +message DeleteEmbeddingConfigResponse { + EmbeddingConfig config = 1; +} + +message VectorCapabilities { + bool supported = 1; + bool storage = 2; + bool indexing = 3; + bool search = 4; + string provider = 5; + optional string reason = 6; +} + +message GetCapabilitiesRequest { + optional string schemaName = 1; +} + +message GetCapabilitiesResponse { + VectorCapabilities capabilities = 1; + repeated string warnings = 2; +} + +message QueueCounts { + int32 waiting = 1; + int32 active = 2; + int32 completed = 3; + int32 failed = 4; + int32 delayed = 5; + int32 paused = 6; +} + +message GetStatusRequest { + optional string schemaName = 1; +} + +message GetStatusResponse { + bool enabled = 1; + bool ready = 2; + VectorCapabilities capabilities = 3; + QueueCounts generationQueue = 4; + QueueCounts backfillQueue = 5; + repeated string warnings = 6; +} + +message BackfillRun { + string id = 1; + string schemaName = 2; + optional string configId = 3; + string state = 4; + optional string cursor = 5; + int32 batchSize = 6; + bool onlyMissing = 7; + optional string filter = 8; + int32 scannedCount = 9; + int32 queuedCount = 10; + int32 processedCount = 11; + int32 failedCount = 12; + optional string startedAt = 13; + optional string finishedAt = 14; + optional string error = 15; + optional string createdAt = 16; + optional string updatedAt = 17; +} + +message StartBackfillRequest { + string schemaName = 1; + optional int32 batchSize = 2; + optional string configId = 3; + optional bool onlyMissing = 4; + optional string filter = 5; +} + +message StartBackfillResponse { + int32 queued = 1; + repeated BackfillRun runs = 2; + repeated string warnings = 3; +} + +message GetBackfillRequest { + string id = 1; +} + +message ListBackfillsRequest { + optional string schemaName = 1; + optional string state = 2; + optional string configId = 3; + optional int32 skip = 4; + optional int32 limit = 5; +} + +message ListBackfillsResponse { + repeated BackfillRun runs = 1; + int32 count = 2; +} + +message CancelBackfillRequest { + string id = 1; +} + +message ResumeBackfillRequest { + string id = 1; +} + +message BackfillMutationResponse { + BackfillRun run = 1; +} + +message SemanticSearchRequest { + string schemaName = 1; + string text = 2; + optional string targetField = 3; + optional string filter = 4; + optional int32 limit = 5; + optional string userId = 6; + optional string scope = 7; + // Honored only when the verified caller is a platform operator module. + optional bool adminOperator = 8; +} + +message SemanticSearchHit { + string document = 1; + double score = 2; + optional double distance = 3; + optional string metric = 4; + optional string provider = 5; +} + +message SemanticSearchResponse { + repeated SemanticSearchHit hits = 1; +} + +service EmbeddingsProvider { + rpc upsertConfig(UpsertConfigRequest) returns (UpsertConfigResponse); + rpc getConfigs(GetConfigsRequest) returns (GetConfigsResponse); + rpc deleteConfig(DeleteEmbeddingConfigRequest) returns (DeleteEmbeddingConfigResponse); + rpc getCapabilities(GetCapabilitiesRequest) returns (GetCapabilitiesResponse); + rpc getStatus(GetStatusRequest) returns (GetStatusResponse); + rpc startBackfill(StartBackfillRequest) returns (StartBackfillResponse); + rpc getBackfill(GetBackfillRequest) returns (BackfillRun); + rpc listBackfills(ListBackfillsRequest) returns (ListBackfillsResponse); + rpc cancelBackfill(CancelBackfillRequest) returns (BackfillMutationResponse); + rpc resumeBackfill(ResumeBackfillRequest) returns (BackfillMutationResponse); + rpc semanticSearch(SemanticSearchRequest) returns (SemanticSearchResponse); +} diff --git a/modules/embeddings/src/index.ts b/modules/embeddings/src/index.ts new file mode 100644 index 000000000..b6b9fa59b --- /dev/null +++ b/modules/embeddings/src/index.ts @@ -0,0 +1,7 @@ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import EmbeddingsModule from './Embeddings.js'; + +const peerManifestRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); +const embeddings = new EmbeddingsModule(peerManifestRoot); +embeddings.start(); diff --git a/modules/embeddings/src/metrics/index.ts b/modules/embeddings/src/metrics/index.ts new file mode 100644 index 000000000..b93f7e713 --- /dev/null +++ b/modules/embeddings/src/metrics/index.ts @@ -0,0 +1,53 @@ +import { MetricType } from '@conduitplatform/grpc-sdk'; + +export default { + generatedEmbeddings: { + type: MetricType.Counter, + config: { + name: 'generated_embeddings_total', + help: 'Tracks the total number of generated embeddings', + }, + }, + failedEmbeddings: { + type: MetricType.Counter, + config: { + name: 'failed_embeddings_total', + help: 'Tracks the total number of failed embedding generation attempts', + }, + }, + skippedEmbeddings: { + type: MetricType.Counter, + config: { + name: 'skipped_embeddings_total', + help: 'Tracks embeddings skipped because the source hash already matched', + }, + }, + retriedEmbeddings: { + type: MetricType.Counter, + config: { + name: 'retried_embeddings_total', + help: 'Tracks embedding generation retries before a terminal outcome', + }, + }, + embeddingBackfillJobs: { + type: MetricType.Counter, + config: { + name: 'embedding_backfill_jobs_total', + help: 'Tracks embedding jobs queued by backfill scans', + }, + }, + malformedEmbeddingEvents: { + type: MetricType.Counter, + config: { + name: 'malformed_embedding_events_total', + help: 'Tracks malformed or oversized embedding bus payloads', + }, + }, + malformedEmbeddingJobs: { + type: MetricType.Counter, + config: { + name: 'malformed_embedding_jobs_total', + help: 'Tracks malformed or oversized embedding queue payloads', + }, + }, +}; diff --git a/modules/embeddings/src/models/BackfillRun.schema.ts b/modules/embeddings/src/models/BackfillRun.schema.ts new file mode 100644 index 000000000..108629a8b --- /dev/null +++ b/modules/embeddings/src/models/BackfillRun.schema.ts @@ -0,0 +1,82 @@ +import { + ConduitModel, + DatabaseProvider, + Indexable, + TYPE, +} from '@conduitplatform/grpc-sdk'; +import { ConduitActiveSchema } from '@conduitplatform/module-tools'; +import { BACKFILL_RUN_STATES, BackfillRunState } from '../utils/backfillRun.js'; + +const schema: ConduitModel = { + _id: TYPE.ObjectId, + schemaName: { type: TYPE.String, required: true }, + configId: { type: TYPE.String, required: false }, + state: { + type: TYPE.String, + enum: [...BACKFILL_RUN_STATES], + required: true, + default: 'queued', + }, + cursor: { type: TYPE.String, required: false }, + batchSize: { type: TYPE.Number, required: true }, + onlyMissing: { type: TYPE.Boolean, default: false }, + filter: { type: TYPE.JSON, required: false }, + scannedCount: { type: TYPE.Number, default: 0 }, + queuedCount: { type: TYPE.Number, default: 0 }, + processedCount: { type: TYPE.Number, default: 0 }, + failedCount: { type: TYPE.Number, default: 0 }, + startedAt: { type: TYPE.Date, required: false }, + finishedAt: { type: TYPE.Date, required: false }, + drainStartedAt: { type: TYPE.Date, required: false }, + error: { type: TYPE.String, required: false }, + createdAt: TYPE.Date, + updatedAt: TYPE.Date, +}; + +const modelOptions = { + timestamps: true, + indexes: [{ fields: ['schemaName', 'state'] }, { fields: ['configId', 'state'] }], + conduit: { + permissions: { + extendable: false, + canCreate: false, + canModify: 'Nothing', + canDelete: false, + }, + }, +} as const; + +export class BackfillRun extends ConduitActiveSchema { + private static _instance: BackfillRun; + _id: string; + schemaName: string; + configId?: string; + state: BackfillRunState; + cursor?: string; + batchSize: number; + onlyMissing: boolean; + filter?: Indexable; + scannedCount: number; + queuedCount: number; + processedCount: number; + failedCount: number; + startedAt?: Date; + finishedAt?: Date; + drainStartedAt?: Date; + error?: string; + createdAt: Date; + updatedAt: Date; + + private constructor(database: DatabaseProvider) { + super(database, BackfillRun.name, schema, modelOptions); + } + + static getInstance(database?: DatabaseProvider) { + if (BackfillRun._instance) return BackfillRun._instance; + if (!database) { + throw new Error('No database instance provided!'); + } + BackfillRun._instance = new BackfillRun(database); + return BackfillRun._instance; + } +} diff --git a/modules/embeddings/src/models/EmbeddingConfig.schema.ts b/modules/embeddings/src/models/EmbeddingConfig.schema.ts new file mode 100644 index 000000000..a2c26e697 --- /dev/null +++ b/modules/embeddings/src/models/EmbeddingConfig.schema.ts @@ -0,0 +1,66 @@ +import { + ConduitModel, + DatabaseProvider, + TYPE, + VectorSimilarity, +} from '@conduitplatform/grpc-sdk'; +import { ConduitActiveSchema } from '@conduitplatform/module-tools'; + +const schema: ConduitModel = { + _id: TYPE.ObjectId, + schemaName: { type: TYPE.String, required: true }, + sourceFields: { type: [TYPE.String], required: true }, + targetField: { type: TYPE.String, required: true }, + provider: { type: TYPE.String, required: true }, + modelName: { type: TYPE.String, required: true }, + dimensions: { type: TYPE.Number, required: true }, + similarity: { + type: TYPE.String, + enum: Object.values(VectorSimilarity), + default: VectorSimilarity.Cosine, + }, + enabled: { type: TYPE.Boolean, default: true }, + createdAt: TYPE.Date, + updatedAt: TYPE.Date, +}; + +const modelOptions = { + timestamps: true, + indexes: [{ fields: ['schemaName', 'targetField'], options: { unique: true } }], + conduit: { + permissions: { + extendable: false, + canCreate: false, + canModify: 'Nothing', + canDelete: false, + }, + }, +} as const; + +export class EmbeddingConfig extends ConduitActiveSchema { + private static _instance: EmbeddingConfig; + _id: string; + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + modelName: string; + dimensions: number; + similarity: VectorSimilarity; + enabled: boolean; + createdAt: Date; + updatedAt: Date; + + private constructor(database: DatabaseProvider) { + super(database, EmbeddingConfig.name, schema, modelOptions); + } + + static getInstance(database?: DatabaseProvider) { + if (EmbeddingConfig._instance) return EmbeddingConfig._instance; + if (!database) { + throw new Error('No database instance provided!'); + } + EmbeddingConfig._instance = new EmbeddingConfig(database); + return EmbeddingConfig._instance; + } +} diff --git a/modules/embeddings/src/models/index.ts b/modules/embeddings/src/models/index.ts new file mode 100644 index 000000000..27cb5ba7b --- /dev/null +++ b/modules/embeddings/src/models/index.ts @@ -0,0 +1,2 @@ +export * from './EmbeddingConfig.schema.js'; +export * from './BackfillRun.schema.js'; diff --git a/modules/embeddings/src/providers/index.test.ts b/modules/embeddings/src/providers/index.test.ts new file mode 100644 index 000000000..7bc18ebe2 --- /dev/null +++ b/modules/embeddings/src/providers/index.test.ts @@ -0,0 +1,85 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { OpenAICompatibleEmbeddingProvider } from './index.js'; + +describe('openai-compatible provider security', () => { + const provider = new OpenAICompatibleEmbeddingProvider({ + lookup: async () => [{ address: '104.18.0.1', family: 4 }], + fetch: async () => { + throw new Error('redirect not allowed'); + }, + }); + + it('rejects redirects, oversize input, and blocked endpoints', async () => { + await assert.rejects( + () => + provider.embed('hello', { + endpoint: 'https://api.openai.com/v1/embeddings', + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + await assert.rejects( + () => + provider.embed('x'.repeat(100), { + endpoint: 'https://api.openai.com/v1/embeddings', + maxInputBytes: 8, + }), + err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, + ); + await assert.rejects( + () => + provider.embed('hello', { + endpoint: 'https://127.0.0.1/v1/embeddings', + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + }); + + it('returns embeddings from a public HTTPS endpoint using the selected model', async () => { + const safe = new OpenAICompatibleEmbeddingProvider({ + lookup: async () => [{ address: '104.18.0.1', family: 4 }], + fetch: async (_url, init) => { + assert.equal(init?.redirect, 'error'); + const headers = new Headers(init?.headers); + assert.equal(headers.get('authorization'), 'Bearer sk-test'); + const body = JSON.parse(String(init?.body ?? '{}')) as { model?: string }; + assert.equal(body.model, 'text-embedding-3-small'); + return new Response(JSON.stringify({ data: [{ embedding: [0.1, 0.2] }] }), { + status: 200, + }); + }, + }); + const vector = await safe.embed('hello', { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-test', + model: 'text-embedding-3-small', + }); + assert.deepEqual(vector, [0.1, 0.2]); + }); + + it('maps provider HTTP failures without leaking the API key', async () => { + const failing = new OpenAICompatibleEmbeddingProvider({ + lookup: async () => [{ address: '104.18.0.1', family: 4 }], + fetch: async () => + new Response('invalid apiKey=sk-test', { + status: 401, + headers: { 'content-type': 'text/plain' }, + }), + }); + await assert.rejects( + () => + failing.embed('hello', { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-test', + model: 'text-embedding-3-small', + }), + err => + err instanceof GrpcError && + err.code === status.UNAVAILABLE && + /HTTP 401/.test(err.message) && + !err.message.includes('sk-test'), + ); + }); +}); diff --git a/modules/embeddings/src/providers/index.ts b/modules/embeddings/src/providers/index.ts new file mode 100644 index 000000000..45341b1e5 --- /dev/null +++ b/modules/embeddings/src/providers/index.ts @@ -0,0 +1,137 @@ +import { createHash } from 'node:crypto'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertSafeEmbeddingEndpoint, + DEFAULT_EMBED_TIMEOUT_MS, + DEFAULT_MAX_EMBED_INPUT_BYTES, + DEFAULT_MAX_EMBED_RESPONSE_BYTES, + readCappedResponse, + type SafeEndpointOptions, +} from '../utils/endpointSecurity.js'; +import { sanitizeErrorMessage } from '../utils/redactConfig.js'; + +export interface EmbeddingProviderConfig { + endpoint?: string; + apiKey?: string; + model?: string; + timeoutMs?: number; + maxInputBytes?: number; + maxResponseBytes?: number; +} + +export interface EmbeddingProvider { + embed(input: string, config: EmbeddingProviderConfig): Promise; +} + +export type EmbeddingFetch = ( + input: string | URL, + init?: RequestInit, +) => Promise; + +export interface EmbeddingProviderDependencies { + fetch?: EmbeddingFetch; + lookup?: SafeEndpointOptions['lookup']; +} + +function mapEmbedFetchError(err: unknown): never { + if (err instanceof GrpcError) throw err; + const message = sanitizeErrorMessage(err); + if (/redirect/i.test(message)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embedding provider redirects are not allowed', + ); + } + if (err instanceof Error && err.name === 'TimeoutError') { + throw new GrpcError(status.DEADLINE_EXCEEDED, 'Embedding provider request timed out'); + } + throw new GrpcError(status.UNAVAILABLE, message); +} + +function parseEmbeddingResponse(bodyText: string): number[] { + let body: { data?: { embedding?: number[] }[] }; + try { + body = JSON.parse(bodyText) as { data?: { embedding?: number[] }[] }; + } catch { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding provider response was not valid JSON', + ); + } + const embedding = body.data?.[0]?.embedding; + if (!embedding?.length) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding provider response did not include an embedding', + ); + } + return embedding; +} + +export class OpenAICompatibleEmbeddingProvider implements EmbeddingProvider { + constructor(private readonly deps: EmbeddingProviderDependencies = {}) {} + + async embed(input: string, config: EmbeddingProviderConfig): Promise { + if (!config.endpoint) { + throw new GrpcError( + status.FAILED_PRECONDITION, + 'Embedding provider endpoint is not configured', + ); + } + if ( + Buffer.byteLength(input) > (config.maxInputBytes ?? DEFAULT_MAX_EMBED_INPUT_BYTES) + ) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding input exceeds the allowed size', + ); + } + await assertSafeEmbeddingEndpoint(config.endpoint, { + lookup: this.deps.lookup, + }); + const fetchImpl = this.deps.fetch ?? fetch; + let response: Response; + try { + response = await fetchImpl(config.endpoint, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(config.apiKey ? { authorization: `Bearer ${config.apiKey}` } : {}), + }, + body: JSON.stringify({ + input, + model: config.model, + }), + redirect: 'error', + signal: AbortSignal.timeout(config.timeoutMs ?? DEFAULT_EMBED_TIMEOUT_MS), + }); + } catch (err) { + mapEmbedFetchError(err); + } + if (!response.ok) { + throw new GrpcError( + status.UNAVAILABLE, + `Embedding provider failed with HTTP ${response.status}`, + ); + } + return parseEmbeddingResponse( + await readCappedResponse( + response, + config.maxResponseBytes ?? DEFAULT_MAX_EMBED_RESPONSE_BYTES, + ), + ); + } +} + +export function getProvider( + name: string, + deps: EmbeddingProviderDependencies = {}, +): EmbeddingProvider { + if (name === 'openai-compatible') return new OpenAICompatibleEmbeddingProvider(deps); + throw new GrpcError(status.INVALID_ARGUMENT, `Unsupported embedding provider: ${name}`); +} + +export function hashEmbeddingInput(input: string) { + return createHash('sha256').update(input).digest('hex'); +} diff --git a/modules/embeddings/src/routes/index.ts b/modules/embeddings/src/routes/index.ts new file mode 100644 index 000000000..a8672b4a1 --- /dev/null +++ b/modules/embeddings/src/routes/index.ts @@ -0,0 +1,90 @@ +import { + ConduitGrpcSdk, + ConduitRouteActions, + ConduitRouteReturnDefinition, + ParsedRouterRequest, + UnparsedRouterResponse, +} from '@conduitplatform/grpc-sdk'; +import { + ConduitJson, + ConduitNumber, + ConduitString, + GrpcServer, + RoutingManager, +} from '@conduitplatform/module-tools'; +import { EmbeddingsApi } from '../api/embeddingsApi.js'; +import { EMBEDDINGS_CLIENT_FORBIDDEN_PATHS } from '../admin/routes.js'; +import { + assertClientSearchSubject, + clampClientSearchLimit, + clientSearchSubject, +} from '../utils/clientSearchContext.js'; + +export class EmbeddingsRoutes { + private readonly routingManager: RoutingManager; + + constructor( + private readonly server: GrpcServer, + private readonly grpcSdk: ConduitGrpcSdk, + private readonly api: EmbeddingsApi, + ) { + this.routingManager = new RoutingManager(this.grpcSdk.router!, this.server); + } + + static clientForbiddenPaths(): string[] { + return [...EMBEDDINGS_CLIENT_FORBIDDEN_PATHS]; + } + + async semanticSearch(call: ParsedRouterRequest): Promise { + const subject = assertClientSearchSubject(clientSearchSubject(call.request.context)); + const filter = call.request.params.filter; + const result = await this.api.semanticSearch( + { + schemaName: call.request.params.schemaName, + text: call.request.params.text, + targetField: call.request.params.targetField, + limit: clampClientSearchLimit(call.request.params.limit), + filter: + filter == null + ? undefined + : typeof filter === 'string' + ? filter + : JSON.stringify(filter), + userId: subject.userId, + scope: subject.scope, + }, + { callerModule: 'router' }, + ); + return { + hits: result.hits.map(hit => ({ + ...hit, + document: JSON.parse(hit.document), + })), + }; + } + + async registerRoutes() { + this.routingManager.clear(); + this.routingManager.route( + { + path: '/search', + action: ConduitRouteActions.POST, + description: + 'Client semantic search by text. User and scope are taken from the authenticated router context; raw vectors, userId, scope, and adminOperator are not accepted. Client limit is capped below the admin/gRPC vector-search maximum.', + bodyParams: { + schemaName: ConduitString.Required, + text: ConduitString.Required, + targetField: ConduitString.Optional, + filter: ConduitJson.Optional, + limit: ConduitNumber.Optional, + }, + middlewares: ['authMiddleware'], + }, + new ConduitRouteReturnDefinition('ClientSemanticSearch', { + hits: [ConduitJson.Required], + }), + this.semanticSearch.bind(this), + ); + await this.routingManager.registerRoutes(); + } +} diff --git a/modules/embeddings/src/utils/backfillExecution.test.ts b/modules/embeddings/src/utils/backfillExecution.test.ts new file mode 100644 index 000000000..84f034cd8 --- /dev/null +++ b/modules/embeddings/src/utils/backfillExecution.test.ts @@ -0,0 +1,668 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { VectorIndexStatus } from '@conduitplatform/grpc-sdk'; +import { + applyBackfillJobOutcome, + backfillRunFromDocument, + cancelBackfillExecution, + parseBackfillControllerJob, + persistableBackfillRun, + persistableNewBackfillRun, + processBackfillControllerJob, + queueBackfillRuns, + resumeBackfillExecution, + type BackfillControllerJobData, + type PersistedBackfillRun, + type ProcessBackfillDeps, +} from './backfillExecution.js'; +import { BackfillGateError } from './backfillGates.js'; +import type { EmbeddingJobData } from './embeddingJobs.js'; +import type { BackfillRunProgress } from './backfillRun.js'; + +const now = new Date('2026-09-06T18:00:00.000Z'); +const capabilities = { + supported: true, + storage: true, + provider: 'mongodb' as const, +}; +const config = { + _id: 'cfg1', + enabled: true, + schemaName: 'Article', + targetField: 'embedding', + dimensions: 3, + similarity: 'cosine', +}; +const readyIndex = { + field: 'embedding', + name: 'embedding_vector', + status: VectorIndexStatus.Ready, + queryable: true, + dimensions: 3, + similarity: 'cosine', +}; + +function memoryStore(initial: PersistedBackfillRun[] = []) { + const runs = new Map( + initial.map(run => [run._id, { ...run }]), + ); + return { + runs, + createRun: async (run: BackfillRunProgress) => { + const created: PersistedBackfillRun = { ...run, _id: `run${runs.size + 1}` }; + runs.set(created._id, created); + return { _id: created._id }; + }, + getRun: async (id: string) => { + const run = runs.get(id); + return run ? { ...run } : null; + }, + saveRun: async (id: string, run: BackfillRunProgress) => { + const existing = runs.get(id); + if (!existing) { + runs.set(id, { + ...run, + ...persistableNewBackfillRun(run), + _id: id, + } as PersistedBackfillRun); + return; + } + Object.assign(existing, persistableBackfillRun(run)); + }, + incrementCounts: async ( + id: string, + patch: { $inc: { processedCount?: number; failedCount?: number } }, + ) => { + const run = runs.get(id); + if (!run || run.state !== 'running') return null; + run.processedCount += patch.$inc.processedCount ?? 0; + run.failedCount += patch.$inc.failedCount ?? 0; + return { ...run }; + }, + }; +} + +function deps( + overrides: Partial & { store: ReturnType }, +): ProcessBackfillDeps { + const pages: Array<{ _id: string }>[] = overrides.findPage + ? [] + : [[{ _id: 'a' }, { _id: 'b' }], [{ _id: 'c' }]]; + let page = 0; + const embeddingJobs: EmbeddingJobData[] = []; + const continuations: BackfillControllerJobData[] = []; + return { + now, + maxBatchSize: 500, + moduleEnabled: true, + getRun: overrides.store.getRun, + saveRun: overrides.store.saveRun, + findPage: async () => pages[page++] ?? [], + enqueueEmbeddingJobs: async jobs => { + embeddingJobs.push(...jobs); + return jobs.length; + }, + enqueueContinuation: async job => { + continuations.push(job); + }, + getCapabilities: async () => capabilities, + getConfig: async id => (id === config._id ? config : null), + getIndexes: async () => [readyIndex], + ...overrides, + embeddingJobs, + continuations, + } as ProcessBackfillDeps & { + embeddingJobs: EmbeddingJobData[]; + continuations: BackfillControllerJobData[]; + }; +} + +describe('queued backfill start', () => { + it('persists queued config-specific runs and enqueues controller jobs without scanning', async () => { + const store = memoryStore(); + const controllerJobs: BackfillControllerJobData[] = []; + const queued = await queueBackfillRuns( + { schemaName: 'Article', batchSize: 2, onlyMissing: true }, + { + moduleEnabled: true, + capabilities, + configs: [config, { ...config, _id: 'cfg2', targetField: 'other' }], + indexes: [readyIndex, { ...readyIndex, field: 'other', name: 'other_vector' }], + createRun: store.createRun, + saveRun: store.saveRun, + findActiveRuns: async configId => + [...store.runs.values()].filter( + run => + run.configId === configId && + (run.state === 'queued' || run.state === 'running'), + ), + enqueueController: async job => { + controllerJobs.push(job); + }, + }, + ); + assert.equal(queued.queued, 2); + assert.equal(queued.runs[0].state, 'queued'); + assert.equal(queued.runs[0].configId, 'cfg1'); + assert.equal(queued.runs[1].configId, 'cfg2'); + assert.deepEqual( + controllerJobs.map(job => job.runId), + queued.runs.map(run => run.id), + ); + const persisted = [...store.runs.values()]; + assert.equal( + persisted.every(run => run.state === 'queued'), + true, + ); + assert.equal( + persisted.every(run => run.scannedCount === 0), + true, + ); + assert.equal( + persisted.every(run => run.onlyMissing === true), + true, + ); + }); + + it('fails closed before persisting when a gate is not met', async () => { + const store = memoryStore(); + await assert.rejects( + () => + queueBackfillRuns( + { schemaName: 'Article' }, + { + moduleEnabled: false, + capabilities, + configs: [config], + indexes: [readyIndex], + createRun: store.createRun, + saveRun: store.saveRun, + findActiveRuns: async () => [], + enqueueController: async () => undefined, + }, + ), + (err: unknown) => + err instanceof BackfillGateError && err.reason === 'module_disabled', + ); + assert.equal(store.runs.size, 0); + }); +}); + +describe('cursor-based backfill continuation', () => { + it('scans bounded pages, caps enqueue, and continues from the cursor', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'queued', + batchSize: 2, + onlyMissing: false, + scannedCount: 0, + queuedCount: 0, + processedCount: 0, + failedCount: 0, + }), + ); + const harness = deps({ + store, + findPage: async (_schema, page) => { + if (page.query._id) return [{ _id: 'c' }]; + return [{ _id: 'a' }, { _id: 'b' }, { _id: 'extra' }]; + }, + }) as ProcessBackfillDeps & { + embeddingJobs: EmbeddingJobData[]; + continuations: BackfillControllerJobData[]; + }; + + const first = await processBackfillControllerJob( + { runId: created._id, cursor: null }, + harness, + ); + assert.equal(first.action, 'continue'); + assert.equal(first.run?.state, 'running'); + assert.equal(first.run?.cursor, 'b'); + assert.equal(first.run?.scannedCount, 2); + assert.equal(first.run?.queuedCount, 2); + assert.equal(harness.embeddingJobs.length, 2); + assert.equal(harness.embeddingJobs[0].configId, 'cfg1'); + assert.equal(harness.embeddingJobs[0].backfillRunId, created._id); + assert.deepEqual(harness.continuations, [{ runId: created._id, cursor: 'b' }]); + + const second = await processBackfillControllerJob(harness.continuations[0], { + ...harness, + enqueueContinuation: async job => { + harness.continuations.push(job); + }, + }); + assert.equal(second.action, 'drain'); + assert.equal(second.run?.cursor, 'c'); + assert.equal(second.run?.scannedCount, 3); + assert.equal(second.run?.queuedCount, 3); + assert.equal(second.run?.state, 'running'); + for (let i = 0; i < 3; i += 1) { + await applyBackfillJobOutcome({ + runId: created._id, + outcome: 'processed', + incrementCounts: store.incrementCounts, + }); + } + const drained = await processBackfillControllerJob( + { runId: created._id, cursor: 'c', drain: true }, + harness, + ); + assert.equal(drained.action, 'completed'); + assert.equal(drained.run?.processedCount, 3); + }); + + it('uses onlyMissing target-field queries and config-specific jobs', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'queued', + batchSize: 1, + onlyMissing: true, + }), + ); + let observedQuery: Record | undefined; + const harness = deps({ + store, + findPage: async (_schema, page) => { + observedQuery = page.query; + return [{ _id: 'a' }]; + }, + }) as ProcessBackfillDeps & { embeddingJobs: EmbeddingJobData[] }; + await processBackfillControllerJob({ runId: created._id, cursor: null }, harness); + assert.equal(observedQuery?.embedding, null); + assert.equal(harness.embeddingJobs[0]?.configId, 'cfg1'); + }); +}); + +describe('backfill cancellation, resume, and counters', () => { + it('stops at cancellation checks and resumes from the saved cursor', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'queued', + batchSize: 2, + }), + ); + const first = deps({ store }); + await processBackfillControllerJob({ runId: created._id, cursor: null }, first); + const running = (await store.getRun(created._id))!; + const canceled = await cancelBackfillExecution({ + run: running, + saveRun: store.saveRun, + now, + }); + assert.equal(canceled.ok, true); + const canceledProcess = await processBackfillControllerJob( + { runId: created._id, cursor: 'b' }, + deps({ store }), + ); + assert.equal(canceledProcess.action, 'canceled'); + + const resumed = await resumeBackfillExecution({ + run: (await store.getRun(created._id))!, + saveRun: store.saveRun, + enqueueController: async () => undefined, + }); + assert.equal(resumed.ok, true); + if (!resumed.ok) return; + assert.equal(resumed.run.state, 'queued'); + assert.equal(resumed.run.cursor, 'b'); + + let query: Record | undefined; + const restarted = await processBackfillControllerJob( + { runId: created._id, cursor: 'b' }, + deps({ + store, + findPage: async (_schema, page) => { + query = page.query; + return [{ _id: 'c' }]; + }, + }), + ); + assert.deepEqual(query?._id, { $gt: 'b' }); + assert.equal(restarted.run?.cursor, 'c'); + assert.equal(restarted.run?.state, 'running'); + assert.equal(restarted.run?.startedAt?.toISOString(), now.toISOString()); + }); + + it('persists processed and failed job counts without exceeding queued work', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'queued', + batchSize: 2, + }), + ); + await processBackfillControllerJob( + { runId: created._id, cursor: null }, + deps({ store }), + ); + const processed = await applyBackfillJobOutcome({ + runId: created._id, + outcome: 'processed', + incrementCounts: store.incrementCounts, + }); + assert.equal(processed.ok, true); + const failed = await applyBackfillJobOutcome({ + runId: created._id, + outcome: 'failed', + incrementCounts: store.incrementCounts, + }); + assert.equal(failed.ok, true); + if (!failed.ok) return; + assert.equal(failed.run.scannedCount, 2); + assert.equal(failed.run.queuedCount, 2); + assert.equal(failed.run.processedCount, 1); + assert.equal(failed.run.failedCount, 1); + }); + + it('keeps concurrent processed and failed increments without lost updates', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'running', + batchSize: 2, + queuedCount: 40, + }), + ); + await Promise.all( + Array.from({ length: 40 }, (_, index) => + applyBackfillJobOutcome({ + runId: created._id, + outcome: index % 5 === 0 ? 'failed' : 'processed', + incrementCounts: async (id, patch) => { + await Promise.resolve(); + return store.incrementCounts(id, patch); + }, + }), + ), + ); + const latest = (await store.getRun(created._id))!; + assert.equal(latest.processedCount, 32); + assert.equal(latest.failedCount, 8); + }); + + it('does not overwrite atomic processed/failed counts when a page save races with workers', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'running', + batchSize: 2, + queuedCount: 0, + }), + ); + const originalSave = store.saveRun; + store.saveRun = async (id, run) => { + await Promise.resolve(); + await originalSave(id, run); + }; + const page = processBackfillControllerJob( + { runId: created._id, cursor: null }, + deps({ + store, + findPage: async () => { + await Promise.resolve(); + return [{ _id: 'a' }, { _id: 'b' }]; + }, + }), + ); + const workers = Promise.all( + Array.from({ length: 10 }, (_, index) => + applyBackfillJobOutcome({ + runId: created._id, + outcome: index % 2 === 0 ? 'processed' : 'failed', + incrementCounts: async (id, patch) => { + await Promise.resolve(); + return store.incrementCounts(id, patch); + }, + }), + ), + ); + await Promise.all([page, workers]); + const latest = (await store.getRun(created._id))!; + assert.equal(latest.processedCount, 5); + assert.equal(latest.failedCount, 5); + assert.equal(latest.scannedCount, 2); + assert.equal(latest.queuedCount, 2); + assert.equal(latest.cursor, 'b'); + }); + + it('fails the running run when the vector index is not queryable', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'queued', + batchSize: 1, + }), + ); + const result = await processBackfillControllerJob( + { runId: created._id, cursor: null }, + deps({ + store, + getIndexes: async () => [ + { + field: 'embedding', + status: VectorIndexStatus.Pending, + queryable: false, + }, + ], + }), + ); + assert.equal(result.action, 'failed'); + assert.equal(result.run?.state, 'failed'); + assert.match(result.run?.error ?? '', /not queryable/); + assert.doesNotMatch(result.run?.error ?? '', /apiKey|Bearer /); + }); +}); + +describe('backfill controller job parsing and persistence mapping', () => { + it('rejects malformed controller payloads and round-trips run documents', () => { + assert.equal(parseBackfillControllerJob({ runId: 'run1', cursor: null }).ok, true); + assert.equal(parseBackfillControllerJob({ runId: '../nope' }).ok, false); + assert.equal(parseBackfillControllerJob({ runId: 'run1', extra: true }).ok, false); + const progress = backfillRunFromDocument({ + _id: 'run1', + schemaName: 'Article', + configId: 'cfg1', + state: 'queued', + batchSize: 10, + onlyMissing: true, + }); + assert.equal(progress.cursor, null); + assert.equal(persistableBackfillRun(progress).onlyMissing, true); + assert.equal('processedCount' in persistableBackfillRun(progress), false); + assert.equal('failedCount' in persistableBackfillRun(progress), false); + assert.equal(persistableNewBackfillRun(progress).processedCount, 0); + assert.equal(persistableNewBackfillRun(progress).failedCount, 0); + }); +}); + +describe('backfill enqueue failures, drain timeout, and start idempotency', () => { + function queueDeps( + store: ReturnType, + extras: { + enqueue?: (job: BackfillControllerJobData) => Promise; + configs?: Array<{ + _id: string; + enabled?: boolean; + schemaName?: string; + targetField?: string; + }>; + } = {}, + ) { + return { + moduleEnabled: true, + capabilities, + configs: extras.configs ?? [config], + indexes: [readyIndex], + createRun: store.createRun, + saveRun: store.saveRun, + findActiveRuns: async (configId: string) => + [...store.runs.values()].filter( + run => + run.configId === configId && + (run.state === 'queued' || run.state === 'running'), + ), + enqueueController: extras.enqueue ?? (async () => undefined), + }; + } + + it('fails a newly created run when controller enqueue throws instead of leaving it queued', async () => { + const store = memoryStore(); + await assert.rejects( + () => + queueBackfillRuns( + { schemaName: 'Article' }, + queueDeps(store, { + enqueue: async () => { + throw new Error('redis down apiKey=sk-secret'); + }, + }), + ), + /redis down/, + ); + const persisted = [...store.runs.values()]; + assert.equal(persisted.length, 1); + assert.equal(persisted[0].state, 'failed'); + assert.match(persisted[0].error ?? '', /redis down/); + assert.doesNotMatch(persisted[0].error ?? '', /sk-secret/); + }); + + it('reuses an active run for the same config instead of creating a duplicate', async () => { + const store = memoryStore(); + const first = await queueBackfillRuns( + { schemaName: 'Article', batchSize: 2 }, + queueDeps(store), + ); + const second = await queueBackfillRuns( + { schemaName: 'Article', batchSize: 50 }, + queueDeps(store), + ); + assert.equal(first.queued, 1); + assert.equal(second.queued, 1); + assert.equal(second.runs[0].id, first.runs[0].id); + assert.equal(store.runs.size, 1); + }); + + it('fails drain polling with a sanitized timeout instead of looping forever', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'running', + batchSize: 2, + queuedCount: 2, + processedCount: 0, + failedCount: 0, + scannedCount: 2, + cursor: 'b', + drainStartedAt: new Date('2026-09-06T17:00:00.000Z'), + }), + ); + const result = await processBackfillControllerJob( + { runId: created._id, cursor: 'b', drain: true }, + deps({ + store, + drainTimeoutMs: 60_000, + now: new Date('2026-09-06T18:00:00.000Z'), + }), + ); + assert.equal(result.action, 'failed'); + assert.equal(result.run?.state, 'failed'); + assert.match(result.run?.error ?? '', /timed out waiting for generation jobs/); + assert.doesNotMatch(result.run?.error ?? '', /apiKey|Bearer /); + }); + + it('records drainStartedAt on the first drain poll then fails after the timeout', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'running', + batchSize: 2, + queuedCount: 2, + processedCount: 0, + failedCount: 0, + scannedCount: 2, + cursor: 'b', + }), + ); + const startedAt = new Date('2026-09-06T18:00:00.000Z'); + const first = await processBackfillControllerJob( + { runId: created._id, cursor: 'b', drain: true }, + deps({ + store, + drainTimeoutMs: 60_000, + now: startedAt, + }), + ); + assert.equal(first.action, 'drain'); + assert.equal(first.run?.drainStartedAt?.toISOString(), startedAt.toISOString()); + const timedOut = await processBackfillControllerJob( + { runId: created._id, cursor: 'b', drain: true }, + deps({ + store, + drainTimeoutMs: 60_000, + now: new Date('2026-09-06T18:01:00.000Z'), + }), + ); + assert.equal(timedOut.action, 'failed'); + assert.match(timedOut.run?.error ?? '', /timed out waiting for generation jobs/); + }); + + it('fails a resumed run when controller enqueue throws', async () => { + const store = memoryStore(); + const created = await store.createRun( + backfillRunFromDocument({ + _id: 'ignored', + schemaName: 'Article', + configId: 'cfg1', + state: 'canceled', + batchSize: 2, + cursor: 'b', + }), + ); + const existing = (await store.getRun(created._id))!; + await assert.rejects( + () => + resumeBackfillExecution({ + run: existing, + saveRun: store.saveRun, + enqueueController: async () => { + throw new Error('queue unavailable Bearer sk-secret'); + }, + }), + /queue unavailable/, + ); + const persisted = (await store.getRun(created._id))!; + assert.equal(persisted.state, 'failed'); + assert.doesNotMatch(persisted.error ?? '', /sk-secret/); + }); +}); diff --git a/modules/embeddings/src/utils/backfillExecution.ts b/modules/embeddings/src/utils/backfillExecution.ts new file mode 100644 index 000000000..3e7cc8093 --- /dev/null +++ b/modules/embeddings/src/utils/backfillExecution.ts @@ -0,0 +1,546 @@ +import type { VectorCapabilities } from '@conduitplatform/grpc-sdk'; +import { + applyBackfillPage, + backfillCountIncrementPatch, + boundBackfillPage, + buildBackfillPageQuery, + cancelBackfillRun, + completeBackfillRun, + createQueuedBackfill, + failBackfillRun, + isActiveBackfillState, + resumeBackfillRun, + startBackfillRun, + type BackfillCountIncrementPatch, + type BackfillPageQuery, + type BackfillRunProgress, + type BackfillRunResult, +} from './backfillRun.js'; +import { + assertBackfillExecutable, + BackfillGateError, + type BackfillConfigGate, + type VectorIndexGate, +} from './backfillGates.js'; +import { EmbeddingJobData, MAX_QUEUE_BATCH_SIZE } from './embeddingJobs.js'; +import { incrementEmbeddingMetric } from './embeddingMetrics.js'; +import { sanitizeErrorMessage } from './redactConfig.js'; + +const IDENTITY = /^[A-Za-z0-9._-]{1,128}$/; + +export const BACKFILL_DRAIN_DELAY_MS = 1000; +export const DEFAULT_BACKFILL_DRAIN_TIMEOUT_MS = 15 * 60 * 1000; +export const BACKFILL_DRAIN_TIMEOUT_MESSAGE = + 'Backfill drain timed out waiting for generation jobs'; + +export interface PersistedBackfillRun extends BackfillRunProgress { + _id: string; +} + +export interface BackfillControllerJobData { + runId: string; + cursor?: string | null; + drain?: boolean; +} + +export type ParsedBackfillControllerJob = + { ok: true; data: BackfillControllerJobData } | { ok: false; reason: string }; + +export interface QueueBackfillInput { + schemaName: string; + batchSize?: number; + configId?: string; + onlyMissing?: boolean; + filter?: unknown; + maxBatchSize?: number; +} + +export interface QueueBackfillDeps { + moduleEnabled: boolean; + capabilities: Pick; + configs: BackfillConfigGate[]; + indexes: readonly VectorIndexGate[]; + createRun: (run: BackfillRunProgress) => Promise<{ _id: string }>; + saveRun: (id: string, run: BackfillRunProgress) => Promise; + findActiveRuns: (configId: string) => Promise; + enqueueController: (job: BackfillControllerJobData) => Promise; +} + +export interface ProcessBackfillDeps { + now?: Date; + maxBatchSize: number; + moduleEnabled: boolean; + getRun: (id: string) => Promise; + saveRun: (id: string, run: BackfillRunProgress) => Promise; + findPage: ( + schemaName: string, + page: BackfillPageQuery, + ) => Promise>; + enqueueEmbeddingJobs: (jobs: EmbeddingJobData[]) => Promise; + enqueueContinuation: (job: BackfillControllerJobData) => Promise; + getCapabilities: ( + schemaName: string, + ) => Promise>; + getConfig: (id: string) => Promise; + getIndexes: (schemaName: string) => Promise; + drainTimeoutMs?: number; +} + +export function backfillRunFromDocument(doc: { + _id: string; + schemaName: string; + configId?: string; + state: BackfillRunProgress['state']; + cursor?: string; + batchSize: number; + onlyMissing?: boolean; + filter?: Record; + scannedCount?: number; + queuedCount?: number; + processedCount?: number; + failedCount?: number; + startedAt?: Date; + finishedAt?: Date; + drainStartedAt?: Date; + error?: string; +}): PersistedBackfillRun { + return { + _id: doc._id, + state: doc.state, + schemaName: doc.schemaName, + ...(doc.configId ? { configId: doc.configId } : {}), + cursor: doc.cursor ?? null, + batchSize: doc.batchSize, + onlyMissing: doc.onlyMissing === true, + filter: doc.filter ?? null, + scannedCount: doc.scannedCount ?? 0, + queuedCount: doc.queuedCount ?? 0, + processedCount: doc.processedCount ?? 0, + failedCount: doc.failedCount ?? 0, + startedAt: doc.startedAt ?? null, + finishedAt: doc.finishedAt ?? null, + drainStartedAt: doc.drainStartedAt ?? null, + error: doc.error ?? null, + }; +} + +export function persistableBackfillRun( + run: BackfillRunProgress, +): Record { + return { + state: run.state, + schemaName: run.schemaName, + configId: run.configId, + cursor: run.cursor ?? undefined, + batchSize: run.batchSize, + onlyMissing: run.onlyMissing, + filter: run.filter ?? undefined, + scannedCount: run.scannedCount, + queuedCount: run.queuedCount, + startedAt: run.startedAt ?? undefined, + finishedAt: run.finishedAt ?? undefined, + drainStartedAt: run.drainStartedAt ?? undefined, + error: run.error ?? undefined, + }; +} + +export function persistableNewBackfillRun( + run: BackfillRunProgress, +): Record { + return { + ...persistableBackfillRun(run), + processedCount: run.processedCount, + failedCount: run.failedCount, + }; +} + +export function parseBackfillControllerJob(value: unknown): ParsedBackfillControllerJob { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return { ok: false, reason: 'malformed' }; + } + const record = value as Record; + const extraKeys = Object.keys(record).filter( + key => !['runId', 'cursor', 'drain'].includes(key), + ); + if (extraKeys.length) return { ok: false, reason: 'malformed' }; + if (typeof record.runId !== 'string' || !IDENTITY.test(record.runId)) { + return { ok: false, reason: 'runId' }; + } + if (record.drain !== undefined && typeof record.drain !== 'boolean') { + return { ok: false, reason: 'drain' }; + } + if ( + record.cursor != null && + (typeof record.cursor !== 'string' || !IDENTITY.test(record.cursor)) + ) { + return { ok: false, reason: 'cursor' }; + } + return { + ok: true, + data: { + runId: record.runId, + cursor: record.cursor ?? null, + ...(record.drain === true ? { drain: true } : {}), + }, + }; +} + +export function selectedBackfillConfigs( + configs: BackfillConfigGate[], + configId?: string, +): BackfillConfigGate[] { + if (configId) { + return configs.filter(config => config._id === configId); + } + return configs.filter(config => config.enabled !== false && config._id); +} + +export async function queueBackfillRuns( + input: QueueBackfillInput, + deps: QueueBackfillDeps, +): Promise<{ + queued: number; + runs: Array<{ id: string; configId?: string; state: string }>; +}> { + const selected = selectedBackfillConfigs(deps.configs, input.configId); + if (!selected.length) { + assertBackfillExecutable({ + moduleEnabled: deps.moduleEnabled, + capabilities: deps.capabilities, + config: null, + indexes: deps.indexes, + }); + } + for (const config of selected) { + assertBackfillExecutable({ + moduleEnabled: deps.moduleEnabled, + capabilities: deps.capabilities, + config, + indexes: deps.indexes, + }); + } + const runs: Array<{ id: string; configId?: string; state: string }> = []; + for (const config of selected) { + const created = createQueuedBackfill({ + schemaName: input.schemaName, + configId: config._id, + batchSize: input.batchSize, + onlyMissing: input.onlyMissing, + filter: input.filter, + maxBatchSize: input.maxBatchSize, + }); + if (!created.ok) { + throw new BackfillGateError( + 'config_not_found', + `Invalid backfill request: ${created.reason}`, + ); + } + if (config._id) { + const active = (await deps.findActiveRuns(config._id)).filter(run => + isActiveBackfillState(run.state), + ); + const existing = active[0]; + if (existing) { + if (existing.state === 'queued') { + await enqueueOrFailRun( + existing._id, + existing, + deps.saveRun, + deps.enqueueController, + { + runId: existing._id, + cursor: existing.cursor ?? null, + }, + ); + } + runs.push({ + id: existing._id, + ...(config._id ? { configId: config._id } : {}), + state: existing.state, + }); + continue; + } + } + const persisted = await deps.createRun(created.run); + const queuedRun: PersistedBackfillRun = { ...created.run, _id: persisted._id }; + await enqueueOrFailRun( + persisted._id, + queuedRun, + deps.saveRun, + deps.enqueueController, + { + runId: persisted._id, + cursor: null, + }, + ); + runs.push({ + id: persisted._id, + ...(config._id ? { configId: config._id } : {}), + state: created.run.state, + }); + } + return { queued: runs.length, runs }; +} + +async function enqueueOrFailRun( + id: string, + run: BackfillRunProgress, + saveRun: (id: string, run: BackfillRunProgress) => Promise, + enqueue: (job: BackfillControllerJobData) => Promise, + job: BackfillControllerJobData, +): Promise { + try { + await enqueue(job); + } catch (err) { + const failed = failBackfillRun(run, sanitizeErrorMessage(err)); + if (failed.ok) { + await saveRun(id, failed.run); + } + throw err; + } +} + +export async function resumeBackfillExecution(args: { + run: PersistedBackfillRun; + saveRun: (id: string, run: BackfillRunProgress) => Promise; + enqueueController: (job: BackfillControllerJobData) => Promise; +}): Promise { + const resumed = resumeBackfillRun(args.run); + if (!resumed.ok) return resumed; + await args.saveRun(args.run._id, resumed.run); + try { + await args.enqueueController({ + runId: args.run._id, + cursor: resumed.run.cursor ?? null, + }); + } catch (err) { + const failed = failBackfillRun(resumed.run, sanitizeErrorMessage(err)); + if (failed.ok) { + await args.saveRun(args.run._id, failed.run); + } + throw err; + } + return resumed; +} + +export async function cancelBackfillExecution(args: { + run: PersistedBackfillRun; + saveRun: (id: string, run: BackfillRunProgress) => Promise; + now?: Date; +}): Promise { + const canceled = cancelBackfillRun(args.run, args.now); + if (!canceled.ok) return canceled; + await args.saveRun(args.run._id, canceled.run); + return canceled; +} + +export async function applyBackfillJobOutcome(args: { + runId: string; + outcome: 'processed' | 'failed'; + incrementCounts: ( + id: string, + patch: BackfillCountIncrementPatch, + ) => Promise; +}): Promise { + const run = await args.incrementCounts( + args.runId, + backfillCountIncrementPatch(args.outcome), + ); + if (!run) return { ok: false, reason: 'not_found' }; + return { ok: true, run }; +} + +export async function processBackfillControllerJob( + rawJob: unknown, + deps: ProcessBackfillDeps, +): Promise<{ action: string; run?: BackfillRunProgress }> { + const parsed = parseBackfillControllerJob(rawJob); + if (!parsed.ok) { + incrementEmbeddingMetric('malformedJobs'); + return { action: 'malformed' }; + } + const persisted = await deps.getRun(parsed.data.runId); + if (!persisted) { + return { action: 'missing' }; + } + if (persisted.state === 'canceled' || persisted.state === 'completed') { + return { action: persisted.state, run: persisted }; + } + if (parsed.data.drain) { + return drainBackfill(persisted, deps); + } + return scanBackfillPage(parsed.data, persisted, deps); +} + +async function startQueuedBackfill( + persisted: PersistedBackfillRun, + deps: ProcessBackfillDeps, + now: Date, +): Promise<{ action: string; run?: BackfillRunProgress } | { run: BackfillRunProgress }> { + let run: BackfillRunProgress = persisted; + if (run.state === 'queued') { + const started = startBackfillRun(run, now); + if (!started.ok) return { action: started.reason, run }; + run = started.run; + await deps.saveRun(persisted._id, run); + } + if (run.state !== 'running') { + return { action: run.state, run }; + } + return { run }; +} + +async function rescheduleStaleBackfill( + persisted: PersistedBackfillRun, + run: BackfillRunProgress, + job: BackfillControllerJobData, + deps: ProcessBackfillDeps, +): Promise<{ action: string; run: BackfillRunProgress } | undefined> { + const jobCursor = job.cursor ?? null; + const runCursor = run.cursor ?? null; + if (jobCursor === runCursor) return undefined; + await deps.enqueueContinuation({ + runId: persisted._id, + cursor: runCursor, + }); + return { action: 'stale', run }; +} + +function pageEmbeddingJobs( + run: BackfillRunProgress, + persistedId: string, + docs: Array<{ _id?: unknown }>, + maxBatchSize: number, +): EmbeddingJobData[] { + return docs + .map(doc => ({ + schemaName: run.schemaName, + documentId: String(doc._id), + ...(run.configId ? { configId: run.configId } : {}), + backfillRunId: persistedId, + })) + .slice(0, Math.min(run.batchSize, maxBatchSize, MAX_QUEUE_BATCH_SIZE)); +} + +async function scanBackfillPage( + job: BackfillControllerJobData, + persisted: PersistedBackfillRun, + deps: ProcessBackfillDeps, +): Promise<{ action: string; run?: BackfillRunProgress }> { + const now = deps.now ?? new Date(); + const started = await startQueuedBackfill(persisted, deps, now); + if ('action' in started) return started; + let run = started.run; + const stale = await rescheduleStaleBackfill(persisted, run, job, deps); + if (stale) return stale; + + try { + if (!run.configId) { + throw new BackfillGateError( + 'config_not_found', + 'Backfill run is missing a config id', + ); + } + const config = await deps.getConfig(run.configId); + assertBackfillExecutable({ + moduleEnabled: deps.moduleEnabled, + capabilities: await deps.getCapabilities(run.schemaName), + config, + indexes: await deps.getIndexes(run.schemaName), + }); + const pageQuery = buildBackfillPageQuery(run, config?.targetField); + if (!pageQuery.ok) { + throw new BackfillGateError( + 'config_not_found', + `Invalid backfill page: ${pageQuery.reason}`, + ); + } + const docs = boundBackfillPage( + await deps.findPage(run.schemaName, pageQuery.page), + run.batchSize, + ); + const jobs = pageEmbeddingJobs(run, persisted._id, docs, deps.maxBatchSize); + const queuedDelta = jobs.length ? await deps.enqueueEmbeddingJobs(jobs) : 0; + incrementEmbeddingMetric('backfill', queuedDelta); + const applied = applyBackfillPage( + run, + docs.map(doc => ({ _id: String(doc._id) })), + queuedDelta, + ); + if (!applied.ok) { + return failPersistedRun(persisted._id, run, applied.reason, deps, now); + } + run = applied.run; + await deps.saveRun(persisted._id, run); + const canceled = await deps.getRun(persisted._id); + if (!canceled || canceled.state === 'canceled') { + return { action: 'canceled', run: canceled ?? run }; + } + if (applied.exhausted) { + return finishOrDrain(persisted._id, run, deps, now); + } + await deps.enqueueContinuation({ + runId: persisted._id, + cursor: run.cursor ?? null, + }); + return { action: 'continue', run }; + } catch (err) { + return failPersistedRun(persisted._id, run, err, deps, now); + } +} + +async function drainBackfill( + persisted: PersistedBackfillRun, + deps: ProcessBackfillDeps, +): Promise<{ action: string; run?: BackfillRunProgress }> { + const latest = (await deps.getRun(persisted._id)) ?? persisted; + if (latest.state === 'canceled') { + return { action: 'canceled', run: latest }; + } + if (latest.state !== 'running') { + return { action: latest.state, run: latest }; + } + return finishOrDrain(latest._id, latest, deps, deps.now ?? new Date()); +} + +async function finishOrDrain( + id: string, + run: BackfillRunProgress, + deps: ProcessBackfillDeps, + now: Date, +): Promise<{ action: string; run?: BackfillRunProgress }> { + if (run.processedCount + run.failedCount >= run.queuedCount) { + const completed = completeBackfillRun(run, now); + if (!completed.ok) return { action: completed.reason, run }; + await deps.saveRun(id, completed.run); + return { action: 'completed', run: completed.run }; + } + const drainStartedAt = run.drainStartedAt ?? now; + const timeoutMs = deps.drainTimeoutMs ?? DEFAULT_BACKFILL_DRAIN_TIMEOUT_MS; + if (now.getTime() - drainStartedAt.getTime() >= timeoutMs) { + return failPersistedRun(id, run, BACKFILL_DRAIN_TIMEOUT_MESSAGE, deps, now); + } + if (!run.drainStartedAt) { + run = { ...run, drainStartedAt }; + await deps.saveRun(id, run); + } + await deps.enqueueContinuation({ + runId: id, + cursor: run.cursor ?? null, + drain: true, + }); + return { action: 'drain', run }; +} + +async function failPersistedRun( + id: string, + run: BackfillRunProgress, + err: unknown, + deps: ProcessBackfillDeps, + now: Date, +): Promise<{ action: string; run?: BackfillRunProgress }> { + const failed = failBackfillRun(run, err, now); + if (!failed.ok) return { action: failed.reason, run }; + await deps.saveRun(id, failed.run); + return { action: 'failed', run: failed.run }; +} diff --git a/modules/embeddings/src/utils/backfillGates.test.ts b/modules/embeddings/src/utils/backfillGates.test.ts new file mode 100644 index 000000000..afe341e21 --- /dev/null +++ b/modules/embeddings/src/utils/backfillGates.test.ts @@ -0,0 +1,266 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { ConduitGrpcSdk, VectorIndexStatus } from '@conduitplatform/grpc-sdk'; +import { + assertBackfillExecutable, + BackfillGateError, + findTargetVectorIndex, + grpcErrorFromBackfillGate, + isEmbeddingVectorIndexQueryable, +} from './backfillGates.js'; + +const capabilities = { + supported: true, + storage: true, + provider: 'mongodb' as const, +}; + +const config = { + _id: 'cfg1', + enabled: true, + schemaName: 'Article', + targetField: 'embedding', + dimensions: 3, + similarity: 'cosine', +}; + +const readyIndex = { + field: 'embedding', + name: 'embedding_vector', + status: VectorIndexStatus.Ready, + queryable: true, + dimensions: 3, + similarity: 'cosine', +}; + +describe('backfill execution gates', () => { + it('blocks disabled modules, unsupported storage, and disabled configs', () => { + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: false, + capabilities, + config, + indexes: [readyIndex], + }), + (err: unknown) => + err instanceof BackfillGateError && err.reason === 'module_disabled', + ); + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities: { + supported: false, + storage: false, + provider: 'unsupported', + reason: 'mysql does not support Conduit vector search', + }, + config, + indexes: [readyIndex], + }), + (err: unknown) => + err instanceof BackfillGateError && + err.reason === 'vector_unsupported' && + /mysql/.test(err.message), + ); + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities: { + supported: true, + storage: false, + provider: 'postgres', + reason: 'pgvector is not available', + }, + config, + indexes: [readyIndex], + }), + (err: unknown) => + err instanceof BackfillGateError && err.reason === 'vector_storage_unavailable', + ); + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities, + config: { ...config, enabled: false }, + indexes: [readyIndex], + }), + (err: unknown) => + err instanceof BackfillGateError && err.reason === 'config_disabled', + ); + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities, + config: null, + indexes: [readyIndex], + }), + (err: unknown) => + err instanceof BackfillGateError && err.reason === 'config_not_found', + ); + }); + + it('requires a queryable vector index and keeps the status actionable', () => { + assert.equal(isEmbeddingVectorIndexQueryable(readyIndex), true); + assert.equal( + isEmbeddingVectorIndexQueryable({ + field: 'embedding', + name: 'embedding_vector', + }), + false, + ); + assert.equal( + isEmbeddingVectorIndexQueryable({ + field: 'embedding', + status: VectorIndexStatus.Pending, + }), + false, + ); + assert.deepEqual(findTargetVectorIndex([readyIndex], 'embedding'), readyIndex); + assert.equal( + findTargetVectorIndex( + [ + readyIndex, + { + field: 'embedding', + name: 'embedding_vector_v2', + status: VectorIndexStatus.Pending, + queryable: false, + }, + ], + 'embedding', + )?.name, + 'embedding_vector_v2', + ); + assert.equal( + findTargetVectorIndex([readyIndex], 'embedding', { + dimensions: 3, + similarity: 'euclidean', + }), + undefined, + ); + assert.equal( + findTargetVectorIndex( + [ + readyIndex, + { + field: 'embedding', + name: 'embedding_vector_v2', + status: VectorIndexStatus.Pending, + queryable: false, + dimensions: 3, + similarity: 'euclidean', + }, + ], + 'embedding', + { dimensions: 3, similarity: 'euclidean' }, + )?.name, + 'embedding_vector_v2', + ); + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities, + config: { ...config, similarity: 'euclidean' }, + indexes: [readyIndex], + }), + (err: unknown) => + err instanceof BackfillGateError && + err.reason === 'index_not_queryable' && + err.indexStatus === 'missing', + ); + assert.throws( + () => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities, + config, + indexes: [ + { + field: 'embedding', + name: 'embedding_vector', + status: VectorIndexStatus.Pending, + queryable: false, + dimensions: 3, + similarity: 'cosine', + }, + ], + }), + (err: unknown) => + err instanceof BackfillGateError && + err.reason === 'index_not_queryable' && + err.indexStatus === VectorIndexStatus.Pending && + /Wait until the index is ready/.test(err.message), + ); + assert.doesNotThrow(() => + assertBackfillExecutable({ + moduleEnabled: true, + capabilities, + config, + indexes: [{ ...readyIndex, method: '' }], + }), + ); + const mapped = grpcErrorFromBackfillGate( + new BackfillGateError( + 'index_not_queryable', + "Vector index for field 'embedding' is not queryable (status: pending). Wait until the index is ready before running a backfill.", + 'pending', + ), + ); + assert.equal(mapped.code, 9); + }); +}); + +describe('embedding metric increments do not accept labels', () => { + it('increments named counters without attaching payload data', async () => { + const { incrementEmbeddingMetric, EMBEDDING_METRICS } = + await import('./embeddingMetrics.js'); + const seen: Array<{ name: string; amount?: number; labels?: unknown }> = []; + const previous = ConduitGrpcSdk.Metrics; + ConduitGrpcSdk.Metrics = { + increment(name: string, amount?: number, labels?: unknown) { + seen.push({ name, amount, labels }); + }, + } as never; + try { + incrementEmbeddingMetric('generated', 2); + incrementEmbeddingMetric('failed'); + incrementEmbeddingMetric('skipped', 1); + incrementEmbeddingMetric('retried', 1); + incrementEmbeddingMetric('backfill', 3); + incrementEmbeddingMetric('malformedEvents'); + incrementEmbeddingMetric('malformedJobs'); + incrementEmbeddingMetric('generated', 0); + } finally { + ConduitGrpcSdk.Metrics = previous; + } + assert.deepEqual( + seen.map(item => item.name), + [ + EMBEDDING_METRICS.generated, + EMBEDDING_METRICS.failed, + EMBEDDING_METRICS.skipped, + EMBEDDING_METRICS.retried, + EMBEDDING_METRICS.backfill, + EMBEDDING_METRICS.malformedEvents, + EMBEDDING_METRICS.malformedJobs, + ], + ); + assert.equal( + seen.every(item => item.labels === undefined), + true, + ); + assert.equal( + seen.some( + item => + JSON.stringify(item).includes('sk-') || JSON.stringify(item).includes('doc'), + ), + false, + ); + }); +}); diff --git a/modules/embeddings/src/utils/backfillGates.ts b/modules/embeddings/src/utils/backfillGates.ts new file mode 100644 index 000000000..d565fd92a --- /dev/null +++ b/modules/embeddings/src/utils/backfillGates.ts @@ -0,0 +1,165 @@ +import { + GrpcError, + VectorCapabilities, + VectorIndexStatus, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + selectEmbeddingVectorIndex, + type EmbeddingVectorIndexContract, + type EmbeddingVectorIndexShape, +} from './configChange.js'; + +export const BACKFILL_GATE_REASONS = [ + 'module_disabled', + 'vector_unsupported', + 'vector_storage_unavailable', + 'config_not_found', + 'config_disabled', + 'index_not_queryable', +] as const; + +export type BackfillGateReason = (typeof BACKFILL_GATE_REASONS)[number]; + +export class BackfillGateError extends Error { + readonly code = 'BACKFILL_GATE' as const; + + constructor( + readonly reason: BackfillGateReason, + message: string, + readonly indexStatus?: string, + ) { + super(message); + this.name = 'BackfillGateError'; + } +} + +export interface BackfillConfigGate { + _id?: string; + enabled?: boolean; + schemaName?: string; + targetField?: string; + dimensions?: number; + similarity?: string; + method?: string; +} + +export interface VectorIndexGate extends EmbeddingVectorIndexShape { + queryable?: boolean; + status?: string; +} + +export type EmbeddingIndexContractInput = Omit; + +export function embeddingIndexContractFromConfig( + config: BackfillConfigGate, +): EmbeddingIndexContractInput | undefined { + if (typeof config.dimensions !== 'number') return undefined; + return { + dimensions: config.dimensions, + similarity: config.similarity, + method: config.method, + }; +} + +export function isEmbeddingVectorIndexQueryable(index?: VectorIndexGate): boolean { + if (!index) return false; + if (index.queryable === false) return false; + const indexStatus = index.status?.toLowerCase(); + if (indexStatus === VectorIndexStatus.Failed || indexStatus === 'failed') { + return false; + } + if ( + (indexStatus === VectorIndexStatus.Pending || indexStatus === 'pending') && + index.queryable !== true + ) { + return false; + } + if (index.queryable === true) return true; + return indexStatus === VectorIndexStatus.Ready || indexStatus === 'ready'; +} + +export function findTargetVectorIndex( + indexes: readonly VectorIndexGate[], + targetField: string, + contract?: EmbeddingIndexContractInput, +): VectorIndexGate | undefined { + return selectEmbeddingVectorIndex(indexes, targetField, contract); +} + +export function assertBackfillExecutable(args: { + moduleEnabled: boolean; + capabilities?: Pick< + VectorCapabilities, + 'supported' | 'storage' | 'provider' | 'reason' + >; + config?: BackfillConfigGate | null; + indexes?: readonly VectorIndexGate[]; +}): void { + if (!args.moduleEnabled) { + throw new BackfillGateError( + 'module_disabled', + 'Embeddings module is disabled; enable it before starting a backfill', + ); + } + const capabilities = args.capabilities; + if (!capabilities?.supported) { + throw new BackfillGateError( + 'vector_unsupported', + capabilities?.reason ?? + 'Database does not support Conduit vector storage; use MongoDB Atlas Vector Search or Postgres pgvector', + ); + } + if (!capabilities.storage) { + throw new BackfillGateError( + 'vector_storage_unavailable', + capabilities.reason ?? + `Vector storage is unavailable for provider '${capabilities.provider}'`, + ); + } + if (!args.config) { + throw new BackfillGateError( + 'config_not_found', + 'No enabled embedding config found for backfill', + ); + } + if (args.config.enabled === false) { + throw new BackfillGateError( + 'config_disabled', + `Embedding config '${args.config._id ?? 'unknown'}' is disabled`, + ); + } + const targetField = args.config.targetField; + if (typeof targetField !== 'string' || !targetField.length) { + throw new BackfillGateError( + 'config_not_found', + 'Embedding config is missing a target vector field', + ); + } + const contract = embeddingIndexContractFromConfig(args.config); + const index = findTargetVectorIndex(args.indexes ?? [], targetField, contract); + if (contract && isEmbeddingVectorIndexQueryable(index)) return; + const indexStatus = contract ? (index?.status ?? 'missing') : 'missing'; + throw new BackfillGateError( + 'index_not_queryable', + `Vector index for field '${targetField}' is not queryable (status: ${indexStatus}). ` + + 'Wait until the index is ready before running a backfill.', + indexStatus, + ); +} + +export function grpcErrorFromBackfillGate(err: BackfillGateError): GrpcError { + switch (err.reason) { + case 'module_disabled': + case 'vector_unsupported': + case 'vector_storage_unavailable': + case 'config_not_found': + case 'config_disabled': + case 'index_not_queryable': + return new GrpcError(status.FAILED_PRECONDITION, err.message); + default: { + const unexpected: never = err.reason; + return new GrpcError(status.INTERNAL, String(unexpected)); + } + } +} diff --git a/modules/embeddings/src/utils/backfillRun.test.ts b/modules/embeddings/src/utils/backfillRun.test.ts new file mode 100644 index 000000000..031ccf2bf --- /dev/null +++ b/modules/embeddings/src/utils/backfillRun.test.ts @@ -0,0 +1,437 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import type { Query } from '@conduitplatform/grpc-sdk'; +import type { BackfillRun } from '../models/BackfillRun.schema.js'; +import { + applyAtomicBackfillCountDelta, + applyBackfillJobCounts, + applyBackfillPage, + BACKFILL_RUN_STATES, + backfillCountIncrementPatch, + boundBackfillBatchSize, + boundBackfillPage, + buildBackfillPageQuery, + canCancelBackfill, + cancelBackfillRun, + completeBackfillRun, + createQueuedBackfill, + DEFAULT_BACKFILL_BATCH_SIZE, + failBackfillRun, + isLegalBackfillTransition, + isResumeEligible, + isSafeBackfillFilter, + LEGAL_BACKFILL_TRANSITIONS, + MAX_BACKFILL_BATCH_SIZE, + MAX_BACKFILL_ERROR_LENGTH, + MAX_BACKFILL_FILTER_BYTES, + MAX_BACKFILL_FILTER_IN_VALUES, + MIN_BACKFILL_BATCH_SIZE, + resumeBackfillRun, + sanitizeBackfillError, + startBackfillRun, + toBackfillCountUpdateQuery, +} from './backfillRun.js'; + +const now = new Date('2026-09-06T16:00:00.000Z'); + +function queuedRun() { + const created = createQueuedBackfill({ + schemaName: 'Article', + configId: 'cfg1', + batchSize: 2, + onlyMissing: true, + filter: { published: true }, + }); + assert.equal(created.ok, true); + if (!created.ok) throw new Error('expected queued run'); + return created.run; +} + +function runningRun() { + const started = startBackfillRun(queuedRun(), now); + assert.equal(started.ok, true); + if (!started.ok) throw new Error('expected running run'); + return started.run; +} + +describe('backfill run state transitions', () => { + it('allows only the documented legal transitions', () => { + assert.deepEqual(BACKFILL_RUN_STATES, [ + 'queued', + 'running', + 'completed', + 'failed', + 'canceled', + ]); + assert.equal(isLegalBackfillTransition('queued', 'running'), true); + assert.equal(isLegalBackfillTransition('queued', 'canceled'), true); + assert.equal(isLegalBackfillTransition('running', 'completed'), true); + assert.equal(isLegalBackfillTransition('running', 'failed'), true); + assert.equal(isLegalBackfillTransition('running', 'canceled'), true); + assert.equal(isLegalBackfillTransition('failed', 'queued'), true); + assert.equal(isLegalBackfillTransition('canceled', 'queued'), true); + assert.equal(isLegalBackfillTransition('queued', 'failed'), true); + assert.equal(isLegalBackfillTransition('queued', 'completed'), false); + assert.equal(isLegalBackfillTransition('running', 'queued'), false); + assert.equal(isLegalBackfillTransition('completed', 'queued'), false); + assert.equal(isLegalBackfillTransition('completed', 'running'), false); + assert.equal(isLegalBackfillTransition('failed', 'running'), false); + assert.equal(isLegalBackfillTransition('canceled', 'running'), false); + assert.deepEqual(LEGAL_BACKFILL_TRANSITIONS.completed, []); + }); + + it('starts a queued run and records startedAt', () => { + const started = startBackfillRun(queuedRun(), now); + assert.equal(started.ok, true); + if (!started.ok) return; + assert.equal(started.run.state, 'running'); + assert.equal(started.run.startedAt?.toISOString(), now.toISOString()); + assert.equal(started.run.finishedAt, null); + assert.equal(started.run.error, null); + }); + + it('rejects illegal transitions without mutating counters', () => { + const completed = completeBackfillRun(runningRun(), now); + assert.equal(completed.ok, true); + if (!completed.ok) return; + const resumed = resumeBackfillRun(completed.run); + assert.equal(resumed.ok, false); + if (resumed.ok) return; + assert.equal(resumed.reason, 'illegal_transition'); + const failedFromCompleted = failBackfillRun(completed.run, 'boom', now); + assert.equal(failedFromCompleted.ok, false); + }); +}); + +describe('backfill pagination and cursor progression', () => { + it('advances the cursor through ordered pages and marks exhaustion', () => { + const first = applyBackfillPage(runningRun(), [{ _id: 'a' }, { _id: 'b' }]); + assert.equal(first.ok, true); + if (!first.ok) return; + assert.equal(first.exhausted, false); + assert.equal(first.run.cursor, 'b'); + assert.equal(first.run.scannedCount, 2); + assert.equal(first.run.queuedCount, 2); + + const query = buildBackfillPageQuery(first.run, 'embedding'); + assert.equal(query.ok, true); + if (!query.ok) return; + assert.deepEqual(query.page, { + query: { published: true, embedding: null, _id: { $gt: 'b' } }, + sort: { _id: 1 }, + limit: 2, + }); + + const last = applyBackfillPage(first.run, [{ _id: 'c' }]); + assert.equal(last.ok, true); + if (!last.ok) return; + assert.equal(last.exhausted, true); + assert.equal(last.run.cursor, 'c'); + assert.equal(last.run.scannedCount, 3); + }); + + it('does not move the cursor on an empty exhausted page', () => { + const empty = applyBackfillPage(runningRun(), []); + assert.equal(empty.ok, true); + if (!empty.ok) return; + assert.equal(empty.exhausted, true); + assert.equal(empty.run.cursor, null); + assert.equal(empty.run.scannedCount, 0); + const completed = completeBackfillRun(empty.run, now); + assert.equal(completed.ok, true); + if (!completed.ok) return; + assert.equal(completed.run.state, 'completed'); + }); + + it('rejects a page that would not advance the cursor', () => { + const first = applyBackfillPage(runningRun(), [{ _id: 'a' }, { _id: 'b' }]); + assert.equal(first.ok, true); + if (!first.ok) return; + const stuck = applyBackfillPage(first.run, [{ _id: 'b' }]); + assert.equal(stuck.ok, false); + if (stuck.ok) return; + assert.equal(stuck.reason, 'cursor'); + }); + + it('owns pagination _id and requires a target field for onlyMissing', () => { + const run = runningRun(); + const missingTarget = buildBackfillPageQuery(run); + assert.equal(missingTarget.ok, false); + const ownedId = buildBackfillPageQuery( + { ...run, cursor: 'doc1', filter: { _id: 'ignored', published: true } }, + 'embedding', + ); + assert.equal(ownedId.ok, true); + if (!ownedId.ok) return; + assert.deepEqual(ownedId.page.query._id, { $gt: 'doc1' }); + assert.equal(ownedId.page.query.published, true); + }); +}); + +describe('backfill counters', () => { + it('tracks scanned, queued, processed, and failed counts', () => { + const paged = applyBackfillPage(runningRun(), [{ _id: 'a' }, { _id: 'b' }], 2); + assert.equal(paged.ok, true); + if (!paged.ok) return; + const progressed = applyBackfillJobCounts(paged.run, { processed: 1, failed: 1 }); + assert.equal(progressed.ok, true); + if (!progressed.ok) return; + assert.equal(progressed.run.scannedCount, 2); + assert.equal(progressed.run.queuedCount, 2); + assert.equal(progressed.run.processedCount, 1); + assert.equal(progressed.run.failedCount, 1); + }); + + it('rejects job counts that exceed queued work or run while not running', () => { + const paged = applyBackfillPage(runningRun(), [{ _id: 'a' }]); + assert.equal(paged.ok, true); + if (!paged.ok) return; + const overflow = applyBackfillJobCounts(paged.run, { processed: 2 }); + assert.equal(overflow.ok, false); + const negative = applyBackfillJobCounts(paged.run, { failed: -1 }); + assert.equal(negative.ok, false); + const queuedCounts = applyBackfillJobCounts(queuedRun(), { processed: 1 }); + assert.equal(queuedCounts.ok, false); + }); + + it('loses concurrent updates when counts are applied via read-modify-write', () => { + const paged = applyBackfillPage(runningRun(), [{ _id: 'a' }, { _id: 'b' }], 2); + assert.equal(paged.ok, true); + if (!paged.ok) return; + const snapshot = { ...paged.run }; + const first = applyBackfillJobCounts(snapshot, { processed: 1 }); + const second = applyBackfillJobCounts(snapshot, { processed: 1 }); + assert.equal(first.ok, true); + assert.equal(second.ok, true); + if (!first.ok || !second.ok) return; + assert.equal(first.run.processedCount, 1); + assert.equal(second.run.processedCount, 1); + }); + + it('keeps concurrent processed and failed increments with an atomic delta', async () => { + const counters = { processedCount: 0, failedCount: 0 }; + assert.deepEqual(backfillCountIncrementPatch('processed'), { + $inc: { processedCount: 1 }, + }); + assert.deepEqual(backfillCountIncrementPatch('failed'), { + $inc: { failedCount: 1 }, + }); + await Promise.all( + Array.from({ length: 40 }, (_, index) => + Promise.resolve( + applyAtomicBackfillCountDelta( + counters, + index % 4 === 0 ? 'failed' : 'processed', + ), + ), + ), + ); + assert.equal(counters.processedCount, 30); + assert.equal(counters.failedCount, 10); + }); + + it('types atomic count patches as Query-compatible $inc updates', () => { + const processed: Query = toBackfillCountUpdateQuery( + backfillCountIncrementPatch('processed'), + ); + const failed: Query = toBackfillCountUpdateQuery( + backfillCountIncrementPatch('failed'), + ); + assert.deepEqual(processed, { $inc: { processedCount: 1 } }); + assert.deepEqual(failed, { $inc: { failedCount: 1 } }); + assert.equal('$set' in processed, false); + assert.equal('$set' in failed, false); + }); +}); + +describe('backfill cancellation and resume', () => { + it('cancels queued and running runs, then allows resume from canceled or failed', () => { + assert.equal(canCancelBackfill('queued'), true); + assert.equal(canCancelBackfill('running'), true); + assert.equal(canCancelBackfill('completed'), false); + assert.equal(isResumeEligible('canceled'), true); + assert.equal(isResumeEligible('failed'), true); + assert.equal(isResumeEligible('completed'), false); + assert.equal(isResumeEligible('running'), false); + + const canceled = cancelBackfillRun(runningRun(), now); + assert.equal(canceled.ok, true); + if (!canceled.ok) return; + assert.equal(canceled.run.state, 'canceled'); + assert.equal(canceled.run.finishedAt?.toISOString(), now.toISOString()); + + const resumed = resumeBackfillRun(canceled.run); + assert.equal(resumed.ok, true); + if (!resumed.ok) return; + assert.equal(resumed.run.state, 'queued'); + assert.equal(resumed.run.finishedAt, null); + assert.equal(resumed.run.error, null); + assert.equal(resumed.run.cursor, canceled.run.cursor); + assert.equal(resumed.run.scannedCount, canceled.run.scannedCount); + + assert.equal(canCancelBackfill('canceled'), false); + const alreadyCanceled = cancelBackfillRun(canceled.run, now); + assert.equal(alreadyCanceled.ok, true); + if (!alreadyCanceled.ok) return; + assert.equal(alreadyCanceled.run.state, 'canceled'); + + const alreadyQueued = resumeBackfillRun(resumed.run); + assert.equal(alreadyQueued.ok, true); + if (!alreadyQueued.ok) return; + assert.equal(alreadyQueued.run.state, 'queued'); + + const queuedFailed = failBackfillRun( + queuedRun(), + 'enqueue failed apiKey=sk-test', + now, + ); + assert.equal(queuedFailed.ok, true); + if (!queuedFailed.ok) return; + assert.equal(queuedFailed.run.state, 'failed'); + assert.doesNotMatch(queuedFailed.run.error ?? '', /sk-test|apiKey=/); + + const failed = failBackfillRun(runningRun(), 'provider timeout', now); + assert.equal(failed.ok, true); + if (!failed.ok) return; + const resumedFailed = resumeBackfillRun(failed.run); + assert.equal(resumedFailed.ok, true); + if (!resumedFailed.ok) return; + assert.equal(resumedFailed.run.state, 'queued'); + assert.equal(resumedFailed.run.error, null); + }); + + it('preserves cursor across cancel and resume so paging can continue', () => { + const paged = applyBackfillPage(runningRun(), [{ _id: 'a' }, { _id: 'b' }]); + assert.equal(paged.ok, true); + if (!paged.ok) return; + const canceled = cancelBackfillRun(paged.run, now); + assert.equal(canceled.ok, true); + if (!canceled.ok) return; + const resumed = resumeBackfillRun(canceled.run); + assert.equal(resumed.ok, true); + if (!resumed.ok) return; + const restarted = startBackfillRun(resumed.run, now); + assert.equal(restarted.ok, true); + if (!restarted.ok) return; + assert.equal(restarted.run.cursor, 'b'); + assert.equal(restarted.run.startedAt?.toISOString(), now.toISOString()); + }); +}); + +describe('backfill bounds', () => { + it('clamps batch size into the configured inclusive range', () => { + assert.deepEqual(boundBackfillBatchSize(undefined), { + ok: true, + batchSize: DEFAULT_BACKFILL_BATCH_SIZE, + }); + assert.deepEqual(boundBackfillBatchSize(0), { + ok: true, + batchSize: MIN_BACKFILL_BATCH_SIZE, + }); + assert.deepEqual(boundBackfillBatchSize(10_000), { + ok: true, + batchSize: MAX_BACKFILL_BATCH_SIZE, + }); + assert.deepEqual(boundBackfillBatchSize(50, 25), { ok: true, batchSize: 25 }); + assert.equal(boundBackfillBatchSize(1.5).ok, false); + assert.equal(boundBackfillBatchSize(Number.NaN).ok, false); + assert.equal(boundBackfillBatchSize(10, 0).ok, false); + }); + + it('bounds pages to batch size and rejects oversized apply calls', () => { + const docs = [{ _id: 'a' }, { _id: 'b' }, { _id: 'c' }]; + assert.deepEqual(boundBackfillPage(docs, 2), [{ _id: 'a' }, { _id: 'b' }]); + const oversized = applyBackfillPage(runningRun(), docs); + assert.equal(oversized.ok, false); + if (oversized.ok) return; + assert.equal(oversized.reason, 'page_size'); + }); + + it('rejects oversized filters, dangerous operators, and invalid identity', () => { + assert.equal(createQueuedBackfill({ schemaName: '../etc' }).ok, false); + assert.equal( + createQueuedBackfill({ schemaName: 'Article', configId: 'bad id' }).ok, + false, + ); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { $where: 'this.password' }, + }).ok, + false, + ); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { title: { $regex: 'a+' } }, + }).ok, + false, + ); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { $or: [{ published: true }] }, + }).ok, + false, + ); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { title: { $exists: true } }, + }).ok, + false, + ); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { body: { $like: '%secret%' } }, + }).ok, + false, + ); + assert.equal(createQueuedBackfill({ schemaName: 'Article', filter: [] }).ok, false); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { body: 'x'.repeat(MAX_BACKFILL_FILTER_BYTES) }, + }).ok, + false, + ); + assert.equal( + createQueuedBackfill({ + schemaName: 'Article', + filter: { + status: { + $in: Array.from({ length: MAX_BACKFILL_FILTER_IN_VALUES + 1 }, () => 'a'), + }, + }, + }).ok, + false, + ); + assert.equal( + isSafeBackfillFilter({ published: true, status: { $in: ['draft', 'live'] } }), + true, + ); + assert.equal( + isSafeBackfillFilter({ $and: [{ published: true }, { views: { $gte: 1 } }] }), + true, + ); + }); +}); + +describe('backfill sanitized errors', () => { + it('redacts secrets and truncates stored failure text', () => { + const failed = failBackfillRun( + runningRun(), + new Error('provider failed apiKey=sk-secret Bearer tok-live'), + now, + ); + assert.equal(failed.ok, true); + if (!failed.ok) return; + assert.equal(failed.run.state, 'failed'); + assert.match(failed.run.error ?? '', /\[REDACTED\]/); + assert.doesNotMatch(failed.run.error ?? '', /sk-secret|tok-live/); + + const long = sanitizeBackfillError('e'.repeat(MAX_BACKFILL_ERROR_LENGTH + 50)); + assert.equal(long.length, MAX_BACKFILL_ERROR_LENGTH); + }); +}); diff --git a/modules/embeddings/src/utils/backfillRun.ts b/modules/embeddings/src/utils/backfillRun.ts new file mode 100644 index 000000000..9f50dc97f --- /dev/null +++ b/modules/embeddings/src/utils/backfillRun.ts @@ -0,0 +1,546 @@ +import type { Query } from '@conduitplatform/grpc-sdk'; +import { sanitizeErrorMessage } from './redactConfig.js'; +import { MAX_QUEUE_BATCH_SIZE } from './embeddingJobs.js'; + +export const BACKFILL_RUN_SCHEMA = 'BackfillRun'; + +export const BACKFILL_RUN_STATES = [ + 'queued', + 'running', + 'completed', + 'failed', + 'canceled', +] as const; + +export type BackfillRunState = (typeof BACKFILL_RUN_STATES)[number]; + +export const LEGAL_BACKFILL_TRANSITIONS: Record< + BackfillRunState, + readonly BackfillRunState[] +> = { + queued: ['running', 'failed', 'canceled'], + running: ['completed', 'failed', 'canceled'], + completed: [], + failed: ['queued'], + canceled: ['queued'], +}; + +export const ACTIVE_BACKFILL_STATES: readonly BackfillRunState[] = ['queued', 'running']; + +export const MIN_BACKFILL_BATCH_SIZE = 1; +export const DEFAULT_BACKFILL_BATCH_SIZE = 100; +export const MAX_BACKFILL_BATCH_SIZE = MAX_QUEUE_BATCH_SIZE; +export const MAX_BACKFILL_ERROR_LENGTH = 1024; +export const MAX_BACKFILL_FILTER_BYTES = 4 * 1024; + +const SCHEMA_NAME = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; +const IDENTITY = /^[A-Za-z0-9._-]{1,128}$/; +const BACKFILL_FILTER_FIELD = /^[A-Za-z_][A-Za-z0-9_]*$/; +const BACKFILL_FILTER_RESERVED_FIELDS = new Set([ + '__proto__', + 'prototype', + 'constructor', +]); + +/** + * Documented safe backfill filter subset. Equality, comparisons, bounded + * `$in`/`$nin`, and `$and` are allowed. Regex, existence, expression, `$or`, + * `$not`, `$like`, and other expensive/operator-injection shapes are rejected. + */ +export const BACKFILL_FILTER_COMPARISON_OPERATORS = [ + '$eq', + '$ne', + '$gt', + '$gte', + '$lt', + '$lte', +] as const; +export const BACKFILL_FILTER_MEMBERSHIP_OPERATORS = ['$in', '$nin'] as const; +export const BACKFILL_FILTER_LOGICAL_OPERATORS = ['$and'] as const; +export const MAX_BACKFILL_FILTER_DEPTH = 3; +export const MAX_BACKFILL_FILTER_KEYS = 16; +export const MAX_BACKFILL_FILTER_IN_VALUES = 32; +export const MAX_BACKFILL_FILTER_AND_BRANCHES = 8; + +export interface BackfillRunProgress { + state: BackfillRunState; + schemaName: string; + configId?: string; + cursor?: string | null; + batchSize: number; + onlyMissing: boolean; + filter?: Record | null; + scannedCount: number; + queuedCount: number; + processedCount: number; + failedCount: number; + startedAt?: Date | null; + finishedAt?: Date | null; + drainStartedAt?: Date | null; + error?: string | null; +} + +export interface CreateBackfillRunInput { + schemaName: string; + configId?: string; + batchSize?: number; + onlyMissing?: boolean; + filter?: unknown; + maxBatchSize?: number; +} + +export type BackfillRunResult = + { ok: true; run: BackfillRunProgress } | { ok: false; reason: string }; + +export interface BackfillPageQuery { + query: Record; + sort: { _id: 1 }; + limit: number; +} + +function unexpectedState(state: never): never { + throw new Error(`Unhandled backfill state: ${String(state)}`); +} + +export function isBackfillRunState(value: unknown): value is BackfillRunState { + return ( + typeof value === 'string' && + (BACKFILL_RUN_STATES as readonly string[]).includes(value) + ); +} + +export function isLegalBackfillTransition( + from: BackfillRunState, + to: BackfillRunState, +): boolean { + return LEGAL_BACKFILL_TRANSITIONS[from].includes(to); +} + +export function isActiveBackfillState(state: BackfillRunState): boolean { + return ACTIVE_BACKFILL_STATES.includes(state); +} + +export function canCancelBackfill(state: BackfillRunState): boolean { + switch (state) { + case 'queued': + case 'running': + return true; + case 'completed': + case 'failed': + case 'canceled': + return false; + default: + return unexpectedState(state); + } +} + +export function isResumeEligible(state: BackfillRunState): boolean { + switch (state) { + case 'failed': + case 'canceled': + return true; + case 'queued': + case 'running': + case 'completed': + return false; + default: + return unexpectedState(state); + } +} + +export function boundBackfillBatchSize( + requested: number | undefined, + maxBatchSize: number = MAX_BACKFILL_BATCH_SIZE, +): { ok: true; batchSize: number } | { ok: false; reason: 'batch_size' } { + if (!Number.isInteger(maxBatchSize) || maxBatchSize < MIN_BACKFILL_BATCH_SIZE) { + return { ok: false, reason: 'batch_size' }; + } + const cappedMax = Math.min(maxBatchSize, MAX_BACKFILL_BATCH_SIZE); + const value = requested ?? DEFAULT_BACKFILL_BATCH_SIZE; + if (typeof value !== 'number' || !Number.isInteger(value) || !Number.isFinite(value)) { + return { ok: false, reason: 'batch_size' }; + } + if (value < MIN_BACKFILL_BATCH_SIZE) { + return { ok: true, batchSize: MIN_BACKFILL_BATCH_SIZE }; + } + return { ok: true, batchSize: Math.min(value, cappedMax) }; +} + +export function boundBackfillPage(docs: readonly T[], batchSize: number): T[] { + if (!Number.isInteger(batchSize) || batchSize < MIN_BACKFILL_BATCH_SIZE) return []; + return docs.slice(0, Math.min(batchSize, MAX_BACKFILL_BATCH_SIZE)); +} + +export function isBackfillPageExhausted(pageLength: number, batchSize: number): boolean { + return pageLength < batchSize; +} + +export function sanitizeBackfillError(err: unknown): string { + return sanitizeErrorMessage(err).slice(0, MAX_BACKFILL_ERROR_LENGTH); +} + +export function createQueuedBackfill(input: CreateBackfillRunInput): BackfillRunResult { + if (typeof input.schemaName !== 'string' || !SCHEMA_NAME.test(input.schemaName)) { + return { ok: false, reason: 'schemaName' }; + } + if ( + input.configId !== undefined && + (typeof input.configId !== 'string' || !IDENTITY.test(input.configId)) + ) { + return { ok: false, reason: 'configId' }; + } + const bounded = boundBackfillBatchSize(input.batchSize, input.maxBatchSize); + if (!bounded.ok) return bounded; + const filter = normalizeFilter(input.filter); + if (!filter.ok) return filter; + return { + ok: true, + run: { + state: 'queued', + schemaName: input.schemaName, + ...(input.configId ? { configId: input.configId } : {}), + cursor: null, + batchSize: bounded.batchSize, + onlyMissing: input.onlyMissing === true, + filter: filter.filter, + scannedCount: 0, + queuedCount: 0, + processedCount: 0, + failedCount: 0, + startedAt: null, + finishedAt: null, + drainStartedAt: null, + error: null, + }, + }; +} + +export function startBackfillRun( + run: BackfillRunProgress, + now: Date = new Date(), +): BackfillRunResult { + return transition(run, 'running', { + startedAt: run.startedAt ?? now, + finishedAt: null, + error: null, + }); +} + +export function cancelBackfillRun( + run: BackfillRunProgress, + now: Date = new Date(), +): BackfillRunResult { + if (run.state === 'canceled') { + return { ok: true, run }; + } + if (!canCancelBackfill(run.state)) { + return { ok: false, reason: 'illegal_transition' }; + } + return transition(run, 'canceled', { + finishedAt: now, + }); +} + +export function failBackfillRun( + run: BackfillRunProgress, + err: unknown, + now: Date = new Date(), +): BackfillRunResult { + return transition(run, 'failed', { + finishedAt: now, + error: sanitizeBackfillError(err), + }); +} + +export function completeBackfillRun( + run: BackfillRunProgress, + now: Date = new Date(), +): BackfillRunResult { + return transition(run, 'completed', { + finishedAt: now, + error: null, + }); +} + +export function resumeBackfillRun(run: BackfillRunProgress): BackfillRunResult { + if (run.state === 'queued' || run.state === 'running') { + return { ok: true, run }; + } + if (!isResumeEligible(run.state)) { + return { ok: false, reason: 'illegal_transition' }; + } + return transition(run, 'queued', { + finishedAt: null, + error: null, + drainStartedAt: null, + }); +} + +export function applyBackfillPage( + run: BackfillRunProgress, + docs: ReadonlyArray<{ _id?: unknown }>, + queuedDelta: number = docs.length, +): BackfillRunResult & { exhausted?: boolean } { + if (run.state !== 'running') { + return { ok: false, reason: 'not_running' }; + } + if (docs.length > run.batchSize) { + return { ok: false, reason: 'page_size' }; + } + if (!Number.isInteger(queuedDelta) || queuedDelta < 0 || queuedDelta > docs.length) { + return { ok: false, reason: 'queued' }; + } + const exhausted = isBackfillPageExhausted(docs.length, run.batchSize); + if (docs.length === 0) { + return { ok: true, run, exhausted }; + } + const ids: string[] = []; + for (const doc of docs) { + if (typeof doc._id !== 'string' || !IDENTITY.test(doc._id)) { + return { ok: false, reason: 'cursor' }; + } + ids.push(doc._id); + } + const cursor = ids[ids.length - 1]; + if (run.cursor && cursor === run.cursor) { + return { ok: false, reason: 'cursor' }; + } + return { + ok: true, + exhausted, + run: { + ...run, + cursor, + scannedCount: run.scannedCount + docs.length, + queuedCount: run.queuedCount + queuedDelta, + }, + }; +} + +type BackfillCountDocument = Pick; + +export type BackfillCountIncrementPatch = Extract< + Query, + { $inc: unknown } +>; + +export function backfillCountIncrementPatch( + outcome: 'processed' | 'failed', +): BackfillCountIncrementPatch { + return { + $inc: outcome === 'processed' ? { processedCount: 1 } : { failedCount: 1 }, + }; +} + +export function toBackfillCountUpdateQuery( + patch: BackfillCountIncrementPatch, +): Query { + return patch; +} + +export function applyAtomicBackfillCountDelta( + counters: { processedCount: number; failedCount: number }, + outcome: 'processed' | 'failed', +): { processedCount: number; failedCount: number } { + const patch = backfillCountIncrementPatch(outcome).$inc; + counters.processedCount += patch.processedCount ?? 0; + counters.failedCount += patch.failedCount ?? 0; + return counters; +} + +export function applyBackfillJobCounts( + run: BackfillRunProgress, + counts: { processed?: number; failed?: number }, +): BackfillRunResult { + if (run.state !== 'running') { + return { ok: false, reason: 'not_running' }; + } + const processedDelta = counts.processed ?? 0; + const failedDelta = counts.failed ?? 0; + if ( + !Number.isInteger(processedDelta) || + processedDelta < 0 || + !Number.isInteger(failedDelta) || + failedDelta < 0 + ) { + return { ok: false, reason: 'counts' }; + } + const processedCount = run.processedCount + processedDelta; + const failedCount = run.failedCount + failedDelta; + if (processedCount + failedCount > run.queuedCount) { + return { ok: false, reason: 'counts' }; + } + return { + ok: true, + run: { + ...run, + processedCount, + failedCount, + }, + }; +} + +export function buildBackfillPageQuery( + run: Pick, + targetField?: string, +): { ok: true; page: BackfillPageQuery } | { ok: false; reason: string } { + if (run.onlyMissing) { + if (typeof targetField !== 'string' || !SCHEMA_NAME.test(targetField)) { + return { ok: false, reason: 'targetField' }; + } + } + const query: Record = { ...(run.filter ?? {}) }; + delete query._id; + if (run.onlyMissing && targetField) { + query[targetField] = null; + } + if (run.cursor) { + if (!IDENTITY.test(run.cursor)) { + return { ok: false, reason: 'cursor' }; + } + query._id = { $gt: run.cursor }; + } + return { + ok: true, + page: { + query, + sort: { _id: 1 }, + limit: run.batchSize, + }, + }; +} + +function transition( + run: BackfillRunProgress, + to: BackfillRunState, + patch: Partial, +): BackfillRunResult { + if (!isLegalBackfillTransition(run.state, to)) { + return { ok: false, reason: 'illegal_transition' }; + } + return { + ok: true, + run: { + ...run, + ...patch, + state: to, + }, + }; +} + +function isPlainFilterObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isBackfillFilterScalar(value: unknown): boolean { + return ( + value === null || + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' + ); +} + +function isSafeBackfillFieldName(field: string): boolean { + return ( + BACKFILL_FILTER_FIELD.test(field) && + !field.startsWith('$') && + !BACKFILL_FILTER_RESERVED_FIELDS.has(field) + ); +} + +function isComparisonOperator( + operator: string, +): operator is (typeof BACKFILL_FILTER_COMPARISON_OPERATORS)[number] { + return (BACKFILL_FILTER_COMPARISON_OPERATORS as readonly string[]).includes(operator); +} + +function isMembershipOperator( + operator: string, +): operator is (typeof BACKFILL_FILTER_MEMBERSHIP_OPERATORS)[number] { + return (BACKFILL_FILTER_MEMBERSHIP_OPERATORS as readonly string[]).includes(operator); +} + +function isSafeComparisonOperand(operator: string, comparison: unknown): boolean { + if (operator === '$eq' || operator === '$ne') { + return isBackfillFilterScalar(comparison); + } + return typeof comparison === 'number' || typeof comparison === 'string'; +} + +function isSafeMembershipOperand(items: unknown): boolean { + return ( + Array.isArray(items) && + items.length <= MAX_BACKFILL_FILTER_IN_VALUES && + items.every(isBackfillFilterScalar) + ); +} + +function isSafeBackfillPredicate(value: unknown, depth: number): boolean { + if (depth > MAX_BACKFILL_FILTER_DEPTH) return false; + if (isBackfillFilterScalar(value)) return true; + if (!isPlainFilterObject(value)) return false; + const operators = Object.keys(value); + if (!operators.length || operators.length > MAX_BACKFILL_FILTER_KEYS) return false; + for (const operator of operators) { + if (isComparisonOperator(operator)) { + if (!isSafeComparisonOperand(operator, value[operator])) return false; + continue; + } + if (isMembershipOperator(operator)) { + if (!isSafeMembershipOperand(value[operator])) return false; + continue; + } + return false; + } + return true; +} + +export function isSafeBackfillFilter(value: unknown, depth = 1): boolean { + if (depth > MAX_BACKFILL_FILTER_DEPTH) return false; + if (!isPlainFilterObject(value)) return false; + const keys = Object.keys(value); + if (keys.length > MAX_BACKFILL_FILTER_KEYS) return false; + for (const key of keys) { + if (key === '$and') { + const branches = value[key]; + if ( + !Array.isArray(branches) || + branches.length === 0 || + branches.length > MAX_BACKFILL_FILTER_AND_BRANCHES + ) { + return false; + } + if (!branches.every(branch => isSafeBackfillFilter(branch, depth + 1))) { + return false; + } + continue; + } + if (key.startsWith('$') || !isSafeBackfillFieldName(key)) return false; + if (!isSafeBackfillPredicate(value[key], depth + 1)) return false; + } + return true; +} + +function normalizeFilter( + filter: unknown, +): { ok: true; filter: Record | null } | { ok: false; reason: string } { + if (filter == null) return { ok: true, filter: null }; + if (!isPlainFilterObject(filter)) { + return { ok: false, reason: 'filter' }; + } + let serialized: string; + try { + serialized = JSON.stringify(filter); + } catch { + return { ok: false, reason: 'filter' }; + } + if (serialized.length > MAX_BACKFILL_FILTER_BYTES) { + return { ok: false, reason: 'filter' }; + } + const parsed = JSON.parse(serialized) as Record; + if (!isSafeBackfillFilter(parsed)) { + return { ok: false, reason: 'filter' }; + } + return { ok: true, filter: parsed }; +} diff --git a/modules/embeddings/src/utils/clientSearchContext.test.ts b/modules/embeddings/src/utils/clientSearchContext.test.ts new file mode 100644 index 000000000..bf2f23e41 --- /dev/null +++ b/modules/embeddings/src/utils/clientSearchContext.test.ts @@ -0,0 +1,41 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertClientSearchSubject, + clampClientSearchLimit, + clientSearchSubject, + CLIENT_SEMANTIC_SEARCH_MAX_LIMIT, +} from './clientSearchContext.js'; + +describe('client semantic search context', () => { + it('accepts text-only search subjects from router context and fail-closes otherwise', () => { + assert.deepEqual( + clientSearchSubject({ user: { _id: 'user-1' }, scope: 'Team:org' }), + { + userId: 'user-1', + scope: 'Team:org', + }, + ); + assert.deepEqual(clientSearchSubject({ user: { _id: 1 } }), {}); + assert.throws( + () => assertClientSearchSubject(clientSearchSubject({})), + (err: unknown) => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + assert.doesNotThrow(() => + assertClientSearchSubject(clientSearchSubject({ user: { _id: 'user-1' } })), + ); + }); + + it('caps client semantic-search limit below the admin/gRPC maximum', () => { + assert.equal(clampClientSearchLimit(undefined), undefined); + assert.equal(clampClientSearchLimit(10), 10); + assert.equal(clampClientSearchLimit(1000), CLIENT_SEMANTIC_SEARCH_MAX_LIMIT); + assert.equal(CLIENT_SEMANTIC_SEARCH_MAX_LIMIT, 50); + assert.throws( + () => clampClientSearchLimit(0), + (err: unknown) => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, + ); + }); +}); diff --git a/modules/embeddings/src/utils/clientSearchContext.ts b/modules/embeddings/src/utils/clientSearchContext.ts new file mode 100644 index 000000000..970547da8 --- /dev/null +++ b/modules/embeddings/src/utils/clientSearchContext.ts @@ -0,0 +1,37 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export const CLIENT_SEMANTIC_SEARCH_MAX_LIMIT = 50; + +export function clientSearchSubject(context?: { + user?: { _id?: unknown }; + scope?: unknown; +}): { userId?: string; scope?: string } { + const userId = context?.user?._id; + const scope = context?.scope; + return { + ...(typeof userId === 'string' && userId.length > 0 ? { userId } : {}), + ...(typeof scope === 'string' && scope.length > 0 ? { scope } : {}), + }; +} + +export function assertClientSearchSubject(subject: { userId?: string; scope?: string }): { + userId?: string; + scope?: string; +} { + if (!subject.userId && !subject.scope) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Semantic search requires an authenticated user or scope from router context', + ); + } + return subject; +} + +export function clampClientSearchLimit(limit?: number): number | undefined { + if (limit === undefined || limit === null) return undefined; + if (!Number.isInteger(limit) || limit < 1) { + throw new GrpcError(status.INVALID_ARGUMENT, 'limit must be a positive integer'); + } + return Math.min(limit, CLIENT_SEMANTIC_SEARCH_MAX_LIMIT); +} diff --git a/modules/embeddings/src/utils/configChange.test.ts b/modules/embeddings/src/utils/configChange.test.ts new file mode 100644 index 000000000..62d81036d --- /dev/null +++ b/modules/embeddings/src/utils/configChange.test.ts @@ -0,0 +1,228 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + diffMaterialEmbeddingConfig, + embeddingConfigFingerprint, + hashFieldsToInvalidate, + hashedEmbeddingSource, + isInPlaceDimensionChange, + materialChangeWarnings, + nextEmbeddingVectorIndexName, + requiresIndexRecreation, + selectEmbeddingVectorIndex, + sameEmbeddingVectorIndexFamily, + embeddingVectorIndexMatchesContract, +} from './configChange.js'; + +const base = { + provider: 'openai-compatible', + modelName: 'text-embedding-3-small', + dimensions: 1536, + sourceFields: ['title', 'body'], + targetField: 'embedding', + similarity: 'cosine', +}; + +describe('material embedding config changes', () => { + it('detects provider, model, dimensions, source fields, target, and similarity changes', () => { + assert.deepEqual(diffMaterialEmbeddingConfig(base, base), []); + assert.deepEqual( + diffMaterialEmbeddingConfig(base, { ...base, sourceFields: ['body', 'title'] }), + [], + ); + assert.deepEqual(diffMaterialEmbeddingConfig(base, { ...base, modelName: 'large' }), [ + 'modelName', + ]); + assert.deepEqual( + diffMaterialEmbeddingConfig(base, { + ...base, + provider: 'other', + dimensions: 768, + sourceFields: ['title'], + targetField: 'vector', + similarity: 'euclidean', + }), + ['provider', 'dimensions', 'sourceFields', 'targetField', 'similarity'], + ); + }); + + it('requires index recreation for dimensions, target field, and similarity', () => { + assert.equal(requiresIndexRecreation(['modelName', 'provider']), false); + assert.equal(requiresIndexRecreation(['sourceFields']), false); + assert.equal(requiresIndexRecreation(['similarity']), true); + assert.equal(requiresIndexRecreation(['dimensions']), true); + assert.equal(requiresIndexRecreation(['targetField']), true); + assert.equal(isInPlaceDimensionChange(base, { ...base, dimensions: 768 }), true); + assert.equal( + isInPlaceDimensionChange(base, { + ...base, + targetField: 'other', + dimensions: 768, + }), + false, + ); + }); + + it('invalidates old hashes via fingerprint and names hash fields to clear', () => { + const hash = (input: string) => input; + const original = hashedEmbeddingSource(hash, 'Hello', base); + const changedModel = hashedEmbeddingSource(hash, 'Hello', { + ...base, + modelName: 'large', + }); + assert.notEqual(original, changedModel); + assert.equal( + embeddingConfigFingerprint(base).includes('text-embedding-3-small'), + true, + ); + assert.deepEqual(hashFieldsToInvalidate(base, { ...base, targetField: 'other' }), [ + 'embeddingSourceHash', + 'otherSourceHash', + ]); + }); + + it('warns that stale vectors require an explicit backfill', () => { + const warnings = materialChangeWarnings(['modelName'], false); + assert.equal( + warnings.some(warning => /invalidated stored source hashes/.test(warning)), + true, + ); + assert.equal( + warnings.some(warning => /explicit backfill/.test(warning)), + true, + ); + assert.equal( + materialChangeWarnings(['similarity'], true).some(warning => + /index recreation is required/i.test(warning), + ), + true, + ); + }); + + it('versions replacement names from live provider-specific indexes', () => { + assert.equal(nextEmbeddingVectorIndexName('embedding', []), 'embedding_vector'); + assert.equal( + nextEmbeddingVectorIndexName('embedding', [ + { field: 'embedding', name: 'embedding_vector' }, + ]), + 'embedding_vector_v2', + ); + assert.equal( + nextEmbeddingVectorIndexName('embedding', [ + { field: 'embedding', name: 'cnd_Article_embedding_vector' }, + { field: 'embedding', name: 'cnd_Article_embedding_vector_v2' }, + ]), + 'cnd_Article_embedding_vector_v3', + ); + assert.equal( + sameEmbeddingVectorIndexFamily('embedding_vector', 'embedding_vector_v2'), + true, + ); + assert.equal( + selectEmbeddingVectorIndex( + [ + { field: 'embedding', name: 'embedding_vector' }, + { field: 'embedding', name: 'embedding_vector_v2' }, + { field: 'title', name: 'title_vector_v9' }, + ], + 'embedding', + )?.name, + 'embedding_vector_v2', + ); + }); + + it('matches live indexes only when field, dimensions, similarity, and method agree', () => { + const cosine = { + field: 'embedding', + name: 'embedding_vector', + dimensions: 3, + similarity: 'cosine', + method: 'hnsw', + }; + const euclidean = { + ...cosine, + name: 'embedding_vector_v2', + similarity: 'euclidean', + }; + const ivf = { ...cosine, method: 'ivfflat' }; + assert.equal( + embeddingVectorIndexMatchesContract(cosine, { + field: 'embedding', + dimensions: 3, + similarity: 'cosine', + }), + true, + ); + assert.equal( + embeddingVectorIndexMatchesContract(cosine, { + field: 'embedding', + dimensions: 3, + similarity: 'euclidean', + }), + false, + ); + assert.equal( + embeddingVectorIndexMatchesContract(cosine, { + field: 'embedding', + dimensions: 8, + similarity: 'cosine', + }), + false, + ); + assert.equal( + embeddingVectorIndexMatchesContract(ivf, { + field: 'embedding', + dimensions: 3, + similarity: 'cosine', + }), + false, + ); + assert.equal( + selectEmbeddingVectorIndex([cosine, euclidean], 'embedding', { + dimensions: 3, + similarity: 'euclidean', + })?.name, + 'embedding_vector_v2', + ); + assert.equal( + selectEmbeddingVectorIndex([cosine], 'embedding', { + dimensions: 3, + similarity: 'euclidean', + }), + undefined, + ); + }); + + it('treats empty and missing index methods as default hnsw', () => { + const missingMethod = { + field: 'embedding', + name: 'embedding_vector', + dimensions: 3, + similarity: 'cosine', + }; + const emptyMethod = { ...missingMethod, method: '' }; + const hnsw = { ...missingMethod, method: 'hnsw' }; + const contract = { + field: 'embedding', + dimensions: 3, + similarity: 'cosine', + }; + assert.equal(embeddingVectorIndexMatchesContract(missingMethod, contract), true); + assert.equal(embeddingVectorIndexMatchesContract(emptyMethod, contract), true); + assert.equal( + embeddingVectorIndexMatchesContract(hnsw, { ...contract, method: '' }), + true, + ); + assert.equal( + selectEmbeddingVectorIndex([missingMethod], 'embedding', { + dimensions: 3, + similarity: 'cosine', + })?.name, + 'embedding_vector', + ); + assert.equal( + nextEmbeddingVectorIndexName('embedding', [missingMethod]), + 'embedding_vector_v2', + ); + }); +}); diff --git a/modules/embeddings/src/utils/configChange.ts b/modules/embeddings/src/utils/configChange.ts new file mode 100644 index 000000000..cf4580735 --- /dev/null +++ b/modules/embeddings/src/utils/configChange.ts @@ -0,0 +1,230 @@ +import { vectorIndexMethodsEquivalent } from '@conduitplatform/grpc-sdk'; + +export const MATERIAL_EMBEDDING_CONFIG_FIELDS = [ + 'provider', + 'modelName', + 'dimensions', + 'sourceFields', + 'targetField', + 'similarity', +] as const; + +export type MaterialEmbeddingConfigField = + (typeof MATERIAL_EMBEDDING_CONFIG_FIELDS)[number]; + +export interface MaterialEmbeddingConfig { + provider: string; + modelName?: string; + dimensions: number; + sourceFields: readonly string[]; + targetField: string; + similarity?: string; +} + +export function normalizeSourceFields(fields: readonly string[]): string[] { + return [...fields].map(field => field.trim()).sort(); +} + +export function embeddingConfigFingerprint(config: MaterialEmbeddingConfig): string { + return JSON.stringify({ + provider: config.provider, + model: config.modelName ?? '', + dimensions: config.dimensions, + sourceFields: normalizeSourceFields(config.sourceFields), + targetField: config.targetField, + similarity: config.similarity ?? '', + }); +} + +export function hashedEmbeddingSource( + hashInput: (input: string) => string, + input: string, + config: MaterialEmbeddingConfig, +): string { + return hashInput(`${embeddingConfigFingerprint(config)}\n${input}`); +} + +export function sameSourceFields( + left: readonly string[] | undefined, + right: readonly string[] | undefined, +): boolean { + return ( + JSON.stringify(normalizeSourceFields(left ?? [])) === + JSON.stringify(normalizeSourceFields(right ?? [])) + ); +} + +export function diffMaterialEmbeddingConfig( + existing: MaterialEmbeddingConfig, + next: MaterialEmbeddingConfig, +): MaterialEmbeddingConfigField[] { + const changed: MaterialEmbeddingConfigField[] = []; + if (existing.provider !== next.provider) changed.push('provider'); + if ((existing.modelName ?? '') !== (next.modelName ?? '')) changed.push('modelName'); + if (existing.dimensions !== next.dimensions) changed.push('dimensions'); + if (!sameSourceFields(existing.sourceFields, next.sourceFields)) { + changed.push('sourceFields'); + } + if (existing.targetField !== next.targetField) changed.push('targetField'); + if ((existing.similarity ?? '') !== (next.similarity ?? '')) { + changed.push('similarity'); + } + return changed; +} + +export function requiresIndexRecreation( + changed: readonly MaterialEmbeddingConfigField[], +): boolean { + return changed.some( + field => field === 'dimensions' || field === 'targetField' || field === 'similarity', + ); +} + +export function isInPlaceDimensionChange( + existing: MaterialEmbeddingConfig, + next: MaterialEmbeddingConfig, +): boolean { + return ( + existing.targetField === next.targetField && existing.dimensions !== next.dimensions + ); +} + +export function sourceHashField(targetField: string): string { + return `${targetField}SourceHash`; +} + +export function hashFieldsToInvalidate( + existing: MaterialEmbeddingConfig, + next: MaterialEmbeddingConfig, +): string[] { + return [ + ...new Set([ + sourceHashField(existing.targetField), + sourceHashField(next.targetField), + ]), + ]; +} + +export function defaultEmbeddingVectorIndexName(field: string): string { + return `${field}_vector`; +} + +export function parseEmbeddingVectorIndexName(name: string): { + base: string; + generation: number; +} { + const match = /^(.*)_v(\d+)$/.exec(name); + if (match) { + return { base: match[1], generation: Number(match[2]) }; + } + return { base: name, generation: 1 }; +} + +export function embeddingVectorIndexGeneration(name?: string): number { + if (typeof name !== 'string' || name.length === 0) return 0; + return parseEmbeddingVectorIndexName(name).generation; +} + +export function sameEmbeddingVectorIndexFamily(left: string, right: string): boolean { + return ( + parseEmbeddingVectorIndexName(left).base === parseEmbeddingVectorIndexName(right).base + ); +} + +export function nextEmbeddingVectorIndexName( + field: string, + indexes: ReadonlyArray<{ field?: string; name?: string }>, +): string { + const names = indexes + .filter( + index => + index.field === field && typeof index.name === 'string' && index.name.length > 0, + ) + .map(index => index.name as string); + if (!names.length) return defaultEmbeddingVectorIndexName(field); + let base = defaultEmbeddingVectorIndexName(field); + let maxGeneration = 0; + for (const name of names) { + const parsed = parseEmbeddingVectorIndexName(name); + if (parsed.generation >= maxGeneration) { + maxGeneration = parsed.generation; + base = parsed.base; + } + } + return `${base}_v${maxGeneration + 1}`; +} + +export interface EmbeddingVectorIndexContract { + field: string; + dimensions: number; + similarity?: string; + method?: string; +} + +export interface EmbeddingVectorIndexShape { + field?: string; + name?: string; + dimensions?: number; + similarity?: string; + method?: string; +} + +export function embeddingVectorIndexMatchesContract( + index: EmbeddingVectorIndexShape, + contract: EmbeddingVectorIndexContract, +): boolean { + if (index.field !== contract.field) return false; + if (typeof index.dimensions !== 'number' || index.dimensions !== contract.dimensions) { + return false; + } + if ((index.similarity ?? '') !== (contract.similarity ?? '')) return false; + return vectorIndexMethodsEquivalent(index.method, contract.method); +} + +export function selectEmbeddingVectorIndex( + indexes: readonly T[], + field: string, + contract?: Omit, +): T | undefined { + const matches = indexes.filter(index => { + if (index.field !== field) return false; + if (!contract) return true; + return embeddingVectorIndexMatchesContract(index, { field, ...contract }); + }); + if (!matches.length) return undefined; + const defaultName = defaultEmbeddingVectorIndexName(field); + return matches.reduce((best, current) => { + const bestGeneration = embeddingVectorIndexGeneration(best.name); + const currentGeneration = embeddingVectorIndexGeneration(current.name); + if (currentGeneration !== bestGeneration) { + return currentGeneration > bestGeneration ? current : best; + } + if (current.name === defaultName) return current; + if (best.name === defaultName) return best; + return best; + }); +} + +export function materialChangeWarnings( + changed: readonly MaterialEmbeddingConfigField[], + scheduledBackfill: boolean, +): string[] { + if (!changed.length) return []; + const warnings = [ + `Material embedding config change (${changed.join(', ')}) invalidated stored source hashes. ` + + 'Existing vectors are stale until an explicit backfill completes.', + ]; + if (requiresIndexRecreation(changed)) { + warnings.push( + 'Vector index recreation is required for this change. Wait until the index is queryable before searching or backfilling.', + ); + } + if (!scheduledBackfill) { + warnings.push( + 'Start an explicit backfill after the vector index is queryable. Stale vectors will not be reused by hash skip.', + ); + } else { + warnings.push('An explicit backfill was scheduled for this config.'); + } + return warnings; +} diff --git a/modules/embeddings/src/utils/embeddingJobs.test.ts b/modules/embeddings/src/utils/embeddingJobs.test.ts new file mode 100644 index 000000000..800c3f804 --- /dev/null +++ b/modules/embeddings/src/utils/embeddingJobs.test.ts @@ -0,0 +1,77 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + dedupeEmbeddingJobs, + embeddingJobId, + isDuplicateJobError, + isInFlightQueueJobState, + isTerminalQueueJobState, + parseEmbeddingJobData, + parseEmbeddingJobBatch, + shouldReplaceRetainedQueueJob, +} from './embeddingJobs.js'; + +describe('embedding job identity', () => { + it('deduplicates jobs by schema, document, and config identity', () => { + const jobs = dedupeEmbeddingJobs([ + { schemaName: 'Article', documentId: 'a' }, + { schemaName: 'Article', documentId: 'a' }, + { schemaName: 'Article', documentId: 'a', configId: 'c1' }, + { schemaName: 'Article', documentId: 'b', configId: 'c1' }, + { schemaName: 'Article', documentId: 'a', configId: 'c1' }, + ]); + assert.deepEqual( + jobs.map(job => embeddingJobId(job)), + ['Article__a', 'Article__a__c1', 'Article__b__c1'], + ); + }); + + it('detects BullMQ duplicate job errors', () => { + assert.equal(isDuplicateJobError(new Error('Job Article__a already exists')), true); + assert.equal(isDuplicateJobError(new Error('redis timeout')), false); + }); + + it('replaces retained completed or failed jobs and only dedupes in-flight work', () => { + assert.equal(isInFlightQueueJobState('waiting'), true); + assert.equal(isInFlightQueueJobState('active'), true); + assert.equal(isInFlightQueueJobState('delayed'), true); + assert.equal(isInFlightQueueJobState('completed'), false); + assert.equal(isTerminalQueueJobState('completed'), true); + assert.equal(isTerminalQueueJobState('failed'), true); + assert.equal(shouldReplaceRetainedQueueJob('completed'), true); + assert.equal(shouldReplaceRetainedQueueJob('failed'), true); + assert.equal(shouldReplaceRetainedQueueJob('waiting'), false); + assert.equal(shouldReplaceRetainedQueueJob('active'), false); + }); + + it('rejects malformed and oversized queue payloads', () => { + assert.equal( + parseEmbeddingJobData({ schemaName: 'Article', documentId: 'a' }).ok, + true, + ); + assert.equal(parseEmbeddingJobData({ schemaName: 'Article' }).ok, false); + assert.equal( + parseEmbeddingJobData({ schemaName: '../etc', documentId: 'a' }).ok, + false, + ); + assert.equal( + parseEmbeddingJobData({ + schemaName: 'Article', + documentId: 'a', + configId: 'c1', + backfillRunId: 'run1', + }).ok, + true, + ); + assert.equal( + parseEmbeddingJobData({ schemaName: 'Article', documentId: 'a', extra: true }).ok, + false, + ); + assert.equal( + parseEmbeddingJobBatch( + new Array(600).fill({ schemaName: 'Article', documentId: 'a' }), + ).length, + 500, + ); + }); +}); diff --git a/modules/embeddings/src/utils/embeddingJobs.ts b/modules/embeddings/src/utils/embeddingJobs.ts new file mode 100644 index 000000000..4c3f740e3 --- /dev/null +++ b/modules/embeddings/src/utils/embeddingJobs.ts @@ -0,0 +1,120 @@ +export interface EmbeddingJobData { + schemaName: string; + documentId: string; + configId?: string; + backfillRunId?: string; +} + +export const MAX_SCHEMA_NAME_LENGTH = 128; +export const MAX_DOCUMENT_ID_LENGTH = 128; +export const MAX_QUEUE_BATCH_SIZE = 500; + +const IDENTITY = /^[A-Za-z0-9._-]{1,128}$/; +const SCHEMA_NAME = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; + +export const IN_FLIGHT_QUEUE_JOB_STATES = [ + 'waiting', + 'active', + 'delayed', + 'paused', + 'waiting-children', + 'prioritized', +] as const; + +export const TERMINAL_QUEUE_JOB_STATES = ['completed', 'failed'] as const; + +export type InFlightQueueJobState = (typeof IN_FLIGHT_QUEUE_JOB_STATES)[number]; +export type TerminalQueueJobState = (typeof TERMINAL_QUEUE_JOB_STATES)[number]; + +export function embeddingJobId(data: EmbeddingJobData): string { + const parts = [data.schemaName, data.documentId]; + if (data.configId) parts.push(data.configId); + return parts.join('__'); +} + +export function isInFlightQueueJobState(state: string): state is InFlightQueueJobState { + return (IN_FLIGHT_QUEUE_JOB_STATES as readonly string[]).includes(state); +} + +export function isTerminalQueueJobState(state: string): state is TerminalQueueJobState { + return (TERMINAL_QUEUE_JOB_STATES as readonly string[]).includes(state); +} + +export function shouldReplaceRetainedQueueJob(state: string): boolean { + return isTerminalQueueJobState(state); +} + +export function dedupeEmbeddingJobs(jobs: EmbeddingJobData[]): EmbeddingJobData[] { + const seen = new Set(); + const unique: EmbeddingJobData[] = []; + for (const job of jobs) { + const id = embeddingJobId(job); + if (seen.has(id)) continue; + seen.add(id); + unique.push(job); + } + return unique; +} + +export function isDuplicateJobError(err: unknown): boolean { + const message = err instanceof Error ? err.message : String(err); + return /already exists/i.test(message); +} + +export type ParsedEmbeddingJob = + { ok: true; data: EmbeddingJobData } | { ok: false; reason: string }; + +function optionalIdentity( + value: unknown, + reason: 'configId' | 'backfillRunId', +): { ok: true; value?: string } | { ok: false; reason: string } { + if (value === undefined) return { ok: true }; + if (typeof value !== 'string' || !IDENTITY.test(value)) { + return { ok: false, reason }; + } + return { ok: true, value }; +} + +export function parseEmbeddingJobData( + value: unknown, + maxBatchIndex?: number, +): ParsedEmbeddingJob { + if (maxBatchIndex !== undefined && maxBatchIndex >= MAX_QUEUE_BATCH_SIZE) { + return { ok: false, reason: 'batch_size' }; + } + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return { ok: false, reason: 'malformed' }; + } + const record = value as Record; + const extraKeys = Object.keys(record).filter( + key => !['schemaName', 'documentId', 'configId', 'backfillRunId'].includes(key), + ); + if (extraKeys.length) return { ok: false, reason: 'malformed' }; + if (typeof record.schemaName !== 'string' || !SCHEMA_NAME.test(record.schemaName)) { + return { ok: false, reason: 'schemaName' }; + } + if (typeof record.documentId !== 'string' || !IDENTITY.test(record.documentId)) { + return { ok: false, reason: 'documentId' }; + } + const configId = optionalIdentity(record.configId, 'configId'); + if (!configId.ok) return configId; + const backfillRunId = optionalIdentity(record.backfillRunId, 'backfillRunId'); + if (!backfillRunId.ok) return backfillRunId; + return { + ok: true, + data: { + schemaName: record.schemaName, + documentId: record.documentId, + ...(configId.value ? { configId: configId.value } : {}), + ...(backfillRunId.value ? { backfillRunId: backfillRunId.value } : {}), + }, + }; +} + +export function parseEmbeddingJobBatch(values: unknown[]): EmbeddingJobData[] { + return values + .slice(0, MAX_QUEUE_BATCH_SIZE) + .map(value => parseEmbeddingJobData(value)) + .filter((parsed): parsed is { ok: true; data: EmbeddingJobData } => parsed.ok) + .map(parsed => parsed.data); +} diff --git a/modules/embeddings/src/utils/embeddingMetrics.ts b/modules/embeddings/src/utils/embeddingMetrics.ts new file mode 100644 index 000000000..6a80009e7 --- /dev/null +++ b/modules/embeddings/src/utils/embeddingMetrics.ts @@ -0,0 +1,21 @@ +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; + +export const EMBEDDING_METRICS = { + generated: 'generated_embeddings_total', + failed: 'failed_embeddings_total', + skipped: 'skipped_embeddings_total', + retried: 'retried_embeddings_total', + backfill: 'embedding_backfill_jobs_total', + malformedEvents: 'malformed_embedding_events_total', + malformedJobs: 'malformed_embedding_jobs_total', +} as const; + +export type EmbeddingMetric = keyof typeof EMBEDDING_METRICS; + +export function incrementEmbeddingMetric( + metric: EmbeddingMetric, + amount: number = 1, +): void { + if (!Number.isFinite(amount) || amount <= 0) return; + ConduitGrpcSdk.Metrics?.increment(EMBEDDING_METRICS[metric], amount); +} diff --git a/modules/embeddings/src/utils/endpointSecurity.test.ts b/modules/embeddings/src/utils/endpointSecurity.test.ts new file mode 100644 index 000000000..39c55082a --- /dev/null +++ b/modules/embeddings/src/utils/endpointSecurity.test.ts @@ -0,0 +1,80 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertSafeEmbeddingEndpoint, + isBlockedIp, + readCappedResponse, +} from './endpointSecurity.js'; + +describe('embedding endpoint SSRF controls', () => { + it('blocks private, link-local, and metadata addresses', () => { + assert.equal(isBlockedIp('127.0.0.1'), true); + assert.equal(isBlockedIp('10.0.0.5'), true); + assert.equal(isBlockedIp('192.168.1.20'), true); + assert.equal(isBlockedIp('169.254.169.254'), true); + assert.equal(isBlockedIp('::1'), true); + assert.equal(isBlockedIp('::ffff:127.0.0.1'), true); + assert.equal(isBlockedIp('8.8.8.8'), false); + }); + + it('requires HTTPS and rejects URL credentials', async () => { + await assert.rejects( + () => assertSafeEmbeddingEndpoint('http://api.openai.com/v1/embeddings'), + err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, + ); + await assert.rejects( + () => + assertSafeEmbeddingEndpoint('https://user:pass@api.openai.com/v1/embeddings', { + lookup: async () => [{ address: '104.18.0.1', family: 4 }], + }), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /credentials/.test(err.message), + ); + }); + + it('derives the hostname from the HTTPS endpoint and allows public resolutions', async () => { + const url = await assertSafeEmbeddingEndpoint( + 'https://api.openai.com/v1/embeddings', + { + lookup: async () => [{ address: '104.18.0.1', family: 4 }], + }, + ); + assert.equal(url.hostname, 'api.openai.com'); + const other = await assertSafeEmbeddingEndpoint( + 'https://evil.example/v1/embeddings', + { + lookup: async () => [{ address: '104.18.0.1', family: 4 }], + }, + ); + assert.equal(other.hostname, 'evil.example'); + }); + + it('rejects DNS results that resolve to private or metadata addresses', async () => { + await assert.rejects( + () => + assertSafeEmbeddingEndpoint('https://api.openai.com/v1/embeddings', { + lookup: async () => [{ address: '169.254.169.254', family: 4 }], + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + await assert.rejects( + () => + assertSafeEmbeddingEndpoint('https://localhost/v1/embeddings', { + lookup: async () => [{ address: '127.0.0.1', family: 4 }], + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + }); + + it('caps response payload size', async () => { + const response = new Response('x'.repeat(20), { status: 200 }); + await assert.rejects( + () => readCappedResponse(response, 8), + err => err instanceof GrpcError && err.code === status.RESOURCE_EXHAUSTED, + ); + }); +}); diff --git a/modules/embeddings/src/utils/endpointSecurity.ts b/modules/embeddings/src/utils/endpointSecurity.ts new file mode 100644 index 000000000..e60bee0ee --- /dev/null +++ b/modules/embeddings/src/utils/endpointSecurity.ts @@ -0,0 +1,137 @@ +import { BlockList, isIP } from 'node:net'; +import { lookup as defaultLookup } from 'node:dns/promises'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export const DEFAULT_EMBED_TIMEOUT_MS = 10_000; +export const DEFAULT_MAX_EMBED_INPUT_BYTES = 32 * 1024; +export const DEFAULT_MAX_EMBED_RESPONSE_BYTES = 1024 * 1024; + +const BLOCKED_HOSTNAMES = new Set([ + 'localhost', + 'metadata', + 'metadata.google.internal', + 'metadata.google.com', +]); + +const privateNetworks = new BlockList(); +privateNetworks.addSubnet('0.0.0.0', 8, 'ipv4'); +privateNetworks.addSubnet('10.0.0.0', 8, 'ipv4'); +privateNetworks.addSubnet('100.64.0.0', 10, 'ipv4'); +privateNetworks.addSubnet('127.0.0.0', 8, 'ipv4'); +privateNetworks.addSubnet('169.254.0.0', 16, 'ipv4'); +privateNetworks.addSubnet('172.16.0.0', 12, 'ipv4'); +privateNetworks.addSubnet('192.168.0.0', 16, 'ipv4'); +privateNetworks.addSubnet('::1', 128, 'ipv6'); +privateNetworks.addAddress('::', 'ipv6'); +privateNetworks.addSubnet('fc00::', 7, 'ipv6'); +privateNetworks.addSubnet('fe80::', 10, 'ipv6'); + +export interface SafeEndpointOptions { + lookup?: ( + hostname: string, + options: { all: true; verbatim: true }, + ) => Promise>; +} + +export function isBlockedIp(address: string): boolean { + const mapped = address.startsWith('::ffff:') ? address.slice(7) : address; + const ipVersion = isIP(mapped); + if (ipVersion === 4) return privateNetworks.check(mapped, 'ipv4'); + if (ipVersion === 6) return privateNetworks.check(mapped, 'ipv6'); + return true; +} + +export async function assertSafeEmbeddingEndpoint( + endpoint: string, + options: SafeEndpointOptions = {}, +): Promise { + let url: URL; + try { + url = new URL(endpoint); + } catch { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding provider endpoint is invalid', + ); + } + if (url.protocol !== 'https:') { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding provider endpoint must use HTTPS', + ); + } + if (url.username || url.password) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Embedding provider endpoint must not include credentials', + ); + } + const hostname = url.hostname.toLowerCase(); + if (BLOCKED_HOSTNAMES.has(hostname)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embedding provider endpoint host is not allowed', + ); + } + if (isIP(hostname)) { + if (isBlockedIp(hostname)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embedding provider endpoint resolves to a blocked address', + ); + } + return url; + } + const lookup = options.lookup ?? defaultLookup; + const resolved = await lookup(hostname, { all: true, verbatim: true }); + const records = Array.isArray(resolved) ? resolved : [resolved]; + if (!records.length) { + throw new GrpcError( + status.FAILED_PRECONDITION, + 'Embedding provider endpoint host could not be resolved', + ); + } + for (const record of records) { + if (isBlockedIp(record.address)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Embedding provider endpoint resolves to a blocked address', + ); + } + } + return url; +} + +export async function readCappedResponse( + response: Response, + maxBytes: number, +): Promise { + if (!response.body) { + const text = await response.text(); + if (Buffer.byteLength(text) > maxBytes) { + throw new GrpcError( + status.RESOURCE_EXHAUSTED, + 'Embedding provider response is too large', + ); + } + return text; + } + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let received = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + received += value.byteLength; + if (received > maxBytes) { + await reader.cancel(); + throw new GrpcError( + status.RESOURCE_EXHAUSTED, + 'Embedding provider response is too large', + ); + } + chunks.push(value); + } + return Buffer.concat(chunks).toString('utf8'); +} diff --git a/modules/embeddings/src/utils/mcpToolNames.test.ts b/modules/embeddings/src/utils/mcpToolNames.test.ts new file mode 100644 index 000000000..f7f8d75e4 --- /dev/null +++ b/modules/embeddings/src/utils/mcpToolNames.test.ts @@ -0,0 +1,92 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { ConduitRouteActions } from '@conduitplatform/grpc-sdk'; +import { ConduitNumber, ConduitString } from '@conduitplatform/module-tools'; +import { embeddingsMcpToolName, embeddingsPublicPath } from './mcpToolNames.js'; +import { + CONFIG_BODY, + EMBEDDINGS_ADMIN_ROUTES, + EMBEDDINGS_CLIENT_FORBIDDEN_PATHS, + EMBEDDINGS_CLIENT_SEARCH_PATH, +} from '../admin/routes.js'; +import { EmbeddingsRoutes } from '../routes/index.js'; + +describe('embeddings MCP tool names', () => { + it('mirrors Hermes route-to-tool naming after the module prefix', () => { + assert.equal(embeddingsPublicPath('/configs'), '/embeddings/configs'); + assert.equal( + embeddingsMcpToolName('GET', '/embeddings/configs'), + 'get_embeddings_configs', + ); + assert.equal( + embeddingsMcpToolName('POST', '/embeddings/backfills/id/cancel'), + 'post_embeddings_backfills_id_cancel', + ); + }); + + it('exposes operator /embeddings/* admin routes with descriptions and MCP names', () => { + const names = EMBEDDINGS_ADMIN_ROUTES.map(route => route.mcpName); + assert.deepEqual(names, [ + 'get_embeddings_configs', + 'post_embeddings_configs', + 'get_embeddings_configs_id', + 'delete_embeddings_configs_id', + 'get_embeddings_capabilities', + 'get_embeddings_status', + 'get_embeddings_backfills', + 'post_embeddings_backfills', + 'get_embeddings_backfills_id', + 'post_embeddings_backfills_id_cancel', + 'post_embeddings_backfills_id_resume', + 'post_embeddings_search', + ]); + for (const route of EMBEDDINGS_ADMIN_ROUTES) { + assert.equal(route.publicPath.startsWith('/embeddings/'), true); + assert.equal(route.clientExposed, false); + assert.equal(route.description.length > 20, true); + assert.match(route.description, /Operator-only/); + assert.equal(route.mcpName, embeddingsMcpToolName(route.action, route.publicPath)); + } + assert.equal( + EMBEDDINGS_ADMIN_ROUTES.some( + route => route.path === '/backfills' && route.action === ConduitRouteActions.POST, + ), + true, + ); + }); + + it('never exposes config, backfill, capabilities, or status as client routes', () => { + assert.deepEqual(EmbeddingsRoutes.clientForbiddenPaths(), [ + '/configs', + '/backfills', + '/capabilities', + '/status', + ]); + for (const path of EMBEDDINGS_CLIENT_FORBIDDEN_PATHS) { + assert.equal( + EMBEDDINGS_ADMIN_ROUTES.some( + route => route.path === path || route.path.startsWith(`${path}/`), + ), + true, + ); + } + assert.equal(EMBEDDINGS_CLIENT_SEARCH_PATH, '/search'); + }); + + it('accepts optional catalogue fields on Admin config upsert', () => { + assert.deepEqual(Object.keys(CONFIG_BODY), [ + 'schemaName', + 'sourceFields', + 'targetField', + 'provider', + 'model', + 'dimensions', + 'similarity', + 'sourceFieldAllowlist', + 'enabled', + ]); + assert.equal(CONFIG_BODY.model, ConduitString.Optional); + assert.equal(CONFIG_BODY.dimensions, ConduitNumber.Optional); + assert.equal(CONFIG_BODY.provider, ConduitString.Optional); + }); +}); diff --git a/modules/embeddings/src/utils/mcpToolNames.ts b/modules/embeddings/src/utils/mcpToolNames.ts new file mode 100644 index 000000000..6e08496f8 --- /dev/null +++ b/modules/embeddings/src/utils/mcpToolNames.ts @@ -0,0 +1,21 @@ +export function embeddingsMcpToolName(action: string, publicPath: string): string { + const cleanPath = publicPath + .replace(/^\/admin\//, '') + .replace(/\//g, '_') + .replace(/[^a-zA-Z0-9_]/g, '') + .toLowerCase(); + return `${action.toLowerCase()}${cleanPath}`; +} + +export function embeddingsPublicPath( + routePath: string, + moduleName = 'embeddings', +): string { + if ( + routePath.startsWith(`/${moduleName}/`) || + routePath.startsWith(`/hook/${moduleName}/`) + ) { + return routePath; + } + return `/${moduleName}${routePath}`; +} diff --git a/modules/embeddings/src/utils/moduleConfigLifecycle.test.ts b/modules/embeddings/src/utils/moduleConfigLifecycle.test.ts new file mode 100644 index 000000000..b2bbfb9de --- /dev/null +++ b/modules/embeddings/src/utils/moduleConfigLifecycle.test.ts @@ -0,0 +1,75 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import convict from 'convict'; +import { + merge, + reconcileStoredModuleConfig, + restoreRedactedSecrets, +} from '@conduitplatform/module-tools'; +import AppConfigSchema, { type Config } from '../config/index.js'; +import { normalizeEmbeddingsConfig } from './providerConfig.js'; + +describe('embeddings module config lifecycle', () => { + it('migrates stored legacy provider settings and keeps them across setConfig', async () => { + const schema = convict(AppConfigSchema); + const stored = normalizeEmbeddingsConfig({ + ...schema.getProperties(), + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-live', + model: 'text-embedding-3-small', + dimensions: 1536, + models: [], + allowedHosts: ['api.openai.com'], + }, + }, + security: { + ...schema.getProperties().security, + requireGrpcKey: true, + }, + } as unknown as Config); + schema.load(stored).validate({ allowed: 'warn' }); + let persistCalls = 0; + const reconciled = await reconcileStoredModuleConfig({ + stored: { + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-live', + model: 'text-embedding-3-small', + dimensions: 1536, + models: [], + allowedHosts: ['api.openai.com'], + }, + }, + security: { requireGrpcKey: true, sourceFieldAllowlist: [] }, + } as unknown as Config, + migrated: schema.getProperties() as Config, + configureOverride: async config => { + persistCalls += 1; + return config; + }, + }); + assert.equal(persistCalls, 1); + schema.load(reconciled.config); + + const previous = schema.getProperties() as Config; + let next = merge(previous, { enabled: true } as Config); + next = restoreRedactedSecrets(next, previous, AppConfigSchema); + next = normalizeEmbeddingsConfig(next); + schema.load(next).validate({ allowed: 'warn' }); + const patched = schema.getProperties(); + const provider = patched.providers['openai-compatible']; + assert.equal(patched.enabled, true); + assert.equal(provider.apiKey, 'sk-live'); + assert.deepEqual(provider.models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + assert.equal(provider.defaultModel, 'text-embedding-3-small'); + assert.equal('model' in provider, false); + assert.equal('dimensions' in provider, false); + assert.equal('allowedHosts' in provider, false); + assert.equal('requireGrpcKey' in patched.security, false); + }); +}); diff --git a/modules/embeddings/src/utils/mutationEvents.test.ts b/modules/embeddings/src/utils/mutationEvents.test.ts new file mode 100644 index 000000000..e74144e7d --- /dev/null +++ b/modules/embeddings/src/utils/mutationEvents.test.ts @@ -0,0 +1,84 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + embeddingOwnedFields, + extractDocumentIds, + isEmbeddingOwnedMutation, + parseBoundedMutationEvent, +} from './mutationEvents.js'; + +describe('embedding mutation event parsing', () => { + it('normalizes create, update, and bulk payloads to unique ids', () => { + const parsed = parseBoundedMutationEvent(JSON.stringify({ _id: 'a', title: 'x' })); + assert.equal(parsed.ok, true); + if (parsed.ok) { + assert.deepEqual(parsed.event, { + payload: { _id: 'a', title: 'x' }, + ids: ['a'], + }); + } + assert.deepEqual(extractDocumentIds([{ _id: 'a' }, { _id: 'b' }, { _id: 'a' }]), [ + 'a', + 'b', + ]); + assert.deepEqual(extractDocumentIds({ ids: ['a', 'b', 'a'] }), ['a', 'b']); + }); + + it('ignores Mongo updateMany result objects that have no document ids', () => { + assert.deepEqual( + extractDocumentIds({ + acknowledged: true, + matchedCount: 4, + modifiedCount: 4, + }), + [], + ); + const parsed = parseBoundedMutationEvent( + JSON.stringify({ acknowledged: true, matchedCount: 2, modifiedCount: 2 }), + ); + assert.equal(parsed.ok, true); + if (parsed.ok) { + assert.equal(parsed.event.ids.length, 0); + } + }); + + it('parses bounded bulk id chunks', () => { + assert.deepEqual(extractDocumentIds([{ _id: '1' }, { _id: '2' }]), ['1', '2']); + }); + + it('treats embedding-owned write-backs as skippable and keeps source updates', () => { + const owned = embeddingOwnedFields([{ targetField: 'embedding' }]); + assert.equal( + isEmbeddingOwnedMutation( + { + _id: 'a', + embedding: [0.1, 0.2], + embeddingSourceHash: 'abc', + updatedAt: 'now', + }, + owned, + ), + true, + ); + assert.equal( + isEmbeddingOwnedMutation({ _id: 'a', title: 'changed', embedding: [0.1] }, owned), + false, + ); + assert.equal(isEmbeddingOwnedMutation({ _id: 'a' }, owned), false); + }); + + it('fails closed on oversized bus payloads instead of crashing', () => { + assert.deepEqual(parseBoundedMutationEvent('{not json'), { + ok: false, + reason: 'malformed', + }); + assert.deepEqual(parseBoundedMutationEvent('x'.repeat(300_000)), { + ok: false, + reason: 'capped', + }); + assert.deepEqual( + parseBoundedMutationEvent(JSON.stringify({ ids: ['a', 'b', 'c'] }), 2), + { ok: false, reason: 'capped' }, + ); + }); +}); diff --git a/modules/embeddings/src/utils/mutationEvents.ts b/modules/embeddings/src/utils/mutationEvents.ts new file mode 100644 index 000000000..169f8f5e9 --- /dev/null +++ b/modules/embeddings/src/utils/mutationEvents.ts @@ -0,0 +1,127 @@ +const META_FIELDS = new Set(['_id', 'id', 'createdAt', 'updatedAt', '__v']); + +export interface ParsedMutationEvent { + payload: unknown; + ids: string[]; +} + +export const MAX_MUTATION_EVENT_BYTES = 256 * 1024; +export const MAX_MUTATION_EVENT_IDS = 500; + +export type MutationEventParseResult = + | { ok: true; event: ParsedMutationEvent } + | { ok: false; reason: 'malformed' | 'capped' }; + +export function parseBoundedMutationEvent( + message: string, + maxIds: number = MAX_MUTATION_EVENT_IDS, +): MutationEventParseResult { + if (typeof message !== 'string' || !message.length) { + return { ok: false, reason: 'malformed' }; + } + if (message.length > MAX_MUTATION_EVENT_BYTES) { + return { ok: false, reason: 'capped' }; + } + let payload: unknown; + try { + payload = JSON.parse(message); + } catch { + return { ok: false, reason: 'malformed' }; + } + const ids = uniqueIds(extractDocumentIds(payload)); + if (ids.length > maxIds) { + return { ok: false, reason: 'capped' }; + } + return { + ok: true, + event: { payload, ids }, + }; +} + +export function extractDocumentIds(payload: unknown): string[] { + if (payload == null) return []; + if (isMongoBulkWriteResult(payload)) return []; + if (isIdEnvelope(payload)) { + return uniqueIds((payload.ids as unknown[]).map(extractId).filter(isPresent)); + } + const docs = normalizeDocs(payload); + return uniqueIds(docs.map(extractId).filter(isPresent)); +} + +export function embeddingOwnedFields(configs: Array<{ targetField: string }>): string[] { + return configs.flatMap(config => [ + config.targetField, + `${config.targetField}SourceHash`, + ]); +} + +export function isEmbeddingOwnedMutation( + payload: unknown, + ownedFields: string[] = [], +): boolean { + const docs = normalizeDocs(payload); + if (!docs.length) return false; + const owned = new Set(ownedFields); + return docs.every(doc => { + const keys = Object.keys(doc).filter(key => !META_FIELDS.has(key)); + if (!keys.length) return false; + return keys.every( + key => owned.has(key) || key.endsWith('SourceHash') || isNumericVector(doc[key]), + ); + }); +} + +function normalizeDocs(payload: unknown): Record[] { + if (payload == null || isMongoBulkWriteResult(payload)) return []; + if (Array.isArray(payload)) { + return payload.filter(isRecord); + } + if (isRecord(payload)) return [payload]; + return []; +} + +function isMongoBulkWriteResult(payload: unknown): boolean { + if (!isRecord(payload) || payload._id !== undefined) return false; + return ( + typeof payload.matchedCount === 'number' || + typeof payload.modifiedCount === 'number' || + typeof payload.nModified === 'number' || + typeof payload.n === 'number' + ); +} + +function isIdEnvelope(payload: unknown): payload is { ids: unknown[] } { + return isRecord(payload) && Array.isArray(payload.ids); +} + +function extractId(value: unknown): string | undefined { + if (value == null) return undefined; + if (typeof value === 'string' || typeof value === 'number') { + const id = String(value); + return id.length ? id : undefined; + } + if (!isRecord(value)) return undefined; + if (value._id !== undefined) return extractId(value._id); + if (value.id !== undefined) return extractId(value.id); + return undefined; +} + +function isNumericVector(value: unknown): boolean { + return ( + Array.isArray(value) && + value.length > 0 && + value.every(item => typeof item === 'number') + ); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function uniqueIds(ids: string[]): string[] { + return [...new Set(ids)]; +} + +function isPresent(value: string | undefined): value is string { + return Boolean(value); +} diff --git a/modules/embeddings/src/utils/operationalStatus.test.ts b/modules/embeddings/src/utils/operationalStatus.test.ts new file mode 100644 index 000000000..850c6ce4a --- /dev/null +++ b/modules/embeddings/src/utils/operationalStatus.test.ts @@ -0,0 +1,238 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError, VectorIndexStatus } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertConfigActivation, + assertSearchExecutable, + capabilityWarnings, + grpcErrorFromSearchGate, + isEmbeddingsReady, + providerReadinessWarnings, + SearchGateError, +} from './operationalStatus.js'; +import { mapBackfillRun, mapEmbeddingConfig, parseSearchHits } from './protoMappers.js'; + +describe('embeddings operational warnings and search gates', () => { + it('warns on unsupported capabilities and missing provider settings without leaking secrets', () => { + const warnings = [ + ...capabilityWarnings({ + supported: false, + storage: false, + indexing: false, + search: false, + provider: 'unsupported', + reason: 'mysql is storage-only', + }), + ...providerReadinessWarnings({ + endpoint: '', + apiKey: 'sk-secret', + models: [], + }), + ]; + assert.equal( + warnings.some(warning => /mysql is storage-only/.test(warning)), + true, + ); + assert.equal( + warnings.some(warning => /model catalogue is empty/.test(warning)), + true, + ); + assert.equal(warnings.join(' ').includes('sk-secret'), false); + assert.deepEqual( + providerReadinessWarnings({ + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-secret', + models: [{ name: 'small', dimensions: 1536 }], + defaultModel: 'missing', + }), + ["Provider default model 'missing' is not in the catalogue"], + ); + assert.equal( + isEmbeddingsReady({ + moduleEnabled: false, + warnings: ['Embeddings module is disabled'], + }), + false, + ); + }); + + it('blocks semantic search when the vector index is not queryable', () => { + assert.throws( + () => + assertSearchExecutable({ + capabilities: { + supported: true, + search: true, + provider: 'mongodb', + }, + config: { + _id: 'cfg1', + enabled: true, + targetField: 'embedding', + dimensions: 3, + similarity: 'cosine', + }, + indexes: [ + { + field: 'embedding', + status: VectorIndexStatus.Pending, + queryable: false, + dimensions: 3, + similarity: 'cosine', + }, + ], + }), + (err: unknown) => + err instanceof SearchGateError && err.reason === 'index_not_queryable', + ); + const mapped = grpcErrorFromSearchGate( + new SearchGateError('index_not_queryable', 'index pending', 'pending'), + ); + assert.equal(mapped.code, status.FAILED_PRECONDITION); + }); + + it('allows search when the live index method is empty or omitted', () => { + assert.doesNotThrow(() => + assertSearchExecutable({ + capabilities: { + supported: true, + search: true, + provider: 'mongodb', + }, + config: { + _id: 'cfg1', + enabled: true, + targetField: 'embedding', + dimensions: 3, + similarity: 'cosine', + }, + indexes: [ + { + field: 'embedding', + name: 'embedding_vector', + status: VectorIndexStatus.Ready, + queryable: true, + dimensions: 3, + similarity: 'cosine', + }, + ], + }), + ); + assert.doesNotThrow(() => + assertSearchExecutable({ + capabilities: { + supported: true, + search: true, + provider: 'mongodb', + }, + config: { + _id: 'cfg1', + enabled: true, + targetField: 'embedding', + dimensions: 3, + similarity: 'cosine', + }, + indexes: [ + { + field: 'embedding', + name: 'embedding_vector', + status: VectorIndexStatus.Ready, + queryable: true, + dimensions: 3, + similarity: 'cosine', + method: '', + }, + ], + }), + ); + }); + + it('denies search and activation when a queryable live index does not match the config contract', () => { + const mismatched = { + field: 'embedding', + name: 'embedding_vector', + status: VectorIndexStatus.Ready, + queryable: true, + dimensions: 3, + similarity: 'cosine', + }; + const config = { + _id: 'cfg1', + enabled: true, + schemaName: 'Article', + targetField: 'embedding', + dimensions: 3, + similarity: 'euclidean', + }; + assert.throws( + () => + assertSearchExecutable({ + capabilities: { + supported: true, + search: true, + provider: 'mongodb', + }, + config, + indexes: [mismatched], + }), + (err: unknown) => + err instanceof SearchGateError && err.reason === 'index_not_queryable', + ); + assert.throws( + () => + assertConfigActivation({ + moduleEnabled: true, + capabilities: { + supported: true, + storage: true, + provider: 'mongodb', + }, + config, + indexes: [mismatched], + }), + (err: unknown) => + err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not queryable/.test(err.message), + ); + }); +}); + +describe('typed proto mappers', () => { + it('maps persisted configs and backfills without JSON-string envelopes', () => { + const config = mapEmbeddingConfig({ + _id: 'cfg1', + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + provider: 'openai-compatible', + modelName: 'text-embedding-3-small', + dimensions: 3, + similarity: 'cosine', + enabled: true, + createdAt: new Date('2026-01-02T00:00:00.000Z'), + }); + assert.equal(config.model, 'text-embedding-3-small'); + assert.equal(config.createdAt, '2026-01-02T00:00:00.000Z'); + const run = mapBackfillRun({ + _id: 'run1', + schemaName: 'Article', + state: 'queued', + batchSize: 10, + onlyMissing: true, + scannedCount: 0, + queuedCount: 0, + processedCount: 0, + failedCount: 0, + filter: { status: 'draft' }, + }); + assert.equal(run.onlyMissing, true); + assert.equal(run.filter, '{"status":"draft"}'); + const hits = parseSearchHits<{ _id: string }>([ + { document: '{"_id":"doc1"}', score: 0.5, metric: 'cosine', provider: 'mongodb' }, + ]); + assert.equal(hits[0].document._id, 'doc1'); + assert.equal(hits[0].score, 0.5); + }); +}); diff --git a/modules/embeddings/src/utils/operationalStatus.ts b/modules/embeddings/src/utils/operationalStatus.ts new file mode 100644 index 000000000..d7e5260eb --- /dev/null +++ b/modules/embeddings/src/utils/operationalStatus.ts @@ -0,0 +1,222 @@ +import { GrpcError, VectorCapabilities } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertBackfillExecutable, + BackfillGateError, + embeddingIndexContractFromConfig, + findTargetVectorIndex, + isEmbeddingVectorIndexQueryable, + type BackfillConfigGate, + type VectorIndexGate, +} from './backfillGates.js'; +import { providerCatalogueIssues } from './providerConfig.js'; +import type { QueueJobCounts } from '../controllers/queue.controller.js'; + +export const SEARCH_GATE_REASONS = [ + 'vector_unsupported', + 'vector_search_unavailable', + 'config_not_found', + 'config_disabled', + 'index_not_queryable', +] as const; + +export type SearchGateReason = (typeof SEARCH_GATE_REASONS)[number]; + +export class SearchGateError extends Error { + readonly code = 'SEARCH_GATE' as const; + + constructor( + readonly reason: SearchGateReason, + message: string, + readonly indexStatus?: string, + ) { + super(message); + this.name = 'SearchGateError'; + } +} + +export function assertSearchExecutable(args: { + capabilities?: Pick; + config?: BackfillConfigGate | null; + indexes?: readonly VectorIndexGate[]; +}): void { + const capabilities = args.capabilities; + if (!capabilities?.supported) { + throw new SearchGateError( + 'vector_unsupported', + capabilities?.reason ?? + 'Database does not support Conduit vector search; use MongoDB Atlas Vector Search or Postgres pgvector', + ); + } + if (!capabilities.search) { + throw new SearchGateError( + 'vector_search_unavailable', + capabilities.reason ?? + `Vector search is unavailable for provider '${capabilities.provider}'`, + ); + } + if (!args.config) { + throw new SearchGateError( + 'config_not_found', + 'No enabled embedding config found for semantic search', + ); + } + if (args.config.enabled === false) { + throw new SearchGateError( + 'config_disabled', + `Embedding config '${args.config._id ?? 'unknown'}' is disabled`, + ); + } + const targetField = args.config.targetField; + if (typeof targetField !== 'string' || !targetField.length) { + throw new SearchGateError( + 'config_not_found', + 'Embedding config is missing a target vector field', + ); + } + const contract = embeddingIndexContractFromConfig(args.config); + const index = findTargetVectorIndex(args.indexes ?? [], targetField, contract); + if (contract && isEmbeddingVectorIndexQueryable(index)) return; + const indexStatus = contract ? (index?.status ?? 'missing') : 'missing'; + throw new SearchGateError( + 'index_not_queryable', + `Vector index for field '${targetField}' is not queryable (status: ${indexStatus}). ` + + 'Wait until the index is ready before running semantic search.', + indexStatus, + ); +} + +export function grpcErrorFromSearchGate(err: SearchGateError): GrpcError { + switch (err.reason) { + case 'vector_unsupported': + case 'vector_search_unavailable': + case 'config_not_found': + case 'config_disabled': + case 'index_not_queryable': + return new GrpcError(status.FAILED_PRECONDITION, err.message); + default: { + const unexpected: never = err.reason; + return new GrpcError(status.INTERNAL, String(unexpected)); + } + } +} + +export function capabilityWarnings( + capabilities?: Pick< + VectorCapabilities, + 'supported' | 'storage' | 'indexing' | 'search' | 'provider' | 'reason' + >, +): string[] { + if (!capabilities) { + return ['Vector capabilities are unavailable']; + } + const warnings: string[] = []; + if (!capabilities.supported) { + warnings.push( + capabilities.reason ?? + `Vector storage is unsupported for provider '${capabilities.provider}'`, + ); + return warnings; + } + if (!capabilities.storage) { + warnings.push( + capabilities.reason ?? + `Vector storage is unavailable for provider '${capabilities.provider}'`, + ); + } + if (!capabilities.indexing) { + warnings.push( + capabilities.reason ?? + `Vector indexing is unavailable for provider '${capabilities.provider}'`, + ); + } + if (!capabilities.search) { + warnings.push( + capabilities.reason ?? + `Vector search is unavailable for provider '${capabilities.provider}'`, + ); + } + return warnings; +} + +export function indexReadinessWarnings( + configs: readonly BackfillConfigGate[], + indexes: readonly VectorIndexGate[], +): string[] { + const warnings: string[] = []; + for (const config of configs) { + const targetField = config.targetField; + if (!targetField) continue; + const contract = embeddingIndexContractFromConfig(config); + const index = findTargetVectorIndex(indexes, targetField, contract); + if (contract && isEmbeddingVectorIndexQueryable(index)) continue; + warnings.push( + `Vector index for field '${targetField}' is not queryable (status: ${ + contract ? (index?.status ?? 'missing') : 'missing' + })`, + ); + } + return warnings; +} + +export function providerReadinessWarnings(provider?: { + endpoint?: string; + apiKey?: string; + models?: Array<{ name?: string; dimensions?: number }>; + defaultModel?: string; +}): string[] { + const warnings: string[] = []; + if (!provider?.endpoint) { + warnings.push('Embedding provider endpoint is not configured'); + } + if (!provider?.apiKey) { + warnings.push('Embedding provider API key is not configured'); + } + warnings.push(...providerCatalogueIssues(provider)); + return warnings; +} + +export function assertConfigActivation(args: { + moduleEnabled: boolean; + capabilities?: Pick< + VectorCapabilities, + 'supported' | 'storage' | 'provider' | 'reason' + >; + config?: BackfillConfigGate | null; + indexes?: readonly VectorIndexGate[]; +}): void { + try { + assertBackfillExecutable({ + moduleEnabled: args.moduleEnabled, + capabilities: args.capabilities, + config: args.config, + indexes: args.indexes, + }); + } catch (err) { + if (err instanceof BackfillGateError && err.reason === 'index_not_queryable') { + throw new GrpcError( + status.FAILED_PRECONDITION, + `${err.message} Save the config with enabled=false until the index is ready.`, + ); + } + throw err; + } +} + +export function emptyQueueCounts(): QueueJobCounts { + return { + waiting: 0, + active: 0, + completed: 0, + failed: 0, + delayed: 0, + paused: 0, + }; +} + +export function isEmbeddingsReady(args: { + moduleEnabled: boolean; + warnings: string[]; +}): boolean { + return args.moduleEnabled && args.warnings.length === 0; +} diff --git a/modules/embeddings/src/utils/processEmbedding.test.ts b/modules/embeddings/src/utils/processEmbedding.test.ts new file mode 100644 index 000000000..5bfd7280d --- /dev/null +++ b/modules/embeddings/src/utils/processEmbedding.test.ts @@ -0,0 +1,107 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { hashedEmbeddingSource } from './configChange.js'; +import { + buildEmbeddingDocumentSelect, + generateEmbeddingsForDocument, +} from './processEmbedding.js'; + +const config = { + sourceFields: ['title', 'body'], + targetField: 'embedding', + dimensions: 2, + provider: 'openai-compatible', + modelName: 'test', +}; + +function hash(input: string) { + return createHash('sha256').update(input).digest('hex'); +} + +describe('embedding generation loop safety', () => { + it('explicitly selects source fields and the hidden source hash', () => { + assert.equal( + buildEmbeddingDocumentSelect([config]), + '+title +body +embeddingSourceHash', + ); + }); + + it('skips provider calls when the source hash already matches', async () => { + const sourceHash = hashedEmbeddingSource(hash, 'Hello\nWorld', config); + let embedCalls = 0; + let updates = 0; + const result = await generateEmbeddingsForDocument({ + doc: { + _id: 'a', + title: 'Hello', + body: 'World', + embeddingSourceHash: sourceHash, + }, + configs: [config], + hashInput: hash, + embed: async () => { + embedCalls += 1; + return [1, 2]; + }, + update: async () => { + updates += 1; + }, + }); + assert.deepEqual(result, { generated: 0, skipped: 1 }); + assert.equal(embedCalls, 0); + assert.equal(updates, 0); + }); + + it('performs one write with event suppression and does not loop on the write-back', async () => { + const sourceHash = hashedEmbeddingSource(hash, 'Hello\nWorld', config); + let embedCalls = 0; + const updates: Array<{ fields: Record; options: unknown }> = []; + const doc: Record = { _id: 'a', title: 'Hello', body: 'World' }; + + const run = () => + generateEmbeddingsForDocument({ + doc, + configs: [config], + hashInput: hash, + embed: async () => { + embedCalls += 1; + return [1, 2]; + }, + update: async (fields, options) => { + updates.push({ fields, options }); + }, + }); + + const first = await run(); + const second = await run(); + + assert.deepEqual(first, { generated: 1, skipped: 0 }); + assert.deepEqual(second, { generated: 0, skipped: 1 }); + assert.equal(embedCalls, 1); + assert.equal(updates.length, 1); + assert.deepEqual(updates[0].options, { suppressEvent: true }); + assert.equal(updates[0].fields.embeddingSourceHash, sourceHash); + }); + + it('does not skip when stored hashes were computed without the material config fingerprint', async () => { + let embedCalls = 0; + const result = await generateEmbeddingsForDocument({ + doc: { + _id: 'a', + title: 'Hello', + body: 'World', + embeddingSourceHash: hash('Hello\nWorld'), + }, + configs: [{ ...config, modelName: 'text-embedding-3-large' }], + hashInput: hash, + embed: async () => { + embedCalls += 1; + return [1, 2]; + }, + update: async () => undefined, + }); + assert.deepEqual(result, { generated: 1, skipped: 0 }); + assert.equal(embedCalls, 1); + }); +}); diff --git a/modules/embeddings/src/utils/processEmbedding.ts b/modules/embeddings/src/utils/processEmbedding.ts new file mode 100644 index 000000000..103c3ced1 --- /dev/null +++ b/modules/embeddings/src/utils/processEmbedding.ts @@ -0,0 +1,85 @@ +import { hashedEmbeddingSource, sourceHashField } from './configChange.js'; + +export { sourceHashField }; + +export interface EmbeddingConfigLike { + sourceFields: string[]; + targetField: string; + dimensions: number; + provider: string; + modelName?: string; + similarity?: string; +} + +export interface EmbeddingGenerationResult { + generated: number; + skipped: number; +} + +export function buildEmbeddingDocumentSelect( + configs: Array<{ sourceFields: string[]; targetField: string }>, +): string { + const fields = new Set(); + for (const config of configs) { + for (const field of config.sourceFields) { + fields.add(`+${field}`); + } + fields.add(`+${sourceHashField(config.targetField)}`); + } + return [...fields].join(' '); +} + +export function buildEmbeddingInput( + doc: Record, + sourceFields: string[], +): string { + return sourceFields.map(field => doc[field] ?? '').join('\n'); +} + +export function shouldSkipEmbedding( + doc: Record, + targetField: string, + sourceHash: string, +): boolean { + return doc[sourceHashField(targetField)] === sourceHash; +} + +export async function generateEmbeddingsForDocument(args: { + doc: Record; + configs: EmbeddingConfigLike[]; + hashInput: (input: string) => string; + embed: (input: string, config: EmbeddingConfigLike) => Promise; + update: ( + fields: Record, + options: { suppressEvent: true }, + ) => Promise; +}): Promise { + let generated = 0; + let skipped = 0; + for (const config of args.configs) { + const input = buildEmbeddingInput(args.doc, config.sourceFields); + const sourceHash = hashedEmbeddingSource(args.hashInput, input, config); + if (shouldSkipEmbedding(args.doc, config.targetField, sourceHash)) { + skipped += 1; + continue; + } + const vector = await args.embed(input, config); + if (vector.length !== config.dimensions) { + throw new Error( + `Embedding provider returned ${vector.length} dimensions; expected ${config.dimensions}`, + ); + } + const hashField = sourceHashField(config.targetField); + await args.update( + { + [config.targetField]: vector, + [hashField]: sourceHash, + }, + { suppressEvent: true }, + ); + args.doc[hashField] = sourceHash; + args.doc[config.targetField] = vector; + generated += 1; + } + return { generated, skipped }; +} diff --git a/modules/embeddings/src/utils/productionSecurity.test.ts b/modules/embeddings/src/utils/productionSecurity.test.ts new file mode 100644 index 000000000..640e44efe --- /dev/null +++ b/modules/embeddings/src/utils/productionSecurity.test.ts @@ -0,0 +1,22 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { assertGrpcKeyRequirement } from './productionSecurity.js'; + +describe('production GRPC_KEY requirement', () => { + it('requires GRPC_KEY when NODE_ENV is production', () => { + assert.throws( + () => assertGrpcKeyRequirement({ NODE_ENV: 'production' }), + err => err instanceof GrpcError && err.code === status.FAILED_PRECONDITION, + ); + assert.doesNotThrow(() => + assertGrpcKeyRequirement({ NODE_ENV: 'production', GRPC_KEY: 'secret' }), + ); + }); + + it('does not require GRPC_KEY outside production', () => { + assert.doesNotThrow(() => assertGrpcKeyRequirement({ NODE_ENV: 'test' })); + assert.doesNotThrow(() => assertGrpcKeyRequirement({ NODE_ENV: 'development' })); + }); +}); diff --git a/modules/embeddings/src/utils/productionSecurity.ts b/modules/embeddings/src/utils/productionSecurity.ts new file mode 100644 index 000000000..310ec54a1 --- /dev/null +++ b/modules/embeddings/src/utils/productionSecurity.ts @@ -0,0 +1,21 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; + +export function assertGrpcKeyRequirement(env: NodeJS.ProcessEnv = process.env): void { + const required = env.NODE_ENV === 'production'; + if (required && !env.GRPC_KEY) { + throw new GrpcError( + status.FAILED_PRECONDITION, + 'GRPC_KEY is required for embeddings in production', + ); + } +} + +export function callerModuleName(metadata?: { + get(key: string): Array; +}): string | undefined { + const value = metadata?.get('module-name')?.[0]; + if (typeof value === 'string' && value.length > 0) return value; + if (Buffer.isBuffer(value) && value.length > 0) return value.toString(); + return undefined; +} diff --git a/modules/embeddings/src/utils/protoMappers.ts b/modules/embeddings/src/utils/protoMappers.ts new file mode 100644 index 000000000..ed3a70c50 --- /dev/null +++ b/modules/embeddings/src/utils/protoMappers.ts @@ -0,0 +1,173 @@ +import type { VectorCapabilities, VectorSearchResult } from '@conduitplatform/grpc-sdk'; +import type { QueueJobCounts } from '../controllers/queue.controller.js'; +import type { PersistedBackfillRun } from './backfillExecution.js'; + +export function toIsoString(value?: Date | string | null): string | undefined { + if (value == null) return undefined; + if (value instanceof Date) { + return Number.isNaN(value.getTime()) ? undefined : value.toISOString(); + } + if (typeof value === 'string' && value.length > 0) return value; + return undefined; +} + +export function parseJsonObject( + value: string | undefined, + field: string, +): Record | undefined { + if (value == null || value === '') return undefined; + try { + const parsed = JSON.parse(value); + if (parsed == null) return undefined; + if (typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('not-object'); + } + return parsed as Record; + } catch { + throw new Error(field); + } +} + +export interface MappedEmbeddingConfig { + id: string; + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + model: string; + dimensions: number; + similarity: string; + enabled: boolean; + createdAt?: string; + updatedAt?: string; +} + +export interface MappedBackfillRun { + id: string; + schemaName: string; + configId?: string; + state: string; + cursor?: string; + batchSize: number; + onlyMissing: boolean; + filter?: string; + scannedCount: number; + queuedCount: number; + processedCount: number; + failedCount: number; + startedAt?: string; + finishedAt?: string; + error?: string; + createdAt?: string; + updatedAt?: string; +} + +export function mapEmbeddingConfig(doc: { + _id: string; + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + modelName?: string; + dimensions: number; + similarity: string; + enabled?: boolean; + createdAt?: Date | string; + updatedAt?: Date | string; +}): MappedEmbeddingConfig { + return { + id: doc._id, + schemaName: doc.schemaName, + sourceFields: [...doc.sourceFields], + targetField: doc.targetField, + provider: doc.provider, + model: doc.modelName ?? '', + dimensions: doc.dimensions, + similarity: doc.similarity, + enabled: doc.enabled !== false, + createdAt: toIsoString(doc.createdAt), + updatedAt: toIsoString(doc.updatedAt), + }; +} + +export function mapBackfillRun( + run: PersistedBackfillRun & { createdAt?: Date | string; updatedAt?: Date | string }, +): MappedBackfillRun { + return { + id: run._id, + schemaName: run.schemaName, + ...(run.configId ? { configId: run.configId } : {}), + state: run.state, + ...(run.cursor ? { cursor: run.cursor } : {}), + batchSize: run.batchSize, + onlyMissing: run.onlyMissing === true, + ...(run.filter ? { filter: JSON.stringify(run.filter) } : {}), + scannedCount: run.scannedCount, + queuedCount: run.queuedCount, + processedCount: run.processedCount, + failedCount: run.failedCount, + startedAt: toIsoString(run.startedAt), + finishedAt: toIsoString(run.finishedAt), + ...(run.error ? { error: run.error } : {}), + createdAt: toIsoString(run.createdAt), + updatedAt: toIsoString(run.updatedAt), + }; +} + +export function mapQueueCounts(counts: QueueJobCounts) { + return { + waiting: counts.waiting, + active: counts.active, + completed: counts.completed, + failed: counts.failed, + delayed: counts.delayed, + paused: counts.paused, + }; +} + +export function mapCapabilities(capabilities: VectorCapabilities) { + return { + supported: capabilities.supported, + storage: capabilities.storage, + indexing: capabilities.indexing, + search: capabilities.search, + provider: capabilities.provider, + ...(capabilities.reason ? { reason: capabilities.reason } : {}), + }; +} + +export function mapSearchHits(results: VectorSearchResult[]): Array<{ + document: string; + score: number; + distance?: number; + metric?: string; + provider?: string; +}> { + return results.map(result => ({ + document: JSON.stringify(result.document ?? {}), + score: result.score, + ...(result.distance != null ? { distance: result.distance } : {}), + ...(result.metric ? { metric: result.metric } : {}), + ...(result.provider ? { provider: result.provider } : {}), + })); +} + +export function parseSearchHits( + hits: Array<{ + document: string; + score: number; + distance?: number; + metric?: string; + provider?: string; + }>, +): VectorSearchResult[] { + return hits.map(hit => ({ + document: JSON.parse(hit.document) as T, + score: hit.score, + ...(hit.distance != null ? { distance: hit.distance } : {}), + ...(hit.metric ? { metric: hit.metric as VectorSearchResult['metric'] } : {}), + ...(hit.provider + ? { provider: hit.provider as VectorSearchResult['provider'] } + : {}), + })); +} diff --git a/modules/embeddings/src/utils/providerConfig.test.ts b/modules/embeddings/src/utils/providerConfig.test.ts new file mode 100644 index 000000000..d1903b04e --- /dev/null +++ b/modules/embeddings/src/utils/providerConfig.test.ts @@ -0,0 +1,182 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + findProviderModel, + normalizeEmbeddingsConfig, + normalizeProviderSettings, + resolveProviderModelName, + assertConfiguredProvider, + resolveCatalogueDimensions, + resolveCatalogueModel, +} from './providerConfig.js'; + +describe('provider model catalogue', () => { + it('migrates a singular model setting into a one-item catalogue', () => { + const migrated = normalizeProviderSettings({ + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-test', + model: 'text-embedding-3-small', + dimensions: 1536, + allowedHosts: ['api.openai.com'], + }); + assert.deepEqual(migrated, { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-test', + models: [{ name: 'text-embedding-3-small', dimensions: 1536 }], + defaultModel: 'text-embedding-3-small', + }); + }); + + it('keeps an existing catalogue and drops legacy host and model fields', () => { + const normalized = normalizeProviderSettings({ + endpoint: 'https://api.openai.com/v1/embeddings', + model: 'legacy-model', + dimensions: 512, + allowedHosts: ['api.openai.com'], + models: [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + { name: 'text-embedding-3-large', dimensions: 3072 }, + ], + defaultModel: 'text-embedding-3-large', + }); + assert.deepEqual(normalized.models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + { name: 'text-embedding-3-large', dimensions: 3072 }, + ]); + assert.equal(normalized.defaultModel, 'text-embedding-3-large'); + assert.equal('model' in normalized, false); + assert.equal('allowedHosts' in normalized, false); + assert.equal('dimensions' in normalized, false); + }); + + it('rejects duplicate names, empty names, and non-positive dimensions', () => { + assert.throws( + () => + normalizeProviderSettings({ + models: [ + { name: 'small', dimensions: 1536 }, + { name: 'small', dimensions: 768 }, + ], + }), + err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, + ); + assert.throws( + () => normalizeProviderSettings({ models: [{ name: ' ', dimensions: 1536 }] }), + err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, + ); + assert.throws( + () => normalizeProviderSettings({ models: [{ name: 'small', dimensions: 0 }] }), + err => err instanceof GrpcError && err.code === status.INVALID_ARGUMENT, + ); + assert.throws( + () => normalizeProviderSettings({ model: 'small' }), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /positive integer/.test(err.message), + ); + }); + + it('requires defaultModel to exist in the catalogue when set', () => { + assert.throws( + () => + normalizeProviderSettings({ + models: [{ name: 'small', dimensions: 1536 }], + defaultModel: 'missing', + }), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /not in the catalogue/.test(err.message), + ); + const normalized = normalizeProviderSettings({ + models: [ + { name: 'small', dimensions: 1536 }, + { name: 'large', dimensions: 3072 }, + ], + }); + assert.equal(normalized.defaultModel, undefined); + assert.equal(resolveProviderModelName(normalized, 'large'), 'large'); + assert.equal(resolveProviderModelName(normalized), 'small'); + assert.equal(findProviderModel(normalized, 'large')?.dimensions, 3072); + assert.equal(findProviderModel(normalized, 'missing'), undefined); + const preferred = normalizeProviderSettings({ + models: [ + { name: 'small', dimensions: 1536 }, + { name: 'large', dimensions: 3072 }, + ], + defaultModel: 'large', + }); + assert.equal(resolveProviderModelName(preferred), 'large'); + assert.equal(resolveCatalogueModel(preferred).name, 'large'); + assert.equal(resolveCatalogueDimensions(resolveCatalogueModel(preferred)), 3072); + }); + + it('resolves configured providers and catalogue dimensions, rejecting mismatches', () => { + const providers = { + 'openai-compatible': { + models: [ + { name: 'small', dimensions: 1536 }, + { name: 'large', dimensions: 3072 }, + ], + defaultModel: 'small', + }, + }; + assert.equal( + assertConfiguredProvider(providers, 'openai-compatible').name, + 'openai-compatible', + ); + assert.throws( + () => assertConfiguredProvider(providers, 'missing'), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /not a configured provider/.test(err.message), + ); + const openai = providers['openai-compatible']; + const [small, large] = openai.models; + assert.equal(resolveCatalogueModel(openai).name, 'small'); + assert.equal(resolveCatalogueDimensions(large, 0), 3072); + assert.throws( + () => resolveCatalogueDimensions(small, 768), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /do not match catalogue dimensions/.test(err.message), + ); + }); + + it('allows an empty catalogue and does not throw on incomplete legacy reads', () => { + assert.deepEqual(normalizeProviderSettings({ endpoint: 'https://api.example/v1' }), { + endpoint: 'https://api.example/v1', + models: [], + }); + assert.deepEqual(normalizeProviderSettings({ model: 'small' }, { strict: false }), { + models: [], + }); + }); + + it('strips requireGrpcKey and returns catalogue-shaped providers', () => { + const normalized = normalizeEmbeddingsConfig({ + enabled: true, + security: { + requireGrpcKey: true, + sourceFieldAllowlist: [], + }, + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + model: 'text-embedding-3-small', + dimensions: 1536, + }, + }, + }); + assert.equal('requireGrpcKey' in normalized.security, false); + assert.deepEqual( + normalizeProviderSettings(normalized.providers['openai-compatible']).models, + [{ name: 'text-embedding-3-small', dimensions: 1536 }], + ); + }); +}); diff --git a/modules/embeddings/src/utils/providerConfig.ts b/modules/embeddings/src/utils/providerConfig.ts new file mode 100644 index 000000000..21cf19e29 --- /dev/null +++ b/modules/embeddings/src/utils/providerConfig.ts @@ -0,0 +1,271 @@ +import { GrpcError } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import type { + EmbeddingProviderModel, + EmbeddingProviderSettings, +} from '../config/index.js'; + +export type { EmbeddingProviderModel, EmbeddingProviderSettings }; + +type CatalogueOptions = { strict?: boolean }; + +function invalidProviderConfig(message: string): GrpcError { + return new GrpcError(status.INVALID_ARGUMENT, message); +} + +function isStrict(options?: CatalogueOptions): boolean { + return options?.strict !== false; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function trimName(value: unknown): string { + return typeof value === 'string' ? value.trim() : ''; +} + +function parseDimensions(value: unknown): number | undefined { + if (typeof value !== 'number' || !Number.isInteger(value) || value <= 0) { + return undefined; + } + return value; +} + +function parseModelEntry(value: unknown, index: number): EmbeddingProviderModel { + if (!isRecord(value)) { + throw invalidProviderConfig(`Provider model at index ${index} is invalid`); + } + const name = trimName(value.name); + if (!name) { + throw invalidProviderConfig( + `Provider model at index ${index} must have a non-empty name`, + ); + } + const dimensions = parseDimensions(value.dimensions); + if (dimensions == null) { + throw invalidProviderConfig( + `Provider model '${name}' dimensions must be a positive integer`, + ); + } + return { name, dimensions }; +} + +function parseModelList( + values: unknown[], + options?: CatalogueOptions, +): EmbeddingProviderModel[] { + const models: EmbeddingProviderModel[] = []; + const names = new Set(); + const strict = isStrict(options); + for (const [index, value] of values.entries()) { + const parsed = strict ? parseModelEntry(value, index) : optionalModelEntry(value); + if (!parsed) continue; + if (names.has(parsed.name)) { + if (strict) { + throw invalidProviderConfig(`Provider model '${parsed.name}' is duplicated`); + } + continue; + } + names.add(parsed.name); + models.push(parsed); + } + return models; +} + +function optionalModelEntry(value: unknown): EmbeddingProviderModel | undefined { + if (!isRecord(value)) return undefined; + const name = trimName(value.name); + const dimensions = parseDimensions(value.dimensions); + if (!name || dimensions == null) return undefined; + return { name, dimensions }; +} + +function migrateLegacyModels( + raw: Record, + options?: CatalogueOptions, +): EmbeddingProviderModel[] | undefined { + if (Array.isArray(raw.models) && raw.models.length > 0) return undefined; + const name = trimName(raw.model); + if (!name) return []; + const dimensions = parseDimensions(raw.dimensions); + if (dimensions == null) { + if (!isStrict(options)) return []; + throw invalidProviderConfig( + `Provider model '${name}' dimensions must be a positive integer`, + ); + } + return [{ name, dimensions }]; +} + +export function providerCatalogueIssues(provider?: { + models?: Array<{ name?: string; dimensions?: number }>; + defaultModel?: string; +}): string[] { + const issues: string[] = []; + const models = provider?.models; + if (models != null && !Array.isArray(models)) { + return ['Embedding provider model catalogue is invalid']; + } + const list = models ?? []; + const names = new Set(); + for (const [index, model] of list.entries()) { + const name = trimName(model?.name); + const dimensions = parseDimensions(model?.dimensions); + if (!name) { + issues.push(`Provider model at index ${index} must have a non-empty name`); + continue; + } + if (dimensions == null) { + issues.push(`Provider model '${name}' dimensions must be a positive integer`); + } + if (names.has(name)) { + issues.push(`Provider model '${name}' is duplicated`); + } + names.add(name); + } + const defaultModel = trimName(provider?.defaultModel); + if (defaultModel && !names.has(defaultModel)) { + issues.push(`Provider default model '${defaultModel}' is not in the catalogue`); + } + if (!list.length) { + issues.push('Embedding provider model catalogue is empty'); + } + return issues; +} + +function resolveDefaultModelName( + models: EmbeddingProviderModel[], + requested: unknown, + migratedFromSingular: boolean, +): string { + const configured = trimName(requested); + if (configured) { + return models.some(model => model.name === configured) ? configured : ''; + } + if (migratedFromSingular) return models[0]?.name ?? ''; + return ''; +} + +export function normalizeProviderSettings( + raw: unknown, + options?: CatalogueOptions, +): EmbeddingProviderSettings { + const source = isRecord(raw) ? raw : {}; + const migrated = migrateLegacyModels(source, options); + const migratedFromSingular = migrated != null && migrated.length > 0; + const models = + migrated ?? + (Array.isArray(source.models) ? parseModelList(source.models, options) : []); + const names = new Set(models.map(model => model.name)); + const configuredDefault = trimName(source.defaultModel); + if (configuredDefault && !names.has(configuredDefault) && isStrict(options)) { + throw invalidProviderConfig( + `Provider default model '${configuredDefault}' is not in the catalogue`, + ); + } + const defaultModel = resolveDefaultModelName( + models, + source.defaultModel, + migratedFromSingular, + ); + return { + ...(typeof source.endpoint === 'string' ? { endpoint: source.endpoint } : {}), + ...(typeof source.apiKey === 'string' ? { apiKey: source.apiKey } : {}), + models, + ...(defaultModel ? { defaultModel } : {}), + }; +} + +export function findProviderModel( + provider: EmbeddingProviderSettings | undefined, + name?: string, +): EmbeddingProviderModel | undefined { + const selected = trimName(name); + if (!selected) return undefined; + return provider?.models?.find(model => model.name === selected); +} + +export function resolveProviderModelName( + provider: EmbeddingProviderSettings | undefined, + selected?: string, +): string { + const trimmed = trimName(selected); + if (trimmed) return trimmed; + const defaultModel = trimName(provider?.defaultModel); + if (defaultModel) return defaultModel; + return provider?.models?.[0]?.name ?? ''; +} + +export function assertConfiguredProvider( + providers: Record | undefined, + requested?: string, +): { name: string; settings: EmbeddingProviderSettings } { + const name = trimName(requested); + if (!name) { + throw invalidProviderConfig('Embedding provider is required'); + } + const settings = providers?.[name]; + if (!settings) { + throw invalidProviderConfig( + `Embedding provider '${name}' is not a configured provider`, + ); + } + return { name, settings }; +} + +export function resolveCatalogueModel( + provider: EmbeddingProviderSettings | undefined, + requested?: string, +): EmbeddingProviderModel { + const name = resolveProviderModelName(provider, requested); + const model = findProviderModel(provider, name); + if (!model) { + throw invalidProviderConfig( + name + ? `Model '${name}' is not in the catalogue for this provider` + : 'Provider model catalogue has no selectable model', + ); + } + return model; +} + +export function resolveCatalogueDimensions( + model: EmbeddingProviderModel, + requested?: number, +): number { + if (requested == null || requested === 0) { + return model.dimensions; + } + if (!Number.isInteger(requested) || requested <= 0) { + throw invalidProviderConfig('dimensions must be a positive integer'); + } + if (requested !== model.dimensions) { + throw invalidProviderConfig( + `Requested dimensions ${requested} do not match catalogue dimensions ${model.dimensions} for model '${model.name}'`, + ); + } + return model.dimensions; +} + +export function normalizeEmbeddingsConfig< + T extends { + providers?: Record; + security?: Record; + }, +>(config: T, options?: CatalogueOptions): T { + const next = { ...config }; + if (isRecord(next.security)) { + const security = { ...next.security }; + delete security.requireGrpcKey; + next.security = security as T['security']; + } + if (isRecord(next.providers)) { + const providers: Record = {}; + for (const [key, value] of Object.entries(next.providers)) { + providers[key] = normalizeProviderSettings(value, options); + } + next.providers = providers as T['providers']; + } + return next; +} diff --git a/modules/embeddings/src/utils/redactConfig.test.ts b/modules/embeddings/src/utils/redactConfig.test.ts new file mode 100644 index 000000000..19f5ac57d --- /dev/null +++ b/modules/embeddings/src/utils/redactConfig.test.ts @@ -0,0 +1,147 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + redactSensitiveConfig, + restoreRedactedSecrets, +} from '@conduitplatform/module-tools'; +import AppConfigSchema from '../config/index.js'; +import { normalizeEmbeddingsConfig } from './providerConfig.js'; +import { redactSecretText } from './redactConfig.js'; + +function assertNoLegacySettingsSurface(config: unknown) { + const serialized = JSON.stringify(config); + assert.doesNotMatch(serialized, /requireGrpcKey/); + assert.doesNotMatch(serialized, /allowedHosts/); + const providers = (config as { providers?: Record> }) + .providers; + for (const provider of Object.values(providers ?? {})) { + assert.equal('model' in provider, false); + assert.equal('allowedHosts' in provider, false); + assert.equal('dimensions' in provider, false); + } + const security = (config as { security?: Record }).security; + if (security) { + assert.equal('requireGrpcKey' in security, false); + } +} + +describe('provider secret redaction', () => { + it('redacts API keys from config objects and error text', () => { + assert.equal( + redactSensitiveConfig({ endpoint: 'https://api.openai.com', apiKey: 'sk-secret' }) + .apiKey, + '[REDACTED]', + ); + assert.match( + redactSecretText('Embedding failed Bearer sk-secret apiKey=sk-secret'), + /\[REDACTED\]/, + ); + assert.doesNotMatch( + redactSecretText('Embedding failed Bearer sk-secret apiKey=sk-secret'), + /sk-secret/, + ); + }); + + it('redacts convict-sensitive and well-known secret keys', () => { + const redacted = redactSensitiveConfig( + { + enabled: true, + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com', + apiKey: 'sk-live', + models: [{ name: 'text-embedding-3-small', dimensions: 1536 }], + }, + }, + }, + { + providers: { + 'openai-compatible': { + apiKey: { format: 'String', default: '', sensitive: true }, + endpoint: { format: 'String', default: '' }, + models: { format: Array, default: [] }, + defaultModel: { format: 'String', default: '' }, + }, + }, + }, + ); + assert.equal(redacted.providers['openai-compatible'].apiKey, '[REDACTED]'); + assert.equal( + redacted.providers['openai-compatible'].endpoint, + 'https://api.openai.com', + ); + assert.deepEqual(redacted.providers['openai-compatible'].models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + }); + + it('keeps Core Admin GET/PATCH catalogue shape while redacting keys', () => { + const legacy = { + enabled: true, + defaultProvider: 'openai-compatible', + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-live', + model: 'text-embedding-3-small', + dimensions: 1536, + allowedHosts: ['api.openai.com'], + }, + }, + security: { + requireGrpcKey: true, + sourceFieldAllowlist: ['summary'], + }, + }; + const patched = normalizeEmbeddingsConfig(legacy); + const getResponse = redactSensitiveConfig(patched, AppConfigSchema); + const monoResponse = redactSensitiveConfig(patched); + for (const redacted of [getResponse, monoResponse]) { + const provider = redacted.providers['openai-compatible'] as { + apiKey?: string; + endpoint?: string; + models?: Array<{ name: string; dimensions: number }>; + defaultModel?: string; + }; + assert.equal(provider.apiKey, '[REDACTED]'); + assert.doesNotMatch(JSON.stringify(redacted), /sk-live/); + assert.equal(provider.endpoint, 'https://api.openai.com/v1/embeddings'); + assert.deepEqual(provider.models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + assert.equal(provider.defaultModel, 'text-embedding-3-small'); + assertNoLegacySettingsSurface(redacted); + } + const emptyKey = redactSensitiveConfig( + normalizeEmbeddingsConfig({ + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: '', + models: [{ name: 'text-embedding-3-small', dimensions: 1536 }], + }, + }, + }), + AppConfigSchema, + ); + assert.equal(emptyKey.providers['openai-compatible'].apiKey, ''); + }); + + it('restores redacted API keys from the currently stored config', () => { + const current = normalizeEmbeddingsConfig({ + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-live', + models: [{ name: 'text-embedding-3-small', dimensions: 1536 }], + }, + }, + }); + const incoming = redactSensitiveConfig(current, AppConfigSchema); + const restored = restoreRedactedSecrets(incoming, current, AppConfigSchema); + assert.equal(restored.providers['openai-compatible'].apiKey, 'sk-live'); + assert.deepEqual(restored.providers['openai-compatible'].models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + }); +}); diff --git a/modules/embeddings/src/utils/redactConfig.ts b/modules/embeddings/src/utils/redactConfig.ts new file mode 100644 index 000000000..131d54295 --- /dev/null +++ b/modules/embeddings/src/utils/redactConfig.ts @@ -0,0 +1,10 @@ +export function redactSecretText(value: string): string { + return value + .replace(/Bearer\s+\S+/gi, 'Bearer [REDACTED]') + .replace(/(apiKey|api_key|password|secret)\s*[:=]\s*\S+/gi, '$1:[REDACTED]'); +} + +export function sanitizeErrorMessage(err: unknown): string { + const message = err instanceof Error ? err.message : String(err); + return redactSecretText(message); +} diff --git a/modules/embeddings/src/utils/schemaPolicy.test.ts b/modules/embeddings/src/utils/schemaPolicy.test.ts new file mode 100644 index 000000000..34106d9eb --- /dev/null +++ b/modules/embeddings/src/utils/schemaPolicy.test.ts @@ -0,0 +1,383 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError, TYPE, VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { + assertCanManageEmbeddingConfig, + assertEmbeddingExtensionAvailability, + assertEmbeddingTargetSchema, + assertSchemaCanReceiveEmbeddings, + assertSemanticSearchAccess, + assertSourceFields, + DATABASE_SYSTEM_SCHEMA_NAMES, + isDeniedEmbeddingSchema, + PLATFORM_INTERNAL_SCHEMA_NAMES, + resolveAdminOperatorContext, + resolveSourceFieldAllowlist, +} from './schemaPolicy.js'; + +describe('embedding schema and source policies', () => { + const extendableEnabled = { + conduit: { cms: { enabled: true }, permissions: { extendable: true } }, + }; + + it('denies system, auth-secret, and embeddings-owned schemas', () => { + assert.equal(isDeniedEmbeddingSchema({ name: 'EmbeddingConfig' }), true); + assert.equal(isDeniedEmbeddingSchema({ name: 'BackfillRun' }), true); + assert.equal( + isDeniedEmbeddingSchema({ name: 'CustomOps', ownerModule: 'embeddings' }), + true, + ); + assert.equal(isDeniedEmbeddingSchema({ name: '_DeclaredSchema' }), true); + assert.equal( + isDeniedEmbeddingSchema({ name: 'Views', ownerModule: 'database' }), + true, + ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'CustomEndpoints', ownerModule: 'database' }), + true, + ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'AccessToken', ownerModule: 'authentication' }), + true, + ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'TwoFactorSecret', ownerModule: 'authentication' }), + true, + ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'Article', ownerModule: 'cms-app' }), + false, + ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'User', ownerModule: 'authentication' }), + false, + ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'Team', ownerModule: 'authentication' }), + false, + ); + assert.equal( + isDeniedEmbeddingSchema({ name: 'File', ownerModule: 'storage' }), + false, + ); + assert.throws( + () => assertEmbeddingTargetSchema({ name: 'RefreshToken' }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + }); + + it('denies platform internal schemas even when they are enabled and extendable', () => { + assert.deepEqual( + [...DATABASE_SYSTEM_SCHEMA_NAMES], + [ + '_DeclaredSchema', + 'MigratedSchemas', + 'CustomEndpoints', + '_PendingSchemas', + 'Views', + ], + ); + for (const name of ['Admin', 'AdminMiddleware', 'AppMiddleware', 'Client'] as const) { + assert.equal(PLATFORM_INTERNAL_SCHEMA_NAMES.has(name), true); + } + const internals = [ + { name: 'Admin', ownerModule: 'core' }, + { name: 'AdminMiddleware', ownerModule: 'core' }, + { name: 'AppMiddleware', ownerModule: 'router' }, + { name: 'Client', ownerModule: 'router' }, + { name: 'Config', ownerModule: 'core' }, + { name: 'CustomEndpoints', ownerModule: 'database' }, + { name: 'ActorIndex', ownerModule: 'authorization' }, + ]; + for (const schema of internals) { + assert.equal(isDeniedEmbeddingSchema(schema), true); + assert.throws( + () => + assertSchemaCanReceiveEmbeddings({ + ...schema, + modelOptions: extendableEnabled, + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + } + assert.equal( + isDeniedEmbeddingSchema({ name: 'FutureCoreDoc', ownerModule: 'core' }), + true, + ); + }); + + it('allows enabled and extendable owner-controlled business schemas', () => { + assert.doesNotThrow(() => + assertSchemaCanReceiveEmbeddings({ + name: 'Article', + ownerModule: 'cms-app', + modelOptions: extendableEnabled, + }), + ); + assert.doesNotThrow(() => + assertSchemaCanReceiveEmbeddings({ + name: 'User', + ownerModule: 'authentication', + modelOptions: { conduit: { permissions: { extendable: true } } }, + }), + ); + assert.doesNotThrow(() => + assertSchemaCanReceiveEmbeddings({ + name: 'Team', + ownerModule: 'authentication', + modelOptions: extendableEnabled, + }), + ); + }); + + it('restricts config and backfill to the schema owner or platform admin', () => { + assert.doesNotThrow(() => + assertCanManageEmbeddingConfig({ + callerModule: 'cms-app', + ownerModule: 'cms-app', + schemaName: 'Article', + }), + ); + assert.doesNotThrow(() => + assertCanManageEmbeddingConfig({ + callerModule: 'database', + ownerModule: 'cms-app', + schemaName: 'Article', + }), + ); + assert.throws( + () => + assertCanManageEmbeddingConfig({ + callerModule: 'chat', + ownerModule: 'cms-app', + schemaName: 'Article', + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + }); + + it('rejects hidden, non-string, and sensitive source fields unless allowlisted', () => { + const schemaFields = { + title: { type: TYPE.String }, + body: { type: TYPE.String }, + password: { type: TYPE.String }, + token: { type: TYPE.String, select: false }, + views: { type: TYPE.Number }, + notes: { type: TYPE.String, select: false }, + }; + assert.doesNotThrow(() => + assertSourceFields({ + sourceFields: ['title', 'body'], + schemaFields, + }), + ); + assert.throws( + () => assertSourceFields({ sourceFields: ['password'], schemaFields }), + /sensitive/, + ); + assert.throws( + () => assertSourceFields({ sourceFields: ['token'], schemaFields }), + /hidden|sensitive/, + ); + assert.throws( + () => assertSourceFields({ sourceFields: ['views'], schemaFields }), + /string-like/, + ); + assert.throws( + () => assertSourceFields({ sourceFields: ['notes'], schemaFields }), + /hidden/, + ); + assert.doesNotThrow(() => + assertSourceFields({ + sourceFields: ['notes'], + schemaFields, + allowlist: ['notes'], + }), + ); + }); + + it('honors caller-supplied sourceFieldAllowlist only for platform-admin context', () => { + assert.deepEqual( + resolveSourceFieldAllowlist({ + operatorAllowlist: ['summary'], + requestAllowlist: ['password', 'notes'], + platformAdmin: false, + }), + ['summary'], + ); + assert.deepEqual( + resolveSourceFieldAllowlist({ + operatorAllowlist: ['summary'], + requestAllowlist: ['password', 'notes'], + platformAdmin: true, + }), + ['summary', 'password', 'notes'], + ); + }); + + it('requires subject, scope, or a verified admin operator for semantic search', () => { + assert.throws( + () => assertSemanticSearchAccess({}), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + assert.doesNotThrow(() => assertSemanticSearchAccess({ userId: 'u1' })); + assert.doesNotThrow(() => + assertSemanticSearchAccess({ + adminOperator: resolveAdminOperatorContext({ + requested: true, + callerModule: 'core', + }), + }), + ); + assert.throws( + () => + resolveAdminOperatorContext({ + requested: true, + callerModule: 'chat', + }), + err => err instanceof GrpcError && err.code === status.PERMISSION_DENIED, + ); + }); + + it('requires an enabled schema and extendable permissions, not CMS enablement alone', () => { + assert.doesNotThrow(() => + assertSchemaCanReceiveEmbeddings({ + name: 'Article', + modelOptions: { + conduit: { cms: { enabled: true }, permissions: { extendable: true } }, + }, + }), + ); + assert.doesNotThrow(() => + assertSchemaCanReceiveEmbeddings({ + name: 'User', + modelOptions: { conduit: { permissions: { extendable: true } } }, + }), + ); + assert.throws( + () => + assertSchemaCanReceiveEmbeddings({ + name: 'Article', + modelOptions: { + conduit: { cms: { enabled: true }, permissions: { extendable: false } }, + }, + }), + err => + err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not extendable/.test(err.message), + ); + assert.throws( + () => + assertSchemaCanReceiveEmbeddings({ + name: 'Article', + modelOptions: { + conduit: { cms: { enabled: true } }, + }, + }), + err => + err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not extendable/.test(err.message), + ); + assert.throws( + () => + assertSchemaCanReceiveEmbeddings({ + name: 'Article', + modelOptions: { + conduit: { cms: { enabled: false }, permissions: { extendable: true } }, + }, + }), + err => + err instanceof GrpcError && + err.code === status.FAILED_PRECONDITION && + /not enabled/.test(err.message), + ); + }); + + it('rejects incompatible vector and hash collisions and allows compatible embeddings extensions', () => { + const proposed = { + schemaName: 'Article', + targetField: 'embedding', + dimensions: 3, + similarity: VectorSimilarity.Cosine, + compiledFields: { + title: { type: TYPE.String }, + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, + embeddingSourceHash: { type: TYPE.String, required: false, select: false }, + }, + extensions: [ + { + ownerModule: 'embeddings', + fields: { + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + select: false, + }, + embeddingSourceHash: { type: TYPE.String, required: false, select: false }, + }, + }, + ], + }; + assert.doesNotThrow(() => assertEmbeddingExtensionAvailability(proposed)); + assert.throws( + () => + assertEmbeddingExtensionAvailability({ + ...proposed, + compiledFields: { + title: { type: TYPE.String }, + embedding: { type: TYPE.String }, + }, + extensions: undefined, + }), + err => err instanceof GrpcError && err.code === status.ALREADY_EXISTS, + ); + assert.throws( + () => + assertEmbeddingExtensionAvailability({ + ...proposed, + dimensions: 8, + }), + err => err instanceof GrpcError && err.code === status.ALREADY_EXISTS, + ); + assert.throws( + () => + assertEmbeddingExtensionAvailability({ + ...proposed, + extensions: [ + { + ownerModule: 'chat', + fields: { + embedding: { + type: TYPE.Vector, + dimensions: 3, + similarity: VectorSimilarity.Cosine, + }, + }, + }, + ], + }), + err => err instanceof GrpcError && err.code === status.ALREADY_EXISTS, + ); + assert.throws( + () => + assertEmbeddingExtensionAvailability({ + ...proposed, + compiledFields: { + title: { type: TYPE.String }, + embeddingSourceHash: { type: TYPE.Number }, + }, + extensions: undefined, + }), + err => err instanceof GrpcError && err.code === status.ALREADY_EXISTS, + ); + }); +}); diff --git a/modules/embeddings/src/utils/schemaPolicy.ts b/modules/embeddings/src/utils/schemaPolicy.ts new file mode 100644 index 000000000..e215c9df9 --- /dev/null +++ b/modules/embeddings/src/utils/schemaPolicy.ts @@ -0,0 +1,381 @@ +import { GrpcError, TYPE } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { BACKFILL_RUN_SCHEMA } from './backfillRun.js'; +import { sourceHashField } from './configChange.js'; + +export const EMBEDDING_CONFIG_SCHEMA = 'EmbeddingConfig'; +export { BACKFILL_RUN_SCHEMA }; +export const EMBEDDINGS_OWNER_MODULE = 'embeddings'; +export const EMBEDDING_OWNED_SCHEMA_NAMES = new Set([ + EMBEDDING_CONFIG_SCHEMA, + BACKFILL_RUN_SCHEMA, +]); +export const CONFIG_OPERATOR_MODULES = ['database', 'core'] as const; +export const SEARCH_OPERATOR_MODULES = ['database', 'core', 'embeddings'] as const; + +export const AUTH_SECRET_SCHEMA_NAMES = new Set([ + 'AccessToken', + 'RefreshToken', + 'Token', + 'TwoFactorSecret', + 'TwoFactorBackUpCodes', + 'BiometricToken', + 'AdminTwoFactorSecret', + 'AdminApiToken', +]); + +export const DATABASE_SYSTEM_SCHEMA_NAMES = new Set([ + '_DeclaredSchema', + 'MigratedSchemas', + 'CustomEndpoints', + '_PendingSchemas', + 'Views', +]); + +export const PLATFORM_INTERNAL_SCHEMA_NAMES = new Set([ + 'Admin', + 'AdminMiddleware', + 'AdminApiToken', + 'AdminTwoFactorSecret', + 'Config', + 'Client', + 'AppMiddleware', + 'ResourceDefinition', + 'Relationship', + 'ObjectIndex', + 'Permission', + 'ActorIndex', +]); + +export const INTERNAL_OWNER_MODULES = new Set(['core', 'router', 'authorization']); + +export const SYSTEM_SCHEMA_NAMES = new Set([ + ...DATABASE_SYSTEM_SCHEMA_NAMES, + ...PLATFORM_INTERNAL_SCHEMA_NAMES, +]); + +const SENSITIVE_FIELD_NAME = + /(password|secret|token|credential|apikey|api_key|private[_-]?key|authorization|refresh[_-]?token|access[_-]?token)/i; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function isSensitiveFieldName(field: string): boolean { + return SENSITIVE_FIELD_NAME.test(field); +} + +export function isStringLikeField(field: unknown): boolean { + if (field === TYPE.String || field === 'String') return true; + if (Array.isArray(field) && field.length === 1) return isStringLikeField(field[0]); + if (!isRecord(field)) return false; + if (field.type === TYPE.String || field.type === 'String') return true; + return Array.isArray(field.type) && isStringLikeField(field.type); +} + +export function isHiddenField(field: unknown): boolean { + return isRecord(field) && field.select === false; +} + +/** + * Explicit denylist for embeddings sources. Platform internals (Database system + * schemas, core/router/authorization) are denied even when extendable. + * Owner-controlled business schemas (including authentication User/Team) are + * not denied by ownerModule alone. + */ +export function isDeniedEmbeddingSchema(schema: { + name: string; + ownerModule?: string; +}): boolean { + if (!schema.name) return true; + if (schema.ownerModule === EMBEDDINGS_OWNER_MODULE) return true; + if (EMBEDDING_OWNED_SCHEMA_NAMES.has(schema.name)) return true; + if (schema.name.startsWith('_')) return true; + if (SYSTEM_SCHEMA_NAMES.has(schema.name)) return true; + if (schema.ownerModule && INTERNAL_OWNER_MODULES.has(schema.ownerModule)) return true; + return AUTH_SECRET_SCHEMA_NAMES.has(schema.name); +} + +export interface EmbeddingSchemaOptions { + conduit?: { + cms?: { enabled?: boolean }; + permissions?: { extendable?: boolean }; + authorization?: { enabled?: boolean }; + }; +} + +function isSchemaExtendable(modelOptions?: EmbeddingSchemaOptions): boolean { + return modelOptions?.conduit?.permissions?.extendable === true; +} + +function isEmbeddingSchemaEnabled(modelOptions?: EmbeddingSchemaOptions): boolean { + if (modelOptions?.conduit?.cms == null) return true; + return modelOptions.conduit.cms.enabled === true; +} + +export function assertEmbeddingTargetSchema(schema: { + name: string; + ownerModule?: string; +}): void { + if (!isDeniedEmbeddingSchema(schema)) return; + throw new GrpcError( + status.PERMISSION_DENIED, + `Schema '${schema.name}' cannot be used as an embedding source`, + ); +} + +export function assertSchemaCanReceiveEmbeddings(schema: { + name: string; + ownerModule?: string; + modelOptions?: EmbeddingSchemaOptions; +}): void { + assertEmbeddingTargetSchema(schema); + if (!isEmbeddingSchemaEnabled(schema.modelOptions)) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Schema '${schema.name}' is not enabled`, + ); + } + if (!isSchemaExtendable(schema.modelOptions)) { + throw new GrpcError( + status.FAILED_PRECONDITION, + `Schema '${schema.name}' is not extendable`, + ); + } +} + +export function canManageEmbeddingConfig(args: { + callerModule?: string; + ownerModule?: string; + operatorModules?: readonly string[]; +}): boolean { + if (!args.callerModule) return false; + if (args.ownerModule && args.callerModule === args.ownerModule) return true; + const operators = args.operatorModules ?? CONFIG_OPERATOR_MODULES; + return operators.includes(args.callerModule); +} + +export function assertCanManageEmbeddingConfig(args: { + callerModule?: string; + ownerModule?: string; + schemaName: string; +}): void { + if (canManageEmbeddingConfig(args)) return; + throw new GrpcError( + status.PERMISSION_DENIED, + `Module '${args.callerModule ?? 'unknown'}' is not allowed to manage embeddings for '${args.schemaName}'`, + ); +} + +export function resolveAdminOperatorContext(args: { + requested?: boolean; + callerModule?: string; + operatorModules?: readonly string[]; +}): boolean { + if (!args.requested) return false; + const operators = args.operatorModules ?? SEARCH_OPERATOR_MODULES; + if (!args.callerModule || !operators.includes(args.callerModule)) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Admin operator context is not allowed for this caller', + ); + } + return true; +} + +export function assertSemanticSearchAccess(args: { + userId?: string; + scope?: string; + adminOperator?: boolean; +}): void { + if (args.userId || args.scope || args.adminOperator) return; + throw new GrpcError( + status.PERMISSION_DENIED, + 'Semantic search requires a subject, scope, or admin operator context', + ); +} + +export function normalizeSourceFieldAllowlist(fields?: string[]): string[] { + return [ + ...new Set( + (fields ?? []).filter(field => typeof field === 'string' && field.length > 0), + ), + ]; +} + +export function resolveSourceFieldAllowlist(args: { + operatorAllowlist?: string[]; + requestAllowlist?: string[]; + platformAdmin?: boolean; +}): string[] { + const operatorAllowlist = normalizeSourceFieldAllowlist(args.operatorAllowlist); + if (!args.platformAdmin) return operatorAllowlist; + return [ + ...new Set([ + ...operatorAllowlist, + ...normalizeSourceFieldAllowlist(args.requestAllowlist), + ]), + ]; +} + +export function assertSourceFields(args: { + sourceFields: string[]; + schemaFields: Record; + allowlist?: string[]; +}): void { + const allowlist = new Set(args.allowlist ?? []); + for (const field of args.sourceFields) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(field)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Invalid source field name '${field}'`, + ); + } + const definition = args.schemaFields[field]; + if (definition === undefined) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Source field '${field}' does not exist on the target schema`, + ); + } + if (!isStringLikeField(definition)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Source field '${field}' must be string-like`, + ); + } + const allowed = allowlist.has(field); + if (!allowed && isHiddenField(definition)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Source field '${field}' is hidden and cannot be embedded`, + ); + } + if (!allowed && isSensitiveFieldName(field)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Source field '${field}' looks sensitive and cannot be embedded`, + ); + } + } +} + +export interface SchemaExtensionInfo { + ownerModule: string; + fields: Record; +} + +export interface EmbeddingExtensionField { + type: string; + dimensions?: number; + similarity?: string; + required?: boolean; + select?: boolean; +} + +export function assertEmbeddingExtensionAvailability(args: { + schemaName: string; + targetField: string; + dimensions: number; + similarity: string; + baseFields?: Record; + compiledFields: Record; + extensions?: SchemaExtensionInfo[]; +}): void { + const hashField = sourceHashField(args.targetField); + const proposed: Record = { + [args.targetField]: { + type: TYPE.Vector, + dimensions: args.dimensions, + similarity: args.similarity, + select: false, + }, + [hashField]: { + type: TYPE.String, + required: false, + select: false, + }, + }; + for (const [name, definition] of Object.entries(proposed)) { + assertExtensionFieldAvailable({ + schemaName: args.schemaName, + fieldName: name, + proposed: definition, + baseFields: args.baseFields, + compiledFields: args.compiledFields, + extensions: args.extensions ?? [], + }); + } +} + +function fieldOwner( + fieldName: string, + extensions: SchemaExtensionInfo[], +): SchemaExtensionInfo | undefined { + return extensions.find(extension => fieldName in extension.fields); +} + +function assertExtensionFieldAvailable(args: { + schemaName: string; + fieldName: string; + proposed: EmbeddingExtensionField; + baseFields?: Record; + compiledFields: Record; + extensions: SchemaExtensionInfo[]; +}): void { + const owned = fieldOwner(args.fieldName, args.extensions); + if (owned) { + if (owned.ownerModule !== EMBEDDINGS_OWNER_MODULE) { + throw extensionCollision(args.schemaName, args.fieldName); + } + if (!isCompatibleEmbeddingField(owned.fields[args.fieldName], args.proposed)) { + throw extensionCollision(args.schemaName, args.fieldName); + } + return; + } + if (args.baseFields && args.fieldName in args.baseFields) { + throw extensionCollision(args.schemaName, args.fieldName); + } + if (!(args.fieldName in args.compiledFields)) return; + if (!isCompatibleEmbeddingField(args.compiledFields[args.fieldName], args.proposed)) { + throw extensionCollision(args.schemaName, args.fieldName); + } +} + +function extensionCollision(schemaName: string, fieldName: string): GrpcError { + return new GrpcError( + status.ALREADY_EXISTS, + `Field '${fieldName}' already exists on schema '${schemaName}' and is not a compatible embeddings extension`, + ); +} + +function fieldType(field: unknown): string | undefined { + if (typeof field === 'string') return field; + if (!isRecord(field)) return undefined; + if (typeof field.type === 'string') return field.type; + return undefined; +} + +function isCompatibleEmbeddingField( + existing: unknown, + proposed: EmbeddingExtensionField, +): boolean { + const type = fieldType(existing); + if (type !== proposed.type) return false; + if (proposed.type === TYPE.Vector) { + if (!isRecord(existing)) return false; + if (existing.dimensions !== proposed.dimensions) return false; + if ( + typeof existing.similarity === 'string' && + existing.similarity !== proposed.similarity + ) { + return false; + } + return true; + } + if (proposed.type === TYPE.String) { + if (isRecord(existing) && existing.required === true) return false; + return isStringLikeField(existing); + } + return false; +} diff --git a/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts b/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts new file mode 100644 index 000000000..04e4a9678 --- /dev/null +++ b/modules/embeddings/src/utils/validateEmbeddingConfig.test.ts @@ -0,0 +1,125 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { GrpcError, VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { validateEmbeddingConfigInput } from './validateEmbeddingConfig.js'; + +describe('validateEmbeddingConfigInput', () => { + const defaults = { + provider: 'openai-compatible', + providers: { + 'openai-compatible': { + models: [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + { name: 'text-embedding-3-large', dimensions: 3072 }, + ], + defaultModel: 'text-embedding-3-small', + }, + }, + }; + const valid = { + schemaName: 'Article', + sourceFields: ['title', 'body'], + targetField: 'embedding', + dimensions: 1536, + }; + + it('accepts object-form config with a supported similarity enum', () => { + const result = validateEmbeddingConfigInput( + { + ...valid, + similarity: VectorSimilarity.DotProduct, + model: 'text-embedding-3-small', + }, + defaults, + ); + assert.equal(result.similarity, VectorSimilarity.DotProduct); + assert.equal(result.provider, 'openai-compatible'); + assert.equal(result.modelName, 'text-embedding-3-small'); + assert.equal(result.dimensions, 1536); + assert.deepEqual(result.sourceFieldAllowlist, []); + }); + + it('defaults omitted similarity to cosine and omitted model to the catalogue default', () => { + const result = validateEmbeddingConfigInput( + { schemaName: 'Article', sourceFields: ['title'], targetField: 'embedding' }, + defaults, + ); + assert.equal(result.similarity, VectorSimilarity.Cosine); + assert.equal(result.modelName, 'text-embedding-3-small'); + assert.equal(result.dimensions, 1536); + }); + + it('rejects missing identity fields', () => { + assert.throws( + () => validateEmbeddingConfigInput({ ...valid, sourceFields: [] }, defaults), + /required/, + ); + }); + + it('rejects unknown providers and models', () => { + assert.throws( + () => validateEmbeddingConfigInput({ ...valid, provider: 'missing' }, defaults), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /not a configured provider/.test(err.message), + ); + assert.throws( + () => validateEmbeddingConfigInput({ ...valid, model: 'missing' }, defaults), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /not in the catalogue/.test(err.message), + ); + }); + + it('rejects explicit dimension mismatches and ignores omitted proto dimensions', () => { + assert.throws( + () => + validateEmbeddingConfigInput( + { ...valid, model: 'text-embedding-3-small', dimensions: 768 }, + defaults, + ), + err => + err instanceof GrpcError && + err.code === status.INVALID_ARGUMENT && + /do not match catalogue dimensions/.test(err.message), + ); + const omitted = validateEmbeddingConfigInput( + { + schemaName: 'Article', + sourceFields: ['title'], + targetField: 'embedding', + model: 'text-embedding-3-large', + dimensions: 0, + }, + defaults, + ); + assert.equal(omitted.dimensions, 3072); + assert.equal(omitted.modelName, 'text-embedding-3-large'); + }); + + it('rejects unsupported similarity values', () => { + assert.throws( + () => validateEmbeddingConfigInput({ ...valid, similarity: 'manhattan' }, defaults), + /Unsupported similarity/, + ); + }); + + it('validates source fields against the schema when provided', () => { + assert.throws( + () => + validateEmbeddingConfigInput(valid, defaults, { + title: { type: 'String' }, + password: { type: 'String' }, + }), + /does not exist/, + ); + const result = validateEmbeddingConfigInput(valid, defaults, { + title: { type: 'String' }, + body: { type: 'String' }, + }); + assert.deepEqual(result.sourceFields, ['title', 'body']); + }); +}); diff --git a/modules/embeddings/src/utils/validateEmbeddingConfig.ts b/modules/embeddings/src/utils/validateEmbeddingConfig.ts new file mode 100644 index 000000000..de568f3e2 --- /dev/null +++ b/modules/embeddings/src/utils/validateEmbeddingConfig.ts @@ -0,0 +1,90 @@ +import { GrpcError, VectorSimilarity } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import type { EmbeddingProviderSettings } from '../config/index.js'; +import { + assertConfiguredProvider, + resolveCatalogueDimensions, + resolveCatalogueModel, +} from './providerConfig.js'; +import { assertSourceFields } from './schemaPolicy.js'; + +export interface EmbeddingConfigInput { + schemaName?: string; + sourceFields?: string[]; + targetField?: string; + provider?: string; + model?: string; + dimensions?: number; + similarity?: string; + sourceFieldAllowlist?: string[]; +} + +export interface ValidatedEmbeddingConfig { + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + modelName: string; + dimensions: number; + similarity: VectorSimilarity; + sourceFieldAllowlist: string[]; +} + +export interface EmbeddingConfigDefaults { + provider: string; + providers: Record; +} + +const SUPPORTED_SIMILARITY = Object.values(VectorSimilarity); + +export function validateEmbeddingConfigInput( + request: EmbeddingConfigInput, + defaults: EmbeddingConfigDefaults, + schemaFields?: Record, +): ValidatedEmbeddingConfig { + if (!request.schemaName || !request.targetField || !request.sourceFields?.length) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'schemaName, targetField, and sourceFields are required', + ); + } + if (!/^[A-Za-z_][A-Za-z0-9_]{0,127}$/.test(request.schemaName)) { + throw new GrpcError(status.INVALID_ARGUMENT, 'schemaName is invalid'); + } + if (!/^[A-Za-z_][A-Za-z0-9_]{0,127}$/.test(request.targetField)) { + throw new GrpcError(status.INVALID_ARGUMENT, 'targetField is invalid'); + } + const { name: provider, settings: providerSettings } = assertConfiguredProvider( + defaults.providers, + request.provider || defaults.provider, + ); + const model = resolveCatalogueModel(providerSettings, request.model); + const dimensions = resolveCatalogueDimensions(model, request.dimensions); + const similarity = request.similarity || VectorSimilarity.Cosine; + if (!SUPPORTED_SIMILARITY.includes(similarity as VectorSimilarity)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Unsupported similarity '${similarity}'. Supported values: ${SUPPORTED_SIMILARITY.join(', ')}`, + ); + } + const sourceFieldAllowlist = (request.sourceFieldAllowlist ?? []).filter( + field => typeof field === 'string' && field.length > 0, + ); + if (schemaFields) { + assertSourceFields({ + sourceFields: request.sourceFields, + schemaFields, + allowlist: sourceFieldAllowlist, + }); + } + return { + schemaName: request.schemaName, + sourceFields: request.sourceFields, + targetField: request.targetField, + provider, + modelName: model.name, + dimensions, + similarity: similarity as VectorSimilarity, + sourceFieldAllowlist, + }; +} diff --git a/modules/embeddings/test/deployment-contract.test.mjs b/modules/embeddings/test/deployment-contract.test.mjs new file mode 100644 index 000000000..7ab11a4a8 --- /dev/null +++ b/modules/embeddings/test/deployment-contract.test.mjs @@ -0,0 +1,157 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { test } from 'node:test'; +import { + IMAGE_TARGETS, + resolveTargets, +} from '../../../scripts/resolve-docker-targets.mjs'; + +const repo = new URL('../../..', import.meta.url); +const readRepo = relativePath => readFileSync(new URL(relativePath, repo), 'utf8'); + +const runbook = readRepo('deploy/embeddings.md'); +const moduleReadme = readRepo('modules/embeddings/README.md'); +const composeSource = readRepo('docker/docker-compose.yml'); +const standaloneCompose = readRepo('docker/docker-compose.standalone.yml'); +const dockerReadme = readRepo('deploy/docker/README.md'); +const k8sReadme = readRepo('deploy/k8s/README.md'); +const convictConfig = readRepo('modules/embeddings/src/config/index.ts'); +const workflow = readRepo('.github/workflows/embeddings-test.yml'); + +const PROFILE_ENABLEMENT_DOCS = [ + ['deploy/embeddings.md', runbook], + ['modules/embeddings/README.md', moduleReadme], + ['docker/docker-compose.yml', composeSource], + ['docker/docker-compose.standalone.yml', standaloneCompose], + ['deploy/docker/README.md', dockerReadme], +]; + +function assertProfileExamplesRequireGrpcKey(source, label) { + const lines = source.split('\n'); + let seen = 0; + for (const [index, line] of lines.entries()) { + if (!line.includes('--profile embeddings')) { + continue; + } + if (/omit `--profile embeddings`|omit --profile embeddings/.test(line)) { + continue; + } + seen += 1; + const window = lines.slice(Math.max(0, index - 3), index + 3).join('\n'); + assert.match( + window, + /export GRPC_KEY=|non-empty `GRPC_KEY`|non-empty GRPC_KEY/, + `${label}:${index + 1} profile enablement must require/export a non-empty GRPC_KEY`, + ); + } + assert.ok(seen > 0, `${label} must document --profile embeddings`); +} + +test('compose profile enablement examples require a non-empty GRPC_KEY', () => { + for (const [label, source] of PROFILE_ENABLEMENT_DOCS) { + assertProfileExamplesRequireGrpcKey(source, label); + } +}); + +test('runbook distinguishes Helm workload install.embeddings.enabled from convict enabled', () => { + assert.match(runbook, /Helm workload `install\.embeddings\.enabled`/); + assert.match(runbook, /Module convict `enabled`/); + assert.match(runbook, /install\.embeddings\.enabled=false/); + assert.match(runbook, /This is not `install\.embeddings\.enabled`/); + assert.doesNotMatch(runbook, /Helm: `install\.embeddings: false`/); + assert.match(k8sReadme, /install\.embeddings\.enabled/); + assert.match(moduleReadme, /install\.embeddings\.enabled/); +}); + +test('rollback retains vector, index, config, and Redis state', () => { + assert.match(runbook, /Rollback \*\*retains\*\*/); + assert.match(runbook, /vector fields, indexes, `EmbeddingConfig` documents/); + assert.match(runbook, /Redis\/BullMQ queue state/); + assert.match( + k8sReadme, + /retained vector\/index\/config\/\nRedis state|retained vector/, + ); +}); + +test('module settings omit gRPC-key and host allowlists in favor of a model catalogue', () => { + assert.doesNotMatch(convictConfig, /requireGrpcKey/); + assert.doesNotMatch(convictConfig, /allowedHosts/); + assert.match(convictConfig, /defaultModel/); + assert.match(convictConfig, /Operator-managed embedding models/); +}); + +test('docs stay default-off and do not claim a published embeddings image', () => { + assert.match(convictConfig, /default: false/); + assert.match(runbook, /not published/); + assert.match(moduleReadme, /not published until a compatible release/); + assert.doesNotMatch( + runbook, + /Image: `docker\.io\/conduitplatform\/embeddings:\$\{IMAGE_TAG\}`/, + ); + assert.match(composeSource, /profiles: \['embeddings'\]/); +}); + +test('compose maps container GRPC_PORT through EMBEDDINGS_GRPC_PORT', () => { + assert.match(composeSource, /GRPC_PORT: '\$\{EMBEDDINGS_GRPC_PORT:-55165\}'/); + assert.match( + composeSource, + /SERVICE_URL: 'conduit-embeddings:\$\{EMBEDDINGS_GRPC_PORT:-55165\}'/, + ); + assert.match( + composeSource, + /'\$\{EMBEDDINGS_GRPC_PORT:-55165\}:\$\{EMBEDDINGS_GRPC_PORT:-55165\}'/, + ); +}); + +test('PR CI runs compose render and target discovery', () => { + assert.match( + workflow, + /docker compose --profile mongodb --profile embeddings config --services/, + ); + assert.match(workflow, /docker compose --profile mongodb config --services/); + assert.match( + workflow, + /env -u GITHUB_OUTPUT node scripts\/resolve-docker-targets\.mjs/, + ); + assert.match(workflow, /docker\/\*\*/); + assert.match(workflow, /scripts\/resolve-docker-targets\.mjs/); +}); + +test('embeddings-only changes select embeddings and exclude standalone', () => { + const standalone = IMAGE_TARGETS.find(entry => entry.target === 'conduit-standalone'); + assert.deepEqual(standalone?.excludePaths, ['modules/embeddings/**']); + + const selected = resolveTargets({ + changedFiles: ['modules/embeddings/src/index.ts'], + forceAll: false, + }).map(entry => entry.target); + + assert.ok(selected.includes('embeddings')); + assert.ok(!selected.includes('conduit-standalone')); + assert.ok(!selected.includes('chat')); +}); + +test('other module rebuilds still select standalone', () => { + const chatSelected = resolveTargets({ + changedFiles: ['modules/chat/src/Chat.ts'], + forceAll: false, + }).map(entry => entry.target); + assert.ok(chatSelected.includes('chat')); + assert.ok(chatSelected.includes('conduit-standalone')); + assert.ok(!chatSelected.includes('embeddings')); + + const mixed = resolveTargets({ + changedFiles: ['modules/embeddings/src/index.ts', 'modules/storage/src/Storage.ts'], + forceAll: false, + }).map(entry => entry.target); + assert.ok(mixed.includes('embeddings')); + assert.ok(mixed.includes('storage')); + assert.ok(mixed.includes('conduit-standalone')); + + const shared = resolveTargets({ + changedFiles: ['docker-bake.hcl'], + forceAll: false, + }).map(entry => entry.target); + assert.ok(shared.includes('embeddings')); + assert.ok(shared.includes('conduit-standalone')); +}); diff --git a/modules/embeddings/test/embedding-contract.test.mjs b/modules/embeddings/test/embedding-contract.test.mjs new file mode 100644 index 000000000..442bdfe8a --- /dev/null +++ b/modules/embeddings/test/embedding-contract.test.mjs @@ -0,0 +1,113 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { test } from 'node:test'; + +const protoSource = readFileSync( + new URL('../src/embeddings.proto', import.meta.url), + 'utf8', +); +const readmeSource = readFileSync(new URL('../README.md', import.meta.url), 'utf8'); +const sdkSource = readFileSync( + new URL('../../../libraries/grpc-sdk/src/modules/embeddings/index.ts', import.meta.url), + 'utf8', +); +const adminRoutesSource = readFileSync( + new URL('../src/admin/routes.ts', import.meta.url), + 'utf8', +); + +test('embeddings proto exposes typed config, status, backfill, and search RPCs', () => { + assert.match( + protoSource, + /rpc upsertConfig\(UpsertConfigRequest\) returns \(UpsertConfigResponse\)/, + ); + assert.match( + protoSource, + /rpc getConfigs\(GetConfigsRequest\) returns \(GetConfigsResponse\)/, + ); + assert.match( + protoSource, + /rpc deleteConfig\(DeleteEmbeddingConfigRequest\) returns \(DeleteEmbeddingConfigResponse\)/, + ); + assert.match( + protoSource, + /rpc getCapabilities\(GetCapabilitiesRequest\) returns \(GetCapabilitiesResponse\)/, + ); + assert.match( + protoSource, + /rpc getStatus\(GetStatusRequest\) returns \(GetStatusResponse\)/, + ); + assert.match( + protoSource, + /rpc startBackfill\(StartBackfillRequest\) returns \(StartBackfillResponse\)/, + ); + assert.match( + protoSource, + /rpc getBackfill\(GetBackfillRequest\) returns \(BackfillRun\)/, + ); + assert.match( + protoSource, + /rpc listBackfills\(ListBackfillsRequest\) returns \(ListBackfillsResponse\)/, + ); + assert.match( + protoSource, + /rpc cancelBackfill\(CancelBackfillRequest\) returns \(BackfillMutationResponse\)/, + ); + assert.match( + protoSource, + /rpc resumeBackfill\(ResumeBackfillRequest\) returns \(BackfillMutationResponse\)/, + ); + assert.match( + protoSource, + /rpc semanticSearch\(SemanticSearchRequest\) returns \(SemanticSearchResponse\)/, + ); + assert.doesNotMatch( + protoSource, + /message EmbeddingConfigResponse \{\n string result = 1;/, + ); + assert.doesNotMatch(protoSource, /message EmbeddingsQueryResponse/); +}); + +test('grpc-sdk embeddings client maps typed proto messages instead of JSON-string envelopes', () => { + assert.match(sdkSource, /upsertConfig\(/); + assert.match(sdkSource, /deleteConfig\(/); + assert.match(sdkSource, /getCapabilities\(/); + assert.match(sdkSource, /getStatus\(/); + assert.match(sdkSource, /getBackfill\(/); + assert.match(sdkSource, /listBackfills\(/); + assert.match(sdkSource, /cancelBackfill\(/); + assert.match(sdkSource, /resumeBackfill\(/); + assert.doesNotMatch(sdkSource, /JSON\.parse\(res\.result\)/); + assert.match(sdkSource, /JSON\.parse\(hit\.document\)/); + assert.match( + protoSource, + /message EmbeddingConfig \{\n string id = 1;[\s\S]*string model = 6;[\s\S]*int32 dimensions = 7;/, + ); + assert.doesNotMatch(protoSource, /string modelName/); + assert.doesNotMatch(protoSource, /string _id/); + assert.match( + protoSource, + /message UpsertConfigRequest \{[\s\S]*string model = 5;[\s\S]*int32 dimensions = 6;[\s\S]*optional bool enabled = 9;/, + ); + assert.match( + protoSource, + /message GetStatusResponse \{\n bool enabled = 1;\n bool ready = 2;\n VectorCapabilities capabilities = 3;/, + ); + assert.match( + sdkSource, + /export interface EmbeddingConfigRecord \{\n id: string;[\s\S]*model: string;[\s\S]*dimensions: number;/, + ); + assert.match( + sdkSource, + /export interface EmbeddingConfigInput \{[\s\S]*model\?: string;[\s\S]*dimensions\?: number;/, + ); + assert.match(adminRoutesSource, /model: ConduitString\.Optional/); + assert.match(adminRoutesSource, /dimensions: ConduitNumber\.Optional/); +}); + +test('deployment docs describe provider configuration and rollout workflow', () => { + assert.match(readmeSource, /openai-compatible/); + assert.match(readmeSource, /backfill/); + assert.match(readmeSource, /semanticSearch/); + assert.match(readmeSource, /\/embeddings\//); +}); diff --git a/modules/embeddings/tsconfig.json b/modules/embeddings/tsconfig.json new file mode 100644 index 000000000..28f0b0240 --- /dev/null +++ b/modules/embeddings/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "NodeNext", + "resolveJsonModule": true, + "skipLibCheck": true, + "declaration": true, + "sourceMap": true, + "outDir": "./dist", + "rootDir": "./src", + "removeComments": true, + "strict": true, + "strictPropertyInitialization": false, + "moduleResolution": "NodeNext", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "src/**/*.test.ts"] +} diff --git a/modules/embeddings/tsconfig.test.json b/modules/embeddings/tsconfig.test.json new file mode 100644 index 000000000..fb6350ab5 --- /dev/null +++ b/modules/embeddings/tsconfig.test.json @@ -0,0 +1,21 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "rootDir": "./src", + "declaration": false, + "sourceMap": false, + "types": ["node"] + }, + "include": [ + "src/utils/**/*.ts", + "src/controllers/**/*.ts", + "src/providers/**/*.ts", + "src/api/**/*.ts", + "src/admin/**/*.ts", + "src/routes/**/*.ts", + "src/models/**/*.ts", + "src/config/**/*.ts" + ], + "exclude": [] +} diff --git a/modules/embeddings/tsup.config.ts b/modules/embeddings/tsup.config.ts new file mode 100644 index 000000000..c21167400 --- /dev/null +++ b/modules/embeddings/tsup.config.ts @@ -0,0 +1,6 @@ +import { createServiceTsupConfig } from '@conduitplatform/service-bundle/tsup'; +import bundleConfig from './service-bundle.config.json' with { type: 'json' }; + +export default createServiceTsupConfig({ + extraExternal: bundleConfig.extraDependencies, +}); diff --git a/packages/core/package.bundle-lock.json b/packages/core/package.bundle-lock.json index 98788f7ac..eb6a81c0d 100644 --- a/packages/core/package.bundle-lock.json +++ b/packages/core/package.bundle-lock.json @@ -16,7 +16,7 @@ "@grpc/proto-loader": "^0.8.1", "@modelcontextprotocol/sdk": "^1.29.0", "@scalar/api-reference": "^1.60.0", - "@scalar/express-api-reference": "^0.10.14", + "@scalar/express-api-reference": "^0.10.16", "@sesamecare-oss/redlock": "^1.4.0", "@socket.io/redis-streams-adapter": "^0.3.1", "abort-controller-x": "^0.5.0", @@ -58,7 +58,7 @@ "socket.io-adapter": "2.5.8", "swagger-ui-express": "5.0.1", "thirty-two": "1.0.2", - "uuid": "^14.0.1", + "uuid": "^14.0.2", "winston": "^3.19.0", "winston-loki": "^6.1.7", "zod": "^4.4.3" @@ -465,9 +465,9 @@ } }, "node_modules/@bufbuild/protobuf": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.14.0.tgz", - "integrity": "sha512-C3UGsiCwSprE2NKIIFA3hCDlpXTMCAXRZuEVp88L1GY36Y41+rYL5fryE+nOFhp4p4JPQvdV8PQ4DWgHgeTE+w==", + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.14.1.tgz", + "integrity": "sha512-agRJn3+EJDUe8AvxTx/LnHA/GErvLE62pSaSk7+MwFOtOv8eWBu/qCq2qoZjBjVZ3C2aiFJCveuSs17KMkYGOw==", "license": "(Apache-2.0 AND BSD-3-Clause)" }, "node_modules/@codemirror/autocomplete": { @@ -604,18 +604,18 @@ } }, "node_modules/@codemirror/state": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.7.1.tgz", - "integrity": "sha512-9QzNDgE4EYDnAHfrTlR2lwiPciiOymLtwKK+8yHQzCc7GXhAP9xdEbEJFy2IWB1j9UGUl9BsgMmTo/ImA02T7A==", + "version": "6.7.4", + "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.7.4.tgz", + "integrity": "sha512-QhQIVRY+xHZDxwOSFrJ1eUMapJBUID3IdeAjf7dHO7zBUzSkyooHiodnalz5MG3iHzwixKMlAAyn7244y537EA==", "license": "MIT", "dependencies": { "@marijn/find-cluster-break": "^1.0.0" } }, "node_modules/@codemirror/view": { - "version": "6.43.9", - "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.43.9.tgz", - "integrity": "sha512-sTuUzTpPMFebRhg6dawChoKKgndIwfjmJgKVxBefPElcU2NwQ6AFroupk0SFqEerQyZOGRfDNnSN8Dw/lMAsXw==", + "version": "6.43.11", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.43.11.tgz", + "integrity": "sha512-2+esucbQX6wB2JYi1eDvdCPFTA31BN8oSy6xCmk3G6CloV11yOvEjYk+gH7kLrP0MuHG94E8WDhjs5oMiu3+Wg==", "license": "MIT", "dependencies": { "@codemirror/state": "^6.7.0", @@ -625,18 +625,18 @@ } }, "node_modules/@colors/colors": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", - "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.1.tgz", + "integrity": "sha512-dTmUJzXSuayBK+hZydEaXd2mhx61qWQwkwaBBY6LyEOVx/L9aQU5ac8eFNEsd9nrD1+zb9zvDCphLSe8g1F4Qw==", "license": "MIT", "engines": { "node": ">=0.1.90" } }, "node_modules/@dabh/diagnostics": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.8.tgz", - "integrity": "sha512-R4MSXTVnuMzGD7bzHdW2ZhhdPC/igELENcq5IjEverBvq5hn1SXCWcsi6eSsdWP0/Ur+SItRRjAktmdoX/8R/Q==", + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.9.tgz", + "integrity": "sha512-R6siwR65Hm+3yfgP7o8DKhNvputQAwfoz9zTc3kyDudnomj2/BcLmD+uGQQPICjuFUp8ounPBU+jmKsocwVVAg==", "license": "MIT", "dependencies": { "@so-ric/colorspace": "^1.1.6", @@ -719,12 +719,12 @@ } }, "node_modules/@graphql-tools/merge": { - "version": "9.2.3", - "resolved": "https://registry.npmjs.org/@graphql-tools/merge/-/merge-9.2.3.tgz", - "integrity": "sha512-cKRoXqJGy2zSRBLvotQpkACbXlHAb0yuHLN0l0ypKGCuL3NnF0zofYalkBJqiBxxxpK0lr3s9uowKFHCKi1/ZQ==", + "version": "9.2.4", + "resolved": "https://registry.npmjs.org/@graphql-tools/merge/-/merge-9.2.4.tgz", + "integrity": "sha512-vV+8uWNWn0+OsqT0r22lZuoT0cTe6fBqBtpLHre2rriLjI/ZTrsOHmabLPTUOyzgFnRrS2PzFSYGL3zKL1Wj4Q==", "license": "MIT", "dependencies": { - "@graphql-tools/utils": "^12.0.0", + "@graphql-tools/utils": "^12.0.1", "tslib": "^2.4.0" }, "engines": { @@ -735,13 +735,13 @@ } }, "node_modules/@graphql-tools/schema": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/@graphql-tools/schema/-/schema-10.1.0.tgz", - "integrity": "sha512-wao48XQnfY631s3jXoNrhEHvCI8mlKXmIuWrR7F6zAdv92VuSOfHoq9P9KL2EnUMgBUnaStnByOx9Mn6RieWDg==", + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/@graphql-tools/schema/-/schema-10.1.1.tgz", + "integrity": "sha512-24jJghRxEW+SG1lbJ45Zg9HEZ8ZKS923ihbFjOicspFLpryxJkWp6gZJ7D8tCVaSAhli4x1+1wDPAfhT8mLqxg==", "license": "MIT", "dependencies": { - "@graphql-tools/merge": "^9.2.3", - "@graphql-tools/utils": "^12.0.0", + "@graphql-tools/merge": "^9.2.4", + "@graphql-tools/utils": "^12.0.1", "tslib": "^2.4.0" }, "engines": { @@ -752,9 +752,9 @@ } }, "node_modules/@graphql-tools/utils": { - "version": "12.0.0", - "resolved": "https://registry.npmjs.org/@graphql-tools/utils/-/utils-12.0.0.tgz", - "integrity": "sha512-aMdIo/l+8j4lhamWAf+MWHr+lOW8zArSBKXspqtKHgt5I2JF9LS55KDLUe/L9AiJOV/O6qJJ60hgYydbnJHbxg==", + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/@graphql-tools/utils/-/utils-12.0.1.tgz", + "integrity": "sha512-8YC6jn4xYDS6YTY6xDAgQAs/nGgvzRaIGHS8GeSfVItQzNK7Ms24CQtE3EJY+amvR+tBnhRSX0N83aGj+V/RIg==", "license": "MIT", "dependencies": { "@graphql-typed-document-node/core": "^3.1.1", @@ -816,9 +816,9 @@ "license": "Apache-2.0" }, "node_modules/@grpc/proto-loader/node_modules/protobufjs": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", - "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -878,18 +878,18 @@ } }, "node_modules/@internationalized/date": { - "version": "3.12.3", - "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.12.3.tgz", - "integrity": "sha512-fuLX+3ZKLsxI73y8b01EG/WjHb6gE6weCqlfawPO27kBWGMh9G1yH6Csv1uU7/cac9H2GHmOMt6CjmuQ1aia4Q==", + "version": "3.12.4", + "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.12.4.tgz", + "integrity": "sha512-M1dEn4c1U1HsSlaVR8upZtSqvXrTkHDfv18H01uCSJyjVLDxnBR38v/fMxecmlwXKR4i9HeZcmgQAPE6A+aGJQ==", "license": "Apache-2.0", "dependencies": { "@swc/helpers": "^0.5.0" } }, "node_modules/@internationalized/number": { - "version": "3.6.7", - "resolved": "https://registry.npmjs.org/@internationalized/number/-/number-3.6.7.tgz", - "integrity": "sha512-3ji1fcrT+FPAK86UqEhB/psHixYo6niWPJtt7+qRaYFynt/BaJG8GhAPimtWUpEiVSTq8ZM8L5psMxGquiB/Vg==", + "version": "3.6.8", + "resolved": "https://registry.npmjs.org/@internationalized/number/-/number-3.6.8.tgz", + "integrity": "sha512-8UmMFia46DUt+k97zKd9fKWXcWHR+k8ae3eYzILETuT2KbIvLyOfac7zesw+sJdRAAZ7Q9pM1Mk22aXp2LD0Ig==", "license": "Apache-2.0", "dependencies": { "@swc/helpers": "^0.5.0" @@ -902,9 +902,9 @@ "license": "MIT" }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", "license": "MIT" }, "node_modules/@js-sdsl/ordered-map": { @@ -1433,24 +1433,24 @@ } }, "node_modules/@scalar/agent-chat": { - "version": "0.12.28", - "resolved": "https://registry.npmjs.org/@scalar/agent-chat/-/agent-chat-0.12.28.tgz", - "integrity": "sha512-N1ZEIKHrOhbB6mSUuaeMbNEK8G0lhXu8aHj8RqPMjKAkm0IbZVGCEkFgRCzaqDMcdsXKqQKygzx2a5pV5gX6Ow==", + "version": "0.12.30", + "resolved": "https://registry.npmjs.org/@scalar/agent-chat/-/agent-chat-0.12.30.tgz", + "integrity": "sha512-1+09CTY/eVLg6ygdrrKyNkhWXnbHcVGrr4Jq4HYr7XogrkIsTCuHCBcpPbYRWOFJ2zRpToPFu5M3R7eLE810zA==", "license": "MIT", "dependencies": { "@ai-sdk/vue": "3.0.33", - "@scalar/api-client": "3.16.3", - "@scalar/components": "0.28.1", - "@scalar/helpers": "0.11.1", + "@scalar/api-client": "3.18.0", + "@scalar/components": "0.29.1", + "@scalar/helpers": "0.11.3", "@scalar/icons": "0.7.6", - "@scalar/json-magic": "0.13.2", + "@scalar/json-magic": "0.13.4", "@scalar/openapi-types": "0.9.5", - "@scalar/schemas": "0.8.3", - "@scalar/themes": "0.17.3", - "@scalar/types": "0.18.2", + "@scalar/schemas": "0.9.0", + "@scalar/themes": "0.17.4", + "@scalar/types": "0.19.0", "@scalar/use-toasts": "0.10.5", "@scalar/validation": "0.6.3", - "@scalar/workspace-store": "0.58.1", + "@scalar/workspace-store": "0.60.0", "@vueuse/core": "13.9.0", "ai": "6.0.33", "js-base64": "^3.9.2", @@ -1463,28 +1463,28 @@ } }, "node_modules/@scalar/api-client": { - "version": "3.16.3", - "resolved": "https://registry.npmjs.org/@scalar/api-client/-/api-client-3.16.3.tgz", - "integrity": "sha512-4F0aZtdnCWZN5EvJQTAlXPzVetniu9pQqPeukck/AUNtvsk59CdNHjd8DjAB3+SS3wv3gfNj0C+obhzjUbCNZg==", + "version": "3.18.0", + "resolved": "https://registry.npmjs.org/@scalar/api-client/-/api-client-3.18.0.tgz", + "integrity": "sha512-FlseC6xWfx0ganh4s0IAOelJ5kF+aJ07BJfjUriwKDdK8Y4oAFkRlneKQNoT/S+4mrNCXdQ/T5VPEjYFlx4Zxw==", "license": "MIT", "dependencies": { "@headlessui/tailwindcss": "^0.2.2", "@headlessui/vue": "1.7.23", - "@scalar/blocks": "0.1.14", - "@scalar/components": "0.28.1", - "@scalar/helpers": "0.11.1", + "@scalar/blocks": "0.1.16", + "@scalar/components": "0.29.1", + "@scalar/helpers": "0.11.3", "@scalar/icons": "0.7.6", - "@scalar/oas-utils": "0.19.14", + "@scalar/oas-utils": "0.19.16", "@scalar/openapi-types": "0.9.5", - "@scalar/sidebar": "0.10.1", - "@scalar/snippetz": "0.9.28", - "@scalar/themes": "0.17.3", + "@scalar/sidebar": "0.11.1", + "@scalar/snippetz": "0.9.30", + "@scalar/themes": "0.17.4", "@scalar/typebox": "^0.1.3", - "@scalar/types": "0.18.2", + "@scalar/types": "0.19.0", "@scalar/use-codemirror": "0.14.15", - "@scalar/use-hooks": "0.4.10", + "@scalar/use-hooks": "0.4.11", "@scalar/use-toasts": "0.10.5", - "@scalar/workspace-store": "0.58.1", + "@scalar/workspace-store": "0.60.0", "@vueuse/core": "13.9.0", "@vueuse/integrations": "13.9.0", "focus-trap": "^7.8.0", @@ -1497,36 +1497,36 @@ "set-cookie-parser": "3.1.0", "vue": "^3.5.40", "yaml": "^2.9.0", - "zod": "^4.3.5" + "zod": "^4.4.3" }, "engines": { "node": ">=22" } }, "node_modules/@scalar/api-reference": { - "version": "1.66.1", - "resolved": "https://registry.npmjs.org/@scalar/api-reference/-/api-reference-1.66.1.tgz", - "integrity": "sha512-+iHSJX8HPUyDGinNiLRV8qgFb+fbNwAjGDWzl8Wg/VNEcbmA40t1ZuL/WoWNqphI2QaPfjiCLtmsIhanWAed3w==", + "version": "1.68.0", + "resolved": "https://registry.npmjs.org/@scalar/api-reference/-/api-reference-1.68.0.tgz", + "integrity": "sha512-rY43w3REwCxp+rDDx/0CncZxmlzISnGTK9zZ8moq0Ij2vRHhLQCJ0/BXut9pBAupVrOZF7MoqKXcG+gISgTu5g==", "license": "MIT", "dependencies": { "@headlessui/vue": "1.7.23", - "@scalar/agent-chat": "0.12.28", - "@scalar/api-client": "3.16.3", - "@scalar/blocks": "0.1.14", + "@scalar/agent-chat": "0.12.30", + "@scalar/api-client": "3.18.0", + "@scalar/blocks": "0.1.16", "@scalar/code-highlight": "0.4.5", - "@scalar/components": "0.28.1", - "@scalar/helpers": "0.11.1", + "@scalar/components": "0.29.1", + "@scalar/helpers": "0.11.3", "@scalar/icons": "0.7.6", - "@scalar/oas-utils": "0.19.14", - "@scalar/schemas": "0.8.3", - "@scalar/sidebar": "0.10.1", - "@scalar/snippetz": "0.9.28", - "@scalar/themes": "0.17.3", - "@scalar/types": "0.18.2", - "@scalar/use-hooks": "0.4.10", + "@scalar/oas-utils": "0.19.16", + "@scalar/schemas": "0.9.0", + "@scalar/sidebar": "0.11.1", + "@scalar/snippetz": "0.9.30", + "@scalar/themes": "0.17.4", + "@scalar/types": "0.19.0", + "@scalar/use-hooks": "0.4.11", "@scalar/use-toasts": "0.10.5", "@scalar/validation": "0.6.3", - "@scalar/workspace-store": "0.58.1", + "@scalar/workspace-store": "0.60.0", "@unhead/vue": "^2.1.4", "@vueuse/core": "13.9.0", "fuse.js": "^7.5.0", @@ -1540,30 +1540,30 @@ } }, "node_modules/@scalar/asyncapi-upgrader": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/@scalar/asyncapi-upgrader/-/asyncapi-upgrader-0.1.7.tgz", - "integrity": "sha512-RU3CrNV77hWiZ9Ik0GJHDf/bEg8C0iF9Rg5b6GDJz3F9Y7ZyBJcgqq++do9GUzPTKzftfOORfQ9180s09XQp7Q==", + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/@scalar/asyncapi-upgrader/-/asyncapi-upgrader-0.1.9.tgz", + "integrity": "sha512-+kK4dp1J8GvOcTU2OCMFwj62VMP3Y0lppjQ2mdfYsczhTTP9saf12vkRzuHQ+ILkLJJXo98km5zj7pEVMdOSPA==", "license": "MIT", "dependencies": { - "@scalar/helpers": "0.11.1" + "@scalar/helpers": "0.11.3" }, "engines": { "node": ">=22" } }, "node_modules/@scalar/blocks": { - "version": "0.1.14", - "resolved": "https://registry.npmjs.org/@scalar/blocks/-/blocks-0.1.14.tgz", - "integrity": "sha512-4goVCRnz8QWCzQIuMV58GUSoj+WJNZBSGS5L6n3kc8TAKY6qiTOf+Unc5oy2DgwHCJJ6AO531hnGDAsMx7sNaQ==", + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@scalar/blocks/-/blocks-0.1.16.tgz", + "integrity": "sha512-k72Dxwj2Bh9jhJkIyTlrpja6QRnvnT5Cb+LH++BGMV70lSu22cyzJBmhtKORTfQVb671s2I7WLUBU6zirxkhsA==", "license": "MIT", "dependencies": { - "@scalar/components": "0.28.1", - "@scalar/helpers": "0.11.1", + "@scalar/components": "0.29.1", + "@scalar/helpers": "0.11.3", "@scalar/icons": "0.7.6", - "@scalar/snippetz": "0.9.28", - "@scalar/themes": "0.17.3", - "@scalar/types": "0.18.2", - "@scalar/workspace-store": "0.58.1", + "@scalar/snippetz": "0.9.30", + "@scalar/themes": "0.17.4", + "@scalar/types": "0.19.0", + "@scalar/workspace-store": "0.60.0", "@types/har-format": "^1.2.16", "js-base64": "^3.9.2", "vue": "^3.5.40" @@ -1573,13 +1573,13 @@ } }, "node_modules/@scalar/client-side-rendering": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@scalar/client-side-rendering/-/client-side-rendering-0.3.9.tgz", - "integrity": "sha512-Gg+VLhreiWHmHN0i9uatEoIxzsr0FaJoq0x+PkypmFgZFniD477DVCAnmWqx+KXlpFg0VtDanDHARIJXZ1fIeQ==", + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@scalar/client-side-rendering/-/client-side-rendering-0.4.0.tgz", + "integrity": "sha512-a8OUod1LZbNqkPv73ijRPL3MH2/3E92I5Awa9mGXtCaIVavY3JcF7+qZW4GOtffihTfuD/CxhZ+CnQhCZds2oQ==", "license": "MIT", "dependencies": { - "@scalar/schemas": "0.8.3", - "@scalar/types": "0.18.2", + "@scalar/schemas": "0.9.0", + "@scalar/types": "0.19.0", "@scalar/validation": "0.6.3" }, "engines": { @@ -1613,9 +1613,9 @@ } }, "node_modules/@scalar/components": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@scalar/components/-/components-0.28.1.tgz", - "integrity": "sha512-mYI2WwvVM6a9E/o6vrft9FKoLP7Chcit5kOc+Iz+MV6xQ63Cjx2e/dACagEXE9O5i3b4+8+8C00iy4p1TS6P0A==", + "version": "0.29.1", + "resolved": "https://registry.npmjs.org/@scalar/components/-/components-0.29.1.tgz", + "integrity": "sha512-mxlJ/3Pv1YqqaXSBU2SLAiTZSGdam3ZyoYW7CAITSoVlruDBIEfSm6a8eABpM9mWSsoNQhmscLEq2QSC0szUPA==", "license": "MIT", "dependencies": { "@floating-ui/utils": "0.2.10", @@ -1623,10 +1623,10 @@ "@headlessui/tailwindcss": "^0.2.2", "@headlessui/vue": "1.7.23", "@scalar/code-highlight": "0.4.5", - "@scalar/helpers": "0.11.1", + "@scalar/helpers": "0.11.3", "@scalar/icons": "0.7.6", - "@scalar/themes": "0.17.3", - "@scalar/use-hooks": "0.4.10", + "@scalar/themes": "0.17.4", + "@scalar/use-hooks": "0.4.11", "@vueuse/core": "13.9.0", "cva": "1.0.0-beta.4", "radix-vue": "^1.9.17", @@ -1638,21 +1638,21 @@ } }, "node_modules/@scalar/express-api-reference": { - "version": "0.10.16", - "resolved": "https://registry.npmjs.org/@scalar/express-api-reference/-/express-api-reference-0.10.16.tgz", - "integrity": "sha512-Jh7gDxGjJZjkJK3nmGVmLL7Ti6EP69Q2KEOnSCI/mtQxjRuavZt3XnYobubkoHUg9aUErvEpDcPH+ANNa4pggg==", + "version": "0.10.18", + "resolved": "https://registry.npmjs.org/@scalar/express-api-reference/-/express-api-reference-0.10.18.tgz", + "integrity": "sha512-PvDEMUNwfMnn0ak7L+rfbN6YwSY4UtM6SaOZPVjvGvHqxt7GrMV8rdax/Vu9sOLK1pL5F92K/UVmu0K9eMEcdQ==", "license": "MIT", "dependencies": { - "@scalar/client-side-rendering": "0.3.9" + "@scalar/client-side-rendering": "0.4.0" }, "engines": { "node": ">=22" } }, "node_modules/@scalar/helpers": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/@scalar/helpers/-/helpers-0.11.1.tgz", - "integrity": "sha512-Knwbe0IYqFk0PPDoOKLasqglBHfyf9/zwWWqFsSNi/AtdjM29wSZXN6p8DFid6iB5B9epYH9YiSgJ6tpD00TEw==", + "version": "0.11.3", + "resolved": "https://registry.npmjs.org/@scalar/helpers/-/helpers-0.11.3.tgz", + "integrity": "sha512-4zPzuNTXObDUtZS93xzAoK83ddHDgBGifv/vKFe6bHqEl5i+IMLTTtEHOaaOZK4dusPEXcXsU5TBSaY/I6Mi+A==", "license": "MIT", "engines": { "node": ">=22" @@ -1674,12 +1674,12 @@ } }, "node_modules/@scalar/json-magic": { - "version": "0.13.2", - "resolved": "https://registry.npmjs.org/@scalar/json-magic/-/json-magic-0.13.2.tgz", - "integrity": "sha512-T8rQw5u7+MSTDpUcd5ShX1taOUxpZMv2b/P6xsahdlv/u68VX/Bq/+uzuAf2xW8IIOy7BEP4MBggle/vMDgAXw==", + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/@scalar/json-magic/-/json-magic-0.13.4.tgz", + "integrity": "sha512-pOZdlzkgLB+/4OlIlzMToV/cr4vsvWy/MtbtJoRcNHIzDnT8sfNtv/cH8MR2pNDg/29sPpTV1HaE4SZ8b9jUOQ==", "license": "MIT", "dependencies": { - "@scalar/helpers": "0.11.1", + "@scalar/helpers": "0.11.3", "pathe": "^2.0.3", "yaml": "^2.9.0" }, @@ -1688,15 +1688,15 @@ } }, "node_modules/@scalar/oas-utils": { - "version": "0.19.14", - "resolved": "https://registry.npmjs.org/@scalar/oas-utils/-/oas-utils-0.19.14.tgz", - "integrity": "sha512-rnVIOK6+oHTc4SeXQBkEj+Kb2xB/VUJxckKGNdHnHHlsLjpN4VXhwUBldYAvV9dA/AENfeMj5ys6GubP0RyNwQ==", + "version": "0.19.16", + "resolved": "https://registry.npmjs.org/@scalar/oas-utils/-/oas-utils-0.19.16.tgz", + "integrity": "sha512-0u0/vd62lEektF9u6d7ywAYwamkrG1xTfxMf5gOkRGTVZJ7jV+J9LoSfUv+NCR3mmQpeGyKSaiD/3/Psa4OwRA==", "license": "MIT", "dependencies": { - "@scalar/helpers": "0.11.1", - "@scalar/themes": "0.17.3", - "@scalar/types": "0.18.2", - "@scalar/workspace-store": "0.58.1", + "@scalar/helpers": "0.11.3", + "@scalar/themes": "0.17.4", + "@scalar/types": "0.19.0", + "@scalar/workspace-store": "0.60.0", "flatted": "^3.4.0", "vue": "^3.5.40", "yaml": "^2.9.0" @@ -1727,12 +1727,12 @@ } }, "node_modules/@scalar/schemas": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/@scalar/schemas/-/schemas-0.8.3.tgz", - "integrity": "sha512-cTjgiJxXFXMqXlFZafXOyTOKm1lUfbDTbe9La+dPcQPe6q0zXAiVtys0IKILQWNrjXPidY0eaB9Va/rd16bfxw==", + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@scalar/schemas/-/schemas-0.9.0.tgz", + "integrity": "sha512-yYRlIWzw+7HuIX4z7rk7tg4y1nERwvvWKbolZOm7LveSTrppllGKyjtnIqS5uXsmJddERxuurSgDW224gGJlFQ==", "license": "MIT", "dependencies": { - "@scalar/helpers": "0.11.1", + "@scalar/helpers": "0.11.3", "@scalar/validation": "0.6.3" }, "engines": { @@ -1740,17 +1740,17 @@ } }, "node_modules/@scalar/sidebar": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/@scalar/sidebar/-/sidebar-0.10.1.tgz", - "integrity": "sha512-RbDJD22tAMGquDh7ItUeVujSvQxLoc7Eo95gPhHaokEYJprBs/B30Es3J1qhNZQ6IVZ0kcxCjuD4haI1h/UmrQ==", + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/@scalar/sidebar/-/sidebar-0.11.1.tgz", + "integrity": "sha512-xXgB0WYWG4WJTFL93WkvoAVdD1H4k+A2n2jgwa/PrUM6RO8TTh+7BLcekpwvPLvXd5muWxzWDTcR413saocphw==", "license": "MIT", "dependencies": { - "@scalar/components": "0.28.1", - "@scalar/helpers": "0.11.1", + "@scalar/components": "0.29.1", + "@scalar/helpers": "0.11.3", "@scalar/icons": "0.7.6", - "@scalar/themes": "0.17.3", - "@scalar/use-hooks": "0.4.10", - "@scalar/workspace-store": "0.58.1", + "@scalar/themes": "0.17.4", + "@scalar/use-hooks": "0.4.11", + "@scalar/workspace-store": "0.60.0", "vue": "^3.5.40" }, "engines": { @@ -1758,13 +1758,13 @@ } }, "node_modules/@scalar/snippetz": { - "version": "0.9.28", - "resolved": "https://registry.npmjs.org/@scalar/snippetz/-/snippetz-0.9.28.tgz", - "integrity": "sha512-xpzQ5NgJDfV5Y5Xmpo2lDZclbXuIRolIm6qAaShNVBvO3q2GYtxJ9RSsrMFTpuk1NIqcQ4WKVAVRllRhYuzlWg==", + "version": "0.9.30", + "resolved": "https://registry.npmjs.org/@scalar/snippetz/-/snippetz-0.9.30.tgz", + "integrity": "sha512-mDluVSGZet1Go8NgJK9s9Z8zNKqePG7zNn8PMCphAzwXNomvMy6j8WRuLDln+Dz33jILfiKlUtv4cnLkmzB+7g==", "license": "MIT", "dependencies": { - "@scalar/helpers": "0.11.1", - "@scalar/types": "0.18.2", + "@scalar/helpers": "0.11.3", + "@scalar/types": "0.19.0", "js-base64": "^3.9.2", "stringify-object": "^6.0.0" }, @@ -1773,9 +1773,9 @@ } }, "node_modules/@scalar/themes": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@scalar/themes/-/themes-0.17.3.tgz", - "integrity": "sha512-QJPHeGCg0hF30IGPjX2nMcMOvBYyd62d5vey1mIC17CLhOe5tLVTn9D6G175D+jPVb0WhVPcSbaax6KxSZTUEQ==", + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@scalar/themes/-/themes-0.17.4.tgz", + "integrity": "sha512-tSCtGLb0noijR8GzgH6H/tlbSuZoLupe66ta6I9FzZxdvgj4SdPM+2Q7E8k1uwPRfrE5NvMyp61s1BsA3DTFsg==", "license": "MIT", "dependencies": { "nanoid": "^5.1.6" @@ -1791,15 +1791,15 @@ "license": "MIT" }, "node_modules/@scalar/types": { - "version": "0.18.2", - "resolved": "https://registry.npmjs.org/@scalar/types/-/types-0.18.2.tgz", - "integrity": "sha512-q7fGMn0IygdLbYk9W4quM0w1caHDfL9FIxsYXNsgIep6uuO0T/t4BOStyf0qyzQ3pjt0B7rWFxO//EM9I7F/Tw==", + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/@scalar/types/-/types-0.19.0.tgz", + "integrity": "sha512-EKeoWgUlP+uepbM/zEHKbsdpBNyOmSrw6DdU/WC0p53gz5uRzI7d/IEecIP9SQMUkResPR9DmWeWfFQftG7vqg==", "license": "MIT", "dependencies": { - "@scalar/helpers": "0.11.1", + "@scalar/helpers": "0.11.3", "nanoid": "^5.1.6", "type-fest": "^5.8.0", - "zod": "^4.3.5" + "zod": "^4.4.3" }, "engines": { "node": ">=22" @@ -1832,11 +1832,12 @@ } }, "node_modules/@scalar/use-hooks": { - "version": "0.4.10", - "resolved": "https://registry.npmjs.org/@scalar/use-hooks/-/use-hooks-0.4.10.tgz", - "integrity": "sha512-YDIohEujqRmPCLpRlE+NTzjKPjeMJZtI4oJcZ5uH2vA1gR8wiaEStK8djBsldOFIn+LmhHagl+HHn6NX054f7Q==", + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/@scalar/use-hooks/-/use-hooks-0.4.11.tgz", + "integrity": "sha512-wCUn9WWKv4abiFZOcjBp8nIcyZ30t7QNPgHbcYu5MXgLgPclYwKb/A+TZiyp63TORWJOK8sp1RzrNGw2CT+m/Q==", "license": "MIT", "dependencies": { + "@scalar/helpers": "0.11.3", "@scalar/use-toasts": "0.10.5", "@scalar/validation": "0.6.3", "@vueuse/core": "13.9.0", @@ -1871,19 +1872,19 @@ } }, "node_modules/@scalar/workspace-store": { - "version": "0.58.1", - "resolved": "https://registry.npmjs.org/@scalar/workspace-store/-/workspace-store-0.58.1.tgz", - "integrity": "sha512-aKBwM7Tp+VzdxqVC971c8uxM8w9cw+RS7JY5V/VHj29HyNwKQTSJX6+qf8gma46bgUK0W15Ra/T8rN1mAR+EIw==", + "version": "0.60.0", + "resolved": "https://registry.npmjs.org/@scalar/workspace-store/-/workspace-store-0.60.0.tgz", + "integrity": "sha512-O3Zp6Olq7+L2Yp6xd7Z9sIQ4VG5SwR2oHkiGTagZBSrqEuvuce5Z93q3NVPux6j3cID/geW0vC0sPJ5LxNTnBA==", "license": "MIT", "dependencies": { - "@scalar/asyncapi-upgrader": "0.1.7", - "@scalar/helpers": "0.11.1", - "@scalar/json-magic": "0.13.2", + "@scalar/asyncapi-upgrader": "0.1.9", + "@scalar/helpers": "0.11.3", + "@scalar/json-magic": "0.13.4", "@scalar/openapi-upgrader": "0.2.15", - "@scalar/schemas": "0.8.3", - "@scalar/snippetz": "0.9.28", + "@scalar/schemas": "0.9.0", + "@scalar/snippetz": "0.9.30", "@scalar/typebox": "0.1.3", - "@scalar/types": "0.18.2", + "@scalar/types": "0.19.0", "@scalar/validation": "0.6.3", "js-base64": "^3.9.2", "type-fest": "^5.8.0", @@ -1978,9 +1979,9 @@ } }, "node_modules/@tanstack/virtual-core": { - "version": "3.17.8", - "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.17.8.tgz", - "integrity": "sha512-BfEvehNpOT75r5Ksc5xW6NZuXujTfb7nlSEyVu4XHG3gdxNg1KqXruWbDewXOUaUYIo4oRbSfkjIajz4MAT8tA==", + "version": "3.17.9", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.17.9.tgz", + "integrity": "sha512-M8Bzy7CCMvUjRiuoHVH9mRjyUVczRs8v8RcUEphLFGc445ZP4KQ15Y1sLqhQqJi/HgGJrxfCHnEnIX0x9mVrBg==", "license": "MIT", "funding": { "type": "github", @@ -1988,12 +1989,12 @@ } }, "node_modules/@tanstack/vue-virtual": { - "version": "3.13.36", - "resolved": "https://registry.npmjs.org/@tanstack/vue-virtual/-/vue-virtual-3.13.36.tgz", - "integrity": "sha512-gKpExv4RbB9luVG+SucTXoqPZv/gzu/Yvz6BNO+8kpNxJ2x+I/ulryzl5W9BRciahZGp5Tls3Dp5XP1ztVGbMw==", + "version": "3.13.37", + "resolved": "https://registry.npmjs.org/@tanstack/vue-virtual/-/vue-virtual-3.13.37.tgz", + "integrity": "sha512-1QNT8EXVKUx537/qvn/tFGs48eITBrrTWcyaMVjD4SCpij7E1LksbXZQLqkXBoJ9lfOVely/8Fgzqe/olQ+drw==", "license": "MIT", "dependencies": { - "@tanstack/virtual-core": "3.17.8" + "@tanstack/virtual-core": "3.17.9" }, "funding": { "type": "github", @@ -2058,9 +2059,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "24.13.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", - "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "version": "24.13.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.4.tgz", + "integrity": "sha512-YJ7EqCstVTzIr0fMr7qul/977en+pQHrfmuKIo6Zr9i75Be21dr3MovcfvGtyvi2HAUrRerWps5sMO9I7WaxDw==", "license": "MIT", "dependencies": { "undici-types": "~7.18.0" @@ -2094,9 +2095,9 @@ } }, "node_modules/@ungap/structured-clone": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.3.tgz", - "integrity": "sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.4.0.tgz", + "integrity": "sha512-1mEZtMKPM09vDmQt5y7YvmN2+DFTP7Tg0EWXdic8/C6VRnpb33e4ghisCIE3WZjsE2N8mf+QV1Zqh7ZFYLWInQ==", "license": "ISC" }, "node_modules/@unhead/vue": { @@ -2125,13 +2126,13 @@ } }, "node_modules/@vue/compiler-core": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.41.tgz", - "integrity": "sha512-q0Xtv/F9w2YO/7htQhtiL+Ev2WCJbe5N2hc+XfgyKkEKqWpSxknmT8QOuGdEKNdjPq0c3F7rNpFkTo3Kfrm7pg==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.42.tgz", + "integrity": "sha512-2Ye1ilMtKXxl8qZUrQ5j0CdgenFp/HFQmta6rfRyfEsTG69L6Wk+tWuNoHYHMx9E8tF2Slvdg1FuwDvAXdy1LQ==", "license": "MIT", "dependencies": { "@babel/parser": "^7.29.8", - "@vue/shared": "3.5.41", + "@vue/shared": "3.5.42", "entities": "^7.0.1", "estree-walker": "^2.0.2", "source-map-js": "^1.2.1" @@ -2150,26 +2151,26 @@ } }, "node_modules/@vue/compiler-dom": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.41.tgz", - "integrity": "sha512-oKacVfNglLvGjnS6BXOlGL7EyG2h8X03pqXCjzotRZUaXGjbrTJUnVAQjrCqUnS+lyu31nwQjZY/d817GmCnfw==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.42.tgz", + "integrity": "sha512-qbhQZEFmycr+ni/qyuccS4sucNN7VAbDfbkvNxWOX2VfgFm90MNs3/UhRNKoPMEIVn0F8gdlYjLPvqxHwHeQOA==", "license": "MIT", "dependencies": { - "@vue/compiler-core": "3.5.41", - "@vue/shared": "3.5.41" + "@vue/compiler-core": "3.5.42", + "@vue/shared": "3.5.42" } }, "node_modules/@vue/compiler-sfc": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.41.tgz", - "integrity": "sha512-XJhip7R2wy6vX3knCxdZN4KracFaZUef58s1KYewqluedHIJaPIVfXoYT7MF1F8nCvv6k8bWWxDC8opMkg1VTQ==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.42.tgz", + "integrity": "sha512-fkCAFB4okcAANGMThboWnScp/gzWjU0ZSkVnjTIiplmMDq2uq0tIB3j+xVu4rhv5rvOgBySCysudmbMd6xRRqw==", "license": "MIT", "dependencies": { "@babel/parser": "^7.29.8", - "@vue/compiler-core": "3.5.41", - "@vue/compiler-dom": "3.5.41", - "@vue/compiler-ssr": "3.5.41", - "@vue/shared": "3.5.41", + "@vue/compiler-core": "3.5.42", + "@vue/compiler-dom": "3.5.42", + "@vue/compiler-ssr": "3.5.42", + "@vue/shared": "3.5.42", "estree-walker": "^2.0.2", "magic-string": "^0.30.21", "postcss": "^8.5.19", @@ -2177,61 +2178,61 @@ } }, "node_modules/@vue/compiler-ssr": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.41.tgz", - "integrity": "sha512-U3v5OejKEGqOI0Wy0+Sz7hGuIFZHA4LSXzrNM3IMIeDyJEBBfTpX26n3SDgToRpP2bLc9FfI2j/kSgcJ8Emq5A==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.42.tgz", + "integrity": "sha512-xmLk3wLkbizPAiLyomjgFFosf2ys9b5Ghb+oh/k2tnvipNz8OFrQOiTcWCzyK7MpBp9KkyGtfvgfLUivbmuGYA==", "license": "MIT", "dependencies": { - "@vue/compiler-dom": "3.5.41", - "@vue/shared": "3.5.41" + "@vue/compiler-dom": "3.5.42", + "@vue/shared": "3.5.42" } }, "node_modules/@vue/reactivity": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.41.tgz", - "integrity": "sha512-rznsqKM0np0x18EjzF8x88MpEhdNsffbvFbckLL5+oUKz1BxAImEmO7J1ArRYSyo6aQaVoBDp7jEkT91OOxydA==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.42.tgz", + "integrity": "sha512-TzNNfKpb7hDxbQltwAut8VDQA5YP+BuRlxntHUuRjyKwlMvmAPbs3unhCvieijifY6vFfVBwsS7wG/C7uq+bEQ==", "license": "MIT", "dependencies": { - "@vue/shared": "3.5.41" + "@vue/shared": "3.5.42" } }, "node_modules/@vue/runtime-core": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.41.tgz", - "integrity": "sha512-Vcry58hiAKwGen9Z1jUZE0feFsNArPCMOImYI8el48A9Idf6DuQYD0U05zZIF2Iad1hGhPSvcbBbAOhNr55fhg==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.42.tgz", + "integrity": "sha512-9uACtuHs7vJGkm5Bp3xu4xRDLFTIYy5DgxpToVjqGIAhAEKwQfsaLvKINhM6nFVp6bZPRFGdDqd1g52MqKsotA==", "license": "MIT", "dependencies": { - "@vue/reactivity": "3.5.41", - "@vue/shared": "3.5.41" + "@vue/reactivity": "3.5.42", + "@vue/shared": "3.5.42" } }, "node_modules/@vue/runtime-dom": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.41.tgz", - "integrity": "sha512-3vVBahVBS9+U6cmXBLyb8nE6/yYo4J/CGI9eVFs3KiMc0YHuudwKyShTD65jtJy/L9PUUxNAFu4cj4LiJ0UFbw==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.42.tgz", + "integrity": "sha512-rsCmhiWLaRxGltLwhlCWyYkFn7WAbKRh0q17eZ1A6Dq6eqc2ACQ61IIryxz0LrsvCzHSilLA9JHovVwM8CNE2g==", "license": "MIT", "dependencies": { - "@vue/reactivity": "3.5.41", - "@vue/runtime-core": "3.5.41", - "@vue/shared": "3.5.41", + "@vue/reactivity": "3.5.42", + "@vue/runtime-core": "3.5.42", + "@vue/shared": "3.5.42", "csstype": "^3.2.3" } }, "node_modules/@vue/server-renderer": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.41.tgz", - "integrity": "sha512-n6hx/pNFfbD6SuyeuMVkvqox8bwf/ET9JlA/kAz/imw8sw++wkqKe2mHX5KutjPpbKE4Z56yTHszoOjGMI9igQ==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.42.tgz", + "integrity": "sha512-2++5dUyYS4gvo7xQXSECUDhB7TS0aOl5SeVfC5qSq1Jgfhjvegw1zqhwTIR3imZ+QYPJQw9gfcFvXGAjGZ7ajQ==", "license": "MIT", "dependencies": { - "@vue/compiler-ssr": "3.5.41", - "@vue/runtime-dom": "3.5.41", - "@vue/shared": "3.5.41" + "@vue/compiler-ssr": "3.5.42", + "@vue/runtime-dom": "3.5.42", + "@vue/shared": "3.5.42" } }, "node_modules/@vue/shared": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.41.tgz", - "integrity": "sha512-IOnwSCma8j+9xJT6b8H0dEYidC80NsYmNMlZxRsukYcSoGaDBohog5hDxzeUXdFeGWFA++vWvxqOmrr96VlqMA==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.42.tgz", + "integrity": "sha512-2rPxex1jQf4jvl9MOHl6YaXCPcrNqz/FstMOEh3QWY+/OME9nQTvl9WYeCwhW7AFjaR0SnngZGlp/wkR6rkI6g==", "license": "MIT" }, "node_modules/@vueuse/core": { @@ -3233,16 +3234,15 @@ } }, "node_modules/engine.io": { - "version": "6.6.9", - "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.6.9.tgz", - "integrity": "sha512-clKkw4C7nJ22mGgoVcCg6V/W/TxdNyIOTr89k2ONZu81qqkddPFDF0LXcbAwhzPD8DjkiRCjzuiO6Y+fkpD4vg==", + "version": "6.6.10", + "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.6.10.tgz", + "integrity": "sha512-9/lX2bdlizlCXMHRMOIm03VBQHQYC7VvydcxtTAUJRxNW1QzM/2PMFSmr6h/lCiMHcyCP6abK+t9Q+j4vekk8Q==", "license": "MIT", "dependencies": { "@types/cors": "^2.8.12", "@types/node": ">=10.0.0", "@types/ws": "^8.5.12", "accepts": "~1.3.4", - "base64id": "2.0.0", "cookie": "~0.7.2", "cors": "~2.8.5", "debug": "~4.4.1", @@ -3466,9 +3466,9 @@ } }, "node_modules/express-rate-limit": { - "version": "8.6.2", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.6.2.tgz", - "integrity": "sha512-YH4ru+eOJxQABscKFfRCy9R7x9QFGdezclVMwwgFFndzS2Xnm0uo6B0ABZsLhcpeptGv2qvuJVWlQr9gQZoC3A==", + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.7.0.tgz", + "integrity": "sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g==", "license": "MIT", "dependencies": { "debug": "^4.4.3", @@ -3563,9 +3563,9 @@ "license": "MIT" }, "node_modules/fast-jwt": { - "version": "6.3.2", - "resolved": "https://registry.npmjs.org/fast-jwt/-/fast-jwt-6.3.2.tgz", - "integrity": "sha512-JTQImpkXVvj+eq7tJImtsHRt1K6ngloEzIx62Qbf9x4tEM2P2EqGpYJSeUoPf/kMn78rImSHfhf08KShR8PauA==", + "version": "6.3.3", + "resolved": "https://registry.npmjs.org/fast-jwt/-/fast-jwt-6.3.3.tgz", + "integrity": "sha512-pQDXx7IHeZT4jSmpE9o80RrBqfrG4fPrl8anazSM5vErIdK1iCc13z/EWX+H0j7liWSRnwTpHswIKMeLYGAckw==", "license": "Apache-2.0", "dependencies": { "@lukeed/ms": "^2.0.2", @@ -3579,9 +3579,9 @@ } }, "node_modules/fast-uri": { - "version": "3.1.6", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz", - "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz", + "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==", "funding": [ { "type": "github", @@ -3897,19 +3897,19 @@ } }, "node_modules/graphql-tools": { - "version": "9.0.34", - "resolved": "https://registry.npmjs.org/graphql-tools/-/graphql-tools-9.0.34.tgz", - "integrity": "sha512-pccboGsOGmF5falh1aKJTX0u7Apot8m66dsIU7aUaVQ8RWpoXqT37j6b1O9ToA0oMuSuS+qLt8LokTmsEsP1Gw==", + "version": "9.0.35", + "resolved": "https://registry.npmjs.org/graphql-tools/-/graphql-tools-9.0.35.tgz", + "integrity": "sha512-bmHHVqGIqkRCb95ShdNS4oMR3bayUz+Q26pVt5JMVG8IpUSMnrjL3uqbNtaTMXn3wLnkaARE039W7hwXpIOE9Q==", "license": "MIT", "dependencies": { - "@graphql-tools/schema": "^10.1.0", + "@graphql-tools/schema": "^10.1.1", "tslib": "^2.4.0" }, "engines": { "node": ">=16.0.0" }, "optionalDependencies": { - "@apollo/client": "~4.2.10" + "@apollo/client": "~4.2.12" }, "peerDependencies": { "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" @@ -4300,9 +4300,9 @@ } }, "node_modules/hono": { - "version": "4.13.4", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.4.tgz", - "integrity": "sha512-AGEwKIyRMHRv1t8Wjwa3LHxQ61X5CqrdFT+4BRNTpqS5aJNnpl5WLjADb7vFlJzI/8uK7T5QLVApCMQKNa3LgQ==", + "version": "4.13.7", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.7.tgz", + "integrity": "sha512-c8/gF9ac8Y78/agExVocyLevgR+JlpNB444Py0FSX8pJoPdYUfUzRcXtYEYGwt6l19qIlVZPN5Mfsw9jFShmQQ==", "license": "MIT", "engines": { "node": ">=16.9.0" @@ -4427,9 +4427,9 @@ } }, "node_modules/ip-address": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz", - "integrity": "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==", + "version": "10.7.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.0.tgz", + "integrity": "sha512-BGFsyJd5mpXp3rK6jIdADLNgpJUK1jnjzvYF8lK+VyDab9JAmqN0YOKDdP17HlgKb2+ehPgDc8EtnRLbGCAMhA==", "license": "MIT", "engines": { "node": ">= 12" @@ -4575,9 +4575,9 @@ "license": "ISC" }, "node_modules/jose": { - "version": "6.2.10", - "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.10.tgz", - "integrity": "sha512-iiW7J9qRFlGxvCOIBDBDxFePQSn7ZMAnrYGhrrOo6siO/MIqwfyilLR27pkfDgUk+raLuzADS8A3S/KLBisc0g==", + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/panva" @@ -4763,6 +4763,15 @@ "node": ">= 12.0.0" } }, + "node_modules/logform/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "license": "MIT", + "engines": { + "node": ">=0.1.90" + } + }, "node_modules/loglevel": { "version": "1.9.2", "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz", @@ -6053,9 +6062,9 @@ } }, "node_modules/postcss": { - "version": "8.5.26", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", - "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", "funding": [ { "type": "opencollective", @@ -6072,7 +6081,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.17", + "nanoid": "^3.3.18", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -6099,9 +6108,9 @@ } }, "node_modules/pretty-ms": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-9.3.0.tgz", - "integrity": "sha512-gjVS5hOP+M3wMm5nmNOucbIrqudzs9v/57bWRHQWLYklXqoXKrVfYW2W9+glfGsqtPgpiz5WwyEEB+ksXIx3gQ==", + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-9.3.1.tgz", + "integrity": "sha512-HzMy3Geq23nVALD/M2LliU+F+M+gVNsvkQWWqeBZ8HDiCgzo6YPJ/Omrmtq24EFrIsk0a3EkQGEd7bDOo+IhGA==", "license": "MIT", "dependencies": { "parse-ms": "^4.0.0" @@ -6117,6 +6126,7 @@ "version": "15.1.3", "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "deprecated": "prom-client has been replaced by @prometheus-io/client", "license": "Apache-2.0", "dependencies": { "@opentelemetry/api": "^1.4.0", @@ -6137,9 +6147,9 @@ } }, "node_modules/protobufjs": { - "version": "8.7.2", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.7.2.tgz", - "integrity": "sha512-oTVHV+oelUBtiu5iTuTNNZ0eLYsXSMxry4cgr30mayNkgIZL6qZ0IOQVPuSWGcyAaXKl/XgqwWHIC3a0khYVBA==", + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.8.0.tgz", + "integrity": "sha512-N3xhQ5yyBx3vQq4gubBfASzYhJGNzeDbjqBpu61g7UVylsN/qyffU96TKWD3GbbLOKF82VGNRNvv1+BFgE31Eg==", "license": "BSD-3-Clause", "dependencies": { "long": "^5.3.2" @@ -6177,9 +6187,9 @@ } }, "node_modules/qs": { - "version": "6.15.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", - "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", "license": "BSD-3-Clause", "dependencies": { "es-define-property": "^1.0.1", @@ -7155,9 +7165,9 @@ } }, "node_modules/swagger-ui-dist": { - "version": "5.32.14", - "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-5.32.14.tgz", - "integrity": "sha512-nOA2pSQhcmODMUQZpJHYKNuwniDUqcOWGNaSCOoZv12FdOSJ9JxV95HtyRGNMqEBj6h6lCNTy20TgZDYTSuUIg==", + "version": "5.32.15", + "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-5.32.15.tgz", + "integrity": "sha512-TSFER+rFQlf1nzk6WvKkMaHTxAPQ3eAAxigFThnxQedSREanfZgSbJFayZVs/ULnSbNdrJOb99vLD6xpb3R3eg==", "license": "Apache-2.0", "dependencies": { "@scarf/scarf": "=1.4.0" @@ -7223,9 +7233,9 @@ "peer": true }, "node_modules/tdigest": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", - "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.3.tgz", + "integrity": "sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==", "license": "MIT", "dependencies": { "bintrees": "1.0.2" @@ -7339,9 +7349,9 @@ "license": "0BSD" }, "node_modules/type-fest": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.8.0.tgz", - "integrity": "sha512-YGYEVz3Fm5iy/AybuA0oyNFq7H4CgQNfRp/qfe8nurE1kuCeNm3/vfm9X4Mtl+qLyaKJUh5xrFZwogr41SMjYA==", + "version": "5.9.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.9.0.tgz", + "integrity": "sha512-yANm3Jr3GiJ1qgJlxGAVxTOIcEOk1rhQHamlXtnrCK7EHP4HeM9OGxtMg/W7HFdrVzw/ZWJKGVIJusVH85sLtw==", "license": "(MIT OR CC0-1.0)", "dependencies": { "tagged-tag": "^1.0.0" @@ -7612,16 +7622,16 @@ } }, "node_modules/vue": { - "version": "3.5.41", - "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.41.tgz", - "integrity": "sha512-2laE0p+aK+/AOPG/XL/WepOs/GlK755LJ1XECi9kDUrz1FKNw8rb2Xzlw9JS1rqEV55nb0ttsKxVlTCcd+R5cg==", + "version": "3.5.42", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.42.tgz", + "integrity": "sha512-4RyHQTbQvOPs3MfvUO1Sg0YRrKNnA0mAVtvpd12Tg1fKDN7OHBUl1IqSn8zGJjK9nI3NkNp8cgTpVrSZC5TTcA==", "license": "MIT", "dependencies": { - "@vue/compiler-dom": "3.5.41", - "@vue/compiler-sfc": "3.5.41", - "@vue/runtime-dom": "3.5.41", - "@vue/server-renderer": "3.5.41", - "@vue/shared": "3.5.41" + "@vue/compiler-dom": "3.5.42", + "@vue/compiler-sfc": "3.5.42", + "@vue/runtime-dom": "3.5.42", + "@vue/server-renderer": "3.5.42", + "@vue/shared": "3.5.42" }, "peerDependencies": { "typescript": "*" @@ -7756,9 +7766,9 @@ "license": "Apache-2.0" }, "node_modules/winston-loki/node_modules/protobufjs": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", - "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -7897,9 +7907,9 @@ } }, "node_modules/zod": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", - "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.0.tgz", + "integrity": "sha512-iIvwyDnebKYpww2ta0DjaNOL8RnVLmPMhgWyOzlW9y0EIIxv5gl+H6Y0ONpt83HqjGqkk78uWp6xWtpxzZdPbw==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/packages/core/package.bundle.json b/packages/core/package.bundle.json index 4288fc87e..6a49d8a0b 100644 --- a/packages/core/package.bundle.json +++ b/packages/core/package.bundle.json @@ -28,7 +28,7 @@ "prom-client": "^15.1.3", "protobufjs": "^8.7.2", "snappy": "7.4.1", - "uuid": "^14.0.1", + "uuid": "^14.0.2", "winston": "^3.19.0", "winston-loki": "^6.1.7", "@apollo/cache-control-types": "^1.0.3", @@ -36,7 +36,7 @@ "@as-integrations/express5": "^1.1.2", "@modelcontextprotocol/sdk": "^1.29.0", "@scalar/api-reference": "^1.60.0", - "@scalar/express-api-reference": "^0.10.14", + "@scalar/express-api-reference": "^0.10.16", "@socket.io/redis-streams-adapter": "^0.3.1", "bcrypt": "^6.0.0", "body-parser": "^2.3.0", diff --git a/packages/core/package.json b/packages/core/package.json index 669147c01..1d115d7e0 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -19,7 +19,8 @@ "prebuild:bundle": "pnpm --filter @conduitplatform/service-bundle run build", "build:bundle": "rimraf bundle && node ../../libraries/service-bundle/dist/cli.js generate-manifest && tsup && node ../../libraries/service-bundle/dist/cli.js copy-assets && node ../../libraries/service-bundle/dist/cli.js generate-lockfile", "prepare": "npm run build", - "prepublish": "npm run build" + "prepublish": "npm run build", + "test": "node --experimental-strip-types --test tests/*.test.ts" }, "license": "ISC", "dependencies": { diff --git a/packages/core/src/admin/routes/GetModuleConfig.route.ts b/packages/core/src/admin/routes/GetModuleConfig.route.ts index 3c1d4db0a..b71b62168 100644 --- a/packages/core/src/admin/routes/GetModuleConfig.route.ts +++ b/packages/core/src/admin/routes/GetModuleConfig.route.ts @@ -5,6 +5,7 @@ import { ConduitRouteReturnDefinition, } from '@conduitplatform/grpc-sdk'; import { ConduitRoute } from '@conduitplatform/hermes'; +import { redactSensitiveConfig } from '@conduitplatform/module-tools'; import convict from 'convict'; export function getModuleConfigRoute( @@ -32,7 +33,7 @@ export function getModuleConfigRoute( } else { finalConfig = JSON.parse(finalConfig); } - return { config: finalConfig }; + return { config: redactSensitiveConfig(finalConfig, configSchema) }; }, ); } diff --git a/packages/core/src/admin/routes/GetMonoConfig.route.ts b/packages/core/src/admin/routes/GetMonoConfig.route.ts index 25b768621..10a2c7009 100644 --- a/packages/core/src/admin/routes/GetMonoConfig.route.ts +++ b/packages/core/src/admin/routes/GetMonoConfig.route.ts @@ -3,7 +3,7 @@ import { ConduitRouteActions, ConduitRouteReturnDefinition, } from '@conduitplatform/grpc-sdk'; -import { ConduitJson } from '@conduitplatform/module-tools'; +import { ConduitJson, redactSensitiveConfig } from '@conduitplatform/module-tools'; import { ConduitRoute } from '@conduitplatform/hermes'; import { ServiceRegistry } from '../../service-discovery/ServiceRegistry.js'; @@ -27,7 +27,11 @@ export function getMonoConfigRoute(grpcSdk: ConduitGrpcSdk) { ].sort(); for (const moduleName of sortedModules) { const moduleConfig = await grpcSdk.state!.getKey(`moduleConfigs.${moduleName}`); - if (moduleConfig) monoConfig.modules[moduleName] = JSON.parse(moduleConfig); + if (moduleConfig) { + monoConfig.modules[moduleName] = redactSensitiveConfig( + JSON.parse(moduleConfig), + ); + } } return { config: monoConfig }; }, diff --git a/packages/core/src/admin/routes/SetModuleConfig.route.ts b/packages/core/src/admin/routes/SetModuleConfig.route.ts index 1bf6ec2f2..adc8cea73 100644 --- a/packages/core/src/admin/routes/SetModuleConfig.route.ts +++ b/packages/core/src/admin/routes/SetModuleConfig.route.ts @@ -9,6 +9,7 @@ import { } from '@conduitplatform/grpc-sdk'; // Removed ConduitCommons import - now using configManager directly import { ConduitRoute } from '@conduitplatform/hermes'; +import { redactSensitiveConfig } from '@conduitplatform/module-tools'; import convict from 'convict'; type SetConfig = (config: { newConfig: string }) => Promise<{ updatedConfig: string }>; @@ -66,7 +67,7 @@ export function setModuleConfigRoute( updatedConfig = JSON.parse(updatedConfig.updatedConfig); } await configManager.set(moduleName, updatedConfig); - return { config: updatedConfig }; + return { config: redactSensitiveConfig(updatedConfig, configSchema) }; }, ); } diff --git a/packages/core/tests/embeddingsConfigLifecycle.test.ts b/packages/core/tests/embeddingsConfigLifecycle.test.ts new file mode 100644 index 000000000..8fc390219 --- /dev/null +++ b/packages/core/tests/embeddingsConfigLifecycle.test.ts @@ -0,0 +1,186 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import convict from 'convict'; +import { merge as lodashMerge } from 'lodash-es'; +import { + merge, + reconcileStoredModuleConfig, + redactSensitiveConfig, + restoreRedactedSecrets, +} from '@conduitplatform/module-tools'; +import { getModuleConfigRoute } from '../dist/admin/routes/GetModuleConfig.route.js'; +import { setModuleConfigRoute } from '../dist/admin/routes/SetModuleConfig.route.js'; +import AppConfigSchema, { + type Config, +} from '../../../modules/embeddings/dist/config/index.js'; +import { normalizeEmbeddingsConfig } from '../../../modules/embeddings/dist/utils/providerConfig.js'; + +const MODULE_NAME = 'embeddings'; +const STORE_KEY = `moduleConfigs.${MODULE_NAME}`; + +const legacyStored = { + enabled: false, + defaultProvider: 'openai-compatible', + providers: { + 'openai-compatible': { + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: 'sk-live', + model: 'text-embedding-3-small', + dimensions: 1536, + models: [], + allowedHosts: ['api.openai.com'], + }, + }, + queue: { + concurrency: 2, + attempts: 3, + maxBatchSize: 500, + drainTimeoutMs: 15 * 60 * 1000, + }, + security: { + requireGrpcKey: true, + sourceFieldAllowlist: [], + maxMutationEventIds: 500, + embedTimeoutMs: 10_000, + maxEmbedInputBytes: 32 * 1024, + maxEmbedResponseBytes: 1024 * 1024, + }, +}; + +function providerOf(config: Record) { + const providers = config.providers as Record>; + return providers['openai-compatible']; +} + +describe('embeddings Admin GET/PATCH config lifecycle', () => { + it('persists catalogue migration so an unrelated PATCH cannot wipe it', async () => { + const store = new Map(); + store.set(STORE_KEY, JSON.stringify(legacyStored)); + const schema = convict(AppConfigSchema); + let configureCalls = 0; + + const local = normalizeEmbeddingsConfig(schema.getProperties() as Config); + const existing = JSON.parse(store.get(STORE_KEY)!) as Config; + const merged = lodashMerge({}, local, existing) as Config; + store.set(STORE_KEY, JSON.stringify(merged)); + + const migrated = normalizeEmbeddingsConfig(merged); + schema.load(migrated).validate({ allowed: 'warn' }); + const persistable = schema.getProperties() as Config; + const reconciled = await reconcileStoredModuleConfig({ + stored: merged, + migrated: persistable, + configureOverride: async next => { + configureCalls += 1; + store.set(STORE_KEY, JSON.stringify(next)); + return next; + }, + }); + schema.load(reconciled.config); + assert.equal(configureCalls, 1); + + const second = await reconcileStoredModuleConfig({ + stored: JSON.parse(store.get(STORE_KEY)!) as Config, + migrated: schema.getProperties() as Config, + configureOverride: async next => { + configureCalls += 1; + store.set(STORE_KEY, JSON.stringify(next)); + return next; + }, + }); + assert.equal(second.persisted, false); + assert.equal(configureCalls, 1); + + const storedAfterLifecycle = JSON.parse(store.get(STORE_KEY)!) as Record< + string, + unknown + >; + const storedProvider = providerOf(storedAfterLifecycle); + assert.deepEqual(storedProvider.models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + assert.equal(storedProvider.defaultModel, 'text-embedding-3-small'); + assert.equal(storedProvider.apiKey, 'sk-live'); + assert.equal('model' in storedProvider, false); + assert.equal('dimensions' in storedProvider, false); + assert.equal('allowedHosts' in storedProvider, false); + const storedSecurity = storedAfterLifecycle.security as Record; + assert.equal('requireGrpcKey' in storedSecurity, false); + + const grpcSdk = { + state: { + getKey: async (key: string) => store.get(key) ?? null, + }, + getModuleClient: () => ({ + setConfig: async ({ newConfig }: { newConfig: string }) => { + const previous = schema.getProperties() as Config; + let next = merge(previous, JSON.parse(newConfig) as Config); + next = restoreRedactedSecrets(next, previous, AppConfigSchema); + next = normalizeEmbeddingsConfig(next); + schema.load(next).validate({ allowed: 'warn' }); + return { updatedConfig: JSON.stringify(schema.getProperties()) }; + }, + }), + }; + const configManager = { + set: async (_name: string, config: unknown) => { + store.set(STORE_KEY, JSON.stringify(config)); + return config; + }, + }; + + const getRoute = getModuleConfigRoute(grpcSdk as never, MODULE_NAME, AppConfigSchema); + const getResponse = await getRoute.executeRequest({} as never); + const getProvider = providerOf(getResponse.config); + assert.equal(getProvider.apiKey, '[REDACTED]'); + assert.deepEqual(getProvider.models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + assert.equal(getProvider.defaultModel, 'text-embedding-3-small'); + assert.equal('model' in getProvider, false); + assert.equal('dimensions' in getProvider, false); + assert.equal('allowedHosts' in getProvider, false); + assert.doesNotMatch(JSON.stringify(getResponse), /sk-live/); + assert.equal( + redactSensitiveConfig(getResponse.config, AppConfigSchema).providers[ + 'openai-compatible' + ].apiKey, + '[REDACTED]', + ); + + const patchRoute = setModuleConfigRoute( + grpcSdk as never, + configManager, + MODULE_NAME, + AppConfigSchema, + ); + const patchResponse = await patchRoute.executeRequest({ + params: { config: { enabled: true } }, + } as never); + assert.equal(patchResponse.config.enabled, true); + assert.equal( + patchResponse.config.providers['openai-compatible'].apiKey, + '[REDACTED]', + ); + assert.deepEqual(patchResponse.config.providers['openai-compatible'].models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + + const storedAfterPatch = JSON.parse(store.get(STORE_KEY)!) as Record; + const patchedProvider = providerOf(storedAfterPatch); + assert.equal(storedAfterPatch.enabled, true); + assert.deepEqual(patchedProvider.models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + assert.equal(patchedProvider.defaultModel, 'text-embedding-3-small'); + assert.equal(patchedProvider.apiKey, 'sk-live'); + assert.equal('model' in patchedProvider, false); + assert.equal('dimensions' in patchedProvider, false); + + const getAfterPatch = await getRoute.executeRequest({} as never); + assert.deepEqual(providerOf(getAfterPatch.config).models, [ + { name: 'text-embedding-3-small', dimensions: 1536 }, + ]); + assert.equal(providerOf(getAfterPatch.config).apiKey, '[REDACTED]'); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ca5262f01..be1bde5f1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -883,6 +883,9 @@ importers: pg-hstore: specifier: ^2.3.4 version: 2.3.4 + pgvector: + specifier: ^0.3.0 + version: 0.3.0 sequelize: specifier: ^6.37.8 version: 6.37.8(mariadb@3.5.4)(mysql2@3.23.1(@types/node@24.13.4))(pg-hstore@2.3.4)(pg@8.22.0)(sqlite3@6.0.1) @@ -936,6 +939,64 @@ importers: specifier: ~6.0.3 version: 6.0.3 + modules/embeddings: + dependencies: + '@bufbuild/protobuf': + specifier: ^2.12.0 + version: 2.14.1 + '@conduitplatform/grpc-sdk': + specifier: workspace:* + version: link:../../libraries/grpc-sdk + '@conduitplatform/module-tools': + specifier: workspace:* + version: link:../../libraries/module-tools + '@grpc/grpc-js': + specifier: ^1.14.4 + version: 1.14.4 + '@grpc/proto-loader': + specifier: ^0.8.1 + version: 0.8.1 + bullmq: + specifier: ^5.79.0 + version: 5.79.0 + convict: + specifier: ^6.2.5 + version: 6.2.5 + ioredis: + specifier: 5.11.1 + version: 5.11.1 + lodash-es: + specifier: ^4.18.1 + version: 4.18.1 + devDependencies: + '@conduitplatform/service-bundle': + specifier: workspace:* + version: link:../../libraries/service-bundle + '@types/convict': + specifier: ^6.1.6 + version: 6.1.6 + '@types/lodash-es': + specifier: ^4.17.12 + version: 4.17.12 + '@types/node': + specifier: 24.13.4 + version: 24.13.4 + copyfiles: + specifier: ^2.4.1 + version: 2.4.1 + rimraf: + specifier: ^6.1.3 + version: 6.1.3 + ts-proto: + specifier: ^2.12.3 + version: 2.12.3 + tsup: + specifier: ^8.5.1 + version: 8.5.1(jiti@2.6.1)(postcss@8.5.15)(typescript@6.0.3)(yaml@2.9.0) + typescript: + specifier: ~6.0.3 + version: 6.0.3 + modules/functions: dependencies: '@conduitplatform/grpc-sdk': @@ -7008,6 +7069,10 @@ packages: pgpass@1.0.5: resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + pgvector@0.3.0: + resolution: {integrity: sha512-+t7qcQD2us8fO8YIq/3lA0gUrD+bVO70MG1MhcDcxJz/OlRGGIIHzFq/4x57Vn/LpzX5wFdfOTLQp9QMPd4ljQ==} + engines: {node: '>=22'} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -15166,6 +15231,8 @@ snapshots: dependencies: split2: 4.2.0 + pgvector@0.3.0: {} + picocolors@1.1.1: {} picomatch@2.3.2: {} diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index d78ebdef7..0694f5968 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -13,6 +13,7 @@ case "$TARGET" in chat) BUILDING_SERVICE="modules/chat" ;; communications) BUILDING_SERVICE="modules/communications" ;; database) BUILDING_SERVICE="modules/database" ;; + embeddings) BUILDING_SERVICE="modules/embeddings" ;; functions) BUILDING_SERVICE="modules/functions" ;; router) BUILDING_SERVICE="modules/router" ;; storage) BUILDING_SERVICE="modules/storage" ;; @@ -21,7 +22,7 @@ case "$TARGET" in all) ;; *) echo "Unknown target: $TARGET" >&2 - echo "Usage: $0 [conduit|authentication|authorization|chat|communications|database|functions|router|storage|conduit-standalone|all]" >&2 + echo "Usage: $0 [conduit|authentication|authorization|chat|communications|database|embeddings|functions|router|storage|conduit-standalone|all]" >&2 exit 1 ;; esac @@ -41,7 +42,7 @@ fi if [ "$TARGET" = "all" ]; then docker buildx bake --file docker-bake.hcl all --set "*.platform=linux/amd64,linux/arm64" -elif [ "$TARGET" = "conduit" ] || [ "$TARGET" = "chat" ] || [ "$TARGET" = "functions" ] || [ "$TARGET" = "storage" ] || [ "$TARGET" = "authentication" ] || [ "$TARGET" = "authorization" ] || [ "$TARGET" = "communications" ] || [ "$TARGET" = "database" ] || [ "$TARGET" = "router" ] || [ "$TARGET" = "conduit-standalone" ]; then +elif [ "$TARGET" = "conduit" ] || [ "$TARGET" = "chat" ] || [ "$TARGET" = "functions" ] || [ "$TARGET" = "storage" ] || [ "$TARGET" = "authentication" ] || [ "$TARGET" = "authorization" ] || [ "$TARGET" = "communications" ] || [ "$TARGET" = "database" ] || [ "$TARGET" = "embeddings" ] || [ "$TARGET" = "router" ] || [ "$TARGET" = "conduit-standalone" ]; then # Bundle-based targets: bake HCL wires conduit-base-bundle-* (BUILD_BUNDLE=1). docker buildx bake --file docker-bake.hcl "$TARGET" \ --set "*.platform=linux/amd64,linux/arm64" diff --git a/scripts/resolve-docker-targets.mjs b/scripts/resolve-docker-targets.mjs index 000ea5ddb..567fc56a6 100644 --- a/scripts/resolve-docker-targets.mjs +++ b/scripts/resolve-docker-targets.mjs @@ -1,6 +1,8 @@ #!/usr/bin/env node import { appendFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; const SHARED_BUILD_PATHS = [ 'Dockerfile', @@ -98,6 +100,19 @@ const IMAGE_TARGETS = [ ...SHARED_BUILD_PATHS, ], }, + { + target: 'embeddings', + image: 'embeddings', + name: 'Build embeddings', + buildingService: 'modules/embeddings', + isBundle: true, + paths: [ + 'modules/embeddings/**', + ...SERVICE_BUNDLE_PATHS, + ...LIBRARY_BUILD_PATHS, + ...SHARED_BUILD_PATHS, + ], + }, { target: 'functions', image: 'functions', @@ -157,6 +172,9 @@ const IMAGE_TARGETS = [ ...SERVICE_BUNDLE_PATHS, ...SHARED_BUILD_PATHS, ], + // Embeddings is intentionally omitted from standalone v1. Keep other + // module path matches so chat/storage/etc. still rebuild standalone. + excludePaths: ['modules/embeddings/**'], }, ]; @@ -178,6 +196,13 @@ function matchesAnyPath(file, patterns) { return patterns.some((pattern) => globMatch(file, pattern)); } +function matchesTarget(file, entry) { + if (Array.isArray(entry.excludePaths) && matchesAnyPath(file, entry.excludePaths)) { + return false; + } + return matchesAnyPath(file, entry.paths); +} + function parseChangedFiles() { const raw = process.env.CHANGED_FILES ?? ''; if (!raw.trim()) { @@ -193,46 +218,72 @@ function shouldBuildAll() { return process.env.FORCE_ALL === 'true'; } -function resolveTargets() { - if (shouldBuildAll()) { +function resolveTargets({ changedFiles, forceAll } = {}) { + if (forceAll ?? shouldBuildAll()) { return IMAGE_TARGETS; } - const changed = parseChangedFiles(); + const changed = changedFiles ?? parseChangedFiles(); if (changed.length === 0) { return IMAGE_TARGETS; } - const selected = IMAGE_TARGETS.filter((entry) => - changed.some((file) => matchesAnyPath(file, entry.paths)), + return IMAGE_TARGETS.filter((entry) => + changed.some((file) => matchesTarget(file, entry)), ); - - return selected; } function writeOutput(matrix, channel) { const payload = JSON.stringify({ include: matrix }); + const result = { + matrix: payload, + channel, + has_targets: matrix.length > 0, + }; + console.log(JSON.stringify(result, null, 2)); const outputFile = process.env.GITHUB_OUTPUT; - if (outputFile) { - appendFileSync(outputFile, `matrix=${payload}\n`, 'utf8'); - appendFileSync(outputFile, `channel=${channel}\n`, 'utf8'); - appendFileSync(outputFile, `has_targets=${matrix.length > 0}\n`, 'utf8'); - } else { - console.log(JSON.stringify({ matrix: payload, channel, has_targets: matrix.length > 0 }, null, 2)); + if (!outputFile) { + return; } + appendFileSync(outputFile, `matrix=${payload}\n`, 'utf8'); + appendFileSync(outputFile, `channel=${channel}\n`, 'utf8'); + appendFileSync(outputFile, `has_targets=${matrix.length > 0}\n`, 'utf8'); } -const channel = - process.env.GITHUB_EVENT_NAME === 'release' ? 'release' : 'dev'; +function isMainModule() { + const entry = process.argv[1]; + if (!entry) { + return false; + } + try { + return import.meta.url === pathToFileURL(resolve(entry)).href; + } catch { + return false; + } +} -const matrix = resolveTargets().map( - ({ target, image, name, buildingService, isBundle }) => ({ - target, - image, - name, - building_service: buildingService, - is_bundle: isBundle === true, - }), -); +if (isMainModule()) { + const channel = + process.env.GITHUB_EVENT_NAME === 'release' ? 'release' : 'dev'; + + const matrix = resolveTargets().map( + ({ target, image, name, buildingService, isBundle }) => ({ + target, + image, + name, + building_service: buildingService, + is_bundle: isBundle === true, + }), + ); + + writeOutput(matrix, channel); +} -writeOutput(matrix, channel); +export { + IMAGE_TARGETS, + globMatch, + matchesAnyPath, + matchesTarget, + parseChangedFiles, + resolveTargets, +}; diff --git a/scripts/verify-service-bundle.sh b/scripts/verify-service-bundle.sh index 509e13357..fe5455337 100755 --- a/scripts/verify-service-bundle.sh +++ b/scripts/verify-service-bundle.sh @@ -51,6 +51,11 @@ case "$SERVICE" in SERVICE_DIR="$ROOT/modules/$SERVICE" SERVICE_PROTOS="database.proto" ;; + embeddings) + PKG="@conduitplatform/embeddings" + SERVICE_DIR="$ROOT/modules/$SERVICE" + SERVICE_PROTOS="embeddings.proto" + ;; router) PKG="@conduitplatform/router" SERVICE_DIR="$ROOT/modules/$SERVICE" diff --git a/scripts/verify-standalone-bundle.sh b/scripts/verify-standalone-bundle.sh index dc8e83f3d..67ec7f911 100755 --- a/scripts/verify-standalone-bundle.sh +++ b/scripts/verify-standalone-bundle.sh @@ -15,6 +15,7 @@ STANDALONE_SERVICES=( "modules/storage:@conduitplatform/storage" "modules/chat:@conduitplatform/chat" ) +# embeddings is not part of standalone v1; it ships as a separate opt-in image. TMP="$(mktemp -d)" trap 'cleanup_all' EXIT diff --git a/standalone.Dockerfile b/standalone.Dockerfile index 248c59beb..94ca75d56 100644 --- a/standalone.Dockerfile +++ b/standalone.Dockerfile @@ -16,7 +16,7 @@ COPY --from=conduit-base /app/packages/core/bundle /app/packages/core/bundle COPY --from=conduit-base /app/packages/core/package.bundle.json /app/packages/core/package.json COPY --from=conduit-base /app/packages/core/package.bundle-lock.json /app/packages/core/package-lock.json -# Modules (standalone PM2 set — functions excluded) +# Modules (standalone PM2 set — functions and embeddings excluded from v1) COPY --from=conduit-base /app/modules/database/bundle /app/modules/database/bundle COPY --from=conduit-base /app/modules/database/package.bundle.json /app/modules/database/package.json COPY --from=conduit-base /app/modules/database/package.bundle-lock.json /app/modules/database/package-lock.json