feat(embeddings): add cross-database vector search

.github/CONTRIBUTING.md

@@ -95,6 +95,7 @@ You may find additional scripts in the `scripts` section of the `package.json` f
     - `authorization`: resource authorization based on Google Zanzibar
     - `chat`: chat room functionality
     - `database`: database engine (MongoDB, PostgreSQL), CMS, CRUD/functional endpoint generation
+    - `embeddings`: opt-in text-to-vector generation and semantic search (disabled by default; not in standalone v1)
     - `email`: email sending with templates support
     - `forms`: form generation and submission
     - `push-notifications`: provides support for push notifications

.github/workflows/embeddings-test.yml

@@ -0,0 +1,137 @@
+name: Embeddings offline tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'modules/embeddings/**'
+      - 'modules/database/**'
+      - 'libraries/hermes/**'
+      - 'libraries/grpc-sdk/**'
+      - 'libraries/module-tools/**'
+      - 'packages/core/**'
+      - 'docker/**'
+      - 'deploy/**'
+      - 'scripts/resolve-docker-targets.mjs'
+      - '.github/workflows/embeddings-test.yml'
+  push:
+    branches:
+      - main
+    paths:
+      - 'modules/embeddings/**'
+      - 'modules/database/**'
+      - 'libraries/hermes/**'
+      - 'libraries/grpc-sdk/**'
+      - 'libraries/module-tools/**'
+      - 'packages/core/**'
+      - 'docker/**'
+      - 'deploy/**'
+      - 'scripts/resolve-docker-targets.mjs'
+      - '.github/workflows/embeddings-test.yml'
+
+permissions:
+  contents: read
+  pull-requests: read
+
+jobs:
+  test:
+    runs-on: ubuntu-24.04
+    name: Embeddings unit and contract tests
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - uses: pnpm/action-setup@v6
+        with:
+          version: 11.5.0
+
+      - uses: actions/setup-node@v7
+        with:
+          node-version: 24
+          cache: pnpm
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          version: '29.x'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile --ignore-scripts
+
+      - name: Build embeddings, database, hermes, grpc-sdk, module-tools, and Core tsc
+        run: >
+          pnpm exec turbo run build
+          --filter=@conduitplatform/embeddings...
+          --filter=@conduitplatform/database...
+          --filter=@conduitplatform/hermes...
+          --filter=@conduitplatform/grpc-sdk...
+          --filter=@conduitplatform/module-tools...
+          --filter=@conduitplatform/core
+
+      # Core `build:bundle` still aliases @conduitplatform/node-2fa to its
+      # TypeScript source because the library's CommonJS dist breaks ESM
+      # bundling. That pre-existing packaging issue is out of this embeddings
+      # offline gate; this job only typechecks Core (`tsc`), it does not run
+      # authentication/Core service-bundle verification.
+
+      - name: Run embeddings offline tests
+        run: pnpm --filter @conduitplatform/embeddings test
+
+      - name: Run database vector and mutation offline tests
+        run: pnpm --filter @conduitplatform/database test --testPathIgnorePatterns=integration
+
+      - name: Run hermes vector offline tests
+        run: pnpm --filter @conduitplatform/hermes test
+
+  deploy-contracts:
+    runs-on: ubuntu-24.04
+    name: Compose render and target discovery
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - uses: actions/setup-node@v7
+        with:
+          node-version: 24
+
+      - name: Render compose with embeddings profile
+        working-directory: docker
+        env:
+          GRPC_KEY: ci-nonempty-grpc-key
+        run: |
+          docker compose --profile mongodb --profile embeddings config --services > /tmp/compose-embeddings-services.txt
+          grep -qx 'embeddings' /tmp/compose-embeddings-services.txt
+          docker compose --profile mongodb --profile embeddings config > /tmp/compose-embeddings.yml
+          grep -q 'ci-nonempty-grpc-key' /tmp/compose-embeddings.yml
+          grep -q '55165' /tmp/compose-embeddings.yml
+
+      - name: Render compose without embeddings profile
+        working-directory: docker
+        run: |
+          docker compose --profile mongodb config --services > /tmp/compose-default-services.txt
+          if grep -qx 'embeddings' /tmp/compose-default-services.txt; then
+            echo 'embeddings service must stay profile-gated' >&2
+            exit 1
+          fi
+
+      - name: Discover docker targets for embeddings-only changes
+        env:
+          CHANGED_FILES: modules/embeddings/src/index.ts
+        run: |
+          env -u GITHUB_OUTPUT node scripts/resolve-docker-targets.mjs > /tmp/targets.json
+          node -e '
+            const fs = require("fs");
+            const data = JSON.parse(fs.readFileSync("/tmp/targets.json", "utf8"));
+            const matrix = JSON.parse(data.matrix);
+            const targets = matrix.include.map((row) => row.target);
+            if (!targets.includes("embeddings")) {
+              throw new Error("expected embeddings target, got " + targets.join(","));
+            }
+            if (targets.includes("conduit-standalone")) {
+              throw new Error("embeddings-only changes must not select standalone: " + targets.join(","));
+            }
+          '
+
+      - name: Run deployment contract tests
+        run: node --test modules/embeddings/test/deployment-contract.test.mjs

.github/workflows/service-bundle-verify.yml

@@ -11,6 +11,7 @@ on:
       - 'modules/authorization/**'
       - 'modules/communications/**'
       - 'modules/database/**'
+      - 'modules/embeddings/**'
       - 'modules/router/**'
       - 'packages/core/**'
       - 'libraries/service-bundle/**'
@@ -39,6 +40,7 @@ on:
       - 'modules/authorization/**'
       - 'modules/communications/**'
       - 'modules/database/**'
+      - 'modules/embeddings/**'
       - 'modules/router/**'
       - 'packages/core/**'
       - 'libraries/service-bundle/**'
@@ -67,7 +69,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        service: [chat, functions, storage, authentication, authorization, communications, database, router, core]
+        service: [chat, functions, storage, authentication, authorization, communications, database, embeddings, router, core]
     name: Verify ${{ matrix.service }} bundle
     steps:
       - name: Checkout

Dockerfile

@@ -28,6 +28,8 @@ RUN pnpm install --frozen-lockfile --ignore-scripts && \
 
 # Compile first, then bundle. Router/authentication turbo branches must not skip
 # build:bundle or COPY --from=conduit-base .../bundle fails in image CI.
+# Standalone v1 (empty BUILDING_SERVICE) bundles core + database/router/authentication/
+# authorization/communications/storage/chat only. Embeddings is a separate image.
 RUN pnpm --filter @conduitplatform/service-bundle run build && \
     if [ -z "$BUILDING_SERVICE" ] ; then npx turbo run build ; \
     elif [ "$BUILDING_SERVICE" = "conduit" ] ; then npx turbo run build --filter=@conduitplatform/core --filter=@conduitplatform/hermes \

README.md

@@ -59,6 +59,7 @@ Can't find what you're interested in? Shoot us a message [on Discord](https://di
 - [Chat](https://getconduit.dev/docs/modules/chat) - Build realtime chat applications.
 - [Database](https://getconduit.dev/docs/modules/database) - Create schemas with auto-generated CRUD and Query-based functional endpoints. Supports MongoDB and PostgreSQL.
 - [Email](https://getconduit.dev/docs/modules/email) - Send emails using multiple supported providers.
+- [Embeddings](modules/embeddings) - Opt-in text-to-vector generation and semantic search. Disabled by default; not included in standalone v1. See the [rollout runbook](deploy/embeddings.md).
 - [Forms](https://getconduit.dev/docs/modules/forms) - Submit forms and have responses forwarded to an email address.
 - [PushNotifications](https://getconduit.dev/docs/modules/push-notifications) - Send push notifications to your users.
 - [Router](https://getconduit.dev/docs/modules/router) - Seamlessly expose REST, GraphQL and WebSockets APIs with auto-generated endpoint documentation.

deploy/docker/README.md

@@ -29,3 +29,8 @@ To run the microservices version:
 - Open the admin panel in your browser at [http://localhost:8080](http://localhost:8080)
 - Open the router in your browser at [http://localhost:8081](http://localhost:8081)
 - You can inject `--profile {profile_name}` command on compose to configure more services
+  (`mongodb` / `postgres` for the database engine, `embeddings` for the embeddings module).
+  Embeddings stays omitted until you pass `--profile embeddings` **and** export a
+  non-empty `GRPC_KEY`. The embeddings image is not published until a compatible
+  release tag exists. Helm workload `install.embeddings.enabled` is separate from
+  module convict `enabled`. See the [embeddings rollout runbook](../embeddings.md).

deploy/embeddings.md

@@ -0,0 +1,91 @@
+# Embeddings rollout and rollback
+
+Embeddings is a separate, disabled-by-default module image. It is not part of
+standalone v1. No embeddings image is published until a compatible release tag
+exists; do not enable the compose profile or Helm workload against `latest`
+until that tag is published. Live MongoDB Atlas, pgvector, Redis, and provider
+suites are **not** covered by CI; operators must complete the capability and
+index readiness checks below before enabling generation or search.
+
+Production containers set `NODE_ENV=production` and **require a non-empty
+`GRPC_KEY`**. Two independent enablement flags exist:
+
+- Helm workload `install.embeddings.enabled` (charts repo, default `false`)
+  only deploys or removes the embeddings process. It does not start workers.
+- Module convict `enabled` (Core config, default `false`) turns on embedding
+  generation workers, mutation subscriptions, and search. Keep this `false`
+  until peer health and vector capabilities are confirmed.
+
+## Compose (opt-in)
+
+```bash
+# A non-empty GRPC_KEY is required; empty values fail in production.
+export GRPC_KEY='replace-with-a-non-empty-key'
+docker compose --profile mongodb --profile embeddings up
+```
+
+- gRPC: `${EMBEDDINGS_GRPC_PORT:-55165}` (container `GRPC_PORT` uses the same value)
+- Metrics: `9192` (Prometheus scrapes `conduit-embeddings:9192`)
+- Image name after a compatible release: `docker.io/conduitplatform/embeddings:<published-tag>`.
+  Compose interpolates `${IMAGE_TAG}`; that tag is not published by this change.
+
+Helm workload `install.embeddings.enabled` is documented in the charts
+repository and remains `false` by default. Setting it to `true` deploys the
+pod with module convict `enabled` still false.
+
+## Rollout order
+
+1. Publish compatible Core, Database, grpc-sdk, **and** the embeddings image
+   tag you will run. Do not start the workload before that tag exists.
+2. Deploy the embeddings **workload** with convict `enabled: false`
+   (`install.embeddings.enabled=true` in Helm, or the compose embeddings
+   profile with a non-empty `GRPC_KEY`). Confirm the process is serving and
+   waiting on / registered with Core. Health stays serving while workers are
+   disabled so operators can configure the module.
+3. Confirm `GRPC_KEY` is set and gRPC peer health is good.
+4. Call `GET /embeddings/capabilities` (or gRPC `getCapabilities`) and verify
+   Database `getVectorCapabilities`: storage, indexing, and search must be
+   true for the target backend (MongoDB Atlas Vector Search or Postgres
+   pgvector). Saving a disabled config may succeed with capability warnings;
+   activation must not.
+5. Configure the HTTPS provider (`endpoint`, `apiKey`, and model catalogue).
+   `GRPC_KEY` is supplied by the deployment (`NODE_ENV=production`), not by
+   module settings. Check `GET /embeddings/status` for provider/index warnings.
+6. Create an embedding config. The first upsert provisions the vector index
+   when Database indexing is available. The config stays disabled until the
+   index for `targetField` is queryable (`status` ready, not pending/failed).
+   If indexing is unavailable, status reports a manual lifecycle warning and
+   the operator must create the index before enabling.
+7. Enable the config only after index readiness. Start a **bounded** backfill
+   (`onlyMissing` recommended). Watch `GET /embeddings/backfills/:id` and
+   `GET /embeddings/status` queue counts. Do not scan collections in the
+   request thread; backfills are queued.
+8. Run a scoped canary semantic search (`POST /embeddings/search` as an
+   operator, or client search with authenticated user/scope). Confirm
+   fail-closed behavior on authorization-enabled schemas.
+9. Enable workers/search for normal traffic (module convict `enabled: true`
+   through Core config). This is not `install.embeddings.enabled`.
+
+## Rollback
+
+1. Disable workers and embedding configs (module convict `enabled: false`
+   and per-config `enabled: false`). Generation and search stop; existing
+   vectors remain.
+2. Scale down or stop the embeddings **workload**:
+   - Compose: omit `--profile embeddings` / `docker compose stop embeddings`
+   - Helm: `install.embeddings.enabled=false` (charts repo). This is the
+     workload flag, not module convict `enabled`.
+3. Roll back the embeddings image and/or chart to the previous **published**
+   version, if any.
+4. Rollback **retains** vector fields, indexes, `EmbeddingConfig` documents,
+   `BackfillRun` records, and Redis/BullMQ queue state. Data and index
+   removal is a separate explicit operator action.
+
+## Residual validation
+
+Offline CI covers unit/contract tests, bundle smoke (`Waiting for Core`),
+image target discovery, and compose rendering. It does not prove Atlas,
+pgvector, Redis queue behavior, a live provider, or a published embeddings
+image. Repeat capability and index readiness checks in the target
+environment before activation. The remaining release prerequisite is
+publishing the first compatible embeddings image tag.

deploy/k8s/README.md

@@ -6,3 +6,11 @@ We've included some basic instructions to get you started on a local k8s cluster
 Current setup includes:
 - [Minikube](minikube.md)
 - [AKS](aks.md)
+
+Embeddings is not part of the standalone image. Helm workload
+`install.embeddings.enabled` (charts repo, default `false`) deploys the
+process; module convict `enabled` is a separate Core config switch. For a
+disabled-by-default embeddings rollout, capability/index readiness, and
+rollback (`install.embeddings.enabled=false`, retained vector/index/config/
+Redis state), see [embeddings.md](../embeddings.md). No embeddings image is
+published until a compatible release tag exists.

docker-bake.hcl

@@ -159,6 +159,24 @@ target "database" {
   }
 }
 
+target "conduit-base-bundle-embeddings" {
+  inherits = ["conduit-base"]
+  args = {
+    BUILDING_SERVICE = "modules/embeddings"
+    BUILD_BUNDLE     = "1"
+  }
+}
+
+target "embeddings" {
+  inherits = ["_runtime"]
+  context    = "modules/embeddings"
+  dockerfile = "Dockerfile"
+  contexts = {
+    conduit-base    = "target:conduit-base-bundle-embeddings"
+    conduit-builder = "target:conduit-builder"
+  }
+}
+
 target "functions" {
   inherits = ["_runtime"]
   context    = "modules/functions"
@@ -230,6 +248,7 @@ group "all" {
     "chat",
     "communications",
     "database",
+    "embeddings",
     "functions",
     "router",
     "storage",

docker/.env

@@ -10,6 +10,7 @@ AUTHN_GRPC_PORT="55162"
 AUTHZ_GRPC_PORT="55169"
 CHAT_GRPC_PORT="55163"
 COMMS_GRPC_PORT="55164"
+EMBEDDINGS_GRPC_PORT="55165"
 STORAGE_GRPC_PORT="55168"
 
 
@@ -30,6 +31,8 @@ DB_CONN_URI="mongodb://conduit:pass@conduit-mongo:27017/conduit?authSource=admin
 #DB_CONN_URI="postgres://conduit:pass@conduit-postgres:5432/conduit"               # profile: postgres
 
 # Security
+# Leave GRPC_KEY empty for default profiles. Enabling '--profile embeddings'
+# requires exporting a non-empty GRPC_KEY; production images refuse empty values.
 CORE_MASTER_KEY="M4ST3RK3Y"
 GRPC_KEY=""
 

docker/docker-compose.standalone.yml

@@ -2,6 +2,10 @@
 # This compose file deploys a "standalone" version of conduit with most modules
 # packaged in a single image. Loki and Prometheus are not deployed, since
 # metrics and logs can be viewed directly from the Docker daemon.
+# Embeddings is not included in standalone v1. After a compatible embeddings
+# image is published, use docker-compose.yml with a non-empty GRPC_KEY:
+#   export GRPC_KEY='replace-with-a-non-empty-key'
+#   docker compose --profile embeddings up
 #-------------------------------------------------------------------------------------------
 
 version: '3.9'