From 37e1840748eb05406b8103daeb043f2b928739d1 Mon Sep 17 00:00:00 2001 From: Diogo Esteves Date: Fri, 16 Oct 2026 10:39:06 +0100 Subject: [PATCH 1/2] fix(cli): embed at 512 dimensions by default, and document embeddings for the Ask box ondocs index with OPENAI_API_KEY and no --embedding built text-embedding-3-small at its full 1,536 dimensions, while the plugins build it at 512 and the docs assume 512. The README's Next.js line and the commands init prints pass no --dimensions, so they shipped the larger index: 3,073,665 bytes against 1,614,214 for docusaurus.io's 1,070 chunks. The CLI's default model, with OpenAI or through AI Gateway, is now built at 512 unless --dimensions gives a size; a model named with --embedding keeps its own size, as in the plugins. An index built at 1,536 keeps working, since the endpoint reads the size from it, and the next build embeds every chunk again once. A plugin build without a model key now adds a second warning line: the Ask box then answers only questions that use enough of the docs' own words, and the key belongs where the site builds as well as where the endpoint runs. The first line stays as troubleshooting.md quotes it. Over a mock index, a similarity counts only from a chunk that shares two of the question's words, not one. On docusaurus.io's docs, 15 off-topic Stack Overflow titles got 6 quoted answers instead of 10; on the ondocs docs, 4 instead of 7. similarityNeedsKeyword takes the number of words; true is still one. The docs present embeddings as the default for the Ask box (OpenAI with a key, Workers AI's BGE-small for the Worker), keyword-only as what the MCP endpoint uses and the no-key fallback, and say that mock mode's gate is permissive. The Worker template's README embeds with Workers AI in its commands and plugin snippets. --- .changeset/embedding-defaults.md | 11 ++ README.md | 4 +- examples/docusaurus/docs/cli.md | 4 +- examples/docusaurus/docs/deployment.md | 2 +- examples/docusaurus/docs/getting-started.md | 2 + examples/docusaurus/docs/local-development.md | 2 +- examples/docusaurus/docs/retrieval.md | 4 +- examples/docusaurus/docs/workers-ai.md | 2 +- examples/nextjs/content/docs/cli.md | 4 +- examples/nextjs/content/docs/deployment.md | 2 +- .../nextjs/content/docs/getting-started.md | 2 + .../nextjs/content/docs/local-development.md | 2 +- examples/nextjs/content/docs/retrieval.md | 4 +- examples/nextjs/content/docs/workers-ai.md | 2 +- .../docs/get-started/getting-started.md | 2 + .../docs/get-started/local-development.md | 2 +- .../src/content/docs/guides/deployment.md | 2 +- .../src/content/docs/guides/retrieval.md | 4 +- .../src/content/docs/guides/workers-ai.md | 2 +- .../src/content/docs/reference/cli.md | 4 +- examples/vitepress/docs/cli.md | 4 +- examples/vitepress/docs/deployment.md | 2 +- examples/vitepress/docs/getting-started.md | 2 + examples/vitepress/docs/local-development.md | 2 +- examples/vitepress/docs/retrieval.md | 4 +- examples/vitepress/docs/workers-ai.md | 2 +- src/cli/main.ts | 14 +- src/integration/build.ts | 8 + src/mock/index.ts | 7 +- src/search/retrieve.ts | 59 +++++-- templates/cloudflare-worker/README.md | 8 +- test/embedding-defaults.test.ts | 148 ++++++++++++++++++ test/mock-gate.test.ts | 79 ++++++++++ 33 files changed, 351 insertions(+), 51 deletions(-) create mode 100644 .changeset/embedding-defaults.md create mode 100644 test/embedding-defaults.test.ts create mode 100644 test/mock-gate.test.ts diff --git a/.changeset/embedding-defaults.md b/.changeset/embedding-defaults.md new file mode 100644 index 00000000..0655fab3 --- /dev/null +++ b/.changeset/embedding-defaults.md @@ -0,0 +1,11 @@ +--- +'ondocs': patch +--- + +`ondocs index` without `--embedding` now embeds with OpenAI's `text-embedding-3-small` at 512 dimensions, as the plugins do, with `OPENAI_API_KEY` or through AI Gateway. Before, it built the model's full 1,536, so a site that followed the README's Next.js line shipped an index about twice the size: 2.93 MB against 1.54 MB for docusaurus.io's 1,070 chunks. `--dimensions` sets another size, and a model you name with `--embedding` is still built at its own size unless `--dimensions` gives one. + +An index the CLI built at 1,536 dimensions needs nothing: the endpoints, `ondocs dev` and `ondocs eval` read the size from the index, and `ondocs index --check` without flags still passes on it. The next `ondocs index` embeds every chunk again, once, at 512 (as a first build does), and reuses those vectors after that. To keep the full size, pass `--dimensions 1536`; that build also embeds every chunk once. An endpoint that sets `dimensions` in `embeddingProviderOptions` by hand must set the size of the index it serves. + +A plugin build without a model key now also says what its keyword-only index means: the Ask box answers only questions that use enough of the docs' own words, the MCP endpoint `init` writes searches by keywords either way, and the key belongs where the site builds as well as where the endpoint runs. `embedding: 'none'` builds the same index without the warning. + +Over a mock index (`-e mock`), a similarity now counts only from a chunk that shares two of the question's words, not one: a single common word in a short section let off-topic questions through. On docusaurus.io's docs, 15 off-topic Stack Overflow titles about git, Bash, SQL and Java got 6 quoted answers instead of 10, as many as from a keyword-only index; on the ondocs docs, 4 instead of 7. Of the 22 answerable questions in the sample eval of docusaurus.io's docs, one more is refused, one that had been answered from a page that does not answer it, and the right page is in the top three for 17 instead of 16. `retrieval.similarityNeedsKeyword` now also takes the number of words: `true` is one, as before. diff --git a/README.md b/README.md index 25f15f7f..01fc76d2 100644 --- a/README.md +++ b/README.md @@ -120,8 +120,8 @@ import 'ondocs/embed/launcher.css'; **Any other site.** Index the HTML it builds, write the endpoint with `npx ondocs init`, and add the script tag: ```sh -npx ondocs index public -o public/ask-index.json -e none # keyword-only; leave out -e none to embed with OPENAI_API_KEY -npx ondocs init --out public # plain HTML needs --out; Hugo, Jekyll, MkDocs and Eleventy are detected +npx ondocs index public -o public/ask-index.json # embeds with OPENAI_API_KEY, best for the Ask box; -e none for keyword-only +npx ondocs init --out public # plain HTML needs --out; Hugo, Jekyll, MkDocs and Eleventy are detected ``` On a GitHub Pages project site, at `https://user.github.io/repo`, add `--base-url /repo` to the index command. diff --git a/examples/docusaurus/docs/cli.md b/examples/docusaurus/docs/cli.md index 42e3a395..9b7d84b7 100644 --- a/examples/docusaurus/docs/cli.md +++ b/examples/docusaurus/docs/cli.md @@ -16,7 +16,7 @@ The `ondocs` command has four subcommands: `index` builds or checks the index, ` | `--check` | off | Exit 1 if the index is stale. Never calls a model. | | `--base-url ` | `/` | URL prefix for pages, such as `/docs` or a full origin. | | `-e, --embedding ` | OpenAI when a key is set | `openai:`, `workers-ai:@cf/`, `/`, `mock[:]` or `none`. | -| `--dimensions ` | the model's | Vector size, for models that support it. With `--check`, the size the index must have. | +| `--dimensions ` | 512 for the default | Vector size, for models that support it. With `--check`, the size the index must have. | | `--chunk-size ` | `1200` | Maximum characters per chunk. | | `--chunk-overlap ` | `150` | Characters shared by consecutive chunks of a section. | | `--ignore ` | none | Skip matching files. Repeatable. | @@ -32,7 +32,7 @@ The `ondocs` command has four subcommands: `index` builds or checks the index, ` ## Choosing the embedding model -Without `--embedding`, the CLI uses `openai:text-embedding-3-small` when `OPENAI_API_KEY` is set, or `openai/text-embedding-3-small` through AI Gateway when only `AI_GATEWAY_API_KEY` is. With neither, it fails rather than quietly building a keyword-only index, and suggests `-e mock` to try it without a key. `-e none` builds a keyword-only index on purpose. `-e workers-ai:@cf/baai/bge-small-en-v1.5` embeds with Workers AI over Cloudflare's REST API, with `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`, for the [Workers AI Worker](./workers-ai.md); its vector size is fixed, so `--dimensions` does not apply. +Without `--embedding`, the CLI uses `openai:text-embedding-3-small` when `OPENAI_API_KEY` is set, or `openai/text-embedding-3-small` through AI Gateway when only `AI_GATEWAY_API_KEY` is, at 512 dimensions, as the plugins do. That is a third of the model's 1,536, and makes the index about half the size: 1.54 MB against 2.93 MB for docusaurus.io's 1,070 chunks. `--dimensions` sets another size. A model you name with `--embedding` is built at its own size unless `--dimensions` gives one. The endpoint reads the size from the index, so an index built at 1,536 dimensions keeps working; the next build embeds every chunk again, once, at the new size. With neither key, the CLI fails rather than quietly building a keyword-only index, and suggests `-e mock` to try it without a key. `-e none` builds a keyword-only index on purpose. It suits agents' MCP search, which `init` writes keyword-only, but the Ask box then answers only questions that use enough of the docs' own words: embed the index for it ([How retrieval works](./retrieval.md#tuning)). `-e workers-ai:@cf/baai/bge-small-en-v1.5` embeds with Workers AI over Cloudflare's REST API, with `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`, for the [Workers AI Worker](./workers-ai.md); its vector size is fixed, so `--dimensions` does not apply. ## Environment and exit codes diff --git a/examples/docusaurus/docs/deployment.md b/examples/docusaurus/docs/deployment.md index da3add75..33a09a29 100644 --- a/examples/docusaurus/docs/deployment.md +++ b/examples/docusaurus/docs/deployment.md @@ -168,7 +168,7 @@ For a static site on Cloudflare Workers, `init` writes `worker/ondocs.ts`, which ## Environment variables -Set your provider's key, such as `OPENAI_API_KEY`, in two places: in the build, if the build embeds the index, and in the function's runtime environment, which embeds questions and calls the model. `init` prints where each goes on your host. The key never reaches the browser: the dialog only talks to your endpoint. A plugin's build without the key writes a keyword-only index, which the endpoint answers from without embedding questions; `ondocs index` writes one with `-e none`. The endpoint still needs the key, because a model writes every answer: without it, the dialog shows "Answers aren’t set up on this site yet." for every question it does not refuse. See [Troubleshooting](./troubleshooting.md#answers-arent-set-up-on-this-site-yet), which also covers the other errors of a first deploy. +Set your provider's key, such as `OPENAI_API_KEY`, in two places: where the site or the index builds, which embeds the pages, and in the function's runtime environment, which embeds questions and calls the model. `init` prints where each goes on your host. The key never reaches the browser: the dialog only talks to your endpoint. Set it in the build too, not only where the endpoint runs: a plugin's build without the key writes a keyword-only index, and says so in a warning (`ondocs index` writes one only with `-e none`). The endpoint answers from it without embedding questions, so the Ask box answers only questions that use enough of the docs' own words, even with the key set where it runs. The MCP endpoint `init` writes searches by keywords either way. The endpoint still needs the key, because a model writes every answer: without it, the dialog shows "Answers aren’t set up on this site yet." for every question it does not refuse. See [Troubleshooting](./troubleshooting.md#answers-arent-set-up-on-this-site-yet), which also covers the other errors of a first deploy. ## Cold starts diff --git a/examples/docusaurus/docs/getting-started.md b/examples/docusaurus/docs/getting-started.md index 0a7019e2..85de9f35 100644 --- a/examples/docusaurus/docs/getting-started.md +++ b/examples/docusaurus/docs/getting-started.md @@ -34,6 +34,8 @@ npx ondocs index ./docs --base-url /docs \ -e openai:text-embedding-3-small --dimensions 512 ``` +Embedding the chunks lets the Ask box match a question by meaning as well as by its words, so it is the default: with `OPENAI_API_KEY` set, the CLI and the plugins embed with `text-embedding-3-small` at 512 dimensions, as above, and a site answered by the [Workers AI Worker](./workers-ai.md) embeds with `-e workers-ai:@cf/baai/bge-small-en-v1.5`. Without a key, `-e none` builds a keyword-only index. It suits agents' MCP search, which `init` writes keyword-only, but the Ask box then answers only questions that use enough of the docs' own words. + With the Docusaurus, Astro, Starlight or VitePress plugin you skip this step: the plugin indexes the built site after every build. ## Add the endpoint diff --git a/examples/docusaurus/docs/local-development.md b/examples/docusaurus/docs/local-development.md index 8ff11e65..386326e8 100644 --- a/examples/docusaurus/docs/local-development.md +++ b/examples/docusaurus/docs/local-development.md @@ -32,7 +32,7 @@ With React, render `` | `/` | URL prefix for pages, such as `/docs` or a full origin. | | `-e, --embedding ` | OpenAI when a key is set | `openai:`, `workers-ai:@cf/`, `/`, `mock[:]` or `none`. | -| `--dimensions ` | the model's | Vector size, for models that support it. With `--check`, the size the index must have. | +| `--dimensions ` | 512 for the default | Vector size, for models that support it. With `--check`, the size the index must have. | | `--chunk-size ` | `1200` | Maximum characters per chunk. | | `--chunk-overlap ` | `150` | Characters shared by consecutive chunks of a section. | | `--ignore ` | none | Skip matching files. Repeatable. | @@ -34,7 +34,7 @@ The `ondocs` command has four subcommands: `index` builds or checks the index, ` ## Choosing the embedding model -Without `--embedding`, the CLI uses `openai:text-embedding-3-small` when `OPENAI_API_KEY` is set, or `openai/text-embedding-3-small` through AI Gateway when only `AI_GATEWAY_API_KEY` is. With neither, it fails rather than quietly building a keyword-only index, and suggests `-e mock` to try it without a key. `-e none` builds a keyword-only index on purpose. `-e workers-ai:@cf/baai/bge-small-en-v1.5` embeds with Workers AI over Cloudflare's REST API, with `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`, for the [Workers AI Worker](/docs/workers-ai); its vector size is fixed, so `--dimensions` does not apply. +Without `--embedding`, the CLI uses `openai:text-embedding-3-small` when `OPENAI_API_KEY` is set, or `openai/text-embedding-3-small` through AI Gateway when only `AI_GATEWAY_API_KEY` is, at 512 dimensions, as the plugins do. That is a third of the model's 1,536, and makes the index about half the size: 1.54 MB against 2.93 MB for docusaurus.io's 1,070 chunks. `--dimensions` sets another size. A model you name with `--embedding` is built at its own size unless `--dimensions` gives one. The endpoint reads the size from the index, so an index built at 1,536 dimensions keeps working; the next build embeds every chunk again, once, at the new size. With neither key, the CLI fails rather than quietly building a keyword-only index, and suggests `-e mock` to try it without a key. `-e none` builds a keyword-only index on purpose. It suits agents' MCP search, which `init` writes keyword-only, but the Ask box then answers only questions that use enough of the docs' own words: embed the index for it ([How retrieval works](/docs/retrieval#tuning)). `-e workers-ai:@cf/baai/bge-small-en-v1.5` embeds with Workers AI over Cloudflare's REST API, with `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`, for the [Workers AI Worker](/docs/workers-ai); its vector size is fixed, so `--dimensions` does not apply. ## Environment and exit codes diff --git a/examples/nextjs/content/docs/deployment.md b/examples/nextjs/content/docs/deployment.md index 1a83ca00..efd11f4c 100644 --- a/examples/nextjs/content/docs/deployment.md +++ b/examples/nextjs/content/docs/deployment.md @@ -170,7 +170,7 @@ For a static site on Cloudflare Workers, `init` writes `worker/ondocs.ts`, which ## Environment variables -Set your provider's key, such as `OPENAI_API_KEY`, in two places: in the build, if the build embeds the index, and in the function's runtime environment, which embeds questions and calls the model. `init` prints where each goes on your host. The key never reaches the browser: the dialog only talks to your endpoint. A plugin's build without the key writes a keyword-only index, which the endpoint answers from without embedding questions; `ondocs index` writes one with `-e none`. The endpoint still needs the key, because a model writes every answer: without it, the dialog shows "Answers aren’t set up on this site yet." for every question it does not refuse. See [Troubleshooting](/docs/troubleshooting#answers-arent-set-up-on-this-site-yet), which also covers the other errors of a first deploy. +Set your provider's key, such as `OPENAI_API_KEY`, in two places: where the site or the index builds, which embeds the pages, and in the function's runtime environment, which embeds questions and calls the model. `init` prints where each goes on your host. The key never reaches the browser: the dialog only talks to your endpoint. Set it in the build too, not only where the endpoint runs: a plugin's build without the key writes a keyword-only index, and says so in a warning (`ondocs index` writes one only with `-e none`). The endpoint answers from it without embedding questions, so the Ask box answers only questions that use enough of the docs' own words, even with the key set where it runs. The MCP endpoint `init` writes searches by keywords either way. The endpoint still needs the key, because a model writes every answer: without it, the dialog shows "Answers aren’t set up on this site yet." for every question it does not refuse. See [Troubleshooting](/docs/troubleshooting#answers-arent-set-up-on-this-site-yet), which also covers the other errors of a first deploy. ## Cold starts diff --git a/examples/nextjs/content/docs/getting-started.md b/examples/nextjs/content/docs/getting-started.md index b5dbc784..1e81b626 100644 --- a/examples/nextjs/content/docs/getting-started.md +++ b/examples/nextjs/content/docs/getting-started.md @@ -35,6 +35,8 @@ npx ondocs index ./docs --base-url /docs \ -e openai:text-embedding-3-small --dimensions 512 ``` +Embedding the chunks lets the Ask box match a question by meaning as well as by its words, so it is the default: with `OPENAI_API_KEY` set, the CLI and the plugins embed with `text-embedding-3-small` at 512 dimensions, as above, and a site answered by the [Workers AI Worker](/docs/workers-ai) embeds with `-e workers-ai:@cf/baai/bge-small-en-v1.5`. Without a key, `-e none` builds a keyword-only index. It suits agents' MCP search, which `init` writes keyword-only, but the Ask box then answers only questions that use enough of the docs' own words. + With the Docusaurus, Astro, Starlight or VitePress plugin you skip this step: the plugin indexes the built site after every build. ## Add the endpoint diff --git a/examples/nextjs/content/docs/local-development.md b/examples/nextjs/content/docs/local-development.md index 81dc2d81..3efb4bd0 100644 --- a/examples/nextjs/content/docs/local-development.md +++ b/examples/nextjs/content/docs/local-development.md @@ -33,7 +33,7 @@ With React, render `` | `/` | URL prefix for pages, such as `/docs` or a full origin. | | `-e, --embedding ` | OpenAI when a key is set | `openai:`, `workers-ai:@cf/`, `/`, `mock[:]` or `none`. | -| `--dimensions ` | the model's | Vector size, for models that support it. With `--check`, the size the index must have. | +| `--dimensions ` | 512 for the default | Vector size, for models that support it. With `--check`, the size the index must have. | | `--chunk-size ` | `1200` | Maximum characters per chunk. | | `--chunk-overlap ` | `150` | Characters shared by consecutive chunks of a section. | | `--ignore ` | none | Skip matching files. Repeatable. | @@ -34,7 +34,7 @@ The `ondocs` command has four subcommands: `index` builds or checks the index, ` ## Choosing the embedding model -Without `--embedding`, the CLI uses `openai:text-embedding-3-small` when `OPENAI_API_KEY` is set, or `openai/text-embedding-3-small` through AI Gateway when only `AI_GATEWAY_API_KEY` is. With neither, it fails rather than quietly building a keyword-only index, and suggests `-e mock` to try it without a key. `-e none` builds a keyword-only index on purpose. `-e workers-ai:@cf/baai/bge-small-en-v1.5` embeds with Workers AI over Cloudflare's REST API, with `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`, for the [Workers AI Worker](/guides/workers-ai/); its vector size is fixed, so `--dimensions` does not apply. +Without `--embedding`, the CLI uses `openai:text-embedding-3-small` when `OPENAI_API_KEY` is set, or `openai/text-embedding-3-small` through AI Gateway when only `AI_GATEWAY_API_KEY` is, at 512 dimensions, as the plugins do. That is a third of the model's 1,536, and makes the index about half the size: 1.54 MB against 2.93 MB for docusaurus.io's 1,070 chunks. `--dimensions` sets another size. A model you name with `--embedding` is built at its own size unless `--dimensions` gives one. The endpoint reads the size from the index, so an index built at 1,536 dimensions keeps working; the next build embeds every chunk again, once, at the new size. With neither key, the CLI fails rather than quietly building a keyword-only index, and suggests `-e mock` to try it without a key. `-e none` builds a keyword-only index on purpose. It suits agents' MCP search, which `init` writes keyword-only, but the Ask box then answers only questions that use enough of the docs' own words: embed the index for it ([How retrieval works](/guides/retrieval/#tuning)). `-e workers-ai:@cf/baai/bge-small-en-v1.5` embeds with Workers AI over Cloudflare's REST API, with `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`, for the [Workers AI Worker](/guides/workers-ai/); its vector size is fixed, so `--dimensions` does not apply. ## Environment and exit codes diff --git a/examples/vitepress/docs/cli.md b/examples/vitepress/docs/cli.md index e02fd796..8e643b70 100644 --- a/examples/vitepress/docs/cli.md +++ b/examples/vitepress/docs/cli.md @@ -18,7 +18,7 @@ The `ondocs` command has four subcommands: `index` builds or checks the index, ` | `--check` | off | Exit 1 if the index is stale. Never calls a model. | | `--base-url ` | `/` | URL prefix for pages, such as `/docs` or a full origin. | | `-e, --embedding ` | OpenAI when a key is set | `openai:`, `workers-ai:@cf/`, `/`, `mock[:]` or `none`. | -| `--dimensions ` | the model's | Vector size, for models that support it. With `--check`, the size the index must have. | +| `--dimensions ` | 512 for the default | Vector size, for models that support it. With `--check`, the size the index must have. | | `--chunk-size ` | `1200` | Maximum characters per chunk. | | `--chunk-overlap ` | `150` | Characters shared by consecutive chunks of a section. | | `--ignore ` | none | Skip matching files. Repeatable. | @@ -34,7 +34,7 @@ The `ondocs` command has four subcommands: `index` builds or checks the index, ` ## Choosing the embedding model -Without `--embedding`, the CLI uses `openai:text-embedding-3-small` when `OPENAI_API_KEY` is set, or `openai/text-embedding-3-small` through AI Gateway when only `AI_GATEWAY_API_KEY` is. With neither, it fails rather than quietly building a keyword-only index, and suggests `-e mock` to try it without a key. `-e none` builds a keyword-only index on purpose. `-e workers-ai:@cf/baai/bge-small-en-v1.5` embeds with Workers AI over Cloudflare's REST API, with `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`, for the [Workers AI Worker](/workers-ai); its vector size is fixed, so `--dimensions` does not apply. +Without `--embedding`, the CLI uses `openai:text-embedding-3-small` when `OPENAI_API_KEY` is set, or `openai/text-embedding-3-small` through AI Gateway when only `AI_GATEWAY_API_KEY` is, at 512 dimensions, as the plugins do. That is a third of the model's 1,536, and makes the index about half the size: 1.54 MB against 2.93 MB for docusaurus.io's 1,070 chunks. `--dimensions` sets another size. A model you name with `--embedding` is built at its own size unless `--dimensions` gives one. The endpoint reads the size from the index, so an index built at 1,536 dimensions keeps working; the next build embeds every chunk again, once, at the new size. With neither key, the CLI fails rather than quietly building a keyword-only index, and suggests `-e mock` to try it without a key. `-e none` builds a keyword-only index on purpose. It suits agents' MCP search, which `init` writes keyword-only, but the Ask box then answers only questions that use enough of the docs' own words: embed the index for it ([How retrieval works](/retrieval#tuning)). `-e workers-ai:@cf/baai/bge-small-en-v1.5` embeds with Workers AI over Cloudflare's REST API, with `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`, for the [Workers AI Worker](/workers-ai); its vector size is fixed, so `--dimensions` does not apply. ## Environment and exit codes diff --git a/examples/vitepress/docs/deployment.md b/examples/vitepress/docs/deployment.md index ab4c06fd..25ad8b68 100644 --- a/examples/vitepress/docs/deployment.md +++ b/examples/vitepress/docs/deployment.md @@ -170,7 +170,7 @@ For a static site on Cloudflare Workers, `init` writes `worker/ondocs.ts`, which ## Environment variables -Set your provider's key, such as `OPENAI_API_KEY`, in two places: in the build, if the build embeds the index, and in the function's runtime environment, which embeds questions and calls the model. `init` prints where each goes on your host. The key never reaches the browser: the dialog only talks to your endpoint. A plugin's build without the key writes a keyword-only index, which the endpoint answers from without embedding questions; `ondocs index` writes one with `-e none`. The endpoint still needs the key, because a model writes every answer: without it, the dialog shows "Answers aren’t set up on this site yet." for every question it does not refuse. See [Troubleshooting](/troubleshooting#answers-arent-set-up-on-this-site-yet), which also covers the other errors of a first deploy. +Set your provider's key, such as `OPENAI_API_KEY`, in two places: where the site or the index builds, which embeds the pages, and in the function's runtime environment, which embeds questions and calls the model. `init` prints where each goes on your host. The key never reaches the browser: the dialog only talks to your endpoint. Set it in the build too, not only where the endpoint runs: a plugin's build without the key writes a keyword-only index, and says so in a warning (`ondocs index` writes one only with `-e none`). The endpoint answers from it without embedding questions, so the Ask box answers only questions that use enough of the docs' own words, even with the key set where it runs. The MCP endpoint `init` writes searches by keywords either way. The endpoint still needs the key, because a model writes every answer: without it, the dialog shows "Answers aren’t set up on this site yet." for every question it does not refuse. See [Troubleshooting](/troubleshooting#answers-arent-set-up-on-this-site-yet), which also covers the other errors of a first deploy. ## Cold starts diff --git a/examples/vitepress/docs/getting-started.md b/examples/vitepress/docs/getting-started.md index c08f2c40..26463f52 100644 --- a/examples/vitepress/docs/getting-started.md +++ b/examples/vitepress/docs/getting-started.md @@ -36,6 +36,8 @@ npx ondocs index ./docs --base-url /docs \ -e openai:text-embedding-3-small --dimensions 512 ``` +Embedding the chunks lets the Ask box match a question by meaning as well as by its words, so it is the default: with `OPENAI_API_KEY` set, the CLI and the plugins embed with `text-embedding-3-small` at 512 dimensions, as above, and a site answered by the [Workers AI Worker](/workers-ai) embeds with `-e workers-ai:@cf/baai/bge-small-en-v1.5`. Without a key, `-e none` builds a keyword-only index. It suits agents' MCP search, which `init` writes keyword-only, but the Ask box then answers only questions that use enough of the docs' own words. + With the Docusaurus, Astro, Starlight or VitePress plugin you skip this step: the plugin indexes the built site after every build. ## Add the endpoint diff --git a/examples/vitepress/docs/local-development.md b/examples/vitepress/docs/local-development.md index 3fc06594..5e9b5bfe 100644 --- a/examples/vitepress/docs/local-development.md +++ b/examples/vitepress/docs/local-development.md @@ -34,7 +34,7 @@ With React, render `/ AI Gateway, needs AI_GATEWAY_API_KEY mock[:] deterministic, offline none keyword-only index - (default: openai:text-embedding-3-small if OPENAI_API_KEY is set) - --dimensions Vector size, for models that support it (e.g. 512). + (default: openai:text-embedding-3-small at 512 dimensions, + if OPENAI_API_KEY is set) + --dimensions Vector size, for models that support it (default: 512 for + the default model, the model's own for one you name). With --check, the size the index must have. --chunk-size Max characters per chunk (default: ${String(DEFAULT_CHUNKING.maxChars)}) --chunk-overlap Characters shared by consecutive chunks (default: ${String(DEFAULT_CHUNKING.overlap)}) @@ -186,9 +189,12 @@ async function chooseEmbedding( }; } if (mode === 'check') return { model: undefined }; - if (io.env.OPENAI_API_KEY) return resolveSpec('openai:text-embedding-3-small', dims, io, mode); + // The default model at the plugins' default size, a third of its own, unless --dimensions + // gives one. A model named with --embedding keeps its own size, as in the plugins. + const size = dims ?? DEFAULT_DIMENSIONS; + if (io.env.OPENAI_API_KEY) return resolveSpec('openai:text-embedding-3-small', size, io, mode); if (io.env.AI_GATEWAY_API_KEY) - return resolveSpec('openai/text-embedding-3-small', dims, io, mode); + return resolveSpec('openai/text-embedding-3-small', size, io, mode); throw new UsageError( 'No embedding model. Set OPENAI_API_KEY or pass --embedding . To try it without a ' + 'key, use --embedding mock (offline, deterministic); --embedding none builds a ' + diff --git a/src/integration/build.ts b/src/integration/build.ts index b9f5bb50..cd9afe8c 100644 --- a/src/integration/build.ts +++ b/src/integration/build.ts @@ -124,6 +124,14 @@ export async function buildEmbedding( 'No embedding model: building a keyword-only index. Set OPENAI_API_KEY or ' + 'AI_GATEWAY_API_KEY at build time, or pass `embedding`, for semantic search.', ); + // What that means for the Ask box: an endpoint with the key embeds questions, not pages. + log.warn( + 'The Ask box then answers only questions that use enough of the docs’ own words, and ' + + 'refuses more of those asked in other words; the MCP endpoint `ondocs init` writes ' + + 'searches by keywords either way. Set the key where the site builds, not only where ' + + "the endpoint runs, or pass `embedding: 'none'` to build a keyword-only index without " + + 'this warning.', + ); return { model: null }; } spec = OPENAI_API_KEY ? `openai:${DEFAULT_MODEL}` : `openai/${DEFAULT_MODEL}`; diff --git a/src/mock/index.ts b/src/mock/index.ts index dc7e455a..4e0d3d37 100644 --- a/src/mock/index.ts +++ b/src/mock/index.ts @@ -21,9 +21,10 @@ import { tokenize } from '../text/tokenize'; * A `minSimilarity` that suits {@link mockEmbeddingModel} at its default size. Chance similarity * between hashed vectors has a standard deviation of about 1/sqrt(dimensions), 0.044 at 512, so * 0.2 sits well clear of noise. Most relevant matches pass on keyword coverage instead. A short - * question can still collide with a short chunk ("France" and "npm" share a bucket at 512), so - * retrieval over a mock index only counts similarity from chunks that share a word with the - * question (`similarityNeedsKeyword`). + * question can still collide with a short chunk ("France" and "npm" share a bucket at 512), and + * one common word, such as "git" or "file", makes a short section similar enough, so retrieval + * over a mock index only counts similarity from chunks that share two of the question's words + * (`similarityNeedsKeyword`). */ export const MOCK_MIN_SIMILARITY = 0.2; diff --git a/src/search/retrieve.ts b/src/search/retrieve.ts index 9122addd..54cee7f5 100644 --- a/src/search/retrieve.ts +++ b/src/search/retrieve.ts @@ -60,12 +60,13 @@ export interface RetrievalOptions { */ minKeywordCoverage?: number; /** - * Whether a chunk that clears `minSimilarity` must also share a word with the question to - * count. Default: `true` for an index built with `mockEmbeddingModel`, whose similarity comes - * from shared words, so without one it is a hash collision; `false` otherwise, since real - * embeddings match meaning without shared words. + * How many of the question's words a chunk that clears `minSimilarity` must also contain to + * count: `true` for one, a number for that many, `false` for none. Default: 2 for an index built + * with `mockEmbeddingModel`, whose similarity comes from shared words, so that without one it + * is a hash collision, and with one it can be a single common word in a short section; `false` + * otherwise, since real embeddings match meaning without shared words. */ - similarityNeedsKeyword?: boolean; + similarityNeedsKeyword?: boolean | number; /** * Whether copies of a section count once. A chunk with the same title, heading and text as * another, such as the same section in two versions of versioned docs, or a page a site serves @@ -110,6 +111,14 @@ export const DEFAULT_RETRIEVAL = { /** The model id `mockEmbeddingModel` records in an index. */ const MOCK_EMBEDDING = /^mock-hash-\d+$/; +/** + * The question's words a chunk of a mock index must contain for its similarity to count. With + * one, a single common word in a short section clears `MOCK_MIN_SIMILARITY`: on docusaurus.io's + * docs, a mock index answered 10 of 15 off-topic Stack Overflow titles (git, Bash, SQL, Java) + * with one, and 6 with two, as many as a keyword-only index of the same docs. + */ +const MOCK_SHARED_WORDS = 2; + /** * `minSimilarity` for embedding models whose similarity scale differs from OpenAI's, as measured: * on docusaurus.io's docs and the 22 in-scope and 6 off-topic questions of the 2026-10-10 audit, @@ -251,12 +260,13 @@ function measure( const similarities = index.vectors && query.vector ? index.vectors.similarities(query.vector) : null; - const needsKeyword = - opts.similarityNeedsKeyword ?? MOCK_EMBEDDING.test(index.embedding?.model ?? ''); + const needed = Number( + opts.similarityNeedsKeyword ?? + (MOCK_EMBEDDING.test(index.embedding?.model ?? '') ? MOCK_SHARED_WORDS : 0), + ); + const sharesWords = sharedWords(index, query.text, keyword.coverage, needed); const relevant = (i: number): boolean => - (similarities !== null && - (similarities[i] ?? 0) >= opts.minSimilarity && - (!needsKeyword || (keyword.coverage[i] ?? 0) > 0)) || + (similarities !== null && (similarities[i] ?? 0) >= opts.minSimilarity && sharesWords(i)) || (keyword.coverage[i] ?? 0) >= opts.minKeywordCoverage; let bestCoverage = 0; @@ -278,6 +288,35 @@ function measure( return { opts, keyword, similarities, relevant, bestCoverage, hitAt }; } +/** + * Whether chunk `i` contains at least `needed` of the question's distinct words, as BM25 reads + * them. One word is any keyword coverage at all; more are counted in the chunk's own words, once + * per chunk, which only the chunks that clear `minSimilarity` get to. + */ +function sharedWords( + index: LoadedIndex, + question: string, + coverage: Float32Array, + needed: number, +): (i: number) => boolean { + if (!(needed > 0)) return () => true; + if (needed <= 1) return (i) => (coverage[i] ?? 0) > 0; + const words = [...new Set(tokenize(question))]; + if (words.length < needed) return () => false; + const counted = new Map(); + return (i) => { + if (!((coverage[i] ?? 0) > 0)) return false; + let shares = counted.get(i); + if (shares === undefined) { + const chunk = index.chunks[i]; + const terms = new Set(chunk ? tokenize(chunkSearchText(chunk)) : []); + shares = words.filter((word) => terms.has(word)).length >= needed; + counted.set(i, shares); + } + return shares; + }; +} + /** * Hybrid retrieval: BM25 and cosine similarity, fused with reciprocal rank fusion, behind a * relevance gate. diff --git a/templates/cloudflare-worker/README.md b/templates/cloudflare-worker/README.md index 7133446a..0769a466 100644 --- a/templates/cloudflare-worker/README.md +++ b/templates/cloudflare-worker/README.md @@ -20,19 +20,19 @@ npx wrangler deploy Wrangler prints the Worker's URL. Point the dialog at it, plus `/api/ask`: -- **Docusaurus:** `plugins: [['ondocs/docusaurus', { endpoint: 'https://my-docs-ask..workers.dev/api/ask' }]]` -- **Starlight:** `ondocs({ endpoint: 'https://my-docs-ask..workers.dev/api/ask' })` +- **Docusaurus:** `plugins: [['ondocs/docusaurus', { endpoint: 'https://my-docs-ask..workers.dev/api/ask', embedding: 'workers-ai:@cf/baai/bge-small-en-v1.5' }]]` +- **Starlight:** `ondocs({ endpoint: 'https://my-docs-ask..workers.dev/api/ask', embedding: 'workers-ai:@cf/baai/bge-small-en-v1.5' })` - **Any other site:** `data-endpoint="https://my-docs-ask..workers.dev/api/ask"` on the script tag `package.json` allows the install scripts of esbuild and workerd, which Wrangler needs to set up their binaries, and declines fsevents (macOS file watching, with a prebuilt fallback). npm 12 runs no install scripts unless they are allowed this way. -The site must serve its index at `SITE_URL/ask-index.json`. The Docusaurus and Starlight plugins write it with every build; for another generator, run `npx ondocs index -o /ask-index.json -e none` after the build. For a project site, at `https://user.github.io/repo`, add `--base-url /repo` to that command: the pages' URLs in the index, which the dialog's citations and the MCP results link to, then start with `/repo`, as the site serves them. +The site must serve its index at `SITE_URL/ask-index.json`. The Docusaurus and Starlight plugins write it with every build; for another generator, run `npx ondocs index -o /ask-index.json -e workers-ai:@cf/baai/bge-small-en-v1.5` after the build. For a project site, at `https://user.github.io/repo`, add `--base-url /repo` to that command: the pages' URLs in the index, which the dialog's citations and the MCP results link to, then start with `/repo`, as the site serves them. Embedding the pages with Workers AI, as `embedding` does in the plugins, needs `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`, a token allowed to run Workers AI, where the site builds. Without them, leave out `embedding`, or pass `-e none`, for a keyword-only index. ## What it does - **It reads the index from the live site,** keeps it in memory, and checks it again every five minutes with a conditional request, so it follows the site's deploys without being redeployed. - **It answers the site's pages across origins,** with CORS headers for `SITE_URL`'s origin and no other. -- **It searches by keywords** for an index built without an embedding model, and by meaning as well for one built with a Workers AI model: `ondocs index -e workers-ai:@cf/baai/bge-small-en-v1.5`, or `embedding: 'workers-ai:@cf/baai/bge-small-en-v1.5'` in the plugins. That build needs `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`, a token allowed to run Workers AI, where the site builds. +- **It searches by meaning and by keywords** for an index embedded with Workers AI, as above, and by keywords only for one built without an embedding model, which needs no secret anywhere. With keywords only, the Ask box answers only questions that use enough of the docs' own words, so embed the index for it. - **It says "I don't know" without calling the model** when nothing in the docs is relevant. - **It limits each visitor** to 10 questions a minute, and the Worker to 100 questions and 300,000 model tokens a day per instance, about the free allowance with the default model. On the Workers Free plan nothing is charged past the allowance; on Workers Paid, raise the budget in `src/index.ts` to what you will spend. - **It serves the same index to agents** at `/api/mcp`, keyword-only, calling no model. diff --git a/test/embedding-defaults.test.ts b/test/embedding-defaults.test.ts new file mode 100644 index 00000000..9415337e --- /dev/null +++ b/test/embedding-defaults.test.ts @@ -0,0 +1,148 @@ +// The embedding an index gets when nothing names one: `ondocs index` builds OpenAI's +// text-embedding-3-small at 512 dimensions, as the plugins do, and a plugin build without a model +// key says what a keyword-only index means for the Ask box. +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { startOpenAIStub } from '../scripts/openai-stub.mjs'; + +import { main } from '../src/cli/main'; +import { parseIndexFile } from '../src/index-file'; +import { buildEmbedding } from '../src/integration/build'; + +let cwd: string; +let stub: Awaited>; + +beforeEach(async () => { + cwd = await mkdtemp(join(tmpdir(), 'ondocs-embedding-defaults-')); + await mkdir(join(cwd, 'content')); + await writeFile(join(cwd, 'content/index.md'), '# Home\n\nWelcome to the docs.\n'); + await writeFile( + join(cwd, 'content/deploy.md'), + '# Deploying\n\nShip it to the edge.\n\n## Rollbacks\n\nRedeploy the previous build.\n', + ); + stub = await startOpenAIStub(); + vi.stubEnv('OPENAI_API_KEY', ''); + vi.stubEnv('AI_GATEWAY_API_KEY', ''); +}); + +afterEach(async () => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + await stub.close(); + await rm(cwd, { recursive: true, force: true }); +}); + +/** `ondocs index content …` with an OpenAI key, against the local stand-in for OpenAI's API. */ +async function index(...args: string[]) { + const stdout: string[] = []; + const stderr: string[] = []; + const code = await main(['index', 'content', ...args], { + cwd, + env: { OPENAI_API_KEY: 'sk-test', OPENAI_BASE_URL: stub.url }, + stdout: (line) => stdout.push(line), + stderr: (line) => stderr.push(line), + }); + return { code, stdout: stdout.join('\n'), stderr: stderr.join('\n') }; +} + +const readIndex = async () => parseIndexFile(await readFile(join(cwd, 'ask-index.json'), 'utf8')); + +describe('ondocs index without --embedding', () => { + it('embeds with text-embedding-3-small at 512 dimensions, as the plugins do', async () => { + const result = await index(); + expect(result.code).toBe(0); + expect((await readIndex()).embedding).toEqual({ + model: 'text-embedding-3-small', + dimensions: 512, + settings: expect.any(String) as string, + }); + }); + + it('builds another size with --dimensions, and a model you name at its own size', async () => { + expect((await index('--dimensions', '1536')).code).toBe(0); + expect((await readIndex()).embedding).toMatchObject({ dimensions: 1536 }); + // The model's full size, as the plugins build a model their `embedding` option names. + expect((await index('-e', 'openai:text-embedding-3-small')).code).toBe(0); + expect((await readIndex()).embedding).toEqual({ + model: 'text-embedding-3-small', + dimensions: 1536, + }); + }); + + it('says so in its help', async () => { + const stdout: string[] = []; + await main(['--help'], { cwd, env: {}, stdout: (line) => stdout.push(line), stderr: vi.fn() }); + expect(stdout.join('\n')).toContain( + '(default: openai:text-embedding-3-small at 512 dimensions,', + ); + }); + + it('rebuilds an index the CLI built at 1,536 dimensions once, and checks it as it is', async () => { + // What `ondocs index` wrote by default before: the model's full size, with no settings. + expect((await index('-e', 'openai:text-embedding-3-small')).code).toBe(0); + const before = await readIndex(); + expect(before.embedding).toEqual({ model: 'text-embedding-3-small', dimensions: 1536 }); + + // A CI check without flags names no model, so the old index is still up to date. + const check = await index('--check'); + expect([check.code, check.stdout]).toEqual([0, expect.stringContaining('is up to date')]); + + // The next build embeds every chunk again, at 512, and reuses none of the old vectors. + const rebuilt = await index(); + const chunks = String(before.chunks.length); + expect(rebuilt.code).toBe(0); + expect(rebuilt.stdout).toContain(`✓ ${chunks} chunks, ${chunks} embedded, 0 reused`); + expect((await readIndex()).embedding).toMatchObject({ dimensions: 512 }); + // And the one after reuses them all. + expect((await index()).stdout).toMatch(/, 0 embedded, \d+ reused/); + }); + + it('embeds through AI Gateway at 512 dimensions when only its key is set', async () => { + const bodies: Record[] = []; + vi.stubEnv('AI_GATEWAY_API_KEY', 'gateway-test'); + vi.stubGlobal( + 'fetch', + vi.fn((_input: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(init?.body as string) as { + values: string[]; + providerOptions?: { openai?: { dimensions?: number } }; + }; + bodies.push(body); + const size = body.providerOptions?.openai?.dimensions ?? 1536; + return Promise.resolve( + Response.json({ + embeddings: body.values.map(() => Array.from({ length: size }, (_, i) => (i ? 0 : 1))), + }), + ); + }), + ); + const code = await main(['index', 'content'], { + cwd, + env: { AI_GATEWAY_API_KEY: 'gateway-test' }, + stdout: vi.fn(), + stderr: vi.fn(), + }); + expect(code).toBe(0); + expect(bodies.at(-1)).toMatchObject({ providerOptions: { openai: { dimensions: 512 } } }); + expect((await readIndex()).embedding).toMatchObject({ + model: 'openai/text-embedding-3-small', + dimensions: 512, + }); + }); +}); + +describe('a plugin build without a model key', () => { + it('says the Ask box then matches words only, and where the key goes', async () => { + const warn = vi.fn(); + expect(await buildEmbedding({}, { info: vi.fn(), warn })).toEqual({ model: null }); + const said = warn.mock.calls.map(([line]) => String(line)).join('\n'); + expect(said).toContain('No embedding model: building a keyword-only index.'); + expect(said).toContain('where the site builds'); + expect(said).toContain('The Ask box then answers only'); + expect(said).toContain("`embedding: 'none'`"); + }); +}); diff --git a/test/mock-gate.test.ts b/test/mock-gate.test.ts new file mode 100644 index 00000000..503d9e65 --- /dev/null +++ b/test/mock-gate.test.ts @@ -0,0 +1,79 @@ +// The relevance gate over a mock index (`ondocs index -e mock`, the README's "Try it without a +// key"): the mock embedder's similarity comes from shared words, so one word in common with a +// short section is enough to clear MOCK_MIN_SIMILARITY. A similarity counts only from a chunk that +// shares two of the question's words. +import { describe, expect, it } from 'vitest'; + +import { buildIndex, loadIndex, retrieve, type RetrievalOptions } from '../src'; +import { MOCK_MIN_SIMILARITY, hashEmbedding, mockEmbeddingModel } from '../src/mock'; + +const page = (id: string, title: string, content: string) => ({ + id: `${id}.md`, + url: `/docs/${id}`, + title, + content, +}); + +const PAGES = [ + page('search', 'Keyword search', 'Keyword search ranks the pages by keyword.'), + page('deploy', 'Deploying', 'Deploy the endpoint to Netlify or Vercel as one function.'), + page('pricing', 'Pricing', 'The plan costs ten dollars a month.'), + page('install', 'Installing', 'Install the package with npm, then build the index.'), + page('dialog', 'The dialog', 'The dialog opens with a shortcut and cites its sources.'), +]; + +/** Shares "plan" and "cost" with the pricing page, and nothing else with any page. */ +const PRICE = 'How much does the yearly plan cost in euros?'; + +async function mockIndex() { + const { index } = await buildIndex({ documents: PAGES, embeddingModel: mockEmbeddingModel() }); + return index; +} + +/** What the endpoint does for a mock index, as `ondocs dev` configures it. */ +const ask = (index: ReturnType, text: string, options: RetrievalOptions = {}) => + retrieve( + index, + { text, vector: hashEmbedding(text, 512) }, + { minSimilarity: MOCK_MIN_SIMILARITY, ...options }, + ); + +describe('the relevance gate over a mock index', () => { + it('refuses a question that shares one word with a short section', async () => { + const index = loadIndex(await mockIndex()); + const question = 'What does the "yield" keyword do in Python?'; + const loose = ask(index, question, { similarityNeedsKeyword: true }); + // One word in common is enough for the similarity... + expect(loose.answerable).toBe(true); + expect(loose.best.similarity).toBeGreaterThan(MOCK_MIN_SIMILARITY); + expect(loose.best.keywordCoverage).toBeLessThan(0.5); + // ...so a mock index asks for two, and says "I don't know". + expect(ask(index, question)).toMatchObject({ answerable: false, hits: [] }); + }); + + it('still answers questions that share two words, or most of their keywords', async () => { + const index = loadIndex(await mockIndex()); + const urls = (text: string) => ask(index, text).hits.map((hit) => hit.chunk.url); + // Two words in common, on similarity: the question's other words are on no page. + const priced = ask(index, PRICE); + expect(priced.hits[0]?.chunk.url).toBe('/docs/pricing'); + expect(priced.best.keywordCoverage).toBeLessThan(0.5); + expect(urls('How do I deploy to Netlify?')[0]).toBe('/docs/deploy'); + // One word, which is the whole question: keyword coverage counts on its own. + expect(urls('shortcut')[0]).toBe('/docs/dialog'); + }); + + it('takes the number of shared words, true for one, or false for none', async () => { + const index = loadIndex(await mockIndex()); + const question = 'What does the "yield" keyword do in Python?'; + expect(ask(index, question, { similarityNeedsKeyword: 1 }).answerable).toBe(true); + expect(ask(index, question, { similarityNeedsKeyword: false }).answerable).toBe(true); + expect(ask(index, PRICE, { similarityNeedsKeyword: 3 }).answerable).toBe(false); + }); + + it('leaves other indexes alone: real embeddings match meaning without shared words', async () => { + const file = await mockIndex(); + const semantic = loadIndex({ ...file, embedding: { ...file.embedding!, model: 'embed-x' } }); + expect(ask(semantic, 'What does the "yield" keyword do in Python?').answerable).toBe(true); + }); +}); From a2c2979c69fd9acf4ea0e75a3844be1f1f139996 Mon Sep 17 00:00:00 2001 From: Diogo Esteves Date: Fri, 16 Oct 2026 11:09:48 +0100 Subject: [PATCH 2/2] fix: keep getting-started within the Next.js JS budget, release as minor The paragraph this branch added to getting-started.md pushed the Next.js example's /docs/getting-started to 180.1 KB of JavaScript against its 180 KB budget, since Next.js serializes the page into its inline RSC script. It is gone: cli.md, retrieval.md and deployment.md already say the same. The four getting-started pages match main again. The changeset is minor: the CLI's default index changes size, so the next build of every CLI user with a key embeds every chunk again, and similarityNeedsKeyword takes a number. On the ondocs docs this branch ships, the two-word mock gate lets 5 of the 15 off-topic titles through, not 4 (one word: 7). The README names the Workers AI Worker's index command, with its Cloudflare variables, or -e none, in the "Any other site" block. The Worker template's README says beside its plugin snippets that the embedding line needs CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN where the site builds. The mock gate counts shared words from the BM25 postings the search already walks (Bm25Result.matchedTerms) instead of tokenizing each candidate chunk again. Results are the same; on docusaurus.io's 1,070 chunk mock index, retrieve() p95 goes from 1.73 ms back to 0.62 ms. --- .changeset/embedding-defaults.md | 4 +- README.md | 2 + examples/docusaurus/docs/getting-started.md | 2 - .../nextjs/content/docs/getting-started.md | 2 - .../docs/get-started/getting-started.md | 2 - examples/vitepress/docs/getting-started.md | 2 - src/search/bm25.ts | 6 ++- src/search/retrieve.ts | 38 +++---------------- templates/cloudflare-worker/README.md | 4 +- test/mock-gate.test.ts | 19 +++++++++- 10 files changed, 36 insertions(+), 45 deletions(-) diff --git a/.changeset/embedding-defaults.md b/.changeset/embedding-defaults.md index 0655fab3..55c66395 100644 --- a/.changeset/embedding-defaults.md +++ b/.changeset/embedding-defaults.md @@ -1,5 +1,5 @@ --- -'ondocs': patch +'ondocs': minor --- `ondocs index` without `--embedding` now embeds with OpenAI's `text-embedding-3-small` at 512 dimensions, as the plugins do, with `OPENAI_API_KEY` or through AI Gateway. Before, it built the model's full 1,536, so a site that followed the README's Next.js line shipped an index about twice the size: 2.93 MB against 1.54 MB for docusaurus.io's 1,070 chunks. `--dimensions` sets another size, and a model you name with `--embedding` is still built at its own size unless `--dimensions` gives one. @@ -8,4 +8,4 @@ An index the CLI built at 1,536 dimensions needs nothing: the endpoints, `ondocs A plugin build without a model key now also says what its keyword-only index means: the Ask box answers only questions that use enough of the docs' own words, the MCP endpoint `init` writes searches by keywords either way, and the key belongs where the site builds as well as where the endpoint runs. `embedding: 'none'` builds the same index without the warning. -Over a mock index (`-e mock`), a similarity now counts only from a chunk that shares two of the question's words, not one: a single common word in a short section let off-topic questions through. On docusaurus.io's docs, 15 off-topic Stack Overflow titles about git, Bash, SQL and Java got 6 quoted answers instead of 10, as many as from a keyword-only index; on the ondocs docs, 4 instead of 7. Of the 22 answerable questions in the sample eval of docusaurus.io's docs, one more is refused, one that had been answered from a page that does not answer it, and the right page is in the top three for 17 instead of 16. `retrieval.similarityNeedsKeyword` now also takes the number of words: `true` is one, as before. +Over a mock index (`-e mock`), a similarity now counts only from a chunk that shares two of the question's words, not one: a single common word in a short section let off-topic questions through. On docusaurus.io's docs, 15 off-topic Stack Overflow titles about git, Bash, SQL and Java got 6 quoted answers instead of 10, as many as from a keyword-only index; on the ondocs docs, 5 instead of 7. Of the 22 answerable questions in the sample eval of docusaurus.io's docs, one more is refused, one that had been answered from a page that does not answer it, and the right page is in the top three for 17 instead of 16. `retrieval.similarityNeedsKeyword` now also takes the number of words: `true` is one, as before. The count comes from `Bm25Index.search`, whose result now also has `matchedTerms`: how many of the query's distinct terms each document contains. diff --git a/README.md b/README.md index 01fc76d2..9ce5e0f0 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,8 @@ import 'ondocs/embed/launcher.css'; ```sh npx ondocs index public -o public/ask-index.json # embeds with OPENAI_API_KEY, best for the Ask box; -e none for keyword-only npx ondocs init --out public # plain HTML needs --out; Hugo, Jekyll, MkDocs and Eleventy are detected +# For the Workers AI Worker (init --host github-pages), index with -e workers-ai:@cf/baai/bge-small-en-v1.5 +# and CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN set, or -e none: it can't embed questions for an OpenAI index ``` On a GitHub Pages project site, at `https://user.github.io/repo`, add `--base-url /repo` to the index command. diff --git a/examples/docusaurus/docs/getting-started.md b/examples/docusaurus/docs/getting-started.md index 85de9f35..0a7019e2 100644 --- a/examples/docusaurus/docs/getting-started.md +++ b/examples/docusaurus/docs/getting-started.md @@ -34,8 +34,6 @@ npx ondocs index ./docs --base-url /docs \ -e openai:text-embedding-3-small --dimensions 512 ``` -Embedding the chunks lets the Ask box match a question by meaning as well as by its words, so it is the default: with `OPENAI_API_KEY` set, the CLI and the plugins embed with `text-embedding-3-small` at 512 dimensions, as above, and a site answered by the [Workers AI Worker](./workers-ai.md) embeds with `-e workers-ai:@cf/baai/bge-small-en-v1.5`. Without a key, `-e none` builds a keyword-only index. It suits agents' MCP search, which `init` writes keyword-only, but the Ask box then answers only questions that use enough of the docs' own words. - With the Docusaurus, Astro, Starlight or VitePress plugin you skip this step: the plugin indexes the built site after every build. ## Add the endpoint diff --git a/examples/nextjs/content/docs/getting-started.md b/examples/nextjs/content/docs/getting-started.md index 1e81b626..b5dbc784 100644 --- a/examples/nextjs/content/docs/getting-started.md +++ b/examples/nextjs/content/docs/getting-started.md @@ -35,8 +35,6 @@ npx ondocs index ./docs --base-url /docs \ -e openai:text-embedding-3-small --dimensions 512 ``` -Embedding the chunks lets the Ask box match a question by meaning as well as by its words, so it is the default: with `OPENAI_API_KEY` set, the CLI and the plugins embed with `text-embedding-3-small` at 512 dimensions, as above, and a site answered by the [Workers AI Worker](/docs/workers-ai) embeds with `-e workers-ai:@cf/baai/bge-small-en-v1.5`. Without a key, `-e none` builds a keyword-only index. It suits agents' MCP search, which `init` writes keyword-only, but the Ask box then answers only questions that use enough of the docs' own words. - With the Docusaurus, Astro, Starlight or VitePress plugin you skip this step: the plugin indexes the built site after every build. ## Add the endpoint diff --git a/examples/starlight/src/content/docs/get-started/getting-started.md b/examples/starlight/src/content/docs/get-started/getting-started.md index b0d29819..6bfa8c72 100644 --- a/examples/starlight/src/content/docs/get-started/getting-started.md +++ b/examples/starlight/src/content/docs/get-started/getting-started.md @@ -36,8 +36,6 @@ npx ondocs index ./docs --base-url /docs \ -e openai:text-embedding-3-small --dimensions 512 ``` -Embedding the chunks lets the Ask box match a question by meaning as well as by its words, so it is the default: with `OPENAI_API_KEY` set, the CLI and the plugins embed with `text-embedding-3-small` at 512 dimensions, as above, and a site answered by the [Workers AI Worker](/guides/workers-ai/) embeds with `-e workers-ai:@cf/baai/bge-small-en-v1.5`. Without a key, `-e none` builds a keyword-only index. It suits agents' MCP search, which `init` writes keyword-only, but the Ask box then answers only questions that use enough of the docs' own words. - With the Docusaurus, Astro, Starlight or VitePress plugin you skip this step: the plugin indexes the built site after every build. ## Add the endpoint diff --git a/examples/vitepress/docs/getting-started.md b/examples/vitepress/docs/getting-started.md index 26463f52..c08f2c40 100644 --- a/examples/vitepress/docs/getting-started.md +++ b/examples/vitepress/docs/getting-started.md @@ -36,8 +36,6 @@ npx ondocs index ./docs --base-url /docs \ -e openai:text-embedding-3-small --dimensions 512 ``` -Embedding the chunks lets the Ask box match a question by meaning as well as by its words, so it is the default: with `OPENAI_API_KEY` set, the CLI and the plugins embed with `text-embedding-3-small` at 512 dimensions, as above, and a site answered by the [Workers AI Worker](/workers-ai) embeds with `-e workers-ai:@cf/baai/bge-small-en-v1.5`. Without a key, `-e none` builds a keyword-only index. It suits agents' MCP search, which `init` writes keyword-only, but the Ask box then answers only questions that use enough of the docs' own words. - With the Docusaurus, Astro, Starlight or VitePress plugin you skip this step: the plugin indexes the built site after every build. ## Add the endpoint diff --git a/src/search/bm25.ts b/src/search/bm25.ts index 43394a7d..047c2757 100644 --- a/src/search/bm25.ts +++ b/src/search/bm25.ts @@ -21,6 +21,8 @@ export interface Bm25Result { scores: Float32Array; /** Query coverage per document, in [0, 1]. */ coverage: Float32Array; + /** How many of the query's distinct terms each document contains. */ + matchedTerms: Uint16Array; /** Indices of documents with a non-zero score, best first. */ ranked: number[]; } @@ -113,6 +115,7 @@ export class Bm25Index { search(queryTerms: readonly string[]): Bm25Result { const scores = new Float32Array(this.size); const matched = new Float32Array(this.size); + const matchedTerms = new Uint16Array(this.size); let totalIdf = 0; for (const term of new Set(queryTerms)) { @@ -128,6 +131,7 @@ export class Bm25Index { const norm = 1 - this.b + (this.b * (this.lengths[doc] ?? 0)) / (this.averageLength || 1); scores[doc] = (scores[doc] ?? 0) + (idf * (tf * (this.k1 + 1))) / (tf + this.k1 * norm); matched[doc] = (matched[doc] ?? 0) + idf; + matchedTerms[doc] = (matchedTerms[doc] ?? 0) + 1; } } @@ -139,6 +143,6 @@ export class Bm25Index { if (totalIdf > 0) { for (const doc of ranked) coverage[doc] = (matched[doc] ?? 0) / totalIdf; } - return { scores, coverage, ranked }; + return { scores, coverage, matchedTerms, ranked }; } } diff --git a/src/search/retrieve.ts b/src/search/retrieve.ts index 54cee7f5..9baeed1c 100644 --- a/src/search/retrieve.ts +++ b/src/search/retrieve.ts @@ -260,13 +260,16 @@ function measure( const similarities = index.vectors && query.vector ? index.vectors.similarities(query.vector) : null; - const needed = Number( + // The question's distinct words a chunk must contain for its similarity to count, as BM25 + // counted them while it scored the chunk. + const sharedWords = Number( opts.similarityNeedsKeyword ?? (MOCK_EMBEDDING.test(index.embedding?.model ?? '') ? MOCK_SHARED_WORDS : 0), ); - const sharesWords = sharedWords(index, query.text, keyword.coverage, needed); const relevant = (i: number): boolean => - (similarities !== null && (similarities[i] ?? 0) >= opts.minSimilarity && sharesWords(i)) || + (similarities !== null && + (similarities[i] ?? 0) >= opts.minSimilarity && + (keyword.matchedTerms[i] ?? 0) >= sharedWords) || (keyword.coverage[i] ?? 0) >= opts.minKeywordCoverage; let bestCoverage = 0; @@ -288,35 +291,6 @@ function measure( return { opts, keyword, similarities, relevant, bestCoverage, hitAt }; } -/** - * Whether chunk `i` contains at least `needed` of the question's distinct words, as BM25 reads - * them. One word is any keyword coverage at all; more are counted in the chunk's own words, once - * per chunk, which only the chunks that clear `minSimilarity` get to. - */ -function sharedWords( - index: LoadedIndex, - question: string, - coverage: Float32Array, - needed: number, -): (i: number) => boolean { - if (!(needed > 0)) return () => true; - if (needed <= 1) return (i) => (coverage[i] ?? 0) > 0; - const words = [...new Set(tokenize(question))]; - if (words.length < needed) return () => false; - const counted = new Map(); - return (i) => { - if (!((coverage[i] ?? 0) > 0)) return false; - let shares = counted.get(i); - if (shares === undefined) { - const chunk = index.chunks[i]; - const terms = new Set(chunk ? tokenize(chunkSearchText(chunk)) : []); - shares = words.filter((word) => terms.has(word)).length >= needed; - counted.set(i, shares); - } - return shares; - }; -} - /** * Hybrid retrieval: BM25 and cosine similarity, fused with reciprocal rank fusion, behind a * relevance gate. diff --git a/templates/cloudflare-worker/README.md b/templates/cloudflare-worker/README.md index 0769a466..28d7e8bd 100644 --- a/templates/cloudflare-worker/README.md +++ b/templates/cloudflare-worker/README.md @@ -24,9 +24,11 @@ Wrangler prints the Worker's URL. Point the dialog at it, plus `/api/ask`: - **Starlight:** `ondocs({ endpoint: 'https://my-docs-ask..workers.dev/api/ask', embedding: 'workers-ai:@cf/baai/bge-small-en-v1.5' })` - **Any other site:** `data-endpoint="https://my-docs-ask..workers.dev/api/ask"` on the script tag +The `embedding` line embeds the pages with Workers AI as the site builds, so it needs `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`, a token allowed to run Workers AI, where the site builds; the build fails without them. Leave it out for a keyword-only index, which needs no secret anywhere. + `package.json` allows the install scripts of esbuild and workerd, which Wrangler needs to set up their binaries, and declines fsevents (macOS file watching, with a prebuilt fallback). npm 12 runs no install scripts unless they are allowed this way. -The site must serve its index at `SITE_URL/ask-index.json`. The Docusaurus and Starlight plugins write it with every build; for another generator, run `npx ondocs index -o /ask-index.json -e workers-ai:@cf/baai/bge-small-en-v1.5` after the build. For a project site, at `https://user.github.io/repo`, add `--base-url /repo` to that command: the pages' URLs in the index, which the dialog's citations and the MCP results link to, then start with `/repo`, as the site serves them. Embedding the pages with Workers AI, as `embedding` does in the plugins, needs `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`, a token allowed to run Workers AI, where the site builds. Without them, leave out `embedding`, or pass `-e none`, for a keyword-only index. +The site must serve its index at `SITE_URL/ask-index.json`. The Docusaurus and Starlight plugins write it with every build; for another generator, run `npx ondocs index -o /ask-index.json -e workers-ai:@cf/baai/bge-small-en-v1.5` after the build. For a project site, at `https://user.github.io/repo`, add `--base-url /repo` to that command: the pages' URLs in the index, which the dialog's citations and the MCP results link to, then start with `/repo`, as the site serves them. The index command needs the same two variables; without them, pass `-e none` for a keyword-only index. ## What it does diff --git a/test/mock-gate.test.ts b/test/mock-gate.test.ts index 503d9e65..3bc04a83 100644 --- a/test/mock-gate.test.ts +++ b/test/mock-gate.test.ts @@ -4,7 +4,14 @@ // shares two of the question's words. import { describe, expect, it } from 'vitest'; -import { buildIndex, loadIndex, retrieve, type RetrievalOptions } from '../src'; +import { + Bm25Index, + buildIndex, + loadIndex, + retrieve, + tokenize, + type RetrievalOptions, +} from '../src'; import { MOCK_MIN_SIMILARITY, hashEmbedding, mockEmbeddingModel } from '../src/mock'; const page = (id: string, title: string, content: string) => ({ @@ -71,6 +78,16 @@ describe('the relevance gate over a mock index', () => { expect(ask(index, PRICE, { similarityNeedsKeyword: 3 }).answerable).toBe(false); }); + it('counts the shared words as BM25 matched them, each distinct word once', () => { + const bm25 = new Bm25Index([ + tokenize('Pricing. The plan costs ten dollars a month.'), + tokenize('Deploying. Deploy the plan.'), + tokenize('Installing. Install the package.'), + ]); + const { matchedTerms } = bm25.search(tokenize('plan dollars plan month')); + expect([...matchedTerms]).toEqual([3, 1, 0]); + }); + it('leaves other indexes alone: real embeddings match meaning without shared words', async () => { const file = await mockIndex(); const semantic = loadIndex({ ...file, embedding: { ...file.embedding!, model: 'embed-x' } });