Rebuild the docs site: contract and SDK reference, readable by AI crawlers

.github/workflows/build.yaml

@@ -6,8 +6,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+      with:
+        # The sitemap reads each page's last commit date from git. The default
+        # shallow clone has no history, so every lastmod would be missing.
+        fetch-depth: 0
     - uses: actions/setup-node@v4
       with:
         node-version: lts/*
-    - run: yarn install
-    - run: yarn build
+    - run: npm ci
+    - run: npm run build
+    # The build only warns when a contract page is behind its source repo. Here
+    # a person is already looking at a diff, so it fails instead.
+    - run: node scripts/sync-reference.mjs --strict

.gitignore

@@ -8,6 +8,10 @@
 .docusaurus
 .cache-loader
 
+# Written by scripts/build-text.mjs on every build
+/static/md
+/static/llms-full.txt
+
 # Misc
 .DS_Store
 .env.local
@@ -16,7 +20,3 @@
 .env.production.local
 
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-
-.yarn

README.md

@@ -1,41 +1,45 @@
-# Website
+# Kokio documentation
 
-This website is built using [Docusaurus](https://docusaurus.io/), a modern static website generator.
+Source for [docs.kokio.app](https://docs.kokio.app), built with [Docusaurus](https://docusaurus.io/).
 
-### Installation
+npm only. There is one lockfile, `package-lock.json`, and Vercel picks the package manager from it.
+
+### Install
 
 ```
-$ yarn
+npm ci
 ```
 
-### Local Development
+### Local development
 
 ```
-$ yarn start
+npm start
 ```
 
-This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server.
+Starts a dev server on port 3000 and reloads on save.
 
 ### Build
 
 ```
-$ yarn build
+npm run build
 ```
 
-This command generates static content into the `build` directory and can be served using any static contents hosting service.
-
-### Deployment
+Writes the static site to `build/`. The prebuild step regenerates the plain-text surfaces that AI crawlers read: one `.md` file per page under `static/md/`, plus `static/llms-full.txt`. Both are gitignored, so build before serving locally or those routes 404.
 
-Using SSH:
+Serve the result with:
 
 ```
-$ USE_SSH=true yarn deploy
+npm run serve
 ```
 
-Not using SSH:
+### Deployment
+
+Vercel builds `main` and deploys it. Nothing to run by hand. Open a PR, merge it, and the change is live.
+
+### Other commands
 
 ```
-$ GIT_USER=<Your GitHub username> yarn deploy
+npm run typecheck          # tsc, no emit
+npm run write-heading-ids  # regenerate explicit heading anchors after editing headings
+npm run clear              # drop the .docusaurus cache
 ```
-
-If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch.

api/ask.mjs

@@ -0,0 +1,194 @@
+/**
+ * POST /api/ask, the endpoint behind the search box.
+ *
+ * Takes a question, returns an answer written from the docs plus the pages it
+ * came from. When the model cannot answer, the pages still come back, so the
+ * box degrades into plain search rather than into an error.
+ *
+ * The response never carries the reason. It can name the key or the model, so
+ * it goes to the logs instead.
+ *
+ * Every question is also posted to Discord when `DISCORD_WEBHOOK_URL` is set,
+ * which is how gaps in the docs get noticed.
+ */
+
+import { answerQuestion, MAX_QUESTION_LENGTH } from "../lib/docs-answer.mjs";
+import { SITE_URL } from "../src/siteCopy.mjs";
+
+const WINDOW_MS = 60 * 1000;
+
+/** Long enough for Discord on a bad day, short enough to never matter. */
+const NOTIFY_TIMEOUT_MS = 1500;
+
+/** Discord's limits on the parts of an embed this uses. */
+const EMBED_TITLE_MAX = 250;
+const EMBED_DESCRIPTION_MAX = 4000;
+const EMBED_FIELD_MAX = 1000;
+
+/** Discord's own green and red, which stay legible in both themes. */
+const COLOUR_ANSWERED = 0x57f287;
+const COLOUR_UNANSWERED = 0xed4245;
+
+function clip(text, max) {
+  return text.length > max ? `${text.slice(0, max - 1)}…` : text;
+}
+
+/** Per reader, and for everyone this instance is serving. */
+const MAX_PER_ADDRESS = 8;
+const MAX_PER_INSTANCE = 60;
+
+const recent = new Map();
+let instanceHits = [];
+let warnedNoWebhook = false;
+
+/**
+ * Requests per address, kept in memory.
+ *
+ * Serverless spreads traffic across instances, so this slows one impatient
+ * reader rather than a determined attacker. The instance cap below is the
+ * cost stop, and the model's own daily quota is the one behind that.
+ */
+function overLimit(address, now) {
+  instanceHits = instanceHits.filter((at) => now - at < WINDOW_MS);
+  instanceHits.push(now);
+  if (instanceHits.length > MAX_PER_INSTANCE) return "instance";
+
+  const seen = (recent.get(address) ?? []).filter((at) => now - at < WINDOW_MS);
+  seen.push(now);
+  recent.set(address, seen);
+  // Addresses stop arriving but never leave, so drop the lot now and then.
+  if (recent.size > 5000) recent.clear();
+
+  return seen.length > MAX_PER_ADDRESS ? "address" : null;
+}
+
+/**
+ * Rejects a browser on another site posting here.
+ *
+ * A request with no Origin, such as curl or a test script, is allowed: the
+ * header only proves where a browser came from, and blocking its absence
+ * stops honest tools without stopping anyone else.
+ */
+function fromAnotherSite(request) {
+  const origin = request.headers.origin;
+  if (!origin) return false;
+  try {
+    return new URL(origin).host !== request.headers.host;
+  } catch {
+    return true;
+  }
+}
+
+/**
+ * Posts what was asked and what came back to a Discord channel.
+ *
+ * Sent before the response, not after. A serverless function can be frozen the
+ * moment it responds, which leaves the post half sent and the channel empty.
+ * The timeout below is what keeps that from costing the reader anything.
+ *
+ * Failures are logged and go no further: a missed notification is not worth
+ * turning into a failed search.
+ */
+async function notify(question, { answer, sources, degraded }) {
+  const url = process.env.DISCORD_WEBHOOK_URL;
+  if (!url) {
+    // Once per instance. Silence here is indistinguishable from a webhook that
+    // is set but refused, and both look the same from the channel.
+    if (!warnedNoWebhook) {
+      warnedNoWebhook = true;
+      console.error("discord notify skipped: DISCORD_WEBHOOK_URL is not set");
+    }
+    return;
+  }
+
+  const embed = {
+    // The stripe down the side is the whole point: a channel of these can be
+    // skimmed for the red ones without reading a word.
+    color: answer ? COLOUR_ANSWERED : COLOUR_UNANSWERED,
+    title: clip(question, EMBED_TITLE_MAX),
+    description: clip(answer ?? "No answer. Closest pages below.", EMBED_DESCRIPTION_MAX),
+    footer: { text: answer ? "answered" : (degraded ?? "no match") },
+    timestamp: new Date().toISOString(),
+  };
+
+  const links = sources
+    .map((source) => `[${source.title}](${SITE_URL}${source.url})`)
+    .join(" · ");
+  if (links) {
+    embed.fields = [{ name: "Pages", value: clip(links, EMBED_FIELD_MAX) }];
+  }
+
+  try {
+    const posted = await fetch(url, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({
+        embeds: [embed],
+        // The question is whatever a reader typed, so it must not be able to
+        // ping the server.
+        allowed_mentions: { parse: [] },
+      }),
+      signal: AbortSignal.timeout(NOTIFY_TIMEOUT_MS),
+    });
+    // A deleted or mistyped webhook answers 401 or 404 rather than throwing,
+    // so without this the channel just stays empty.
+    if (!posted.ok) {
+      console.error(`discord notify rejected: ${posted.status} ${await posted.text()}`);
+    }
+  } catch (error) {
+    console.error("discord notify failed:", error.message);
+  }
+}
+
+function send(response, status, body) {
+  response.status(status);
+  response.setHeader("content-type", "application/json; charset=utf-8");
+  response.setHeader("cache-control", "no-store");
+  response.end(JSON.stringify(body));
+}
+
+export default async function handler(request, response) {
+  if (request.method !== "POST") return send(response, 405, { error: "use POST" });
+  if (fromAnotherSite(request)) return send(response, 403, { error: "wrong origin" });
+
+  const type = request.headers["content-type"] ?? "";
+  if (!type.includes("application/json")) {
+    return send(response, 415, { error: "send JSON" });
+  }
+
+  const address =
+    request.headers["x-forwarded-for"]?.split(",")[0]?.trim() ?? "unknown";
+  const limit = overLimit(address, Date.now());
+  if (limit) {
+    if (limit === "instance") console.error("ask: instance rate limit hit");
+    return send(response, 429, { error: "rate_limited" });
+  }
+
+  let question = "";
+  try {
+    const body =
+      typeof request.body === "string" ? JSON.parse(request.body) : request.body;
+    question = String(body?.question ?? "").trim();
+  } catch {
+    return send(response, 400, { error: "body must be JSON" });
+  }
+
+  if (!question) return send(response, 400, { error: "question is required" });
+  if (question.length > MAX_QUESTION_LENGTH) {
+    return send(response, 400, { error: "question is too long" });
+  }
+
+  try {
+    const { answer, sources, degraded, detail, cached } = await answerQuestion(question, {
+      apiKey: process.env.GEMINI_API_KEY,
+    });
+    if (detail) console.error("ask:", degraded ?? "answered", detail);
+
+    // A repeat inside the cache window was posted the first time it was asked.
+    if (!cached) await notify(question, { answer, sources, degraded });
+    return send(response, 200, { answer, sources, degraded });
+  } catch (error) {
+    console.error("ask failed:", error);
+    return send(response, 500, { error: "could not answer" });
+  }
+}