docs: serve raw Markdown versions of every page for AI agents

website/docusaurus.config.js

@@ -25,6 +25,12 @@ const config = {
     mermaid: true,
   },
 
+  plugins: [
+    // Serves a raw Markdown copy of every page (page URL + ".md") and
+    // llms-full.txt, so AI agents can read the docs without parsing HTML.
+    require.resolve('./plugins/raw-markdown'),
+  ],
+
   themes: [
     '@docusaurus/theme-mermaid',
     [

website/plugins/raw-markdown/index.js

@@ -0,0 +1,259 @@
+// Serves a raw Markdown copy of every docs page so AI agents can read the
+// docs without parsing HTML.
+//
+// After each build this plugin:
+//   1. Copies every docs/**/*.md file into the build output at the page's
+//      public URL plus ".md" (e.g. /slack/approvals.md). The homepage is
+//      written to /index.md.
+//   2. Writes /llms-full.txt: the whole docs site as one Markdown file, in
+//      sidebar order, with each page's canonical URL noted above it.
+//
+// The URL for each file is derived the same way Docusaurus derives it
+// (frontmatter slug if present, README/index files map to their folder,
+// everything else maps to its path). As a safety net, every derived URL is
+// checked against the routes Docusaurus actually built, and the build fails
+// if one does not match, so a future rename or custom slug cannot silently
+// publish markdown at a dead URL.
+
+const fs = require('fs');
+const path = require('path');
+
+const SITE_URL = 'https://docs.kite.ai';
+
+// Reads the value of `slug:` from a file's frontmatter block, if any.
+function readFrontmatterSlug(markdown) {
+  if (!markdown.startsWith('---\n')) return null;
+  const end = markdown.indexOf('\n---', 4);
+  if (end === -1) return null;
+  const frontmatter = markdown.slice(4, end);
+  const match = frontmatter.match(/^slug:\s*(.+)\s*$/m);
+  return match ? match[1].trim() : null;
+}
+
+// Removes the frontmatter block, leaving just the page body.
+function stripFrontmatter(markdown) {
+  if (!markdown.startsWith('---\n')) return markdown;
+  const end = markdown.indexOf('\n---', 4);
+  if (end === -1) return markdown;
+  return markdown.slice(markdown.indexOf('\n', end + 1) + 1).replace(/^\s+/, '');
+}
+
+// Derives the page route for a docs source file, e.g.
+// "slack/approvals.md" -> "/slack/approvals", "account/README.md" -> "/account".
+function routeForFile(relPath, markdown) {
+  const slug = readFrontmatterSlug(markdown);
+  const dir = path.posix.dirname(relPath);
+  if (slug) {
+    if (slug.startsWith('/')) return slug === '/' ? '/' : slug.replace(/\/$/, '');
+    const joined = path.posix.join(dir === '.' ? '' : dir, slug);
+    return '/' + joined;
+  }
+  const base = path.posix.basename(relPath, '.md');
+  const isIndex = base === 'README' || base === 'index' || base === path.posix.basename(dir);
+  const routePath = isIndex ? (dir === '.' ? '' : dir) : dir === '.' ? base : `${dir}/${base}`;
+  return routePath === '' ? '/' : '/' + routePath;
+}
+
+function listMarkdownFiles(dir, base = dir) {
+  const out = [];
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) out.push(...listMarkdownFiles(full, base));
+    else if (entry.name.endsWith('.md')) out.push(path.relative(base, full).split(path.sep).join('/'));
+  }
+  return out;
+}
+
+// Collects categories whose index page is auto-generated by Docusaurus
+// (link type "generated-index"). These have no source .md file, so the
+// plugin synthesizes one from the category's title, description, and the
+// pages directly inside it.
+function collectGeneratedIndexes(items, out = []) {
+  for (const item of items) {
+    if (item && item.type === 'category') {
+      if (item.link && item.link.type === 'generated-index') {
+        const childIds = (item.items || [])
+          .map((child) => {
+            if (typeof child === 'string') return child;
+            if (child && child.type === 'category' && child.link && child.link.id) return child.link.id;
+            if (child && child.type === 'doc' && child.id) return child.id;
+            return null;
+          })
+          .filter(Boolean);
+        out.push({
+          title: item.link.title || item.label,
+          description: item.link.description || '',
+          slug: item.link.slug,
+          childIds,
+        });
+      }
+      collectGeneratedIndexes(item.items || [], out);
+    }
+  }
+  return out;
+}
+
+// Reads the value of `title:` from a file's frontmatter block, if any.
+function readFrontmatterTitle(markdown) {
+  if (!markdown.startsWith('---\n')) return null;
+  const end = markdown.indexOf('\n---', 4);
+  if (end === -1) return null;
+  const match = markdown.slice(4, end).match(/^title:\s*(.+)\s*$/m);
+  return match ? match[1].trim() : null;
+}
+
+// Flattens sidebars.js into an ordered list of doc ids (category link docs
+// come before the docs inside the category).
+function flattenSidebar(items, out = []) {
+  for (const item of items) {
+    if (typeof item === 'string') {
+      out.push(item);
+    } else if (item && item.type === 'category') {
+      if (item.link && item.link.type === 'doc' && item.link.id) out.push(item.link.id);
+      flattenSidebar(item.items || [], out);
+    } else if (item && item.type === 'doc' && item.id) {
+      out.push(item.id);
+    }
+  }
+  return out;
+}
+
+// Fails the build when static/llms.txt and the docs disagree, so the index
+// cannot drift when pages are added, renamed, or removed. Two checks:
+//   1. Every docs page (and every generated category index) is listed.
+//   2. Every docs.kite.ai URL mentioned in llms.txt points at a real page.
+// The summary lines in llms.txt stay hand-written; this only checks coverage.
+function checkLlmsIndex(siteDir, mustBeListed, routes) {
+  const llmsPath = path.join(siteDir, 'static', 'llms.txt');
+  const llmsText = fs.readFileSync(llmsPath, 'utf8');
+
+  // Normalizes a docs.kite.ai URL down to the page route it refers to,
+  // e.g. "https://docs.kite.ai/slack/approvals.md." -> "/slack/approvals".
+  // Returns null for URLs that are fine but are not pages (llms.txt itself).
+  const toRoute = (url) => {
+    let p = url.replace(SITE_URL, '').replace(/[).,]+$/, '');
+    if (p === '/llms.txt' || p === '/llms-full.txt') return null;
+    if (p.endsWith('.md')) p = p.slice(0, -3);
+    if (p !== '/') p = p.replace(/\/$/, '');
+    return p === '' || p === '/index' ? '/' : p;
+  };
+
+  const listedRoutes = new Set(
+    (llmsText.match(/https:\/\/docs\.kite\.ai[^\s)]*/g) || []).map(toRoute).filter(Boolean)
+  );
+
+  const problems = [];
+  for (const { route, source } of mustBeListed) {
+    if (!listedRoutes.has(route)) {
+      problems.push(`The page ${source} exists in the docs but is not listed in static/llms.txt (expected ${SITE_URL}${route === '/' ? '/' : route}).`);
+    }
+  }
+  for (const listed of listedRoutes) {
+    if (!routes.has(listed)) {
+      problems.push(`static/llms.txt mentions ${SITE_URL}${listed}, but no page exists at that URL.`);
+    }
+  }
+
+  if (problems.length > 0) {
+    throw new Error(
+      'raw-markdown plugin: static/llms.txt is out of date with the docs.\n' +
+        problems.map((p) => `  - ${p}`).join('\n') +
+        '\nUpdate static/llms.txt to match the docs (add, rename, or remove the entries above), then rebuild.'
+    );
+  }
+}
+
+module.exports = function rawMarkdownPlugin(context) {
+  return {
+    name: 'raw-markdown',
+
+    async postBuild({ outDir, routesPaths }) {
+      const docsDir = path.join(context.siteDir, 'docs');
+      const routes = new Set(routesPaths.map((r) => (r === '/' ? '/' : r.replace(/\/$/, ''))));
+
+      const pages = listMarkdownFiles(docsDir).map((relPath) => {
+        const markdown = fs.readFileSync(path.join(docsDir, relPath), 'utf8');
+        return { relPath, markdown, route: routeForFile(relPath, markdown) };
+      });
+
+      const badRoutes = pages.filter((p) => !routes.has(p.route));
+      if (badRoutes.length > 0) {
+        const details = badRoutes.map((p) => `  ${p.relPath} -> ${p.route}`).join('\n');
+        throw new Error(
+          `raw-markdown plugin: derived URLs not found among built routes:\n${details}\n` +
+            'The route derivation in plugins/raw-markdown/index.js no longer matches how ' +
+            'Docusaurus routes these files (a rename or a custom slug is the usual cause).'
+        );
+      }
+
+      // 1. One .md file per page, at the page URL plus ".md".
+      for (const page of pages) {
+        const outPath =
+          page.route === '/' ? path.join(outDir, 'index.md') : path.join(outDir, `${page.route.slice(1)}.md`);
+        fs.mkdirSync(path.dirname(outPath), { recursive: true });
+        fs.writeFileSync(outPath, page.markdown);
+      }
+
+      // sidebars.js uses "export default"; require() wraps that in a module
+      // object whose default property holds the config.
+      const sidebarsModule = require(path.join(context.siteDir, 'sidebars.js'));
+      const sidebars = sidebarsModule.default ?? sidebarsModule;
+      const byId = new Map(pages.map((p) => [p.relPath.replace(/\.md$/, ''), p]));
+
+      // 2. A synthesized .md file for each auto-generated category index page,
+      // so appending .md works for those URLs too.
+      const generatedIndexes = collectGeneratedIndexes(sidebars.docsSidebar);
+      for (const generated of generatedIndexes) {
+        if (!routes.has(generated.slug)) {
+          throw new Error(`raw-markdown plugin: generated index slug "${generated.slug}" is not a built route.`);
+        }
+        const links = generated.childIds.map((id) => {
+          const page = byId.get(id);
+          if (!page) throw new Error(`raw-markdown plugin: sidebar doc id "${id}" has no matching file under docs/.`);
+          const title = readFrontmatterTitle(page.markdown) || id;
+          return `- [${title}](${SITE_URL}${page.route})`;
+        });
+        const body = `# ${generated.title}\n\n${generated.description}\n\n${links.join('\n')}\n`;
+        const outPath = path.join(outDir, `${generated.slug.slice(1)}.md`);
+        fs.mkdirSync(path.dirname(outPath), { recursive: true });
+        fs.writeFileSync(outPath, body);
+      }
+
+      // 3. Fail the build if llms.txt no longer matches the pages that exist.
+      const mustBeListed = [
+        ...pages.map((p) => ({ route: p.route, source: `docs/${p.relPath}` })),
+        ...generatedIndexes.map((g) => ({
+          route: g.slug,
+          source: `the generated "${g.title}" index in sidebars.js`,
+        })),
+      ];
+      checkLlmsIndex(context.siteDir, mustBeListed, routes);
+
+      // 4. llms-full.txt: every page in sidebar order, in one file.
+      const orderedIds = flattenSidebar(sidebars.docsSidebar);
+      const orderedPages = orderedIds.map((id) => {
+        const page = byId.get(id);
+        if (!page) throw new Error(`raw-markdown plugin: sidebar doc id "${id}" has no matching file under docs/.`);
+        return page;
+      });
+
+      const header =
+        '# Kite Docs\n\n' +
+        '> Kite is your AI marketer in Slack. It researches your business, builds a growth strategy, and ships the work that brings you new customers.\n\n' +
+        `This file contains every page of the Kite help center (${SITE_URL}) as Markdown. ` +
+        'Each page begins with its canonical URL. A per-page index is at ' +
+        `${SITE_URL}/llms.txt, and each page is also available on its own by appending .md to its URL.\n`;
+
+      const body = orderedPages
+        .map((page) => {
+          const url = page.route === '/' ? SITE_URL + '/' : SITE_URL + page.route;
+          return `---\nCanonical URL: ${url}\n---\n\n${stripFrontmatter(page.markdown).trim()}\n`;
+        })
+        .join('\n');
+
+      fs.writeFileSync(path.join(outDir, 'llms-full.txt'), `${header}\n${body}`);
+
+      console.log(`[raw-markdown] Wrote ${pages.length} .md pages and llms-full.txt to ${outDir}`);
+    },
+  };
+};