diff --git a/.github/scripts/check-search-heap.mjs b/.github/scripts/check-search-heap.mjs new file mode 100644 index 00000000..414c4e57 --- /dev/null +++ b/.github/scripts/check-search-heap.mjs @@ -0,0 +1,179 @@ +#!/usr/bin/env node +/** + * The search heap gate for #393: the BUILT `/api/search` route stays under a + * heap budget while it is asked in every locale `apps/docs/lib/i18n.ts` + * declares. + * + * #390 added eleven ja pages, every gate was green, and after the deploy + * `/api/search` answered 503 in two or three locales per smoke run, so the site + * rolled back twice. A Cloudflare Worker isolate has 128 MB. The route kept + * every locale's Orama index (30–40 MB each) in one map, so the second or third + * locale an isolate served pushed it over. Nothing measured that before the + * deploy: the route is dynamic, and `check-search-locales.mjs` asks the right + * questions but not how much memory the answers cost. Measured on the built + * route at `09784e7`, heap after a forced GC: 58 MB after the first locale, + * 283 MB after all eight. + * + * So this gate asks the way an isolate is asked. It loads the route handler out + * of `apps/docs/.next/`, calls its `GET` in-process with `?query=permissions` + * in each locale in turn, and then once more in each, and reads `heapUsed` + * after a forced GC after every request: + * + * - `threw`, `status`: a request that throws or answers anything but 200. + * A heap reading after a failed request measures nothing. + * - `over-budget`: the heap after any request is above `BUDGET`. 100 MB + * leaves 28 MB of the isolate's 128 MB for what Node's `heapUsed` does not + * count (code, the Worker runtime) and for the peak while an index builds. + * - `still-growing`: the second round settles more than `GROWTH` above the + * first round's highest reading. Whatever the route keeps per locale has to + * stop growing once every locale has been asked; a leak under the budget + * today is the next content PR's 503. + * + * Run it after `pnpm turbo run build`, because a missing build fails it. It + * needs `--expose-gc` and re-runs itself with the flag when started without it. + * `--self-test` runs every rule against fake handlers and fake heap readings, + * and needs no build. + */ +import { spawnSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const ROUTE = 'apps/docs/.next/server/app/api/search/route.js'; +const QUERY = 'permissions'; +const MB = 1024 * 1024; +const BUDGET = 100 * MB; // `over-budget`: of a Worker isolate's 128 MB +const GROWTH = 8 * MB; // `still-growing`: how far round two may settle above round one + +const RULES = ['threw', 'status', 'over-budget', 'still-growing']; + +/** `languages` out of `i18n.ts`. Throws rather than guessing. */ +function readLanguages(text) { + const list = /languages:\s*\[([^\]]+)\]/.exec(text)?.[1]; + const languages = list?.split(',').map((s) => s.trim().replace(/['"]/g, '')).filter(Boolean) ?? []; + if (!languages.length) throw new Error('could not read languages[] out of apps/docs/lib/i18n.ts'); + return languages; +} + +/** One request. A rejection is recorded, because Next turns it into a 500. */ +async function ask(GET, locale) { + const url = `http://localhost/api/search?locale=${encodeURIComponent(locale)}&query=${QUERY}`; + try { + const res = await GET(new Request(url)); + await res.arrayBuffer(); + return { status: res.status }; + } catch (err) { + return { threw: err?.code ?? String(err?.message ?? err).split('\n')[0] }; + } +} + +const mb = (bytes) => `${(bytes / MB).toFixed(1)} MB`; + +/** Two rounds over `languages`; `heap()` reads `heapUsed` after a forced GC. */ +async function evaluate(GET, languages, heap) { + const findings = []; + const rows = []; + const add = (rule, detail) => findings.push({ rule, detail }); + const peaks = [0, 0]; + const settled = [0, 0]; + rows.push(['route loaded', mb(heap())]); + for (const round of [0, 1]) { + for (const locale of languages) { + const r = await ask(GET, locale); + const label = `${round ? 'again ' : ''}${locale}`; + if (r.threw) add('threw', `${label}: the handler threw ${r.threw}, which Next serves as a 500`); + else if (r.status !== 200) add('status', `${label}: HTTP ${r.status}`); + if (r.threw || r.status !== 200) return { findings, rows }; + const used = heap(); + rows.push([label, mb(used)]); + if (used > BUDGET) add('over-budget', `${label}: ${mb(used)} after GC, above the ${mb(BUDGET)} budget`); + peaks[round] = Math.max(peaks[round], used); + settled[round] = used; + } + } + if (settled[1] - peaks[0] > GROWTH) { + add('still-growing', `round two settled at ${mb(settled[1])}, ${mb(settled[1] - peaks[0])} above round one's highest ${mb(peaks[0])}, more than ${mb(GROWTH)}`); + } + return { findings, rows }; +} + +async function gate() { + if (typeof globalThis.gc !== 'function') { + const child = spawnSync(process.execPath, ['--expose-gc', fileURLToPath(import.meta.url), ...process.argv.slice(2)], { stdio: 'inherit' }); + return child.status ?? 1; + } + const route = join(ROOT, ROUTE); + if (!existsSync(route)) { + console.error(`✗ search heap: ${ROUTE} does not exist. Run pnpm turbo run build first.`); + return 1; + } + const languages = readLanguages(readFileSync(join(ROOT, 'apps/docs/lib/i18n.ts'), 'utf8')); + const GET = (await createRequire(import.meta.url)(route))?.routeModule?.userland?.GET; + if (typeof GET !== 'function') { + console.error(`✗ search heap: ${ROUTE} exposes no routeModule.userland.GET, so nothing was measured.`); + return 1; + } + const heap = () => { + globalThis.gc(); + globalThis.gc(); + return process.memoryUsage().heapUsed; + }; + const { findings, rows } = await evaluate(GET, languages, heap); + console.log(['| after | heapUsed after GC |', '|:--|--:|', ...rows.map(([a, b]) => `| ${a} | ${b} |`)].join('\n')); + for (const f of findings) console.error(` (${f.rule}) ${f.detail}`); + if (findings.length) { + console.error(`\n✗ search heap: ${findings.length} finding(s). The rules are in this script's header.`); + return 1; + } + console.log(`\n✓ search heap: every locale asked twice, heap stayed under ${mb(BUDGET)} and stopped growing`); + return 0; +} + +/* Self-test: one clean fake, then one fake per rule that must break exactly that rule. */ +const LANGS = ['en', 'ja', 'de']; +const ok = async () => new Response('[]'); +/** Heap readings in order: route loaded, then one per request across both rounds. */ +const readings = (...values) => { + const queue = values.map((v) => v * MB); + return () => queue.shift() ?? queue.at(-1) ?? 0; +}; +const CASES = [ + ['clean: levels off under the budget', ok, readings(18, 60, 85, 86, 86, 85, 86), []], + ['ja throws', async (req) => (new URL(req.url).searchParams.get('locale') === 'ja' ? Promise.reject(new Error('x')) : new Response('[]')), readings(18, 60), ['threw']], + ['ja answers 503, as the Worker did', async (req) => new Response('', { status: new URL(req.url).searchParams.get('locale') === 'ja' ? 503 : 200 }), readings(18, 60), ['status']], + ['every index kept: over the budget', ok, readings(18, 58, 98, 137, 137, 137, 137), ['over-budget', 'over-budget', 'over-budget', 'over-budget']], + ['a slow leak under the budget', ok, readings(18, 60, 64, 68, 72, 76, 80), ['still-growing']], + ['round two at round one\'s peak is level', ok, readings(18, 60, 90, 70, 90, 70, 90), []], +]; + +async function selfTest() { + let failed = 0; + const fired = new Set(); + for (const [name, GET, heap, want] of CASES) { + const got = (await evaluate(GET, LANGS, heap)).findings.map((f) => f.rule).sort(); + got.forEach((r) => fired.add(r)); + const pass = JSON.stringify(got) === JSON.stringify([...want].sort()); + if (!pass) failed += 1; + console.log(`${pass ? '✓' : '✗'} ${name}: ${got.length ? got.join(', ') : 'no findings'}`); + } + const silent = RULES.filter((r) => !fired.has(r)); + if (silent.length) { + failed += 1; + console.log(`✗ rules no fixture makes fire: ${silent.join(', ')}`); + } + const langsOk = readLanguages("defineI18n({ languages: ['en', 'zh-Hans'] })").join() === 'en,zh-Hans'; + let unreadable = false; + try { + readLanguages('defineI18n({})'); + } catch { + unreadable = true; + } + if (!langsOk || !unreadable) failed += 1; + console.log(`${langsOk ? '✓' : '✗'} i18n.ts is read; ${unreadable ? '✓' : '✗'} an unreadable one is rejected`); + console.log(failed ? `\n✗ search heap self-test: ${failed} failure(s)` : `\n✓ search heap self-test: ${CASES.length} cases, all ${RULES.length} rules can fire`); + return failed ? 1 : 0; +} + +process.exitCode = process.argv.includes('--self-test') ? await selfTest() : await gate(); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 266f3410..0ed37e43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,6 +199,18 @@ jobs: shell: bash run: node .github/scripts/check-search-locales.mjs | tee -a "$GITHUB_STEP_SUMMARY" + # #393: #390's eleven ja pages passed every step above, and after the + # deploy `/api/search` answered 503 and the site rolled back twice. A + # Worker isolate has 128 MB, and the route kept every locale's index + # (30–40 MB each). This step asks the BUILT handler in every locale, then + # in each once more, the way one isolate is asked, and fails when the + # heap after a forced GC goes above 100 MB or is still growing on the + # second pass. A content PR that would tip the Worker over goes red here, + # not after the deploy. Same footing as the step above. + - name: Search fits the Worker's memory + shell: bash + run: node --expose-gc .github/scripts/check-search-heap.mjs | tee -a "$GITHUB_STEP_SUMMARY" + - run: pnpm turbo run test # Defect 2 of #269: the deploy used to run its own `pnpm install` and its diff --git a/apps/docs/app/api/search/route.ts b/apps/docs/app/api/search/route.ts index 46004afc..b41e3342 100644 --- a/apps/docs/app/api/search/route.ts +++ b/apps/docs/app/api/search/route.ts @@ -1,5 +1,6 @@ import { source } from '@/lib/source'; import { i18n } from '@/lib/i18n'; +import { searchData } from '@/.search'; import { createSearchAPI, type AdvancedIndex, @@ -118,22 +119,65 @@ function breadcrumbs(locale: Locale, url: string): string[] | undefined { return names.filter((name): name is string => typeof name === 'string' && name.length > 0); } -/** `locale`'s pages, indexed as `createFromSource` indexes them. */ +type StructuredData = AdvancedIndex['structuredData']; + +/** + * One locale's `structuredData` — each page's headings and paragraphs — keyed + * by file path under `content/docs`, from `.search/` (#393). + * + * `scripts/gen-search-data.mjs` writes it before every build, with the same + * remark chain fumadocs-mdx compiles the pages with. Reading it from there + * instead of from `page.data.load()` is the point: `load()` imports the page's + * compiled module, the module registry keeps it for the life of the isolate, + * and one locale's page modules are 4–5 MB of heap that outlived its index. + * The module imported here is one string, and the unzipped, parsed copy is + * dropped once the index has the text. + */ +async function texts(locale: Locale): Promise> { + // Gzipped JSON in base64, because the module is Worker upload too. + const packed = (await searchData[locale]()).default; + const bytes = Uint8Array.from(atob(packed), (c) => c.charCodeAt(0)); + const json = new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip')); + return JSON.parse(await new Response(json).text()); +} + +/** + * `locale`'s pages, indexed as `createFromSource` indexes them. A page that + * has no translation in `locale` is the default language's file, so its text + * is read from the default language's module. + */ async function indexes(locale: Locale): Promise { + const [own, fallback] = await Promise.all([ + texts(locale), + locale === i18n.defaultLanguage ? undefined : texts(i18n.defaultLanguage), + ]); return Promise.all( - source.getPages(locale).map(async (page) => ({ - id: page.url, - url: page.url, - title: page.data.title, - description: page.data.description, - structuredData: (await page.data.load()).structuredData, - breadcrumbs: breadcrumbs(locale, page.url), - })), + source.getPages(locale).map(async (page) => { + const file = page.data.info.path; + let structuredData = own[file] ?? fallback?.[file]; + if (!structuredData) { + // `next dev` serves pages added since it wrote `.search/`; a build + // writes it first, so there a missing page is a stale `.search/`. + if (process.env.NODE_ENV !== 'development') { + throw new Error(`/api/search: no search data for ${file}; run scripts/gen-search-data.mjs`); + } + structuredData = (await page.data.load()).structuredData; + } + return { + id: page.url, + url: page.url, + title: page.data.title, + description: page.data.description, + structuredData, + breadcrumbs: breadcrumbs(locale, page.url), + }; + }), ); } /** - * One index per locale, built on that locale's first search (#301). + * One index per locale, built on that locale's first search (#301), and only + * the most recent `KEEP` of them kept (#393, below). * * `createFromSource` with i18n builds every locale's index on the first search * in any locale, so the first reader after a cold start paid for all eight: @@ -156,16 +200,38 @@ async function indexes(locale: Locale): Promise { */ const servers = new Map(); +/** + * How many locales' indexes one isolate keeps (#393). + * + * One index is 30–40 MB of heap, and a Cloudflare Worker isolate has 128 MB in + * all. The map above used to keep every locale an isolate had served, so the + * second or third locale pushed it over and `/api/search` answered 503: on the + * built route at `09784e7`, heap after a forced GC grew from 58 MB after the + * first locale to 283 MB after all eight. Keeping the most recently used one + * holds it level. A search in another locale then pays that locale's build + * again (0.5–1.5 s), and a second entry would leave too little room for that + * build's peak. `.github/scripts/check-search-heap.mjs` measures the level + * after every build. + */ +const KEEP = 1; + function server(locale: Locale): SearchAPI { let found = servers.get(locale); - if (!found) { + if (found) { + // Most recently used last, so the first key is the one to drop. + servers.delete(locale); + } else { const options: Partial = localeOptions[locale]; found = createSearchAPI('advanced', { ...options, search: { ...options.search, sortBy: byWeightedScore }, indexes: () => indexes(locale), }); - servers.set(locale, found); + } + servers.set(locale, found); + for (const oldest of servers.keys()) { + if (servers.size <= KEEP) break; + servers.delete(oldest); } return found; } diff --git a/apps/docs/package.json b/apps/docs/package.json index f7a3034d..7d875e3c 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -5,12 +5,12 @@ "description": "ObjectOS Documentation Site", "license": "Apache-2.0", "scripts": { - "dev": "next dev", + "dev": "fumadocs-mdx && node scripts/gen-search-data.mjs && next dev", "gen:zh-hant": "node scripts/gen-zh-hant.mjs", - "build": "next build", + "build": "fumadocs-mdx && node scripts/gen-search-data.mjs && next build", "start": "next start", "lint": "next lint", - "type-check": "next typegen && fumadocs-mdx && tsc --noEmit && node scripts/check-frontmatter.mjs", + "type-check": "next typegen && fumadocs-mdx && node scripts/gen-search-data.mjs && tsc --noEmit && node scripts/check-frontmatter.mjs", "postinstall": "fumadocs-mdx", "preview": "opennextjs-cloudflare build && opennextjs-cloudflare preview", "deploy": "opennextjs-cloudflare build && opennextjs-cloudflare deploy", diff --git a/apps/docs/scripts/gen-search-data.mjs b/apps/docs/scripts/gen-search-data.mjs new file mode 100644 index 00000000..59c20e38 --- /dev/null +++ b/apps/docs/scripts/gen-search-data.mjs @@ -0,0 +1,121 @@ +#!/usr/bin/env node +/** + * Writes the text `/api/search` indexes, one module per locale (#393). + * + * The route used to read each page's `structuredData` (its headings and + * paragraphs) through `page.data.load()`, which imports the page's compiled + * MDX module. That is the whole rendered page, highlighted code included, and + * a module once imported stays in the module registry for the life of the + * isolate. Building one locale's index therefore left every page module of + * that locale resident after the index was gone. Measured on the built route + * at `09784e7`, with only one index kept: 58 MB after English, 86 MB after all + * eight locales, about 94 MB with ja fully translated, against a Worker + * isolate's 128 MB. + * + * So the same `structuredData` is computed here, before `next build`, and + * written as one string per locale to `.search/.ts`: the JSON, + * gzipped, in base64. The route imports the locale it is asked for, unzips and + * parses it. What stays resident is that string, not the page modules. + * + * Gzipped because the string is also Worker upload, which has a budget the + * `Worker bundle fits the size budget` CI step holds: as plain JSON the + * English, zh-Hans and zh-Hant modules were about 1 MB each, 3.5 MB of the + * 6 MB headroom main had at `09784e7`; gzipped and in base64, about a third. + * + * It is the same `structuredData`, not a lookalike. The processor is + * fumadocs-mdx's own remark chain, built with the public `applyMdxPreset` from + * the compiled `source.config` (`.source/source.config.mjs`, written by the + * `fumadocs-mdx` CLI), plus `remarkInclude`, which `buildMDX` puts first. + * `remarkStructure` runs in the remark phase, so the rehype plugins, which run + * after it and only shape the rendered HTML, are left out: that is + * `rehypeCode`, and it is most of the cost. + * + * A file's locale is its suffix, as `source.config.ts` reads it: `foo.ja.mdx` + * is `ja`, `foo.mdx` is the default language. A locale whose page falls back + * to English reads that page from the default language's module, so English + * text is written once. + * + * The output directory carries its own `.gitignore`, because it is a build + * product. Run from `apps/docs`, after `fumadocs-mdx`, as the `build`, + * `type-check` and `dev` scripts do. + */ +import { applyMdxPreset, remarkInclude } from 'fumadocs-mdx/config'; +import { frontmatter } from 'fumadocs-core/content/md/frontmatter'; +import { createRequire } from 'node:module'; +import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join, relative, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { gzipSync } from 'node:zlib'; + +const APP = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const CONTENT = resolve(APP, '../../content/docs'); +const CONFIG = join(APP, '.source/source.config.mjs'); +const OUT = join(APP, '.search'); + +/** `languages` and `defaultLanguage` out of `lib/i18n.ts`, read as text. Throws rather than guessing. */ +function readI18n() { + const text = readFileSync(join(APP, 'lib/i18n.ts'), 'utf8'); + const languages = /languages:\s*\[([^\]]+)\]/.exec(text)?.[1]?.split(',').map((s) => s.trim().replace(/['"]/g, '')).filter(Boolean) ?? []; + const defaultLanguage = /defaultLanguage:\s*['"]([^'"]+)['"]/.exec(text)?.[1]; + if (!languages.length || !languages.includes(defaultLanguage)) { + throw new Error('could not read languages[] and defaultLanguage out of apps/docs/lib/i18n.ts'); + } + return { languages, defaultLanguage }; +} + +function walk(dir) { + return readdirSync(dir, { withFileTypes: true }) + .flatMap((e) => (e.isDirectory() ? walk(join(dir, e.name)) : e.name.endsWith('.mdx') ? [join(dir, e.name)] : [])) + .sort(); +} + +if (!existsSync(CONFIG)) { + console.error(`✗ search data: ${relative(process.cwd(), CONFIG)} does not exist. Run \`fumadocs-mdx\` first.`); + process.exit(1); +} + +const { languages, defaultLanguage } = readI18n(); +// @mdx-js/mdx is fumadocs-mdx's dependency, not this app's, so it is resolved from there: the copy buildMDX uses. +const { createProcessor } = await import( + pathToFileURL(createRequire(import.meta.resolve('fumadocs-mdx')).resolve('@mdx-js/mdx')).href +); +const config = (await import(pathToFileURL(CONFIG).href)).default; +const mdx = await applyMdxPreset({ ...config.mdxOptions, rehypeCodeOptions: false })('bundler'); +const processor = createProcessor({ + outputFormat: 'program', + ...mdx, + remarkPlugins: [remarkInclude, ...(mdx.remarkPlugins ?? [])], + rehypePlugins: [], + format: 'mdx', +}); + +const byLocale = new Map(languages.map((l) => [l, {}])); +for (const path of walk(CONTENT)) { + const file = relative(CONTENT, path).split('\\').join('/'); + const locale = languages.find((l) => l !== defaultLanguage && file.endsWith(`.${l}.mdx`)) ?? defaultLanguage; + const vfile = await processor.process({ + path, + value: frontmatter(readFileSync(path, 'utf8')).content, + data: { frontmatter: {}, _getProcessor: () => processor }, + }); + if (!vfile.data.structuredData) throw new Error(`${file}: remarkStructure wrote no structuredData`); + byLocale.get(locale)[file] = vfile.data.structuredData; +} + +rmSync(OUT, { recursive: true, force: true }); +mkdirSync(OUT, { recursive: true }); +writeFileSync(join(OUT, '.gitignore'), '*\n'); +const banner = '// Generated by scripts/gen-search-data.mjs before every build. Do not edit.\n'; +for (const [locale, pages] of byLocale) { + // A string, not an object literal: the route parses it when it builds the + // index, and the parsed copy is garbage once the index holds the text. + const packed = gzipSync(JSON.stringify(pages), { level: 9 }).toString('base64'); + writeFileSync(join(OUT, `${locale}.ts`), `${banner}export default ${JSON.stringify(packed)};\n`); +} +const loaders = languages.map((l) => ` ${JSON.stringify(l)}: () => import(${JSON.stringify(`./${l}`)}),`).join('\n'); +writeFileSync( + join(OUT, 'index.ts'), + `${banner}/** Each locale's \`structuredData\`, keyed by file path under content/docs, as gzipped JSON in base64. */\nexport const searchData = {\n${loaders}\n} as const;\n`, +); +const sizes = [...byLocale].map(([l, pages]) => `${l} ${Object.keys(pages).length}`).join(', '); +console.log(`✓ search data: wrote .search/ (pages per locale: ${sizes})`); diff --git a/tools/ci-scripts/run-self-tests.mjs b/tools/ci-scripts/run-self-tests.mjs index f5cbadad..fb1f15f6 100644 --- a/tools/ci-scripts/run-self-tests.mjs +++ b/tools/ci-scripts/run-self-tests.mjs @@ -114,6 +114,13 @@ * malformed pin or SHA is refused, that the same commit bumps nothing, and that * the pull-request body stays inside GitHub's size limit with every upstream * reference inside a fence. + * + * `check-search-heap.mjs` is listed for its `--self-test` only (#393), on the + * footing of `check-search-locales.mjs`: its gate mode loads the built search + * route out of `apps/docs/.next/`, so it is a `ci.yml` step after the build. + * Its fixtures are fake handlers and fake heap readings. They prove that a + * failed request, a reading above the budget, and a heap still growing on the + * second pass each go red. */ import { readFileSync, readdirSync, existsSync } from 'node:fs'; @@ -137,6 +144,7 @@ const SELF_TESTED = [ 'check-translation-ownership.mjs', 'check-positioning.mjs', 'check-search-locales.mjs', + 'check-search-heap.mjs', 'check-doc-samples.mjs', 'bump-upstream-pin.mjs', ];