diff --git a/nuxt.config.ts b/nuxt.config.ts index 7c58fc53..24cbc84b 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -292,7 +292,43 @@ export default defineNuxtConfig({ }, aiReady: { - llmsTxt: { directive: { html: true } }, + llmsTxt: { + directive: { html: true }, + pages: 'sitemap', + // Retired URLs remain in the search removal sitemap only. + sitemaps: ['pages', 'skills', 'demos'], + markdownLinks: true, + maxLength: 80_000, + sections: [ + { + title: 'Agent setup', + links: [ + { title: 'Set up skilld', href: 'https://skilld.dev/agent.md', description: 'Setup steps an Agent reads and follows: the CLI, the MCP server, or a plain URL fetch.' }, + { title: 'OpenAPI', href: 'https://skilld.dev/api/v1/openapi.json', description: 'The public API contract.' }, + ], + }, + { + title: 'Indexes', + links: [ + { title: 'AI sitemap', href: 'https://skilld.dev/ai-sitemap.xml', description: 'Every AI-ready page, including pages that are noindex for Google.' }, + { title: 'Sitemap', href: 'https://skilld.dev/sitemap_index.xml', description: 'XML sitemap for search engines and crawlers.' }, + ], + }, + { + title: 'Skills', + description: 'Every Skill page and track in the skilld sitemap.', + sitemap: 'skills', + path: '/llms/skills.txt', + }, + { + title: 'Skill Demos', + description: 'Each demo is one recorded run: the prompt, and what the Agent built with the Skill.', + sitemap: 'demos', + path: '/llms/demos.txt', + }, + ], + }, + llmsFullTxt: false, database: { type: 'd1', bindingName: 'DB', @@ -305,9 +341,6 @@ export default defineNuxtConfig({ aiInput: true, aiTrain: true, }, - // `server/middleware/llms-txt.ts` serves /llms.txt and its nested files - // from the sitemaps, ahead of the module's route. The 28.5 MB - // llms-full.txt inline dump is retired below (routeRules). // The module's negotiation also reads User-Agent, which a shared cache // cannot key on. server/handlers/content-negotiation.ts decides from // Accept and Sec-Fetch-Dest instead. Explicit `.md` URLs and the diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0c1e1f7d..3875c6b5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -386,7 +386,7 @@ catalogs: version: 4.146.0 overrides: - nuxt-ai-ready: 3.3.0 + nuxt-ai-ready: 3.3.1 patchedDependencies: hookable@5.5.3: 3b44da13a743bfa2eab02aef7246065e4dfd56e9878c206f34b51322832cfb2c @@ -2247,76 +2247,76 @@ packages: engines: {node: '>=18'} hasBin: true - '@mdream/js@2.0.1': - resolution: {integrity: sha512-1br+LISK+zpWTOWakKteZTQ/PjdKKcL4ZXssBBGZaVRbtHepXUQP7g2ZVIvUEucbAmmYagBGV5T21r612NXcag==} + '@mdream/js@2.0.3': + resolution: {integrity: sha512-n35jmVwE663LBHr9iawaIbO+zpJ5EoCFD9UpG4XRYHs//YUqIX322d/F+QuF08Yi64dpFfzz4p4/joTnVZDvJA==} hasBin: true - '@mdream/rust-android-arm-eabi@2.0.1': - resolution: {integrity: sha512-kPnJLpUoreSuz5aLfw02RQgVgi+nnHXoMaJLoel3COF3tP0zcX1L5k3y9T4Eh3x5g3GDs9dpdMO0ENr29dpZJg==} + '@mdream/rust-android-arm-eabi@2.0.3': + resolution: {integrity: sha512-tL0Oq5xm4Bwph3NA2Ymuk2obGBuHZfN+TUPlBgowzuPv+yZK2G4zQ4xr4gmYQBVG6xPt9pygiM6hLMm6Q3xn1g==} engines: {node: '>= 22.0.0'} cpu: [arm] os: [android] - '@mdream/rust-android-arm64@2.0.1': - resolution: {integrity: sha512-t42oAiMhGf20eBAJPoHuoCdNeu8exgL2dKE1DsHJL1U0Eh4B84DFnDlqjtjsF0bZB4g0rHHImwd3vIi5zxzkDA==} + '@mdream/rust-android-arm64@2.0.3': + resolution: {integrity: sha512-9yAyPh2yF2rjkKHviDmJwOiZE8oIIwQIP7XCCVrkzNfg0hxJikom6UUwKvTTmH9B/2NafUv6clIV0fl4hZ9TwA==} engines: {node: '>= 22.0.0'} cpu: [arm64] os: [android] - '@mdream/rust-darwin-arm64@2.0.1': - resolution: {integrity: sha512-nsLyhSBA1+MK/7owbPfkbwDEp6RxMpM+LVGaYa9JKt2jH7AyGwHjadp6PnQu777EsDF7b37/tXaoVejTXVP3CQ==} + '@mdream/rust-darwin-arm64@2.0.3': + resolution: {integrity: sha512-0tArymmWZ1Q1CVrTsdePxzuibB00+kJ8qPWIGh8sNiaXm9TKj+G1OprOxZYzZNerQKKZKF5MpMhKgYcyQdYliw==} engines: {node: '>= 22.0.0'} cpu: [arm64] os: [darwin] - '@mdream/rust-darwin-x64@2.0.1': - resolution: {integrity: sha512-C269lQq62V3LEv2581jr2mnCsyyIlN4ExHAWwP/K9ZTo9UK1Ke/AMSa50aXNBT+JZIo/7CrtOSnECFcnUS5jGQ==} + '@mdream/rust-darwin-x64@2.0.3': + resolution: {integrity: sha512-NvKFknrbNOLlK75Mb8/FbrGlJ69qyq6q7pNHbVqclrF79RgdTnL5UbAGv/EYquEa5BfQgpyINAZgtEcvxwILzA==} engines: {node: '>= 22.0.0'} cpu: [x64] os: [darwin] - '@mdream/rust-freebsd-x64@2.0.1': - resolution: {integrity: sha512-HDmbFbcr5fUIWn1jmqMY7lVI9UcKPZlG6NYnyq351c0SjBfXbHJGVZ3tEuF0pi8nH+glhrm89mhDV2krjhuasw==} + '@mdream/rust-freebsd-x64@2.0.3': + resolution: {integrity: sha512-JFV6wtVp2yZRldJqsOeVYd+d11RxVYYytLeT6lWNzYR2hlANFTDHAvq00rhWS2aZW6ZzHU+oHyNbxO5MXJgxTg==} engines: {node: '>= 22.0.0'} cpu: [x64] os: [freebsd] - '@mdream/rust-linux-arm-gnueabihf@2.0.1': - resolution: {integrity: sha512-26uSbt3BGzwsTIL6+j4e9z5fCSY0kShx6MFTYByVFGas0nIXWNiAYhuOxDqR4drgIdUeDb4s7QwXX6WtxsLWZA==} + '@mdream/rust-linux-arm-gnueabihf@2.0.3': + resolution: {integrity: sha512-QVLP41g2oItpV6wqNyKF3CkoDoz9sd/Sh8X27i4oBKy4jH3F7MjvEXX7+e548XY0iSrzCQq4MGSwG1tUx9jmmQ==} engines: {node: '>= 22.0.0'} cpu: [arm] os: [linux] - '@mdream/rust-linux-arm64-gnu@2.0.1': - resolution: {integrity: sha512-EherloN6J/njIOKPM7qY3xDQvQIP0GL57sAAPUbJ8GGAM8DHDPRXLLHyd9z+saypHwF/mXns+G7x3GwEZUQIrA==} + '@mdream/rust-linux-arm64-gnu@2.0.3': + resolution: {integrity: sha512-5E06gnO+NWrjOoTszCidCv/IivLJj8joPEw6Wn8H3Qc6wRniUrlqe2LoOOTgNHyT2GR7hPXRP6bBEbMQYucD+A==} engines: {node: '>= 22.0.0'} cpu: [arm64] os: [linux] libc: [glibc] - '@mdream/rust-linux-arm64-musl@2.0.1': - resolution: {integrity: sha512-8nNq3I/nBzRAOcUCGsJ0FvCS0dqxwuXljR/zoi6dlwULoin43z+xshOUOWyL8XMmdmAGNK9fCBPqhp8SAgxyEQ==} + '@mdream/rust-linux-arm64-musl@2.0.3': + resolution: {integrity: sha512-06b/C+5Nt3iEaR/u4aKU4Zm9zYBFvAqWwOGzmkLoRj6gEk4gAteSMNePsx8nA3UGbm34WEOOD+vG6Wj8yU14yw==} engines: {node: '>= 22.0.0'} cpu: [arm64] os: [linux] libc: [musl] - '@mdream/rust-linux-x64-gnu@2.0.1': - resolution: {integrity: sha512-VE2u4z0YM31eIFprVIJfHGBOqEsbnI9/dqyiBrqGn9in+7cguRuxpkoKWaQC1c6DPZOk8GWmkl4Dtoq9SiUCog==} + '@mdream/rust-linux-x64-gnu@2.0.3': + resolution: {integrity: sha512-qmlIZS7vp+g/J9CFJJkUSP5cStVR64gfzwK0tmswbbGnIMgCfb3wn/KBTaJSn7Fhvvjlwh6CNFwmJ6jMd9TXfg==} engines: {node: '>= 22.0.0'} cpu: [x64] os: [linux] libc: [glibc] - '@mdream/rust-linux-x64-musl@2.0.1': - resolution: {integrity: sha512-UtzHzsXfPlw7Nfr2pB7u7U8qYop0e2XAnDRsjdIbGunHjF8bkJ6246L+oOQhoCaC50oK9v77Gk1Qqtgvhvqgyw==} + '@mdream/rust-linux-x64-musl@2.0.3': + resolution: {integrity: sha512-7WBEn+xmmVL/WT3du40kY0ilQoom6GlLKIyL2F2rvdRgenyjZ2eq34YYdg3DQa+mDcZZAsKn/TAR9xSzz8o+gw==} engines: {node: '>= 22.0.0'} cpu: [x64] os: [linux] libc: [musl] - '@mdream/rust-wasm32-wasi@2.0.1': - resolution: {integrity: sha512-pCSab2uruCcEhWyR5674Moh+9tphtLQDQvPOC2X0OdDmAe35Xf2TYT6rl0QoKSxRqqPRlOoMEpwEXueTtYCDNQ==} + '@mdream/rust-wasm32-wasi@2.0.3': + resolution: {integrity: sha512-gjKh5NMj7Yr537HfiTZxlrRbmJ36S1mS16ytCJMwsJ9H0VtCWYSAdTFQJI27S6yLl+yAFZgtO2rPnqlnX2kaQA==} engines: {node: '>= 22.0.0'} cpu: [wasm32] bundledDependencies: @@ -2327,14 +2327,14 @@ packages: - '@emnapi/wasi-threads' - tslib - '@mdream/rust-win32-arm64-msvc@2.0.1': - resolution: {integrity: sha512-MDi6Adxmsp6EuPLwv5WfIGDP4xn/fXDwi985avz2q/3x0Bz623xqKb07GkwBrsZ1K2I2mGivDZc/fTtBuBiwmQ==} + '@mdream/rust-win32-arm64-msvc@2.0.3': + resolution: {integrity: sha512-Ru7jM9QdrhJBspyCe43kwH7WHso/8WiE76AAQEho9OWo2xoiDKxPI02ecdAzRHbmhIVybUvPAbY4/MEzfr3/kA==} engines: {node: '>= 22.0.0'} cpu: [arm64] os: [win32] - '@mdream/rust-win32-x64-msvc@2.0.1': - resolution: {integrity: sha512-u5kZlejphCRiQam2zprGWbv4b8Mv9ZKmx0coLhndDK9PnbaVd/IyNAA0jG0H6CbuevLBf4D+3TxUeFAz5itpXQ==} + '@mdream/rust-win32-x64-msvc@2.0.3': + resolution: {integrity: sha512-QiVAh+WDxHxYt3pkJeGdhVgm7MH5SEa2odSXcIannPC1bu6ylorZ1B9Xw9vxMKk/BmIHpFi+16r1FIEohziP6w==} engines: {node: '>= 22.0.0'} cpu: [x64] os: [win32] @@ -7238,8 +7238,8 @@ packages: mdn-data@2.29.0: resolution: {integrity: sha512-pVxQFCcaYUEAH853+v7yoI/qzhxXSq1bTb9obMYGYAN1c3Hen+XDCEvr296XhstrwlSTNgOR7mCSD4JPjbJe5A==} - mdream@2.0.1: - resolution: {integrity: sha512-QcQgjArVbvTTY7kfbTH065UYymjc9OMedLrI7UjErukEHfKxjKxx9MmKU+8s0d9xD8MDdfD4HAi1vOt7i7FRyA==} + mdream@2.0.3: + resolution: {integrity: sha512-MQTuW2cHn/aHXH6zc23PmWys7vzw+yP8V8e6YUNkXK9tHA2dORNz0X8mkKqYOqHBDXWIyjYHtxVVFtpwactXoQ==} hasBin: true mdurl@2.1.0: @@ -7578,8 +7578,8 @@ packages: nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} - nuxt-ai-ready@3.3.0: - resolution: {integrity: sha512-n1imTpXUmVblc7x8x52dyOSheEO32j3yAvvxzMBxo7uIsBCuiT2BJn5jLJRw3YUu0u5tKsZ0wCQUKtlmLqn/eg==} + nuxt-ai-ready@3.3.1: + resolution: {integrity: sha512-sgOLdQlVHM1apqXRcnGMMp34h2ppFBw0f5UF9GdkL19eeE4LNa6XvxwqPSuu9TpI+X0Luu1CMvAiYqa6Dxw/5w==} engines: {node: ^22.22.3 || ^24.15.0 || >=26.0.0} hasBin: true peerDependencies: @@ -11398,48 +11398,48 @@ snapshots: - encoding - supports-color - '@mdream/js@2.0.1': + '@mdream/js@2.0.3': dependencies: cac: 7.1.0 pathe: 2.0.3 - '@mdream/rust-android-arm-eabi@2.0.1': + '@mdream/rust-android-arm-eabi@2.0.3': optional: true - '@mdream/rust-android-arm64@2.0.1': + '@mdream/rust-android-arm64@2.0.3': optional: true - '@mdream/rust-darwin-arm64@2.0.1': + '@mdream/rust-darwin-arm64@2.0.3': optional: true - '@mdream/rust-darwin-x64@2.0.1': + '@mdream/rust-darwin-x64@2.0.3': optional: true - '@mdream/rust-freebsd-x64@2.0.1': + '@mdream/rust-freebsd-x64@2.0.3': optional: true - '@mdream/rust-linux-arm-gnueabihf@2.0.1': + '@mdream/rust-linux-arm-gnueabihf@2.0.3': optional: true - '@mdream/rust-linux-arm64-gnu@2.0.1': + '@mdream/rust-linux-arm64-gnu@2.0.3': optional: true - '@mdream/rust-linux-arm64-musl@2.0.1': + '@mdream/rust-linux-arm64-musl@2.0.3': optional: true - '@mdream/rust-linux-x64-gnu@2.0.1': + '@mdream/rust-linux-x64-gnu@2.0.3': optional: true - '@mdream/rust-linux-x64-musl@2.0.1': + '@mdream/rust-linux-x64-musl@2.0.3': optional: true - '@mdream/rust-wasm32-wasi@2.0.1': + '@mdream/rust-wasm32-wasi@2.0.3': optional: true - '@mdream/rust-win32-arm64-msvc@2.0.1': + '@mdream/rust-win32-arm64-msvc@2.0.3': optional: true - '@mdream/rust-win32-x64-msvc@2.0.1': + '@mdream/rust-win32-x64-msvc@2.0.3': optional: true '@modelcontextprotocol/client@2.0.0': @@ -13284,7 +13284,7 @@ snapshots: '@nuxtjs/robots': 7.0.2(@nuxt/schema@4.6.1)(confbox@0.3.1)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(oxc-parser@0.143.0)(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6) '@nuxtjs/sitemap': 9.1.0(@nuxt/schema@4.6.1)(@oxc-project/types@0.144.0)(confbox@0.3.1)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6) nuxt: 4.6.1(02c719891bc06f27f791554366e92673) - nuxt-ai-ready: 3.3.0(@nuxt/schema@4.6.1)(@oxc-project/types@0.144.0)(dotenv@17.4.2)(drizzle-orm@1.0.0-rc.4(@cloudflare/workers-types@5.20260901.1)(@opentelemetry/api@1.9.1)(better-sqlite3@13.0.3)(valibot@1.4.2(typescript@6.0.3))(zod@4.6.6))(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(oxc-parser@0.143.0)(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6) + nuxt-ai-ready: 3.3.1(@nuxt/schema@4.6.1)(@oxc-project/types@0.144.0)(dotenv@17.4.2)(drizzle-orm@1.0.0-rc.4(@cloudflare/workers-types@5.20260901.1)(@opentelemetry/api@1.9.1)(better-sqlite3@13.0.3)(valibot@1.4.2(typescript@6.0.3))(zod@4.6.6))(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(oxc-parser@0.143.0)(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6) nuxt-link-checker: 6.0.1(@nuxt/schema@4.6.1)(aws4fetch@1.0.20)(confbox@0.3.1)(db0@0.3.4(better-sqlite3@13.0.3)(drizzle-orm@1.0.0-rc.4(@cloudflare/workers-types@5.20260901.1)(@opentelemetry/api@1.9.1)(better-sqlite3@13.0.3)(valibot@1.4.2(typescript@6.0.3))(zod@4.6.6)))(dotenv@17.4.2)(giget@3.3.1)(ioredis@5.11.1(supports-color@10.2.2))(jiti@2.7.0)(mlly@1.8.2)(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(oxc-parser@0.143.0)(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6) nuxt-og-image: 7.0.3(cea27c93ad18112218efc54a25e950de) nuxt-schema-org: 7.0.1(@nuxt/schema@4.6.1)(@unhead/vue@3.4.3(@oxc-project/types@0.144.0)(esbuild@0.28.2)(lightningcss@1.33.0)(oxc-parser@0.143.0)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1))(vue@3.5.43(typescript@6.0.3)))(confbox@0.3.1)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(oxc-parser@0.143.0)(rolldown@1.2.12)(unhead@3.4.3(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6) @@ -17446,21 +17446,21 @@ snapshots: mdn-data@2.29.0: {} - mdream@2.0.1: + mdream@2.0.3: optionalDependencies: - '@mdream/rust-android-arm-eabi': 2.0.1 - '@mdream/rust-android-arm64': 2.0.1 - '@mdream/rust-darwin-arm64': 2.0.1 - '@mdream/rust-darwin-x64': 2.0.1 - '@mdream/rust-freebsd-x64': 2.0.1 - '@mdream/rust-linux-arm-gnueabihf': 2.0.1 - '@mdream/rust-linux-arm64-gnu': 2.0.1 - '@mdream/rust-linux-arm64-musl': 2.0.1 - '@mdream/rust-linux-x64-gnu': 2.0.1 - '@mdream/rust-linux-x64-musl': 2.0.1 - '@mdream/rust-wasm32-wasi': 2.0.1 - '@mdream/rust-win32-arm64-msvc': 2.0.1 - '@mdream/rust-win32-x64-msvc': 2.0.1 + '@mdream/rust-android-arm-eabi': 2.0.3 + '@mdream/rust-android-arm64': 2.0.3 + '@mdream/rust-darwin-arm64': 2.0.3 + '@mdream/rust-darwin-x64': 2.0.3 + '@mdream/rust-freebsd-x64': 2.0.3 + '@mdream/rust-linux-arm-gnueabihf': 2.0.3 + '@mdream/rust-linux-arm64-gnu': 2.0.3 + '@mdream/rust-linux-arm64-musl': 2.0.3 + '@mdream/rust-linux-x64-gnu': 2.0.3 + '@mdream/rust-linux-x64-musl': 2.0.3 + '@mdream/rust-wasm32-wasi': 2.0.3 + '@mdream/rust-win32-arm64-msvc': 2.0.3 + '@mdream/rust-win32-x64-msvc': 2.0.3 mdurl@2.1.0: {} @@ -17993,9 +17993,9 @@ snapshots: dependencies: boolbase: 1.0.0 - nuxt-ai-ready@3.3.0(@nuxt/schema@4.6.1)(@oxc-project/types@0.144.0)(dotenv@17.4.2)(drizzle-orm@1.0.0-rc.4(@cloudflare/workers-types@5.20260901.1)(@opentelemetry/api@1.9.1)(better-sqlite3@13.0.3)(valibot@1.4.2(typescript@6.0.3))(zod@4.6.6))(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(oxc-parser@0.143.0)(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6): + nuxt-ai-ready@3.3.1(@nuxt/schema@4.6.1)(@oxc-project/types@0.144.0)(dotenv@17.4.2)(drizzle-orm@1.0.0-rc.4(@cloudflare/workers-types@5.20260901.1)(@opentelemetry/api@1.9.1)(better-sqlite3@13.0.3)(valibot@1.4.2(typescript@6.0.3))(zod@4.6.6))(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(oxc-parser@0.143.0)(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6): dependencies: - '@mdream/js': 2.0.1 + '@mdream/js': 2.0.3 '@nuxt/kit': 4.6.1(@nuxt/schema@4.6.1)(confbox@0.3.1)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(oxc-parser@0.143.0)(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1))) '@nuxtjs/robots': 7.0.2(@nuxt/schema@4.6.1)(confbox@0.3.1)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(oxc-parser@0.143.0)(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6) '@nuxtjs/sitemap': 9.1.0(@nuxt/schema@4.6.1)(@oxc-project/types@0.144.0)(confbox@0.3.1)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6) @@ -18003,7 +18003,7 @@ snapshots: confbox: 0.3.1 consola: 3.4.2 defu: 6.1.7 - mdream: 2.0.1 + mdream: 2.0.3 nuxt: 4.6.1(02c719891bc06f27f791554366e92673) nuxt-site-config: 5.0.3(@nuxt/schema@4.6.1)(confbox@0.3.1)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(oxc-parser@0.143.0)(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6) nuxtseo-shared: 6.0.2(@nuxt/schema@4.6.1)(confbox@0.3.1)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(nuxt-site-config@5.0.3(@nuxt/schema@4.6.1)(confbox@0.3.1)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magic-string@1.2.3)(mlly@1.8.2)(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(oxc-parser@0.143.0)(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6))(nuxt@4.6.1(02c719891bc06f27f791554366e92673))(oxc-parser@0.143.0)(rolldown@1.2.12)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.12)(rollup@4.62.4)(vite@8.3.2(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.1)))(vue@3.5.43(typescript@6.0.3))(zod@4.6.6) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 586aa1ef..1f47478f 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -97,7 +97,7 @@ minimumReleaseAgeExclude: - '@nuxtjs/robots@7.0.2' - '@nuxtjs/sitemap@9.1.0' - nuxt-og-image@7.0.3 - - nuxt-ai-ready@3.3.0 + - nuxt-ai-ready@3.3.1 - nuxt-skew-protection@2.0.2 - nuxt-schema-org@7.0.1 - nuxt-seo-utils@9.0.0 @@ -123,6 +123,21 @@ minimumReleaseAgeExclude: - '@takumi-rs/helpers@2.15.0' - '@takumi-rs/wasm@2.15.0' - sitemapd@0.2.3 + - '@mdream/js@2.0.3' + - '@mdream/rust-android-arm-eabi@2.0.3' + - '@mdream/rust-android-arm64@2.0.3' + - '@mdream/rust-darwin-arm64@2.0.3' + - '@mdream/rust-darwin-x64@2.0.3' + - '@mdream/rust-freebsd-x64@2.0.3' + - '@mdream/rust-linux-arm-gnueabihf@2.0.3' + - '@mdream/rust-linux-arm64-gnu@2.0.3' + - '@mdream/rust-linux-arm64-musl@2.0.3' + - '@mdream/rust-linux-x64-gnu@2.0.3' + - '@mdream/rust-linux-x64-musl@2.0.3' + - '@mdream/rust-wasm32-wasi@2.0.3' + - '@mdream/rust-win32-arm64-msvc@2.0.3' + - '@mdream/rust-win32-x64-msvc@2.0.3' + - mdream@2.0.3 shellEmulator: true autoInstallPeers: false @@ -249,7 +264,7 @@ packages: - workers/* overrides: - nuxt-ai-ready: 3.3.0 + nuxt-ai-ready: 3.3.1 patchedDependencies: hookable@5.5.3: patches/hookable@5.5.3.patch hookable@6.1.1: patches/hookable@6.patch diff --git a/server/middleware/llms-txt.ts b/server/middleware/llms-txt.ts deleted file mode 100644 index ff68822c..00000000 --- a/server/middleware/llms-txt.ts +++ /dev/null @@ -1,94 +0,0 @@ -import type { H3Event } from 'h3' -import type { LlmsSection, PageMeta, SitemapPage } from '#shared/llms-txt' -import { assertMethod } from 'h3' -import { - isPagesSitemap, - LLMS_INDEX_PATH, - LLMS_SECTIONS, - llmsSectionFor, - parseSitemapLocs, - readableSitemap, - renderLlmsIndex, - renderLlmsSection, - SITE_ORIGIN, - sitemapName, - sitemapPages, -} from '#shared/llms-txt' -import { cached } from '#shared/server/cache' - -// The root llms.txt and its nested files. A middleware, not a route: -// nuxt-ai-ready registers its own `/llms.txt` route, and a project middleware -// runs before it (see agent-md.ts). `shared/llms-txt.ts` says why this site -// replaces the module's file. - -const CACHE_SECONDS = 600 -const STALE_SECONDS = 3600 -/** D1 allows 100 bound parameters per statement. */ -const META_BATCH = 100 - -export default defineEventHandler(async (event) => { - const path = event.path.split('?')[0]! - const section = llmsSectionFor(path) - if (path !== LLMS_INDEX_PATH && !section) - return - assertMethod(event, 'GET', true) - - const body = await cached({ - storage: useStorage('cache'), - key: `llms-txt:v1:${path}`, - ttlSeconds: CACHE_SECONDS, - staleSeconds: STALE_SECONDS, - compute: () => buildLlmsFile(event, section), - schedule: promise => event.waitUntil(promise), - }) - setHeader(event, 'Content-Type', 'text/plain; charset=utf-8') - setHeader(event, 'Cache-Control', `public, max-age=${CACHE_SECONDS}, stale-while-revalidate=${STALE_SECONDS}`) - return body -}) - -async function buildLlmsFile(event: H3Event, section: LlmsSection | null): Promise { - const sitemaps = await readSitemaps(event) - const site = getSiteConfig(event) - const llmsSite = { name: site.name || 'skilld', description: site.description || '' } - if (section) { - const pages = sitemaps.get(section.sitemap) ?? [] - return renderLlmsSection({ site: llmsSite, section, pages, meta: await readPageMeta(event, pages) }) - } - const pages = [...sitemaps.entries()].filter(([name]) => isPagesSitemap(name)).flatMap(([, list]) => list) - return renderLlmsIndex({ - site: llmsSite, - pages, - sections: LLMS_SECTIONS.filter(entry => (sitemaps.get(entry.sitemap)?.length ?? 0) > 0), - meta: await readPageMeta(event, pages), - }) -} - -/** Pages by sitemap name, read from the sitemap index the site serves. */ -async function readSitemaps(event: H3Event): Promise> { - const index = await event.$fetch('/sitemap_index.xml', { responseType: 'text' }) - const byName = new Map() - for (const loc of parseSitemapLocs(index)) { - const name = sitemapName(loc) - if (!name || !readableSitemap(name)) - continue - const xml = await event.$fetch(new URL(loc, SITE_ORIGIN).pathname, { responseType: 'text' }) - byName.set(name, [...byName.get(name) ?? [], ...parseSitemapLocs(xml)]) - } - return new Map([...byName].map(([name, locs]) => [name, sitemapPages(locs)])) -} - -/** Titles and descriptions that nuxt-ai-ready stored when it rendered each page. */ -async function readPageMeta(event: H3Event, pages: readonly SitemapPage[]): Promise> { - const db = event.context.platform.db - const meta = new Map() - for (let i = 0; i < pages.length; i += META_BATCH) { - const routes = pages.slice(i, i + META_BATCH).map(page => page.path) - const { results } = await db - .prepare(`SELECT route, title, description FROM ai_ready_pages WHERE is_error = 0 AND route IN (${routes.map(() => '?').join(', ')})`) - .bind(...routes) - .all<{ route: string, title: string, description: string }>() - for (const row of results ?? []) - meta.set(row.route, { title: row.title, description: row.description }) - } - return meta -} diff --git a/shared/llms-txt.ts b/shared/llms-txt.ts index a1a40867..4a2f408b 100644 --- a/shared/llms-txt.ts +++ b/shared/llms-txt.ts @@ -1,214 +1,5 @@ -/** - * llms.txt as a small root index plus one nested file per sitemap. - * - * nuxt-ai-ready lists every `ai_ready_pages` row in one file. Here that was - * 1.48M characters and 11,872 links, most of them retired `/skills//...` - * URLs that answer 301, 404, or 410. Agents read about the first 100k - * characters. These files list the sitemaps instead, so they follow the same - * admission rules as search, and each file stays far under that limit. - */ - -export const SITE_ORIGIN = 'https://skilld.dev' - -export const LLMS_INDEX_PATH = '/llms.txt' - -/** The skilld Skill this site publishes, so `nuxt.config.ts` and llms.txt name it alike. */ +/** The Skill this site publishes through nuxt-ai-ready's Agent Skills index. */ export const REGISTRY_SKILL = { name: 'skilld-registry', description: 'Search skilld.dev and return verified commands to run or install a skill.', } as const - -/** A nested llms file: the pages of one sitemap, linked from the root index. */ -export interface LlmsSection { - path: `/llms/${string}.txt` - /** The sitemap name, `skills` for `/__sitemap__/skills-0.xml`. */ - sitemap: string - title: string - description: string -} - -/** The root index lists this sitemap inline. */ -const PAGES_SITEMAP = 'pages' - -/** - * The sitemap of retired URLs (301, 404, 410), so Google drops them sooner. - * It must never reach an Agent. - */ -const RETIRED_SITEMAP = 'retired' - -export const LLMS_SECTIONS: readonly LlmsSection[] = [ - { - path: '/llms/skills.txt', - sitemap: 'skills', - title: 'Skills', - description: 'Every Skill page and track in the skilld sitemap.', - }, - { - path: '/llms/demos.txt', - sitemap: 'demos', - title: 'Skill Demos', - description: 'Each demo is one recorded run: the prompt, and what the Agent built with the Skill.', - }, -] - -/** Title and description of a page, from the `ai_ready_pages` row the Markdown renderer wrote. */ -export interface PageMeta { - title: string - description: string -} - -/** A page path, such as `/gh/antfu/skills/antfu`. */ -export interface SitemapPage { - path: string -} - -const LOC_RE = /([^<]+)<\/loc>/g -const SITEMAP_NAME_RE = /\/__sitemap__\/(.+?)(?:-\d+)?\.xml$/ -const TITLE_SUFFIX_RE = /\s+·\s+skilld$/ - -function decodeXml(value: string): string { - return value - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/'/g, '\'') - .replace(/&/g, '&') -} - -/** Every `` of a sitemap or sitemap index, decoded. */ -export function parseSitemapLocs(xml: string): string[] { - return [...xml.matchAll(LOC_RE)].map(match => decodeXml(match[1]!.trim())) -} - -/** `skills` for `https://skilld.dev/__sitemap__/skills-0.xml`. Null for any other URL. */ -export function sitemapName(loc: string): string | null { - const pathname = URL.canParse(loc, SITE_ORIGIN) ? new URL(loc, SITE_ORIGIN).pathname : '' - return SITEMAP_NAME_RE.exec(pathname)?.[1] ?? null -} - -/** The sitemaps an llms file reads. The retired sitemap is never one of them. */ -export function readableSitemap(name: string): boolean { - return name !== RETIRED_SITEMAP -} - -/** - * Page paths from sitemap locs, in sitemap order. A loc with a query string is - * a view of a page already listed, and a loc on another origin is not ours. - */ -export function sitemapPages(locs: readonly string[]): SitemapPage[] { - const seen = new Set() - const pages: SitemapPage[] = [] - for (const loc of locs) { - if (!URL.canParse(loc, SITE_ORIGIN)) - continue - const url = new URL(loc, SITE_ORIGIN) - if (url.origin !== SITE_ORIGIN || url.search) - continue - const path = url.pathname.length > 1 ? url.pathname.replace(/\/+$/, '') : '/' - if (seen.has(path)) - continue - seen.add(path) - pages.push({ path }) - } - return pages -} - -/** The Markdown URL of a page: `/index.md` for the homepage, `.md` otherwise. */ -export function markdownUrl(path: string): string { - return `${SITE_ORIGIN}${path === '/' ? '/index' : path}.md` -} - -function oneLine(value: string): string { - return value.replace(/\s+/g, ' ').trim() -} - -function linkText(value: string): string { - return oneLine(value).replace(/[[\]]/g, '\\$&') -} - -function linkLine(title: string, url: string, description: string): string { - const note = oneLine(description) - return `- [${linkText(title)}](${url})${note ? `: ${note}` : ''}` -} - -function pageLine(page: SitemapPage, meta: ReadonlyMap): string { - const row = meta.get(page.path) - const title = row?.title.replace(TITLE_SUFFIX_RE, '').trim() || page.path - return linkLine(title, markdownUrl(page.path), row?.description ?? '') -} - -export interface LlmsSite { - name: string - description: string -} - -function header(site: LlmsSite, description: string): string[] { - return [`# ${site.name}`, '', `> ${oneLine(description)}`, '', `Canonical Origin: ${SITE_ORIGIN}/`, ''] -} - -export interface LlmsIndexInput { - site: LlmsSite - /** Pages from the pages sitemap, listed inline. */ - pages: readonly SitemapPage[] - /** Nested files that have at least one page. */ - sections: readonly LlmsSection[] - meta: ReadonlyMap -} - -/** The root `/llms.txt`. */ -export function renderLlmsIndex(input: LlmsIndexInput): string { - const lines = [ - ...header(input.site, input.site.description), - 'Each page link below opens the Markdown version of that page.', - '', - '## Agent setup', - '', - linkLine('Set up skilld', `${SITE_ORIGIN}/agent.md`, 'Setup steps an Agent reads and follows: the CLI, the MCP server, or a plain URL fetch.'), - linkLine(REGISTRY_SKILL.name, `${SITE_ORIGIN}/SKILL.md`, REGISTRY_SKILL.description), - linkLine('Agent Skills index', `${SITE_ORIGIN}/.well-known/agent-skills/index.json`, 'Every Skill this site publishes, with a sha256 digest for each one.'), - linkLine('MCP server card', `${SITE_ORIGIN}/api/mcp/server-card`, 'The discovery MCP server and how to connect to it.'), - linkLine('API catalog', `${SITE_ORIGIN}/.well-known/api-catalog`, 'The public API and the MCP server, as an RFC 9727 linkset.'), - linkLine('OpenAPI', `${SITE_ORIGIN}/api/v1/openapi.json`, 'The public API contract.'), - '', - '## Indexes', - '', - ...input.sections.map(section => linkLine(section.title, `${SITE_ORIGIN}${section.path}`, section.description)), - linkLine('AI sitemap', `${SITE_ORIGIN}/ai-sitemap.xml`, 'Every AI-ready page, including pages that are noindex for Google.'), - linkLine('Sitemap', `${SITE_ORIGIN}/sitemap_index.xml`, 'XML sitemap for search engines and crawlers.'), - linkLine('robots.txt', `${SITE_ORIGIN}/robots.txt`, 'Crawler rules and permissions.'), - ] - if (input.pages.length) { - lines.push('', '## Pages', '', ...input.pages.map(page => pageLine(page, input.meta))) - } - return `${lines.join('\n')}\n` -} - -export interface LlmsSectionInput { - site: LlmsSite - section: LlmsSection - pages: readonly SitemapPage[] - meta: ReadonlyMap -} - -/** One nested file, such as `/llms/skills.txt`. */ -export function renderLlmsSection(input: LlmsSectionInput): string { - const lines = [ - ...header({ ...input.site, name: `${input.site.name}: ${input.section.title}` }, input.section.description), - `The index of every file is ${SITE_ORIGIN}${LLMS_INDEX_PATH}.`, - '', - `## ${input.section.title}`, - '', - ...input.pages.map(page => pageLine(page, input.meta)), - ] - return `${lines.join('\n')}\n` -} - -/** The section a nested file path serves, or null. */ -export function llmsSectionFor(path: string): LlmsSection | null { - return LLMS_SECTIONS.find(section => section.path === path) ?? null -} - -/** Which sitemap feeds the root index, so the shell knows what to read. */ -export function isPagesSitemap(name: string): boolean { - return name === PAGES_SITEMAP -} diff --git a/test/e2e/agent-discovery.spec.ts b/test/e2e/agent-discovery.spec.ts index 50a1fa46..598ba287 100644 --- a/test/e2e/agent-discovery.spec.ts +++ b/test/e2e/agent-discovery.spec.ts @@ -1,6 +1,7 @@ import { expect, test } from './test-utils' const LINK_RE = /\]\((https:\/\/skilld\.dev[^)\s]+)\)/g +const DISCOVERY_PATH_RE = /^\/llms\/[^/]+\.txt$/i test.describe('agent discovery', () => { test('the AI sitemap answers HEAD the way it answers GET', async ({ request, baseURL }) => { @@ -19,19 +20,56 @@ test.describe('agent discovery', () => { const index = await request.get(`${baseURL}/llms.txt`) expect(index.status()).toBe(200) const text = await index.text() - expect(text.length).toBeLessThan(100_000) + expect(text.length).toBeLessThanOrEqual(80_000) expect(text).not.toContain('llms-full.txt') + expect(text).toContain('https://skilld.dev/agent.md') + expect(text).toContain('https://skilld.dev/.well-known/agent-skills/index.json') + expect(text).toContain('https://skilld.dev/SKILL.md') + expect(text).toContain('https://skilld.dev/.well-known/api-catalog') + expect(text).toContain('https://skilld.dev/api/v1/openapi.json') + expect(text).toContain('https://skilld.dev/ai-sitemap.xml') - const nested = [...text.matchAll(LINK_RE)].map(match => match[1]!).filter(url => url.includes('/llms/')) - const links = [...text.matchAll(LINK_RE)].map(match => match[1]!) - for (const url of nested) { + const visited = new Set(['https://skilld.dev/llms.txt']) + const links = new Set([...text.matchAll(LINK_RE)].map(match => match[1]!)) + const pending = [...links].filter(url => DISCOVERY_PATH_RE.test(new URL(url).pathname)) + while (pending.length) { + const url = pending.shift()! + if (visited.has(url)) + continue + visited.add(url) const res = await request.get(local(url)) expect(res.status(), url).toBe(200) - links.push(...[...(await res.text()).matchAll(LINK_RE)].map(match => match[1]!)) + const child = await res.text() + expect(child.length, url).toBeLessThanOrEqual(80_000) + expect(child, url).not.toContain('llms-full.txt') + expect(child, url).not.toContain('/__sitemap__/retired') + for (const match of child.matchAll(LINK_RE)) { + const href = match[1]! + links.add(href) + if (DISCOVERY_PATH_RE.test(new URL(href).pathname)) + pending.push(href) + } + const head = await request.head(local(url)) + expect(head.status(), url).toBe(200) + expect(head.headers()['content-type'], url).toContain('text/plain') } - for (const url of new Set(links)) { + for (const url of links) { + expect(new URL(url).pathname, url).not.toMatch(/^\/skills\/[^/]+\/[^/]+\/[^/]+$/) + expect(new URL(url).pathname, url).not.toContain('/__sitemap__/retired') const res = await request.get(local(url), { maxRedirects: 0 }) expect(res.status(), url).toBe(200) } }) + + test('generated discovery files enforce their HTTP methods and unknown paths', async ({ request }) => { + const head = await request.head('/llms.txt') + expect(head.status()).toBe(200) + expect(head.headers()['content-type']).toContain('text/plain') + expect(await head.body()).toHaveLength(0) + expect((await request.post('/llms.txt')).status()).toBe(405) + expect((await request.get('/llms/missing-discovery-file.txt')).status()).toBe(404) + const full = await request.get('/llms-full.txt', { maxRedirects: 0 }) + expect(full.status()).toBe(302) + expect(full.headers().location).toBe('/llms.txt') + }) }) diff --git a/test/unit/llms-txt.test.ts b/test/unit/llms-txt.test.ts deleted file mode 100644 index bbc7e3ce..00000000 --- a/test/unit/llms-txt.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - LLMS_SECTIONS, - parseSitemapLocs, - readableSitemap, - renderLlmsIndex, - renderLlmsSection, - sitemapName, - sitemapPages, -} from '../../shared/llms-txt' - -const site = { name: 'skilld', description: 'Curated agent skills by humans' } - -describe('llms.txt sitemap reading', () => { - it('names each child sitemap, chunked or not, and never reads the retired one', () => { - const index = ` - https://skilld.dev/__sitemap__/pages.xml - https://skilld.dev/__sitemap__/skills-0.xml - https://skilld.dev/__sitemap__/retired-0.xml - ` - const names = parseSitemapLocs(index).map(sitemapName) - expect(names).toEqual(['pages', 'skills', 'retired']) - expect(names.filter(name => name && readableSitemap(name))).toEqual(['pages', 'skills']) - expect(sitemapName('https://skilld.dev/gh/antfu/skills/antfu')).toBeNull() - }) - - it('keeps one path per page, without query views or other origins', () => { - const locs = parseSitemapLocs(` - https://skilld.dev/skills/trending - https://skilld.dev/skills/trending?range=all&page=2 - https://skilld.dev/learn/create-agent-skills/ - https://example.com/elsewhere - https://skilld.dev/ - `) - expect(sitemapPages(locs)).toEqual([ - { path: '/skills/trending' }, - { path: '/learn/create-agent-skills' }, - { path: '/' }, - ]) - }) -}) - -describe('llms.txt rendering', () => { - const meta = new Map([ - ['/', { title: 'Curated agent skills by humans · skilld', description: 'Agent skills written by real maintainers.' }], - ['/gh/antfu/skills/antfu', { title: 'antfu [skill] · skilld', description: 'Anthony Fu\'s\nconventions.' }], - ]) - - it('links every nested file and each page\'s Markdown, and never llms-full.txt', () => { - const text = renderLlmsIndex({ site, pages: [{ path: '/' }, { path: '/verify' }], sections: LLMS_SECTIONS, meta }) - expect(text.startsWith('# skilld\n\n> Curated agent skills by humans\n')).toBe(true) - for (const section of LLMS_SECTIONS) - expect(text).toContain(`](https://skilld.dev${section.path})`) - expect(text).toContain('- [Curated agent skills by humans](https://skilld.dev/index.md): Agent skills written by real maintainers.') - // No stored title yet: the path stands in. - expect(text).toContain('- [/verify](https://skilld.dev/verify.md)') - expect(text).not.toContain('llms-full.txt') - expect(text).not.toContain('](https://skilld.dev/api/mcp)') - }) - - it('renders a nested file that points back to the index and keeps links parseable', () => { - const [skills] = LLMS_SECTIONS - const text = renderLlmsSection({ site, section: skills!, pages: [{ path: '/gh/antfu/skills/antfu' }], meta }) - expect(text).toContain('https://skilld.dev/llms.txt') - expect(text).toContain('- [antfu \\[skill\\]](https://skilld.dev/gh/antfu/skills/antfu.md): Anthony Fu\'s conventions.') - }) -})