diff --git a/docs/content/2.guides/7.robot-recipes.md b/docs/content/2.guides/7.robot-recipes.md index b79a908d..0725fbdf 100644 --- a/docs/content/2.guides/7.robot-recipes.md +++ b/docs/content/2.guides/7.robot-recipes.md @@ -51,6 +51,21 @@ export default defineNuxtConfig({ This adds a disallow rule for the training, AI search, and user-directed retrieval agents listed in the [`AiBots` constant](https://github.com/nuxt-modules/robots/blob/main/src/const.ts). Blocking search and retrieval agents can reduce your site's visibility and citations in AI-generated answers. Robots.txt is voluntary, and some user-triggered agents may ignore its rules. +To opt out of training while leaving dedicated AI search and answer agents unblocked, use `'training'`: + +```ts [nuxt.config.ts] +export default defineNuxtConfig({ + robots: { + blockAiBots: 'training' + } +}) +``` + +This blocks a curated subset of training crawlers and training opt-out tokens. +Existing rules still apply to search and answer agents. +Google-Extended also controls Gemini grounding, so this mode opts out of that use too. +See [`blockAiBots`](/docs/robots/api/config) for the token list and operator limits. + ### Blocking Privileged Pages If you have pages that require authentication or are only available to certain users, you should block these from being indexed. diff --git a/docs/content/4.api/1.config.md b/docs/content/4.api/1.config.md index b82f93cc..c35314f6 100644 --- a/docs/content/4.api/1.config.md +++ b/docs/content/4.api/1.config.md @@ -136,16 +136,31 @@ export default defineNuxtConfig({ }) ``` -## `blockAiBots: boolean`{lang="ts"} +## `blockAiBots: boolean | 'training'`{lang="ts"} - Default: `false`{lang="ts"} -Adds a rule group that disallows `/` for the training, AI search, and user-directed retrieval agents defined in the [`AiBots` constant](https://github.com/nuxt-modules/robots/blob/main/src/const.ts). Blocking search and retrieval agents can reduce your site's visibility and citations in AI-generated answers. Some user-triggered agents may ignore robots.txt rules. +Adds a rule group that disallows `/` for selected AI tokens. +Blocking search and retrieval agents can reduce your site's visibility and citations in AI-generated answers. +Some user-triggered agents may ignore robots.txt rules. + +- `false`: Adds no AI blocking group. +- `true`: Blocks every token in `AiBots`, including search and user-directed retrieval agents. +- `'training'`: Blocks `GPTBot`, `ClaudeBot`, `Google-Extended`, `meta-externalagent`, `Applebot-Extended`, and `CCBot`. + +The `'training'` mode leaves dedicated search and answer agents unblocked, including OpenAI, Claude, and Perplexity agents. +Other robots.txt groups and site indexing rules still apply. +The module maintains this curated list locally, with operator sources in [`src/const.ts`](https://github.com/nuxt-modules/robots/blob/main/src/const.ts). + +[Google-Extended](https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended) controls both Gemini training and grounding. +Blocking it also opts out of grounding in Gemini Apps and Vertex AI, while preserving Google Search inclusion. +[CCBot](https://commoncrawl.org/about) supplies a shared web archive used for training and other purposes. +Robots.txt cannot guarantee that content stays out of all training datasets. ```ts twoslash export default defineNuxtConfig({ robots: { - blockAiBots: true + blockAiBots: 'training' } }) ``` diff --git a/skills/nuxtjs-robots/SKILL.md b/skills/nuxtjs-robots/SKILL.md index ee05677a..ee5b9371 100644 --- a/skills/nuxtjs-robots/SKILL.md +++ b/skills/nuxtjs-robots/SKILL.md @@ -95,7 +95,7 @@ export default defineNuxtConfig({ disallow: ['/admin'], allow: ['/admin/login'], sitemap: '/sitemap.xml', // made absolute with site.url - blockAiBots: true, // GPTBot, ClaudeBot, Google-Extended, and others + blockAiBots: 'training', // Training opt-out. Use true to also block AI search and user fetches. groups: [ { userAgent: ['Googlebot'], disallow: ['/private'], comment: ['Google only'] }, ], @@ -148,6 +148,8 @@ Nuxt Content v3 frontmatter and Nuxt i18n path expansion: [references/integratio - An `app.baseURL` other than `/` turns off robots.txt and logs an error. Crawlers only read `/robots.txt` at the host root. - Nuxt Content frontmatter `robots` sets the meta tag but not the header, and it needs `zod` 4 in your dependencies. See [references/integrations.md](references/integrations.md). - The key is `blockAiBots`. Some docs prose spells it `blockAIBots`. +- Use `blockAiBots: 'training'` to leave dedicated AI search and user fetch agents unblocked. Existing rules still apply. +- Training mode includes Google-Extended, which also controls grounding in Gemini Apps and Vertex AI. ## Version limits diff --git a/src/const.ts b/src/const.ts index eb07a477..5f223685 100644 --- a/src/const.ts +++ b/src/const.ts @@ -80,3 +80,24 @@ export const AiBots = [ 'ICC-Crawler', 'SemrushBot-OCOB', ] + +/** + * Training crawlers and training opt-out tokens with documented operator support. + * Keep this curated subset local. Check operator documentation before changing it. + * Dedicated search and user-directed retrieval agents belong only in AiBots. + * Google-Extended also controls Gemini grounding. CCBot supplies a shared web archive. + */ +export const AiTrainingBots = [ + // https://developers.openai.com/api/docs/bots + 'GPTBot', + // https://privacy.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler + 'ClaudeBot', + // https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended + 'Google-Extended', + // https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/ + 'meta-externalagent', + // https://support.apple.com/en-us/119829 + 'Applebot-Extended', + // https://commoncrawl.org/about + 'CCBot', +] diff --git a/src/module.ts b/src/module.ts index 4a0abcb8..afd1b27d 100644 --- a/src/module.ts +++ b/src/module.ts @@ -21,7 +21,7 @@ import { import { updateSiteConfig, useSiteConfig } from 'nuxt-site-config/kit' import { setupNitroRuntimeCompatibility, setupRuntimeAliases, useModuleLogger } from 'nuxtseo-shared/kit' import { withoutTrailingSlash, withTrailingSlash } from 'ufo' -import { AiBots, NonHelpfulBots } from './const' +import { AiBots, AiTrainingBots, NonHelpfulBots } from './const' import { setupDevToolsUI } from './devtools' import { resolveRobotsHeaderRules } from './header-rules' import { mapPathForI18nPages, resolveI18nConfig, splitPathForI18nLocales } from './i18n' @@ -135,11 +135,13 @@ export interface ModuleOptions { */ blockNonSeoBots: boolean /** - * Blocks AI crawlers. + * Blocks all listed AI crawlers with true, or training tokens with 'training'. + * Dedicated AI search and user-directed retrieval agents are excluded from 'training'. + * Google-Extended also controls Gemini grounding. * * @default false */ - blockAiBots: boolean + blockAiBots: boolean | 'training' /** * Override the auto i18n configuration. */ @@ -374,7 +376,7 @@ export default defineNuxtModule({ if (config.blockAiBots) { config.groups.push({ - userAgent: AiBots, + userAgent: config.blockAiBots === 'training' ? AiTrainingBots : AiBots, comment: ['Block AI Crawlers'], disallow: ['/'], }) diff --git a/test/e2e/ai-bots.test.ts b/test/e2e/ai-bots.test.ts new file mode 100644 index 00000000..47315a02 --- /dev/null +++ b/test/e2e/ai-bots.test.ts @@ -0,0 +1,22 @@ +import { createResolver } from '@nuxt/kit' +import { $fetch, setup } from '@nuxt/test-utils' +import { describe, expect, it } from 'vitest' +import { AiBots, parseRobotsTxt } from '../../src/util' + +const { resolve } = createResolver(import.meta.url) + +await setup({ + rootDir: resolve('../fixtures/basic'), + build: true, + server: true, + nuxtConfig: { robots: { blockAiBots: true } }, +}) + +describe('all AI bots opt-out', () => { + it('keeps true blocking the complete existing list', async () => { + const { groups } = parseRobotsTxt(await $fetch('/robots.txt')) + const blocked = groups.filter(group => group.disallow.includes('/')).flatMap(group => group.userAgent) + expect(blocked).toEqual(expect.arrayContaining(AiBots)) + expect(blocked).toEqual(expect.arrayContaining(['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'PerplexityBot'])) + }) +}) diff --git a/test/e2e/ai-training.test.ts b/test/e2e/ai-training.test.ts new file mode 100644 index 00000000..c2a5faf7 --- /dev/null +++ b/test/e2e/ai-training.test.ts @@ -0,0 +1,57 @@ +import { createResolver } from '@nuxt/kit' +import { $fetch, fetch, setup } from '@nuxt/test-utils' +import { describe, expect, it } from 'vitest' +import { parseRobotsTxt } from '../../src/util' + +const { resolve } = createResolver(import.meta.url) + +await setup({ + rootDir: resolve('../fixtures/basic'), + build: true, + server: true, + nuxtConfig: { + robots: { + blockAiBots: 'training', + groups: [{ userAgent: ['CustomBot'], disallow: ['/private'] }], + }, + }, +}) + +describe('training opt-out', () => { + it('blocks training tokens without blocking dedicated search and answer agents', async () => { + const { groups } = parseRobotsTxt(await $fetch('/robots.txt')) + const blocked = groups.filter(group => group.disallow.includes('/')).flatMap(group => group.userAgent) + expect(blocked).toEqual(expect.arrayContaining([ + 'GPTBot', + 'ClaudeBot', + 'Google-Extended', + 'Applebot-Extended', + 'meta-externalagent', + 'CCBot', + ])) + for (const bot of [ + 'OAI-SearchBot', + 'ChatGPT-User', + 'Claude-SearchBot', + 'Claude-User', + 'PerplexityBot', + 'Perplexity-User', + 'MistralAI-Index', + 'MistralAI-User', + 'Amzn-SearchBot', + 'Amzn-User', + 'DuckAssistBot', + 'meta-externalfetcher', + ]) { + expect(blocked).not.toContain(bot) + } + expect(groups.find(group => group.userAgent.includes('CustomBot'))?.disallow).toEqual(['/private']) + }) + + it('keeps ordinary search crawlers and user fetches indexable', async () => { + for (const bot of ['Googlebot', 'OAI-SearchBot', 'ChatGPT-User', 'Claude-SearchBot', 'PerplexityBot']) { + const response = await fetch('/', { headers: { 'User-Agent': bot } }) + expect(response.headers.get('x-robots-tag')).not.toContain('noindex') + } + }) +}) diff --git a/test/types/templates.test-d.ts b/test/types/templates.test-d.ts index f2e339eb..de3ed21f 100644 --- a/test/types/templates.test-d.ts +++ b/test/types/templates.test-d.ts @@ -1,6 +1,7 @@ import type { HookRobotsConfigContext, HookRobotsTxtContext, + ModuleOptions, RobotsContext, RobotsValue, } from '@nuxtjs/robots' @@ -12,6 +13,14 @@ import type { NuxtConfig } from 'nuxt/schema' import { extendRouteRules } from '@nuxt/kit' import { describe, expectTypeOf, it } from 'vitest' +describe('AI bot configuration', () => { + it('accepts training mode and boolean modes in module options', () => { + expectTypeOf<{ blockAiBots: 'training' }>().toExtend>() + expectTypeOf<{ blockAiBots: true }>().toExtend>() + expectTypeOf<{ blockAiBots: false }>().toExtend>() + }) +}) + // Tests the generated type augmentations from .nuxt/types/nuxt-robots-*.d.ts. // Requires `nuxi prepare` to have been run so .nuxt/ exists. // The playground targets Nuxt 4 which augments 'nitropack/types'.