feat: support blocking AI training crawlers

docs/content/2.guides/7.robot-recipes.md

@@ -51,6 +51,21 @@ export default defineNuxtConfig({
 
 This adds a disallow rule for the training, AI search, and user-directed retrieval agents listed in the [`AiBots` constant](https://github.com/nuxt-modules/robots/blob/main/src/const.ts). Blocking search and retrieval agents can reduce your site's visibility and citations in AI-generated answers. Robots.txt is voluntary, and some user-triggered agents may ignore its rules.
 
+To opt out of training while leaving dedicated AI search and answer agents unblocked, use `'training'`:
+
+```ts [nuxt.config.ts]
+export default defineNuxtConfig({
+  robots: {
+    blockAiBots: 'training'
+  }
+})
+```
+
+This blocks a curated subset of training crawlers and training opt-out tokens.
+Existing rules still apply to search and answer agents.
+Google-Extended also controls Gemini grounding, so this mode opts out of that use too.
+See [`blockAiBots`](/docs/robots/api/config) for the token list and operator limits.
+
 ### Blocking Privileged Pages
 
 If you have pages that require authentication or are only available to certain users, you should block these from being indexed.

docs/content/4.api/1.config.md

@@ -136,16 +136,31 @@ export default defineNuxtConfig({
 })
 ```
 
-## `blockAiBots: boolean`{lang="ts"}
+## `blockAiBots: boolean | 'training'`{lang="ts"}
 
 - Default: `false`{lang="ts"}
 
-Adds a rule group that disallows `/` for the training, AI search, and user-directed retrieval agents defined in the [`AiBots` constant](https://github.com/nuxt-modules/robots/blob/main/src/const.ts). Blocking search and retrieval agents can reduce your site's visibility and citations in AI-generated answers. Some user-triggered agents may ignore robots.txt rules.
+Adds a rule group that disallows `/` for selected AI tokens.
+Blocking search and retrieval agents can reduce your site's visibility and citations in AI-generated answers.
+Some user-triggered agents may ignore robots.txt rules.
+
+- `false`: Adds no AI blocking group.
+- `true`: Blocks every token in `AiBots`, including search and user-directed retrieval agents.
+- `'training'`: Blocks `GPTBot`, `ClaudeBot`, `Google-Extended`, `meta-externalagent`, `Applebot-Extended`, and `CCBot`.
+
+The `'training'` mode leaves dedicated search and answer agents unblocked, including OpenAI, Claude, and Perplexity agents.
+Other robots.txt groups and site indexing rules still apply.
+The module maintains this curated list locally, with operator sources in [`src/const.ts`](https://github.com/nuxt-modules/robots/blob/main/src/const.ts).
+
+[Google-Extended](https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended) controls both Gemini training and grounding.
+Blocking it also opts out of grounding in Gemini Apps and Vertex AI, while preserving Google Search inclusion.
+[CCBot](https://commoncrawl.org/about) supplies a shared web archive used for training and other purposes.
+Robots.txt cannot guarantee that content stays out of all training datasets.
 
 ```ts twoslash
 export default defineNuxtConfig({
   robots: {
-    blockAiBots: true
+    blockAiBots: 'training'
   }
 })
 ```

skills/nuxtjs-robots/SKILL.md

@@ -95,7 +95,7 @@ export default defineNuxtConfig({
     disallow: ['/admin'],
     allow: ['/admin/login'],
     sitemap: '/sitemap.xml', // made absolute with site.url
-    blockAiBots: true, // GPTBot, ClaudeBot, Google-Extended, and others
+    blockAiBots: 'training', // Training opt-out. Use true to also block AI search and user fetches.
     groups: [
       { userAgent: ['Googlebot'], disallow: ['/private'], comment: ['Google only'] },
     ],
@@ -148,6 +148,8 @@ Nuxt Content v3 frontmatter and Nuxt i18n path expansion: [references/integratio
 - An `app.baseURL` other than `/` turns off robots.txt and logs an error. Crawlers only read `/robots.txt` at the host root.
 - Nuxt Content frontmatter `robots` sets the meta tag but not the header, and it needs `zod` 4 in your dependencies. See [references/integrations.md](references/integrations.md).
 - The key is `blockAiBots`. Some docs prose spells it `blockAIBots`.
+- Use `blockAiBots: 'training'` to leave dedicated AI search and user fetch agents unblocked. Existing rules still apply.
+- Training mode includes Google-Extended, which also controls grounding in Gemini Apps and Vertex AI.
 
 ## Version limits
 

src/const.ts

@@ -80,3 +80,24 @@ export const AiBots = [
   'ICC-Crawler',
   'SemrushBot-OCOB',
 ]
+
+/**
+ * Training crawlers and training opt-out tokens with documented operator support.
+ * Keep this curated subset local. Check operator documentation before changing it.
+ * Dedicated search and user-directed retrieval agents belong only in AiBots.
+ * Google-Extended also controls Gemini grounding. CCBot supplies a shared web archive.
+ */
+export const AiTrainingBots = [
+  // https://developers.openai.com/api/docs/bots
+  'GPTBot',
+  // https://privacy.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
+  'ClaudeBot',
+  // https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended
+  'Google-Extended',
+  // https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/
+  'meta-externalagent',
+  // https://support.apple.com/en-us/119829
+  'Applebot-Extended',
+  // https://commoncrawl.org/about
+  'CCBot',
+]

src/module.ts

@@ -21,7 +21,7 @@ import {
 import { updateSiteConfig, useSiteConfig } from 'nuxt-site-config/kit'
 import { setupNitroRuntimeCompatibility, setupRuntimeAliases, useModuleLogger } from 'nuxtseo-shared/kit'
 import { withoutTrailingSlash, withTrailingSlash } from 'ufo'
-import { AiBots, NonHelpfulBots } from './const'
+import { AiBots, AiTrainingBots, NonHelpfulBots } from './const'
 import { setupDevToolsUI } from './devtools'
 import { resolveRobotsHeaderRules } from './header-rules'
 import { mapPathForI18nPages, resolveI18nConfig, splitPathForI18nLocales } from './i18n'
@@ -135,11 +135,13 @@ export interface ModuleOptions {
    */
   blockNonSeoBots: boolean
   /**
-   * Blocks AI crawlers.
+   * Blocks all listed AI crawlers with true, or training tokens with 'training'.
+   * Dedicated AI search and user-directed retrieval agents are excluded from 'training'.
+   * Google-Extended also controls Gemini grounding.
    *
    * @default false
    */
-  blockAiBots: boolean
+  blockAiBots: boolean | 'training'
   /**
    * Override the auto i18n configuration.
    */
@@ -374,7 +376,7 @@ export default defineNuxtModule<ModuleOptions>({
 
     if (config.blockAiBots) {
       config.groups.push({
-        userAgent: AiBots,
+        userAgent: config.blockAiBots === 'training' ? AiTrainingBots : AiBots,
         comment: ['Block AI Crawlers'],
         disallow: ['/'],
       })

test/e2e/ai-bots.test.ts

@@ -0,0 +1,22 @@
+import { createResolver } from '@nuxt/kit'
+import { $fetch, setup } from '@nuxt/test-utils'
+import { describe, expect, it } from 'vitest'
+import { AiBots, parseRobotsTxt } from '../../src/util'
+
+const { resolve } = createResolver(import.meta.url)
+
+await setup({
+  rootDir: resolve('../fixtures/basic'),
+  build: true,
+  server: true,
+  nuxtConfig: { robots: { blockAiBots: true } },
+})
+
+describe('all AI bots opt-out', () => {
+  it('keeps true blocking the complete existing list', async () => {
+    const { groups } = parseRobotsTxt(await $fetch('/robots.txt'))
+    const blocked = groups.filter(group => group.disallow.includes('/')).flatMap(group => group.userAgent)
+    expect(blocked).toEqual(expect.arrayContaining(AiBots))
+    expect(blocked).toEqual(expect.arrayContaining(['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'PerplexityBot']))
+  })
+})

test/e2e/ai-training.test.ts

@@ -0,0 +1,57 @@
+import { createResolver } from '@nuxt/kit'
+import { $fetch, fetch, setup } from '@nuxt/test-utils'
+import { describe, expect, it } from 'vitest'
+import { parseRobotsTxt } from '../../src/util'
+
+const { resolve } = createResolver(import.meta.url)
+
+await setup({
+  rootDir: resolve('../fixtures/basic'),
+  build: true,
+  server: true,
+  nuxtConfig: {
+    robots: {
+      blockAiBots: 'training',
+      groups: [{ userAgent: ['CustomBot'], disallow: ['/private'] }],
+    },
+  },
+})
+
+describe('training opt-out', () => {
+  it('blocks training tokens without blocking dedicated search and answer agents', async () => {
+    const { groups } = parseRobotsTxt(await $fetch('/robots.txt'))
+    const blocked = groups.filter(group => group.disallow.includes('/')).flatMap(group => group.userAgent)
+    expect(blocked).toEqual(expect.arrayContaining([
+      'GPTBot',
+      'ClaudeBot',
+      'Google-Extended',
+      'Applebot-Extended',
+      'meta-externalagent',
+      'CCBot',
+    ]))
+    for (const bot of [
+      'OAI-SearchBot',
+      'ChatGPT-User',
+      'Claude-SearchBot',
+      'Claude-User',
+      'PerplexityBot',
+      'Perplexity-User',
+      'MistralAI-Index',
+      'MistralAI-User',
+      'Amzn-SearchBot',
+      'Amzn-User',
+      'DuckAssistBot',
+      'meta-externalfetcher',
+    ]) {
+      expect(blocked).not.toContain(bot)
+    }
+    expect(groups.find(group => group.userAgent.includes('CustomBot'))?.disallow).toEqual(['/private'])
+  })
+
+  it('keeps ordinary search crawlers and user fetches indexable', async () => {
+    for (const bot of ['Googlebot', 'OAI-SearchBot', 'ChatGPT-User', 'Claude-SearchBot', 'PerplexityBot']) {
+      const response = await fetch('/', { headers: { 'User-Agent': bot } })
+      expect(response.headers.get('x-robots-tag')).not.toContain('noindex')
+    }
+  })
+})

test/types/templates.test-d.ts

@@ -1,6 +1,7 @@
 import type {
   HookRobotsConfigContext,
   HookRobotsTxtContext,
+  ModuleOptions,
   RobotsContext,
   RobotsValue,
 } from '@nuxtjs/robots'
@@ -12,6 +13,14 @@ import type { NuxtConfig } from 'nuxt/schema'
 import { extendRouteRules } from '@nuxt/kit'
 import { describe, expectTypeOf, it } from 'vitest'
 
+describe('AI bot configuration', () => {
+  it('accepts training mode and boolean modes in module options', () => {
+    expectTypeOf<{ blockAiBots: 'training' }>().toExtend<Partial<ModuleOptions>>()
+    expectTypeOf<{ blockAiBots: true }>().toExtend<Partial<ModuleOptions>>()
+    expectTypeOf<{ blockAiBots: false }>().toExtend<Partial<ModuleOptions>>()
+  })
+})
+
 // Tests the generated type augmentations from .nuxt/types/nuxt-robots-*.d.ts.
 // Requires `nuxi prepare` to have been run so .nuxt/ exists.
 // The playground targets Nuxt 4 which augments 'nitropack/types'.