diff --git a/docs/content/2.guides/7.robot-recipes.md b/docs/content/2.guides/7.robot-recipes.md index d3e1c193..5731f86f 100644 --- a/docs/content/2.guides/7.robot-recipes.md +++ b/docs/content/2.guides/7.robot-recipes.md @@ -49,7 +49,9 @@ export default defineNuxtConfig({ }) ``` -This will block the following AI crawlers: `GPTBot`, `ChatGPT-User`, `Claude-Web`, `anthropic-ai`, `Applebot-Extended`, `Bytespider`, `CCBot`, `cohere-ai`, `Diffbot`, `FacebookBot`, `Google-Extended`, `ImagesiftBot`, `PerplexityBot`, `OmigiliBot`, `Omigili` +This will block the AI crawlers listed in the `AiBots` constant, which follows the [ai.robots.txt](https://github.com/ai-robots-txt/ai.robots.txt) reference list: `GPTBot`, `ChatGPT-User`, `OAI-SearchBot`, `ClaudeBot`, `Claude-User`, `Claude-SearchBot`, `Claude-Web`, `anthropic-ai`, `Google-Extended`, `Google-CloudVertexBot`, `Google-NotebookLM`, `Gemini-Deep-Research`, `GoogleOther`, `meta-externalagent`, `meta-externalfetcher`, `FacebookBot`, `Amazonbot`, `bedrockbot`, `Applebot-Extended`, `PerplexityBot`, `Perplexity-User`, `Bytespider`, `TikTokSpider`, `omgili`, `omgilibot`, `Webzio-Extended`, `YandexAdditional`, `YandexAdditionalBot`, `CCBot`, `cohere-ai`, `Diffbot`, `ImagesiftBot`, `MistralAI-User`, `DeepSeekBot`, `AI2Bot`, `Ai2Bot-Dolma`, `YouBot`, `Timpibot`, `PanguBot`, `DuckAssistBot`, `FirecrawlAgent`, `ICC-Crawler`, `Kangaroo Bot`, `SemrushBot-OCOB`, `img2dataset` + +Search crawlers are not affected. `Googlebot`, `Applebot` (Siri and Spotlight) and `facebookexternalhit` (Open Graph link previews) are deliberately left out, so blocking AI bots never costs you search visibility or social previews. ### Blocking Privileged Pages diff --git a/src/const.ts b/src/const.ts index dd3abbed..91047050 100644 --- a/src/const.ts +++ b/src/const.ts @@ -19,20 +19,71 @@ export const NonHelpfulBots = [ 'magpie-crawler', ] +/** + * A list of AI crawler user agents, both training and retrieval. + * + * Search crawlers are deliberately excluded, so `blockAiBots` never costs a site its search + * visibility. This is why `Googlebot`, `Applebot` (Siri, Spotlight) and `facebookexternalhit` + * (Open Graph previews) are absent, while their AI-only counterparts `Google-Extended` and + * `Applebot-Extended` are listed. + * + * @credits https://github.com/ai-robots-txt/ai.robots.txt + */ export const AiBots = [ + // OpenAI 'GPTBot', 'ChatGPT-User', + 'OAI-SearchBot', + // Anthropic + 'ClaudeBot', + 'Claude-User', + 'Claude-SearchBot', 'Claude-Web', 'anthropic-ai', + // Google + 'Google-Extended', + 'Google-CloudVertexBot', + 'Google-NotebookLM', + 'Gemini-Deep-Research', + 'GoogleOther', + // Meta + 'meta-externalagent', + 'meta-externalfetcher', + 'FacebookBot', + // Amazon + 'Amazonbot', + 'bedrockbot', + // Apple 'Applebot-Extended', + // Perplexity + 'PerplexityBot', + 'Perplexity-User', + // ByteDance 'Bytespider', + 'TikTokSpider', + // Webz.io + 'omgili', + 'omgilibot', + 'Webzio-Extended', + // Yandex + 'YandexAdditional', + 'YandexAdditionalBot', + // Others 'CCBot', 'cohere-ai', 'Diffbot', - 'FacebookBot', - 'Google-Extended', 'ImagesiftBot', - 'PerplexityBot', - 'OmigiliBot', - 'Omigili', + 'MistralAI-User', + 'DeepSeekBot', + 'AI2Bot', + 'Ai2Bot-Dolma', + 'YouBot', + 'Timpibot', + 'PanguBot', + 'DuckAssistBot', + 'FirecrawlAgent', + 'ICC-Crawler', + 'Kangaroo Bot', + 'SemrushBot-OCOB', + 'img2dataset', ]