From 3a6192bd0d97c5917d220e984fc55c52501ed9c7 Mon Sep 17 00:00:00 2001
From: hlassiege <hlassiege@pm.me>
Date: Fri, 17 Jul 2026 13:45:46 +0200
Subject: [PATCH] fix: add missing ai crawlers and correct omgili spelling

The AiBots list had not kept up with the crawlers vendors actually run.

ClaudeBot, the crawler Anthropic uses today, was absent: the list only carried
the legacy Claude-Web and anthropic-ai names, so blockAiBots did not block
Anthropic at all. Same for OAI-SearchBot, meta-externalagent, Amazonbot,
MistralAI-User, DeepSeekBot and others.

Omigili/OmigiliBot were misspellings of omgili/omgilibot (Webz.io) and matched
no real user agent.

Names now follow the ai.robots.txt reference list, restricted to AI crawlers.
Search crawlers stay out by design, so blockAiBots never costs search
visibility: Googlebot, Applebot (Siri, Spotlight) and facebookexternalhit (Open
Graph previews) are deliberately absent, while Google-Extended and
Applebot-Extended are listed.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 docs/content/2.guides/7.robot-recipes.md |  4 +-
 src/const.ts                             | 61 ++++++++++++++++++++++--
 2 files changed, 59 insertions(+), 6 deletions(-)

diff --git a/docs/content/2.guides/7.robot-recipes.md b/docs/content/2.guides/7.robot-recipes.md
index d3e1c193..5731f86f 100644
--- a/docs/content/2.guides/7.robot-recipes.md
+++ b/docs/content/2.guides/7.robot-recipes.md
@@ -49,7 +49,9 @@ export default defineNuxtConfig({
 })
 ```
 
-This will block the following AI crawlers: `GPTBot`, `ChatGPT-User`, `Claude-Web`, `anthropic-ai`, `Applebot-Extended`, `Bytespider`, `CCBot`, `cohere-ai`, `Diffbot`, `FacebookBot`, `Google-Extended`, `ImagesiftBot`, `PerplexityBot`, `OmigiliBot`, `Omigili`
+This will block the AI crawlers listed in the `AiBots` constant, which follows the [ai.robots.txt](https://github.com/ai-robots-txt/ai.robots.txt) reference list: `GPTBot`, `ChatGPT-User`, `OAI-SearchBot`, `ClaudeBot`, `Claude-User`, `Claude-SearchBot`, `Claude-Web`, `anthropic-ai`, `Google-Extended`, `Google-CloudVertexBot`, `Google-NotebookLM`, `Gemini-Deep-Research`, `GoogleOther`, `meta-externalagent`, `meta-externalfetcher`, `FacebookBot`, `Amazonbot`, `bedrockbot`, `Applebot-Extended`, `PerplexityBot`, `Perplexity-User`, `Bytespider`, `TikTokSpider`, `omgili`, `omgilibot`, `Webzio-Extended`, `YandexAdditional`, `YandexAdditionalBot`, `CCBot`, `cohere-ai`, `Diffbot`, `ImagesiftBot`, `MistralAI-User`, `DeepSeekBot`, `AI2Bot`, `Ai2Bot-Dolma`, `YouBot`, `Timpibot`, `PanguBot`, `DuckAssistBot`, `FirecrawlAgent`, `ICC-Crawler`, `Kangaroo Bot`, `SemrushBot-OCOB`, `img2dataset`
+
+Search crawlers are not affected. `Googlebot`, `Applebot` (Siri and Spotlight) and `facebookexternalhit` (Open Graph link previews) are deliberately left out, so blocking AI bots never costs you search visibility or social previews.
 
 ### Blocking Privileged Pages
 
diff --git a/src/const.ts b/src/const.ts
index dd3abbed..91047050 100644
--- a/src/const.ts
+++ b/src/const.ts
@@ -19,20 +19,71 @@ export const NonHelpfulBots = [
   'magpie-crawler',
 ]
 
+/**
+ * A list of AI crawler user agents, both training and retrieval.
+ *
+ * Search crawlers are deliberately excluded, so `blockAiBots` never costs a site its search
+ * visibility. This is why `Googlebot`, `Applebot` (Siri, Spotlight) and `facebookexternalhit`
+ * (Open Graph previews) are absent, while their AI-only counterparts `Google-Extended` and
+ * `Applebot-Extended` are listed.
+ *
+ * @credits https://github.com/ai-robots-txt/ai.robots.txt
+ */
 export const AiBots = [
+  // OpenAI
   'GPTBot',
   'ChatGPT-User',
+  'OAI-SearchBot',
+  // Anthropic
+  'ClaudeBot',
+  'Claude-User',
+  'Claude-SearchBot',
   'Claude-Web',
   'anthropic-ai',
+  // Google
+  'Google-Extended',
+  'Google-CloudVertexBot',
+  'Google-NotebookLM',
+  'Gemini-Deep-Research',
+  'GoogleOther',
+  // Meta
+  'meta-externalagent',
+  'meta-externalfetcher',
+  'FacebookBot',
+  // Amazon
+  'Amazonbot',
+  'bedrockbot',
+  // Apple
   'Applebot-Extended',
+  // Perplexity
+  'PerplexityBot',
+  'Perplexity-User',
+  // ByteDance
   'Bytespider',
+  'TikTokSpider',
+  // Webz.io
+  'omgili',
+  'omgilibot',
+  'Webzio-Extended',
+  // Yandex
+  'YandexAdditional',
+  'YandexAdditionalBot',
+  // Others
   'CCBot',
   'cohere-ai',
   'Diffbot',
-  'FacebookBot',
-  'Google-Extended',
   'ImagesiftBot',
-  'PerplexityBot',
-  'OmigiliBot',
-  'Omigili',
+  'MistralAI-User',
+  'DeepSeekBot',
+  'AI2Bot',
+  'Ai2Bot-Dolma',
+  'YouBot',
+  'Timpibot',
+  'PanguBot',
+  'DuckAssistBot',
+  'FirecrawlAgent',
+  'ICC-Crawler',
+  'Kangaroo Bot',
+  'SemrushBot-OCOB',
+  'img2dataset',
 ]