{
  "name": "AI crawler user agents",
  "source": "https://marketingsignals.com/tools/geo-checker/ai-crawlers",
  "publisher": "Marketing Signals",
  "license": "CC BY 4.0",
  "lastVerified": "2026-11-19",
  "crawlers": [
    {
      "token": "OAI-SearchBot",
      "operator": "OpenAI",
      "product": "ChatGPT search",
      "purpose": "search",
      "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot",
      "docsUrl": "https://developers.openai.com/api/docs/bots",
      "ipRangesUrl": "https://openai.com/searchbot.json",
      "honoursRobots": "yes",
      "note": "Controls whether pages appear in ChatGPT search answers. Independent of GPTBot.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "ChatGPT-User",
      "operator": "OpenAI",
      "product": "ChatGPT",
      "purpose": "user",
      "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
      "docsUrl": "https://developers.openai.com/api/docs/bots",
      "ipRangesUrl": "https://openai.com/chatgpt-user.json",
      "honoursRobots": "may-not",
      "note": "Visits a page when a ChatGPT user asks it to. OpenAI says robots.txt rules may not apply. Not used for search inclusion.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "GPTBot",
      "operator": "OpenAI",
      "product": "OpenAI model training",
      "purpose": "training",
      "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot",
      "docsUrl": "https://developers.openai.com/api/docs/bots",
      "ipRangesUrl": "https://openai.com/gptbot.json",
      "honoursRobots": "yes",
      "note": "Training opt-out only. Blocking it does not remove you from ChatGPT search.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "OAI-AdsBot",
      "operator": "OpenAI",
      "product": "ChatGPT ads",
      "purpose": "other",
      "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-AdsBot/1.0; +https://openai.com/adsbot",
      "docsUrl": "https://developers.openai.com/api/docs/bots",
      "ipRangesUrl": "https://openai.com/adsbot.json",
      "honoursRobots": "unverified",
      "note": "Only visits landing pages submitted as ads.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Claude-SearchBot",
      "operator": "Anthropic",
      "product": "Claude search",
      "purpose": "search",
      "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com)",
      "docsUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "honoursRobots": "yes",
      "note": "Anthropic says blocking it may reduce your visibility in Claude search results.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Claude-User",
      "operator": "Anthropic",
      "product": "Claude",
      "purpose": "user",
      "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +https://www.anthropic.com)",
      "docsUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "honoursRobots": "yes",
      "note": "Fetches pages when a Claude user asks. Anthropic says blocking it stops Claude retrieving your content for users.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "ClaudeBot",
      "operator": "Anthropic",
      "product": "Anthropic model training",
      "purpose": "training",
      "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
      "docsUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "honoursRobots": "yes",
      "note": "Training opt-out. Anthropic advises against IP blocking, which also stops it reading robots.txt.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "anthropic-ai",
      "operator": "Anthropic",
      "product": "none (retired)",
      "purpose": "deprecated",
      "docsUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "honoursRobots": "n/a",
      "note": "Old token that Anthropic no longer documents. Rules for it do nothing: use ClaudeBot, Claude-User and Claude-SearchBot.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Claude-Web",
      "operator": "Anthropic",
      "product": "none (retired)",
      "purpose": "deprecated",
      "docsUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "honoursRobots": "n/a",
      "note": "Old token that Anthropic no longer documents. Rules for it do nothing.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "PerplexityBot",
      "operator": "Perplexity",
      "product": "Perplexity search",
      "purpose": "search",
      "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
      "docsUrl": "https://docs.perplexity.ai/guides/bots",
      "ipRangesUrl": "https://www.perplexity.com/perplexitybot.json",
      "honoursRobots": "yes",
      "note": "Surfaces and links sites in Perplexity answers. Perplexity says it is not used to train foundation models.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Perplexity-User",
      "operator": "Perplexity",
      "product": "Perplexity",
      "purpose": "user",
      "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)",
      "docsUrl": "https://docs.perplexity.ai/guides/bots",
      "ipRangesUrl": "https://www.perplexity.com/perplexity-user.json",
      "honoursRobots": "may-not",
      "note": "Fetches a page when a user asks. Perplexity says it generally ignores robots.txt.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Googlebot",
      "operator": "Google",
      "product": "Google Search, AI Overviews and AI Mode",
      "purpose": "search",
      "ua": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
      "docsUrl": "https://developers.google.com/search/docs/crawling-indexing/googlebot",
      "honoursRobots": "yes",
      "note": "Google Search, including AI Overviews and AI Mode, uses Googlebot. There is no separate AI Overviews crawler.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Google-Extended",
      "operator": "Google",
      "product": "Gemini training and grounding",
      "purpose": "control",
      "docsUrl": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "honoursRobots": "yes",
      "note": "A robots.txt token only (no crawler of its own). Controls Gemini training and grounding; does not affect Google Search or AI Overviews.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Bingbot",
      "operator": "Microsoft",
      "product": "Bing, Copilot and partner AI",
      "purpose": "search",
      "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/131.0.0.0 Safari/537.36",
      "docsUrl": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "honoursRobots": "yes",
      "note": "Powers Bing search and Copilot answers. Bing Webmaster Tools has an AI Performance report.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Applebot",
      "operator": "Apple",
      "product": "Siri, Spotlight, Safari and Apple Intelligence answers",
      "purpose": "search",
      "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)",
      "docsUrl": "https://support.apple.com/en-us/119829",
      "honoursRobots": "yes",
      "fallback": "Googlebot",
      "note": "Follows your Googlebot rules when robots.txt does not name Applebot.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Applebot-Extended",
      "operator": "Apple",
      "product": "Apple model training",
      "purpose": "control",
      "docsUrl": "https://support.apple.com/en-us/119829",
      "honoursRobots": "yes",
      "note": "A robots.txt token only: opts out of Apple using your content to train its models. Does not crawl.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Meta-WebIndexer",
      "operator": "Meta",
      "product": "Meta AI search",
      "purpose": "search",
      "docsUrl": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "honoursRobots": "yes",
      "note": "Indexes pages for citations in Meta AI answers.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "meta-externalagent",
      "operator": "Meta",
      "product": "Meta AI training and indexing",
      "purpose": "training",
      "docsUrl": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "honoursRobots": "yes",
      "note": "Crawls for training and indexing.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Meta-ExternalFetcher",
      "operator": "Meta",
      "product": "Meta AI",
      "purpose": "user",
      "docsUrl": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "honoursRobots": "may-not",
      "note": "Fetches a page for a user; Meta says it may bypass robots.txt.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Amzn-SearchBot",
      "operator": "Amazon",
      "product": "Alexa and Amazon search",
      "purpose": "search",
      "docsUrl": "https://developer.amazon.com/docs/amazonbot/amazonbot.html",
      "honoursRobots": "yes",
      "fallback": "Googlebot",
      "note": "Search experiences such as Alexa. Not used for training. Follows other search crawlers’ rules when not named.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Amzn-User",
      "operator": "Amazon",
      "product": "Alexa",
      "purpose": "user",
      "docsUrl": "https://developer.amazon.com/docs/amazonbot/amazonbot.html",
      "honoursRobots": "may-not",
      "note": "User-requested fetches; may not follow every robots.txt rule.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Amazonbot",
      "operator": "Amazon",
      "product": "Amazon model training",
      "purpose": "training",
      "docsUrl": "https://developer.amazon.com/docs/amazonbot/amazonbot.html",
      "honoursRobots": "yes",
      "note": "May be used to train Amazon AI models. Honours noarchive as a training opt-out.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "CCBot",
      "operator": "Common Crawl",
      "product": "Common Crawl (open datasets)",
      "purpose": "training",
      "docsUrl": "https://commoncrawl.org/ccbot",
      "honoursRobots": "yes",
      "note": "Feeds the open Common Crawl corpus used to train many models.",
      "lastVerified": "2026-11-19"
    },
    {
      "token": "Bytespider",
      "operator": "ByteDance",
      "product": "ByteDance models",
      "purpose": "training",
      "docsUrl": "https://www.bytedance.com/",
      "honoursRobots": "unverified",
      "note": "ByteDance publishes no crawler policy, so whether it honours robots.txt is unverified.",
      "lastVerified": "2026-11-19"
    }
  ]
}