{
  "name": "Machine Readiness crawler registry",
  "version": "2026-10-24",
  "url": "https://machinereadiness.com/crawlers",
  "license": "CC BY 4.0",
  "licenseUrl": "https://creativecommons.org/licenses/by/4.0/",
  "attribution": "Machine Readiness crawler registry, https://machinereadiness.com/crawlers",
  "purposes": {
    "training": "Training only",
    "search": "AI search index",
    "user": "Fetches when a user asks",
    "answers": "AI answers",
    "research": "Research crawl",
    "owner": "Only for the site owner",
    "retired": "Retired token"
  },
  "count": 51,
  "crawlers": [
    {
      "token": "GPTBot",
      "vendor": "OpenAI",
      "purpose": "training",
      "purposeLabel": "Training only",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/gptbot"
    },
    {
      "token": "OAI-SearchBot",
      "vendor": "OpenAI",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/oai-searchbot"
    },
    {
      "token": "ChatGPT-User",
      "vendor": "OpenAI",
      "purpose": "user",
      "purposeLabel": "Fetches when a user asks",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/chatgpt-user"
    },
    {
      "token": "ClaudeBot",
      "vendor": "Anthropic",
      "purpose": "training",
      "purposeLabel": "Training only",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/claudebot"
    },
    {
      "token": "Claude-User",
      "vendor": "Anthropic",
      "purpose": "user",
      "purposeLabel": "Fetches when a user asks",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/claude-user"
    },
    {
      "token": "Claude-SearchBot",
      "vendor": "Anthropic",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/claude-searchbot"
    },
    {
      "token": "Claude-Web",
      "vendor": "Anthropic",
      "purpose": "retired",
      "purposeLabel": "Retired token",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/claude-web"
    },
    {
      "token": "anthropic-ai",
      "vendor": "Anthropic",
      "purpose": "retired",
      "purposeLabel": "Retired token",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/anthropic-ai"
    },
    {
      "token": "Google-Extended",
      "vendor": "Google AI",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/google-extended"
    },
    {
      "token": "GoogleOther",
      "vendor": "Google AI",
      "purpose": "research",
      "purposeLabel": "Research crawl",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/googleother"
    },
    {
      "token": "Gemini-Deep-Research",
      "vendor": "Google AI",
      "purpose": "user",
      "purposeLabel": "Fetches when a user asks",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/gemini-deep-research"
    },
    {
      "token": "Google-NotebookLM",
      "vendor": "Google AI",
      "purpose": "user",
      "purposeLabel": "Fetches when a user asks",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/google-notebooklm"
    },
    {
      "token": "Google-CloudVertexBot",
      "vendor": "Google AI",
      "purpose": "owner",
      "purposeLabel": "Only for the site owner",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/google-cloudvertexbot"
    },
    {
      "token": "GoogleOther-Image",
      "vendor": "Google AI",
      "purpose": "research",
      "purposeLabel": "Research crawl",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/googleother-image"
    },
    {
      "token": "GoogleOther-Video",
      "vendor": "Google AI",
      "purpose": "research",
      "purposeLabel": "Research crawl",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/googleother-video"
    },
    {
      "token": "PerplexityBot",
      "vendor": "Perplexity",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/perplexitybot"
    },
    {
      "token": "Perplexity-User",
      "vendor": "Perplexity",
      "purpose": "user",
      "purposeLabel": "Fetches when a user asks",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/perplexity-user"
    },
    {
      "token": "GrokBot",
      "vendor": "xAI / Grok",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/grokbot"
    },
    {
      "token": "Grok-DeepSearch",
      "vendor": "xAI / Grok",
      "purpose": "user",
      "purposeLabel": "Fetches when a user asks",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/grok-deepsearch"
    },
    {
      "token": "xAI-Grok",
      "vendor": "xAI / Grok",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/xai-grok"
    },
    {
      "token": "Copilot",
      "vendor": "Microsoft Copilot",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/copilot"
    },
    {
      "token": "CopilotNative",
      "vendor": "Microsoft Copilot",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/copilotnative"
    },
    {
      "token": "meta-externalagent",
      "vendor": "Meta AI",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/meta-externalagent"
    },
    {
      "token": "meta-externalfetcher",
      "vendor": "Meta AI",
      "purpose": "user",
      "purposeLabel": "Fetches when a user asks",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/meta-externalfetcher"
    },
    {
      "token": "FacebookBot",
      "vendor": "Meta AI",
      "purpose": "training",
      "purposeLabel": "Training only",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/facebookbot"
    },
    {
      "token": "Applebot-Extended",
      "vendor": "Apple Intelligence",
      "purpose": "training",
      "purposeLabel": "Training only",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/applebot-extended"
    },
    {
      "token": "Amazonbot",
      "vendor": "Amazon",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/amazonbot"
    },
    {
      "token": "PetalBot",
      "vendor": "Huawei",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/petalbot"
    },
    {
      "token": "YandexAdditional",
      "vendor": "Yandex AI",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/yandexadditional"
    },
    {
      "token": "YandexAdditionalBot",
      "vendor": "Yandex AI",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/yandexadditionalbot"
    },
    {
      "token": "DeepSeekBot",
      "vendor": "DeepSeek",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/deepseekbot"
    },
    {
      "token": "MistralAI-User",
      "vendor": "Mistral",
      "purpose": "user",
      "purposeLabel": "Fetches when a user asks",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/mistralai-user"
    },
    {
      "token": "cohere-ai",
      "vendor": "Cohere",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/cohere-ai"
    },
    {
      "token": "cohere-training-data-crawler",
      "vendor": "Cohere",
      "purpose": "training",
      "purposeLabel": "Training only",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/cohere-training-data-crawler"
    },
    {
      "token": "CCBot",
      "vendor": "Common Crawl",
      "purpose": "training",
      "purposeLabel": "Training only",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/ccbot"
    },
    {
      "token": "DuckAssistBot",
      "vendor": "DuckDuckGo AI",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/duckassistbot"
    },
    {
      "token": "Bytespider",
      "vendor": "Bytedance",
      "purpose": "training",
      "purposeLabel": "Training only",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/bytespider"
    },
    {
      "token": "Diffbot",
      "vendor": "Diffbot",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/diffbot"
    },
    {
      "token": "AI2Bot",
      "vendor": "Allen AI",
      "purpose": "training",
      "purposeLabel": "Training only",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/ai2bot"
    },
    {
      "token": "AI2Bot-Dolma",
      "vendor": "Allen AI",
      "purpose": "training",
      "purposeLabel": "Training only",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/ai2bot-dolma"
    },
    {
      "token": "PhindBot",
      "vendor": "Phind",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/phindbot"
    },
    {
      "token": "YouBot",
      "vendor": "You.com",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/youbot"
    },
    {
      "token": "Kagibot",
      "vendor": "Kagi",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/kagibot"
    },
    {
      "token": "AndiBot",
      "vendor": "Andi",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/andibot"
    },
    {
      "token": "Timpibot",
      "vendor": "Timpi",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/timpibot"
    },
    {
      "token": "SiderAI",
      "vendor": "Sider",
      "purpose": "user",
      "purposeLabel": "Fetches when a user asks",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/siderai"
    },
    {
      "token": "HuggingFace-Bot",
      "vendor": "HuggingFace",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/huggingface-bot"
    },
    {
      "token": "Quora-Bot",
      "vendor": "Quora / Poe",
      "purpose": "answers",
      "purposeLabel": "AI answers",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/quora-bot"
    },
    {
      "token": "GensparkBot",
      "vendor": "Genspark",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/gensparkbot"
    },
    {
      "token": "NeevaBot",
      "vendor": "Neeva",
      "purpose": "retired",
      "purposeLabel": "Retired token",
      "scored": false,
      "reference": "https://www.crawlreadiness.com/crawlers/neevabot"
    },
    {
      "token": "Elicit-Bot",
      "vendor": "Elicit",
      "purpose": "user",
      "purposeLabel": "Fetches when a user asks",
      "scored": true,
      "reference": "https://www.crawlreadiness.com/crawlers/elicit-bot"
    }
  ]
}