{ "category": "ai-search", "label": "AI search crawlers", "description": "Build the retrieval index an assistant answers and cites from. These are the crawlers that send you traffic; blocking them is the expensive mistake in this space.", "count": 8, "crawlers": [ { "slug": "amazonbot", "name": "Amazonbot", "operator": "Amazon", "operator_slug": "amazon", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "Amazonbot", "user_agent_substring": "Amazonbot", "user_agent_example": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Amazon's crawler, feeding Alexa's ability to answer questions from the web and Amazon's own search and assistant products.", "cost_of_blocking": "Alexa and Amazon's assistants stop answering from your pages. Verify with reverse DNS to crawl.amazonbot.amazon before trusting the user-agent.", "operator_docs": "https://developer.amazon.com/amazonbot", "html_url": "https://www.pathwren.workers.dev/crawler/amazonbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/amazonbot.json", "last_reviewed": "2026-09-01" }, { "slug": "claude-searchbot", "name": "Claude-SearchBot", "operator": "Anthropic", "operator_slug": "anthropic", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "Claude-SearchBot", "user_agent_substring": "Claude-SearchBot", "user_agent_example": "Mozilla/5.0 (compatible; Claude-SearchBot/1.0; +Claude-SearchBot@anthropic.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Indexes pages so Claude's web search can find and cite them. Separate token from the training crawler, so search visibility and training consent are independent decisions.", "cost_of_blocking": "You stop appearing in Claude's search results and citations.", "operator_docs": "https://support.anthropic.com/en/articles/8896518", "html_url": "https://www.pathwren.workers.dev/crawler/claude-searchbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/claude-searchbot.json", "last_reviewed": "2026-09-01" }, { "slug": "claude-web", "name": "Claude-Web", "operator": "Anthropic", "operator_slug": "anthropic", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "Claude-Web", "user_agent_substring": "Claude-Web", "user_agent_example": "Mozilla/5.0 (compatible; Claude-Web/1.0)", "respects_robots_txt": "n-a", "respects_robots_txt_label": "control token only — no crawler", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "An earlier Anthropic token for user-facing web access, superseded by Claude-User and Claude-SearchBot. Kept here because it appears in most published robots.txt templates.", "cost_of_blocking": "None in practice. Retain the rule; expect no traffic.", "operator_docs": "https://support.anthropic.com/en/articles/8896518", "html_url": "https://www.pathwren.workers.dev/crawler/claude-web.html", "json_url": "https://www.pathwren.workers.dev/crawler/claude-web.json", "last_reviewed": "2026-09-01" }, { "slug": "duckassistbot", "name": "DuckAssistBot", "operator": "DuckDuckGo", "operator_slug": "duckduckgo", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "DuckAssistBot", "user_agent_substring": "DuckAssistBot", "user_agent_example": "Mozilla/5.0 (compatible; DuckAssistBot/1.0; +https://duckduckgo.com/duckassistbot)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches pages so DuckAssist can generate and cite answers inside DuckDuckGo.", "cost_of_blocking": "No DuckAssist answers or citations from your site. Ordinary DuckDuckGo results are unaffected.", "operator_docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/", "html_url": "https://www.pathwren.workers.dev/crawler/duckassistbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/duckassistbot.json", "last_reviewed": "2026-09-01" }, { "slug": "google-cloudvertexbot", "name": "Google-CloudVertexBot", "operator": "Google", "operator_slug": "google", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "Google-CloudVertexBot", "user_agent_substring": "Google-CloudVertexBot", "user_agent_example": "Mozilla/5.0 (compatible; Google-CloudVertexBot/1.0; +https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json", "ipv4_prefix_count": 135, "ipv6_prefix_count": 135, "what_it_is": "Crawls a site on behalf of a Vertex AI Agent Builder customer who is building an agent over that site. It only visits sites the customer has asked it to.", "cost_of_blocking": "Third parties can no longer build Vertex AI agents that read your site. Irrelevant to Google Search.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/google-cloudvertexbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-cloudvertexbot.json", "last_reviewed": "2026-09-01" }, { "slug": "oai-searchbot", "name": "OAI-SearchBot", "operator": "OpenAI", "operator_slug": "openai", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "OAI-SearchBot", "user_agent_substring": "OAI-SearchBot", "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/searchbot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-searchbot.json", "ipv4_prefix_count": 35, "ipv6_prefix_count": 0, "what_it_is": "Builds the index ChatGPT search answers from. Content it collects is used for retrieval and citation, not for model training.", "cost_of_blocking": "High. Blocking this removes you from ChatGPT search results and from the source links ChatGPT shows. This is the single most expensive block on this list for anyone who wants to be cited by an assistant.", "operator_docs": "https://platform.openai.com/docs/bots", "html_url": "https://www.pathwren.workers.dev/crawler/oai-searchbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/oai-searchbot.json", "last_reviewed": "2026-09-01" }, { "slug": "perplexitybot", "name": "PerplexityBot", "operator": "Perplexity", "operator_slug": "perplexity", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "PerplexityBot", "user_agent_substring": "PerplexityBot", "user_agent_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://www.perplexity.ai/perplexitybot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/perplexity-bot.json", "ipv4_prefix_count": 8, "ipv6_prefix_count": 0, "what_it_is": "Builds Perplexity's search index. Perplexity is citation-heavy by product design, so inclusion here converts to referral traffic more directly than most AI surfaces.", "cost_of_blocking": "You stop being indexed and cited by Perplexity, and lose the referral clicks its citations produce.", "operator_docs": "https://docs.perplexity.ai/guides/bots", "html_url": "https://www.pathwren.workers.dev/crawler/perplexitybot.html", "json_url": "https://www.pathwren.workers.dev/crawler/perplexitybot.json", "last_reviewed": "2026-09-01" }, { "slug": "youbot", "name": "YouBot", "operator": "You.com", "operator_slug": "you", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "YouBot", "user_agent_substring": "YouBot", "user_agent_example": "Mozilla/5.0 (compatible; YouBot (+http://www.you.com))", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "You.com's crawler, feeding its AI search product and its search API.", "cost_of_blocking": "Removal from You.com's index and from answers built on its API.", "operator_docs": "https://about.you.com/youbot/", "html_url": "https://www.pathwren.workers.dev/crawler/youbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/youbot.json", "last_reviewed": "2026-09-01" } ] }