{
 "category": "search",
 "label": "Search engines",
 "description": "Classic index-and-rank crawlers. Several also feed their operator's generative answers, which is why the AI opt-out for Google and Apple is a token rather than a block.",
 "count": 13,
 "crawlers": [
  {
   "slug": "applebot",
   "name": "Applebot",
   "operator": "Apple",
   "operator_slug": "apple",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Applebot",
   "user_agent_substring": "Applebot",
   "user_agent_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://search.developer.apple.com/applebot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/apple-applebot.json",
   "ipv4_prefix_count": 33,
   "ipv6_prefix_count": 0,
   "what_it_is": "Powers Siri, Spotlight and Safari suggestions. Blocking it is a search decision, not an AI decision — the AI decision has its own token.",
   "cost_of_blocking": "You disappear from Siri, Spotlight and Safari search suggestions across Apple's install base.",
   "operator_docs": "https://support.apple.com/en-us/119829",
   "html_url": "https://www.pathwren.workers.dev/crawler/applebot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/applebot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "baiduspider",
   "name": "Baiduspider",
   "operator": "Baidu",
   "operator_slug": "baidu",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Baiduspider",
   "user_agent_substring": "Baiduspider",
   "user_agent_example": "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Baidu's search crawler, and the ingest path for Baidu's Ernie-backed answers.",
   "cost_of_blocking": "Removal from Baidu Search, which matters only if you want Chinese-language traffic.",
   "operator_docs": "https://help.baidu.com/question?prod_id=99&class=0&id=3001",
   "html_url": "https://www.pathwren.workers.dev/crawler/baiduspider.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/baiduspider.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "bingbot",
   "name": "bingbot",
   "operator": "Microsoft",
   "operator_slug": "microsoft",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "bingbot",
   "user_agent_substring": "bingbot",
   "user_agent_example": "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://www.bing.com/toolbox/bingbot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/bing-bingbot.json",
   "ipv4_prefix_count": 28,
   "ipv6_prefix_count": 0,
   "what_it_is": "Bing's only crawler, and therefore also the crawler behind Microsoft Copilot's grounding. Microsoft's documented way to keep search indexing while refusing generative reuse is the nocache / noarchive robots meta directive, not a separate user-agent.",
   "cost_of_blocking": "Very high and very wide: Bing, Copilot, DuckDuckGo and several assistants that resell Bing's index all lose you at once. Use nocache/noarchive rather than blocking.",
   "operator_docs": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
   "html_url": "https://www.pathwren.workers.dev/crawler/bingbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/bingbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "duckduckbot",
   "name": "DuckDuckBot",
   "operator": "DuckDuckGo",
   "operator_slug": "duckduckgo",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "DuckDuckBot",
   "user_agent_substring": "DuckDuckBot",
   "user_agent_example": "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://duckduckgo.com/duckduckbot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/duckduckgo-duckduckbot.json",
   "ipv4_prefix_count": 481,
   "ipv6_prefix_count": 0,
   "what_it_is": "DuckDuckGo's own crawler. Note that the bulk of DuckDuckGo's web results come from Bing, so blocking bingbot removes you from DuckDuckGo whether or not you allow this one.",
   "cost_of_blocking": "Limited on its own; the real DuckDuckGo lever is bingbot.",
   "operator_docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/duckduckbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/duckduckbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "googlebot",
   "name": "Googlebot",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Googlebot",
   "user_agent_substring": "Googlebot",
   "user_agent_example": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json",
   "ipv4_prefix_count": 169,
   "ipv6_prefix_count": 146,
   "what_it_is": "The classic search crawler. It is also the crawler behind AI Overviews: Google does not run a separate bot for them, which is why the only AI opt-out is the Google-Extended token and not a Googlebot block.",
   "cost_of_blocking": "Total. You leave Google Search. Never block this to avoid AI use; use Google-Extended instead.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googlebot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googlebot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "googlebot-image",
   "name": "Googlebot-Image",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Googlebot-Image",
   "user_agent_substring": "Googlebot-Image",
   "user_agent_example": "Googlebot-Image/1.0",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json",
   "ipv4_prefix_count": 169,
   "ipv6_prefix_count": 146,
   "what_it_is": "Image indexing for Google Images. A separate token so you can leave images out of search without leaving search.",
   "cost_of_blocking": "Your images stop appearing in Google Images.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googlebot-image.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googlebot-image.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "googlebot-news",
   "name": "Googlebot-News",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Googlebot-News",
   "user_agent_substring": "Googlebot-News",
   "user_agent_example": "(uses the Googlebot user-agent; controlled by the Googlebot-News robots token)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json",
   "ipv4_prefix_count": 169,
   "ipv6_prefix_count": 146,
   "what_it_is": "A robots.txt token controlling inclusion in Google News. It does not have its own user-agent string; the fetch arrives as Googlebot.",
   "cost_of_blocking": "Removal from Google News, with normal Search unaffected.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googlebot-news.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googlebot-news.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "petalbot",
   "name": "PetalBot",
   "operator": "Huawei",
   "operator_slug": "huawei",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "PetalBot",
   "user_agent_substring": "PetalBot",
   "user_agent_example": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Huawei's crawler for Petal Search, shipped as the default search on Huawei devices.",
   "cost_of_blocking": "Removal from Petal Search. Frequently blocked for volume rather than for policy.",
   "operator_docs": "https://aspiegel.com/petalbot",
   "html_url": "https://www.pathwren.workers.dev/crawler/petalbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/petalbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "seznambot",
   "name": "SeznamBot",
   "operator": "Seznam",
   "operator_slug": "seznam",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "SeznamBot",
   "user_agent_substring": "SeznamBot",
   "user_agent_example": "Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/en/seznambot-intro/)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Seznam's crawler — the dominant search engine in the Czech Republic and one of the few national engines with its own index.",
   "cost_of_blocking": "Removal from Seznam. Also removes you from its IndexNow endpoint's usefulness.",
   "operator_docs": "https://napoveda.seznam.cz/en/seznamzbozi/subject-matter-crawler/",
   "html_url": "https://www.pathwren.workers.dev/crawler/seznambot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/seznambot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "storebot-google",
   "name": "Storebot-Google",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Storebot-Google",
   "user_agent_substring": "Storebot-Google",
   "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36 (compatible; Storebot-Google/1.0; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 135,
   "ipv6_prefix_count": 135,
   "what_it_is": "Checks shopping and checkout flows for Google's shopping surfaces.",
   "cost_of_blocking": "Product listings may lose shopping-specific enrichment. Irrelevant to non-commerce sites.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/storebot-google.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/storebot-google.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "timpibot",
   "name": "Timpibot",
   "operator": "Timpi",
   "operator_slug": "timpi",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Timpibot",
   "user_agent_substring": "Timpibot",
   "user_agent_example": "Mozilla/5.0 (compatible; Timpibot/0.1; +https://timpi.io)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "A distributed crawler building an independent search index outside the Google/Bing duopoly.",
   "cost_of_blocking": "Absence from a small independent index.",
   "operator_docs": "https://timpi.io/",
   "html_url": "https://www.pathwren.workers.dev/crawler/timpibot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/timpibot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "yandexbot",
   "name": "YandexBot",
   "operator": "Yandex",
   "operator_slug": "yandex",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "YandexBot",
   "user_agent_substring": "YandexBot",
   "user_agent_example": "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Yandex's search crawler, which also feeds Alice and Yandex's generative answers.",
   "cost_of_blocking": "Removal from Yandex Search. Verify with reverse DNS to a yandex.ru, yandex.net or yandex.com host — YandexBot is among the most-spoofed user-agents there is.",
   "operator_docs": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html",
   "html_url": "https://www.pathwren.workers.dev/crawler/yandexbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/yandexbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "yeti",
   "name": "Yeti",
   "operator": "Naver",
   "operator_slug": "naver",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Yeti",
   "user_agent_substring": "Yeti",
   "user_agent_example": "Mozilla/5.0 (compatible; Yeti/1.1; +https://naver.me/spd)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Naver's crawler. Naver is South Korea's largest search portal and runs its own index and its own generative answers.",
   "cost_of_blocking": "Removal from Naver, which is most of Korean search.",
   "operator_docs": "https://searchadvisor.naver.com/guide/seo-basic-crawl",
   "html_url": "https://www.pathwren.workers.dev/crawler/yeti.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/yeti.json",
   "last_reviewed": "2026-09-01"
  }
 ]
}