{
 "package": "ai-crawler-robots",
 "data_version": "2026-09-01T19:26:24+00:00",
 "generated_at": "2026-09-01T19:26:24+00:00",
 "source": "AI Crawler Index — https://www.pathwren.workers.dev — independent, non-commercial; data CC0-1.0, code MIT",
 "self_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/data.json",
 "docs_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/",
 "index_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/",
 "license": {
  "code": "MIT",
  "data": "CC0-1.0"
 },
 "ai_categories": [
  "ai-training",
  "ai-search",
  "user-fetch",
  "dataset"
 ],
 "categories": {
  "ai-training": {
   "label": "AI training crawlers",
   "description": "Collect pages in bulk so that a model can be trained or fine-tuned on them. Blocking these removes you from future training sets and changes nothing a user sees today."
  },
  "ai-search": {
   "label": "AI search crawlers",
   "description": "Build the retrieval index an assistant answers and cites from. These are the crawlers that send you traffic; blocking them is the expensive mistake in this space."
  },
  "user-fetch": {
   "label": "User-triggered fetchers",
   "description": "Fetch one page because a person asked for it, right then. One human intent, one request. Blocking them produces a visible error for a real reader."
  },
  "dataset": {
   "label": "Corpus and dataset builders",
   "description": "Crawl the web into a published or resold dataset that other people train on. Highest leverage per block, longest delay before any effect."
  },
  "search": {
   "label": "Search engines",
   "description": "Classic index-and-rank crawlers. Several also feed their operator's generative answers, which is why the AI opt-out for Google and Apple is a token rather than a block."
  },
  "seo": {
   "label": "SEO and backlink crawlers",
   "description": "Commercial link-graph tooling. No user-facing effect either way, and usually a large share of your bot bandwidth."
  },
  "archive": {
   "label": "Archivers",
   "description": "Preservation crawlers. Their output is public and permanent, which makes them a separate decision from the AI one."
  },
  "preview": {
   "label": "Link preview fetchers",
   "description": "Read your Open Graph tags when someone shares a link. Blocking these is almost always an accident."
  },
  "tool": {
   "label": "Tools and frameworks",
   "description": "Not operators: crawling software anyone can run. The party behind the request is unknown, so treat them as a rate-limit question rather than a consent question."
  }
 },
 "crawlers": [
  {
   "slug": "ahrefsbot",
   "name": "AhrefsBot",
   "operator": "Ahrefs",
   "operator_slug": "ahrefs",
   "token": "AhrefsBot",
   "ua": "AhrefsBot",
   "ua_example": "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "No user-facing effect. Ahrefs honours Crawl-delay, so rate-limiting is usually better than blocking.",
   "what_it_is": "Ahrefs' backlink crawler, and one of the largest non-search crawlers on the web by request volume.",
   "docs": "https://ahrefs.com/robot"
  },
  {
   "slug": "ai2bot",
   "name": "AI2Bot",
   "operator": "Allen Institute for AI",
   "operator_slug": "ai2",
   "token": "AI2Bot",
   "ua": "AI2Bot",
   "ua_example": "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Excluded from open research datasets. Worth a deliberate decision: this is the category where 'blocking AI' also blocks the open, auditable end of it.",
   "what_it_is": "The Allen Institute's crawler, gathering pages for open research corpora such as Dolma that underpin fully open models like OLMo.",
   "docs": "https://allenai.org/crawler"
  },
  {
   "slug": "ai2bot-dolma",
   "name": "Ai2Bot-Dolma",
   "operator": "Allen Institute for AI",
   "operator_slug": "ai2",
   "token": "Ai2Bot-Dolma",
   "ua": "Ai2Bot-Dolma",
   "ua_example": "Mozilla/5.0 (compatible) Ai2Bot-Dolma (+https://www.allenai.org/crawler)",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Same as AI2Bot: exclusion from an open, published training corpus.",
   "what_it_is": "The variant of AI2's crawler named for the Dolma corpus specifically.",
   "docs": "https://allenai.org/crawler"
  },
  {
   "slug": "amazonbot",
   "name": "Amazonbot",
   "operator": "Amazon",
   "operator_slug": "amazon",
   "token": "Amazonbot",
   "ua": "Amazonbot",
   "ua_example": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Alexa and Amazon's assistants stop answering from your pages. Verify with reverse DNS to crawl.amazonbot.amazon before trusting the user-agent.",
   "what_it_is": "Amazon's crawler, feeding Alexa's ability to answer questions from the web and Amazon's own search and assistant products.",
   "docs": "https://developer.amazon.com/amazonbot"
  },
  {
   "slug": "anthropic-ai",
   "name": "anthropic-ai",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "token": "anthropic-ai",
   "ua": "anthropic-ai",
   "ua_example": "(no live crawler currently identifies with this string)",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "n-a",
   "respects_label": "control token only — no crawler",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "None. Nothing crawls under this name today; keeping the rule is harmless insurance.",
   "what_it_is": "A legacy robots.txt token from before Anthropic consolidated on ClaudeBot. It is still widely present in robots.txt files and costs nothing to keep, but it is a control token rather than a bot you will see in logs.",
   "docs": "https://support.anthropic.com/en/articles/8896518"
  },
  {
   "slug": "applebot",
   "name": "Applebot",
   "operator": "Apple",
   "operator_slug": "apple",
   "token": "Applebot",
   "ua": "Applebot",
   "ua_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://search.developer.apple.com/applebot.json",
   "cost_of_blocking": "You disappear from Siri, Spotlight and Safari search suggestions across Apple's install base.",
   "what_it_is": "Powers Siri, Spotlight and Safari suggestions. Blocking it is a search decision, not an AI decision — the AI decision has its own token.",
   "docs": "https://support.apple.com/en-us/119829"
  },
  {
   "slug": "applebot-extended",
   "name": "Applebot-Extended",
   "operator": "Apple",
   "operator_slug": "apple",
   "token": "Applebot-Extended",
   "ua": "(control token only — no crawler)",
   "ua_example": "(none: Applebot-Extended never appears as a user-agent)",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "n-a",
   "respects_label": "control token only — no crawler",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Excluded from Apple Intelligence training. Siri, Spotlight and Safari suggestions are unaffected.",
   "what_it_is": "Apple's counterpart to Google-Extended: a robots.txt token that withdraws consent for Apple Intelligence and Apple foundation-model training, without touching Applebot's search crawl.",
   "docs": "https://support.apple.com/en-us/119829"
  },
  {
   "slug": "archive-org-bot",
   "name": "archive.org_bot",
   "operator": "Internet Archive",
   "operator_slug": "internetarchive",
   "token": "archive.org_bot",
   "ua": "archive.org_bot",
   "ua_example": "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)",
   "category": "archive",
   "category_label": "Archivers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Your site stops being preserved. When it dies, it is gone. Consider this one separately from the AI question.",
   "what_it_is": "The Wayback Machine's crawler. Preservation rather than AI, but it lands in the same 'is this bot welcome' decision and its output is a public corpus.",
   "docs": "https://archive.org/details/archive.org_bot"
  },
  {
   "slug": "baiduspider",
   "name": "Baiduspider",
   "operator": "Baidu",
   "operator_slug": "baidu",
   "token": "Baiduspider",
   "ua": "Baiduspider",
   "ua_example": "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Removal from Baidu Search, which matters only if you want Chinese-language traffic.",
   "what_it_is": "Baidu's search crawler, and the ingest path for Baidu's Ernie-backed answers.",
   "docs": "https://help.baidu.com/question?prod_id=99&class=0&id=3001"
  },
  {
   "slug": "bingbot",
   "name": "bingbot",
   "operator": "Microsoft",
   "operator_slug": "microsoft",
   "token": "bingbot",
   "ua": "bingbot",
   "ua_example": "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://www.bing.com/toolbox/bingbot.json",
   "cost_of_blocking": "Very high and very wide: Bing, Copilot, DuckDuckGo and several assistants that resell Bing's index all lose you at once. Use nocache/noarchive rather than blocking.",
   "what_it_is": "Bing's only crawler, and therefore also the crawler behind Microsoft Copilot's grounding. Microsoft's documented way to keep search indexing while refusing generative reuse is the nocache / noarchive robots meta directive, not a separate user-agent.",
   "docs": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0"
  },
  {
   "slug": "bytespider",
   "name": "Bytespider",
   "operator": "ByteDance",
   "operator_slug": "bytedance",
   "token": "Bytespider",
   "ua": "Bytespider",
   "ua_example": "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "disputed",
   "respects_label": "compliance disputed",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Little to lose. If you want it gone, expect to block by user-agent at the edge rather than to ask politely in robots.txt.",
   "what_it_is": "ByteDance's crawler, associated with training data collection for Doubao and related models. Repeatedly reported by CDNs and site operators as the highest-volume AI crawler on the web and as inconsistent about robots.txt.",
   "docs": "https://www.bytespider.net/"
  },
  {
   "slug": "ccbot",
   "name": "CCBot",
   "operator": "Common Crawl",
   "operator_slug": "commoncrawl",
   "token": "CCBot",
   "ua": "CCBot",
   "ua_example": "CCBot/2.0 (https://commoncrawl.org/faq/)",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Future Common Crawl snapshots exclude you, so downstream training sets lose you too — but only going forward. Existing snapshots are permanent and blocking today does not retract them.",
   "what_it_is": "Common Crawl's corpus builder. It trains nothing itself, but its archive is an input to most open and many closed LLM training sets, which makes it the highest-leverage single entry on this list.",
   "docs": "https://commoncrawl.org/faq"
  },
  {
   "slug": "chatgpt-user",
   "name": "ChatGPT-User",
   "operator": "OpenAI",
   "operator_slug": "openai",
   "token": "ChatGPT-User",
   "ua": "ChatGPT-User",
   "ua_example": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; ChatGPT-User/1.0; +https://openai.com/bot",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://openai.com/chatgpt-user.json",
   "cost_of_blocking": "ChatGPT cannot open your pages when a user explicitly asks it to. The user sees a fetch failure. This is usually the last bot anyone means to block.",
   "what_it_is": "Fetches a single page at the moment a user or a ChatGPT agent asks for it — a pasted link, a browsing step, an Operator task. One human intent, one request. OpenAI states these fetches are not used for training.",
   "docs": "https://platform.openai.com/docs/bots"
  },
  {
   "slug": "claude-searchbot",
   "name": "Claude-SearchBot",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "token": "Claude-SearchBot",
   "ua": "Claude-SearchBot",
   "ua_example": "Mozilla/5.0 (compatible; Claude-SearchBot/1.0; +Claude-SearchBot@anthropic.com)",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "You stop appearing in Claude's search results and citations.",
   "what_it_is": "Indexes pages so Claude's web search can find and cite them. Separate token from the training crawler, so search visibility and training consent are independent decisions.",
   "docs": "https://support.anthropic.com/en/articles/8896518"
  },
  {
   "slug": "claude-user",
   "name": "Claude-User",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "token": "Claude-User",
   "ua": "Claude-User",
   "ua_example": "Mozilla/5.0 (compatible; Claude-User/1.0; +Claude-User@anthropic.com)",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Claude reports a fetch failure to a user who asked for your page by name.",
   "what_it_is": "Fetches a page because a Claude user asked Claude to read it, at that moment.",
   "docs": "https://support.anthropic.com/en/articles/8896518"
  },
  {
   "slug": "claude-web",
   "name": "Claude-Web",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "token": "Claude-Web",
   "ua": "Claude-Web",
   "ua_example": "Mozilla/5.0 (compatible; Claude-Web/1.0)",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "respects": "n-a",
   "respects_label": "control token only — no crawler",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "None in practice. Retain the rule; expect no traffic.",
   "what_it_is": "An earlier Anthropic token for user-facing web access, superseded by Claude-User and Claude-SearchBot. Kept here because it appears in most published robots.txt templates.",
   "docs": "https://support.anthropic.com/en/articles/8896518"
  },
  {
   "slug": "claudebot",
   "name": "ClaudeBot",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "token": "ClaudeBot",
   "ua": "ClaudeBot",
   "ua_example": "Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Content excluded from training data for future Claude models. No effect on Claude's ability to fetch a link a user gives it.",
   "what_it_is": "Anthropic's bulk crawler, gathering pages that may be used to train Claude models.",
   "docs": "https://support.anthropic.com/en/articles/8896518"
  },
  {
   "slug": "cohere-ai",
   "name": "cohere-ai",
   "operator": "Cohere",
   "operator_slug": "cohere",
   "token": "cohere-ai",
   "ua": "cohere-ai",
   "ua_example": "Mozilla/5.0 (compatible; cohere-ai)",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Cohere-powered assistants cannot read your pages on request.",
   "what_it_is": "Cohere's fetcher, used when its assistant products need a page.",
   "docs": "https://cohere.com/"
  },
  {
   "slug": "cohere-training-data-crawler",
   "name": "cohere-training-data-crawler",
   "operator": "Cohere",
   "operator_slug": "cohere",
   "token": "cohere-training-data-crawler",
   "ua": "cohere-training-data-crawler",
   "ua_example": "Mozilla/5.0 (compatible; cohere-training-data-crawler)",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Excluded from Cohere model training.",
   "what_it_is": "Cohere's separately-named bulk crawler for model training data, split out so consent for training and consent for retrieval can differ.",
   "docs": "https://cohere.com/"
  },
  {
   "slug": "diffbot",
   "name": "Diffbot",
   "operator": "Diffbot",
   "operator_slug": "diffbot",
   "token": "Diffbot",
   "ua": "Diffbot",
   "ua_example": "Mozilla/5.0 (compatible; Diffbot/0.1; +http://www.diffbot.com)",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Your facts stop entering a widely-licensed knowledge graph. Whether that is a loss depends on whether you want to be a machine-readable entity.",
   "what_it_is": "Extracts structured records from pages to build a commercial knowledge graph that is resold and used for retrieval and training.",
   "docs": "https://docs.diffbot.com/"
  },
  {
   "slug": "duckassistbot",
   "name": "DuckAssistBot",
   "operator": "DuckDuckGo",
   "operator_slug": "duckduckgo",
   "token": "DuckAssistBot",
   "ua": "DuckAssistBot",
   "ua_example": "Mozilla/5.0 (compatible; DuckAssistBot/1.0; +https://duckduckgo.com/duckassistbot)",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "No DuckAssist answers or citations from your site. Ordinary DuckDuckGo results are unaffected.",
   "what_it_is": "Fetches pages so DuckAssist can generate and cite answers inside DuckDuckGo.",
   "docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/"
  },
  {
   "slug": "duckduckbot",
   "name": "DuckDuckBot",
   "operator": "DuckDuckGo",
   "operator_slug": "duckduckgo",
   "token": "DuckDuckBot",
   "ua": "DuckDuckBot",
   "ua_example": "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://duckduckgo.com/duckduckbot.json",
   "cost_of_blocking": "Limited on its own; the real DuckDuckGo lever is bingbot.",
   "what_it_is": "DuckDuckGo's own crawler. Note that the bulk of DuckDuckGo's web results come from Bing, so blocking bingbot removes you from DuckDuckGo whether or not you allow this one.",
   "docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/"
  },
  {
   "slug": "facebookbot",
   "name": "FacebookBot",
   "operator": "Meta",
   "operator_slug": "meta",
   "token": "FacebookBot",
   "ua": "FacebookBot",
   "ua_example": "FacebookBot/1.0 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Negligible today. Keep the rule; expect little traffic.",
   "what_it_is": "Meta's older speech- and language-corpus crawler, largely superseded by meta-externalagent but still listed as a valid robots token.",
   "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers"
  },
  {
   "slug": "facebookexternalhit",
   "name": "facebookexternalhit",
   "operator": "Meta",
   "operator_slug": "meta",
   "token": "facebookexternalhit",
   "ua": "facebookexternalhit",
   "ua_example": "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
   "category": "preview",
   "category_label": "Link preview fetchers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Severe and usually accidental. Your links share as bare grey boxes with no title, image or description across Facebook, Instagram, Messenger and WhatsApp. Almost nobody means to block this.",
   "what_it_is": "The link unfurler: it reads your Open Graph tags when somebody shares your URL on a Meta property.",
   "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers"
  },
  {
   "slug": "firecrawlagent",
   "name": "FirecrawlAgent",
   "operator": "Firecrawl",
   "operator_slug": "firecrawl",
   "token": "FirecrawlAgent",
   "ua": "FirecrawlAgent",
   "ua_example": "Mozilla/5.0 (compatible; FirecrawlAgent/1.0; +https://firecrawl.dev)",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Applications built on Firecrawl cannot read your pages. This is increasingly how agents fetch the web, so it is a bigger block than its name suggests.",
   "what_it_is": "A hosted scrape-to-markdown service that LLM applications call to read pages. The requester is whoever is building on it, not Firecrawl itself, so volume and intent vary wildly.",
   "docs": "https://docs.firecrawl.dev/"
  },
  {
   "slug": "google-cloudvertexbot",
   "name": "Google-CloudVertexBot",
   "operator": "Google",
   "operator_slug": "google",
   "token": "Google-CloudVertexBot",
   "ua": "Google-CloudVertexBot",
   "ua_example": "Mozilla/5.0 (compatible; Google-CloudVertexBot/1.0; +https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers)",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "cost_of_blocking": "Third parties can no longer build Vertex AI agents that read your site. Irrelevant to Google Search.",
   "what_it_is": "Crawls a site on behalf of a Vertex AI Agent Builder customer who is building an agent over that site. It only visits sites the customer has asked it to.",
   "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
  },
  {
   "slug": "google-extended",
   "name": "Google-Extended",
   "operator": "Google",
   "operator_slug": "google",
   "token": "Google-Extended",
   "ua": "(control token only — no crawler)",
   "ua_example": "(none: Google-Extended never appears as a user-agent)",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "n-a",
   "respects_label": "control token only — no crawler",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "You are excluded from Gemini grounding and Gemini training. Google Search ranking and indexing are explicitly unaffected. This is the cleanest 'no training, keep my search traffic' lever that exists.",
   "what_it_is": "Not a crawler. A robots.txt token that tells Google whether pages Googlebot already fetched may be used to train and ground Gemini. You will never see it in an access log; disallowing it changes what Google does with content it fetched under a different name.",
   "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
  },
  {
   "slug": "google-inspectiontool",
   "name": "Google-InspectionTool",
   "operator": "Google",
   "operator_slug": "google",
   "token": "Google-InspectionTool",
   "ua": "Google-InspectionTool",
   "ua_example": "Mozilla/5.0 (compatible; Google-InspectionTool/1.0;)",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "cost_of_blocking": "Your own Search Console live tests stop working. Blocking this only hurts you.",
   "what_it_is": "The fetcher behind Search Console's URL Inspection and the Rich Results Test. It runs when a site owner clicks a button.",
   "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
  },
  {
   "slug": "googlebot",
   "name": "Googlebot",
   "operator": "Google",
   "operator_slug": "google",
   "token": "Googlebot",
   "ua": "Googlebot",
   "ua_example": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "cost_of_blocking": "Total. You leave Google Search. Never block this to avoid AI use; use Google-Extended instead.",
   "what_it_is": "The classic search crawler. It is also the crawler behind AI Overviews: Google does not run a separate bot for them, which is why the only AI opt-out is the Google-Extended token and not a Googlebot block.",
   "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
  },
  {
   "slug": "googlebot-image",
   "name": "Googlebot-Image",
   "operator": "Google",
   "operator_slug": "google",
   "token": "Googlebot-Image",
   "ua": "Googlebot-Image",
   "ua_example": "Googlebot-Image/1.0",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "cost_of_blocking": "Your images stop appearing in Google Images.",
   "what_it_is": "Image indexing for Google Images. A separate token so you can leave images out of search without leaving search.",
   "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
  },
  {
   "slug": "googlebot-news",
   "name": "Googlebot-News",
   "operator": "Google",
   "operator_slug": "google",
   "token": "Googlebot-News",
   "ua": "Googlebot-News",
   "ua_example": "(uses the Googlebot user-agent; controlled by the Googlebot-News robots token)",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "cost_of_blocking": "Removal from Google News, with normal Search unaffected.",
   "what_it_is": "A robots.txt token controlling inclusion in Google News. It does not have its own user-agent string; the fetch arrives as Googlebot.",
   "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
  },
  {
   "slug": "googleother",
   "name": "GoogleOther",
   "operator": "Google",
   "operator_slug": "google",
   "token": "GoogleOther",
   "ua": "GoogleOther",
   "ua_example": "Mozilla/5.0 (compatible; GoogleOther)",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "cost_of_blocking": "No effect on Search indexing. Blocks internal Google research and product fetches.",
   "what_it_is": "A generic fetcher used by Google product teams for one-off crawls and research, including data collection that does not belong to Search.",
   "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
  },
  {
   "slug": "gptbot",
   "name": "GPTBot",
   "operator": "OpenAI",
   "operator_slug": "openai",
   "token": "GPTBot",
   "ua": "GPTBot",
   "ua_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2; +https://openai.com/gptbot",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://openai.com/gptbot.json",
   "cost_of_blocking": "Your content is excluded from training data for future OpenAI models. No effect on ChatGPT search visibility, on citations, or on links a user pastes into ChatGPT.",
   "what_it_is": "OpenAI's bulk crawler. Pages it fetches may be used to train future OpenAI foundation models. It is not the bot that puts you in ChatGPT's search results, and blocking it does not remove you from them.",
   "docs": "https://platform.openai.com/docs/bots"
  },
  {
   "slug": "ia-archiver",
   "name": "ia_archiver",
   "operator": "Internet Archive",
   "operator_slug": "internetarchive",
   "token": "ia_archiver",
   "ua": "ia_archiver",
   "ua_example": "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)",
   "category": "archive",
   "category_label": "Archivers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Negligible today; retain for tidiness.",
   "what_it_is": "The legacy Alexa/Internet Archive token, still present in most robots.txt files and still occasionally honoured.",
   "docs": "https://archive.org/details/archive.org_bot"
  },
  {
   "slug": "imagesiftbot",
   "name": "ImagesiftBot",
   "operator": "Hive AI",
   "operator_slug": "hive",
   "token": "ImagesiftBot",
   "ua": "ImagesiftBot",
   "ua_example": "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Your images stop entering an image dataset and reverse-image index.",
   "what_it_is": "Crawls images for Hive AI's reverse-image and dataset products. Image-heavy sites see this one long before they see the text crawlers.",
   "docs": "https://imagesift.com/about"
  },
  {
   "slug": "img2dataset",
   "name": "img2dataset",
   "operator": "LAION / img2dataset",
   "operator_slug": "laion",
   "token": "img2dataset",
   "ua": "img2dataset",
   "ua_example": "img2dataset",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Your images are skipped when someone materialises an image-text dataset that references them.",
   "what_it_is": "The tool used to turn image-URL lists such as LAION's into downloaded training sets. It is run by whoever is building a dataset, not by a single operator.",
   "docs": "https://github.com/rom1504/img2dataset"
  },
  {
   "slug": "meta-externalagent",
   "name": "meta-externalagent",
   "operator": "Meta",
   "operator_slug": "meta",
   "token": "meta-externalagent",
   "ua": "meta-externalagent",
   "ua_example": "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Excluded from Meta AI training. Link previews on Facebook, Instagram and WhatsApp are unaffected — those are a different bot.",
   "what_it_is": "Meta's AI crawler, gathering training data for Llama and Meta AI. It replaced the older FacebookBot name for this purpose.",
   "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers"
  },
  {
   "slug": "meta-externalfetcher",
   "name": "meta-externalfetcher",
   "operator": "Meta",
   "operator_slug": "meta",
   "token": "meta-externalfetcher",
   "ua": "meta-externalfetcher",
   "ua_example": "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Meta AI cannot read pages users hand it.",
   "what_it_is": "Fetches a page when a Meta AI user asks about a specific link.",
   "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers"
  },
  {
   "slug": "mistralai-user",
   "name": "MistralAI-User",
   "operator": "Mistral AI",
   "operator_slug": "mistral",
   "token": "MistralAI-User",
   "ua": "MistralAI-User",
   "ua_example": "Mozilla/5.0 (compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Le Chat cannot open links your readers give it.",
   "what_it_is": "Fetches a page when a Le Chat user asks Mistral's assistant to read it.",
   "docs": "https://docs.mistral.ai/"
  },
  {
   "slug": "oai-searchbot",
   "name": "OAI-SearchBot",
   "operator": "OpenAI",
   "operator_slug": "openai",
   "token": "OAI-SearchBot",
   "ua": "OAI-SearchBot",
   "ua_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://openai.com/searchbot.json",
   "cost_of_blocking": "High. Blocking this removes you from ChatGPT search results and from the source links ChatGPT shows. This is the single most expensive block on this list for anyone who wants to be cited by an assistant.",
   "what_it_is": "Builds the index ChatGPT search answers from. Content it collects is used for retrieval and citation, not for model training.",
   "docs": "https://platform.openai.com/docs/bots"
  },
  {
   "slug": "omgili",
   "name": "omgili",
   "operator": "Webz.io",
   "operator_slug": "webz",
   "token": "omgili",
   "ua": "omgili",
   "ua_example": "omgili/0.5 +http://omgili.com",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Same as omgilibot.",
   "what_it_is": "The older robots token for the same Webz.io collection, still honoured and still worth listing.",
   "docs": "https://webz.io/blog/machine-learning/"
  },
  {
   "slug": "omgilibot",
   "name": "omgilibot",
   "operator": "Webz.io",
   "operator_slug": "webz",
   "token": "omgilibot",
   "ua": "omgilibot",
   "ua_example": "omgilibot/0.4; +http://omgili.com",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Exclusion from a commercial dataset resold to third parties.",
   "what_it_is": "Webz.io's crawler, collecting web and forum text sold as datasets, including to model builders.",
   "docs": "https://webz.io/blog/machine-learning/"
  },
  {
   "slug": "perplexity-user",
   "name": "Perplexity-User",
   "operator": "Perplexity",
   "operator_slug": "perplexity",
   "token": "Perplexity-User",
   "ua": "Perplexity-User",
   "ua_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "respects": "by-design-no",
   "respects_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://www.perplexity.ai/perplexity-user.json",
   "cost_of_blocking": "Not controllable via robots.txt. If you must stop it, verify by the published IP ranges and block at the edge — and accept that users who ask for your page get an error.",
   "what_it_is": "Fetches a page because a Perplexity user asked for it. Perplexity documents that this fetch is user-initiated and is therefore not governed by robots.txt — a robots rule will not stop it, by stated policy.",
   "docs": "https://docs.perplexity.ai/guides/bots"
  },
  {
   "slug": "perplexitybot",
   "name": "PerplexityBot",
   "operator": "Perplexity",
   "operator_slug": "perplexity",
   "token": "PerplexityBot",
   "ua": "PerplexityBot",
   "ua_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://www.perplexity.ai/perplexitybot.json",
   "cost_of_blocking": "You stop being indexed and cited by Perplexity, and lose the referral clicks its citations produce.",
   "what_it_is": "Builds Perplexity's search index. Perplexity is citation-heavy by product design, so inclusion here converts to referral traffic more directly than most AI surfaces.",
   "docs": "https://docs.perplexity.ai/guides/bots"
  },
  {
   "slug": "petalbot",
   "name": "PetalBot",
   "operator": "Huawei",
   "operator_slug": "huawei",
   "token": "PetalBot",
   "ua": "PetalBot",
   "ua_example": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Removal from Petal Search. Frequently blocked for volume rather than for policy.",
   "what_it_is": "Huawei's crawler for Petal Search, shipped as the default search on Huawei devices.",
   "docs": "https://aspiegel.com/petalbot"
  },
  {
   "slug": "scrapy",
   "name": "Scrapy",
   "operator": "Scrapy project",
   "operator_slug": "scrapy",
   "token": "Scrapy",
   "ua": "Scrapy",
   "ua_example": "Scrapy/2.11.0 (+https://scrapy.org)",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "You block a very large tail of unattributed one-off crawlers, and also every well-behaved researcher who did not change the default.",
   "what_it_is": "Not an operator: the default user-agent of the most common Python crawling framework. Anyone can be behind it. Modern Scrapy obeys robots.txt by default, which is why the default UA is still worth a rule.",
   "docs": "https://scrapy.org/"
  },
  {
   "slug": "semrushbot",
   "name": "SemrushBot",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "token": "SemrushBot",
   "ua": "SemrushBot",
   "ua_example": "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Your competitors' Semrush reports get thinner, and so do yours. No user-facing effect.",
   "what_it_is": "Semrush's backlink and keyword crawler. It is not an AI crawler, but it is usually in the top three by volume on any site, and it is the cheapest block on this list.",
   "docs": "https://www.semrush.com/bot/"
  },
  {
   "slug": "semrushbot-ocob",
   "name": "SemrushBot-OCOB",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "token": "SemrushBot-OCOB",
   "ua": "SemrushBot-OCOB",
   "ua_example": "Mozilla/5.0 (compatible; SemrushBot-OCOB/1.0; +http://www.semrush.com/bot.html)",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Exclusion from Semrush's AI corpus, with its SEO crawl unaffected.",
   "what_it_is": "Semrush's separately-tokenised crawler for its AI content tooling, split out so SEO crawling and AI reuse can be answered differently.",
   "docs": "https://www.semrush.com/bot/"
  },
  {
   "slug": "seznambot",
   "name": "SeznamBot",
   "operator": "Seznam",
   "operator_slug": "seznam",
   "token": "SeznamBot",
   "ua": "SeznamBot",
   "ua_example": "Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/en/seznambot-intro/)",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Removal from Seznam. Also removes you from its IndexNow endpoint's usefulness.",
   "what_it_is": "Seznam's crawler — the dominant search engine in the Czech Republic and one of the few national engines with its own index.",
   "docs": "https://napoveda.seznam.cz/en/seznamzbozi/subject-matter-crawler/"
  },
  {
   "slug": "storebot-google",
   "name": "Storebot-Google",
   "operator": "Google",
   "operator_slug": "google",
   "token": "Storebot-Google",
   "ua": "Storebot-Google",
   "ua_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36 (compatible; Storebot-Google/1.0; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "cost_of_blocking": "Product listings may lose shopping-specific enrichment. Irrelevant to non-commerce sites.",
   "what_it_is": "Checks shopping and checkout flows for Google's shopping surfaces.",
   "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
  },
  {
   "slug": "tiktokspider",
   "name": "TikTokSpider",
   "operator": "ByteDance",
   "operator_slug": "bytedance",
   "token": "TikTokSpider",
   "ua": "TikTokSpider",
   "ua_example": "Mozilla/5.0 (compatible; TikTokSpider; ttspider-feedback@tiktok.com)",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "disputed",
   "respects_label": "compliance disputed",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Little to lose unless TikTok search referral matters to you.",
   "what_it_is": "A second ByteDance crawler identifying with TikTok, collecting page content for the same family of models.",
   "docs": "https://www.bytespider.net/"
  },
  {
   "slug": "timpibot",
   "name": "Timpibot",
   "operator": "Timpi",
   "operator_slug": "timpi",
   "token": "Timpibot",
   "ua": "Timpibot",
   "ua_example": "Mozilla/5.0 (compatible; Timpibot/0.1; +https://timpi.io)",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Absence from a small independent index.",
   "what_it_is": "A distributed crawler building an independent search index outside the Google/Bing duopoly.",
   "docs": "https://timpi.io/"
  },
  {
   "slug": "webzio-extended",
   "name": "Webzio-Extended",
   "operator": "Webz.io",
   "operator_slug": "webz",
   "token": "Webzio-Extended",
   "ua": "Webzio-Extended",
   "ua_example": "Mozilla/5.0 (compatible; Webzio-Extended/1.0)",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Your content is excluded from the AI-training tier of Webz.io's product while ordinary collection continues.",
   "what_it_is": "Webz.io's opt-out token specifically for AI training reuse, in the pattern Google and Apple established.",
   "docs": "https://webz.io/blog/machine-learning/"
  },
  {
   "slug": "yandexbot",
   "name": "YandexBot",
   "operator": "Yandex",
   "operator_slug": "yandex",
   "token": "YandexBot",
   "ua": "YandexBot",
   "ua_example": "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Removal from Yandex Search. Verify with reverse DNS to a yandex.ru, yandex.net or yandex.com host — YandexBot is among the most-spoofed user-agents there is.",
   "what_it_is": "Yandex's search crawler, which also feeds Alice and Yandex's generative answers.",
   "docs": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html"
  },
  {
   "slug": "yeti",
   "name": "Yeti",
   "operator": "Naver",
   "operator_slug": "naver",
   "token": "Yeti",
   "ua": "Yeti",
   "ua_example": "Mozilla/5.0 (compatible; Yeti/1.1; +https://naver.me/spd)",
   "category": "search",
   "category_label": "Search engines",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Removal from Naver, which is most of Korean search.",
   "what_it_is": "Naver's crawler. Naver is South Korea's largest search portal and runs its own index and its own generative answers.",
   "docs": "https://searchadvisor.naver.com/guide/seo-basic-crawl"
  },
  {
   "slug": "youbot",
   "name": "YouBot",
   "operator": "You.com",
   "operator_slug": "you",
   "token": "YouBot",
   "ua": "YouBot",
   "ua_example": "Mozilla/5.0 (compatible; YouBot (+http://www.you.com))",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "respects": "documented",
   "respects_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "cost_of_blocking": "Removal from You.com's index and from answers built on its API.",
   "what_it_is": "You.com's crawler, feeding its AI search product and its search API.",
   "docs": "https://about.you.com/youbot/"
  }
 ],
 "policies": [
  {
   "slug": "allow-all",
   "title": "Allow everything, explicitly",
   "summary": "Every crawler on this index is named and allowed. Use when you want maximum reach into search and assistants and have nothing to withhold.",
   "detail": "An empty robots.txt already allows everything, so this file is not about permission — it is about being explicit. Naming each token means a later change is a one-line diff instead of a rewrite, and it documents that the allow was a decision. This is the policy this site itself serves.",
   "crawlers": [
    "ahrefsbot",
    "ai2bot",
    "ai2bot-dolma",
    "amazonbot",
    "anthropic-ai",
    "applebot",
    "applebot-extended",
    "archive-org-bot",
    "baiduspider",
    "bingbot",
    "bytespider",
    "ccbot",
    "chatgpt-user",
    "claude-searchbot",
    "claude-user",
    "claude-web",
    "claudebot",
    "cohere-ai",
    "cohere-training-data-crawler",
    "diffbot",
    "duckassistbot",
    "duckduckbot",
    "facebookbot",
    "facebookexternalhit",
    "firecrawlagent",
    "google-cloudvertexbot",
    "google-extended",
    "google-inspectiontool",
    "googlebot",
    "googlebot-image",
    "googlebot-news",
    "googleother",
    "gptbot",
    "ia-archiver",
    "imagesiftbot",
    "img2dataset",
    "meta-externalagent",
    "meta-externalfetcher",
    "mistralai-user",
    "oai-searchbot",
    "omgili",
    "omgilibot",
    "perplexity-user",
    "perplexitybot",
    "petalbot",
    "scrapy",
    "semrushbot",
    "semrushbot-ocob",
    "seznambot",
    "storebot-google",
    "tiktokspider",
    "timpibot",
    "webzio-extended",
    "yandexbot",
    "yeti",
    "youbot"
   ],
   "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/robots/allow-all.txt",
   "robots_txt": "# AI Crawler Index — policy: allow-all\n# Allow everything, explicitly\n# Every crawler on this index is named and allowed. Use when you want maximum reach into search and assistants and have nothing to withhold.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/allow-all.html\n# 56 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AhrefsBot\nAllow: /\n\nUser-agent: AI2Bot\nAllow: /\n\nUser-agent: Ai2Bot-Dolma\nAllow: /\n\nUser-agent: Amazonbot\nAllow: /\n\nUser-agent: anthropic-ai   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: Applebot\nAllow: /\n\nUser-agent: Applebot-Extended   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: archive.org_bot\nAllow: /\n\nUser-agent: Baiduspider\nAllow: /\n\nUser-agent: bingbot\nAllow: /\n\nUser-agent: Bytespider   # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: Claude-SearchBot\nAllow: /\n\nUser-agent: Claude-User\nAllow: /\n\nUser-agent: Claude-Web   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: cohere-ai\nAllow: /\n\nUser-agent: cohere-training-data-crawler\nAllow: /\n\nUser-agent: Diffbot\nAllow: /\n\nUser-agent: DuckAssistBot\nAllow: /\n\nUser-agent: DuckDuckBot\nAllow: /\n\nUser-agent: FacebookBot\nAllow: /\n\nUser-agent: facebookexternalhit\nAllow: /\n\nUser-agent: FirecrawlAgent\nAllow: /\n\nUser-agent: Google-CloudVertexBot\nAllow: /\n\nUser-agent: Google-Extended   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: Google-InspectionTool\nAllow: /\n\nUser-agent: Googlebot\nAllow: /\n\nUser-agent: Googlebot-Image\nAllow: /\n\nUser-agent: Googlebot-News\nAllow: /\n\nUser-agent: GoogleOther\nAllow: /\n\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ia_archiver\nAllow: /\n\nUser-agent: ImagesiftBot\nAllow: /\n\nUser-agent: img2dataset\nAllow: /\n\nUser-agent: meta-externalagent\nAllow: /\n\nUser-agent: meta-externalfetcher\nAllow: /\n\nUser-agent: MistralAI-User\nAllow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: omgili\nAllow: /\n\nUser-agent: omgilibot\nAllow: /\n\nUser-agent: Perplexity-User   # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: PetalBot\nAllow: /\n\nUser-agent: Scrapy\nAllow: /\n\nUser-agent: SemrushBot\nAllow: /\n\nUser-agent: SemrushBot-OCOB\nAllow: /\n\nUser-agent: SeznamBot\nAllow: /\n\nUser-agent: Storebot-Google\nAllow: /\n\nUser-agent: TikTokSpider   # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: Timpibot\nAllow: /\n\nUser-agent: Webzio-Extended\nAllow: /\n\nUser-agent: YandexBot\nAllow: /\n\nUser-agent: Yeti\nAllow: /\n\nUser-agent: YouBot\nAllow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n"
  },
  {
   "slug": "block-ai-training",
   "title": "Block AI training, keep AI search",
   "summary": "Refuse the crawlers that feed model training. Keep the ones that put you in ChatGPT, Claude, Perplexity and Gemini answers.",
   "detail": "The distinction most people actually want, and the one that is easy to get wrong: GPTBot trains, OAI-SearchBot indexes for citation. Blocking both loses you the traffic and gains you nothing extra. Google and Apple have no separate crawler at all — Google-Extended and Applebot-Extended are pure control tokens, so they belong in this file while Googlebot and Applebot must not.",
   "crawlers": [
    "anthropic-ai",
    "applebot-extended",
    "bytespider",
    "claudebot",
    "cohere-training-data-crawler",
    "facebookbot",
    "google-extended",
    "googleother",
    "gptbot",
    "meta-externalagent",
    "semrushbot-ocob",
    "tiktokspider",
    "webzio-extended"
   ],
   "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/robots/block-ai-training.txt",
   "robots_txt": "# AI Crawler Index — policy: block-ai-training\n# Block AI training, keep AI search\n# Refuse the crawlers that feed model training. Keep the ones that put you in ChatGPT, Claude, Perplexity and Gemini answers.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/block-ai-training.html\n# 13 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: anthropic-ai   # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: Applebot-Extended   # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: Bytespider   # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: FacebookBot\nDisallow: /\n\nUser-agent: Google-Extended   # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: SemrushBot-OCOB\nDisallow: /\n\nUser-agent: TikTokSpider   # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: Webzio-Extended\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n"
  },
  {
   "slug": "block-all-ai",
   "title": "Block every AI crawler",
   "summary": "Training, AI search, user-triggered fetches and corpus builders, all refused. Classic search engines still allowed.",
   "detail": "The maximal AI opt-out that still leaves you in Google and Bing. Understand the price before deploying it: you will not be cited by any assistant, and when a reader explicitly asks ChatGPT or Claude to open your page, they get an error. Note also that Perplexity-User and Bytespider are listed here but documented as not governed by robots.txt, so this file is a statement of intent for those two, not an enforcement mechanism.",
   "crawlers": [
    "ai2bot",
    "ai2bot-dolma",
    "amazonbot",
    "anthropic-ai",
    "applebot-extended",
    "bytespider",
    "ccbot",
    "chatgpt-user",
    "claude-searchbot",
    "claude-user",
    "claude-web",
    "claudebot",
    "cohere-ai",
    "cohere-training-data-crawler",
    "diffbot",
    "duckassistbot",
    "facebookbot",
    "google-cloudvertexbot",
    "google-extended",
    "googleother",
    "gptbot",
    "imagesiftbot",
    "img2dataset",
    "meta-externalagent",
    "meta-externalfetcher",
    "mistralai-user",
    "oai-searchbot",
    "omgili",
    "omgilibot",
    "perplexity-user",
    "perplexitybot",
    "semrushbot-ocob",
    "tiktokspider",
    "webzio-extended",
    "youbot"
   ],
   "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/robots/block-all-ai.txt",
   "robots_txt": "# AI Crawler Index — policy: block-all-ai\n# Block every AI crawler\n# Training, AI search, user-triggered fetches and corpus builders, all refused. Classic search engines still allowed.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/block-all-ai.html\n# 35 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: anthropic-ai   # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: Applebot-Extended   # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: Bytespider   # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: Claude-Web   # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: FacebookBot\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: Google-Extended   # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: omgilibot\nDisallow: /\n\nUser-agent: Perplexity-User   # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: SemrushBot-OCOB\nDisallow: /\n\nUser-agent: TikTokSpider   # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: Webzio-Extended\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n"
  },
  {
   "slug": "block-datasets",
   "title": "Block corpus and dataset builders",
   "summary": "Refuse the crawlers whose output is a dataset other people train on: Common Crawl, AI2, Webz.io, Diffbot, ImagesiftBot.",
   "detail": "These are the highest-leverage blocks per line, because one crawl becomes many downstream training runs. It is also the block with the longest delay before it has any effect, and no effect at all on archives already published.",
   "crawlers": [
    "ai2bot",
    "ai2bot-dolma",
    "ccbot",
    "diffbot",
    "imagesiftbot",
    "img2dataset",
    "omgili",
    "omgilibot"
   ],
   "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/robots/block-datasets.txt",
   "robots_txt": "# AI Crawler Index — policy: block-datasets\n# Block corpus and dataset builders\n# Refuse the crawlers whose output is a dataset other people train on: Common Crawl, AI2, Webz.io, Diffbot, ImagesiftBot.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/block-datasets.html\n# 8 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: omgilibot\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n"
  },
  {
   "slug": "allow-ai-search-only",
   "title": "Allow AI search and user fetches, block the rest",
   "summary": "Be findable and citable in assistants without contributing to training corpora.",
   "detail": "The inverse framing of block-ai-training, written as an allowlist so the default for anything new is deny. Fetches a user explicitly asked for stay allowed, because refusing those produces a visible error for a real person who wanted your page.",
   "crawlers": [
    "amazonbot",
    "applebot",
    "baiduspider",
    "bingbot",
    "chatgpt-user",
    "claude-searchbot",
    "claude-user",
    "claude-web",
    "cohere-ai",
    "duckassistbot",
    "duckduckbot",
    "facebookexternalhit",
    "google-cloudvertexbot",
    "googlebot",
    "googlebot-image",
    "googlebot-news",
    "meta-externalfetcher",
    "mistralai-user",
    "oai-searchbot",
    "perplexity-user",
    "perplexitybot",
    "petalbot",
    "seznambot",
    "storebot-google",
    "timpibot",
    "yandexbot",
    "yeti",
    "youbot"
   ],
   "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/robots/allow-ai-search-only.txt",
   "robots_txt": "# AI Crawler Index — policy: allow-ai-search-only\n# Allow AI search and user fetches, block the rest\n# Be findable and citable in assistants without contributing to training corpora.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/allow-ai-search-only.html\n# 28 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: Amazonbot\nAllow: /\n\nUser-agent: Applebot\nAllow: /\n\nUser-agent: Baiduspider\nAllow: /\n\nUser-agent: bingbot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: Claude-SearchBot\nAllow: /\n\nUser-agent: Claude-User\nAllow: /\n\nUser-agent: Claude-Web   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: cohere-ai\nAllow: /\n\nUser-agent: DuckAssistBot\nAllow: /\n\nUser-agent: DuckDuckBot\nAllow: /\n\nUser-agent: facebookexternalhit\nAllow: /\n\nUser-agent: Google-CloudVertexBot\nAllow: /\n\nUser-agent: Googlebot\nAllow: /\n\nUser-agent: Googlebot-Image\nAllow: /\n\nUser-agent: Googlebot-News\nAllow: /\n\nUser-agent: meta-externalfetcher\nAllow: /\n\nUser-agent: MistralAI-User\nAllow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: Perplexity-User   # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: PetalBot\nAllow: /\n\nUser-agent: SeznamBot\nAllow: /\n\nUser-agent: Storebot-Google\nAllow: /\n\nUser-agent: Timpibot\nAllow: /\n\nUser-agent: YandexBot\nAllow: /\n\nUser-agent: Yeti\nAllow: /\n\nUser-agent: YouBot\nAllow: /\n\n# Anything not named above is refused.\nUser-agent: *\nDisallow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n"
  },
  {
   "slug": "block-seo-tools",
   "title": "Block SEO and backlink crawlers",
   "summary": "Ahrefs, Semrush and friends. No user-facing consequence, and often the largest single slice of your bot traffic.",
   "detail": "The cheapest bandwidth saving available to most sites, and the one nobody regrets. The only cost is that your own dashboards on those tools get thinner. Consider Crawl-delay first: both major operators honour it.",
   "crawlers": [
    "ahrefsbot",
    "semrushbot"
   ],
   "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/robots/block-seo-tools.txt",
   "robots_txt": "# AI Crawler Index — policy: block-seo-tools\n# Block SEO and backlink crawlers\n# Ahrefs, Semrush and friends. No user-facing consequence, and often the largest single slice of your bot traffic.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/block-seo-tools.html\n# 2 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: SemrushBot\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n"
  },
  {
   "slug": "block-disputed",
   "title": "Block the crawlers with disputed robots compliance",
   "summary": "The ones repeatedly reported as ignoring robots.txt. Included for completeness — expect to enforce this at the edge instead.",
   "detail": "A robots.txt rule is a request. For the operators in this file the request is documented as unreliable or explicitly not applicable, so the honest use of this file is as a record of intent that sits alongside a real block by user-agent or by IP at your CDN.",
   "crawlers": [
    "bytespider",
    "perplexity-user",
    "tiktokspider"
   ],
   "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/robots/block-disputed.txt",
   "robots_txt": "# AI Crawler Index — policy: block-disputed\n# Block the crawlers with disputed robots compliance\n# The ones repeatedly reported as ignoring robots.txt. Included for completeness — expect to enforce this at the edge instead.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/block-disputed.html\n# 3 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: Bytespider   # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: Perplexity-User   # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: TikTokSpider   # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n"
  },
  {
   "slug": "maximum-ai-visibility",
   "title": "Maximum AI visibility",
   "summary": "Allow every AI crawler and every search engine; refuse only SEO scrapers. For sites whose goal is to be found and cited by machines.",
   "detail": "If your content exists to be read by assistants — documentation, reference data, an API — every block costs you and none of them protect anything. Pair this with an llms.txt, a sitemap, and per-item JSON, and the crawlers can actually use what they find.",
   "crawlers": [
    "ai2bot",
    "ai2bot-dolma",
    "amazonbot",
    "anthropic-ai",
    "applebot",
    "applebot-extended",
    "archive-org-bot",
    "baiduspider",
    "bingbot",
    "bytespider",
    "ccbot",
    "chatgpt-user",
    "claude-searchbot",
    "claude-user",
    "claude-web",
    "claudebot",
    "cohere-ai",
    "cohere-training-data-crawler",
    "diffbot",
    "duckassistbot",
    "duckduckbot",
    "facebookbot",
    "facebookexternalhit",
    "firecrawlagent",
    "google-cloudvertexbot",
    "google-extended",
    "google-inspectiontool",
    "googlebot",
    "googlebot-image",
    "googlebot-news",
    "googleother",
    "gptbot",
    "ia-archiver",
    "imagesiftbot",
    "img2dataset",
    "meta-externalagent",
    "meta-externalfetcher",
    "mistralai-user",
    "oai-searchbot",
    "omgili",
    "omgilibot",
    "perplexity-user",
    "perplexitybot",
    "petalbot",
    "scrapy",
    "semrushbot-ocob",
    "seznambot",
    "storebot-google",
    "tiktokspider",
    "timpibot",
    "webzio-extended",
    "yandexbot",
    "yeti",
    "youbot"
   ],
   "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/robots/maximum-ai-visibility.txt",
   "robots_txt": "# AI Crawler Index — policy: maximum-ai-visibility\n# Maximum AI visibility\n# Allow every AI crawler and every search engine; refuse only SEO scrapers. For sites whose goal is to be found and cited by machines.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/maximum-ai-visibility.html\n# 54 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AI2Bot\nAllow: /\n\nUser-agent: Ai2Bot-Dolma\nAllow: /\n\nUser-agent: Amazonbot\nAllow: /\n\nUser-agent: anthropic-ai   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: Applebot\nAllow: /\n\nUser-agent: Applebot-Extended   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: archive.org_bot\nAllow: /\n\nUser-agent: Baiduspider\nAllow: /\n\nUser-agent: bingbot\nAllow: /\n\nUser-agent: Bytespider   # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: Claude-SearchBot\nAllow: /\n\nUser-agent: Claude-User\nAllow: /\n\nUser-agent: Claude-Web   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: cohere-ai\nAllow: /\n\nUser-agent: cohere-training-data-crawler\nAllow: /\n\nUser-agent: Diffbot\nAllow: /\n\nUser-agent: DuckAssistBot\nAllow: /\n\nUser-agent: DuckDuckBot\nAllow: /\n\nUser-agent: FacebookBot\nAllow: /\n\nUser-agent: facebookexternalhit\nAllow: /\n\nUser-agent: FirecrawlAgent\nAllow: /\n\nUser-agent: Google-CloudVertexBot\nAllow: /\n\nUser-agent: Google-Extended   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: Google-InspectionTool\nAllow: /\n\nUser-agent: Googlebot\nAllow: /\n\nUser-agent: Googlebot-Image\nAllow: /\n\nUser-agent: Googlebot-News\nAllow: /\n\nUser-agent: GoogleOther\nAllow: /\n\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ia_archiver\nAllow: /\n\nUser-agent: ImagesiftBot\nAllow: /\n\nUser-agent: img2dataset\nAllow: /\n\nUser-agent: meta-externalagent\nAllow: /\n\nUser-agent: meta-externalfetcher\nAllow: /\n\nUser-agent: MistralAI-User\nAllow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: omgili\nAllow: /\n\nUser-agent: omgilibot\nAllow: /\n\nUser-agent: Perplexity-User   # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: PetalBot\nAllow: /\n\nUser-agent: Scrapy\nAllow: /\n\nUser-agent: SemrushBot-OCOB\nAllow: /\n\nUser-agent: SeznamBot\nAllow: /\n\nUser-agent: Storebot-Google\nAllow: /\n\nUser-agent: TikTokSpider   # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: Timpibot\nAllow: /\n\nUser-agent: Webzio-Extended\nAllow: /\n\nUser-agent: YandexBot\nAllow: /\n\nUser-agent: Yeti\nAllow: /\n\nUser-agent: YouBot\nAllow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n"
  }
 ],
 "note": "Presence in this table means the operator documents that crawler. A user-agent is a claim; nothing here is proof of identity."
}