{
 "name": "AI Crawler Index",
 "description": "Every AI crawler on the web, what it is for, what blocking it costs you, and the IP ranges its operator publishes — as JSON, CSV, robots.txt and regex.",
 "url": "https://www.pathwren.workers.dev",
 "generated_at": "2026-09-01T00:39:07+00:00",
 "version": "2026-09-01",
 "license": "CC0-1.0",
 "source_note": "Robots tokens, user-agent strings and documentation URLs are taken from each operator's published documentation, cited per record in operator_docs. The category, the prose in what_it_is and the cost_of_blocking assessment are this index's own.",
 "count": 56,
 "categories": {
  "ai-training": {
   "label": "AI training crawlers",
   "description": "Collect pages in bulk so that a model can be trained or fine-tuned on them. Blocking these removes you from future training sets and changes nothing a user sees today."
  },
  "ai-search": {
   "label": "AI search crawlers",
   "description": "Build the retrieval index an assistant answers and cites from. These are the crawlers that send you traffic; blocking them is the expensive mistake in this space."
  },
  "user-fetch": {
   "label": "User-triggered fetchers",
   "description": "Fetch one page because a person asked for it, right then. One human intent, one request. Blocking them produces a visible error for a real reader."
  },
  "dataset": {
   "label": "Corpus and dataset builders",
   "description": "Crawl the web into a published or resold dataset that other people train on. Highest leverage per block, longest delay before any effect."
  },
  "search": {
   "label": "Search engines",
   "description": "Classic index-and-rank crawlers. Several also feed their operator's generative answers, which is why the AI opt-out for Google and Apple is a token rather than a block."
  },
  "seo": {
   "label": "SEO and backlink crawlers",
   "description": "Commercial link-graph tooling. No user-facing effect either way, and usually a large share of your bot bandwidth."
  },
  "archive": {
   "label": "Archivers",
   "description": "Preservation crawlers. Their output is public and permanent, which makes them a separate decision from the AI one."
  },
  "preview": {
   "label": "Link preview fetchers",
   "description": "Read your Open Graph tags when someone shares a link. Blocking these is almost always an accident."
  },
  "tool": {
   "label": "Tools and frameworks",
   "description": "Not operators: crawling software anyone can run. The party behind the request is unknown, so treat them as a rate-limit question rather than a consent question."
  }
 },
 "endpoints": {
  "agents_json": "https://www.pathwren.workers.dev/data/agents.json",
  "agents_csv": "https://www.pathwren.workers.dev/data/agents.csv",
  "user_agents_txt": "https://www.pathwren.workers.dev/data/user-agents.txt",
  "robots_tokens_txt": "https://www.pathwren.workers.dev/data/robots-tokens.txt",
  "ua_regex_json": "https://www.pathwren.workers.dev/data/ua-regex.json",
  "ip_ranges_json": "https://www.pathwren.workers.dev/ip-ranges/all.json",
  "ip_ranges_txt": "https://www.pathwren.workers.dev/ip-ranges/all.txt",
  "status_json": "https://www.pathwren.workers.dev/status.json",
  "openapi": "https://www.pathwren.workers.dev/openapi.json"
 },
 "crawlers": [
  {
   "slug": "gptbot",
   "name": "GPTBot",
   "operator": "OpenAI",
   "operator_slug": "openai",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "GPTBot",
   "user_agent_substring": "GPTBot",
   "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2; +https://openai.com/gptbot",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://openai.com/gptbot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-gptbot.json",
   "ipv4_prefix_count": 21,
   "ipv6_prefix_count": 0,
   "what_it_is": "OpenAI's bulk crawler. Pages it fetches may be used to train future OpenAI foundation models. It is not the bot that puts you in ChatGPT's search results, and blocking it does not remove you from them.",
   "cost_of_blocking": "Your content is excluded from training data for future OpenAI models. No effect on ChatGPT search visibility, on citations, or on links a user pastes into ChatGPT.",
   "operator_docs": "https://platform.openai.com/docs/bots",
   "html_url": "https://www.pathwren.workers.dev/crawler/gptbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/gptbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "oai-searchbot",
   "name": "OAI-SearchBot",
   "operator": "OpenAI",
   "operator_slug": "openai",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "robots_token": "OAI-SearchBot",
   "user_agent_substring": "OAI-SearchBot",
   "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://openai.com/searchbot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-searchbot.json",
   "ipv4_prefix_count": 35,
   "ipv6_prefix_count": 0,
   "what_it_is": "Builds the index ChatGPT search answers from. Content it collects is used for retrieval and citation, not for model training.",
   "cost_of_blocking": "High. Blocking this removes you from ChatGPT search results and from the source links ChatGPT shows. This is the single most expensive block on this list for anyone who wants to be cited by an assistant.",
   "operator_docs": "https://platform.openai.com/docs/bots",
   "html_url": "https://www.pathwren.workers.dev/crawler/oai-searchbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/oai-searchbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "chatgpt-user",
   "name": "ChatGPT-User",
   "operator": "OpenAI",
   "operator_slug": "openai",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "ChatGPT-User",
   "user_agent_substring": "ChatGPT-User",
   "user_agent_example": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; ChatGPT-User/1.0; +https://openai.com/bot",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://openai.com/chatgpt-user.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-chatgpt-user.json",
   "ipv4_prefix_count": 204,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a single page at the moment a user or a ChatGPT agent asks for it — a pasted link, a browsing step, an Operator task. One human intent, one request. OpenAI states these fetches are not used for training.",
   "cost_of_blocking": "ChatGPT cannot open your pages when a user explicitly asks it to. The user sees a fetch failure. This is usually the last bot anyone means to block.",
   "operator_docs": "https://platform.openai.com/docs/bots",
   "html_url": "https://www.pathwren.workers.dev/crawler/chatgpt-user.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/chatgpt-user.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "claudebot",
   "name": "ClaudeBot",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "ClaudeBot",
   "user_agent_substring": "ClaudeBot",
   "user_agent_example": "Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Anthropic's bulk crawler, gathering pages that may be used to train Claude models.",
   "cost_of_blocking": "Content excluded from training data for future Claude models. No effect on Claude's ability to fetch a link a user gives it.",
   "operator_docs": "https://support.anthropic.com/en/articles/8896518",
   "html_url": "https://www.pathwren.workers.dev/crawler/claudebot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/claudebot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "claude-searchbot",
   "name": "Claude-SearchBot",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "robots_token": "Claude-SearchBot",
   "user_agent_substring": "Claude-SearchBot",
   "user_agent_example": "Mozilla/5.0 (compatible; Claude-SearchBot/1.0; +Claude-SearchBot@anthropic.com)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Indexes pages so Claude's web search can find and cite them. Separate token from the training crawler, so search visibility and training consent are independent decisions.",
   "cost_of_blocking": "You stop appearing in Claude's search results and citations.",
   "operator_docs": "https://support.anthropic.com/en/articles/8896518",
   "html_url": "https://www.pathwren.workers.dev/crawler/claude-searchbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/claude-searchbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "claude-user",
   "name": "Claude-User",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "Claude-User",
   "user_agent_substring": "Claude-User",
   "user_agent_example": "Mozilla/5.0 (compatible; Claude-User/1.0; +Claude-User@anthropic.com)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a page because a Claude user asked Claude to read it, at that moment.",
   "cost_of_blocking": "Claude reports a fetch failure to a user who asked for your page by name.",
   "operator_docs": "https://support.anthropic.com/en/articles/8896518",
   "html_url": "https://www.pathwren.workers.dev/crawler/claude-user.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/claude-user.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "anthropic-ai",
   "name": "anthropic-ai",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "anthropic-ai",
   "user_agent_substring": "anthropic-ai",
   "user_agent_example": "(no live crawler currently identifies with this string)",
   "respects_robots_txt": "n-a",
   "respects_robots_txt_label": "control token only — no crawler",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "A legacy robots.txt token from before Anthropic consolidated on ClaudeBot. It is still widely present in robots.txt files and costs nothing to keep, but it is a control token rather than a bot you will see in logs.",
   "cost_of_blocking": "None. Nothing crawls under this name today; keeping the rule is harmless insurance.",
   "operator_docs": "https://support.anthropic.com/en/articles/8896518",
   "html_url": "https://www.pathwren.workers.dev/crawler/anthropic-ai.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/anthropic-ai.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "claude-web",
   "name": "Claude-Web",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "robots_token": "Claude-Web",
   "user_agent_substring": "Claude-Web",
   "user_agent_example": "Mozilla/5.0 (compatible; Claude-Web/1.0)",
   "respects_robots_txt": "n-a",
   "respects_robots_txt_label": "control token only — no crawler",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "An earlier Anthropic token for user-facing web access, superseded by Claude-User and Claude-SearchBot. Kept here because it appears in most published robots.txt templates.",
   "cost_of_blocking": "None in practice. Retain the rule; expect no traffic.",
   "operator_docs": "https://support.anthropic.com/en/articles/8896518",
   "html_url": "https://www.pathwren.workers.dev/crawler/claude-web.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/claude-web.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "google-extended",
   "name": "Google-Extended",
   "operator": "Google",
   "operator_slug": "google",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "Google-Extended",
   "user_agent_substring": "(control token only — no crawler)",
   "user_agent_example": "(none: Google-Extended never appears as a user-agent)",
   "respects_robots_txt": "n-a",
   "respects_robots_txt_label": "control token only — no crawler",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Not a crawler. A robots.txt token that tells Google whether pages Googlebot already fetched may be used to train and ground Gemini. You will never see it in an access log; disallowing it changes what Google does with content it fetched under a different name.",
   "cost_of_blocking": "You are excluded from Gemini grounding and Gemini training. Google Search ranking and indexing are explicitly unaffected. This is the cleanest 'no training, keep my search traffic' lever that exists.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-extended.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-extended.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "googlebot",
   "name": "Googlebot",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Googlebot",
   "user_agent_substring": "Googlebot",
   "user_agent_example": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json",
   "ipv4_prefix_count": 169,
   "ipv6_prefix_count": 146,
   "what_it_is": "The classic search crawler. It is also the crawler behind AI Overviews: Google does not run a separate bot for them, which is why the only AI opt-out is the Google-Extended token and not a Googlebot block.",
   "cost_of_blocking": "Total. You leave Google Search. Never block this to avoid AI use; use Google-Extended instead.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googlebot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googlebot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "googleother",
   "name": "GoogleOther",
   "operator": "Google",
   "operator_slug": "google",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "GoogleOther",
   "user_agent_substring": "GoogleOther",
   "user_agent_example": "Mozilla/5.0 (compatible; GoogleOther)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 135,
   "ipv6_prefix_count": 135,
   "what_it_is": "A generic fetcher used by Google product teams for one-off crawls and research, including data collection that does not belong to Search.",
   "cost_of_blocking": "No effect on Search indexing. Blocks internal Google research and product fetches.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googleother.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googleother.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "google-cloudvertexbot",
   "name": "Google-CloudVertexBot",
   "operator": "Google",
   "operator_slug": "google",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "robots_token": "Google-CloudVertexBot",
   "user_agent_substring": "Google-CloudVertexBot",
   "user_agent_example": "Mozilla/5.0 (compatible; Google-CloudVertexBot/1.0; +https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 135,
   "ipv6_prefix_count": 135,
   "what_it_is": "Crawls a site on behalf of a Vertex AI Agent Builder customer who is building an agent over that site. It only visits sites the customer has asked it to.",
   "cost_of_blocking": "Third parties can no longer build Vertex AI agents that read your site. Irrelevant to Google Search.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-cloudvertexbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-cloudvertexbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "google-inspectiontool",
   "name": "Google-InspectionTool",
   "operator": "Google",
   "operator_slug": "google",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "Google-InspectionTool",
   "user_agent_substring": "Google-InspectionTool",
   "user_agent_example": "Mozilla/5.0 (compatible; Google-InspectionTool/1.0;)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 135,
   "ipv6_prefix_count": 135,
   "what_it_is": "The fetcher behind Search Console's URL Inspection and the Rich Results Test. It runs when a site owner clicks a button.",
   "cost_of_blocking": "Your own Search Console live tests stop working. Blocking this only hurts you.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-inspectiontool.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-inspectiontool.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "googlebot-image",
   "name": "Googlebot-Image",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Googlebot-Image",
   "user_agent_substring": "Googlebot-Image",
   "user_agent_example": "Googlebot-Image/1.0",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json",
   "ipv4_prefix_count": 169,
   "ipv6_prefix_count": 146,
   "what_it_is": "Image indexing for Google Images. A separate token so you can leave images out of search without leaving search.",
   "cost_of_blocking": "Your images stop appearing in Google Images.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googlebot-image.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googlebot-image.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "googlebot-news",
   "name": "Googlebot-News",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Googlebot-News",
   "user_agent_substring": "Googlebot-News",
   "user_agent_example": "(uses the Googlebot user-agent; controlled by the Googlebot-News robots token)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json",
   "ipv4_prefix_count": 169,
   "ipv6_prefix_count": 146,
   "what_it_is": "A robots.txt token controlling inclusion in Google News. It does not have its own user-agent string; the fetch arrives as Googlebot.",
   "cost_of_blocking": "Removal from Google News, with normal Search unaffected.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googlebot-news.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googlebot-news.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "storebot-google",
   "name": "Storebot-Google",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Storebot-Google",
   "user_agent_substring": "Storebot-Google",
   "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36 (compatible; Storebot-Google/1.0; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 135,
   "ipv6_prefix_count": 135,
   "what_it_is": "Checks shopping and checkout flows for Google's shopping surfaces.",
   "cost_of_blocking": "Product listings may lose shopping-specific enrichment. Irrelevant to non-commerce sites.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/storebot-google.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/storebot-google.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "bingbot",
   "name": "bingbot",
   "operator": "Microsoft",
   "operator_slug": "microsoft",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "bingbot",
   "user_agent_substring": "bingbot",
   "user_agent_example": "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://www.bing.com/toolbox/bingbot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/bing-bingbot.json",
   "ipv4_prefix_count": 28,
   "ipv6_prefix_count": 0,
   "what_it_is": "Bing's only crawler, and therefore also the crawler behind Microsoft Copilot's grounding. Microsoft's documented way to keep search indexing while refusing generative reuse is the nocache / noarchive robots meta directive, not a separate user-agent.",
   "cost_of_blocking": "Very high and very wide: Bing, Copilot, DuckDuckGo and several assistants that resell Bing's index all lose you at once. Use nocache/noarchive rather than blocking.",
   "operator_docs": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
   "html_url": "https://www.pathwren.workers.dev/crawler/bingbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/bingbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "applebot",
   "name": "Applebot",
   "operator": "Apple",
   "operator_slug": "apple",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Applebot",
   "user_agent_substring": "Applebot",
   "user_agent_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://search.developer.apple.com/applebot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/apple-applebot.json",
   "ipv4_prefix_count": 33,
   "ipv6_prefix_count": 0,
   "what_it_is": "Powers Siri, Spotlight and Safari suggestions. Blocking it is a search decision, not an AI decision — the AI decision has its own token.",
   "cost_of_blocking": "You disappear from Siri, Spotlight and Safari search suggestions across Apple's install base.",
   "operator_docs": "https://support.apple.com/en-us/119829",
   "html_url": "https://www.pathwren.workers.dev/crawler/applebot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/applebot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "applebot-extended",
   "name": "Applebot-Extended",
   "operator": "Apple",
   "operator_slug": "apple",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "Applebot-Extended",
   "user_agent_substring": "(control token only — no crawler)",
   "user_agent_example": "(none: Applebot-Extended never appears as a user-agent)",
   "respects_robots_txt": "n-a",
   "respects_robots_txt_label": "control token only — no crawler",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Apple's counterpart to Google-Extended: a robots.txt token that withdraws consent for Apple Intelligence and Apple foundation-model training, without touching Applebot's search crawl.",
   "cost_of_blocking": "Excluded from Apple Intelligence training. Siri, Spotlight and Safari suggestions are unaffected.",
   "operator_docs": "https://support.apple.com/en-us/119829",
   "html_url": "https://www.pathwren.workers.dev/crawler/applebot-extended.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/applebot-extended.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "perplexitybot",
   "name": "PerplexityBot",
   "operator": "Perplexity",
   "operator_slug": "perplexity",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "robots_token": "PerplexityBot",
   "user_agent_substring": "PerplexityBot",
   "user_agent_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://www.perplexity.ai/perplexitybot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/perplexity-bot.json",
   "ipv4_prefix_count": 8,
   "ipv6_prefix_count": 0,
   "what_it_is": "Builds Perplexity's search index. Perplexity is citation-heavy by product design, so inclusion here converts to referral traffic more directly than most AI surfaces.",
   "cost_of_blocking": "You stop being indexed and cited by Perplexity, and lose the referral clicks its citations produce.",
   "operator_docs": "https://docs.perplexity.ai/guides/bots",
   "html_url": "https://www.pathwren.workers.dev/crawler/perplexitybot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/perplexitybot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "perplexity-user",
   "name": "Perplexity-User",
   "operator": "Perplexity",
   "operator_slug": "perplexity",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "Perplexity-User",
   "user_agent_substring": "Perplexity-User",
   "user_agent_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://www.perplexity.ai/perplexity-user.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/perplexity-user.json",
   "ipv4_prefix_count": 4,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a page because a Perplexity user asked for it. Perplexity documents that this fetch is user-initiated and is therefore not governed by robots.txt — a robots rule will not stop it, by stated policy.",
   "cost_of_blocking": "Not controllable via robots.txt. If you must stop it, verify by the published IP ranges and block at the edge — and accept that users who ask for your page get an error.",
   "operator_docs": "https://docs.perplexity.ai/guides/bots",
   "html_url": "https://www.pathwren.workers.dev/crawler/perplexity-user.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/perplexity-user.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "ccbot",
   "name": "CCBot",
   "operator": "Common Crawl",
   "operator_slug": "commoncrawl",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "robots_token": "CCBot",
   "user_agent_substring": "CCBot",
   "user_agent_example": "CCBot/2.0 (https://commoncrawl.org/faq/)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Common Crawl's corpus builder. It trains nothing itself, but its archive is an input to most open and many closed LLM training sets, which makes it the highest-leverage single entry on this list.",
   "cost_of_blocking": "Future Common Crawl snapshots exclude you, so downstream training sets lose you too — but only going forward. Existing snapshots are permanent and blocking today does not retract them.",
   "operator_docs": "https://commoncrawl.org/faq",
   "html_url": "https://www.pathwren.workers.dev/crawler/ccbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/ccbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "bytespider",
   "name": "Bytespider",
   "operator": "ByteDance",
   "operator_slug": "bytedance",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "Bytespider",
   "user_agent_substring": "Bytespider",
   "user_agent_example": "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)",
   "respects_robots_txt": "disputed",
   "respects_robots_txt_label": "compliance disputed",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "ByteDance's crawler, associated with training data collection for Doubao and related models. Repeatedly reported by CDNs and site operators as the highest-volume AI crawler on the web and as inconsistent about robots.txt.",
   "cost_of_blocking": "Little to lose. If you want it gone, expect to block by user-agent at the edge rather than to ask politely in robots.txt.",
   "operator_docs": "https://www.bytespider.net/",
   "html_url": "https://www.pathwren.workers.dev/crawler/bytespider.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/bytespider.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "tiktokspider",
   "name": "TikTokSpider",
   "operator": "ByteDance",
   "operator_slug": "bytedance",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "TikTokSpider",
   "user_agent_substring": "TikTokSpider",
   "user_agent_example": "Mozilla/5.0 (compatible; TikTokSpider; ttspider-feedback@tiktok.com)",
   "respects_robots_txt": "disputed",
   "respects_robots_txt_label": "compliance disputed",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "A second ByteDance crawler identifying with TikTok, collecting page content for the same family of models.",
   "cost_of_blocking": "Little to lose unless TikTok search referral matters to you.",
   "operator_docs": "https://www.bytespider.net/",
   "html_url": "https://www.pathwren.workers.dev/crawler/tiktokspider.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/tiktokspider.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "meta-externalagent",
   "name": "meta-externalagent",
   "operator": "Meta",
   "operator_slug": "meta",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "meta-externalagent",
   "user_agent_substring": "meta-externalagent",
   "user_agent_example": "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Meta's AI crawler, gathering training data for Llama and Meta AI. It replaced the older FacebookBot name for this purpose.",
   "cost_of_blocking": "Excluded from Meta AI training. Link previews on Facebook, Instagram and WhatsApp are unaffected — those are a different bot.",
   "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/meta-externalagent.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/meta-externalagent.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "meta-externalfetcher",
   "name": "meta-externalfetcher",
   "operator": "Meta",
   "operator_slug": "meta",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "meta-externalfetcher",
   "user_agent_substring": "meta-externalfetcher",
   "user_agent_example": "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a page when a Meta AI user asks about a specific link.",
   "cost_of_blocking": "Meta AI cannot read pages users hand it.",
   "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "facebookexternalhit",
   "name": "facebookexternalhit",
   "operator": "Meta",
   "operator_slug": "meta",
   "category": "preview",
   "category_label": "Link preview fetchers",
   "robots_token": "facebookexternalhit",
   "user_agent_substring": "facebookexternalhit",
   "user_agent_example": "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The link unfurler: it reads your Open Graph tags when somebody shares your URL on a Meta property.",
   "cost_of_blocking": "Severe and usually accidental. Your links share as bare grey boxes with no title, image or description across Facebook, Instagram, Messenger and WhatsApp. Almost nobody means to block this.",
   "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/facebookexternalhit.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/facebookexternalhit.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "facebookbot",
   "name": "FacebookBot",
   "operator": "Meta",
   "operator_slug": "meta",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "FacebookBot",
   "user_agent_substring": "FacebookBot",
   "user_agent_example": "FacebookBot/1.0 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Meta's older speech- and language-corpus crawler, largely superseded by meta-externalagent but still listed as a valid robots token.",
   "cost_of_blocking": "Negligible today. Keep the rule; expect little traffic.",
   "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/facebookbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/facebookbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "amazonbot",
   "name": "Amazonbot",
   "operator": "Amazon",
   "operator_slug": "amazon",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "robots_token": "Amazonbot",
   "user_agent_substring": "Amazonbot",
   "user_agent_example": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Amazon's crawler, feeding Alexa's ability to answer questions from the web and Amazon's own search and assistant products.",
   "cost_of_blocking": "Alexa and Amazon's assistants stop answering from your pages. Verify with reverse DNS to crawl.amazonbot.amazon before trusting the user-agent.",
   "operator_docs": "https://developer.amazon.com/amazonbot",
   "html_url": "https://www.pathwren.workers.dev/crawler/amazonbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/amazonbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "duckassistbot",
   "name": "DuckAssistBot",
   "operator": "DuckDuckGo",
   "operator_slug": "duckduckgo",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "robots_token": "DuckAssistBot",
   "user_agent_substring": "DuckAssistBot",
   "user_agent_example": "Mozilla/5.0 (compatible; DuckAssistBot/1.0; +https://duckduckgo.com/duckassistbot)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches pages so DuckAssist can generate and cite answers inside DuckDuckGo.",
   "cost_of_blocking": "No DuckAssist answers or citations from your site. Ordinary DuckDuckGo results are unaffected.",
   "operator_docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/duckassistbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/duckassistbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "duckduckbot",
   "name": "DuckDuckBot",
   "operator": "DuckDuckGo",
   "operator_slug": "duckduckgo",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "DuckDuckBot",
   "user_agent_substring": "DuckDuckBot",
   "user_agent_example": "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://duckduckgo.com/duckduckbot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/duckduckgo-duckduckbot.json",
   "ipv4_prefix_count": 481,
   "ipv6_prefix_count": 0,
   "what_it_is": "DuckDuckGo's own crawler. Note that the bulk of DuckDuckGo's web results come from Bing, so blocking bingbot removes you from DuckDuckGo whether or not you allow this one.",
   "cost_of_blocking": "Limited on its own; the real DuckDuckGo lever is bingbot.",
   "operator_docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/duckduckbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/duckduckbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "ai2bot",
   "name": "AI2Bot",
   "operator": "Allen Institute for AI",
   "operator_slug": "ai2",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "robots_token": "AI2Bot",
   "user_agent_substring": "AI2Bot",
   "user_agent_example": "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The Allen Institute's crawler, gathering pages for open research corpora such as Dolma that underpin fully open models like OLMo.",
   "cost_of_blocking": "Excluded from open research datasets. Worth a deliberate decision: this is the category where 'blocking AI' also blocks the open, auditable end of it.",
   "operator_docs": "https://allenai.org/crawler",
   "html_url": "https://www.pathwren.workers.dev/crawler/ai2bot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/ai2bot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "ai2bot-dolma",
   "name": "Ai2Bot-Dolma",
   "operator": "Allen Institute for AI",
   "operator_slug": "ai2",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "robots_token": "Ai2Bot-Dolma",
   "user_agent_substring": "Ai2Bot-Dolma",
   "user_agent_example": "Mozilla/5.0 (compatible) Ai2Bot-Dolma (+https://www.allenai.org/crawler)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The variant of AI2's crawler named for the Dolma corpus specifically.",
   "cost_of_blocking": "Same as AI2Bot: exclusion from an open, published training corpus.",
   "operator_docs": "https://allenai.org/crawler",
   "html_url": "https://www.pathwren.workers.dev/crawler/ai2bot-dolma.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/ai2bot-dolma.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "cohere-ai",
   "name": "cohere-ai",
   "operator": "Cohere",
   "operator_slug": "cohere",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "cohere-ai",
   "user_agent_substring": "cohere-ai",
   "user_agent_example": "Mozilla/5.0 (compatible; cohere-ai)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Cohere's fetcher, used when its assistant products need a page.",
   "cost_of_blocking": "Cohere-powered assistants cannot read your pages on request.",
   "operator_docs": "https://cohere.com/",
   "html_url": "https://www.pathwren.workers.dev/crawler/cohere-ai.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/cohere-ai.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "cohere-training-data-crawler",
   "name": "cohere-training-data-crawler",
   "operator": "Cohere",
   "operator_slug": "cohere",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "cohere-training-data-crawler",
   "user_agent_substring": "cohere-training-data-crawler",
   "user_agent_example": "Mozilla/5.0 (compatible; cohere-training-data-crawler)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Cohere's separately-named bulk crawler for model training data, split out so consent for training and consent for retrieval can differ.",
   "cost_of_blocking": "Excluded from Cohere model training.",
   "operator_docs": "https://cohere.com/",
   "html_url": "https://www.pathwren.workers.dev/crawler/cohere-training-data-crawler.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/cohere-training-data-crawler.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "mistralai-user",
   "name": "MistralAI-User",
   "operator": "Mistral AI",
   "operator_slug": "mistral",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "MistralAI-User",
   "user_agent_substring": "MistralAI-User",
   "user_agent_example": "Mozilla/5.0 (compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a page when a Le Chat user asks Mistral's assistant to read it.",
   "cost_of_blocking": "Le Chat cannot open links your readers give it.",
   "operator_docs": "https://docs.mistral.ai/",
   "html_url": "https://www.pathwren.workers.dev/crawler/mistralai-user.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/mistralai-user.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "youbot",
   "name": "YouBot",
   "operator": "You.com",
   "operator_slug": "you",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "robots_token": "YouBot",
   "user_agent_substring": "YouBot",
   "user_agent_example": "Mozilla/5.0 (compatible; YouBot (+http://www.you.com))",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "You.com's crawler, feeding its AI search product and its search API.",
   "cost_of_blocking": "Removal from You.com's index and from answers built on its API.",
   "operator_docs": "https://about.you.com/youbot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/youbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/youbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "diffbot",
   "name": "Diffbot",
   "operator": "Diffbot",
   "operator_slug": "diffbot",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "robots_token": "Diffbot",
   "user_agent_substring": "Diffbot",
   "user_agent_example": "Mozilla/5.0 (compatible; Diffbot/0.1; +http://www.diffbot.com)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Extracts structured records from pages to build a commercial knowledge graph that is resold and used for retrieval and training.",
   "cost_of_blocking": "Your facts stop entering a widely-licensed knowledge graph. Whether that is a loss depends on whether you want to be a machine-readable entity.",
   "operator_docs": "https://docs.diffbot.com/",
   "html_url": "https://www.pathwren.workers.dev/crawler/diffbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/diffbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "omgilibot",
   "name": "omgilibot",
   "operator": "Webz.io",
   "operator_slug": "webz",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "robots_token": "omgilibot",
   "user_agent_substring": "omgilibot",
   "user_agent_example": "omgilibot/0.4; +http://omgili.com",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Webz.io's crawler, collecting web and forum text sold as datasets, including to model builders.",
   "cost_of_blocking": "Exclusion from a commercial dataset resold to third parties.",
   "operator_docs": "https://webz.io/blog/machine-learning/",
   "html_url": "https://www.pathwren.workers.dev/crawler/omgilibot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/omgilibot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "omgili",
   "name": "omgili",
   "operator": "Webz.io",
   "operator_slug": "webz",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "robots_token": "omgili",
   "user_agent_substring": "omgili",
   "user_agent_example": "omgili/0.5 +http://omgili.com",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The older robots token for the same Webz.io collection, still honoured and still worth listing.",
   "cost_of_blocking": "Same as omgilibot.",
   "operator_docs": "https://webz.io/blog/machine-learning/",
   "html_url": "https://www.pathwren.workers.dev/crawler/omgili.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/omgili.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "webzio-extended",
   "name": "Webzio-Extended",
   "operator": "Webz.io",
   "operator_slug": "webz",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "Webzio-Extended",
   "user_agent_substring": "Webzio-Extended",
   "user_agent_example": "Mozilla/5.0 (compatible; Webzio-Extended/1.0)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Webz.io's opt-out token specifically for AI training reuse, in the pattern Google and Apple established.",
   "cost_of_blocking": "Your content is excluded from the AI-training tier of Webz.io's product while ordinary collection continues.",
   "operator_docs": "https://webz.io/blog/machine-learning/",
   "html_url": "https://www.pathwren.workers.dev/crawler/webzio-extended.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/webzio-extended.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "imagesiftbot",
   "name": "ImagesiftBot",
   "operator": "Hive AI",
   "operator_slug": "hive",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "robots_token": "ImagesiftBot",
   "user_agent_substring": "ImagesiftBot",
   "user_agent_example": "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Crawls images for Hive AI's reverse-image and dataset products. Image-heavy sites see this one long before they see the text crawlers.",
   "cost_of_blocking": "Your images stop entering an image dataset and reverse-image index.",
   "operator_docs": "https://imagesift.com/about",
   "html_url": "https://www.pathwren.workers.dev/crawler/imagesiftbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/imagesiftbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "timpibot",
   "name": "Timpibot",
   "operator": "Timpi",
   "operator_slug": "timpi",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Timpibot",
   "user_agent_substring": "Timpibot",
   "user_agent_example": "Mozilla/5.0 (compatible; Timpibot/0.1; +https://timpi.io)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "A distributed crawler building an independent search index outside the Google/Bing duopoly.",
   "cost_of_blocking": "Absence from a small independent index.",
   "operator_docs": "https://timpi.io/",
   "html_url": "https://www.pathwren.workers.dev/crawler/timpibot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/timpibot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "semrushbot",
   "name": "SemrushBot",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "SemrushBot",
   "user_agent_substring": "SemrushBot",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Semrush's backlink and keyword crawler. It is not an AI crawler, but it is usually in the top three by volume on any site, and it is the cheapest block on this list.",
   "cost_of_blocking": "Your competitors' Semrush reports get thinner, and so do yours. No user-facing effect.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "semrushbot-ocob",
   "name": "SemrushBot-OCOB",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "SemrushBot-OCOB",
   "user_agent_substring": "SemrushBot-OCOB",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-OCOB/1.0; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Semrush's separately-tokenised crawler for its AI content tooling, split out so SEO crawling and AI reuse can be answered differently.",
   "cost_of_blocking": "Exclusion from Semrush's AI corpus, with its SEO crawl unaffected.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ocob.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ocob.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "ahrefsbot",
   "name": "AhrefsBot",
   "operator": "Ahrefs",
   "operator_slug": "ahrefs",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "AhrefsBot",
   "user_agent_substring": "AhrefsBot",
   "user_agent_example": "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Ahrefs' backlink crawler, and one of the largest non-search crawlers on the web by request volume.",
   "cost_of_blocking": "No user-facing effect. Ahrefs honours Crawl-delay, so rate-limiting is usually better than blocking.",
   "operator_docs": "https://ahrefs.com/robot",
   "html_url": "https://www.pathwren.workers.dev/crawler/ahrefsbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/ahrefsbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "archive-org-bot",
   "name": "archive.org_bot",
   "operator": "Internet Archive",
   "operator_slug": "internetarchive",
   "category": "archive",
   "category_label": "Archivers",
   "robots_token": "archive.org_bot",
   "user_agent_substring": "archive.org_bot",
   "user_agent_example": "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The Wayback Machine's crawler. Preservation rather than AI, but it lands in the same 'is this bot welcome' decision and its output is a public corpus.",
   "cost_of_blocking": "Your site stops being preserved. When it dies, it is gone. Consider this one separately from the AI question.",
   "operator_docs": "https://archive.org/details/archive.org_bot",
   "html_url": "https://www.pathwren.workers.dev/crawler/archive-org-bot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/archive-org-bot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "ia-archiver",
   "name": "ia_archiver",
   "operator": "Internet Archive",
   "operator_slug": "internetarchive",
   "category": "archive",
   "category_label": "Archivers",
   "robots_token": "ia_archiver",
   "user_agent_substring": "ia_archiver",
   "user_agent_example": "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The legacy Alexa/Internet Archive token, still present in most robots.txt files and still occasionally honoured.",
   "cost_of_blocking": "Negligible today; retain for tidiness.",
   "operator_docs": "https://archive.org/details/archive.org_bot",
   "html_url": "https://www.pathwren.workers.dev/crawler/ia-archiver.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/ia-archiver.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "yandexbot",
   "name": "YandexBot",
   "operator": "Yandex",
   "operator_slug": "yandex",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "YandexBot",
   "user_agent_substring": "YandexBot",
   "user_agent_example": "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Yandex's search crawler, which also feeds Alice and Yandex's generative answers.",
   "cost_of_blocking": "Removal from Yandex Search. Verify with reverse DNS to a yandex.ru, yandex.net or yandex.com host — YandexBot is among the most-spoofed user-agents there is.",
   "operator_docs": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html",
   "html_url": "https://www.pathwren.workers.dev/crawler/yandexbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/yandexbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "baiduspider",
   "name": "Baiduspider",
   "operator": "Baidu",
   "operator_slug": "baidu",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Baiduspider",
   "user_agent_substring": "Baiduspider",
   "user_agent_example": "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Baidu's search crawler, and the ingest path for Baidu's Ernie-backed answers.",
   "cost_of_blocking": "Removal from Baidu Search, which matters only if you want Chinese-language traffic.",
   "operator_docs": "https://help.baidu.com/question?prod_id=99&class=0&id=3001",
   "html_url": "https://www.pathwren.workers.dev/crawler/baiduspider.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/baiduspider.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "seznambot",
   "name": "SeznamBot",
   "operator": "Seznam",
   "operator_slug": "seznam",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "SeznamBot",
   "user_agent_substring": "SeznamBot",
   "user_agent_example": "Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/en/seznambot-intro/)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Seznam's crawler — the dominant search engine in the Czech Republic and one of the few national engines with its own index.",
   "cost_of_blocking": "Removal from Seznam. Also removes you from its IndexNow endpoint's usefulness.",
   "operator_docs": "https://napoveda.seznam.cz/en/seznamzbozi/subject-matter-crawler/",
   "html_url": "https://www.pathwren.workers.dev/crawler/seznambot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/seznambot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "yeti",
   "name": "Yeti",
   "operator": "Naver",
   "operator_slug": "naver",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Yeti",
   "user_agent_substring": "Yeti",
   "user_agent_example": "Mozilla/5.0 (compatible; Yeti/1.1; +https://naver.me/spd)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Naver's crawler. Naver is South Korea's largest search portal and runs its own index and its own generative answers.",
   "cost_of_blocking": "Removal from Naver, which is most of Korean search.",
   "operator_docs": "https://searchadvisor.naver.com/guide/seo-basic-crawl",
   "html_url": "https://www.pathwren.workers.dev/crawler/yeti.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/yeti.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "petalbot",
   "name": "PetalBot",
   "operator": "Huawei",
   "operator_slug": "huawei",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "PetalBot",
   "user_agent_substring": "PetalBot",
   "user_agent_example": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Huawei's crawler for Petal Search, shipped as the default search on Huawei devices.",
   "cost_of_blocking": "Removal from Petal Search. Frequently blocked for volume rather than for policy.",
   "operator_docs": "https://aspiegel.com/petalbot",
   "html_url": "https://www.pathwren.workers.dev/crawler/petalbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/petalbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "firecrawlagent",
   "name": "FirecrawlAgent",
   "operator": "Firecrawl",
   "operator_slug": "firecrawl",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "FirecrawlAgent",
   "user_agent_substring": "FirecrawlAgent",
   "user_agent_example": "Mozilla/5.0 (compatible; FirecrawlAgent/1.0; +https://firecrawl.dev)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "A hosted scrape-to-markdown service that LLM applications call to read pages. The requester is whoever is building on it, not Firecrawl itself, so volume and intent vary wildly.",
   "cost_of_blocking": "Applications built on Firecrawl cannot read your pages. This is increasingly how agents fetch the web, so it is a bigger block than its name suggests.",
   "operator_docs": "https://docs.firecrawl.dev/",
   "html_url": "https://www.pathwren.workers.dev/crawler/firecrawlagent.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/firecrawlagent.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "scrapy",
   "name": "Scrapy",
   "operator": "Scrapy project",
   "operator_slug": "scrapy",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "Scrapy",
   "user_agent_substring": "Scrapy",
   "user_agent_example": "Scrapy/2.11.0 (+https://scrapy.org)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Not an operator: the default user-agent of the most common Python crawling framework. Anyone can be behind it. Modern Scrapy obeys robots.txt by default, which is why the default UA is still worth a rule.",
   "cost_of_blocking": "You block a very large tail of unattributed one-off crawlers, and also every well-behaved researcher who did not change the default.",
   "operator_docs": "https://scrapy.org/",
   "html_url": "https://www.pathwren.workers.dev/crawler/scrapy.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/scrapy.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "img2dataset",
   "name": "img2dataset",
   "operator": "LAION / img2dataset",
   "operator_slug": "laion",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "robots_token": "img2dataset",
   "user_agent_substring": "img2dataset",
   "user_agent_example": "img2dataset",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The tool used to turn image-URL lists such as LAION's into downloaded training sets. It is run by whoever is building a dataset, not by a single operator.",
   "cost_of_blocking": "Your images are skipped when someone materialises an image-text dataset that references them.",
   "operator_docs": "https://github.com/rom1504/img2dataset",
   "html_url": "https://www.pathwren.workers.dev/crawler/img2dataset.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/img2dataset.json",
   "last_reviewed": "2026-09-01"
  }
 ]
}