{
 "category": "user-fetch",
 "label": "User-triggered fetchers",
 "description": "Fetch one page because a person asked for it, right then. One human intent, one request. Blocking them produces a visible error for a real reader.",
 "count": 6,
 "crawlers": [
  {
   "slug": "chatgpt-user",
   "name": "ChatGPT-User",
   "operator": "OpenAI",
   "operator_slug": "openai",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "ChatGPT-User",
   "user_agent_substring": "ChatGPT-User",
   "user_agent_example": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; ChatGPT-User/1.0; +https://openai.com/bot",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://openai.com/chatgpt-user.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-chatgpt-user.json",
   "ipv4_prefix_count": 204,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a single page at the moment a user or a ChatGPT agent asks for it — a pasted link, a browsing step, an Operator task. One human intent, one request. OpenAI states these fetches are not used for training.",
   "cost_of_blocking": "ChatGPT cannot open your pages when a user explicitly asks it to. The user sees a fetch failure. This is usually the last bot anyone means to block.",
   "operator_docs": "https://platform.openai.com/docs/bots",
   "html_url": "https://www.pathwren.workers.dev/crawler/chatgpt-user.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/chatgpt-user.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "claude-user",
   "name": "Claude-User",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "Claude-User",
   "user_agent_substring": "Claude-User",
   "user_agent_example": "Mozilla/5.0 (compatible; Claude-User/1.0; +Claude-User@anthropic.com)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a page because a Claude user asked Claude to read it, at that moment.",
   "cost_of_blocking": "Claude reports a fetch failure to a user who asked for your page by name.",
   "operator_docs": "https://support.anthropic.com/en/articles/8896518",
   "html_url": "https://www.pathwren.workers.dev/crawler/claude-user.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/claude-user.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "cohere-ai",
   "name": "cohere-ai",
   "operator": "Cohere",
   "operator_slug": "cohere",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "cohere-ai",
   "user_agent_substring": "cohere-ai",
   "user_agent_example": "Mozilla/5.0 (compatible; cohere-ai)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Cohere's fetcher, used when its assistant products need a page.",
   "cost_of_blocking": "Cohere-powered assistants cannot read your pages on request.",
   "operator_docs": "https://cohere.com/",
   "html_url": "https://www.pathwren.workers.dev/crawler/cohere-ai.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/cohere-ai.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "meta-externalfetcher",
   "name": "meta-externalfetcher",
   "operator": "Meta",
   "operator_slug": "meta",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "meta-externalfetcher",
   "user_agent_substring": "meta-externalfetcher",
   "user_agent_example": "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a page when a Meta AI user asks about a specific link.",
   "cost_of_blocking": "Meta AI cannot read pages users hand it.",
   "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "mistralai-user",
   "name": "MistralAI-User",
   "operator": "Mistral AI",
   "operator_slug": "mistral",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "MistralAI-User",
   "user_agent_substring": "MistralAI-User",
   "user_agent_example": "Mozilla/5.0 (compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a page when a Le Chat user asks Mistral's assistant to read it.",
   "cost_of_blocking": "Le Chat cannot open links your readers give it.",
   "operator_docs": "https://docs.mistral.ai/",
   "html_url": "https://www.pathwren.workers.dev/crawler/mistralai-user.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/mistralai-user.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "perplexity-user",
   "name": "Perplexity-User",
   "operator": "Perplexity",
   "operator_slug": "perplexity",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "Perplexity-User",
   "user_agent_substring": "Perplexity-User",
   "user_agent_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://www.perplexity.ai/perplexity-user.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/perplexity-user.json",
   "ipv4_prefix_count": 4,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a page because a Perplexity user asked for it. Perplexity documents that this fetch is user-initiated and is therefore not governed by robots.txt — a robots rule will not stop it, by stated policy.",
   "cost_of_blocking": "Not controllable via robots.txt. If you must stop it, verify by the published IP ranges and block at the edge — and accept that users who ask for your page get an error.",
   "operator_docs": "https://docs.perplexity.ai/guides/bots",
   "html_url": "https://www.pathwren.workers.dev/crawler/perplexity-user.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/perplexity-user.json",
   "last_reviewed": "2026-09-01"
  }
 ]
}