{
 "operator": "Anthropic",
 "slug": "anthropic",
 "docs": "https://support.anthropic.com/en/articles/8896518",
 "crawler_count": 5,
 "crawlers": [
  {
   "slug": "anthropic-ai",
   "name": "anthropic-ai",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "anthropic-ai",
   "user_agent_substring": "anthropic-ai",
   "user_agent_example": "(no live crawler currently identifies with this string)",
   "respects_robots_txt": "n-a",
   "respects_robots_txt_label": "control token only — no crawler",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "A legacy robots.txt token from before Anthropic consolidated on ClaudeBot. It is still widely present in robots.txt files and costs nothing to keep, but it is a control token rather than a bot you will see in logs.",
   "cost_of_blocking": "None. Nothing crawls under this name today; keeping the rule is harmless insurance.",
   "operator_docs": "https://support.anthropic.com/en/articles/8896518",
   "html_url": "https://www.pathwren.workers.dev/crawler/anthropic-ai.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/anthropic-ai.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "claude-searchbot",
   "name": "Claude-SearchBot",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "robots_token": "Claude-SearchBot",
   "user_agent_substring": "Claude-SearchBot",
   "user_agent_example": "Mozilla/5.0 (compatible; Claude-SearchBot/1.0; +Claude-SearchBot@anthropic.com)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Indexes pages so Claude's web search can find and cite them. Separate token from the training crawler, so search visibility and training consent are independent decisions.",
   "cost_of_blocking": "You stop appearing in Claude's search results and citations.",
   "operator_docs": "https://support.anthropic.com/en/articles/8896518",
   "html_url": "https://www.pathwren.workers.dev/crawler/claude-searchbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/claude-searchbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "claude-user",
   "name": "Claude-User",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "Claude-User",
   "user_agent_substring": "Claude-User",
   "user_agent_example": "Mozilla/5.0 (compatible; Claude-User/1.0; +Claude-User@anthropic.com)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a page because a Claude user asked Claude to read it, at that moment.",
   "cost_of_blocking": "Claude reports a fetch failure to a user who asked for your page by name.",
   "operator_docs": "https://support.anthropic.com/en/articles/8896518",
   "html_url": "https://www.pathwren.workers.dev/crawler/claude-user.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/claude-user.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "claude-web",
   "name": "Claude-Web",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "robots_token": "Claude-Web",
   "user_agent_substring": "Claude-Web",
   "user_agent_example": "Mozilla/5.0 (compatible; Claude-Web/1.0)",
   "respects_robots_txt": "n-a",
   "respects_robots_txt_label": "control token only — no crawler",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "An earlier Anthropic token for user-facing web access, superseded by Claude-User and Claude-SearchBot. Kept here because it appears in most published robots.txt templates.",
   "cost_of_blocking": "None in practice. Retain the rule; expect no traffic.",
   "operator_docs": "https://support.anthropic.com/en/articles/8896518",
   "html_url": "https://www.pathwren.workers.dev/crawler/claude-web.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/claude-web.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "claudebot",
   "name": "ClaudeBot",
   "operator": "Anthropic",
   "operator_slug": "anthropic",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "ClaudeBot",
   "user_agent_substring": "ClaudeBot",
   "user_agent_example": "Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Anthropic's bulk crawler, gathering pages that may be used to train Claude models.",
   "cost_of_blocking": "Content excluded from training data for future Claude models. No effect on Claude's ability to fetch a link a user gives it.",
   "operator_docs": "https://support.anthropic.com/en/articles/8896518",
   "html_url": "https://www.pathwren.workers.dev/crawler/claudebot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/claudebot.json",
   "last_reviewed": "2026-09-01"
  }
 ],
 "ip_range_endpoints": []
}