{
 "operator": "Cloudflare",
 "slug": "cloudflare",
 "docs": "https://developers.cloudflare.com/ai-search/",
 "crawler_count": 1,
 "crawlers": [
  {
   "slug": "cloudflare-autorag",
   "name": "Cloudflare-AutoRAG",
   "operator": "Cloudflare",
   "operator_slug": "cloudflare",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "robots_token": "Cloudflare-AutoRAG",
   "user_agent_substring": "Cloudflare-AutoRAG",
   "user_agent_example": "Cloudflare-AutoRAG",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The crawler behind Cloudflare's AI Search / AutoRAG, which indexes a website into a retrieval index for an application. Cloudflare's own documentation warns that a bot-blocking rule on your zone will also stop this crawler and tells you to allow-list it.",
   "cost_of_blocking": "Applications built on Cloudflare AI Search cannot retrieve your pages. If you are the one building the index over your own site, blocking it breaks your own product.",
   "operator_docs": "https://developers.cloudflare.com/ai-search/",
   "html_url": "https://www.pathwren.workers.dev/crawler/cloudflare-autorag.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/cloudflare-autorag.json",
   "last_reviewed": "2026-09-01"
  }
 ],
 "ip_range_endpoints": []
}