{
 "operator": "Screaming Frog",
 "slug": "screamingfrog",
 "docs": "https://www.screamingfrog.co.uk/seo-spider/user-agent/",
 "crawler_count": 1,
 "crawlers": [
  {
   "slug": "screaming-frog-seo-spider",
   "name": "Screaming Frog SEO Spider",
   "operator": "Screaming Frog",
   "operator_slug": "screamingfrog",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "Screaming Frog SEO Spider",
   "user_agent_substring": "Screaming Frog SEO Spider",
   "user_agent_example": "Screaming Frog SEO Spider/21.0",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Not an operator: desktop crawling software that anybody can point at any site. The default user-agent identifies the tool, not who is running it, and the operator of the moment is whoever pressed start.",
   "cost_of_blocking": "You block a consultant auditing your own site as often as you block a stranger. Treat it as a rate-limit question, not a consent one — and note that the user-agent is configurable, so a block is advisory.",
   "operator_docs": "https://www.screamingfrog.co.uk/seo-spider/user-agent/",
   "html_url": "https://www.pathwren.workers.dev/crawler/screaming-frog-seo-spider.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/screaming-frog-seo-spider.json",
   "last_reviewed": "2026-09-01"
  }
 ],
 "ip_range_endpoints": []
}