{
 "slug": "scrapy",
 "name": "Scrapy",
 "operator": "Scrapy project",
 "operator_slug": "scrapy",
 "category": "tool",
 "category_label": "Tools and frameworks",
 "robots_token": "Scrapy",
 "user_agent_substring": "Scrapy",
 "user_agent_example": "Scrapy/2.11.0 (+https://scrapy.org)",
 "respects_robots_txt": "documented",
 "respects_robots_txt_label": "obeys robots.txt (documented)",
 "verification_method": "none",
 "verification_label": "no published verification method",
 "published_ip_ranges_url": null,
 "ip_ranges_endpoint": null,
 "ipv4_prefix_count": 0,
 "ipv6_prefix_count": 0,
 "what_it_is": "Not an operator: the default user-agent of the most common Python crawling framework. Anyone can be behind it. Modern Scrapy obeys robots.txt by default, which is why the default UA is still worth a rule.",
 "cost_of_blocking": "You block a very large tail of unattributed one-off crawlers, and also every well-behaved researcher who did not change the default.",
 "operator_docs": "https://scrapy.org/",
 "html_url": "https://www.pathwren.workers.dev/crawler/scrapy.html",
 "json_url": "https://www.pathwren.workers.dev/crawler/scrapy.json",
 "last_reviewed": "2026-09-01"
}