{
 "category": "seo",
 "label": "SEO and backlink crawlers",
 "description": "Commercial link-graph tooling. No user-facing effect either way, and usually a large share of your bot bandwidth.",
 "count": 2,
 "crawlers": [
  {
   "slug": "ahrefsbot",
   "name": "AhrefsBot",
   "operator": "Ahrefs",
   "operator_slug": "ahrefs",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "AhrefsBot",
   "user_agent_substring": "AhrefsBot",
   "user_agent_example": "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Ahrefs' backlink crawler, and one of the largest non-search crawlers on the web by request volume.",
   "cost_of_blocking": "No user-facing effect. Ahrefs honours Crawl-delay, so rate-limiting is usually better than blocking.",
   "operator_docs": "https://ahrefs.com/robot",
   "html_url": "https://www.pathwren.workers.dev/crawler/ahrefsbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/ahrefsbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "semrushbot",
   "name": "SemrushBot",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "SemrushBot",
   "user_agent_substring": "SemrushBot",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Semrush's backlink and keyword crawler. It is not an AI crawler, but it is usually in the top three by volume on any site, and it is the cheapest block on this list.",
   "cost_of_blocking": "Your competitors' Semrush reports get thinner, and so do yours. No user-facing effect.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot.json",
   "last_reviewed": "2026-09-01"
  }
 ]
}