{
 "operator": "Semrush",
 "slug": "semrush",
 "docs": "https://www.semrush.com/bot/",
 "crawler_count": 2,
 "crawlers": [
  {
   "slug": "semrushbot",
   "name": "SemrushBot",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "SemrushBot",
   "user_agent_substring": "SemrushBot",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Semrush's backlink and keyword crawler. It is not an AI crawler, but it is usually in the top three by volume on any site, and it is the cheapest block on this list.",
   "cost_of_blocking": "Your competitors' Semrush reports get thinner, and so do yours. No user-facing effect.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "semrushbot-ocob",
   "name": "SemrushBot-OCOB",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "SemrushBot-OCOB",
   "user_agent_substring": "SemrushBot-OCOB",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-OCOB/1.0; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Semrush's separately-tokenised crawler for its AI content tooling, split out so SEO crawling and AI reuse can be answered differently.",
   "cost_of_blocking": "Exclusion from Semrush's AI corpus, with its SEO crawl unaffected.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ocob.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ocob.json",
   "last_reviewed": "2026-09-01"
  }
 ],
 "ip_range_endpoints": []
}