{
 "tool": "classify-ua",
 "endpoint": "https://www.pathwren.workers.dev/tools/classify-ua",
 "asked": {
  "ua": "SEMrushBot"
 },
 "summary": "SemrushBot — Semrush (SEO and backlink crawlers).\nMatched on \"SemrushBot\".\n\nWhat it is: Semrush's backlink and keyword crawler. It is not an AI crawler, but it is usually in the top three by volume on any site, and it is the cheapest block on this list.\n\nCost of blocking it: Your competitors' Semrush reports get thinner, and so do yours. No user-facing effect.\n\nrobots.txt token: SemrushBot · robots.txt: obeys robots.txt (documented) · verification: reverse DNS\n\nA user-agent string is a claim, not a proof: anyone can send it. Before you act on a match, check the address with is_verified_crawler_ip, or use the operator's reverse-DNS method where there are no published ranges.",
 "answer": {
  "user_agent": "SEMrushBot",
  "matched": true,
  "matched_on": "SemrushBot",
  "crawler": {
   "slug": "semrushbot",
   "name": "SemrushBot",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "SemrushBot",
   "user_agent_substring": "SemrushBot",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Semrush's backlink and keyword crawler. It is not an AI crawler, but it is usually in the top three by volume on any site, and it is the cheapest block on this list.",
   "cost_of_blocking": "Your competitors' Semrush reports get thinner, and so do yours. No user-facing effect.",
   "operator_docs": "https://www.semrush.com/bot/",
   "docs_url": "https://www.pathwren.workers.dev/crawler/semrushbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot.json",
   "last_reviewed": "2026-09-12"
  },
  "also_matched": [],
  "verification": {
   "method": "reverse-dns",
   "label": "reverse DNS",
   "published_ip_ranges": null,
   "how": "Operator verifies by forward-confirmed reverse DNS, not by a published prefix list; an IP check here will usually not match."
  },
  "caveat": "A user-agent string is a claim, not a proof: anyone can send it. Before you act on a match, check the address with is_verified_crawler_ip, or use the operator's reverse-DNS method where there are no published ranges.",
  "source": "https://www.pathwren.workers.dev"
 },
 "answered_by": {
  "note": "The MCP tool itself answered this. One implementation, two doors — this endpoint holds no copy of it.",
  "server": "ai-crawler-index",
  "endpoint": "https://www.pathwren.workers.dev/mcp",
  "tool": "classify_user_agent",
  "protocol": "MCP streamable-http, JSON-RPC 2.0 tools/call"
 },
 "caveats": [
  "A user-agent is a claim, not proof. Pair it with /tools/verify-crawler, which is the address half of the same question."
 ],
 "docs": "https://www.pathwren.workers.dev/tools/classify-ua.html?s=client-dossiers",
 "catalogue": "https://www.pathwren.workers.dev/tools/index.json?s=client-dossiers",
 "more": {
  "who actually crawls this host": "https://www.pathwren.workers.dev/bot/index.html?s=client-dossiers",
  "every named client as one file": "https://www.pathwren.workers.dev/data/observed-clients.json?s=client-dossiers"
 },
 "auth": "none — no account, no key, no handshake. CORS open, cacheable, CC0-1.0.",
 "license": "CC0-1.0",
 "independent": true,
 "_about": {
  "page": "https://www.pathwren.workers.dev/bot/claudebot.html?s=client-dossiers",
  "what": "What this host recorded about the client that made this request — the requests, the status codes and the user-agent string, from its own log. Nothing else: no address is published, nothing is looked up anywhere, and there is no account to close.",
  "matched_on": "the User-Agent header you sent. If we have no page for it yet this points at the index of all of them.",
  "index": "https://www.pathwren.workers.dev/bot/index.html?s=client-dossiers"
 }
}