{
 "slug": "block-seo-tools",
 "title": "Block SEO and backlink crawlers",
 "summary": "Ahrefs, Semrush and friends. No user-facing consequence, and often the largest single slice of your bot traffic.",
 "detail": "The cheapest bandwidth saving available to most sites, and the one nobody regrets. The only cost is that your own dashboards on those tools get thinner. Consider Crawl-delay first: both major operators honour it.",
 "crawler_count": 2,
 "crawlers": [
  "ahrefsbot",
  "semrushbot"
 ],
 "robots_txt_url": "https://www.pathwren.workers.dev/robots/block-seo-tools.txt",
 "robots_txt": "# AI Crawler Index — policy: block-seo-tools\n# Block SEO and backlink crawlers\n# Ahrefs, Semrush and friends. No user-facing consequence, and often the largest single slice of your bot traffic.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/block-seo-tools.html\n# 2 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: SemrushBot\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n",
 "generated": "2026-09-01"
}