{
 "category": "archive",
 "label": "Archivers",
 "description": "Preservation crawlers. Their output is public and permanent, which makes them a separate decision from the AI one.",
 "count": 2,
 "crawlers": [
  {
   "slug": "archive-org-bot",
   "name": "archive.org_bot",
   "operator": "Internet Archive",
   "operator_slug": "internetarchive",
   "category": "archive",
   "category_label": "Archivers",
   "robots_token": "archive.org_bot",
   "user_agent_substring": "archive.org_bot",
   "user_agent_example": "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The Wayback Machine's crawler. Preservation rather than AI, but it lands in the same 'is this bot welcome' decision and its output is a public corpus.",
   "cost_of_blocking": "Your site stops being preserved. When it dies, it is gone. Consider this one separately from the AI question.",
   "operator_docs": "https://archive.org/details/archive.org_bot",
   "html_url": "https://www.pathwren.workers.dev/crawler/archive-org-bot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/archive-org-bot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "ia-archiver",
   "name": "ia_archiver",
   "operator": "Internet Archive",
   "operator_slug": "internetarchive",
   "category": "archive",
   "category_label": "Archivers",
   "robots_token": "ia_archiver",
   "user_agent_substring": "ia_archiver",
   "user_agent_example": "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The legacy Alexa/Internet Archive token, still present in most robots.txt files and still occasionally honoured.",
   "cost_of_blocking": "Negligible today; retain for tidiness.",
   "operator_docs": "https://archive.org/details/archive.org_bot",
   "html_url": "https://www.pathwren.workers.dev/crawler/ia-archiver.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/ia-archiver.json",
   "last_reviewed": "2026-09-01"
  }
 ]
}