{
 "operator": "Allen Institute for AI",
 "slug": "ai2",
 "docs": "https://allenai.org/crawler",
 "crawler_count": 2,
 "crawlers": [
  {
   "slug": "ai2bot",
   "name": "AI2Bot",
   "operator": "Allen Institute for AI",
   "operator_slug": "ai2",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "robots_token": "AI2Bot",
   "user_agent_substring": "AI2Bot",
   "user_agent_example": "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The Allen Institute's crawler, gathering pages for open research corpora such as Dolma that underpin fully open models like OLMo.",
   "cost_of_blocking": "Excluded from open research datasets. Worth a deliberate decision: this is the category where 'blocking AI' also blocks the open, auditable end of it.",
   "operator_docs": "https://allenai.org/crawler",
   "html_url": "https://www.pathwren.workers.dev/crawler/ai2bot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/ai2bot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "ai2bot-dolma",
   "name": "Ai2Bot-Dolma",
   "operator": "Allen Institute for AI",
   "operator_slug": "ai2",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "robots_token": "Ai2Bot-Dolma",
   "user_agent_substring": "Ai2Bot-Dolma",
   "user_agent_example": "Mozilla/5.0 (compatible) Ai2Bot-Dolma (+https://www.allenai.org/crawler)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The variant of AI2's crawler named for the Dolma corpus specifically.",
   "cost_of_blocking": "Same as AI2Bot: exclusion from an open, published training corpus.",
   "operator_docs": "https://allenai.org/crawler",
   "html_url": "https://www.pathwren.workers.dev/crawler/ai2bot-dolma.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/ai2bot-dolma.json",
   "last_reviewed": "2026-09-01"
  }
 ],
 "ip_range_endpoints": []
}