{ "category": "dataset", "label": "Corpus and dataset builders", "description": "Crawl the web into a published or resold dataset that other people train on. Highest leverage per block, longest delay before any effect.", "count": 8, "crawlers": [ { "slug": "ai2bot", "name": "AI2Bot", "operator": "Allen Institute for AI", "operator_slug": "ai2", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "AI2Bot", "user_agent_substring": "AI2Bot", "user_agent_example": "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The Allen Institute's crawler, gathering pages for open research corpora such as Dolma that underpin fully open models like OLMo.", "cost_of_blocking": "Excluded from open research datasets. Worth a deliberate decision: this is the category where 'blocking AI' also blocks the open, auditable end of it.", "operator_docs": "https://allenai.org/crawler", "html_url": "https://www.pathwren.workers.dev/crawler/ai2bot.html", "json_url": "https://www.pathwren.workers.dev/crawler/ai2bot.json", "last_reviewed": "2026-09-01" }, { "slug": "ai2bot-dolma", "name": "Ai2Bot-Dolma", "operator": "Allen Institute for AI", "operator_slug": "ai2", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "Ai2Bot-Dolma", "user_agent_substring": "Ai2Bot-Dolma", "user_agent_example": "Mozilla/5.0 (compatible) Ai2Bot-Dolma (+https://www.allenai.org/crawler)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The variant of AI2's crawler named for the Dolma corpus specifically.", "cost_of_blocking": "Same as AI2Bot: exclusion from an open, published training corpus.", "operator_docs": "https://allenai.org/crawler", "html_url": "https://www.pathwren.workers.dev/crawler/ai2bot-dolma.html", "json_url": "https://www.pathwren.workers.dev/crawler/ai2bot-dolma.json", "last_reviewed": "2026-09-01" }, { "slug": "ccbot", "name": "CCBot", "operator": "Common Crawl", "operator_slug": "commoncrawl", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "CCBot", "user_agent_substring": "CCBot", "user_agent_example": "CCBot/2.0 (https://commoncrawl.org/faq/)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Common Crawl's corpus builder. It trains nothing itself, but its archive is an input to most open and many closed LLM training sets, which makes it the highest-leverage single entry on this list.", "cost_of_blocking": "Future Common Crawl snapshots exclude you, so downstream training sets lose you too — but only going forward. Existing snapshots are permanent and blocking today does not retract them.", "operator_docs": "https://commoncrawl.org/faq", "html_url": "https://www.pathwren.workers.dev/crawler/ccbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/ccbot.json", "last_reviewed": "2026-09-01" }, { "slug": "diffbot", "name": "Diffbot", "operator": "Diffbot", "operator_slug": "diffbot", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "Diffbot", "user_agent_substring": "Diffbot", "user_agent_example": "Mozilla/5.0 (compatible; Diffbot/0.1; +http://www.diffbot.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Extracts structured records from pages to build a commercial knowledge graph that is resold and used for retrieval and training.", "cost_of_blocking": "Your facts stop entering a widely-licensed knowledge graph. Whether that is a loss depends on whether you want to be a machine-readable entity.", "operator_docs": "https://docs.diffbot.com/", "html_url": "https://www.pathwren.workers.dev/crawler/diffbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/diffbot.json", "last_reviewed": "2026-09-01" }, { "slug": "imagesiftbot", "name": "ImagesiftBot", "operator": "Hive AI", "operator_slug": "hive", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "ImagesiftBot", "user_agent_substring": "ImagesiftBot", "user_agent_example": "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Crawls images for Hive AI's reverse-image and dataset products. Image-heavy sites see this one long before they see the text crawlers.", "cost_of_blocking": "Your images stop entering an image dataset and reverse-image index.", "operator_docs": "https://imagesift.com/about", "html_url": "https://www.pathwren.workers.dev/crawler/imagesiftbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/imagesiftbot.json", "last_reviewed": "2026-09-01" }, { "slug": "img2dataset", "name": "img2dataset", "operator": "LAION / img2dataset", "operator_slug": "laion", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "img2dataset", "user_agent_substring": "img2dataset", "user_agent_example": "img2dataset", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The tool used to turn image-URL lists such as LAION's into downloaded training sets. It is run by whoever is building a dataset, not by a single operator.", "cost_of_blocking": "Your images are skipped when someone materialises an image-text dataset that references them.", "operator_docs": "https://github.com/rom1504/img2dataset", "html_url": "https://www.pathwren.workers.dev/crawler/img2dataset.html", "json_url": "https://www.pathwren.workers.dev/crawler/img2dataset.json", "last_reviewed": "2026-09-01" }, { "slug": "omgili", "name": "omgili", "operator": "Webz.io", "operator_slug": "webz", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "omgili", "user_agent_substring": "omgili", "user_agent_example": "omgili/0.5 +http://omgili.com", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The older robots token for the same Webz.io collection, still honoured and still worth listing.", "cost_of_blocking": "Same as omgilibot.", "operator_docs": "https://webz.io/blog/machine-learning/", "html_url": "https://www.pathwren.workers.dev/crawler/omgili.html", "json_url": "https://www.pathwren.workers.dev/crawler/omgili.json", "last_reviewed": "2026-09-01" }, { "slug": "omgilibot", "name": "omgilibot", "operator": "Webz.io", "operator_slug": "webz", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "omgilibot", "user_agent_substring": "omgilibot", "user_agent_example": "omgilibot/0.4; +http://omgili.com", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Webz.io's crawler, collecting web and forum text sold as datasets, including to model builders.", "cost_of_blocking": "Exclusion from a commercial dataset resold to third parties.", "operator_docs": "https://webz.io/blog/machine-learning/", "html_url": "https://www.pathwren.workers.dev/crawler/omgilibot.html", "json_url": "https://www.pathwren.workers.dev/crawler/omgilibot.json", "last_reviewed": "2026-09-01" } ] }