{ "name": "AI Crawler Index", "description": "Every AI crawler on the web, what it is for, what blocking it costs you, and the IP ranges its operator publishes — as JSON, CSV, robots.txt and regex.", "url": "https://www.pathwren.workers.dev", "generated_at": "2026-09-01T21:52:59+00:00", "version": "2026-09-01", "license": "CC0-1.0", "source_note": "Robots tokens, user-agent strings and documentation URLs are taken from each operator's published documentation, cited per record in operator_docs. The category, the prose in what_it_is and the cost_of_blocking assessment are this index's own.", "count": 150, "categories": { "ai-training": { "label": "AI training crawlers", "description": "Collect pages in bulk so that a model can be trained or fine-tuned on them. Blocking these removes you from future training sets and changes nothing a user sees today." }, "ai-search": { "label": "AI search crawlers", "description": "Build the retrieval index an assistant answers and cites from. These are the crawlers that send you traffic; blocking them is the expensive mistake in this space." }, "user-fetch": { "label": "User-triggered fetchers", "description": "Fetch one page because a person asked for it, right then. One human intent, one request. Blocking them produces a visible error for a real reader." }, "dataset": { "label": "Corpus and dataset builders", "description": "Crawl the web into a published or resold dataset that other people train on. Highest leverage per block, longest delay before any effect." }, "search": { "label": "Search engines", "description": "Classic index-and-rank crawlers. Several also feed their operator's generative answers, which is why the AI opt-out for Google and Apple is a token rather than a block." }, "seo": { "label": "SEO and backlink crawlers", "description": "Commercial link-graph tooling. No user-facing effect either way, and usually a large share of your bot bandwidth." }, "archive": { "label": "Archivers", "description": "Preservation crawlers. Their output is public and permanent, which makes them a separate decision from the AI one." }, "preview": { "label": "Link preview fetchers", "description": "Read your Open Graph tags when someone shares a link. Blocking these is almost always an accident." }, "tool": { "label": "Tools and frameworks", "description": "Not operators: crawling software anyone can run. The party behind the request is unknown, so treat them as a rate-limit question rather than a consent question." } }, "endpoints": { "agents_json": "https://www.pathwren.workers.dev/data/agents.json", "agents_csv": "https://www.pathwren.workers.dev/data/agents.csv", "user_agents_txt": "https://www.pathwren.workers.dev/data/user-agents.txt", "robots_tokens_txt": "https://www.pathwren.workers.dev/data/robots-tokens.txt", "ua_regex_json": "https://www.pathwren.workers.dev/data/ua-regex.json", "ip_ranges_json": "https://www.pathwren.workers.dev/ip-ranges/all.json", "ip_ranges_txt": "https://www.pathwren.workers.dev/ip-ranges/all.txt", "status_json": "https://www.pathwren.workers.dev/status.json", "openapi": "https://www.pathwren.workers.dev/openapi.json", "changes": "https://www.pathwren.workers.dev/changes.json?since=105" }, "links": [ { "rel": "self", "href": "https://www.pathwren.workers.dev/data/agents.json", "type": "application/json" }, { "rel": "changes", "href": "https://www.pathwren.workers.dev/changes.json?since=105", "type": "application/json", "title": "What changed since your cursor — poll this instead of re-downloading this document", "cursor_param": "since", "head_cursor": 105, "min_poll_seconds": 21600, "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 700 bytes, or a 304 with no body if you send back the ETag." }, { "rel": "related", "href": "https://www.pathwren.workers.dev/ip-ranges/all.json", "type": "application/json", "title": "Every operator-published prefix, unioned and grouped by source" }, { "rel": "related", "href": "https://www.pathwren.workers.dev/status.json", "type": "application/json", "title": "Freshness of every upstream IP-range source" }, { "rel": "related", "href": "https://www.pathwren.workers.dev/data/ua-regex.json", "type": "application/json", "title": "Compiled user-agent regexes for a log classifier or a WAF" }, { "rel": "service-desc", "href": "https://www.pathwren.workers.dev/openapi.json", "type": "application/json", "title": "Every read endpoint, described formally" } ], "crawlers": [ { "slug": "gptbot", "name": "GPTBot", "operator": "OpenAI", "operator_slug": "openai", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "GPTBot", "user_agent_substring": "GPTBot", "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2; +https://openai.com/gptbot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/gptbot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-gptbot.json", "ipv4_prefix_count": 21, "ipv6_prefix_count": 0, "what_it_is": "OpenAI's bulk crawler. Pages it fetches may be used to train future OpenAI foundation models. It is not the bot that puts you in ChatGPT's search results, and blocking it does not remove you from them.", "cost_of_blocking": "Your content is excluded from training data for future OpenAI models. No effect on ChatGPT search visibility, on citations, or on links a user pastes into ChatGPT.", "operator_docs": "https://platform.openai.com/docs/bots", "html_url": "https://www.pathwren.workers.dev/crawler/gptbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/gptbot.json", "last_reviewed": "2026-09-01" }, { "slug": "oai-searchbot", "name": "OAI-SearchBot", "operator": "OpenAI", "operator_slug": "openai", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "OAI-SearchBot", "user_agent_substring": "OAI-SearchBot", "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/searchbot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-searchbot.json", "ipv4_prefix_count": 35, "ipv6_prefix_count": 0, "what_it_is": "Builds the index ChatGPT search answers from. Content it collects is used for retrieval and citation, not for model training.", "cost_of_blocking": "High. Blocking this removes you from ChatGPT search results and from the source links ChatGPT shows. This is the single most expensive block on this list for anyone who wants to be cited by an assistant.", "operator_docs": "https://platform.openai.com/docs/bots", "html_url": "https://www.pathwren.workers.dev/crawler/oai-searchbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/oai-searchbot.json", "last_reviewed": "2026-09-01" }, { "slug": "chatgpt-user", "name": "ChatGPT-User", "operator": "OpenAI", "operator_slug": "openai", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "ChatGPT-User", "user_agent_substring": "ChatGPT-User", "user_agent_example": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; ChatGPT-User/1.0; +https://openai.com/bot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/chatgpt-user.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-chatgpt-user.json", "ipv4_prefix_count": 204, "ipv6_prefix_count": 0, "what_it_is": "Fetches a single page at the moment a user or a ChatGPT agent asks for it — a pasted link, a browsing step, an Operator task. One human intent, one request. OpenAI states these fetches are not used for training.", "cost_of_blocking": "ChatGPT cannot open your pages when a user explicitly asks it to. The user sees a fetch failure. This is usually the last bot anyone means to block.", "operator_docs": "https://platform.openai.com/docs/bots", "html_url": "https://www.pathwren.workers.dev/crawler/chatgpt-user.html", "json_url": "https://www.pathwren.workers.dev/crawler/chatgpt-user.json", "last_reviewed": "2026-09-01" }, { "slug": "claudebot", "name": "ClaudeBot", "operator": "Anthropic", "operator_slug": "anthropic", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "ClaudeBot", "user_agent_substring": "ClaudeBot", "user_agent_example": "Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Anthropic's bulk crawler, gathering pages that may be used to train Claude models.", "cost_of_blocking": "Content excluded from training data for future Claude models. No effect on Claude's ability to fetch a link a user gives it.", "operator_docs": "https://support.anthropic.com/en/articles/8896518", "html_url": "https://www.pathwren.workers.dev/crawler/claudebot.html", "json_url": "https://www.pathwren.workers.dev/crawler/claudebot.json", "last_reviewed": "2026-09-01" }, { "slug": "claude-searchbot", "name": "Claude-SearchBot", "operator": "Anthropic", "operator_slug": "anthropic", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "Claude-SearchBot", "user_agent_substring": "Claude-SearchBot", "user_agent_example": "Mozilla/5.0 (compatible; Claude-SearchBot/1.0; +Claude-SearchBot@anthropic.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Indexes pages so Claude's web search can find and cite them. Separate token from the training crawler, so search visibility and training consent are independent decisions.", "cost_of_blocking": "You stop appearing in Claude's search results and citations.", "operator_docs": "https://support.anthropic.com/en/articles/8896518", "html_url": "https://www.pathwren.workers.dev/crawler/claude-searchbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/claude-searchbot.json", "last_reviewed": "2026-09-01" }, { "slug": "claude-user", "name": "Claude-User", "operator": "Anthropic", "operator_slug": "anthropic", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "Claude-User", "user_agent_substring": "Claude-User", "user_agent_example": "Mozilla/5.0 (compatible; Claude-User/1.0; +Claude-User@anthropic.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches a page because a Claude user asked Claude to read it, at that moment.", "cost_of_blocking": "Claude reports a fetch failure to a user who asked for your page by name.", "operator_docs": "https://support.anthropic.com/en/articles/8896518", "html_url": "https://www.pathwren.workers.dev/crawler/claude-user.html", "json_url": "https://www.pathwren.workers.dev/crawler/claude-user.json", "last_reviewed": "2026-09-01" }, { "slug": "anthropic-ai", "name": "anthropic-ai", "operator": "Anthropic", "operator_slug": "anthropic", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "anthropic-ai", "user_agent_substring": "anthropic-ai", "user_agent_example": "(no live crawler currently identifies with this string)", "respects_robots_txt": "n-a", "respects_robots_txt_label": "control token only — no crawler", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "A legacy robots.txt token from before Anthropic consolidated on ClaudeBot. It is still widely present in robots.txt files and costs nothing to keep, but it is a control token rather than a bot you will see in logs.", "cost_of_blocking": "None. Nothing crawls under this name today; keeping the rule is harmless insurance.", "operator_docs": "https://support.anthropic.com/en/articles/8896518", "html_url": "https://www.pathwren.workers.dev/crawler/anthropic-ai.html", "json_url": "https://www.pathwren.workers.dev/crawler/anthropic-ai.json", "last_reviewed": "2026-09-01" }, { "slug": "claude-web", "name": "Claude-Web", "operator": "Anthropic", "operator_slug": "anthropic", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "Claude-Web", "user_agent_substring": "Claude-Web", "user_agent_example": "Mozilla/5.0 (compatible; Claude-Web/1.0)", "respects_robots_txt": "n-a", "respects_robots_txt_label": "control token only — no crawler", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "An earlier Anthropic token for user-facing web access, superseded by Claude-User and Claude-SearchBot. Kept here because it appears in most published robots.txt templates.", "cost_of_blocking": "None in practice. Retain the rule; expect no traffic.", "operator_docs": "https://support.anthropic.com/en/articles/8896518", "html_url": "https://www.pathwren.workers.dev/crawler/claude-web.html", "json_url": "https://www.pathwren.workers.dev/crawler/claude-web.json", "last_reviewed": "2026-09-01" }, { "slug": "google-extended", "name": "Google-Extended", "operator": "Google", "operator_slug": "google", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "Google-Extended", "user_agent_substring": "(control token only — no crawler)", "user_agent_example": "(none: Google-Extended never appears as a user-agent)", "respects_robots_txt": "n-a", "respects_robots_txt_label": "control token only — no crawler", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Not a crawler. A robots.txt token that tells Google whether pages Googlebot already fetched may be used to train and ground Gemini. You will never see it in an access log; disallowing it changes what Google does with content it fetched under a different name.", "cost_of_blocking": "You are excluded from Gemini grounding and Gemini training. Google Search ranking and indexing are explicitly unaffected. This is the cleanest 'no training, keep my search traffic' lever that exists.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/google-extended.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-extended.json", "last_reviewed": "2026-09-01" }, { "slug": "googlebot", "name": "Googlebot", "operator": "Google", "operator_slug": "google", "category": "search", "category_label": "Search engines", "robots_token": "Googlebot", "user_agent_substring": "Googlebot", "user_agent_example": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json", "ipv4_prefix_count": 170, "ipv6_prefix_count": 147, "what_it_is": "The classic search crawler. It is also the crawler behind AI Overviews: Google does not run a separate bot for them, which is why the only AI opt-out is the Google-Extended token and not a Googlebot block.", "cost_of_blocking": "Total. You leave Google Search. Never block this to avoid AI use; use Google-Extended instead.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/googlebot.html", "json_url": "https://www.pathwren.workers.dev/crawler/googlebot.json", "last_reviewed": "2026-09-01" }, { "slug": "googleother", "name": "GoogleOther", "operator": "Google", "operator_slug": "google", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "GoogleOther", "user_agent_substring": "GoogleOther", "user_agent_example": "Mozilla/5.0 (compatible; GoogleOther)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json", "ipv4_prefix_count": 136, "ipv6_prefix_count": 136, "what_it_is": "A generic fetcher used by Google product teams for one-off crawls and research, including data collection that does not belong to Search.", "cost_of_blocking": "No effect on Search indexing. Blocks internal Google research and product fetches.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/googleother.html", "json_url": "https://www.pathwren.workers.dev/crawler/googleother.json", "last_reviewed": "2026-09-01" }, { "slug": "google-cloudvertexbot", "name": "Google-CloudVertexBot", "operator": "Google", "operator_slug": "google", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "Google-CloudVertexBot", "user_agent_substring": "Google-CloudVertexBot", "user_agent_example": "Mozilla/5.0 (compatible; Google-CloudVertexBot/1.0; +https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json", "ipv4_prefix_count": 136, "ipv6_prefix_count": 136, "what_it_is": "Crawls a site on behalf of a Vertex AI Agent Builder customer who is building an agent over that site. It only visits sites the customer has asked it to.", "cost_of_blocking": "Third parties can no longer build Vertex AI agents that read your site. Irrelevant to Google Search.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/google-cloudvertexbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-cloudvertexbot.json", "last_reviewed": "2026-09-01" }, { "slug": "google-inspectiontool", "name": "Google-InspectionTool", "operator": "Google", "operator_slug": "google", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "Google-InspectionTool", "user_agent_substring": "Google-InspectionTool", "user_agent_example": "Mozilla/5.0 (compatible; Google-InspectionTool/1.0;)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json", "ipv4_prefix_count": 136, "ipv6_prefix_count": 136, "what_it_is": "The fetcher behind Search Console's URL Inspection and the Rich Results Test. It runs when a site owner clicks a button.", "cost_of_blocking": "Your own Search Console live tests stop working. Blocking this only hurts you.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/google-inspectiontool.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-inspectiontool.json", "last_reviewed": "2026-09-01" }, { "slug": "googlebot-image", "name": "Googlebot-Image", "operator": "Google", "operator_slug": "google", "category": "search", "category_label": "Search engines", "robots_token": "Googlebot-Image", "user_agent_substring": "Googlebot-Image", "user_agent_example": "Googlebot-Image/1.0", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json", "ipv4_prefix_count": 170, "ipv6_prefix_count": 147, "what_it_is": "Image indexing for Google Images. A separate token so you can leave images out of search without leaving search.", "cost_of_blocking": "Your images stop appearing in Google Images.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/googlebot-image.html", "json_url": "https://www.pathwren.workers.dev/crawler/googlebot-image.json", "last_reviewed": "2026-09-01" }, { "slug": "googlebot-news", "name": "Googlebot-News", "operator": "Google", "operator_slug": "google", "category": "search", "category_label": "Search engines", "robots_token": "Googlebot-News", "user_agent_substring": "Googlebot-News", "user_agent_example": "(uses the Googlebot user-agent; controlled by the Googlebot-News robots token)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json", "ipv4_prefix_count": 170, "ipv6_prefix_count": 147, "what_it_is": "A robots.txt token controlling inclusion in Google News. It does not have its own user-agent string; the fetch arrives as Googlebot.", "cost_of_blocking": "Removal from Google News, with normal Search unaffected.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/googlebot-news.html", "json_url": "https://www.pathwren.workers.dev/crawler/googlebot-news.json", "last_reviewed": "2026-09-01" }, { "slug": "storebot-google", "name": "Storebot-Google", "operator": "Google", "operator_slug": "google", "category": "search", "category_label": "Search engines", "robots_token": "Storebot-Google", "user_agent_substring": "Storebot-Google", "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36 (compatible; Storebot-Google/1.0; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json", "ipv4_prefix_count": 136, "ipv6_prefix_count": 136, "what_it_is": "Checks shopping and checkout flows for Google's shopping surfaces.", "cost_of_blocking": "Product listings may lose shopping-specific enrichment. Irrelevant to non-commerce sites.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/storebot-google.html", "json_url": "https://www.pathwren.workers.dev/crawler/storebot-google.json", "last_reviewed": "2026-09-01" }, { "slug": "bingbot", "name": "bingbot", "operator": "Microsoft", "operator_slug": "microsoft", "category": "search", "category_label": "Search engines", "robots_token": "bingbot", "user_agent_substring": "bingbot", "user_agent_example": "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://www.bing.com/toolbox/bingbot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/bing-bingbot.json", "ipv4_prefix_count": 28, "ipv6_prefix_count": 0, "what_it_is": "Bing's only crawler, and therefore also the crawler behind Microsoft Copilot's grounding. Microsoft's documented way to keep search indexing while refusing generative reuse is the nocache / noarchive robots meta directive, not a separate user-agent.", "cost_of_blocking": "Very high and very wide: Bing, Copilot, DuckDuckGo and several assistants that resell Bing's index all lose you at once. Use nocache/noarchive rather than blocking.", "operator_docs": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "html_url": "https://www.pathwren.workers.dev/crawler/bingbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/bingbot.json", "last_reviewed": "2026-09-01" }, { "slug": "applebot", "name": "Applebot", "operator": "Apple", "operator_slug": "apple", "category": "search", "category_label": "Search engines", "robots_token": "Applebot", "user_agent_substring": "Applebot", "user_agent_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://search.developer.apple.com/applebot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/apple-applebot.json", "ipv4_prefix_count": 33, "ipv6_prefix_count": 0, "what_it_is": "Powers Siri, Spotlight and Safari suggestions. Blocking it is a search decision, not an AI decision — the AI decision has its own token.", "cost_of_blocking": "You disappear from Siri, Spotlight and Safari search suggestions across Apple's install base.", "operator_docs": "https://support.apple.com/en-us/119829", "html_url": "https://www.pathwren.workers.dev/crawler/applebot.html", "json_url": "https://www.pathwren.workers.dev/crawler/applebot.json", "last_reviewed": "2026-09-01" }, { "slug": "applebot-extended", "name": "Applebot-Extended", "operator": "Apple", "operator_slug": "apple", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "Applebot-Extended", "user_agent_substring": "(control token only — no crawler)", "user_agent_example": "(none: Applebot-Extended never appears as a user-agent)", "respects_robots_txt": "n-a", "respects_robots_txt_label": "control token only — no crawler", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Apple's counterpart to Google-Extended: a robots.txt token that withdraws consent for Apple Intelligence and Apple foundation-model training, without touching Applebot's search crawl.", "cost_of_blocking": "Excluded from Apple Intelligence training. Siri, Spotlight and Safari suggestions are unaffected.", "operator_docs": "https://support.apple.com/en-us/119829", "html_url": "https://www.pathwren.workers.dev/crawler/applebot-extended.html", "json_url": "https://www.pathwren.workers.dev/crawler/applebot-extended.json", "last_reviewed": "2026-09-01" }, { "slug": "perplexitybot", "name": "PerplexityBot", "operator": "Perplexity", "operator_slug": "perplexity", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "PerplexityBot", "user_agent_substring": "PerplexityBot", "user_agent_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://www.perplexity.ai/perplexitybot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/perplexity-bot.json", "ipv4_prefix_count": 8, "ipv6_prefix_count": 0, "what_it_is": "Builds Perplexity's search index. Perplexity is citation-heavy by product design, so inclusion here converts to referral traffic more directly than most AI surfaces.", "cost_of_blocking": "You stop being indexed and cited by Perplexity, and lose the referral clicks its citations produce.", "operator_docs": "https://docs.perplexity.ai/guides/bots", "html_url": "https://www.pathwren.workers.dev/crawler/perplexitybot.html", "json_url": "https://www.pathwren.workers.dev/crawler/perplexitybot.json", "last_reviewed": "2026-09-01" }, { "slug": "perplexity-user", "name": "Perplexity-User", "operator": "Perplexity", "operator_slug": "perplexity", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "Perplexity-User", "user_agent_substring": "Perplexity-User", "user_agent_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://www.perplexity.ai/perplexity-user.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/perplexity-user.json", "ipv4_prefix_count": 4, "ipv6_prefix_count": 0, "what_it_is": "Fetches a page because a Perplexity user asked for it. Perplexity documents that this fetch is user-initiated and is therefore not governed by robots.txt — a robots rule will not stop it, by stated policy.", "cost_of_blocking": "Not controllable via robots.txt. If you must stop it, verify by the published IP ranges and block at the edge — and accept that users who ask for your page get an error.", "operator_docs": "https://docs.perplexity.ai/guides/bots", "html_url": "https://www.pathwren.workers.dev/crawler/perplexity-user.html", "json_url": "https://www.pathwren.workers.dev/crawler/perplexity-user.json", "last_reviewed": "2026-09-01" }, { "slug": "ccbot", "name": "CCBot", "operator": "Common Crawl", "operator_slug": "commoncrawl", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "CCBot", "user_agent_substring": "CCBot", "user_agent_example": "CCBot/2.0 (https://commoncrawl.org/faq/)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://index.commoncrawl.org/ccbot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/commoncrawl-ccbot.json", "ipv4_prefix_count": 4, "ipv6_prefix_count": 1, "what_it_is": "Common Crawl's corpus builder. It trains nothing itself, but its archive is an input to most open and many closed LLM training sets, which makes it the highest-leverage single entry on this list.", "cost_of_blocking": "Future Common Crawl snapshots exclude you, so downstream training sets lose you too — but only going forward. Existing snapshots are permanent and blocking today does not retract them.", "operator_docs": "https://commoncrawl.org/ccbot", "html_url": "https://www.pathwren.workers.dev/crawler/ccbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/ccbot.json", "last_reviewed": "2026-09-01" }, { "slug": "bytespider", "name": "Bytespider", "operator": "ByteDance", "operator_slug": "bytedance", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "Bytespider", "user_agent_substring": "Bytespider", "user_agent_example": "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)", "respects_robots_txt": "disputed", "respects_robots_txt_label": "compliance disputed", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "ByteDance's crawler, associated with training data collection for Doubao and related models. Repeatedly reported by CDNs and site operators as the highest-volume AI crawler on the web and as inconsistent about robots.txt.", "cost_of_blocking": "Little to lose. If you want it gone, expect to block by user-agent at the edge rather than to ask politely in robots.txt.", "operator_docs": "https://www.bytespider.net/", "html_url": "https://www.pathwren.workers.dev/crawler/bytespider.html", "json_url": "https://www.pathwren.workers.dev/crawler/bytespider.json", "last_reviewed": "2026-09-01" }, { "slug": "tiktokspider", "name": "TikTokSpider", "operator": "ByteDance", "operator_slug": "bytedance", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "TikTokSpider", "user_agent_substring": "TikTokSpider", "user_agent_example": "Mozilla/5.0 (compatible; TikTokSpider; ttspider-feedback@tiktok.com)", "respects_robots_txt": "disputed", "respects_robots_txt_label": "compliance disputed", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "A second ByteDance crawler identifying with TikTok, collecting page content for the same family of models.", "cost_of_blocking": "Little to lose unless TikTok search referral matters to you.", "operator_docs": "https://www.bytespider.net/", "html_url": "https://www.pathwren.workers.dev/crawler/tiktokspider.html", "json_url": "https://www.pathwren.workers.dev/crawler/tiktokspider.json", "last_reviewed": "2026-09-01" }, { "slug": "meta-externalagent", "name": "meta-externalagent", "operator": "Meta", "operator_slug": "meta", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "meta-externalagent", "user_agent_substring": "meta-externalagent", "user_agent_example": "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Meta's AI crawler, gathering training data for Llama and Meta AI. It replaced the older FacebookBot name for this purpose.", "cost_of_blocking": "Excluded from Meta AI training. Link previews on Facebook, Instagram and WhatsApp are unaffected — those are a different bot.", "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/meta-externalagent.html", "json_url": "https://www.pathwren.workers.dev/crawler/meta-externalagent.json", "last_reviewed": "2026-09-01" }, { "slug": "meta-externalfetcher", "name": "meta-externalfetcher", "operator": "Meta", "operator_slug": "meta", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "meta-externalfetcher", "user_agent_substring": "meta-externalfetcher", "user_agent_example": "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches a page when a Meta AI user asks about a specific link.", "cost_of_blocking": "Meta AI cannot read pages users hand it.", "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.html", "json_url": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.json", "last_reviewed": "2026-09-01" }, { "slug": "facebookexternalhit", "name": "facebookexternalhit", "operator": "Meta", "operator_slug": "meta", "category": "preview", "category_label": "Link preview fetchers", "robots_token": "facebookexternalhit", "user_agent_substring": "facebookexternalhit", "user_agent_example": "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The link unfurler: it reads your Open Graph tags when somebody shares your URL on a Meta property.", "cost_of_blocking": "Severe and usually accidental. Your links share as bare grey boxes with no title, image or description across Facebook, Instagram, Messenger and WhatsApp. Almost nobody means to block this.", "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/facebookexternalhit.html", "json_url": "https://www.pathwren.workers.dev/crawler/facebookexternalhit.json", "last_reviewed": "2026-09-01" }, { "slug": "facebookbot", "name": "FacebookBot", "operator": "Meta", "operator_slug": "meta", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "FacebookBot", "user_agent_substring": "FacebookBot", "user_agent_example": "FacebookBot/1.0 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Meta's older speech- and language-corpus crawler, largely superseded by meta-externalagent but still listed as a valid robots token.", "cost_of_blocking": "Negligible today. Keep the rule; expect little traffic.", "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/facebookbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/facebookbot.json", "last_reviewed": "2026-09-01" }, { "slug": "amazonbot", "name": "Amazonbot", "operator": "Amazon", "operator_slug": "amazon", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "Amazonbot", "user_agent_substring": "Amazonbot", "user_agent_example": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Amazon's crawler, feeding Alexa's ability to answer questions from the web and Amazon's own search and assistant products.", "cost_of_blocking": "Alexa and Amazon's assistants stop answering from your pages. Verify with reverse DNS to crawl.amazonbot.amazon before trusting the user-agent.", "operator_docs": "https://developer.amazon.com/amazonbot", "html_url": "https://www.pathwren.workers.dev/crawler/amazonbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/amazonbot.json", "last_reviewed": "2026-09-01" }, { "slug": "duckassistbot", "name": "DuckAssistBot", "operator": "DuckDuckGo", "operator_slug": "duckduckgo", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "DuckAssistBot", "user_agent_substring": "DuckAssistBot", "user_agent_example": "Mozilla/5.0 (compatible; DuckAssistBot/1.0; +https://duckduckgo.com/duckassistbot)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches pages so DuckAssist can generate and cite answers inside DuckDuckGo.", "cost_of_blocking": "No DuckAssist answers or citations from your site. Ordinary DuckDuckGo results are unaffected.", "operator_docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/", "html_url": "https://www.pathwren.workers.dev/crawler/duckassistbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/duckassistbot.json", "last_reviewed": "2026-09-01" }, { "slug": "duckduckbot", "name": "DuckDuckBot", "operator": "DuckDuckGo", "operator_slug": "duckduckgo", "category": "search", "category_label": "Search engines", "robots_token": "DuckDuckBot", "user_agent_substring": "DuckDuckBot", "user_agent_example": "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://duckduckgo.com/duckduckbot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/duckduckgo-duckduckbot.json", "ipv4_prefix_count": 486, "ipv6_prefix_count": 0, "what_it_is": "DuckDuckGo's own crawler. Note that the bulk of DuckDuckGo's web results come from Bing, so blocking bingbot removes you from DuckDuckGo whether or not you allow this one.", "cost_of_blocking": "Limited on its own; the real DuckDuckGo lever is bingbot.", "operator_docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/", "html_url": "https://www.pathwren.workers.dev/crawler/duckduckbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/duckduckbot.json", "last_reviewed": "2026-09-01" }, { "slug": "ai2bot", "name": "AI2Bot", "operator": "Allen Institute for AI", "operator_slug": "ai2", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "AI2Bot", "user_agent_substring": "AI2Bot", "user_agent_example": "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The Allen Institute's crawler, gathering pages for open research corpora such as Dolma that underpin fully open models like OLMo.", "cost_of_blocking": "Excluded from open research datasets. Worth a deliberate decision: this is the category where 'blocking AI' also blocks the open, auditable end of it.", "operator_docs": "https://allenai.org/crawler", "html_url": "https://www.pathwren.workers.dev/crawler/ai2bot.html", "json_url": "https://www.pathwren.workers.dev/crawler/ai2bot.json", "last_reviewed": "2026-09-01" }, { "slug": "ai2bot-dolma", "name": "Ai2Bot-Dolma", "operator": "Allen Institute for AI", "operator_slug": "ai2", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "Ai2Bot-Dolma", "user_agent_substring": "Ai2Bot-Dolma", "user_agent_example": "Mozilla/5.0 (compatible) Ai2Bot-Dolma (+https://www.allenai.org/crawler)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The variant of AI2's crawler named for the Dolma corpus specifically.", "cost_of_blocking": "Same as AI2Bot: exclusion from an open, published training corpus.", "operator_docs": "https://allenai.org/crawler", "html_url": "https://www.pathwren.workers.dev/crawler/ai2bot-dolma.html", "json_url": "https://www.pathwren.workers.dev/crawler/ai2bot-dolma.json", "last_reviewed": "2026-09-01" }, { "slug": "cohere-ai", "name": "cohere-ai", "operator": "Cohere", "operator_slug": "cohere", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "cohere-ai", "user_agent_substring": "cohere-ai", "user_agent_example": "Mozilla/5.0 (compatible; cohere-ai)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Cohere's fetcher, used when its assistant products need a page.", "cost_of_blocking": "Cohere-powered assistants cannot read your pages on request.", "operator_docs": "https://cohere.com/", "html_url": "https://www.pathwren.workers.dev/crawler/cohere-ai.html", "json_url": "https://www.pathwren.workers.dev/crawler/cohere-ai.json", "last_reviewed": "2026-09-01" }, { "slug": "cohere-training-data-crawler", "name": "cohere-training-data-crawler", "operator": "Cohere", "operator_slug": "cohere", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "cohere-training-data-crawler", "user_agent_substring": "cohere-training-data-crawler", "user_agent_example": "Mozilla/5.0 (compatible; cohere-training-data-crawler)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Cohere's separately-named bulk crawler for model training data, split out so consent for training and consent for retrieval can differ.", "cost_of_blocking": "Excluded from Cohere model training.", "operator_docs": "https://cohere.com/", "html_url": "https://www.pathwren.workers.dev/crawler/cohere-training-data-crawler.html", "json_url": "https://www.pathwren.workers.dev/crawler/cohere-training-data-crawler.json", "last_reviewed": "2026-09-01" }, { "slug": "mistralai-user", "name": "MistralAI-User", "operator": "Mistral AI", "operator_slug": "mistral", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "MistralAI-User", "user_agent_substring": "MistralAI-User", "user_agent_example": "Mozilla/5.0 (compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches a page when a Le Chat user asks Mistral's assistant to read it.", "cost_of_blocking": "Le Chat cannot open links your readers give it.", "operator_docs": "https://docs.mistral.ai/", "html_url": "https://www.pathwren.workers.dev/crawler/mistralai-user.html", "json_url": "https://www.pathwren.workers.dev/crawler/mistralai-user.json", "last_reviewed": "2026-09-01" }, { "slug": "youbot", "name": "YouBot", "operator": "You.com", "operator_slug": "you", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "YouBot", "user_agent_substring": "YouBot", "user_agent_example": "Mozilla/5.0 (compatible; YouBot (+http://www.you.com))", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "You.com's crawler, feeding its AI search product and its search API.", "cost_of_blocking": "Removal from You.com's index and from answers built on its API.", "operator_docs": "https://about.you.com/youbot/", "html_url": "https://www.pathwren.workers.dev/crawler/youbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/youbot.json", "last_reviewed": "2026-09-01" }, { "slug": "diffbot", "name": "Diffbot", "operator": "Diffbot", "operator_slug": "diffbot", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "Diffbot", "user_agent_substring": "Diffbot", "user_agent_example": "Mozilla/5.0 (compatible; Diffbot/0.1; +http://www.diffbot.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Extracts structured records from pages to build a commercial knowledge graph that is resold and used for retrieval and training.", "cost_of_blocking": "Your facts stop entering a widely-licensed knowledge graph. Whether that is a loss depends on whether you want to be a machine-readable entity.", "operator_docs": "https://docs.diffbot.com/", "html_url": "https://www.pathwren.workers.dev/crawler/diffbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/diffbot.json", "last_reviewed": "2026-09-01" }, { "slug": "omgilibot", "name": "omgilibot", "operator": "Webz.io", "operator_slug": "webz", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "omgilibot", "user_agent_substring": "omgilibot", "user_agent_example": "omgilibot/0.4; +http://omgili.com", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Webz.io's crawler, collecting web and forum text sold as datasets, including to model builders.", "cost_of_blocking": "Exclusion from a commercial dataset resold to third parties.", "operator_docs": "https://webz.io/blog/machine-learning/", "html_url": "https://www.pathwren.workers.dev/crawler/omgilibot.html", "json_url": "https://www.pathwren.workers.dev/crawler/omgilibot.json", "last_reviewed": "2026-09-01" }, { "slug": "omgili", "name": "omgili", "operator": "Webz.io", "operator_slug": "webz", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "omgili", "user_agent_substring": "omgili", "user_agent_example": "omgili/0.5 +http://omgili.com", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The older robots token for the same Webz.io collection, still honoured and still worth listing.", "cost_of_blocking": "Same as omgilibot.", "operator_docs": "https://webz.io/blog/machine-learning/", "html_url": "https://www.pathwren.workers.dev/crawler/omgili.html", "json_url": "https://www.pathwren.workers.dev/crawler/omgili.json", "last_reviewed": "2026-09-01" }, { "slug": "webzio-extended", "name": "Webzio-Extended", "operator": "Webz.io", "operator_slug": "webz", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "Webzio-Extended", "user_agent_substring": "Webzio-Extended", "user_agent_example": "Mozilla/5.0 (compatible; Webzio-Extended/1.0)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Webz.io's opt-out token specifically for AI training reuse, in the pattern Google and Apple established.", "cost_of_blocking": "Your content is excluded from the AI-training tier of Webz.io's product while ordinary collection continues.", "operator_docs": "https://webz.io/blog/machine-learning/", "html_url": "https://www.pathwren.workers.dev/crawler/webzio-extended.html", "json_url": "https://www.pathwren.workers.dev/crawler/webzio-extended.json", "last_reviewed": "2026-09-01" }, { "slug": "imagesiftbot", "name": "ImagesiftBot", "operator": "Hive AI", "operator_slug": "hive", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "ImagesiftBot", "user_agent_substring": "ImagesiftBot", "user_agent_example": "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Crawls images for Hive AI's reverse-image and dataset products. Image-heavy sites see this one long before they see the text crawlers.", "cost_of_blocking": "Your images stop entering an image dataset and reverse-image index.", "operator_docs": "https://imagesift.com/about", "html_url": "https://www.pathwren.workers.dev/crawler/imagesiftbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/imagesiftbot.json", "last_reviewed": "2026-09-01" }, { "slug": "timpibot", "name": "Timpibot", "operator": "Timpi", "operator_slug": "timpi", "category": "search", "category_label": "Search engines", "robots_token": "Timpibot", "user_agent_substring": "Timpibot", "user_agent_example": "Mozilla/5.0 (compatible; Timpibot/0.1; +https://timpi.io)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "A distributed crawler building an independent search index outside the Google/Bing duopoly.", "cost_of_blocking": "Absence from a small independent index.", "operator_docs": "https://timpi.io/", "html_url": "https://www.pathwren.workers.dev/crawler/timpibot.html", "json_url": "https://www.pathwren.workers.dev/crawler/timpibot.json", "last_reviewed": "2026-09-01" }, { "slug": "semrushbot", "name": "SemrushBot", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SemrushBot", "user_agent_substring": "SemrushBot", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Semrush's backlink and keyword crawler. It is not an AI crawler, but it is usually in the top three by volume on any site, and it is the cheapest block on this list.", "cost_of_blocking": "Your competitors' Semrush reports get thinner, and so do yours. No user-facing effect.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot.json", "last_reviewed": "2026-09-01" }, { "slug": "semrushbot-ocob", "name": "SemrushBot-OCOB", "operator": "Semrush", "operator_slug": "semrush", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "SemrushBot-OCOB", "user_agent_substring": "SemrushBot-OCOB", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-OCOB/1.0; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Semrush's separately-tokenised crawler for its AI content tooling, split out so SEO crawling and AI reuse can be answered differently.", "cost_of_blocking": "Exclusion from Semrush's AI corpus, with its SEO crawl unaffected.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ocob.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ocob.json", "last_reviewed": "2026-09-01" }, { "slug": "ahrefsbot", "name": "AhrefsBot", "operator": "Ahrefs", "operator_slug": "ahrefs", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "AhrefsBot", "user_agent_substring": "AhrefsBot", "user_agent_example": "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/ahrefs-crawler.json", "ipv4_prefix_count": 81, "ipv6_prefix_count": 0, "what_it_is": "Ahrefs' backlink crawler, and one of the largest non-search crawlers on the web by request volume.", "cost_of_blocking": "No user-facing effect. Ahrefs honours Crawl-delay, so rate-limiting is usually better than blocking.", "operator_docs": "https://ahrefs.com/robot", "html_url": "https://www.pathwren.workers.dev/crawler/ahrefsbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/ahrefsbot.json", "last_reviewed": "2026-09-01" }, { "slug": "archive-org-bot", "name": "archive.org_bot", "operator": "Internet Archive", "operator_slug": "internetarchive", "category": "archive", "category_label": "Archivers", "robots_token": "archive.org_bot", "user_agent_substring": "archive.org_bot", "user_agent_example": "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The Wayback Machine's crawler. Preservation rather than AI, but it lands in the same 'is this bot welcome' decision and its output is a public corpus.", "cost_of_blocking": "Your site stops being preserved. When it dies, it is gone. Consider this one separately from the AI question.", "operator_docs": "https://archive.org/details/archive.org_bot", "html_url": "https://www.pathwren.workers.dev/crawler/archive-org-bot.html", "json_url": "https://www.pathwren.workers.dev/crawler/archive-org-bot.json", "last_reviewed": "2026-09-01" }, { "slug": "ia-archiver", "name": "ia_archiver", "operator": "Internet Archive", "operator_slug": "internetarchive", "category": "archive", "category_label": "Archivers", "robots_token": "ia_archiver", "user_agent_substring": "ia_archiver", "user_agent_example": "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The legacy Alexa/Internet Archive token, still present in most robots.txt files and still occasionally honoured.", "cost_of_blocking": "Negligible today; retain for tidiness.", "operator_docs": "https://archive.org/details/archive.org_bot", "html_url": "https://www.pathwren.workers.dev/crawler/ia-archiver.html", "json_url": "https://www.pathwren.workers.dev/crawler/ia-archiver.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexbot", "name": "YandexBot", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexBot", "user_agent_substring": "YandexBot", "user_agent_example": "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Yandex's search crawler, which also feeds Alice and Yandex's generative answers.", "cost_of_blocking": "Removal from Yandex Search. Verify with reverse DNS to a yandex.ru, yandex.net or yandex.com host — YandexBot is among the most-spoofed user-agents there is.", "operator_docs": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html", "html_url": "https://www.pathwren.workers.dev/crawler/yandexbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexbot.json", "last_reviewed": "2026-09-01" }, { "slug": "baiduspider", "name": "Baiduspider", "operator": "Baidu", "operator_slug": "baidu", "category": "search", "category_label": "Search engines", "robots_token": "Baiduspider", "user_agent_substring": "Baiduspider", "user_agent_example": "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Baidu's search crawler, and the ingest path for Baidu's Ernie-backed answers.", "cost_of_blocking": "Removal from Baidu Search, which matters only if you want Chinese-language traffic.", "operator_docs": "https://help.baidu.com/question?prod_id=99&class=0&id=3001", "html_url": "https://www.pathwren.workers.dev/crawler/baiduspider.html", "json_url": "https://www.pathwren.workers.dev/crawler/baiduspider.json", "last_reviewed": "2026-09-01" }, { "slug": "seznambot", "name": "SeznamBot", "operator": "Seznam", "operator_slug": "seznam", "category": "search", "category_label": "Search engines", "robots_token": "SeznamBot", "user_agent_substring": "SeznamBot", "user_agent_example": "Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/en/seznambot-intro/)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Seznam's crawler — the dominant search engine in the Czech Republic and one of the few national engines with its own index.", "cost_of_blocking": "Removal from Seznam. Also removes you from its IndexNow endpoint's usefulness.", "operator_docs": "https://napoveda.seznam.cz/en/seznamzbozi/subject-matter-crawler/", "html_url": "https://www.pathwren.workers.dev/crawler/seznambot.html", "json_url": "https://www.pathwren.workers.dev/crawler/seznambot.json", "last_reviewed": "2026-09-01" }, { "slug": "yeti", "name": "Yeti", "operator": "Naver", "operator_slug": "naver", "category": "search", "category_label": "Search engines", "robots_token": "Yeti", "user_agent_substring": "Yeti", "user_agent_example": "Mozilla/5.0 (compatible; Yeti/1.1; +https://naver.me/spd)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Naver's crawler. Naver is South Korea's largest search portal and runs its own index and its own generative answers.", "cost_of_blocking": "Removal from Naver, which is most of Korean search.", "operator_docs": "https://searchadvisor.naver.com/guide/seo-basic-crawl", "html_url": "https://www.pathwren.workers.dev/crawler/yeti.html", "json_url": "https://www.pathwren.workers.dev/crawler/yeti.json", "last_reviewed": "2026-09-01" }, { "slug": "petalbot", "name": "PetalBot", "operator": "Huawei", "operator_slug": "huawei", "category": "search", "category_label": "Search engines", "robots_token": "PetalBot", "user_agent_substring": "PetalBot", "user_agent_example": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Huawei's crawler for Petal Search, shipped as the default search on Huawei devices.", "cost_of_blocking": "Removal from Petal Search. Frequently blocked for volume rather than for policy.", "operator_docs": "https://aspiegel.com/petalbot", "html_url": "https://www.pathwren.workers.dev/crawler/petalbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/petalbot.json", "last_reviewed": "2026-09-01" }, { "slug": "firecrawlagent", "name": "FirecrawlAgent", "operator": "Firecrawl", "operator_slug": "firecrawl", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "FirecrawlAgent", "user_agent_substring": "FirecrawlAgent", "user_agent_example": "Mozilla/5.0 (compatible; FirecrawlAgent/1.0; +https://firecrawl.dev)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "A hosted scrape-to-markdown service that LLM applications call to read pages. The requester is whoever is building on it, not Firecrawl itself, so volume and intent vary wildly.", "cost_of_blocking": "Applications built on Firecrawl cannot read your pages. This is increasingly how agents fetch the web, so it is a bigger block than its name suggests.", "operator_docs": "https://docs.firecrawl.dev/", "html_url": "https://www.pathwren.workers.dev/crawler/firecrawlagent.html", "json_url": "https://www.pathwren.workers.dev/crawler/firecrawlagent.json", "last_reviewed": "2026-09-01" }, { "slug": "scrapy", "name": "Scrapy", "operator": "Scrapy project", "operator_slug": "scrapy", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "Scrapy", "user_agent_substring": "Scrapy", "user_agent_example": "Scrapy/2.11.0 (+https://scrapy.org)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Not an operator: the default user-agent of the most common Python crawling framework. Anyone can be behind it. Modern Scrapy obeys robots.txt by default, which is why the default UA is still worth a rule.", "cost_of_blocking": "You block a very large tail of unattributed one-off crawlers, and also every well-behaved researcher who did not change the default.", "operator_docs": "https://scrapy.org/", "html_url": "https://www.pathwren.workers.dev/crawler/scrapy.html", "json_url": "https://www.pathwren.workers.dev/crawler/scrapy.json", "last_reviewed": "2026-09-01" }, { "slug": "img2dataset", "name": "img2dataset", "operator": "LAION / img2dataset", "operator_slug": "laion", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "img2dataset", "user_agent_substring": "img2dataset", "user_agent_example": "img2dataset", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The tool used to turn image-URL lists such as LAION's into downloaded training sets. It is run by whoever is building a dataset, not by a single operator.", "cost_of_blocking": "Your images are skipped when someone materialises an image-text dataset that references them.", "operator_docs": "https://github.com/rom1504/img2dataset", "html_url": "https://www.pathwren.workers.dev/crawler/img2dataset.html", "json_url": "https://www.pathwren.workers.dev/crawler/img2dataset.json", "last_reviewed": "2026-09-01" }, { "slug": "googlebot-video", "name": "Googlebot-Video", "operator": "Google", "operator_slug": "google", "category": "search", "category_label": "Search engines", "robots_token": "Googlebot-Video", "user_agent_substring": "Googlebot-Video", "user_agent_example": "Googlebot-Video/1.0", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json", "ipv4_prefix_count": 170, "ipv6_prefix_count": 147, "what_it_is": "The video half of Googlebot. It crawls video files and the pages around them for Google Video search, and it is matched by a robots.txt group for Googlebot as well as by its own token.", "cost_of_blocking": "Your videos leave Google video search. A rule for Googlebot already covers it, so blocking this token alone is usually a mistake of precision rather than of intent.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/googlebot-video.html", "json_url": "https://www.pathwren.workers.dev/crawler/googlebot-video.json", "last_reviewed": "2026-09-01" }, { "slug": "googleother-image", "name": "GoogleOther-Image", "operator": "Google", "operator_slug": "google", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "GoogleOther-Image", "user_agent_substring": "GoogleOther-Image", "user_agent_example": "GoogleOther-Image/1.0", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json", "ipv4_prefix_count": 136, "ipv6_prefix_count": 136, "what_it_is": "The image variant of GoogleOther: one-off fetches by Google product and research teams that are not Search. It also answers to a GoogleOther group in robots.txt.", "cost_of_blocking": "Google teams outside Search stop fetching your images. Image Search itself is unaffected — that is Googlebot-Image.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/googleother-image.html", "json_url": "https://www.pathwren.workers.dev/crawler/googleother-image.json", "last_reviewed": "2026-09-01" }, { "slug": "googleother-video", "name": "GoogleOther-Video", "operator": "Google", "operator_slug": "google", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "GoogleOther-Video", "user_agent_substring": "GoogleOther-Video", "user_agent_example": "GoogleOther-Video/1.0", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json", "ipv4_prefix_count": 136, "ipv6_prefix_count": 136, "what_it_is": "The video variant of GoogleOther, used for internal Google fetches that do not belong to Search.", "cost_of_blocking": "No effect on Search or on Google Video search. Blocks internal Google research fetches of your video files.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/googleother-video.html", "json_url": "https://www.pathwren.workers.dev/crawler/googleother-video.json", "last_reviewed": "2026-09-01" }, { "slug": "apis-google", "name": "APIs-Google", "operator": "Google", "operator_slug": "google", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "APIs-Google", "user_agent_substring": "APIs-Google", "user_agent_example": "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json", "ipv4_prefix_count": 136, "ipv6_prefix_count": 136, "what_it_is": "Delivers push notifications for Google APIs to a webhook you registered. It is a special-case crawler: it ignores the robots.txt * group, because the fetch is a delivery to an address you asked it to deliver to.", "cost_of_blocking": "Google API push notifications stop arriving at your endpoint. This only affects services you set up yourself; there is no search or AI consequence.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/apis-google.html", "json_url": "https://www.pathwren.workers.dev/crawler/apis-google.json", "last_reviewed": "2026-09-01" }, { "slug": "adsbot-google", "name": "AdsBot-Google", "operator": "Google", "operator_slug": "google", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "AdsBot-Google", "user_agent_substring": "AdsBot-Google", "user_agent_example": "AdsBot-Google (+http://www.google.com/adsbot.html)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json", "ipv4_prefix_count": 136, "ipv6_prefix_count": 136, "what_it_is": "Checks the quality of desktop landing pages for Google Ads. Google documents that it ignores the robots.txt * group with the ad publisher's permission, and obeys a group named for its own token.", "cost_of_blocking": "Google Ads cannot score your landing pages, which lowers Ad Rank on the ads pointing at them. If you do not buy ads, blocking it costs nothing but bandwidth savings.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/adsbot-google.html", "json_url": "https://www.pathwren.workers.dev/crawler/adsbot-google.json", "last_reviewed": "2026-09-01" }, { "slug": "adsbot-google-mobile", "name": "AdsBot-Google-Mobile", "operator": "Google", "operator_slug": "google", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "AdsBot-Google-Mobile", "user_agent_substring": "AdsBot-Google-Mobile", "user_agent_example": "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json", "ipv4_prefix_count": 136, "ipv6_prefix_count": 136, "what_it_is": "The mobile-web landing page checker for Google Ads. Same rules as AdsBot-Google: the * group does not apply to it, its own token does.", "cost_of_blocking": "Mobile ad landing pages go unscored and the ads pointing at them rank worse. No effect on organic search.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile.html", "json_url": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile.json", "last_reviewed": "2026-09-01" }, { "slug": "adsbot-google-mobile-apps", "name": "AdsBot-Google-Mobile-Apps", "operator": "Google", "operator_slug": "google", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "AdsBot-Google-Mobile-Apps", "user_agent_substring": "AdsBot-Google-Mobile-Apps", "user_agent_example": "AdsBot-Google-Mobile-Apps", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json", "ipv4_prefix_count": 136, "ipv6_prefix_count": 136, "what_it_is": "Checks Android app landing pages for Google Ads. It obeys a group named for its own token and, per Google, follows the AdsBot-Google rules otherwise.", "cost_of_blocking": "App-install ad landing pages go unscored. Nothing organic changes.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile-apps.html", "json_url": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile-apps.json", "last_reviewed": "2026-09-01" }, { "slug": "mediapartners-google", "name": "Mediapartners-Google", "operator": "Google", "operator_slug": "google", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "Mediapartners-Google", "user_agent_substring": "Mediapartners-Google", "user_agent_example": "Mediapartners-Google", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json", "ipv4_prefix_count": 136, "ipv6_prefix_count": 136, "what_it_is": "The AdSense crawler. It reads a page so AdSense can choose relevant ads for it, and it is a special-case crawler that ignores the robots.txt * group.", "cost_of_blocking": "Pages it cannot read get generic, lower-value AdSense ads or none at all. This is the one block on this list that costs you money directly if you run AdSense.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/mediapartners-google.html", "json_url": "https://www.pathwren.workers.dev/crawler/mediapartners-google.json", "last_reviewed": "2026-09-01" }, { "slug": "google-safety", "name": "Google-Safety", "operator": "Google", "operator_slug": "google", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "Google-Safety", "user_agent_substring": "Google-Safety", "user_agent_example": "Google-Safety", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Google's abuse-investigation fetcher: malware review, phishing reports and similar. Google documents that it ignores robots.txt entirely, and a robots.txt rule for it does nothing.", "cost_of_blocking": "Nothing you can control. The rule is ignored by design; listing the token is documentation, not enforcement.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/google-safety.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-safety.json", "last_reviewed": "2026-09-01" }, { "slug": "feedfetcher-google", "name": "FeedFetcher-Google", "operator": "Google", "operator_slug": "google", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "FeedFetcher-Google", "user_agent_substring": "FeedFetcher-Google", "user_agent_example": "FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json", "ipv4_prefix_count": 529, "ipv6_prefix_count": 529, "what_it_is": "Crawls RSS and Atom feeds for Google News and WebSub. It is a user-triggered fetcher, and Google documents that those generally ignore robots.txt because a person asked for the fetch. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "cost_of_blocking": "Feed-driven Google products stop seeing your updates. A robots.txt rule will not stop it — block by user-agent at the edge if you mean it.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/feedfetcher-google.html", "json_url": "https://www.pathwren.workers.dev/crawler/feedfetcher-google.json", "last_reviewed": "2026-09-01" }, { "slug": "google-read-aloud", "name": "Google-Read-Aloud", "operator": "Google", "operator_slug": "google", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "Google-Read-Aloud", "user_agent_substring": "Google-Read-Aloud", "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 (compatible; Google-Read-Aloud; +https://support.google.com/webmasters/answer/1061943)", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json", "ipv4_prefix_count": 529, "ipv6_prefix_count": 529, "what_it_is": "Fetches a page so Google can read it out loud with text-to-speech, at the moment a user asks. Formerly google-speakr. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "cost_of_blocking": "A reader who asked Google to read your page aloud — often someone using it for accessibility — gets an error instead.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/google-read-aloud.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-read-aloud.json", "last_reviewed": "2026-09-01" }, { "slug": "google-site-verification", "name": "Google-Site-Verification", "operator": "Google", "operator_slug": "google", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "Google-Site-Verification", "user_agent_substring": "Google-Site-Verification", "user_agent_example": "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json", "ipv4_prefix_count": 529, "ipv6_prefix_count": 529, "what_it_is": "Fetches the token file or meta tag that proves you own a site, when you click verify in Search Console. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "cost_of_blocking": "Your own Search Console verification fails. Blocking this only ever hurts the person doing the blocking.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/google-site-verification.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-site-verification.json", "last_reviewed": "2026-09-01" }, { "slug": "google-cws", "name": "Google-CWS", "operator": "Google", "operator_slug": "google", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "Google-CWS", "user_agent_substring": "Google-CWS", "user_agent_example": "Mozilla/5.0 (compatible; Google-CWS)", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json", "ipv4_prefix_count": 529, "ipv6_prefix_count": 529, "what_it_is": "The Chrome Web Store fetcher. It requests the URLs a developer put in the metadata of a Chrome extension or theme. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "cost_of_blocking": "Chrome Web Store listings that point at your pages cannot fetch them. Relevant only if you publish extensions.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/google-cws.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-cws.json", "last_reviewed": "2026-09-01" }, { "slug": "google-pinpoint", "name": "Google-Pinpoint", "operator": "Google", "operator_slug": "google", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "Google-Pinpoint", "user_agent_substring": "Google-Pinpoint", "user_agent_example": "Google-Pinpoint", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json", "ipv4_prefix_count": 529, "ipv6_prefix_count": 529, "what_it_is": "Fetches individual URLs that a Pinpoint user — usually a journalist or researcher — added as a source to their own document collection. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "cost_of_blocking": "A researcher who explicitly added your page to a collection cannot load it.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/google-pinpoint.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-pinpoint.json", "last_reviewed": "2026-09-01" }, { "slug": "googleproducer", "name": "GoogleProducer", "operator": "Google", "operator_slug": "google", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "GoogleProducer", "user_agent_substring": "GoogleProducer", "user_agent_example": "GoogleProducer; (+https://developers.google.com/search/docs/crawling-indexing/google-producer)", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json", "ipv4_prefix_count": 529, "ipv6_prefix_count": 529, "what_it_is": "Google Publisher Center: fetches the feeds a publisher explicitly supplied for Google News landing pages. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "cost_of_blocking": "Your own Google News landing pages stop updating. Only publishers who configured Publisher Center are affected.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/googleproducer.html", "json_url": "https://www.pathwren.workers.dev/crawler/googleproducer.json", "last_reviewed": "2026-09-01" }, { "slug": "googlemessages", "name": "GoogleMessages", "operator": "Google", "operator_slug": "google", "category": "preview", "category_label": "Link preview fetchers", "robots_token": "GoogleMessages", "user_agent_substring": "GoogleMessages", "user_agent_example": "GoogleMessages", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json", "ipv4_prefix_count": 529, "ipv6_prefix_count": 529, "what_it_is": "Generates the link preview when somebody sends one of your URLs in Google Messages. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "cost_of_blocking": "Your links appear as bare URLs with no title or image in Google Messages chats. A preview fetcher is almost never the one you meant to block.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/googlemessages.html", "json_url": "https://www.pathwren.workers.dev/crawler/googlemessages.json", "last_reviewed": "2026-09-01" }, { "slug": "google-gemininotebook", "name": "Google-GeminiNotebook", "operator": "Google", "operator_slug": "google", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "Google-GeminiNotebook", "user_agent_substring": "Google-GeminiNotebook", "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 (compatible; Google-GeminiNotebook; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-gemininotebook)", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json", "ipv4_prefix_count": 529, "ipv6_prefix_count": 529, "what_it_is": "Fetches a URL a Gemini Notebook (formerly NotebookLM) user added as a source to their notebook. The former agent string Google-NotebookLM is documented as supported until August 2026. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "cost_of_blocking": "A user who deliberately added your page as a research source gets nothing. This is a citation-shaped fetch, not a training crawl.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/google-gemininotebook.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-gemininotebook.json", "last_reviewed": "2026-09-01" }, { "slug": "google-agent", "name": "Google-Agent", "operator": "Google", "operator_slug": "google", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "Google-Agent", "user_agent_substring": "Google-Agent", "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent) Chrome/W.X.Y.Z Safari/537.36", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-agents.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered-agents.json", "ipv4_prefix_count": 3, "ipv6_prefix_count": 1, "what_it_is": "Agents hosted on Google infrastructure navigating the web and taking actions on a user's request. Google names one prefix list for it — user-triggered-agents.json — and is separately experimenting with Web Bot Auth under the identity https://agent.bot.goog.", "cost_of_blocking": "Google-hosted agents cannot complete a task on your site for a user who asked them to. This is the agentic-commerce fetch: blocking it removes you from what an assistant can actually do rather than from what it can say.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/google-agent.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-agent.json", "last_reviewed": "2026-09-01" }, { "slug": "yandeximages", "name": "YandexImages", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexImages", "user_agent_substring": "YandexImages", "user_agent_example": "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Indexes images for Yandex Images. Yandex's robot table marks it as taking the general robots.txt rules into account.", "cost_of_blocking": "Your images leave Yandex Images, which is a large share of image search in Russian-speaking markets.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandeximages.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandeximages.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexvideo", "name": "YandexVideo", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexVideo", "user_agent_substring": "YandexVideo", "user_agent_example": "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Indexes video for Yandex video search. Obeys the general robots.txt rules per Yandex's own table.", "cost_of_blocking": "Removal from Yandex video search. Note that a second robot, YandexVideoParser, does the same job and is documented as NOT taking the general rules into account.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexvideo.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexvideo.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexmedia", "name": "YandexMedia", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexMedia", "user_agent_substring": "YandexMedia", "user_agent_example": "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Indexes multimedia data for Yandex. Takes the general robots.txt rules into account.", "cost_of_blocking": "Your multimedia content stops appearing in Yandex's media surfaces.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexmedia.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexmedia.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexblogs", "name": "YandexBlogs", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexBlogs", "user_agent_substring": "YandexBlogs", "user_agent_example": "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Yandex's blog-search robot; it indexes post comments as well as posts.", "cost_of_blocking": "Comment threads and blog posts stop being findable through Yandex blog search.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexblogs.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexblogs.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexmarket", "name": "YandexMarket", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexMarket", "user_agent_substring": "YandexMarket", "user_agent_example": "Mozilla/5.0 (compatible; YandexMarket/1.0; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The robot behind Yandex Market, Yandex's shopping comparison service. Version 1.0 is documented as obeying the general rules; version 2.0 is documented as not.", "cost_of_blocking": "Your products stop being listed and priced in Yandex Market. For a retailer in that market this is a revenue block, not a bandwidth one.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexmarket.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexmarket.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexwebmaster", "name": "YandexWebmaster", "operator": "Yandex", "operator_slug": "yandex", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "YandexWebmaster", "user_agent_substring": "YandexWebmaster", "user_agent_example": "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The fetcher behind Yandex Webmaster, the console a site owner uses to inspect their own site.", "cost_of_blocking": "Your own Yandex Webmaster checks stop working. Blocking this only hurts you.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexwebmaster.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexwebmaster.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexmobilebot", "name": "YandexMobileBot", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexMobileBot", "user_agent_substring": "YandexMobileBot", "user_agent_example": "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Decides whether a page's layout is suitable for mobile devices. Yandex's table marks it as NOT taking the general robots.txt rules into account, so a * group does not stop it — a group named YandexMobileBot does.", "cost_of_blocking": "Yandex loses its mobile-friendliness signal for your pages, which affects how they are ranked and rendered on phones.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexmobilebot.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexmobilebot.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexfavicons", "name": "YandexFavicons", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexFavicons", "user_agent_substring": "YandexFavicons", "user_agent_example": "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Downloads your favicon so Yandex can show it beside your result. Documented as not taking the general robots.txt rules into account.", "cost_of_blocking": "Your results in Yandex lose their icon. Cosmetic, and a * rule will not achieve it anyway.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexfavicons.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexfavicons.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexcalendar", "name": "YandexCalendar", "operator": "Yandex", "operator_slug": "yandex", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "YandexCalendar", "user_agent_substring": "YandexCalendar", "user_agent_example": "Mozilla/5.0 (compatible; YandexCalendar/1.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Downloads calendar files a user subscribed to. Yandex notes these files are often in directories that are disallowed for indexing, which is why the general rules are not applied.", "cost_of_blocking": "Users who subscribed to a calendar you publish stop receiving updates.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexcalendar.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexcalendar.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexdirect", "name": "YandexDirect", "operator": "Yandex", "operator_slug": "yandex", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "YandexDirect", "user_agent_substring": "YandexDirect", "user_agent_example": "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Reads the content of Yandex Advertising Network partner pages to work out their topic so relevant ads can be matched. Documented as not taking the general robots.txt rules into account.", "cost_of_blocking": "Ads on your pages become less relevant and earn less. Relevant only if you monetise with Yandex's network.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexdirect.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexdirect.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexmetrika", "name": "YandexMetrika", "operator": "Yandex", "operator_slug": "yandex", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "YandexMetrika", "user_agent_substring": "YandexMetrika", "user_agent_example": "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Yandex Metrica's own fetcher. Two of its versions — the 2.0 yabs01 availability checker and the 4.0 CSS cache for Webvisor — are documented in Yandex's table as not using robots.txt at all.", "cost_of_blocking": "Nothing you can enforce through robots.txt. If you run Metrica, its session replay loses your stylesheets and renders your pages wrong.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexmetrika.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexmetrika.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexrenderresourcesbot", "name": "YandexRenderResourcesBot", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexRenderResourcesBot", "user_agent_substring": "YandexRenderResourcesBot", "user_agent_example": "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Loads the CSS, JavaScript and images Yandex needs to render a page. Yandex documents the exact rule: it ignores robots.txt for a resource when the HTML page using it is allowed, and does not fetch the resource when that page is disallowed.", "cost_of_blocking": "Yandex renders your pages without their stylesheets or scripts and ranks what it sees. This is the classic accidental self-inflicted ranking loss.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexrenderresourcesbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexrenderresourcesbot.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexscreenshotbot", "name": "YandexScreenshotBot", "operator": "Yandex", "operator_slug": "yandex", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "YandexScreenshotBot", "user_agent_substring": "YandexScreenshotBot", "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36 (compatible; YandexScreenshotBot/3.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Takes a screenshot of a page. Documented as not taking the general robots.txt rules into account.", "cost_of_blocking": "Yandex surfaces that show a page thumbnail show nothing for you.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexscreenshotbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexscreenshotbot.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexadditional", "name": "YandexAdditional", "operator": "Yandex", "operator_slug": "yandex", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "YandexAdditional", "user_agent_substring": "YandexAdditional", "user_agent_example": "Mozilla/5.0 (compatible; YandexAdditional/1.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The token that controls whether already-indexed pages may appear in Search with Yandex AI answers. Yandex's table says it makes no indexing requests of its own — it exists so a site can opt out of the generative answer without leaving the index.", "cost_of_blocking": "You disappear from Yandex's AI answers while staying in Yandex Search. This is Yandex's equivalent of Google-Extended, and it is the cheap opt-out most people are looking for.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexadditional.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexadditional.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexadditionalbot", "name": "YandexAdditionalBot", "operator": "Yandex", "operator_slug": "yandex", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "YandexAdditionalBot", "user_agent_substring": "YandexAdditionalBot", "user_agent_example": "Mozilla/5.0 (compatible; YandexAdditionalBot/1.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The second token Yandex publishes for the same AI-answers opt-out. Both names appear in Yandex's own robot list, so a robots.txt that names only one of them is half a policy.", "cost_of_blocking": "Same as YandexAdditional: out of Yandex's AI answers, still in Yandex Search. Name both tokens or neither.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexadditionalbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexadditionalbot.json", "last_reviewed": "2026-09-01" }, { "slug": "yandexcombot", "name": "YandexComBot", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexComBot", "user_agent_substring": "YandexComBot", "user_agent_example": "Mozilla/5.0 (compatible; YandexComBot/3.0; +http://ya.cc/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Indexes content for Yandex search in languages other than Russian. Yandex documents that it can index content when there is no explicit robot-specific restriction — a * group is not one.", "cost_of_blocking": "You leave Yandex's non-Russian index. A rule naming this token is the only one that works on it.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexcombot.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexcombot.json", "last_reviewed": "2026-09-01" }, { "slug": "siteauditbot", "name": "SiteAuditBot", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SiteAuditBot", "user_agent_substring": "SiteAuditBot", "user_agent_example": "Mozilla/5.0 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Semrush's Site Audit crawler: it walks a site a customer owns and reports technical SEO problems. Semrush names it as the token to block for that product.", "cost_of_blocking": "Site Audit reports on your own domain stop working. If a customer is auditing your site with your permission, blocking this breaks their tooling and nothing of yours.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/siteauditbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/siteauditbot.json", "last_reviewed": "2026-09-01" }, { "slug": "semrushbot-ba", "name": "SemrushBot-BA", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SemrushBot-BA", "user_agent_substring": "SemrushBot-BA", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The Backlink Audit crawler. It re-checks links pointing at a customer's site, which means it lands on the sites doing the linking.", "cost_of_blocking": "None to you. It costs the site being audited a little accuracy in their backlink report.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ba.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ba.json", "last_reviewed": "2026-09-01" }, { "slug": "semrushbot-si", "name": "SemrushBot-SI", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SemrushBot-SI", "user_agent_substring": "SemrushBot-SI", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-SI; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches pages for the On Page SEO Checker and similar advisory tools.", "cost_of_blocking": "Nothing user-facing. Semrush customers lose on-page suggestions for pages on your domain.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-si.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-si.json", "last_reviewed": "2026-09-01" }, { "slug": "semrushbot-swa", "name": "SemrushBot-SWA", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SemrushBot-SWA", "user_agent_substring": "SemrushBot-SWA", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-SWA; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Checks whether a URL is reachable, for the SEO Writing Assistant. One request per URL a writer references, not a crawl.", "cost_of_blocking": "Writers using Semrush's assistant see your links reported as unreachable.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-swa.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-swa.json", "last_reviewed": "2026-09-01" }, { "slug": "splitsignalbot", "name": "SplitSignalBot", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SplitSignalBot", "user_agent_substring": "SplitSignalBot", "user_agent_example": "Mozilla/5.0 (compatible; SplitSignalBot; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Runs SEO A/B tests on a customer's own site with the SplitSignal tool.", "cost_of_blocking": "A site owner's own A/B testing stops. Only relevant on domains whose owner uses the product.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/splitsignalbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/splitsignalbot.json", "last_reviewed": "2026-09-01" }, { "slug": "semrushbot-ft", "name": "SemrushBot-FT", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SemrushBot-FT", "user_agent_substring": "SemrushBot-FT", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-FT; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches full text for the Plagiarism Checker and similar text-comparison tools.", "cost_of_blocking": "Your text stops being compared against other people's submissions — which also means copies of your text are less likely to be caught.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ft.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ft.json", "last_reviewed": "2026-09-01" }, { "slug": "semrushbot-esi", "name": "SemrushBot-ESI", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SemrushBot-ESI", "user_agent_substring": "SemrushBot-ESI", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-ESI; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The crawler for Semrush Enterprise Site Intelligence, the enterprise tier's own site analysis.", "cost_of_blocking": "Enterprise customers lose analysis of your domain. Nothing user-facing.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-esi.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-esi.json", "last_reviewed": "2026-09-01" }, { "slug": "ahrefssiteaudit", "name": "AhrefsSiteAudit", "operator": "Ahrefs", "operator_slug": "ahrefs", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "AhrefsSiteAudit", "user_agent_substring": "AhrefsSiteAudit", "user_agent_example": "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/site-audit)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/ahrefs-crawler.json", "ipv4_prefix_count": 81, "ipv6_prefix_count": 0, "what_it_is": "Ahrefs' site-audit crawler, separate from AhrefsBot. Ahrefs documents that it obeys robots.txt by default, and that a verified site owner can ask for it to be allowed to ignore robots.txt on their own site so the audit can see disallowed sections.", "cost_of_blocking": "Site owners auditing your domain get an incomplete report. Blocking it saves bandwidth and costs you nothing in search.", "operator_docs": "https://ahrefs.com/robot", "html_url": "https://www.pathwren.workers.dev/crawler/ahrefssiteaudit.html", "json_url": "https://www.pathwren.workers.dev/crawler/ahrefssiteaudit.json", "last_reviewed": "2026-09-01" }, { "slug": "mj12bot", "name": "MJ12bot", "operator": "Majestic", "operator_slug": "majestic", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "MJ12bot", "user_agent_substring": "MJ12bot", "user_agent_example": "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Majestic's link-graph crawler, run as a distributed community project. Majestic states plainly that it cannot restrict the bot to a fixed set of addresses, and offers a pre-arranged ident string in the request headers instead.", "cost_of_blocking": "You leave the Majestic backlink index. No search or AI effect. It supports Crawl-delay, which is usually the better answer than a block.", "operator_docs": "https://mj12bot.com/", "html_url": "https://www.pathwren.workers.dev/crawler/mj12bot.html", "json_url": "https://www.pathwren.workers.dev/crawler/mj12bot.json", "last_reviewed": "2026-09-01" }, { "slug": "dotbot", "name": "DotBot", "operator": "Moz", "operator_slug": "moz", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "dotbot", "user_agent_substring": "dotbot", "user_agent_example": "Mozilla/5.0 (compatible; DotBot/1.2; +https://opensiteexplorer.org/dotbot; help@moz.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Moz's crawler for Link Explorer. Moz documents that it respects robots.txt and that dotbot is the token to name.", "cost_of_blocking": "You leave Moz's link index, so Domain Authority and link reports about your site get thinner. Nothing a reader or an assistant sees changes.", "operator_docs": "https://moz.com/help/moz-procedures/crawlers/dotbot", "html_url": "https://www.pathwren.workers.dev/crawler/dotbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/dotbot.json", "last_reviewed": "2026-09-01" }, { "slug": "rogerbot", "name": "rogerbot", "operator": "Moz", "operator_slug": "moz", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "rogerbot", "user_agent_substring": "rogerbot", "user_agent_example": "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler@moz.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Moz's Campaign crawler, which audits a site its own owner registered. Moz states there is no IP range for it — identification is by user-agent only.", "cost_of_blocking": "Moz Pro site audits of your domain stop. If the domain is yours and you use Moz, blocking this breaks your own reports.", "operator_docs": "https://moz.com/help/moz-procedures/crawlers/rogerbot", "html_url": "https://www.pathwren.workers.dev/crawler/rogerbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/rogerbot.json", "last_reviewed": "2026-09-01" }, { "slug": "dataforseobot", "name": "DataForSeoBot", "operator": "DataForSEO", "operator_slug": "dataforseo", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "DataForSeoBot", "user_agent_substring": "DataForSeoBot", "user_agent_example": "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Builds the backlink and SERP datasets DataForSEO resells through its API, so one crawl reaches many downstream tools.", "cost_of_blocking": "You leave a dataset that a long tail of SEO products is built on. No user-facing effect.", "operator_docs": "https://dataforseo.com/dataforseo-bot", "html_url": "https://www.pathwren.workers.dev/crawler/dataforseobot.html", "json_url": "https://www.pathwren.workers.dev/crawler/dataforseobot.json", "last_reviewed": "2026-09-01" }, { "slug": "serpstatbot", "name": "serpstatbot", "operator": "Serpstat", "operator_slug": "serpstat", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "serpstatbot", "user_agent_substring": "serpstatbot", "user_agent_example": "serpstatbot/2.1 (advanced backlink tracking bot; https://serpstatbot.com/; abuse@serpstatbot.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Serpstat's backlink crawler. It documents support for Crawl-delay up to 20 seconds, including a delay set on the * group.", "cost_of_blocking": "You leave Serpstat's link index. Try Crawl-delay first — they honour it, and a slow crawler is cheaper to keep than to fight.", "operator_docs": "https://serpstatbot.com/", "html_url": "https://www.pathwren.workers.dev/crawler/serpstatbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/serpstatbot.json", "last_reviewed": "2026-09-01" }, { "slug": "barkrowler", "name": "Barkrowler", "operator": "Babbar", "operator_slug": "babbar", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "barkrowler", "user_agent_substring": "barkrowler", "user_agent_example": "Mozilla/5.0 (compatible; Barkrowler/0.9; +https://babbar.tech/crawler)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Babbar's crawler, which builds the link graph behind their French-market SEO tooling.", "cost_of_blocking": "You leave Babbar's index. No effect on search or assistants.", "operator_docs": "https://babbar.tech/crawler", "html_url": "https://www.pathwren.workers.dev/crawler/barkrowler.html", "json_url": "https://www.pathwren.workers.dev/crawler/barkrowler.json", "last_reviewed": "2026-09-01" }, { "slug": "screaming-frog-seo-spider", "name": "Screaming Frog SEO Spider", "operator": "Screaming Frog", "operator_slug": "screamingfrog", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "Screaming Frog SEO Spider", "user_agent_substring": "Screaming Frog SEO Spider", "user_agent_example": "Screaming Frog SEO Spider/21.0", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Not an operator: desktop crawling software that anybody can point at any site. The default user-agent identifies the tool, not who is running it, and the operator of the moment is whoever pressed start.", "cost_of_blocking": "You block a consultant auditing your own site as often as you block a stranger. Treat it as a rate-limit question, not a consent one — and note that the user-agent is configurable, so a block is advisory.", "operator_docs": "https://www.screamingfrog.co.uk/seo-spider/user-agent/", "html_url": "https://www.pathwren.workers.dev/crawler/screaming-frog-seo-spider.html", "json_url": "https://www.pathwren.workers.dev/crawler/screaming-frog-seo-spider.json", "last_reviewed": "2026-09-01" }, { "slug": "seokicks", "name": "SEOkicks", "operator": "SEOkicks", "operator_slug": "seokicks", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SEOkicks", "user_agent_substring": "SEOkicks", "user_agent_example": "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "A German backlink index. Its documentation names SEOkicks as the user-agent to use in robots.txt.", "cost_of_blocking": "You leave a regional backlink index. Nothing else changes.", "operator_docs": "https://www.seokicks.de/robot.html", "html_url": "https://www.pathwren.workers.dev/crawler/seokicks.html", "json_url": "https://www.pathwren.workers.dev/crawler/seokicks.json", "last_reviewed": "2026-09-01" }, { "slug": "mojeekbot", "name": "MojeekBot", "operator": "Mojeek", "operator_slug": "mojeek", "category": "search", "category_label": "Search engines", "robots_token": "MojeekBot", "user_agent_substring": "MojeekBot", "user_agent_example": "Mozilla/5.0 (compatible; MojeekBot/0.11; +https://www.mojeek.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Mojeek's crawler. Mojeek runs one of the few genuinely independent web indexes — not a front end over Bing or Google — so it is one of the few blocks that removes you from an index nobody else can put you back into. Its documentation states it obeys the first record whose User-Agent contains MojeekBot, falling back to *.", "cost_of_blocking": "You leave an independent index that other privacy-focused search products draw on. Small traffic, disproportionate long-term cost to web plurality.", "operator_docs": "https://www.mojeek.com/bot.html", "html_url": "https://www.pathwren.workers.dev/crawler/mojeekbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/mojeekbot.json", "last_reviewed": "2026-09-01" }, { "slug": "kagibot", "name": "Kagibot", "operator": "Kagi", "operator_slug": "kagi", "category": "search", "category_label": "Search engines", "robots_token": "Kagibot", "user_agent_substring": "Kagibot", "user_agent_example": "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The crawler for Kagi, a paid, ad-free search engine with its own index and its own assistant.", "cost_of_blocking": "You leave Kagi's index. Kagi's users are paying to search and skew technical; per visitor this is an expensive block.", "operator_docs": "https://kagi.com/bot", "html_url": "https://www.pathwren.workers.dev/crawler/kagibot.html", "json_url": "https://www.pathwren.workers.dev/crawler/kagibot.json", "last_reviewed": "2026-09-01" }, { "slug": "qwantbot", "name": "Qwantbot", "operator": "Qwant", "operator_slug": "qwant", "category": "search", "category_label": "Search engines", "robots_token": "Qwantbot", "user_agent_substring": "Qwantbot", "user_agent_example": "Mozilla/5.0 (compatible; Qwantbot/1.0_12345; +https://help.qwant.com/bot/)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Qwant's crawler. Qwant documents that the string Qwantbot always appears in its user-agents whatever the crawler version, which is what makes a substring match safe here.", "cost_of_blocking": "You leave the index behind Qwant, the French privacy-focused engine, and the products that federate it.", "operator_docs": "https://help.qwant.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/qwantbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/qwantbot.json", "last_reviewed": "2026-09-01" }, { "slug": "qwantbot-news", "name": "Qwantbot-news", "operator": "Qwant", "operator_slug": "qwant", "category": "search", "category_label": "Search engines", "robots_token": "Qwantbot-news", "user_agent_substring": "Qwantbot-news", "user_agent_example": "Mozilla/5.0 (compatible; Qwantbot-news/2.0; +https://help.qwant.com/bot/)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The news variant of Qwant's crawler, documented alongside the main one and carrying the same Qwantbot substring.", "cost_of_blocking": "Your articles stop appearing in Qwant News. A rule for Qwantbot as a substring already catches both.", "operator_docs": "https://help.qwant.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/qwantbot-news.html", "json_url": "https://www.pathwren.workers.dev/crawler/qwantbot-news.json", "last_reviewed": "2026-09-01" }, { "slug": "slackbot-linkexpanding", "name": "Slackbot-LinkExpanding", "operator": "Slack", "operator_slug": "slack", "category": "preview", "category_label": "Link preview fetchers", "robots_token": "Slackbot-LinkExpanding", "user_agent_substring": "Slackbot-LinkExpanding", "user_agent_example": "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches a page to build the unfurl card when somebody pastes your link into Slack. One paste, one fetch.", "cost_of_blocking": "Your links appear in Slack as bare URLs. Inside working teams that quietly costs you clicks, and nothing is gained.", "operator_docs": "https://api.slack.com/robots", "html_url": "https://www.pathwren.workers.dev/crawler/slackbot-linkexpanding.html", "json_url": "https://www.pathwren.workers.dev/crawler/slackbot-linkexpanding.json", "last_reviewed": "2026-09-01" }, { "slug": "slackbot", "name": "Slackbot", "operator": "Slack", "operator_slug": "slack", "category": "preview", "category_label": "Link preview fetchers", "robots_token": "Slackbot", "user_agent_substring": "Slackbot", "user_agent_example": "Slackbot 1.0 (+https://api.slack.com/robots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The other half of Slack's pair: the agent that reads robots.txt and handles Slack's non-unfurl fetches. Slack documents both strings on one page.", "cost_of_blocking": "Slack stops being able to read your robots.txt, which is a strange thing to want. Block the link expander instead if that is the goal.", "operator_docs": "https://api.slack.com/robots", "html_url": "https://www.pathwren.workers.dev/crawler/slackbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/slackbot.json", "last_reviewed": "2026-09-01" }, { "slug": "pinterestbot", "name": "Pinterestbot", "operator": "Pinterest", "operator_slug": "pinterest", "category": "search", "category_label": "Search engines", "robots_token": "Pinterestbot", "user_agent_substring": "Pinterestbot", "user_agent_example": "Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Pinterest's crawler. It indexes pages so people can find them on Pinterest and re-reads product pages to keep price and title on a Pin current. Pinterest states that content it crawls is not used to train their Canvas image generation model.", "cost_of_blocking": "Pins pointing at your site go stale — wrong prices, dead links — and new content stops being indexed. For a retailer this is one of the more expensive blocks on the list.", "operator_docs": "https://help.pinterest.com/en/business/article/pinterest-crawler", "html_url": "https://www.pathwren.workers.dev/crawler/pinterestbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/pinterestbot.json", "last_reviewed": "2026-09-01" }, { "slug": "bedrockbot", "name": "bedrockbot", "operator": "Amazon", "operator_slug": "amazon", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "bedrockbot", "user_agent_substring": "bedrockbot", "user_agent_example": "bedrockbot-UUID", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The web crawler an AWS customer points at URLs they chose, to build a knowledge base for a Bedrock application. AWS documents that it respects robots.txt and that the user-agent carries a per-customer suffix, so you can allow or refuse one customer's crawl by naming bedrockbot-UUID.", "cost_of_blocking": "Companies building retrieval applications on Bedrock cannot include your pages. This is a RAG block, not a training block: nothing is being trained, but nothing can cite you either.", "operator_docs": "https://docs.aws.amazon.com/bedrock/latest/userguide/webcrawl-data-source-connector.html", "html_url": "https://www.pathwren.workers.dev/crawler/bedrockbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/bedrockbot.json", "last_reviewed": "2026-09-01" }, { "slug": "cloudflare-autorag", "name": "Cloudflare-AutoRAG", "operator": "Cloudflare", "operator_slug": "cloudflare", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "Cloudflare-AutoRAG", "user_agent_substring": "Cloudflare-AutoRAG", "user_agent_example": "Cloudflare-AutoRAG", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The crawler behind Cloudflare's AI Search / AutoRAG, which indexes a website into a retrieval index for an application. Cloudflare's own documentation warns that a bot-blocking rule on your zone will also stop this crawler and tells you to allow-list it.", "cost_of_blocking": "Applications built on Cloudflare AI Search cannot retrieve your pages. If you are the one building the index over your own site, blocking it breaks your own product.", "operator_docs": "https://developers.cloudflare.com/ai-search/", "html_url": "https://www.pathwren.workers.dev/crawler/cloudflare-autorag.html", "json_url": "https://www.pathwren.workers.dev/crawler/cloudflare-autorag.json", "last_reviewed": "2026-09-01" }, { "slug": "exasearchbot", "name": "ExaSearchBot", "operator": "Exa", "operator_slug": "exa", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "ExaSearchBot", "user_agent_substring": "ExaSearchBot", "user_agent_example": "ExaSearchBot", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Exa's crawler. It discovers and indexes public pages so they can be retrieved and cited through Exa's search API, which is one of the common retrieval backends behind agent frameworks.", "cost_of_blocking": "Agents built on Exa's API stop finding you. Exa publishes no statement about robots.txt compliance, so treat the rule as a request.", "operator_docs": "https://exa.ai", "html_url": "https://www.pathwren.workers.dev/crawler/exasearchbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/exasearchbot.json", "last_reviewed": "2026-09-01" }, { "slug": "shapbot", "name": "ShapBot", "operator": "Parallel", "operator_slug": "parallel", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "ShapBot", "user_agent_substring": "ShapBot", "user_agent_example": "ShapBot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Parallel's crawler. It collects and structures web content to power the search, extraction and deep-research APIs that Parallel sells to agent builders.", "cost_of_blocking": "Agents using Parallel's research API lose you as a source. Parallel documents robots.txt compliance, so a rule works.", "operator_docs": "https://docs.parallel.ai/features/crawler", "html_url": "https://www.pathwren.workers.dev/crawler/shapbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/shapbot.json", "last_reviewed": "2026-09-01" }, { "slug": "terracotta", "name": "TerraCotta", "operator": "Ceramic AI", "operator_slug": "ceramic", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "TerraCotta", "user_agent_substring": "TerraCotta", "user_agent_example": "TerraCotta", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Ceramic AI's crawler, which indexes public content for a web-scale search API aimed at LLMs and agents.", "cost_of_blocking": "You are absent from another agent-facing retrieval index. Ceramic documents that it obeys robots.txt.", "operator_docs": "https://ceramic.ai/", "html_url": "https://www.pathwren.workers.dev/crawler/terracotta.html", "json_url": "https://www.pathwren.workers.dev/crawler/terracotta.json", "last_reviewed": "2026-09-01" }, { "slug": "crawlspace", "name": "Crawlspace", "operator": "Crawlspace", "operator_slug": "crawlspace", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "Crawlspace", "user_agent_substring": "Crawlspace", "user_agent_example": "Crawlspace", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "A crawling platform: customers run their own crawls on it to feed agents, RAG pipelines and structured-data workflows. Like Firecrawl, the party behind any given request is the customer, not the platform.", "cost_of_blocking": "Whatever any Crawlspace customer was building over your pages stops working. Volume and intent vary per customer, so this is a rate-limit decision more than a consent one.", "operator_docs": "https://crawlspace.dev", "html_url": "https://www.pathwren.workers.dev/crawler/crawlspace.html", "json_url": "https://www.pathwren.workers.dev/crawler/crawlspace.json", "last_reviewed": "2026-09-01" }, { "slug": "panscient", "name": "Panscient", "operator": "Panscient", "operator_slug": "panscient", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "panscient.com", "user_agent_substring": "panscient.com", "user_agent_example": "Mozilla/5.0 (compatible; panscient.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Compiles structured data about businesses and business professionals using machine learning. Panscient's FAQ states it obeys robots.txt.", "cost_of_blocking": "Your company pages stop feeding a business-data product. No effect on search or assistants.", "operator_docs": "https://panscient.com/faq.htm", "html_url": "https://www.pathwren.workers.dev/crawler/panscient.html", "json_url": "https://www.pathwren.workers.dev/crawler/panscient.json", "last_reviewed": "2026-09-01" }, { "slug": "sbintuitionsbot", "name": "SBIntuitionsBot", "operator": "SB Intuitions", "operator_slug": "sbintuitions", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "SBIntuitionsBot", "user_agent_substring": "SBIntuitionsBot", "user_agent_example": "SBIntuitionsBot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "SB Intuitions is SoftBank's Japanese LLM lab; this crawler gathers data used in that model development and in information analysis. The operator publishes a dedicated bot page.", "cost_of_blocking": "Your content is excluded from a Japanese-language foundation-model corpus. Nothing user-facing changes.", "operator_docs": "https://www.sbintuitions.co.jp/en/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/sbintuitionsbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/sbintuitionsbot.json", "last_reviewed": "2026-09-01" }, { "slug": "icc-crawler", "name": "ICC-Crawler", "operator": "NICT", "operator_slug": "nict", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "ICC-Crawler", "user_agent_substring": "ICC-Crawler", "user_agent_example": "ICC-Crawler", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Operated by NICT, Japan's national information and communications research institute. The collected data supports AI research and, per the operator, is also provided to third parties including commercial companies.", "cost_of_blocking": "You are excluded from a national research corpus and from the commercial redistributions of it. This is a dataset-shaped block: one refusal, many downstream effects.", "operator_docs": "https://www.nict.go.jp/en/", "html_url": "https://www.pathwren.workers.dev/crawler/icc-crawler.html", "json_url": "https://www.pathwren.workers.dev/crawler/icc-crawler.json", "last_reviewed": "2026-09-01" }, { "slug": "cotoyogi", "name": "Cotoyogi", "operator": "ROIS-DS", "operator_slug": "rois", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "Cotoyogi", "user_agent_substring": "Cotoyogi", "user_agent_example": "Cotoyogi", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "A crawler run by ROIS-DS, a Japanese inter-university research organisation, collecting Japanese-language text for AI training. It publishes a crawler page in English and Japanese.", "cost_of_blocking": "Your Japanese-language content is left out of an academic training corpus.", "operator_docs": "https://ds.rois.ac.jp/en_center8/en_crawler/", "html_url": "https://www.pathwren.workers.dev/crawler/cotoyogi.html", "json_url": "https://www.pathwren.workers.dev/crawler/cotoyogi.json", "last_reviewed": "2026-09-01" }, { "slug": "isscyberriskcrawler", "name": "ISSCyberRiskCrawler", "operator": "ISS Corporate Solutions", "operator_slug": "iss", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "ISSCyberRiskCrawler", "user_agent_substring": "ISSCyberRiskCrawler", "user_agent_example": "ISSCyberRiskCrawler", "respects_robots_txt": "disputed", "respects_robots_txt_label": "compliance disputed", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Crawls in order to train models that score a company's cyber risk. The ai.robots.txt dataset records the operator as not respecting robots.txt; ISS publishes no compliance statement of its own.", "cost_of_blocking": "A rule here is a statement of intent. Your organisation's public footprint still gets scored — by a model trained on everybody else.", "operator_docs": "https://iss-cyber.com", "html_url": "https://www.pathwren.workers.dev/crawler/isscyberriskcrawler.html", "json_url": "https://www.pathwren.workers.dev/crawler/isscyberriskcrawler.json", "last_reviewed": "2026-09-01" }, { "slug": "sidetrade-indexer-bot", "name": "Sidetrade indexer bot", "operator": "Sidetrade", "operator_slug": "sidetrade", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "Sidetrade indexer bot", "user_agent_substring": "Sidetrade indexer bot", "user_agent_example": "Sidetrade indexer bot", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Sidetrade extracts web data for a range of uses including training its AI products for order-to-cash and customer-data work.", "cost_of_blocking": "Exclusion from a commercial B2B dataset. The operator publishes no robots.txt statement, so treat the rule as a request rather than a control.", "operator_docs": "https://www.sidetrade.com", "html_url": "https://www.pathwren.workers.dev/crawler/sidetrade-indexer-bot.html", "json_url": "https://www.pathwren.workers.dev/crawler/sidetrade-indexer-bot.json", "last_reviewed": "2026-09-01" }, { "slug": "yak", "name": "YaK", "operator": "Meltwater", "operator_slug": "meltwater", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "YaK", "user_agent_substring": "YaK", "user_agent_example": "YaK", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Meltwater's crawler, feeding the live data stream behind its media-monitoring and consumer-intelligence suite.", "cost_of_blocking": "Your content stops appearing in Meltwater's media monitoring — which is how PR teams find out you were mentioned. Some publishers want to be in it.", "operator_docs": "https://www.meltwater.com/en/suite/consumer-intelligence", "html_url": "https://www.pathwren.workers.dev/crawler/yak.html", "json_url": "https://www.pathwren.workers.dev/crawler/yak.json", "last_reviewed": "2026-09-01" }, { "slug": "atlassian-bot", "name": "atlassian-bot", "operator": "Atlassian", "operator_slug": "atlassian", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "atlassian-bot", "user_agent_substring": "atlassian-bot", "user_agent_example": "atlassian-bot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Indexes a website so it can be searched and cited by Rovo, Atlassian's generative assistant inside Jira and Confluence. Atlassian's documentation walks a customer through editing robots.txt for it, which is as close to a compliance statement as this list gets.", "cost_of_blocking": "Rovo cannot answer from your public documentation. If your customers live inside Atlassian tools, this is a support-deflection block.", "operator_docs": "https://support.atlassian.com/organization-administration/docs/connect-custom-website-to-rovo/", "html_url": "https://www.pathwren.workers.dev/crawler/atlassian-bot.html", "json_url": "https://www.pathwren.workers.dev/crawler/atlassian-bot.json", "last_reviewed": "2026-09-01" }, { "slug": "klaviyoaibot", "name": "KlaviyoAIBot", "operator": "Klaviyo", "operator_slug": "klaviyo", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "KlaviyoAIBot", "user_agent_substring": "KlaviyoAIBot", "user_agent_example": "KlaviyoAIBot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches pages from domains a Klaviyo customer has explicitly connected to their own account, to power Klaviyo's Kai customer agent. It is scoped to connected domains rather than the open web.", "cost_of_blocking": "If the connected domain is yours, blocking this breaks the agent you configured. If it is not, this bot should not be reaching you at all.", "operator_docs": "https://help.klaviyo.com/hc/en-us/articles/40496146232219", "html_url": "https://www.pathwren.workers.dev/crawler/klaviyoaibot.html", "json_url": "https://www.pathwren.workers.dev/crawler/klaviyoaibot.json", "last_reviewed": "2026-09-01" }, { "slug": "quillbot", "name": "QuillBot", "operator": "QuillBot", "operator_slug": "quillbot", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "QuillBot", "user_agent_substring": "QuillBot", "user_agent_example": "QuillBot", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Operated by QuillBot as part of its writing, paraphrasing and AI-detection products. The dataset also records a second token, quillbot.com, for the same operator.", "cost_of_blocking": "Exclusion from QuillBot's corpus. No compliance statement is published, so the rule is a request.", "operator_docs": "https://quillbot.com", "html_url": "https://www.pathwren.workers.dev/crawler/quillbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/quillbot.json", "last_reviewed": "2026-09-01" }, { "slug": "phindbot", "name": "PhindBot", "operator": "Phind", "operator_slug": "phind", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "PhindBot", "user_agent_substring": "PhindBot", "user_agent_example": "PhindBot", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Phind is an answer engine for developers that combines live web search with its own models. This is the crawler behind those answers.", "cost_of_blocking": "You stop being cited in answers to technical questions — which, for documentation and reference sites, is the exact audience most worth keeping.", "operator_docs": "https://www.phind.com/", "html_url": "https://www.pathwren.workers.dev/crawler/phindbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/phindbot.json", "last_reviewed": "2026-09-01" }, { "slug": "andibot", "name": "Andibot", "operator": "Andi", "operator_slug": "andi", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "Andibot", "user_agent_substring": "Andibot", "user_agent_example": "Andibot", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The crawler for Andi, a small generative search assistant that summarises pages rather than listing them.", "cost_of_blocking": "You disappear from another assistant's answers. Andi publishes no robots.txt statement.", "operator_docs": "https://andisearch.com/", "html_url": "https://www.pathwren.workers.dev/crawler/andibot.html", "json_url": "https://www.pathwren.workers.dev/crawler/andibot.json", "last_reviewed": "2026-09-01" }, { "slug": "anomura", "name": "Anomura", "operator": "Direqt", "operator_slug": "direqt", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "Anomura", "user_agent_substring": "Anomura", "user_agent_example": "Anomura", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Direqt's search crawler. It indexes the sites of Direqt's own publisher customers so their on-site chatbots can answer from them.", "cost_of_blocking": "If you are the publisher, this breaks the assistant you put on your own pages. If you are not, it should not be crawling you.", "operator_docs": "https://direqt.ai", "html_url": "https://www.pathwren.workers.dev/crawler/anomura.html", "json_url": "https://www.pathwren.workers.dev/crawler/anomura.json", "last_reviewed": "2026-09-01" }, { "slug": "aiwebindex", "name": "AIWebIndex", "operator": "Lyrenth", "operator_slug": "lyrenth", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "AIWebIndex", "user_agent_substring": "AIWebIndex", "user_agent_example": "AIWebIndex", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Builds an index of public pages and serves them to AI agents as extracted readable text, with attribution and a link back. Lyrenth publishes a crawler policy stating it does not train foundation models on what it collects and that it obeys robots.txt.", "cost_of_blocking": "Agents reading through this index stop seeing you — including the attribution and link back that make it a referral rather than a summary.", "operator_docs": "https://lyrenth.com/crawler-policy", "html_url": "https://www.pathwren.workers.dev/crawler/aiwebindex.html", "json_url": "https://www.pathwren.workers.dev/crawler/aiwebindex.json", "last_reviewed": "2026-09-01" }, { "slug": "factset-spyderbot", "name": "Factset_spyderbot", "operator": "FactSet", "operator_slug": "factset", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "Factset_spyderbot", "user_agent_substring": "Factset_spyderbot", "user_agent_example": "Factset_spyderbot", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "FactSet's crawler, collecting data used in AI model training for its financial data and analytics products.", "cost_of_blocking": "Exclusion from a financial-data vendor's corpus. Relevant mostly to companies whose filings and disclosures are being read.", "operator_docs": "https://www.factset.com/ai", "html_url": "https://www.pathwren.workers.dev/crawler/factset-spyderbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/factset-spyderbot.json", "last_reviewed": "2026-09-01" }, { "slug": "poseidon-research-crawler", "name": "Poseidon Research Crawler", "operator": "Poseidon Research", "operator_slug": "poseidon", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "Poseidon Research Crawler", "user_agent_substring": "Poseidon Research Crawler", "user_agent_example": "Poseidon Research Crawler", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "A crawler run by Poseidon Research, a lab working on interpretability research for AI systems.", "cost_of_blocking": "Exclusion from an interpretability research corpus. No published compliance statement.", "operator_docs": "https://www.poseidonresearch.com", "html_url": "https://www.pathwren.workers.dev/crawler/poseidon-research-crawler.html", "json_url": "https://www.pathwren.workers.dev/crawler/poseidon-research-crawler.json", "last_reviewed": "2026-09-01" }, { "slug": "qualifiedbot", "name": "QualifiedBot", "operator": "Qualified", "operator_slug": "qualified", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "QualifiedBot", "user_agent_substring": "QualifiedBot", "user_agent_example": "QualifiedBot", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Analyses a customer's website so Qualified's AI sales chatbots can answer questions about it in context.", "cost_of_blocking": "A chatbot on a site that licensed the product loses context. If that site is yours, this block is self-inflicted.", "operator_docs": "https://www.qualified.com", "html_url": "https://www.pathwren.workers.dev/crawler/qualifiedbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/qualifiedbot.json", "last_reviewed": "2026-09-01" }, { "slug": "reflectionbot", "name": "Reflectionbot", "operator": "Reflection AI", "operator_slug": "reflection", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "Reflectionbot", "user_agent_substring": "Reflectionbot", "user_agent_example": "Reflectionbot", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "An undocumented crawler whose user-agent links to Reflection AI, a company building AI models. The link in the user-agent is the only public statement of purpose that exists.", "cost_of_blocking": "Unknown by construction — which is itself the reason some people block it. Nothing user-facing depends on it.", "operator_docs": "https://reflection.ai/", "html_url": "https://www.pathwren.workers.dev/crawler/reflectionbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/reflectionbot.json", "last_reviewed": "2026-09-01" }, { "slug": "thinkbot", "name": "Thinkbot", "operator": "Thinkbot", "operator_slug": "thinkbot", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "Thinkbot", "user_agent_substring": "Thinkbot", "user_agent_example": "Thinkbot", "respects_robots_txt": "disputed", "respects_robots_txt_label": "compliance disputed", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Collects pages for analysis of how sites are adopting AI and automation. The ai.robots.txt dataset records the operator as not respecting robots.txt.", "cost_of_blocking": "Exclusion from a market-research dataset. Expect to enforce this at the edge rather than in robots.txt.", "operator_docs": "https://www.thinkbot.agency", "html_url": "https://www.pathwren.workers.dev/crawler/thinkbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/thinkbot.json", "last_reviewed": "2026-09-01" }, { "slug": "aihitbot", "name": "aiHitBot", "operator": "aiHit", "operator_slug": "aihit", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "aiHitBot", "user_agent_substring": "aiHitBot", "user_agent_example": "aiHitBot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "aiHit's automated collector, building a company dataset from public company websites.", "cost_of_blocking": "Your company record in a B2B dataset goes stale. Documented as respecting robots.txt, so the rule works.", "operator_docs": "https://www.aihitdata.com/about", "html_url": "https://www.pathwren.workers.dev/crawler/aihitbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/aihitbot.json", "last_reviewed": "2026-09-01" }, { "slug": "linguee-bot", "name": "Linguee Bot", "operator": "Linguee", "operator_slug": "linguee", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "Linguee Bot", "user_agent_substring": "Linguee Bot", "user_agent_example": "Linguee Bot", "respects_robots_txt": "disputed", "respects_robots_txt_label": "compliance disputed", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Gathers bilingual text for Linguee's translation corpus and the machine translation trained on it. Recorded in the ai.robots.txt dataset as not respecting robots.txt.", "cost_of_blocking": "Multilingual pages stop feeding a translation corpus. If your site is translated, being in it is usually a benefit.", "operator_docs": "https://www.linguee.com", "html_url": "https://www.pathwren.workers.dev/crawler/linguee-bot.html", "json_url": "https://www.pathwren.workers.dev/crawler/linguee-bot.json", "last_reviewed": "2026-09-01" }, { "slug": "lightpanda", "name": "Lightpanda", "operator": "Lightpanda", "operator_slug": "lightpanda", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "Lightpanda", "user_agent_substring": "Lightpanda", "user_agent_example": "Lightpanda", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "A purpose-built headless browser for AI and automation — a runtime, not an operator. Whether robots.txt is honoured is left to whoever runs it, which is what its maintainers say themselves.", "cost_of_blocking": "You block a browser, not a company: the same rule stops a scraper and a legitimate automation a customer of yours is running.", "operator_docs": "https://lightpanda.io/", "html_url": "https://www.pathwren.workers.dev/crawler/lightpanda.html", "json_url": "https://www.pathwren.workers.dev/crawler/lightpanda.json", "last_reviewed": "2026-09-01" }, { "slug": "laiondownloader", "name": "LAIONDownloader", "operator": "LAION / img2dataset", "operator_slug": "laion", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "LAIONDownloader", "user_agent_substring": "LAIONDownloader", "user_agent_example": "LAIONDownloader", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "LAION's downloader, used to materialise the image and text datasets the non-profit publishes for machine-learning research. LAION's own FAQ is the source for its robots.txt position.", "cost_of_blocking": "Your media is skipped when an open research dataset is built from URL lists. Once a dataset is published, a later block does not remove you from it.", "operator_docs": "https://laion.ai/faq/", "html_url": "https://www.pathwren.workers.dev/crawler/laiondownloader.html", "json_url": "https://www.pathwren.workers.dev/crawler/laiondownloader.json", "last_reviewed": "2026-09-01" }, { "slug": "velenpublicwebcrawler", "name": "VelenPublicWebCrawler", "operator": "Hunter (Velen)", "operator_slug": "hunter", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "VelenPublicWebCrawler", "user_agent_substring": "VelenPublicWebCrawler", "user_agent_example": "VelenPublicWebCrawler", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Hunter's crawler, written in Go, building business datasets and machine-learning models from public pages. Its page states it follows robots.txt and meta directives and never fetches more than one page every two seconds.", "cost_of_blocking": "Your company pages stop feeding a B2B contact and company dataset. The crawl rate it documents makes this one of the cheapest visitors to simply allow.", "operator_docs": "https://velen.io/", "html_url": "https://www.pathwren.workers.dev/crawler/velenpublicwebcrawler.html", "json_url": "https://www.pathwren.workers.dev/crawler/velenpublicwebcrawler.json", "last_reviewed": "2026-09-01" }, { "slug": "awariosmartbot", "name": "AwarioSmartBot", "operator": "Awario", "operator_slug": "awario", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "AwarioSmartBot", "user_agent_substring": "AwarioSmartBot", "user_agent_example": "AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Awario's brand-monitoring crawler. It documents one request per three seconds, honours Crawl-delay, and states it does not use consecutive IP blocks so identification is by user-agent only.", "cost_of_blocking": "Mentions of brands on your pages stop being surfaced to the people monitoring them — including, quite possibly, your own.", "operator_docs": "https://awario.com/bots.html", "html_url": "https://www.pathwren.workers.dev/crawler/awariosmartbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/awariosmartbot.json", "last_reviewed": "2026-09-01" }, { "slug": "awariorssbot", "name": "AwarioRssBot", "operator": "Awario", "operator_slug": "awario", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "AwarioRssBot", "user_agent_substring": "AwarioRssBot", "user_agent_example": "AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The feed-reading half of Awario's pair, documented on the same page and under the same crawl-rate policy.", "cost_of_blocking": "Your RSS updates stop reaching Awario's monitoring. Block both tokens or neither.", "operator_docs": "https://awario.com/bots.html", "html_url": "https://www.pathwren.workers.dev/crawler/awariorssbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/awariorssbot.json", "last_reviewed": "2026-09-01" }, { "slug": "echoboxbot", "name": "EchoboxBot", "operator": "Echobox", "operator_slug": "echobox", "category": "dataset", "category_label": "Corpus and dataset builders", "robots_token": "EchoboxBot", "user_agent_substring": "EchoboxBot", "user_agent_example": "EchoboxBot", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Collects data supporting Echobox's AI-driven social and email distribution products, which publishers use to schedule and target their own content.", "cost_of_blocking": "Publishers using Echobox get worse scheduling decisions about your articles. No compliance statement is published.", "operator_docs": "https://echobox.com", "html_url": "https://www.pathwren.workers.dev/crawler/echoboxbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/echoboxbot.json", "last_reviewed": "2026-09-01" }, { "slug": "meta-webindexer", "name": "Meta-WebIndexer", "operator": "Meta", "operator_slug": "meta", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "Meta-WebIndexer", "user_agent_substring": "Meta-WebIndexer", "user_agent_example": "Meta-WebIndexer", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Per Meta's crawler documentation, Meta-WebIndexer navigates the web to improve the quality of Meta AI's search results. It is a third Meta token alongside Meta-ExternalAgent and Meta-ExternalFetcher, and the newest of them.", "cost_of_blocking": "You leave the index Meta AI answers from across Facebook, Instagram and WhatsApp — the largest assistant install base there is. A robots.txt that names the two older Meta tokens does not cover this one.", "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "html_url": "https://www.pathwren.workers.dev/crawler/meta-webindexer.html", "json_url": "https://www.pathwren.workers.dev/crawler/meta-webindexer.json", "last_reviewed": "2026-09-01" }, { "slug": "chatgpt-agent", "name": "ChatGPT Agent", "operator": "OpenAI", "operator_slug": "openai", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "ChatGPT-User", "user_agent_substring": "ChatGPT Agent", "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/chatgpt-user.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-chatgpt-user.json", "ipv4_prefix_count": 204, "ipv6_prefix_count": 0, "what_it_is": "ChatGPT's agent mode driving a real browser: it navigates and interacts with sites to finish a multi-step task a user gave it. OpenAI governs it with the ChatGPT-User token and the ChatGPT-User prefix list rather than a token of its own, so the robots rule and the address check are the same ones.", "cost_of_blocking": "Agentic tasks a user asked for — booking, comparing, filling a form on your site — fail. This is the fetch that ends in a transaction, so it is the most expensive user-triggered block on this list.", "operator_docs": "https://platform.openai.com/docs/bots", "html_url": "https://www.pathwren.workers.dev/crawler/chatgpt-agent.html", "json_url": "https://www.pathwren.workers.dev/crawler/chatgpt-agent.json", "last_reviewed": "2026-09-01" }, { "slug": "wpbot", "name": "wpbot", "operator": "QuantumCloud", "operator_slug": "quantumcloud", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "wpbot", "user_agent_substring": "wpbot", "user_agent_example": "wpbot", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Supports the AI Chatbot for WordPress plugin: it reads pages so the plugin can answer from a site's own content. The operator provides an opt-out through a form rather than through robots.txt.", "cost_of_blocking": "A WordPress site running that plugin loses its own content as an answer source. Only relevant where the plugin is installed.", "operator_docs": "https://www.quantumcloud.com", "html_url": "https://www.pathwren.workers.dev/crawler/wpbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/wpbot.json", "last_reviewed": "2026-09-01" }, { "slug": "crawl4ai", "name": "Crawl4AI", "operator": "Crawl4AI project", "operator_slug": "crawl4ai", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "Crawl4AI", "user_agent_substring": "Crawl4AI", "user_agent_example": "Crawl4AI", "respects_robots_txt": "undocumented", "respects_robots_txt_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "An open-source LLM-oriented crawler and scraper library, run by whoever installs it. Like Scrapy, the default user-agent identifies the software and says nothing about who is behind the request.", "cost_of_blocking": "You block a library, not an operator: the rule catches a researcher and a bulk scraper equally, and anyone who edits one config line is not caught at all.", "operator_docs": "https://github.com/unclecode/crawl4ai", "html_url": "https://www.pathwren.workers.dev/crawler/crawl4ai.html", "json_url": "https://www.pathwren.workers.dev/crawler/crawl4ai.json", "last_reviewed": "2026-09-01" } ] }