{ "package": "ai-crawler-logs", "data_version": "2026-09-01T21:52:59+00:00", "generated_at": "2026-09-01T21:52:59+00:00", "source": "AI Crawler Index — https://www.pathwren.workers.dev — independent, non-commercial; data CC0-1.0, code MIT", "self_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/data.json", "docs_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/", "index_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/", "license": { "code": "MIT", "data": "CC0-1.0" }, "ai_categories": [ "ai-training", "ai-search", "user-fetch", "dataset" ], "categories": { "ai-training": { "label": "AI training crawlers", "description": "Collect pages in bulk so that a model can be trained or fine-tuned on them. Blocking these removes you from future training sets and changes nothing a user sees today." }, "ai-search": { "label": "AI search crawlers", "description": "Build the retrieval index an assistant answers and cites from. These are the crawlers that send you traffic; blocking them is the expensive mistake in this space." }, "user-fetch": { "label": "User-triggered fetchers", "description": "Fetch one page because a person asked for it, right then. One human intent, one request. Blocking them produces a visible error for a real reader." }, "dataset": { "label": "Corpus and dataset builders", "description": "Crawl the web into a published or resold dataset that other people train on. Highest leverage per block, longest delay before any effect." }, "search": { "label": "Search engines", "description": "Classic index-and-rank crawlers. Several also feed their operator's generative answers, which is why the AI opt-out for Google and Apple is a token rather than a block." }, "seo": { "label": "SEO and backlink crawlers", "description": "Commercial link-graph tooling. No user-facing effect either way, and usually a large share of your bot bandwidth." }, "archive": { "label": "Archivers", "description": "Preservation crawlers. Their output is public and permanent, which makes them a separate decision from the AI one." }, "preview": { "label": "Link preview fetchers", "description": "Read your Open Graph tags when someone shares a link. Blocking these is almost always an accident." }, "tool": { "label": "Tools and frameworks", "description": "Not operators: crawling software anyone can run. The party behind the request is unknown, so treat them as a rate-limit question rather than a consent question." } }, "crawlers": [ { "slug": "adsbot-google", "name": "AdsBot-Google", "operator": "Google", "operator_slug": "google", "token": "AdsBot-Google", "ua": "AdsBot-Google", "ua_example": "AdsBot-Google (+http://www.google.com/adsbot.html)", "category": "tool", "category_label": "Tools and frameworks", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "cost_of_blocking": "Google Ads cannot score your landing pages, which lowers Ad Rank on the ads pointing at them. If you do not buy ads, blocking it costs nothing but bandwidth savings.", "what_it_is": "Checks the quality of desktop landing pages for Google Ads. Google documents that it ignores the robots.txt * group with the ad publisher's permission, and obeys a group named for its own token.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers" }, { "slug": "adsbot-google-mobile", "name": "AdsBot-Google-Mobile", "operator": "Google", "operator_slug": "google", "token": "AdsBot-Google-Mobile", "ua": "AdsBot-Google-Mobile", "ua_example": "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)", "category": "tool", "category_label": "Tools and frameworks", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "cost_of_blocking": "Mobile ad landing pages go unscored and the ads pointing at them rank worse. No effect on organic search.", "what_it_is": "The mobile-web landing page checker for Google Ads. Same rules as AdsBot-Google: the * group does not apply to it, its own token does.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers" }, { "slug": "adsbot-google-mobile-apps", "name": "AdsBot-Google-Mobile-Apps", "operator": "Google", "operator_slug": "google", "token": "AdsBot-Google-Mobile-Apps", "ua": "AdsBot-Google-Mobile-Apps", "ua_example": "AdsBot-Google-Mobile-Apps", "category": "tool", "category_label": "Tools and frameworks", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "cost_of_blocking": "App-install ad landing pages go unscored. Nothing organic changes.", "what_it_is": "Checks Android app landing pages for Google Ads. It obeys a group named for its own token and, per Google, follows the AdsBot-Google rules otherwise.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers" }, { "slug": "ahrefsbot", "name": "AhrefsBot", "operator": "Ahrefs", "operator_slug": "ahrefs", "token": "AhrefsBot", "ua": "AhrefsBot", "ua_example": "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges", "cost_of_blocking": "No user-facing effect. Ahrefs honours Crawl-delay, so rate-limiting is usually better than blocking.", "what_it_is": "Ahrefs' backlink crawler, and one of the largest non-search crawlers on the web by request volume.", "docs": "https://ahrefs.com/robot" }, { "slug": "ahrefssiteaudit", "name": "AhrefsSiteAudit", "operator": "Ahrefs", "operator_slug": "ahrefs", "token": "AhrefsSiteAudit", "ua": "AhrefsSiteAudit", "ua_example": "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/site-audit)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges", "cost_of_blocking": "Site owners auditing your domain get an incomplete report. Blocking it saves bandwidth and costs you nothing in search.", "what_it_is": "Ahrefs' site-audit crawler, separate from AhrefsBot. Ahrefs documents that it obeys robots.txt by default, and that a verified site owner can ask for it to be allowed to ignore robots.txt on their own site so the audit can see disallowed sections.", "docs": "https://ahrefs.com/robot" }, { "slug": "ai2bot", "name": "AI2Bot", "operator": "Allen Institute for AI", "operator_slug": "ai2", "token": "AI2Bot", "ua": "AI2Bot", "ua_example": "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Excluded from open research datasets. Worth a deliberate decision: this is the category where 'blocking AI' also blocks the open, auditable end of it.", "what_it_is": "The Allen Institute's crawler, gathering pages for open research corpora such as Dolma that underpin fully open models like OLMo.", "docs": "https://allenai.org/crawler" }, { "slug": "ai2bot-dolma", "name": "Ai2Bot-Dolma", "operator": "Allen Institute for AI", "operator_slug": "ai2", "token": "Ai2Bot-Dolma", "ua": "Ai2Bot-Dolma", "ua_example": "Mozilla/5.0 (compatible) Ai2Bot-Dolma (+https://www.allenai.org/crawler)", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Same as AI2Bot: exclusion from an open, published training corpus.", "what_it_is": "The variant of AI2's crawler named for the Dolma corpus specifically.", "docs": "https://allenai.org/crawler" }, { "slug": "aihitbot", "name": "aiHitBot", "operator": "aiHit", "operator_slug": "aihit", "token": "aiHitBot", "ua": "aiHitBot", "ua_example": "aiHitBot", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your company record in a B2B dataset goes stale. Documented as respecting robots.txt, so the rule works.", "what_it_is": "aiHit's automated collector, building a company dataset from public company websites.", "docs": "https://www.aihitdata.com/about" }, { "slug": "aiwebindex", "name": "AIWebIndex", "operator": "Lyrenth", "operator_slug": "lyrenth", "token": "AIWebIndex", "ua": "AIWebIndex", "ua_example": "AIWebIndex", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Agents reading through this index stop seeing you — including the attribution and link back that make it a referral rather than a summary.", "what_it_is": "Builds an index of public pages and serves them to AI agents as extracted readable text, with attribution and a link back. Lyrenth publishes a crawler policy stating it does not train foundation models on what it collects and that it obeys robots.txt.", "docs": "https://lyrenth.com/crawler-policy" }, { "slug": "amazonbot", "name": "Amazonbot", "operator": "Amazon", "operator_slug": "amazon", "token": "Amazonbot", "ua": "Amazonbot", "ua_example": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Alexa and Amazon's assistants stop answering from your pages. Verify with reverse DNS to crawl.amazonbot.amazon before trusting the user-agent.", "what_it_is": "Amazon's crawler, feeding Alexa's ability to answer questions from the web and Amazon's own search and assistant products.", "docs": "https://developer.amazon.com/amazonbot" }, { "slug": "andibot", "name": "Andibot", "operator": "Andi", "operator_slug": "andi", "token": "Andibot", "ua": "Andibot", "ua_example": "Andibot", "category": "ai-search", "category_label": "AI search crawlers", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You disappear from another assistant's answers. Andi publishes no robots.txt statement.", "what_it_is": "The crawler for Andi, a small generative search assistant that summarises pages rather than listing them.", "docs": "https://andisearch.com/" }, { "slug": "anomura", "name": "Anomura", "operator": "Direqt", "operator_slug": "direqt", "token": "Anomura", "ua": "Anomura", "ua_example": "Anomura", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "If you are the publisher, this breaks the assistant you put on your own pages. If you are not, it should not be crawling you.", "what_it_is": "Direqt's search crawler. It indexes the sites of Direqt's own publisher customers so their on-site chatbots can answer from them.", "docs": "https://direqt.ai" }, { "slug": "anthropic-ai", "name": "anthropic-ai", "operator": "Anthropic", "operator_slug": "anthropic", "token": "anthropic-ai", "ua": "anthropic-ai", "ua_example": "(no live crawler currently identifies with this string)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "n-a", "respects_label": "control token only — no crawler", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "None. Nothing crawls under this name today; keeping the rule is harmless insurance.", "what_it_is": "A legacy robots.txt token from before Anthropic consolidated on ClaudeBot. It is still widely present in robots.txt files and costs nothing to keep, but it is a control token rather than a bot you will see in logs.", "docs": "https://support.anthropic.com/en/articles/8896518" }, { "slug": "apis-google", "name": "APIs-Google", "operator": "Google", "operator_slug": "google", "token": "APIs-Google", "ua": "APIs-Google", "ua_example": "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)", "category": "tool", "category_label": "Tools and frameworks", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "cost_of_blocking": "Google API push notifications stop arriving at your endpoint. This only affects services you set up yourself; there is no search or AI consequence.", "what_it_is": "Delivers push notifications for Google APIs to a webhook you registered. It is a special-case crawler: it ignores the robots.txt * group, because the fetch is a delivery to an address you asked it to deliver to.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers" }, { "slug": "applebot", "name": "Applebot", "operator": "Apple", "operator_slug": "apple", "token": "Applebot", "ua": "Applebot", "ua_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://search.developer.apple.com/applebot.json", "cost_of_blocking": "You disappear from Siri, Spotlight and Safari search suggestions across Apple's install base.", "what_it_is": "Powers Siri, Spotlight and Safari suggestions. Blocking it is a search decision, not an AI decision — the AI decision has its own token.", "docs": "https://support.apple.com/en-us/119829" }, { "slug": "applebot-extended", "name": "Applebot-Extended", "operator": "Apple", "operator_slug": "apple", "token": "Applebot-Extended", "ua": "(control token only — no crawler)", "ua_example": "(none: Applebot-Extended never appears as a user-agent)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "n-a", "respects_label": "control token only — no crawler", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Excluded from Apple Intelligence training. Siri, Spotlight and Safari suggestions are unaffected.", "what_it_is": "Apple's counterpart to Google-Extended: a robots.txt token that withdraws consent for Apple Intelligence and Apple foundation-model training, without touching Applebot's search crawl.", "docs": "https://support.apple.com/en-us/119829" }, { "slug": "archive-org-bot", "name": "archive.org_bot", "operator": "Internet Archive", "operator_slug": "internetarchive", "token": "archive.org_bot", "ua": "archive.org_bot", "ua_example": "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)", "category": "archive", "category_label": "Archivers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your site stops being preserved. When it dies, it is gone. Consider this one separately from the AI question.", "what_it_is": "The Wayback Machine's crawler. Preservation rather than AI, but it lands in the same 'is this bot welcome' decision and its output is a public corpus.", "docs": "https://archive.org/details/archive.org_bot" }, { "slug": "atlassian-bot", "name": "atlassian-bot", "operator": "Atlassian", "operator_slug": "atlassian", "token": "atlassian-bot", "ua": "atlassian-bot", "ua_example": "atlassian-bot", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Rovo cannot answer from your public documentation. If your customers live inside Atlassian tools, this is a support-deflection block.", "what_it_is": "Indexes a website so it can be searched and cited by Rovo, Atlassian's generative assistant inside Jira and Confluence. Atlassian's documentation walks a customer through editing robots.txt for it, which is as close to a compliance statement as this list gets.", "docs": "https://support.atlassian.com/organization-administration/docs/connect-custom-website-to-rovo/" }, { "slug": "awariorssbot", "name": "AwarioRssBot", "operator": "Awario", "operator_slug": "awario", "token": "AwarioRssBot", "ua": "AwarioRssBot", "ua_example": "AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your RSS updates stop reaching Awario's monitoring. Block both tokens or neither.", "what_it_is": "The feed-reading half of Awario's pair, documented on the same page and under the same crawl-rate policy.", "docs": "https://awario.com/bots.html" }, { "slug": "awariosmartbot", "name": "AwarioSmartBot", "operator": "Awario", "operator_slug": "awario", "token": "AwarioSmartBot", "ua": "AwarioSmartBot", "ua_example": "AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Mentions of brands on your pages stop being surfaced to the people monitoring them — including, quite possibly, your own.", "what_it_is": "Awario's brand-monitoring crawler. It documents one request per three seconds, honours Crawl-delay, and states it does not use consecutive IP blocks so identification is by user-agent only.", "docs": "https://awario.com/bots.html" }, { "slug": "baiduspider", "name": "Baiduspider", "operator": "Baidu", "operator_slug": "baidu", "token": "Baiduspider", "ua": "Baiduspider", "ua_example": "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Removal from Baidu Search, which matters only if you want Chinese-language traffic.", "what_it_is": "Baidu's search crawler, and the ingest path for Baidu's Ernie-backed answers.", "docs": "https://help.baidu.com/question?prod_id=99&class=0&id=3001" }, { "slug": "barkrowler", "name": "Barkrowler", "operator": "Babbar", "operator_slug": "babbar", "token": "barkrowler", "ua": "barkrowler", "ua_example": "Mozilla/5.0 (compatible; Barkrowler/0.9; +https://babbar.tech/crawler)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You leave Babbar's index. No effect on search or assistants.", "what_it_is": "Babbar's crawler, which builds the link graph behind their French-market SEO tooling.", "docs": "https://babbar.tech/crawler" }, { "slug": "bedrockbot", "name": "bedrockbot", "operator": "Amazon", "operator_slug": "amazon", "token": "bedrockbot", "ua": "bedrockbot", "ua_example": "bedrockbot-UUID", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Companies building retrieval applications on Bedrock cannot include your pages. This is a RAG block, not a training block: nothing is being trained, but nothing can cite you either.", "what_it_is": "The web crawler an AWS customer points at URLs they chose, to build a knowledge base for a Bedrock application. AWS documents that it respects robots.txt and that the user-agent carries a per-customer suffix, so you can allow or refuse one customer's crawl by naming bedrockbot-UUID.", "docs": "https://docs.aws.amazon.com/bedrock/latest/userguide/webcrawl-data-source-connector.html" }, { "slug": "bingbot", "name": "bingbot", "operator": "Microsoft", "operator_slug": "microsoft", "token": "bingbot", "ua": "bingbot", "ua_example": "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://www.bing.com/toolbox/bingbot.json", "cost_of_blocking": "Very high and very wide: Bing, Copilot, DuckDuckGo and several assistants that resell Bing's index all lose you at once. Use nocache/noarchive rather than blocking.", "what_it_is": "Bing's only crawler, and therefore also the crawler behind Microsoft Copilot's grounding. Microsoft's documented way to keep search indexing while refusing generative reuse is the nocache / noarchive robots meta directive, not a separate user-agent.", "docs": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0" }, { "slug": "bytespider", "name": "Bytespider", "operator": "ByteDance", "operator_slug": "bytedance", "token": "Bytespider", "ua": "Bytespider", "ua_example": "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "disputed", "respects_label": "compliance disputed", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Little to lose. If you want it gone, expect to block by user-agent at the edge rather than to ask politely in robots.txt.", "what_it_is": "ByteDance's crawler, associated with training data collection for Doubao and related models. Repeatedly reported by CDNs and site operators as the highest-volume AI crawler on the web and as inconsistent about robots.txt.", "docs": "https://www.bytespider.net/" }, { "slug": "ccbot", "name": "CCBot", "operator": "Common Crawl", "operator_slug": "commoncrawl", "token": "CCBot", "ua": "CCBot", "ua_example": "CCBot/2.0 (https://commoncrawl.org/faq/)", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://index.commoncrawl.org/ccbot.json", "cost_of_blocking": "Future Common Crawl snapshots exclude you, so downstream training sets lose you too — but only going forward. Existing snapshots are permanent and blocking today does not retract them.", "what_it_is": "Common Crawl's corpus builder. It trains nothing itself, but its archive is an input to most open and many closed LLM training sets, which makes it the highest-leverage single entry on this list.", "docs": "https://commoncrawl.org/ccbot" }, { "slug": "chatgpt-agent", "name": "ChatGPT Agent", "operator": "OpenAI", "operator_slug": "openai", "token": "ChatGPT-User", "ua": "ChatGPT Agent", "ua_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot", "category": "user-fetch", "category_label": "User-triggered fetchers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/chatgpt-user.json", "cost_of_blocking": "Agentic tasks a user asked for — booking, comparing, filling a form on your site — fail. This is the fetch that ends in a transaction, so it is the most expensive user-triggered block on this list.", "what_it_is": "ChatGPT's agent mode driving a real browser: it navigates and interacts with sites to finish a multi-step task a user gave it. OpenAI governs it with the ChatGPT-User token and the ChatGPT-User prefix list rather than a token of its own, so the robots rule and the address check are the same ones.", "docs": "https://platform.openai.com/docs/bots" }, { "slug": "chatgpt-user", "name": "ChatGPT-User", "operator": "OpenAI", "operator_slug": "openai", "token": "ChatGPT-User", "ua": "ChatGPT-User", "ua_example": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; ChatGPT-User/1.0; +https://openai.com/bot", "category": "user-fetch", "category_label": "User-triggered fetchers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/chatgpt-user.json", "cost_of_blocking": "ChatGPT cannot open your pages when a user explicitly asks it to. The user sees a fetch failure. This is usually the last bot anyone means to block.", "what_it_is": "Fetches a single page at the moment a user or a ChatGPT agent asks for it — a pasted link, a browsing step, an Operator task. One human intent, one request. OpenAI states these fetches are not used for training.", "docs": "https://platform.openai.com/docs/bots" }, { "slug": "claude-searchbot", "name": "Claude-SearchBot", "operator": "Anthropic", "operator_slug": "anthropic", "token": "Claude-SearchBot", "ua": "Claude-SearchBot", "ua_example": "Mozilla/5.0 (compatible; Claude-SearchBot/1.0; +Claude-SearchBot@anthropic.com)", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You stop appearing in Claude's search results and citations.", "what_it_is": "Indexes pages so Claude's web search can find and cite them. Separate token from the training crawler, so search visibility and training consent are independent decisions.", "docs": "https://support.anthropic.com/en/articles/8896518" }, { "slug": "claude-user", "name": "Claude-User", "operator": "Anthropic", "operator_slug": "anthropic", "token": "Claude-User", "ua": "Claude-User", "ua_example": "Mozilla/5.0 (compatible; Claude-User/1.0; +Claude-User@anthropic.com)", "category": "user-fetch", "category_label": "User-triggered fetchers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Claude reports a fetch failure to a user who asked for your page by name.", "what_it_is": "Fetches a page because a Claude user asked Claude to read it, at that moment.", "docs": "https://support.anthropic.com/en/articles/8896518" }, { "slug": "claude-web", "name": "Claude-Web", "operator": "Anthropic", "operator_slug": "anthropic", "token": "Claude-Web", "ua": "Claude-Web", "ua_example": "Mozilla/5.0 (compatible; Claude-Web/1.0)", "category": "ai-search", "category_label": "AI search crawlers", "respects": "n-a", "respects_label": "control token only — no crawler", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "None in practice. Retain the rule; expect no traffic.", "what_it_is": "An earlier Anthropic token for user-facing web access, superseded by Claude-User and Claude-SearchBot. Kept here because it appears in most published robots.txt templates.", "docs": "https://support.anthropic.com/en/articles/8896518" }, { "slug": "claudebot", "name": "ClaudeBot", "operator": "Anthropic", "operator_slug": "anthropic", "token": "ClaudeBot", "ua": "ClaudeBot", "ua_example": "Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Content excluded from training data for future Claude models. No effect on Claude's ability to fetch a link a user gives it.", "what_it_is": "Anthropic's bulk crawler, gathering pages that may be used to train Claude models.", "docs": "https://support.anthropic.com/en/articles/8896518" }, { "slug": "cloudflare-autorag", "name": "Cloudflare-AutoRAG", "operator": "Cloudflare", "operator_slug": "cloudflare", "token": "Cloudflare-AutoRAG", "ua": "Cloudflare-AutoRAG", "ua_example": "Cloudflare-AutoRAG", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Applications built on Cloudflare AI Search cannot retrieve your pages. If you are the one building the index over your own site, blocking it breaks your own product.", "what_it_is": "The crawler behind Cloudflare's AI Search / AutoRAG, which indexes a website into a retrieval index for an application. Cloudflare's own documentation warns that a bot-blocking rule on your zone will also stop this crawler and tells you to allow-list it.", "docs": "https://developers.cloudflare.com/ai-search/" }, { "slug": "cohere-ai", "name": "cohere-ai", "operator": "Cohere", "operator_slug": "cohere", "token": "cohere-ai", "ua": "cohere-ai", "ua_example": "Mozilla/5.0 (compatible; cohere-ai)", "category": "user-fetch", "category_label": "User-triggered fetchers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Cohere-powered assistants cannot read your pages on request.", "what_it_is": "Cohere's fetcher, used when its assistant products need a page.", "docs": "https://cohere.com/" }, { "slug": "cohere-training-data-crawler", "name": "cohere-training-data-crawler", "operator": "Cohere", "operator_slug": "cohere", "token": "cohere-training-data-crawler", "ua": "cohere-training-data-crawler", "ua_example": "Mozilla/5.0 (compatible; cohere-training-data-crawler)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Excluded from Cohere model training.", "what_it_is": "Cohere's separately-named bulk crawler for model training data, split out so consent for training and consent for retrieval can differ.", "docs": "https://cohere.com/" }, { "slug": "cotoyogi", "name": "Cotoyogi", "operator": "ROIS-DS", "operator_slug": "rois", "token": "Cotoyogi", "ua": "Cotoyogi", "ua_example": "Cotoyogi", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your Japanese-language content is left out of an academic training corpus.", "what_it_is": "A crawler run by ROIS-DS, a Japanese inter-university research organisation, collecting Japanese-language text for AI training. It publishes a crawler page in English and Japanese.", "docs": "https://ds.rois.ac.jp/en_center8/en_crawler/" }, { "slug": "crawl4ai", "name": "Crawl4AI", "operator": "Crawl4AI project", "operator_slug": "crawl4ai", "token": "Crawl4AI", "ua": "Crawl4AI", "ua_example": "Crawl4AI", "category": "tool", "category_label": "Tools and frameworks", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You block a library, not an operator: the rule catches a researcher and a bulk scraper equally, and anyone who edits one config line is not caught at all.", "what_it_is": "An open-source LLM-oriented crawler and scraper library, run by whoever installs it. Like Scrapy, the default user-agent identifies the software and says nothing about who is behind the request.", "docs": "https://github.com/unclecode/crawl4ai" }, { "slug": "crawlspace", "name": "Crawlspace", "operator": "Crawlspace", "operator_slug": "crawlspace", "token": "Crawlspace", "ua": "Crawlspace", "ua_example": "Crawlspace", "category": "tool", "category_label": "Tools and frameworks", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Whatever any Crawlspace customer was building over your pages stops working. Volume and intent vary per customer, so this is a rate-limit decision more than a consent one.", "what_it_is": "A crawling platform: customers run their own crawls on it to feed agents, RAG pipelines and structured-data workflows. Like Firecrawl, the party behind any given request is the customer, not the platform.", "docs": "https://crawlspace.dev" }, { "slug": "dataforseobot", "name": "DataForSeoBot", "operator": "DataForSEO", "operator_slug": "dataforseo", "token": "DataForSeoBot", "ua": "DataForSeoBot", "ua_example": "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You leave a dataset that a long tail of SEO products is built on. No user-facing effect.", "what_it_is": "Builds the backlink and SERP datasets DataForSEO resells through its API, so one crawl reaches many downstream tools.", "docs": "https://dataforseo.com/dataforseo-bot" }, { "slug": "diffbot", "name": "Diffbot", "operator": "Diffbot", "operator_slug": "diffbot", "token": "Diffbot", "ua": "Diffbot", "ua_example": "Mozilla/5.0 (compatible; Diffbot/0.1; +http://www.diffbot.com)", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your facts stop entering a widely-licensed knowledge graph. Whether that is a loss depends on whether you want to be a machine-readable entity.", "what_it_is": "Extracts structured records from pages to build a commercial knowledge graph that is resold and used for retrieval and training.", "docs": "https://docs.diffbot.com/" }, { "slug": "dotbot", "name": "DotBot", "operator": "Moz", "operator_slug": "moz", "token": "dotbot", "ua": "dotbot", "ua_example": "Mozilla/5.0 (compatible; DotBot/1.2; +https://opensiteexplorer.org/dotbot; help@moz.com)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You leave Moz's link index, so Domain Authority and link reports about your site get thinner. Nothing a reader or an assistant sees changes.", "what_it_is": "Moz's crawler for Link Explorer. Moz documents that it respects robots.txt and that dotbot is the token to name.", "docs": "https://moz.com/help/moz-procedures/crawlers/dotbot" }, { "slug": "duckassistbot", "name": "DuckAssistBot", "operator": "DuckDuckGo", "operator_slug": "duckduckgo", "token": "DuckAssistBot", "ua": "DuckAssistBot", "ua_example": "Mozilla/5.0 (compatible; DuckAssistBot/1.0; +https://duckduckgo.com/duckassistbot)", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "No DuckAssist answers or citations from your site. Ordinary DuckDuckGo results are unaffected.", "what_it_is": "Fetches pages so DuckAssist can generate and cite answers inside DuckDuckGo.", "docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/" }, { "slug": "duckduckbot", "name": "DuckDuckBot", "operator": "DuckDuckGo", "operator_slug": "duckduckgo", "token": "DuckDuckBot", "ua": "DuckDuckBot", "ua_example": "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://duckduckgo.com/duckduckbot.json", "cost_of_blocking": "Limited on its own; the real DuckDuckGo lever is bingbot.", "what_it_is": "DuckDuckGo's own crawler. Note that the bulk of DuckDuckGo's web results come from Bing, so blocking bingbot removes you from DuckDuckGo whether or not you allow this one.", "docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/" }, { "slug": "echoboxbot", "name": "EchoboxBot", "operator": "Echobox", "operator_slug": "echobox", "token": "EchoboxBot", "ua": "EchoboxBot", "ua_example": "EchoboxBot", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Publishers using Echobox get worse scheduling decisions about your articles. No compliance statement is published.", "what_it_is": "Collects data supporting Echobox's AI-driven social and email distribution products, which publishers use to schedule and target their own content.", "docs": "https://echobox.com" }, { "slug": "exasearchbot", "name": "ExaSearchBot", "operator": "Exa", "operator_slug": "exa", "token": "ExaSearchBot", "ua": "ExaSearchBot", "ua_example": "ExaSearchBot", "category": "ai-search", "category_label": "AI search crawlers", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Agents built on Exa's API stop finding you. Exa publishes no statement about robots.txt compliance, so treat the rule as a request.", "what_it_is": "Exa's crawler. It discovers and indexes public pages so they can be retrieved and cited through Exa's search API, which is one of the common retrieval backends behind agent frameworks.", "docs": "https://exa.ai" }, { "slug": "facebookbot", "name": "FacebookBot", "operator": "Meta", "operator_slug": "meta", "token": "FacebookBot", "ua": "FacebookBot", "ua_example": "FacebookBot/1.0 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Negligible today. Keep the rule; expect little traffic.", "what_it_is": "Meta's older speech- and language-corpus crawler, largely superseded by meta-externalagent but still listed as a valid robots token.", "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers" }, { "slug": "facebookexternalhit", "name": "facebookexternalhit", "operator": "Meta", "operator_slug": "meta", "token": "facebookexternalhit", "ua": "facebookexternalhit", "ua_example": "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", "category": "preview", "category_label": "Link preview fetchers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Severe and usually accidental. Your links share as bare grey boxes with no title, image or description across Facebook, Instagram, Messenger and WhatsApp. Almost nobody means to block this.", "what_it_is": "The link unfurler: it reads your Open Graph tags when somebody shares your URL on a Meta property.", "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers" }, { "slug": "factset-spyderbot", "name": "Factset_spyderbot", "operator": "FactSet", "operator_slug": "factset", "token": "Factset_spyderbot", "ua": "Factset_spyderbot", "ua_example": "Factset_spyderbot", "category": "ai-training", "category_label": "AI training crawlers", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Exclusion from a financial-data vendor's corpus. Relevant mostly to companies whose filings and disclosures are being read.", "what_it_is": "FactSet's crawler, collecting data used in AI model training for its financial data and analytics products.", "docs": "https://www.factset.com/ai" }, { "slug": "feedfetcher-google", "name": "FeedFetcher-Google", "operator": "Google", "operator_slug": "google", "token": "FeedFetcher-Google", "ua": "FeedFetcher-Google", "ua_example": "FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)", "category": "tool", "category_label": "Tools and frameworks", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "cost_of_blocking": "Feed-driven Google products stop seeing your updates. A robots.txt rule will not stop it — block by user-agent at the edge if you mean it.", "what_it_is": "Crawls RSS and Atom feeds for Google News and WebSub. It is a user-triggered fetcher, and Google documents that those generally ignore robots.txt because a person asked for the fetch. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers" }, { "slug": "firecrawlagent", "name": "FirecrawlAgent", "operator": "Firecrawl", "operator_slug": "firecrawl", "token": "FirecrawlAgent", "ua": "FirecrawlAgent", "ua_example": "Mozilla/5.0 (compatible; FirecrawlAgent/1.0; +https://firecrawl.dev)", "category": "tool", "category_label": "Tools and frameworks", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Applications built on Firecrawl cannot read your pages. This is increasingly how agents fetch the web, so it is a bigger block than its name suggests.", "what_it_is": "A hosted scrape-to-markdown service that LLM applications call to read pages. The requester is whoever is building on it, not Firecrawl itself, so volume and intent vary wildly.", "docs": "https://docs.firecrawl.dev/" }, { "slug": "google-agent", "name": "Google-Agent", "operator": "Google", "operator_slug": "google", "token": "Google-Agent", "ua": "Google-Agent", "ua_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent) Chrome/W.X.Y.Z Safari/537.36", "category": "user-fetch", "category_label": "User-triggered fetchers", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-agents.json", "cost_of_blocking": "Google-hosted agents cannot complete a task on your site for a user who asked them to. This is the agentic-commerce fetch: blocking it removes you from what an assistant can actually do rather than from what it can say.", "what_it_is": "Agents hosted on Google infrastructure navigating the web and taking actions on a user's request. Google names one prefix list for it — user-triggered-agents.json — and is separately experimenting with Web Bot Auth under the identity https://agent.bot.goog.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers" }, { "slug": "google-cloudvertexbot", "name": "Google-CloudVertexBot", "operator": "Google", "operator_slug": "google", "token": "Google-CloudVertexBot", "ua": "Google-CloudVertexBot", "ua_example": "Mozilla/5.0 (compatible; Google-CloudVertexBot/1.0; +https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers)", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "cost_of_blocking": "Third parties can no longer build Vertex AI agents that read your site. Irrelevant to Google Search.", "what_it_is": "Crawls a site on behalf of a Vertex AI Agent Builder customer who is building an agent over that site. It only visits sites the customer has asked it to.", "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers" }, { "slug": "google-cws", "name": "Google-CWS", "operator": "Google", "operator_slug": "google", "token": "Google-CWS", "ua": "Google-CWS", "ua_example": "Mozilla/5.0 (compatible; Google-CWS)", "category": "tool", "category_label": "Tools and frameworks", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "cost_of_blocking": "Chrome Web Store listings that point at your pages cannot fetch them. Relevant only if you publish extensions.", "what_it_is": "The Chrome Web Store fetcher. It requests the URLs a developer put in the metadata of a Chrome extension or theme. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers" }, { "slug": "google-extended", "name": "Google-Extended", "operator": "Google", "operator_slug": "google", "token": "Google-Extended", "ua": "(control token only — no crawler)", "ua_example": "(none: Google-Extended never appears as a user-agent)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "n-a", "respects_label": "control token only — no crawler", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You are excluded from Gemini grounding and Gemini training. Google Search ranking and indexing are explicitly unaffected. This is the cleanest 'no training, keep my search traffic' lever that exists.", "what_it_is": "Not a crawler. A robots.txt token that tells Google whether pages Googlebot already fetched may be used to train and ground Gemini. You will never see it in an access log; disallowing it changes what Google does with content it fetched under a different name.", "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers" }, { "slug": "google-gemininotebook", "name": "Google-GeminiNotebook", "operator": "Google", "operator_slug": "google", "token": "Google-GeminiNotebook", "ua": "Google-GeminiNotebook", "ua_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 (compatible; Google-GeminiNotebook; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-gemininotebook)", "category": "user-fetch", "category_label": "User-triggered fetchers", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "cost_of_blocking": "A user who deliberately added your page as a research source gets nothing. This is a citation-shaped fetch, not a training crawl.", "what_it_is": "Fetches a URL a Gemini Notebook (formerly NotebookLM) user added as a source to their notebook. The former agent string Google-NotebookLM is documented as supported until August 2026. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers" }, { "slug": "google-inspectiontool", "name": "Google-InspectionTool", "operator": "Google", "operator_slug": "google", "token": "Google-InspectionTool", "ua": "Google-InspectionTool", "ua_example": "Mozilla/5.0 (compatible; Google-InspectionTool/1.0;)", "category": "tool", "category_label": "Tools and frameworks", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "cost_of_blocking": "Your own Search Console live tests stop working. Blocking this only hurts you.", "what_it_is": "The fetcher behind Search Console's URL Inspection and the Rich Results Test. It runs when a site owner clicks a button.", "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers" }, { "slug": "google-pinpoint", "name": "Google-Pinpoint", "operator": "Google", "operator_slug": "google", "token": "Google-Pinpoint", "ua": "Google-Pinpoint", "ua_example": "Google-Pinpoint", "category": "user-fetch", "category_label": "User-triggered fetchers", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "cost_of_blocking": "A researcher who explicitly added your page to a collection cannot load it.", "what_it_is": "Fetches individual URLs that a Pinpoint user — usually a journalist or researcher — added as a source to their own document collection. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers" }, { "slug": "google-read-aloud", "name": "Google-Read-Aloud", "operator": "Google", "operator_slug": "google", "token": "Google-Read-Aloud", "ua": "Google-Read-Aloud", "ua_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 (compatible; Google-Read-Aloud; +https://support.google.com/webmasters/answer/1061943)", "category": "user-fetch", "category_label": "User-triggered fetchers", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "cost_of_blocking": "A reader who asked Google to read your page aloud — often someone using it for accessibility — gets an error instead.", "what_it_is": "Fetches a page so Google can read it out loud with text-to-speech, at the moment a user asks. Formerly google-speakr. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers" }, { "slug": "google-safety", "name": "Google-Safety", "operator": "Google", "operator_slug": "google", "token": "Google-Safety", "ua": "Google-Safety", "ua_example": "Google-Safety", "category": "tool", "category_label": "Tools and frameworks", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Nothing you can control. The rule is ignored by design; listing the token is documentation, not enforcement.", "what_it_is": "Google's abuse-investigation fetcher: malware review, phishing reports and similar. Google documents that it ignores robots.txt entirely, and a robots.txt rule for it does nothing.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers" }, { "slug": "google-site-verification", "name": "Google-Site-Verification", "operator": "Google", "operator_slug": "google", "token": "Google-Site-Verification", "ua": "Google-Site-Verification", "ua_example": "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)", "category": "tool", "category_label": "Tools and frameworks", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "cost_of_blocking": "Your own Search Console verification fails. Blocking this only ever hurts the person doing the blocking.", "what_it_is": "Fetches the token file or meta tag that proves you own a site, when you click verify in Search Console. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers" }, { "slug": "googlebot", "name": "Googlebot", "operator": "Google", "operator_slug": "google", "token": "Googlebot", "ua": "Googlebot", "ua_example": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "cost_of_blocking": "Total. You leave Google Search. Never block this to avoid AI use; use Google-Extended instead.", "what_it_is": "The classic search crawler. It is also the crawler behind AI Overviews: Google does not run a separate bot for them, which is why the only AI opt-out is the Google-Extended token and not a Googlebot block.", "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers" }, { "slug": "googlebot-image", "name": "Googlebot-Image", "operator": "Google", "operator_slug": "google", "token": "Googlebot-Image", "ua": "Googlebot-Image", "ua_example": "Googlebot-Image/1.0", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "cost_of_blocking": "Your images stop appearing in Google Images.", "what_it_is": "Image indexing for Google Images. A separate token so you can leave images out of search without leaving search.", "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers" }, { "slug": "googlebot-news", "name": "Googlebot-News", "operator": "Google", "operator_slug": "google", "token": "Googlebot-News", "ua": "Googlebot-News", "ua_example": "(uses the Googlebot user-agent; controlled by the Googlebot-News robots token)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "cost_of_blocking": "Removal from Google News, with normal Search unaffected.", "what_it_is": "A robots.txt token controlling inclusion in Google News. It does not have its own user-agent string; the fetch arrives as Googlebot.", "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers" }, { "slug": "googlebot-video", "name": "Googlebot-Video", "operator": "Google", "operator_slug": "google", "token": "Googlebot-Video", "ua": "Googlebot-Video", "ua_example": "Googlebot-Video/1.0", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "cost_of_blocking": "Your videos leave Google video search. A rule for Googlebot already covers it, so blocking this token alone is usually a mistake of precision rather than of intent.", "what_it_is": "The video half of Googlebot. It crawls video files and the pages around them for Google Video search, and it is matched by a robots.txt group for Googlebot as well as by its own token.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers" }, { "slug": "googlemessages", "name": "GoogleMessages", "operator": "Google", "operator_slug": "google", "token": "GoogleMessages", "ua": "GoogleMessages", "ua_example": "GoogleMessages", "category": "preview", "category_label": "Link preview fetchers", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "cost_of_blocking": "Your links appear as bare URLs with no title or image in Google Messages chats. A preview fetcher is almost never the one you meant to block.", "what_it_is": "Generates the link preview when somebody sends one of your URLs in Google Messages. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers" }, { "slug": "googleother", "name": "GoogleOther", "operator": "Google", "operator_slug": "google", "token": "GoogleOther", "ua": "GoogleOther", "ua_example": "Mozilla/5.0 (compatible; GoogleOther)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "cost_of_blocking": "No effect on Search indexing. Blocks internal Google research and product fetches.", "what_it_is": "A generic fetcher used by Google product teams for one-off crawls and research, including data collection that does not belong to Search.", "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers" }, { "slug": "googleother-image", "name": "GoogleOther-Image", "operator": "Google", "operator_slug": "google", "token": "GoogleOther-Image", "ua": "GoogleOther-Image", "ua_example": "GoogleOther-Image/1.0", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "cost_of_blocking": "Google teams outside Search stop fetching your images. Image Search itself is unaffected — that is Googlebot-Image.", "what_it_is": "The image variant of GoogleOther: one-off fetches by Google product and research teams that are not Search. It also answers to a GoogleOther group in robots.txt.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers" }, { "slug": "googleother-video", "name": "GoogleOther-Video", "operator": "Google", "operator_slug": "google", "token": "GoogleOther-Video", "ua": "GoogleOther-Video", "ua_example": "GoogleOther-Video/1.0", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "cost_of_blocking": "No effect on Search or on Google Video search. Blocks internal Google research fetches of your video files.", "what_it_is": "The video variant of GoogleOther, used for internal Google fetches that do not belong to Search.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers" }, { "slug": "googleproducer", "name": "GoogleProducer", "operator": "Google", "operator_slug": "google", "token": "GoogleProducer", "ua": "GoogleProducer", "ua_example": "GoogleProducer; (+https://developers.google.com/search/docs/crawling-indexing/google-producer)", "category": "tool", "category_label": "Tools and frameworks", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "cost_of_blocking": "Your own Google News landing pages stop updating. Only publishers who configured Publisher Center are affected.", "what_it_is": "Google Publisher Center: fetches the feeds a publisher explicitly supplied for Google News landing pages. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers" }, { "slug": "gptbot", "name": "GPTBot", "operator": "OpenAI", "operator_slug": "openai", "token": "GPTBot", "ua": "GPTBot", "ua_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2; +https://openai.com/gptbot", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/gptbot.json", "cost_of_blocking": "Your content is excluded from training data for future OpenAI models. No effect on ChatGPT search visibility, on citations, or on links a user pastes into ChatGPT.", "what_it_is": "OpenAI's bulk crawler. Pages it fetches may be used to train future OpenAI foundation models. It is not the bot that puts you in ChatGPT's search results, and blocking it does not remove you from them.", "docs": "https://platform.openai.com/docs/bots" }, { "slug": "ia-archiver", "name": "ia_archiver", "operator": "Internet Archive", "operator_slug": "internetarchive", "token": "ia_archiver", "ua": "ia_archiver", "ua_example": "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)", "category": "archive", "category_label": "Archivers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Negligible today; retain for tidiness.", "what_it_is": "The legacy Alexa/Internet Archive token, still present in most robots.txt files and still occasionally honoured.", "docs": "https://archive.org/details/archive.org_bot" }, { "slug": "icc-crawler", "name": "ICC-Crawler", "operator": "NICT", "operator_slug": "nict", "token": "ICC-Crawler", "ua": "ICC-Crawler", "ua_example": "ICC-Crawler", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You are excluded from a national research corpus and from the commercial redistributions of it. This is a dataset-shaped block: one refusal, many downstream effects.", "what_it_is": "Operated by NICT, Japan's national information and communications research institute. The collected data supports AI research and, per the operator, is also provided to third parties including commercial companies.", "docs": "https://www.nict.go.jp/en/" }, { "slug": "imagesiftbot", "name": "ImagesiftBot", "operator": "Hive AI", "operator_slug": "hive", "token": "ImagesiftBot", "ua": "ImagesiftBot", "ua_example": "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your images stop entering an image dataset and reverse-image index.", "what_it_is": "Crawls images for Hive AI's reverse-image and dataset products. Image-heavy sites see this one long before they see the text crawlers.", "docs": "https://imagesift.com/about" }, { "slug": "img2dataset", "name": "img2dataset", "operator": "LAION / img2dataset", "operator_slug": "laion", "token": "img2dataset", "ua": "img2dataset", "ua_example": "img2dataset", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your images are skipped when someone materialises an image-text dataset that references them.", "what_it_is": "The tool used to turn image-URL lists such as LAION's into downloaded training sets. It is run by whoever is building a dataset, not by a single operator.", "docs": "https://github.com/rom1504/img2dataset" }, { "slug": "isscyberriskcrawler", "name": "ISSCyberRiskCrawler", "operator": "ISS Corporate Solutions", "operator_slug": "iss", "token": "ISSCyberRiskCrawler", "ua": "ISSCyberRiskCrawler", "ua_example": "ISSCyberRiskCrawler", "category": "ai-training", "category_label": "AI training crawlers", "respects": "disputed", "respects_label": "compliance disputed", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "A rule here is a statement of intent. Your organisation's public footprint still gets scored — by a model trained on everybody else.", "what_it_is": "Crawls in order to train models that score a company's cyber risk. The ai.robots.txt dataset records the operator as not respecting robots.txt; ISS publishes no compliance statement of its own.", "docs": "https://iss-cyber.com" }, { "slug": "kagibot", "name": "Kagibot", "operator": "Kagi", "operator_slug": "kagi", "token": "Kagibot", "ua": "Kagibot", "ua_example": "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You leave Kagi's index. Kagi's users are paying to search and skew technical; per visitor this is an expensive block.", "what_it_is": "The crawler for Kagi, a paid, ad-free search engine with its own index and its own assistant.", "docs": "https://kagi.com/bot" }, { "slug": "klaviyoaibot", "name": "KlaviyoAIBot", "operator": "Klaviyo", "operator_slug": "klaviyo", "token": "KlaviyoAIBot", "ua": "KlaviyoAIBot", "ua_example": "KlaviyoAIBot", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "If the connected domain is yours, blocking this breaks the agent you configured. If it is not, this bot should not be reaching you at all.", "what_it_is": "Fetches pages from domains a Klaviyo customer has explicitly connected to their own account, to power Klaviyo's Kai customer agent. It is scoped to connected domains rather than the open web.", "docs": "https://help.klaviyo.com/hc/en-us/articles/40496146232219" }, { "slug": "laiondownloader", "name": "LAIONDownloader", "operator": "LAION / img2dataset", "operator_slug": "laion", "token": "LAIONDownloader", "ua": "LAIONDownloader", "ua_example": "LAIONDownloader", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your media is skipped when an open research dataset is built from URL lists. Once a dataset is published, a later block does not remove you from it.", "what_it_is": "LAION's downloader, used to materialise the image and text datasets the non-profit publishes for machine-learning research. LAION's own FAQ is the source for its robots.txt position.", "docs": "https://laion.ai/faq/" }, { "slug": "lightpanda", "name": "Lightpanda", "operator": "Lightpanda", "operator_slug": "lightpanda", "token": "Lightpanda", "ua": "Lightpanda", "ua_example": "Lightpanda", "category": "tool", "category_label": "Tools and frameworks", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You block a browser, not a company: the same rule stops a scraper and a legitimate automation a customer of yours is running.", "what_it_is": "A purpose-built headless browser for AI and automation — a runtime, not an operator. Whether robots.txt is honoured is left to whoever runs it, which is what its maintainers say themselves.", "docs": "https://lightpanda.io/" }, { "slug": "linguee-bot", "name": "Linguee Bot", "operator": "Linguee", "operator_slug": "linguee", "token": "Linguee Bot", "ua": "Linguee Bot", "ua_example": "Linguee Bot", "category": "ai-training", "category_label": "AI training crawlers", "respects": "disputed", "respects_label": "compliance disputed", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Multilingual pages stop feeding a translation corpus. If your site is translated, being in it is usually a benefit.", "what_it_is": "Gathers bilingual text for Linguee's translation corpus and the machine translation trained on it. Recorded in the ai.robots.txt dataset as not respecting robots.txt.", "docs": "https://www.linguee.com" }, { "slug": "mediapartners-google", "name": "Mediapartners-Google", "operator": "Google", "operator_slug": "google", "token": "Mediapartners-Google", "ua": "Mediapartners-Google", "ua_example": "Mediapartners-Google", "category": "tool", "category_label": "Tools and frameworks", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "cost_of_blocking": "Pages it cannot read get generic, lower-value AdSense ads or none at all. This is the one block on this list that costs you money directly if you run AdSense.", "what_it_is": "The AdSense crawler. It reads a page so AdSense can choose relevant ads for it, and it is a special-case crawler that ignores the robots.txt * group.", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers" }, { "slug": "meta-externalagent", "name": "meta-externalagent", "operator": "Meta", "operator_slug": "meta", "token": "meta-externalagent", "ua": "meta-externalagent", "ua_example": "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Excluded from Meta AI training. Link previews on Facebook, Instagram and WhatsApp are unaffected — those are a different bot.", "what_it_is": "Meta's AI crawler, gathering training data for Llama and Meta AI. It replaced the older FacebookBot name for this purpose.", "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers" }, { "slug": "meta-externalfetcher", "name": "meta-externalfetcher", "operator": "Meta", "operator_slug": "meta", "token": "meta-externalfetcher", "ua": "meta-externalfetcher", "ua_example": "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "category": "user-fetch", "category_label": "User-triggered fetchers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Meta AI cannot read pages users hand it.", "what_it_is": "Fetches a page when a Meta AI user asks about a specific link.", "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers" }, { "slug": "meta-webindexer", "name": "Meta-WebIndexer", "operator": "Meta", "operator_slug": "meta", "token": "Meta-WebIndexer", "ua": "Meta-WebIndexer", "ua_example": "Meta-WebIndexer", "category": "ai-search", "category_label": "AI search crawlers", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You leave the index Meta AI answers from across Facebook, Instagram and WhatsApp — the largest assistant install base there is. A robots.txt that names the two older Meta tokens does not cover this one.", "what_it_is": "Per Meta's crawler documentation, Meta-WebIndexer navigates the web to improve the quality of Meta AI's search results. It is a third Meta token alongside Meta-ExternalAgent and Meta-ExternalFetcher, and the newest of them.", "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/" }, { "slug": "mistralai-user", "name": "MistralAI-User", "operator": "Mistral AI", "operator_slug": "mistral", "token": "MistralAI-User", "ua": "MistralAI-User", "ua_example": "Mozilla/5.0 (compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)", "category": "user-fetch", "category_label": "User-triggered fetchers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Le Chat cannot open links your readers give it.", "what_it_is": "Fetches a page when a Le Chat user asks Mistral's assistant to read it.", "docs": "https://docs.mistral.ai/" }, { "slug": "mj12bot", "name": "MJ12bot", "operator": "Majestic", "operator_slug": "majestic", "token": "MJ12bot", "ua": "MJ12bot", "ua_example": "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You leave the Majestic backlink index. No search or AI effect. It supports Crawl-delay, which is usually the better answer than a block.", "what_it_is": "Majestic's link-graph crawler, run as a distributed community project. Majestic states plainly that it cannot restrict the bot to a fixed set of addresses, and offers a pre-arranged ident string in the request headers instead.", "docs": "https://mj12bot.com/" }, { "slug": "mojeekbot", "name": "MojeekBot", "operator": "Mojeek", "operator_slug": "mojeek", "token": "MojeekBot", "ua": "MojeekBot", "ua_example": "Mozilla/5.0 (compatible; MojeekBot/0.11; +https://www.mojeek.com/bot.html)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You leave an independent index that other privacy-focused search products draw on. Small traffic, disproportionate long-term cost to web plurality.", "what_it_is": "Mojeek's crawler. Mojeek runs one of the few genuinely independent web indexes — not a front end over Bing or Google — so it is one of the few blocks that removes you from an index nobody else can put you back into. Its documentation states it obeys the first record whose User-Agent contains MojeekBot, falling back to *.", "docs": "https://www.mojeek.com/bot.html" }, { "slug": "oai-searchbot", "name": "OAI-SearchBot", "operator": "OpenAI", "operator_slug": "openai", "token": "OAI-SearchBot", "ua": "OAI-SearchBot", "ua_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/searchbot.json", "cost_of_blocking": "High. Blocking this removes you from ChatGPT search results and from the source links ChatGPT shows. This is the single most expensive block on this list for anyone who wants to be cited by an assistant.", "what_it_is": "Builds the index ChatGPT search answers from. Content it collects is used for retrieval and citation, not for model training.", "docs": "https://platform.openai.com/docs/bots" }, { "slug": "omgili", "name": "omgili", "operator": "Webz.io", "operator_slug": "webz", "token": "omgili", "ua": "omgili", "ua_example": "omgili/0.5 +http://omgili.com", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Same as omgilibot.", "what_it_is": "The older robots token for the same Webz.io collection, still honoured and still worth listing.", "docs": "https://webz.io/blog/machine-learning/" }, { "slug": "omgilibot", "name": "omgilibot", "operator": "Webz.io", "operator_slug": "webz", "token": "omgilibot", "ua": "omgilibot", "ua_example": "omgilibot/0.4; +http://omgili.com", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Exclusion from a commercial dataset resold to third parties.", "what_it_is": "Webz.io's crawler, collecting web and forum text sold as datasets, including to model builders.", "docs": "https://webz.io/blog/machine-learning/" }, { "slug": "panscient", "name": "Panscient", "operator": "Panscient", "operator_slug": "panscient", "token": "panscient.com", "ua": "panscient.com", "ua_example": "Mozilla/5.0 (compatible; panscient.com)", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your company pages stop feeding a business-data product. No effect on search or assistants.", "what_it_is": "Compiles structured data about businesses and business professionals using machine learning. Panscient's FAQ states it obeys robots.txt.", "docs": "https://panscient.com/faq.htm" }, { "slug": "perplexity-user", "name": "Perplexity-User", "operator": "Perplexity", "operator_slug": "perplexity", "token": "Perplexity-User", "ua": "Perplexity-User", "ua_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user", "category": "user-fetch", "category_label": "User-triggered fetchers", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://www.perplexity.ai/perplexity-user.json", "cost_of_blocking": "Not controllable via robots.txt. If you must stop it, verify by the published IP ranges and block at the edge — and accept that users who ask for your page get an error.", "what_it_is": "Fetches a page because a Perplexity user asked for it. Perplexity documents that this fetch is user-initiated and is therefore not governed by robots.txt — a robots rule will not stop it, by stated policy.", "docs": "https://docs.perplexity.ai/guides/bots" }, { "slug": "perplexitybot", "name": "PerplexityBot", "operator": "Perplexity", "operator_slug": "perplexity", "token": "PerplexityBot", "ua": "PerplexityBot", "ua_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://www.perplexity.ai/perplexitybot.json", "cost_of_blocking": "You stop being indexed and cited by Perplexity, and lose the referral clicks its citations produce.", "what_it_is": "Builds Perplexity's search index. Perplexity is citation-heavy by product design, so inclusion here converts to referral traffic more directly than most AI surfaces.", "docs": "https://docs.perplexity.ai/guides/bots" }, { "slug": "petalbot", "name": "PetalBot", "operator": "Huawei", "operator_slug": "huawei", "token": "PetalBot", "ua": "PetalBot", "ua_example": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Removal from Petal Search. Frequently blocked for volume rather than for policy.", "what_it_is": "Huawei's crawler for Petal Search, shipped as the default search on Huawei devices.", "docs": "https://aspiegel.com/petalbot" }, { "slug": "phindbot", "name": "PhindBot", "operator": "Phind", "operator_slug": "phind", "token": "PhindBot", "ua": "PhindBot", "ua_example": "PhindBot", "category": "ai-search", "category_label": "AI search crawlers", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You stop being cited in answers to technical questions — which, for documentation and reference sites, is the exact audience most worth keeping.", "what_it_is": "Phind is an answer engine for developers that combines live web search with its own models. This is the crawler behind those answers.", "docs": "https://www.phind.com/" }, { "slug": "pinterestbot", "name": "Pinterestbot", "operator": "Pinterest", "operator_slug": "pinterest", "token": "Pinterestbot", "ua": "Pinterestbot", "ua_example": "Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Pins pointing at your site go stale — wrong prices, dead links — and new content stops being indexed. For a retailer this is one of the more expensive blocks on the list.", "what_it_is": "Pinterest's crawler. It indexes pages so people can find them on Pinterest and re-reads product pages to keep price and title on a Pin current. Pinterest states that content it crawls is not used to train their Canvas image generation model.", "docs": "https://help.pinterest.com/en/business/article/pinterest-crawler" }, { "slug": "poseidon-research-crawler", "name": "Poseidon Research Crawler", "operator": "Poseidon Research", "operator_slug": "poseidon", "token": "Poseidon Research Crawler", "ua": "Poseidon Research Crawler", "ua_example": "Poseidon Research Crawler", "category": "ai-training", "category_label": "AI training crawlers", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Exclusion from an interpretability research corpus. No published compliance statement.", "what_it_is": "A crawler run by Poseidon Research, a lab working on interpretability research for AI systems.", "docs": "https://www.poseidonresearch.com" }, { "slug": "qualifiedbot", "name": "QualifiedBot", "operator": "Qualified", "operator_slug": "qualified", "token": "QualifiedBot", "ua": "QualifiedBot", "ua_example": "QualifiedBot", "category": "ai-search", "category_label": "AI search crawlers", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "A chatbot on a site that licensed the product loses context. If that site is yours, this block is self-inflicted.", "what_it_is": "Analyses a customer's website so Qualified's AI sales chatbots can answer questions about it in context.", "docs": "https://www.qualified.com" }, { "slug": "quillbot", "name": "QuillBot", "operator": "QuillBot", "operator_slug": "quillbot", "token": "QuillBot", "ua": "QuillBot", "ua_example": "QuillBot", "category": "ai-training", "category_label": "AI training crawlers", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Exclusion from QuillBot's corpus. No compliance statement is published, so the rule is a request.", "what_it_is": "Operated by QuillBot as part of its writing, paraphrasing and AI-detection products. The dataset also records a second token, quillbot.com, for the same operator.", "docs": "https://quillbot.com" }, { "slug": "qwantbot", "name": "Qwantbot", "operator": "Qwant", "operator_slug": "qwant", "token": "Qwantbot", "ua": "Qwantbot", "ua_example": "Mozilla/5.0 (compatible; Qwantbot/1.0_12345; +https://help.qwant.com/bot/)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You leave the index behind Qwant, the French privacy-focused engine, and the products that federate it.", "what_it_is": "Qwant's crawler. Qwant documents that the string Qwantbot always appears in its user-agents whatever the crawler version, which is what makes a substring match safe here.", "docs": "https://help.qwant.com/bot/" }, { "slug": "qwantbot-news", "name": "Qwantbot-news", "operator": "Qwant", "operator_slug": "qwant", "token": "Qwantbot-news", "ua": "Qwantbot-news", "ua_example": "Mozilla/5.0 (compatible; Qwantbot-news/2.0; +https://help.qwant.com/bot/)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your articles stop appearing in Qwant News. A rule for Qwantbot as a substring already catches both.", "what_it_is": "The news variant of Qwant's crawler, documented alongside the main one and carrying the same Qwantbot substring.", "docs": "https://help.qwant.com/bot/" }, { "slug": "reflectionbot", "name": "Reflectionbot", "operator": "Reflection AI", "operator_slug": "reflection", "token": "Reflectionbot", "ua": "Reflectionbot", "ua_example": "Reflectionbot", "category": "ai-training", "category_label": "AI training crawlers", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Unknown by construction — which is itself the reason some people block it. Nothing user-facing depends on it.", "what_it_is": "An undocumented crawler whose user-agent links to Reflection AI, a company building AI models. The link in the user-agent is the only public statement of purpose that exists.", "docs": "https://reflection.ai/" }, { "slug": "rogerbot", "name": "rogerbot", "operator": "Moz", "operator_slug": "moz", "token": "rogerbot", "ua": "rogerbot", "ua_example": "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler@moz.com)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Moz Pro site audits of your domain stop. If the domain is yours and you use Moz, blocking this breaks your own reports.", "what_it_is": "Moz's Campaign crawler, which audits a site its own owner registered. Moz states there is no IP range for it — identification is by user-agent only.", "docs": "https://moz.com/help/moz-procedures/crawlers/rogerbot" }, { "slug": "sbintuitionsbot", "name": "SBIntuitionsBot", "operator": "SB Intuitions", "operator_slug": "sbintuitions", "token": "SBIntuitionsBot", "ua": "SBIntuitionsBot", "ua_example": "SBIntuitionsBot", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your content is excluded from a Japanese-language foundation-model corpus. Nothing user-facing changes.", "what_it_is": "SB Intuitions is SoftBank's Japanese LLM lab; this crawler gathers data used in that model development and in information analysis. The operator publishes a dedicated bot page.", "docs": "https://www.sbintuitions.co.jp/en/bot/" }, { "slug": "scrapy", "name": "Scrapy", "operator": "Scrapy project", "operator_slug": "scrapy", "token": "Scrapy", "ua": "Scrapy", "ua_example": "Scrapy/2.11.0 (+https://scrapy.org)", "category": "tool", "category_label": "Tools and frameworks", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You block a very large tail of unattributed one-off crawlers, and also every well-behaved researcher who did not change the default.", "what_it_is": "Not an operator: the default user-agent of the most common Python crawling framework. Anyone can be behind it. Modern Scrapy obeys robots.txt by default, which is why the default UA is still worth a rule.", "docs": "https://scrapy.org/" }, { "slug": "screaming-frog-seo-spider", "name": "Screaming Frog SEO Spider", "operator": "Screaming Frog", "operator_slug": "screamingfrog", "token": "Screaming Frog SEO Spider", "ua": "Screaming Frog SEO Spider", "ua_example": "Screaming Frog SEO Spider/21.0", "category": "tool", "category_label": "Tools and frameworks", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You block a consultant auditing your own site as often as you block a stranger. Treat it as a rate-limit question, not a consent one — and note that the user-agent is configurable, so a block is advisory.", "what_it_is": "Not an operator: desktop crawling software that anybody can point at any site. The default user-agent identifies the tool, not who is running it, and the operator of the moment is whoever pressed start.", "docs": "https://www.screamingfrog.co.uk/seo-spider/user-agent/" }, { "slug": "semrushbot", "name": "SemrushBot", "operator": "Semrush", "operator_slug": "semrush", "token": "SemrushBot", "ua": "SemrushBot", "ua_example": "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Your competitors' Semrush reports get thinner, and so do yours. No user-facing effect.", "what_it_is": "Semrush's backlink and keyword crawler. It is not an AI crawler, but it is usually in the top three by volume on any site, and it is the cheapest block on this list.", "docs": "https://www.semrush.com/bot/" }, { "slug": "semrushbot-ba", "name": "SemrushBot-BA", "operator": "Semrush", "operator_slug": "semrush", "token": "SemrushBot-BA", "ua": "SemrushBot-BA", "ua_example": "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "None to you. It costs the site being audited a little accuracy in their backlink report.", "what_it_is": "The Backlink Audit crawler. It re-checks links pointing at a customer's site, which means it lands on the sites doing the linking.", "docs": "https://www.semrush.com/bot/" }, { "slug": "semrushbot-esi", "name": "SemrushBot-ESI", "operator": "Semrush", "operator_slug": "semrush", "token": "SemrushBot-ESI", "ua": "SemrushBot-ESI", "ua_example": "Mozilla/5.0 (compatible; SemrushBot-ESI; +http://www.semrush.com/bot.html)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Enterprise customers lose analysis of your domain. Nothing user-facing.", "what_it_is": "The crawler for Semrush Enterprise Site Intelligence, the enterprise tier's own site analysis.", "docs": "https://www.semrush.com/bot/" }, { "slug": "semrushbot-ft", "name": "SemrushBot-FT", "operator": "Semrush", "operator_slug": "semrush", "token": "SemrushBot-FT", "ua": "SemrushBot-FT", "ua_example": "Mozilla/5.0 (compatible; SemrushBot-FT; +http://www.semrush.com/bot.html)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your text stops being compared against other people's submissions — which also means copies of your text are less likely to be caught.", "what_it_is": "Fetches full text for the Plagiarism Checker and similar text-comparison tools.", "docs": "https://www.semrush.com/bot/" }, { "slug": "semrushbot-ocob", "name": "SemrushBot-OCOB", "operator": "Semrush", "operator_slug": "semrush", "token": "SemrushBot-OCOB", "ua": "SemrushBot-OCOB", "ua_example": "Mozilla/5.0 (compatible; SemrushBot-OCOB/1.0; +http://www.semrush.com/bot.html)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Exclusion from Semrush's AI corpus, with its SEO crawl unaffected.", "what_it_is": "Semrush's separately-tokenised crawler for its AI content tooling, split out so SEO crawling and AI reuse can be answered differently.", "docs": "https://www.semrush.com/bot/" }, { "slug": "semrushbot-si", "name": "SemrushBot-SI", "operator": "Semrush", "operator_slug": "semrush", "token": "SemrushBot-SI", "ua": "SemrushBot-SI", "ua_example": "Mozilla/5.0 (compatible; SemrushBot-SI; +http://www.semrush.com/bot.html)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Nothing user-facing. Semrush customers lose on-page suggestions for pages on your domain.", "what_it_is": "Fetches pages for the On Page SEO Checker and similar advisory tools.", "docs": "https://www.semrush.com/bot/" }, { "slug": "semrushbot-swa", "name": "SemrushBot-SWA", "operator": "Semrush", "operator_slug": "semrush", "token": "SemrushBot-SWA", "ua": "SemrushBot-SWA", "ua_example": "Mozilla/5.0 (compatible; SemrushBot-SWA; +http://www.semrush.com/bot.html)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Writers using Semrush's assistant see your links reported as unreachable.", "what_it_is": "Checks whether a URL is reachable, for the SEO Writing Assistant. One request per URL a writer references, not a crawl.", "docs": "https://www.semrush.com/bot/" }, { "slug": "seokicks", "name": "SEOkicks", "operator": "SEOkicks", "operator_slug": "seokicks", "token": "SEOkicks", "ua": "SEOkicks", "ua_example": "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You leave a regional backlink index. Nothing else changes.", "what_it_is": "A German backlink index. Its documentation names SEOkicks as the user-agent to use in robots.txt.", "docs": "https://www.seokicks.de/robot.html" }, { "slug": "serpstatbot", "name": "serpstatbot", "operator": "Serpstat", "operator_slug": "serpstat", "token": "serpstatbot", "ua": "serpstatbot", "ua_example": "serpstatbot/2.1 (advanced backlink tracking bot; https://serpstatbot.com/; abuse@serpstatbot.com)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You leave Serpstat's link index. Try Crawl-delay first — they honour it, and a slow crawler is cheaper to keep than to fight.", "what_it_is": "Serpstat's backlink crawler. It documents support for Crawl-delay up to 20 seconds, including a delay set on the * group.", "docs": "https://serpstatbot.com/" }, { "slug": "seznambot", "name": "SeznamBot", "operator": "Seznam", "operator_slug": "seznam", "token": "SeznamBot", "ua": "SeznamBot", "ua_example": "Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/en/seznambot-intro/)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Removal from Seznam. Also removes you from its IndexNow endpoint's usefulness.", "what_it_is": "Seznam's crawler — the dominant search engine in the Czech Republic and one of the few national engines with its own index.", "docs": "https://napoveda.seznam.cz/en/seznamzbozi/subject-matter-crawler/" }, { "slug": "shapbot", "name": "ShapBot", "operator": "Parallel", "operator_slug": "parallel", "token": "ShapBot", "ua": "ShapBot", "ua_example": "ShapBot", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Agents using Parallel's research API lose you as a source. Parallel documents robots.txt compliance, so a rule works.", "what_it_is": "Parallel's crawler. It collects and structures web content to power the search, extraction and deep-research APIs that Parallel sells to agent builders.", "docs": "https://docs.parallel.ai/features/crawler" }, { "slug": "sidetrade-indexer-bot", "name": "Sidetrade indexer bot", "operator": "Sidetrade", "operator_slug": "sidetrade", "token": "Sidetrade indexer bot", "ua": "Sidetrade indexer bot", "ua_example": "Sidetrade indexer bot", "category": "ai-training", "category_label": "AI training crawlers", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Exclusion from a commercial B2B dataset. The operator publishes no robots.txt statement, so treat the rule as a request rather than a control.", "what_it_is": "Sidetrade extracts web data for a range of uses including training its AI products for order-to-cash and customer-data work.", "docs": "https://www.sidetrade.com" }, { "slug": "siteauditbot", "name": "SiteAuditBot", "operator": "Semrush", "operator_slug": "semrush", "token": "SiteAuditBot", "ua": "SiteAuditBot", "ua_example": "Mozilla/5.0 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Site Audit reports on your own domain stop working. If a customer is auditing your site with your permission, blocking this breaks their tooling and nothing of yours.", "what_it_is": "Semrush's Site Audit crawler: it walks a site a customer owns and reports technical SEO problems. Semrush names it as the token to block for that product.", "docs": "https://www.semrush.com/bot/" }, { "slug": "slackbot", "name": "Slackbot", "operator": "Slack", "operator_slug": "slack", "token": "Slackbot", "ua": "Slackbot", "ua_example": "Slackbot 1.0 (+https://api.slack.com/robots)", "category": "preview", "category_label": "Link preview fetchers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Slack stops being able to read your robots.txt, which is a strange thing to want. Block the link expander instead if that is the goal.", "what_it_is": "The other half of Slack's pair: the agent that reads robots.txt and handles Slack's non-unfurl fetches. Slack documents both strings on one page.", "docs": "https://api.slack.com/robots" }, { "slug": "slackbot-linkexpanding", "name": "Slackbot-LinkExpanding", "operator": "Slack", "operator_slug": "slack", "token": "Slackbot-LinkExpanding", "ua": "Slackbot-LinkExpanding", "ua_example": "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", "category": "preview", "category_label": "Link preview fetchers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your links appear in Slack as bare URLs. Inside working teams that quietly costs you clicks, and nothing is gained.", "what_it_is": "Fetches a page to build the unfurl card when somebody pastes your link into Slack. One paste, one fetch.", "docs": "https://api.slack.com/robots" }, { "slug": "splitsignalbot", "name": "SplitSignalBot", "operator": "Semrush", "operator_slug": "semrush", "token": "SplitSignalBot", "ua": "SplitSignalBot", "ua_example": "Mozilla/5.0 (compatible; SplitSignalBot; +http://www.semrush.com/bot.html)", "category": "seo", "category_label": "SEO and backlink crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "A site owner's own A/B testing stops. Only relevant on domains whose owner uses the product.", "what_it_is": "Runs SEO A/B tests on a customer's own site with the SplitSignal tool.", "docs": "https://www.semrush.com/bot/" }, { "slug": "storebot-google", "name": "Storebot-Google", "operator": "Google", "operator_slug": "google", "token": "Storebot-Google", "ua": "Storebot-Google", "ua_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36 (compatible; Storebot-Google/1.0; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "cost_of_blocking": "Product listings may lose shopping-specific enrichment. Irrelevant to non-commerce sites.", "what_it_is": "Checks shopping and checkout flows for Google's shopping surfaces.", "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers" }, { "slug": "terracotta", "name": "TerraCotta", "operator": "Ceramic AI", "operator_slug": "ceramic", "token": "TerraCotta", "ua": "TerraCotta", "ua_example": "TerraCotta", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "You are absent from another agent-facing retrieval index. Ceramic documents that it obeys robots.txt.", "what_it_is": "Ceramic AI's crawler, which indexes public content for a web-scale search API aimed at LLMs and agents.", "docs": "https://ceramic.ai/" }, { "slug": "thinkbot", "name": "Thinkbot", "operator": "Thinkbot", "operator_slug": "thinkbot", "token": "Thinkbot", "ua": "Thinkbot", "ua_example": "Thinkbot", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "disputed", "respects_label": "compliance disputed", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Exclusion from a market-research dataset. Expect to enforce this at the edge rather than in robots.txt.", "what_it_is": "Collects pages for analysis of how sites are adopting AI and automation. The ai.robots.txt dataset records the operator as not respecting robots.txt.", "docs": "https://www.thinkbot.agency" }, { "slug": "tiktokspider", "name": "TikTokSpider", "operator": "ByteDance", "operator_slug": "bytedance", "token": "TikTokSpider", "ua": "TikTokSpider", "ua_example": "Mozilla/5.0 (compatible; TikTokSpider; ttspider-feedback@tiktok.com)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "disputed", "respects_label": "compliance disputed", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Little to lose unless TikTok search referral matters to you.", "what_it_is": "A second ByteDance crawler identifying with TikTok, collecting page content for the same family of models.", "docs": "https://www.bytespider.net/" }, { "slug": "timpibot", "name": "Timpibot", "operator": "Timpi", "operator_slug": "timpi", "token": "Timpibot", "ua": "Timpibot", "ua_example": "Mozilla/5.0 (compatible; Timpibot/0.1; +https://timpi.io)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Absence from a small independent index.", "what_it_is": "A distributed crawler building an independent search index outside the Google/Bing duopoly.", "docs": "https://timpi.io/" }, { "slug": "velenpublicwebcrawler", "name": "VelenPublicWebCrawler", "operator": "Hunter (Velen)", "operator_slug": "hunter", "token": "VelenPublicWebCrawler", "ua": "VelenPublicWebCrawler", "ua_example": "VelenPublicWebCrawler", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your company pages stop feeding a B2B contact and company dataset. The crawl rate it documents makes this one of the cheapest visitors to simply allow.", "what_it_is": "Hunter's crawler, written in Go, building business datasets and machine-learning models from public pages. Its page states it follows robots.txt and meta directives and never fetches more than one page every two seconds.", "docs": "https://velen.io/" }, { "slug": "webzio-extended", "name": "Webzio-Extended", "operator": "Webz.io", "operator_slug": "webz", "token": "Webzio-Extended", "ua": "Webzio-Extended", "ua_example": "Mozilla/5.0 (compatible; Webzio-Extended/1.0)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your content is excluded from the AI-training tier of Webz.io's product while ordinary collection continues.", "what_it_is": "Webz.io's opt-out token specifically for AI training reuse, in the pattern Google and Apple established.", "docs": "https://webz.io/blog/machine-learning/" }, { "slug": "wpbot", "name": "wpbot", "operator": "QuantumCloud", "operator_slug": "quantumcloud", "token": "wpbot", "ua": "wpbot", "ua_example": "wpbot", "category": "tool", "category_label": "Tools and frameworks", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "A WordPress site running that plugin loses its own content as an answer source. Only relevant where the plugin is installed.", "what_it_is": "Supports the AI Chatbot for WordPress plugin: it reads pages so the plugin can answer from a site's own content. The operator provides an opt-out through a form rather than through robots.txt.", "docs": "https://www.quantumcloud.com" }, { "slug": "yak", "name": "YaK", "operator": "Meltwater", "operator_slug": "meltwater", "token": "YaK", "ua": "YaK", "ua_example": "YaK", "category": "dataset", "category_label": "Corpus and dataset builders", "respects": "undocumented", "respects_label": "operator publishes no robots.txt statement", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Your content stops appearing in Meltwater's media monitoring — which is how PR teams find out you were mentioned. Some publishers want to be in it.", "what_it_is": "Meltwater's crawler, feeding the live data stream behind its media-monitoring and consumer-intelligence suite.", "docs": "https://www.meltwater.com/en/suite/consumer-intelligence" }, { "slug": "yandexadditional", "name": "YandexAdditional", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexAdditional", "ua": "YandexAdditional", "ua_example": "Mozilla/5.0 (compatible; YandexAdditional/1.0; +http://yandex.com/bots)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "You disappear from Yandex's AI answers while staying in Yandex Search. This is Yandex's equivalent of Google-Extended, and it is the cheap opt-out most people are looking for.", "what_it_is": "The token that controls whether already-indexed pages may appear in Search with Yandex AI answers. Yandex's table says it makes no indexing requests of its own — it exists so a site can opt out of the generative answer without leaving the index.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexadditionalbot", "name": "YandexAdditionalBot", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexAdditionalBot", "ua": "YandexAdditionalBot", "ua_example": "Mozilla/5.0 (compatible; YandexAdditionalBot/1.0; +http://yandex.com/bots)", "category": "ai-training", "category_label": "AI training crawlers", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Same as YandexAdditional: out of Yandex's AI answers, still in Yandex Search. Name both tokens or neither.", "what_it_is": "The second token Yandex publishes for the same AI-answers opt-out. Both names appear in Yandex's own robot list, so a robots.txt that names only one of them is half a policy.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexblogs", "name": "YandexBlogs", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexBlogs", "ua": "YandexBlogs", "ua_example": "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Comment threads and blog posts stop being findable through Yandex blog search.", "what_it_is": "Yandex's blog-search robot; it indexes post comments as well as posts.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexbot", "name": "YandexBot", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexBot", "ua": "YandexBot", "ua_example": "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Removal from Yandex Search. Verify with reverse DNS to a yandex.ru, yandex.net or yandex.com host — YandexBot is among the most-spoofed user-agents there is.", "what_it_is": "Yandex's search crawler, which also feeds Alice and Yandex's generative answers.", "docs": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html" }, { "slug": "yandexcalendar", "name": "YandexCalendar", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexCalendar", "ua": "YandexCalendar", "ua_example": "Mozilla/5.0 (compatible; YandexCalendar/1.0; +http://yandex.com/bots)", "category": "user-fetch", "category_label": "User-triggered fetchers", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Users who subscribed to a calendar you publish stop receiving updates.", "what_it_is": "Downloads calendar files a user subscribed to. Yandex notes these files are often in directories that are disallowed for indexing, which is why the general rules are not applied.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexcombot", "name": "YandexComBot", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexComBot", "ua": "YandexComBot", "ua_example": "Mozilla/5.0 (compatible; YandexComBot/3.0; +http://ya.cc/bots)", "category": "search", "category_label": "Search engines", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "You leave Yandex's non-Russian index. A rule naming this token is the only one that works on it.", "what_it_is": "Indexes content for Yandex search in languages other than Russian. Yandex documents that it can index content when there is no explicit robot-specific restriction — a * group is not one.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexdirect", "name": "YandexDirect", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexDirect", "ua": "YandexDirect", "ua_example": "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)", "category": "tool", "category_label": "Tools and frameworks", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Ads on your pages become less relevant and earn less. Relevant only if you monetise with Yandex's network.", "what_it_is": "Reads the content of Yandex Advertising Network partner pages to work out their topic so relevant ads can be matched. Documented as not taking the general robots.txt rules into account.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexfavicons", "name": "YandexFavicons", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexFavicons", "ua": "YandexFavicons", "ua_example": "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)", "category": "search", "category_label": "Search engines", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Your results in Yandex lose their icon. Cosmetic, and a * rule will not achieve it anyway.", "what_it_is": "Downloads your favicon so Yandex can show it beside your result. Documented as not taking the general robots.txt rules into account.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandeximages", "name": "YandexImages", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexImages", "ua": "YandexImages", "ua_example": "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Your images leave Yandex Images, which is a large share of image search in Russian-speaking markets.", "what_it_is": "Indexes images for Yandex Images. Yandex's robot table marks it as taking the general robots.txt rules into account.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexmarket", "name": "YandexMarket", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexMarket", "ua": "YandexMarket", "ua_example": "Mozilla/5.0 (compatible; YandexMarket/1.0; +http://yandex.com/bots)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Your products stop being listed and priced in Yandex Market. For a retailer in that market this is a revenue block, not a bandwidth one.", "what_it_is": "The robot behind Yandex Market, Yandex's shopping comparison service. Version 1.0 is documented as obeying the general rules; version 2.0 is documented as not.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexmedia", "name": "YandexMedia", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexMedia", "ua": "YandexMedia", "ua_example": "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Your multimedia content stops appearing in Yandex's media surfaces.", "what_it_is": "Indexes multimedia data for Yandex. Takes the general robots.txt rules into account.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexmetrika", "name": "YandexMetrika", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexMetrika", "ua": "YandexMetrika", "ua_example": "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)", "category": "tool", "category_label": "Tools and frameworks", "respects": "by-design-no", "respects_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Nothing you can enforce through robots.txt. If you run Metrica, its session replay loses your stylesheets and renders your pages wrong.", "what_it_is": "Yandex Metrica's own fetcher. Two of its versions — the 2.0 yabs01 availability checker and the 4.0 CSS cache for Webvisor — are documented in Yandex's table as not using robots.txt at all.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexmobilebot", "name": "YandexMobileBot", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexMobileBot", "ua": "YandexMobileBot", "ua_example": "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)", "category": "search", "category_label": "Search engines", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Yandex loses its mobile-friendliness signal for your pages, which affects how they are ranked and rendered on phones.", "what_it_is": "Decides whether a page's layout is suitable for mobile devices. Yandex's table marks it as NOT taking the general robots.txt rules into account, so a * group does not stop it — a group named YandexMobileBot does.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexrenderresourcesbot", "name": "YandexRenderResourcesBot", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexRenderResourcesBot", "ua": "YandexRenderResourcesBot", "ua_example": "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots)", "category": "search", "category_label": "Search engines", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Yandex renders your pages without their stylesheets or scripts and ranks what it sees. This is the classic accidental self-inflicted ranking loss.", "what_it_is": "Loads the CSS, JavaScript and images Yandex needs to render a page. Yandex documents the exact rule: it ignores robots.txt for a resource when the HTML page using it is allowed, and does not fetch the resource when that page is disallowed.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexscreenshotbot", "name": "YandexScreenshotBot", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexScreenshotBot", "ua": "YandexScreenshotBot", "ua_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36 (compatible; YandexScreenshotBot/3.0; +http://yandex.com/bots)", "category": "tool", "category_label": "Tools and frameworks", "respects": "own-token-only", "respects_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Yandex surfaces that show a page thumbnail show nothing for you.", "what_it_is": "Takes a screenshot of a page. Documented as not taking the general robots.txt rules into account.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexvideo", "name": "YandexVideo", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexVideo", "ua": "YandexVideo", "ua_example": "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Removal from Yandex video search. Note that a second robot, YandexVideoParser, does the same job and is documented as NOT taking the general rules into account.", "what_it_is": "Indexes video for Yandex video search. Obeys the general robots.txt rules per Yandex's own table.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yandexwebmaster", "name": "YandexWebmaster", "operator": "Yandex", "operator_slug": "yandex", "token": "YandexWebmaster", "ua": "YandexWebmaster", "ua_example": "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)", "category": "tool", "category_label": "Tools and frameworks", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Your own Yandex Webmaster checks stop working. Blocking this only hurts you.", "what_it_is": "The fetcher behind Yandex Webmaster, the console a site owner uses to inspect their own site.", "docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" }, { "slug": "yeti", "name": "Yeti", "operator": "Naver", "operator_slug": "naver", "token": "Yeti", "ua": "Yeti", "ua_example": "Mozilla/5.0 (compatible; Yeti/1.1; +https://naver.me/spd)", "category": "search", "category_label": "Search engines", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "cost_of_blocking": "Removal from Naver, which is most of Korean search.", "what_it_is": "Naver's crawler. Naver is South Korea's largest search portal and runs its own index and its own generative answers.", "docs": "https://searchadvisor.naver.com/guide/seo-basic-crawl" }, { "slug": "youbot", "name": "YouBot", "operator": "You.com", "operator_slug": "you", "token": "YouBot", "ua": "YouBot", "ua_example": "Mozilla/5.0 (compatible; YouBot (+http://www.you.com))", "category": "ai-search", "category_label": "AI search crawlers", "respects": "documented", "respects_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "cost_of_blocking": "Removal from You.com's index and from answers built on its API.", "what_it_is": "You.com's crawler, feeding its AI search product and its search API.", "docs": "https://about.you.com/youbot/" } ], "policies": [ { "slug": "allow-all", "title": "Allow everything, explicitly", "summary": "Every crawler on this index is named and allowed. Use when you want maximum reach into search and assistants and have nothing to withhold.", "detail": "An empty robots.txt already allows everything, so this file is not about permission — it is about being explicit. Naming each token means a later change is a one-line diff instead of a rewrite, and it documents that the allow was a decision. This is the policy this site itself serves.", "crawlers": [ "adsbot-google", "adsbot-google-mobile", "adsbot-google-mobile-apps", "ahrefsbot", "ahrefssiteaudit", "ai2bot", "ai2bot-dolma", "aihitbot", "aiwebindex", "amazonbot", "andibot", "anomura", "anthropic-ai", "apis-google", "applebot", "applebot-extended", "archive-org-bot", "atlassian-bot", "awariorssbot", "awariosmartbot", "baiduspider", "barkrowler", "bedrockbot", "bingbot", "bytespider", "ccbot", "chatgpt-agent", "chatgpt-user", "claude-searchbot", "claude-user", "claude-web", "claudebot", "cloudflare-autorag", "cohere-ai", "cohere-training-data-crawler", "cotoyogi", "crawl4ai", "crawlspace", "dataforseobot", "diffbot", "dotbot", "duckassistbot", "duckduckbot", "echoboxbot", "exasearchbot", "facebookbot", "facebookexternalhit", "factset-spyderbot", "feedfetcher-google", "firecrawlagent", "google-agent", "google-cloudvertexbot", "google-cws", "google-extended", "google-gemininotebook", "google-inspectiontool", "google-pinpoint", "google-read-aloud", "google-safety", "google-site-verification", "googlebot", "googlebot-image", "googlebot-news", "googlebot-video", "googlemessages", "googleother", "googleother-image", "googleother-video", "googleproducer", "gptbot", "ia-archiver", "icc-crawler", "imagesiftbot", "img2dataset", "isscyberriskcrawler", "kagibot", "klaviyoaibot", "laiondownloader", "lightpanda", "linguee-bot", "mediapartners-google", "meta-externalagent", "meta-externalfetcher", "meta-webindexer", "mistralai-user", "mj12bot", "mojeekbot", "oai-searchbot", "omgili", "omgilibot", "panscient", "perplexity-user", "perplexitybot", "petalbot", "phindbot", "pinterestbot", "poseidon-research-crawler", "qualifiedbot", "quillbot", "qwantbot", "qwantbot-news", "reflectionbot", "rogerbot", "sbintuitionsbot", "scrapy", "screaming-frog-seo-spider", "semrushbot", "semrushbot-ba", "semrushbot-esi", "semrushbot-ft", "semrushbot-ocob", "semrushbot-si", "semrushbot-swa", "seokicks", "serpstatbot", "seznambot", "shapbot", "sidetrade-indexer-bot", "siteauditbot", "slackbot", "slackbot-linkexpanding", "splitsignalbot", "storebot-google", "terracotta", "thinkbot", "tiktokspider", "timpibot", "velenpublicwebcrawler", "webzio-extended", "wpbot", "yak", "yandexadditional", "yandexadditionalbot", "yandexblogs", "yandexbot", "yandexcalendar", "yandexcombot", "yandexdirect", "yandexfavicons", "yandeximages", "yandexmarket", "yandexmedia", "yandexmetrika", "yandexmobilebot", "yandexrenderresourcesbot", "yandexscreenshotbot", "yandexvideo", "yandexwebmaster", "yeti", "youbot" ], "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/robots/allow-all.txt", "robots_txt": "# AI Crawler Index — policy: allow-all\n# Allow everything, explicitly\n# Every crawler on this index is named and allowed. Use when you want maximum reach into search and assistants and have nothing to withhold.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/allow-all.html\n# 150 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AdsBot-Google\nAllow: /\n\nUser-agent: AdsBot-Google-Mobile\nAllow: /\n\nUser-agent: AdsBot-Google-Mobile-Apps\nAllow: /\n\nUser-agent: AhrefsBot\nAllow: /\n\nUser-agent: AhrefsSiteAudit\nAllow: /\n\nUser-agent: AI2Bot\nAllow: /\n\nUser-agent: Ai2Bot-Dolma\nAllow: /\n\nUser-agent: aiHitBot\nAllow: /\n\nUser-agent: AIWebIndex\nAllow: /\n\nUser-agent: Amazonbot\nAllow: /\n\nUser-agent: Andibot\nAllow: /\n\nUser-agent: Anomura\nAllow: /\n\nUser-agent: anthropic-ai # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: APIs-Google\nAllow: /\n\nUser-agent: Applebot\nAllow: /\n\nUser-agent: Applebot-Extended # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: archive.org_bot\nAllow: /\n\nUser-agent: atlassian-bot\nAllow: /\n\nUser-agent: AwarioRssBot\nAllow: /\n\nUser-agent: AwarioSmartBot\nAllow: /\n\nUser-agent: Baiduspider\nAllow: /\n\nUser-agent: barkrowler\nAllow: /\n\nUser-agent: bedrockbot\nAllow: /\n\nUser-agent: bingbot\nAllow: /\n\nUser-agent: Bytespider # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: Claude-SearchBot\nAllow: /\n\nUser-agent: Claude-User\nAllow: /\n\nUser-agent: Claude-Web # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: Cloudflare-AutoRAG\nAllow: /\n\nUser-agent: cohere-ai\nAllow: /\n\nUser-agent: cohere-training-data-crawler\nAllow: /\n\nUser-agent: Cotoyogi\nAllow: /\n\nUser-agent: Crawl4AI\nAllow: /\n\nUser-agent: Crawlspace\nAllow: /\n\nUser-agent: DataForSeoBot\nAllow: /\n\nUser-agent: Diffbot\nAllow: /\n\nUser-agent: dotbot\nAllow: /\n\nUser-agent: DuckAssistBot\nAllow: /\n\nUser-agent: DuckDuckBot\nAllow: /\n\nUser-agent: EchoboxBot\nAllow: /\n\nUser-agent: ExaSearchBot\nAllow: /\n\nUser-agent: FacebookBot\nAllow: /\n\nUser-agent: facebookexternalhit\nAllow: /\n\nUser-agent: Factset_spyderbot\nAllow: /\n\nUser-agent: FeedFetcher-Google # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: FirecrawlAgent\nAllow: /\n\nUser-agent: Google-Agent # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-CloudVertexBot\nAllow: /\n\nUser-agent: Google-CWS # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-Extended # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: Google-GeminiNotebook # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-InspectionTool\nAllow: /\n\nUser-agent: Google-Pinpoint # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-Read-Aloud # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-Safety # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-Site-Verification # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Googlebot\nAllow: /\n\nUser-agent: Googlebot-Image\nAllow: /\n\nUser-agent: Googlebot-News\nAllow: /\n\nUser-agent: Googlebot-Video\nAllow: /\n\nUser-agent: GoogleMessages # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: GoogleOther\nAllow: /\n\nUser-agent: GoogleOther-Image\nAllow: /\n\nUser-agent: GoogleOther-Video\nAllow: /\n\nUser-agent: GoogleProducer # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ia_archiver\nAllow: /\n\nUser-agent: ICC-Crawler\nAllow: /\n\nUser-agent: ImagesiftBot\nAllow: /\n\nUser-agent: img2dataset\nAllow: /\n\nUser-agent: ISSCyberRiskCrawler # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: Kagibot\nAllow: /\n\nUser-agent: KlaviyoAIBot\nAllow: /\n\nUser-agent: LAIONDownloader # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Lightpanda\nAllow: /\n\nUser-agent: Linguee Bot # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: Mediapartners-Google\nAllow: /\n\nUser-agent: meta-externalagent\nAllow: /\n\nUser-agent: meta-externalfetcher\nAllow: /\n\nUser-agent: Meta-WebIndexer\nAllow: /\n\nUser-agent: MistralAI-User\nAllow: /\n\nUser-agent: MJ12bot\nAllow: /\n\nUser-agent: MojeekBot\nAllow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: omgili\nAllow: /\n\nUser-agent: omgilibot\nAllow: /\n\nUser-agent: panscient.com\nAllow: /\n\nUser-agent: Perplexity-User # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: PetalBot\nAllow: /\n\nUser-agent: PhindBot\nAllow: /\n\nUser-agent: Pinterestbot\nAllow: /\n\nUser-agent: Poseidon Research Crawler\nAllow: /\n\nUser-agent: QualifiedBot\nAllow: /\n\nUser-agent: QuillBot\nAllow: /\n\nUser-agent: Qwantbot\nAllow: /\n\nUser-agent: Qwantbot-news\nAllow: /\n\nUser-agent: Reflectionbot\nAllow: /\n\nUser-agent: rogerbot\nAllow: /\n\nUser-agent: SBIntuitionsBot\nAllow: /\n\nUser-agent: Scrapy\nAllow: /\n\nUser-agent: Screaming Frog SEO Spider\nAllow: /\n\nUser-agent: SemrushBot\nAllow: /\n\nUser-agent: SemrushBot-BA\nAllow: /\n\nUser-agent: SemrushBot-ESI\nAllow: /\n\nUser-agent: SemrushBot-FT\nAllow: /\n\nUser-agent: SemrushBot-OCOB\nAllow: /\n\nUser-agent: SemrushBot-SI\nAllow: /\n\nUser-agent: SemrushBot-SWA\nAllow: /\n\nUser-agent: SEOkicks\nAllow: /\n\nUser-agent: serpstatbot\nAllow: /\n\nUser-agent: SeznamBot\nAllow: /\n\nUser-agent: ShapBot\nAllow: /\n\nUser-agent: Sidetrade indexer bot\nAllow: /\n\nUser-agent: SiteAuditBot\nAllow: /\n\nUser-agent: Slackbot\nAllow: /\n\nUser-agent: Slackbot-LinkExpanding\nAllow: /\n\nUser-agent: SplitSignalBot\nAllow: /\n\nUser-agent: Storebot-Google\nAllow: /\n\nUser-agent: TerraCotta\nAllow: /\n\nUser-agent: Thinkbot # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: TikTokSpider # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: Timpibot\nAllow: /\n\nUser-agent: VelenPublicWebCrawler\nAllow: /\n\nUser-agent: Webzio-Extended\nAllow: /\n\nUser-agent: wpbot\nAllow: /\n\nUser-agent: YaK\nAllow: /\n\nUser-agent: YandexAdditional\nAllow: /\n\nUser-agent: YandexAdditionalBot\nAllow: /\n\nUser-agent: YandexBlogs\nAllow: /\n\nUser-agent: YandexBot\nAllow: /\n\nUser-agent: YandexCalendar\nAllow: /\n\nUser-agent: YandexComBot\nAllow: /\n\nUser-agent: YandexDirect\nAllow: /\n\nUser-agent: YandexFavicons\nAllow: /\n\nUser-agent: YandexImages\nAllow: /\n\nUser-agent: YandexMarket\nAllow: /\n\nUser-agent: YandexMedia\nAllow: /\n\nUser-agent: YandexMetrika # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: YandexMobileBot\nAllow: /\n\nUser-agent: YandexRenderResourcesBot\nAllow: /\n\nUser-agent: YandexScreenshotBot\nAllow: /\n\nUser-agent: YandexVideo\nAllow: /\n\nUser-agent: YandexWebmaster\nAllow: /\n\nUser-agent: Yeti\nAllow: /\n\nUser-agent: YouBot\nAllow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n" }, { "slug": "block-ai-training", "title": "Block AI training, keep AI search", "summary": "Refuse the crawlers that feed model training. Keep the ones that put you in ChatGPT, Claude, Perplexity and Gemini answers.", "detail": "The distinction most people actually want, and the one that is easy to get wrong: GPTBot trains, OAI-SearchBot indexes for citation. Blocking both loses you the traffic and gains you nothing extra. Google and Apple have no separate crawler at all — Google-Extended and Applebot-Extended are pure control tokens, so they belong in this file while Googlebot and Applebot must not.", "crawlers": [ "anthropic-ai", "applebot-extended", "bytespider", "claudebot", "cohere-training-data-crawler", "cotoyogi", "facebookbot", "factset-spyderbot", "google-extended", "googleother", "googleother-image", "googleother-video", "gptbot", "icc-crawler", "isscyberriskcrawler", "linguee-bot", "meta-externalagent", "poseidon-research-crawler", "quillbot", "reflectionbot", "sbintuitionsbot", "semrushbot-ocob", "sidetrade-indexer-bot", "tiktokspider", "webzio-extended", "yandexadditional", "yandexadditionalbot" ], "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/robots/block-ai-training.txt", "robots_txt": "# AI Crawler Index — policy: block-ai-training\n# Block AI training, keep AI search\n# Refuse the crawlers that feed model training. Keep the ones that put you in ChatGPT, Claude, Perplexity and Gemini answers.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/block-ai-training.html\n# 27 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: anthropic-ai # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: Applebot-Extended # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: Bytespider # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: FacebookBot\nDisallow: /\n\nUser-agent: Factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Extended # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: GoogleOther-Image\nDisallow: /\n\nUser-agent: GoogleOther-Video\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: Linguee Bot # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Poseidon Research Crawler\nDisallow: /\n\nUser-agent: QuillBot\nDisallow: /\n\nUser-agent: Reflectionbot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: SemrushBot-OCOB\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: TikTokSpider # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: Webzio-Extended\nDisallow: /\n\nUser-agent: YandexAdditional\nDisallow: /\n\nUser-agent: YandexAdditionalBot\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n" }, { "slug": "block-all-ai", "title": "Block every AI crawler", "summary": "Training, AI search, user-triggered fetches and corpus builders, all refused. Classic search engines still allowed.", "detail": "The maximal AI opt-out that still leaves you in Google and Bing. Understand the price before deploying it: you will not be cited by any assistant, and when a reader explicitly asks ChatGPT or Claude to open your page, they get an error. Note also that Perplexity-User and Bytespider are listed here but documented as not governed by robots.txt, so this file is a statement of intent for those two, not an enforcement mechanism.", "crawlers": [ "ai2bot", "ai2bot-dolma", "aihitbot", "aiwebindex", "amazonbot", "andibot", "anomura", "anthropic-ai", "applebot-extended", "atlassian-bot", "awariorssbot", "awariosmartbot", "bedrockbot", "bytespider", "ccbot", "chatgpt-agent", "chatgpt-user", "claude-searchbot", "claude-user", "claude-web", "claudebot", "cloudflare-autorag", "cohere-ai", "cohere-training-data-crawler", "cotoyogi", "diffbot", "duckassistbot", "echoboxbot", "exasearchbot", "facebookbot", "factset-spyderbot", "google-agent", "google-cloudvertexbot", "google-extended", "google-gemininotebook", "google-pinpoint", "google-read-aloud", "googleother", "googleother-image", "googleother-video", "gptbot", "icc-crawler", "imagesiftbot", "img2dataset", "isscyberriskcrawler", "klaviyoaibot", "laiondownloader", "linguee-bot", "meta-externalagent", "meta-externalfetcher", "meta-webindexer", "mistralai-user", "oai-searchbot", "omgili", "omgilibot", "panscient", "perplexity-user", "perplexitybot", "phindbot", "poseidon-research-crawler", "qualifiedbot", "quillbot", "reflectionbot", "sbintuitionsbot", "semrushbot-ocob", "shapbot", "sidetrade-indexer-bot", "terracotta", "thinkbot", "tiktokspider", "velenpublicwebcrawler", "webzio-extended", "yak", "yandexadditional", "yandexadditionalbot", "yandexcalendar", "youbot" ], "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/robots/block-all-ai.txt", "robots_txt": "# AI Crawler Index — policy: block-all-ai\n# Block every AI crawler\n# Training, AI search, user-triggered fetches and corpus builders, all refused. Classic search engines still allowed.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/block-all-ai.html\n# 77 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: AIWebIndex\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: anthropic-ai # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: Applebot-Extended # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: AwarioRssBot\nDisallow: /\n\nUser-agent: AwarioSmartBot\nDisallow: /\n\nUser-agent: bedrockbot\nDisallow: /\n\nUser-agent: Bytespider # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: Claude-Web # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: ExaSearchBot\nDisallow: /\n\nUser-agent: FacebookBot\nDisallow: /\n\nUser-agent: Factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Agent # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: Google-Extended # control token, no crawler uses this user-agent\nDisallow: /\n\nUser-agent: Google-GeminiNotebook # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Google-Pinpoint # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Google-Read-Aloud # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: GoogleOther-Image\nDisallow: /\n\nUser-agent: GoogleOther-Video\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: LAIONDownloader # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Linguee Bot # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: Meta-WebIndexer\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: omgilibot\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: Perplexity-User # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poseidon Research Crawler\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: QuillBot\nDisallow: /\n\nUser-agent: Reflectionbot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: SemrushBot-OCOB\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: Thinkbot # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: TikTokSpider # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: Webzio-Extended\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: YandexAdditional\nDisallow: /\n\nUser-agent: YandexAdditionalBot\nDisallow: /\n\nUser-agent: YandexCalendar\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n" }, { "slug": "block-datasets", "title": "Block corpus and dataset builders", "summary": "Refuse the crawlers whose output is a dataset other people train on: Common Crawl, AI2, Webz.io, Diffbot, ImagesiftBot.", "detail": "These are the highest-leverage blocks per line, because one crawl becomes many downstream training runs. It is also the block with the longest delay before it has any effect, and no effect at all on archives already published.", "crawlers": [ "ai2bot", "ai2bot-dolma", "aihitbot", "awariorssbot", "awariosmartbot", "ccbot", "diffbot", "echoboxbot", "imagesiftbot", "img2dataset", "laiondownloader", "omgili", "omgilibot", "panscient", "thinkbot", "velenpublicwebcrawler", "yak" ], "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/robots/block-datasets.txt", "robots_txt": "# AI Crawler Index — policy: block-datasets\n# Block corpus and dataset builders\n# Refuse the crawlers whose output is a dataset other people train on: Common Crawl, AI2, Webz.io, Diffbot, ImagesiftBot.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/block-datasets.html\n# 17 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: AwarioRssBot\nDisallow: /\n\nUser-agent: AwarioSmartBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: LAIONDownloader # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: omgilibot\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: Thinkbot # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n" }, { "slug": "allow-ai-search-only", "title": "Allow AI search and user fetches, block the rest", "summary": "Be findable and citable in assistants without contributing to training corpora.", "detail": "The inverse framing of block-ai-training, written as an allowlist so the default for anything new is deny. Fetches a user explicitly asked for stay allowed, because refusing those produces a visible error for a real person who wanted your page.", "crawlers": [ "aiwebindex", "amazonbot", "andibot", "anomura", "applebot", "atlassian-bot", "baiduspider", "bedrockbot", "bingbot", "chatgpt-agent", "chatgpt-user", "claude-searchbot", "claude-user", "claude-web", "cloudflare-autorag", "cohere-ai", "duckassistbot", "duckduckbot", "exasearchbot", "facebookexternalhit", "google-agent", "google-cloudvertexbot", "google-gemininotebook", "google-pinpoint", "google-read-aloud", "googlebot", "googlebot-image", "googlebot-news", "googlebot-video", "googlemessages", "kagibot", "klaviyoaibot", "meta-externalfetcher", "meta-webindexer", "mistralai-user", "mojeekbot", "oai-searchbot", "perplexity-user", "perplexitybot", "petalbot", "phindbot", "pinterestbot", "qualifiedbot", "qwantbot", "qwantbot-news", "seznambot", "shapbot", "slackbot", "slackbot-linkexpanding", "storebot-google", "terracotta", "timpibot", "yandexblogs", "yandexbot", "yandexcalendar", "yandexcombot", "yandexfavicons", "yandeximages", "yandexmarket", "yandexmedia", "yandexmobilebot", "yandexrenderresourcesbot", "yandexvideo", "yeti", "youbot" ], "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/robots/allow-ai-search-only.txt", "robots_txt": "# AI Crawler Index — policy: allow-ai-search-only\n# Allow AI search and user fetches, block the rest\n# Be findable and citable in assistants without contributing to training corpora.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/allow-ai-search-only.html\n# 65 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AIWebIndex\nAllow: /\n\nUser-agent: Amazonbot\nAllow: /\n\nUser-agent: Andibot\nAllow: /\n\nUser-agent: Anomura\nAllow: /\n\nUser-agent: Applebot\nAllow: /\n\nUser-agent: atlassian-bot\nAllow: /\n\nUser-agent: Baiduspider\nAllow: /\n\nUser-agent: bedrockbot\nAllow: /\n\nUser-agent: bingbot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: Claude-SearchBot\nAllow: /\n\nUser-agent: Claude-User\nAllow: /\n\nUser-agent: Claude-Web # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: Cloudflare-AutoRAG\nAllow: /\n\nUser-agent: cohere-ai\nAllow: /\n\nUser-agent: DuckAssistBot\nAllow: /\n\nUser-agent: DuckDuckBot\nAllow: /\n\nUser-agent: ExaSearchBot\nAllow: /\n\nUser-agent: facebookexternalhit\nAllow: /\n\nUser-agent: Google-Agent # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-CloudVertexBot\nAllow: /\n\nUser-agent: Google-GeminiNotebook # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-Pinpoint # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-Read-Aloud # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Googlebot\nAllow: /\n\nUser-agent: Googlebot-Image\nAllow: /\n\nUser-agent: Googlebot-News\nAllow: /\n\nUser-agent: Googlebot-Video\nAllow: /\n\nUser-agent: GoogleMessages # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Kagibot\nAllow: /\n\nUser-agent: KlaviyoAIBot\nAllow: /\n\nUser-agent: meta-externalfetcher\nAllow: /\n\nUser-agent: Meta-WebIndexer\nAllow: /\n\nUser-agent: MistralAI-User\nAllow: /\n\nUser-agent: MojeekBot\nAllow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: Perplexity-User # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: PetalBot\nAllow: /\n\nUser-agent: PhindBot\nAllow: /\n\nUser-agent: Pinterestbot\nAllow: /\n\nUser-agent: QualifiedBot\nAllow: /\n\nUser-agent: Qwantbot\nAllow: /\n\nUser-agent: Qwantbot-news\nAllow: /\n\nUser-agent: SeznamBot\nAllow: /\n\nUser-agent: ShapBot\nAllow: /\n\nUser-agent: Slackbot\nAllow: /\n\nUser-agent: Slackbot-LinkExpanding\nAllow: /\n\nUser-agent: Storebot-Google\nAllow: /\n\nUser-agent: TerraCotta\nAllow: /\n\nUser-agent: Timpibot\nAllow: /\n\nUser-agent: YandexBlogs\nAllow: /\n\nUser-agent: YandexBot\nAllow: /\n\nUser-agent: YandexCalendar\nAllow: /\n\nUser-agent: YandexComBot\nAllow: /\n\nUser-agent: YandexFavicons\nAllow: /\n\nUser-agent: YandexImages\nAllow: /\n\nUser-agent: YandexMarket\nAllow: /\n\nUser-agent: YandexMedia\nAllow: /\n\nUser-agent: YandexMobileBot\nAllow: /\n\nUser-agent: YandexRenderResourcesBot\nAllow: /\n\nUser-agent: YandexVideo\nAllow: /\n\nUser-agent: Yeti\nAllow: /\n\nUser-agent: YouBot\nAllow: /\n\n# Anything not named above is refused.\nUser-agent: *\nDisallow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n" }, { "slug": "block-seo-tools", "title": "Block SEO and backlink crawlers", "summary": "Ahrefs, Semrush and friends. No user-facing consequence, and often the largest single slice of your bot traffic.", "detail": "The cheapest bandwidth saving available to most sites, and the one nobody regrets. The only cost is that your own dashboards on those tools get thinner. Consider Crawl-delay first: both major operators honour it.", "crawlers": [ "ahrefsbot", "ahrefssiteaudit", "barkrowler", "dataforseobot", "dotbot", "mj12bot", "rogerbot", "semrushbot", "semrushbot-ba", "semrushbot-esi", "semrushbot-ft", "semrushbot-si", "semrushbot-swa", "seokicks", "serpstatbot", "siteauditbot", "splitsignalbot" ], "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/robots/block-seo-tools.txt", "robots_txt": "# AI Crawler Index — policy: block-seo-tools\n# Block SEO and backlink crawlers\n# Ahrefs, Semrush and friends. No user-facing consequence, and often the largest single slice of your bot traffic.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/block-seo-tools.html\n# 17 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: AhrefsSiteAudit\nDisallow: /\n\nUser-agent: barkrowler\nDisallow: /\n\nUser-agent: DataForSeoBot\nDisallow: /\n\nUser-agent: dotbot\nDisallow: /\n\nUser-agent: MJ12bot\nDisallow: /\n\nUser-agent: rogerbot\nDisallow: /\n\nUser-agent: SemrushBot\nDisallow: /\n\nUser-agent: SemrushBot-BA\nDisallow: /\n\nUser-agent: SemrushBot-ESI\nDisallow: /\n\nUser-agent: SemrushBot-FT\nDisallow: /\n\nUser-agent: SemrushBot-SI\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: SEOkicks\nDisallow: /\n\nUser-agent: serpstatbot\nDisallow: /\n\nUser-agent: SiteAuditBot\nDisallow: /\n\nUser-agent: SplitSignalBot\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n" }, { "slug": "block-disputed", "title": "Block the crawlers with disputed robots compliance", "summary": "The ones repeatedly reported as ignoring robots.txt. Included for completeness — expect to enforce this at the edge instead.", "detail": "A robots.txt rule is a request. For the operators in this file the request is documented as unreliable or explicitly not applicable, so the honest use of this file is as a record of intent that sits alongside a real block by user-agent or by IP at your CDN.", "crawlers": [ "bytespider", "feedfetcher-google", "google-agent", "google-cws", "google-gemininotebook", "google-pinpoint", "google-read-aloud", "google-safety", "google-site-verification", "googlemessages", "googleproducer", "isscyberriskcrawler", "laiondownloader", "linguee-bot", "perplexity-user", "thinkbot", "tiktokspider", "yandexmetrika" ], "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/robots/block-disputed.txt", "robots_txt": "# AI Crawler Index — policy: block-disputed\n# Block the crawlers with disputed robots compliance\n# The ones repeatedly reported as ignoring robots.txt. Included for completeness — expect to enforce this at the edge instead.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/block-disputed.html\n# 18 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: Bytespider # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: FeedFetcher-Google # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Google-Agent # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Google-CWS # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Google-GeminiNotebook # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Google-Pinpoint # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Google-Read-Aloud # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Google-Safety # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Google-Site-Verification # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: GoogleMessages # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: GoogleProducer # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: LAIONDownloader # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Linguee Bot # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: Perplexity-User # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: Thinkbot # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: TikTokSpider # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: YandexMetrika # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n" }, { "slug": "maximum-ai-visibility", "title": "Maximum AI visibility", "summary": "Allow every AI crawler and every search engine; refuse only SEO scrapers. For sites whose goal is to be found and cited by machines.", "detail": "If your content exists to be read by assistants — documentation, reference data, an API — every block costs you and none of them protect anything. Pair this with an llms.txt, a sitemap, and per-item JSON, and the crawlers can actually use what they find.", "crawlers": [ "adsbot-google", "adsbot-google-mobile", "adsbot-google-mobile-apps", "ai2bot", "ai2bot-dolma", "aihitbot", "aiwebindex", "amazonbot", "andibot", "anomura", "anthropic-ai", "apis-google", "applebot", "applebot-extended", "archive-org-bot", "atlassian-bot", "awariorssbot", "awariosmartbot", "baiduspider", "bedrockbot", "bingbot", "bytespider", "ccbot", "chatgpt-agent", "chatgpt-user", "claude-searchbot", "claude-user", "claude-web", "claudebot", "cloudflare-autorag", "cohere-ai", "cohere-training-data-crawler", "cotoyogi", "crawl4ai", "crawlspace", "diffbot", "duckassistbot", "duckduckbot", "echoboxbot", "exasearchbot", "facebookbot", "facebookexternalhit", "factset-spyderbot", "feedfetcher-google", "firecrawlagent", "google-agent", "google-cloudvertexbot", "google-cws", "google-extended", "google-gemininotebook", "google-inspectiontool", "google-pinpoint", "google-read-aloud", "google-safety", "google-site-verification", "googlebot", "googlebot-image", "googlebot-news", "googlebot-video", "googlemessages", "googleother", "googleother-image", "googleother-video", "googleproducer", "gptbot", "ia-archiver", "icc-crawler", "imagesiftbot", "img2dataset", "isscyberriskcrawler", "kagibot", "klaviyoaibot", "laiondownloader", "lightpanda", "linguee-bot", "mediapartners-google", "meta-externalagent", "meta-externalfetcher", "meta-webindexer", "mistralai-user", "mojeekbot", "oai-searchbot", "omgili", "omgilibot", "panscient", "perplexity-user", "perplexitybot", "petalbot", "phindbot", "pinterestbot", "poseidon-research-crawler", "qualifiedbot", "quillbot", "qwantbot", "qwantbot-news", "reflectionbot", "sbintuitionsbot", "scrapy", "screaming-frog-seo-spider", "semrushbot-ocob", "seznambot", "shapbot", "sidetrade-indexer-bot", "slackbot", "slackbot-linkexpanding", "storebot-google", "terracotta", "thinkbot", "tiktokspider", "timpibot", "velenpublicwebcrawler", "webzio-extended", "wpbot", "yak", "yandexadditional", "yandexadditionalbot", "yandexblogs", "yandexbot", "yandexcalendar", "yandexcombot", "yandexdirect", "yandexfavicons", "yandeximages", "yandexmarket", "yandexmedia", "yandexmetrika", "yandexmobilebot", "yandexrenderresourcesbot", "yandexscreenshotbot", "yandexvideo", "yandexwebmaster", "yeti", "youbot" ], "robots_txt_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/robots/maximum-ai-visibility.txt", "robots_txt": "# AI Crawler Index — policy: maximum-ai-visibility\n# Maximum AI visibility\n# Allow every AI crawler and every search engine; refuse only SEO scrapers. For sites whose goal is to be found and cited by machines.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/maximum-ai-visibility.html\n# 133 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AdsBot-Google\nAllow: /\n\nUser-agent: AdsBot-Google-Mobile\nAllow: /\n\nUser-agent: AdsBot-Google-Mobile-Apps\nAllow: /\n\nUser-agent: AI2Bot\nAllow: /\n\nUser-agent: Ai2Bot-Dolma\nAllow: /\n\nUser-agent: aiHitBot\nAllow: /\n\nUser-agent: AIWebIndex\nAllow: /\n\nUser-agent: Amazonbot\nAllow: /\n\nUser-agent: Andibot\nAllow: /\n\nUser-agent: Anomura\nAllow: /\n\nUser-agent: anthropic-ai # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: APIs-Google\nAllow: /\n\nUser-agent: Applebot\nAllow: /\n\nUser-agent: Applebot-Extended # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: archive.org_bot\nAllow: /\n\nUser-agent: atlassian-bot\nAllow: /\n\nUser-agent: AwarioRssBot\nAllow: /\n\nUser-agent: AwarioSmartBot\nAllow: /\n\nUser-agent: Baiduspider\nAllow: /\n\nUser-agent: bedrockbot\nAllow: /\n\nUser-agent: bingbot\nAllow: /\n\nUser-agent: Bytespider # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: Claude-SearchBot\nAllow: /\n\nUser-agent: Claude-User\nAllow: /\n\nUser-agent: Claude-Web # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: Cloudflare-AutoRAG\nAllow: /\n\nUser-agent: cohere-ai\nAllow: /\n\nUser-agent: cohere-training-data-crawler\nAllow: /\n\nUser-agent: Cotoyogi\nAllow: /\n\nUser-agent: Crawl4AI\nAllow: /\n\nUser-agent: Crawlspace\nAllow: /\n\nUser-agent: Diffbot\nAllow: /\n\nUser-agent: DuckAssistBot\nAllow: /\n\nUser-agent: DuckDuckBot\nAllow: /\n\nUser-agent: EchoboxBot\nAllow: /\n\nUser-agent: ExaSearchBot\nAllow: /\n\nUser-agent: FacebookBot\nAllow: /\n\nUser-agent: facebookexternalhit\nAllow: /\n\nUser-agent: Factset_spyderbot\nAllow: /\n\nUser-agent: FeedFetcher-Google # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: FirecrawlAgent\nAllow: /\n\nUser-agent: Google-Agent # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-CloudVertexBot\nAllow: /\n\nUser-agent: Google-CWS # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-Extended # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: Google-GeminiNotebook # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-InspectionTool\nAllow: /\n\nUser-agent: Google-Pinpoint # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-Read-Aloud # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-Safety # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Google-Site-Verification # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Googlebot\nAllow: /\n\nUser-agent: Googlebot-Image\nAllow: /\n\nUser-agent: Googlebot-News\nAllow: /\n\nUser-agent: Googlebot-Video\nAllow: /\n\nUser-agent: GoogleMessages # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: GoogleOther\nAllow: /\n\nUser-agent: GoogleOther-Image\nAllow: /\n\nUser-agent: GoogleOther-Video\nAllow: /\n\nUser-agent: GoogleProducer # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ia_archiver\nAllow: /\n\nUser-agent: ICC-Crawler\nAllow: /\n\nUser-agent: ImagesiftBot\nAllow: /\n\nUser-agent: img2dataset\nAllow: /\n\nUser-agent: ISSCyberRiskCrawler # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: Kagibot\nAllow: /\n\nUser-agent: KlaviyoAIBot\nAllow: /\n\nUser-agent: LAIONDownloader # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: Lightpanda\nAllow: /\n\nUser-agent: Linguee Bot # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: Mediapartners-Google\nAllow: /\n\nUser-agent: meta-externalagent\nAllow: /\n\nUser-agent: meta-externalfetcher\nAllow: /\n\nUser-agent: Meta-WebIndexer\nAllow: /\n\nUser-agent: MistralAI-User\nAllow: /\n\nUser-agent: MojeekBot\nAllow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: omgili\nAllow: /\n\nUser-agent: omgilibot\nAllow: /\n\nUser-agent: panscient.com\nAllow: /\n\nUser-agent: Perplexity-User # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: PetalBot\nAllow: /\n\nUser-agent: PhindBot\nAllow: /\n\nUser-agent: Pinterestbot\nAllow: /\n\nUser-agent: Poseidon Research Crawler\nAllow: /\n\nUser-agent: QualifiedBot\nAllow: /\n\nUser-agent: QuillBot\nAllow: /\n\nUser-agent: Qwantbot\nAllow: /\n\nUser-agent: Qwantbot-news\nAllow: /\n\nUser-agent: Reflectionbot\nAllow: /\n\nUser-agent: SBIntuitionsBot\nAllow: /\n\nUser-agent: Scrapy\nAllow: /\n\nUser-agent: Screaming Frog SEO Spider\nAllow: /\n\nUser-agent: SemrushBot-OCOB\nAllow: /\n\nUser-agent: SeznamBot\nAllow: /\n\nUser-agent: ShapBot\nAllow: /\n\nUser-agent: Sidetrade indexer bot\nAllow: /\n\nUser-agent: Slackbot\nAllow: /\n\nUser-agent: Slackbot-LinkExpanding\nAllow: /\n\nUser-agent: Storebot-Google\nAllow: /\n\nUser-agent: TerraCotta\nAllow: /\n\nUser-agent: Thinkbot # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: TikTokSpider # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: Timpibot\nAllow: /\n\nUser-agent: VelenPublicWebCrawler\nAllow: /\n\nUser-agent: Webzio-Extended\nAllow: /\n\nUser-agent: wpbot\nAllow: /\n\nUser-agent: YaK\nAllow: /\n\nUser-agent: YandexAdditional\nAllow: /\n\nUser-agent: YandexAdditionalBot\nAllow: /\n\nUser-agent: YandexBlogs\nAllow: /\n\nUser-agent: YandexBot\nAllow: /\n\nUser-agent: YandexCalendar\nAllow: /\n\nUser-agent: YandexComBot\nAllow: /\n\nUser-agent: YandexDirect\nAllow: /\n\nUser-agent: YandexFavicons\nAllow: /\n\nUser-agent: YandexImages\nAllow: /\n\nUser-agent: YandexMarket\nAllow: /\n\nUser-agent: YandexMedia\nAllow: /\n\nUser-agent: YandexMetrika # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: YandexMobileBot\nAllow: /\n\nUser-agent: YandexRenderResourcesBot\nAllow: /\n\nUser-agent: YandexScreenshotBot\nAllow: /\n\nUser-agent: YandexVideo\nAllow: /\n\nUser-agent: YandexWebmaster\nAllow: /\n\nUser-agent: Yeti\nAllow: /\n\nUser-agent: YouBot\nAllow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n" } ], "note": "Presence in this table means the operator documents that crawler. A user-agent is a claim; nothing here is proof of identity." }