{
 "slug": "maximum-ai-visibility",
 "title": "Maximum AI visibility",
 "summary": "Allow every AI crawler and every search engine; refuse only SEO scrapers. For sites whose goal is to be found and cited by machines.",
 "detail": "If your content exists to be read by assistants — documentation, reference data, an API — every block costs you and none of them protect anything. Pair this with an llms.txt, a sitemap, and per-item JSON, and the crawlers can actually use what they find.",
 "crawler_count": 54,
 "crawlers": [
  "ai2bot",
  "ai2bot-dolma",
  "amazonbot",
  "anthropic-ai",
  "applebot",
  "applebot-extended",
  "archive-org-bot",
  "baiduspider",
  "bingbot",
  "bytespider",
  "ccbot",
  "chatgpt-user",
  "claude-searchbot",
  "claude-user",
  "claude-web",
  "claudebot",
  "cohere-ai",
  "cohere-training-data-crawler",
  "diffbot",
  "duckassistbot",
  "duckduckbot",
  "facebookbot",
  "facebookexternalhit",
  "firecrawlagent",
  "google-cloudvertexbot",
  "google-extended",
  "google-inspectiontool",
  "googlebot",
  "googlebot-image",
  "googlebot-news",
  "googleother",
  "gptbot",
  "ia-archiver",
  "imagesiftbot",
  "img2dataset",
  "meta-externalagent",
  "meta-externalfetcher",
  "mistralai-user",
  "oai-searchbot",
  "omgili",
  "omgilibot",
  "perplexity-user",
  "perplexitybot",
  "petalbot",
  "scrapy",
  "semrushbot-ocob",
  "seznambot",
  "storebot-google",
  "tiktokspider",
  "timpibot",
  "webzio-extended",
  "yandexbot",
  "yeti",
  "youbot"
 ],
 "robots_txt_url": "https://www.pathwren.workers.dev/robots/maximum-ai-visibility.txt",
 "robots_txt": "# AI Crawler Index — policy: maximum-ai-visibility\n# Maximum AI visibility\n# Allow every AI crawler and every search engine; refuse only SEO scrapers. For sites whose goal is to be found and cited by machines.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/maximum-ai-visibility.html\n# 54 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AI2Bot\nAllow: /\n\nUser-agent: Ai2Bot-Dolma\nAllow: /\n\nUser-agent: Amazonbot\nAllow: /\n\nUser-agent: anthropic-ai   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: Applebot\nAllow: /\n\nUser-agent: Applebot-Extended   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: archive.org_bot\nAllow: /\n\nUser-agent: Baiduspider\nAllow: /\n\nUser-agent: bingbot\nAllow: /\n\nUser-agent: Bytespider   # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: Claude-SearchBot\nAllow: /\n\nUser-agent: Claude-User\nAllow: /\n\nUser-agent: Claude-Web   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: cohere-ai\nAllow: /\n\nUser-agent: cohere-training-data-crawler\nAllow: /\n\nUser-agent: Diffbot\nAllow: /\n\nUser-agent: DuckAssistBot\nAllow: /\n\nUser-agent: DuckDuckBot\nAllow: /\n\nUser-agent: FacebookBot\nAllow: /\n\nUser-agent: facebookexternalhit\nAllow: /\n\nUser-agent: FirecrawlAgent\nAllow: /\n\nUser-agent: Google-CloudVertexBot\nAllow: /\n\nUser-agent: Google-Extended   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: Google-InspectionTool\nAllow: /\n\nUser-agent: Googlebot\nAllow: /\n\nUser-agent: Googlebot-Image\nAllow: /\n\nUser-agent: Googlebot-News\nAllow: /\n\nUser-agent: GoogleOther\nAllow: /\n\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ia_archiver\nAllow: /\n\nUser-agent: ImagesiftBot\nAllow: /\n\nUser-agent: img2dataset\nAllow: /\n\nUser-agent: meta-externalagent\nAllow: /\n\nUser-agent: meta-externalfetcher\nAllow: /\n\nUser-agent: MistralAI-User\nAllow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: omgili\nAllow: /\n\nUser-agent: omgilibot\nAllow: /\n\nUser-agent: Perplexity-User   # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: PetalBot\nAllow: /\n\nUser-agent: Scrapy\nAllow: /\n\nUser-agent: SemrushBot-OCOB\nAllow: /\n\nUser-agent: SeznamBot\nAllow: /\n\nUser-agent: Storebot-Google\nAllow: /\n\nUser-agent: TikTokSpider   # compliance disputed; enforce at the edge\nAllow: /\n\nUser-agent: Timpibot\nAllow: /\n\nUser-agent: Webzio-Extended\nAllow: /\n\nUser-agent: YandexBot\nAllow: /\n\nUser-agent: Yeti\nAllow: /\n\nUser-agent: YouBot\nAllow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n",
 "generated": "2026-09-01"
}