{
 "slug": "allow-ai-search-only",
 "title": "Allow AI search and user fetches, block the rest",
 "summary": "Be findable and citable in assistants without contributing to training corpora.",
 "detail": "The inverse framing of block-ai-training, written as an allowlist so the default for anything new is deny. Fetches a user explicitly asked for stay allowed, because refusing those produces a visible error for a real person who wanted your page.",
 "crawler_count": 28,
 "crawlers": [
  "amazonbot",
  "applebot",
  "baiduspider",
  "bingbot",
  "chatgpt-user",
  "claude-searchbot",
  "claude-user",
  "claude-web",
  "cohere-ai",
  "duckassistbot",
  "duckduckbot",
  "facebookexternalhit",
  "google-cloudvertexbot",
  "googlebot",
  "googlebot-image",
  "googlebot-news",
  "meta-externalfetcher",
  "mistralai-user",
  "oai-searchbot",
  "perplexity-user",
  "perplexitybot",
  "petalbot",
  "seznambot",
  "storebot-google",
  "timpibot",
  "yandexbot",
  "yeti",
  "youbot"
 ],
 "robots_txt_url": "https://www.pathwren.workers.dev/robots/allow-ai-search-only.txt",
 "robots_txt": "# AI Crawler Index — policy: allow-ai-search-only\n# Allow AI search and user fetches, block the rest\n# Be findable and citable in assistants without contributing to training corpora.\n# Generated 2026-09-01 from https://www.pathwren.workers.dev/policy/allow-ai-search-only.html\n# 28 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: Amazonbot\nAllow: /\n\nUser-agent: Applebot\nAllow: /\n\nUser-agent: Baiduspider\nAllow: /\n\nUser-agent: bingbot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: Claude-SearchBot\nAllow: /\n\nUser-agent: Claude-User\nAllow: /\n\nUser-agent: Claude-Web   # control token, no crawler uses this user-agent\nAllow: /\n\nUser-agent: cohere-ai\nAllow: /\n\nUser-agent: DuckAssistBot\nAllow: /\n\nUser-agent: DuckDuckBot\nAllow: /\n\nUser-agent: facebookexternalhit\nAllow: /\n\nUser-agent: Google-CloudVertexBot\nAllow: /\n\nUser-agent: Googlebot\nAllow: /\n\nUser-agent: Googlebot-Image\nAllow: /\n\nUser-agent: Googlebot-News\nAllow: /\n\nUser-agent: meta-externalfetcher\nAllow: /\n\nUser-agent: MistralAI-User\nAllow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: Perplexity-User   # operator states robots.txt does not apply; enforce at the edge\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: PetalBot\nAllow: /\n\nUser-agent: SeznamBot\nAllow: /\n\nUser-agent: Storebot-Google\nAllow: /\n\nUser-agent: Timpibot\nAllow: /\n\nUser-agent: YandexBot\nAllow: /\n\nUser-agent: Yeti\nAllow: /\n\nUser-agent: YouBot\nAllow: /\n\n# Anything not named above is refused.\nUser-agent: *\nDisallow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n",
 "generated": "2026-09-01"
}