# AI Crawler Index — nginx classification map
# curl -s https://www.pathwren.workers.dev/snippet/nginx-map.txt
map $http_user_agent $ai_crawler {
    default             "";
    "~*AhrefsBot"  "seo";
    "~*AI2Bot"  "dataset";
    "~*Ai2Bot\-Dolma"  "dataset";
    "~*Amazonbot"  "ai-search";
    "~*anthropic\-ai"  "ai-training";
    "~*Applebot"  "search";
    "~*archive\.org_bot"  "archive";
    "~*Baiduspider"  "search";
    "~*bingbot"  "search";
    "~*Bytespider"  "ai-training";
    "~*CCBot"  "dataset";
    "~*ChatGPT\-User"  "user-fetch";
    "~*Claude\-SearchBot"  "ai-search";
    "~*Claude\-User"  "user-fetch";
    "~*Claude\-Web"  "ai-search";
    "~*ClaudeBot"  "ai-training";
    "~*cohere\-ai"  "user-fetch";
    "~*cohere\-training\-data\-crawler"  "ai-training";
    "~*Diffbot"  "dataset";
    "~*DuckAssistBot"  "ai-search";
    "~*DuckDuckBot"  "search";
    "~*FacebookBot"  "ai-training";
    "~*facebookexternalhit"  "preview";
    "~*FirecrawlAgent"  "tool";
    "~*Google\-CloudVertexBot"  "ai-search";
    "~*Google\-InspectionTool"  "tool";
    "~*Googlebot"  "search";
    "~*Googlebot\-Image"  "search";
    "~*Googlebot\-News"  "search";
    "~*GoogleOther"  "ai-training";
    "~*GPTBot"  "ai-training";
    "~*ia_archiver"  "archive";
    "~*ImagesiftBot"  "dataset";
    "~*img2dataset"  "dataset";
    "~*meta\-externalagent"  "ai-training";
    "~*meta\-externalfetcher"  "user-fetch";
    "~*MistralAI\-User"  "user-fetch";
    "~*OAI\-SearchBot"  "ai-search";
    "~*omgili"  "dataset";
    "~*omgilibot"  "dataset";
    "~*Perplexity\-User"  "user-fetch";
    "~*PerplexityBot"  "ai-search";
    "~*PetalBot"  "search";
    "~*Scrapy"  "tool";
    "~*SemrushBot"  "seo";
    "~*SemrushBot\-OCOB"  "ai-training";
    "~*SeznamBot"  "search";
    "~*Storebot\-Google"  "search";
    "~*TikTokSpider"  "ai-training";
    "~*Timpibot"  "search";
    "~*Webzio\-Extended"  "ai-training";
    "~*YandexBot"  "search";
    "~*Yeti"  "search";
    "~*YouBot"  "ai-search";
}

# Then, in a server block: log it, rate-limit it, or refuse it.
#   access_log /var/log/nginx/ai.log combined if=$ai_crawler;
#   if ($ai_crawler = "disputed") { return 403; }