A map block that turns the user-agent into a category, so you can rate-limit training crawlers, allow AI search, and 403 the disputed ones without maintaining three separate if-chains.
curl -s https://www.pathwren.workers.dev/snippet/nginx-map.txt -o nginx-map.conf
# AI Crawler Index — nginx classification map
# curl -s https://www.pathwren.workers.dev/snippet/nginx-map.txt
map $http_user_agent $ai_crawler {
default "";
"~*AhrefsBot" "seo";
"~*AI2Bot" "dataset";
"~*Ai2Bot\-Dolma" "dataset";
"~*Amazonbot" "ai-search";
"~*anthropic\-ai" "ai-training";
"~*Applebot" "search";
"~*archive\.org_bot" "archive";
"~*Baiduspider" "search";
"~*bingbot" "search";
"~*Bytespider" "ai-training";
"~*CCBot" "dataset";
"~*ChatGPT\-User" "user-fetch";
"~*Claude\-SearchBot" "ai-search";
"~*Claude\-User" "user-fetch";
"~*Claude\-Web" "ai-search";
"~*ClaudeBot" "ai-training";
"~*cohere\-ai" "user-fetch";
"~*cohere\-training\-data\-crawler" "ai-training";
"~*Diffbot" "dataset";
"~*DuckAssistBot" "ai-search";
"~*DuckDuckBot" "search";
"~*FacebookBot" "ai-training";
"~*facebookexternalhit" "preview";
"~*FirecrawlAgent" "tool";
"~*Google\-CloudVertexBot" "ai-search";
"~*Google\-InspectionTool" "tool";
"~*Googlebot" "search";
"~*Googlebot\-Image" "search";
"~*Googlebot\-News" "search";
"~*GoogleOther" "ai-training";
"~*GPTBot" "ai-training";
"~*ia_archiver" "archive";
"~*ImagesiftBot" "dataset";
"~*img2dataset" "dataset";
"~*meta\-externalagent" "ai-training";
"~*meta\-externalfetcher" "user-fetch";
"~*MistralAI\-User" "user-fetch";
"~*OAI\-SearchBot" "ai-search";
"~*omgili" "dataset";
"~*omgilibot" "dataset";
"~*Perplexity\-User" "user-fetch";
"~*PerplexityBot" "ai-search";
"~*PetalBot" "search";
"~*Scrapy" "tool";
"~*SemrushBot" "seo";
"~*SemrushBot\-OCOB" "ai-training";
"~*SeznamBot" "search";
"~*Storebot\-Google" "search";
"~*TikTokSpider" "ai-training";
"~*Timpibot" "search";
"~*Webzio\-Extended" "ai-training";
"~*YandexBot" "search";
"~*Yeti" "search";
"~*YouBot" "ai-search";
}
# Then, in a server block: log it, rate-limit it, or refuse it.
# access_log /var/log/nginx/ai.log combined if=$ai_crawler;
# if ($ai_crawler = "disputed") { return 403; }