{
 "slug": "nginx-map",
 "title": "nginx: classify and block by user-agent",
 "language": "nginx",
 "suggested_filename": "nginx-map.conf",
 "description": "A map block that turns the user-agent into a category, so you can rate-limit training crawlers, allow AI search, and 403 the disputed ones without maintaining three separate if-chains.",
 "url": "https://www.pathwren.workers.dev/snippet/nginx-map.txt",
 "body": "# AI Crawler Index — nginx classification map\n# curl -s https://www.pathwren.workers.dev/snippet/nginx-map.txt\nmap $http_user_agent $ai_crawler {\n    default             \"\";\n    \"~*AhrefsBot\"  \"seo\";\n    \"~*AI2Bot\"  \"dataset\";\n    \"~*Ai2Bot\\-Dolma\"  \"dataset\";\n    \"~*Amazonbot\"  \"ai-search\";\n    \"~*anthropic\\-ai\"  \"ai-training\";\n    \"~*Applebot\"  \"search\";\n    \"~*archive\\.org_bot\"  \"archive\";\n    \"~*Baiduspider\"  \"search\";\n    \"~*bingbot\"  \"search\";\n    \"~*Bytespider\"  \"ai-training\";\n    \"~*CCBot\"  \"dataset\";\n    \"~*ChatGPT\\-User\"  \"user-fetch\";\n    \"~*Claude\\-SearchBot\"  \"ai-search\";\n    \"~*Claude\\-User\"  \"user-fetch\";\n    \"~*Claude\\-Web\"  \"ai-search\";\n    \"~*ClaudeBot\"  \"ai-training\";\n    \"~*cohere\\-ai\"  \"user-fetch\";\n    \"~*cohere\\-training\\-data\\-crawler\"  \"ai-training\";\n    \"~*Diffbot\"  \"dataset\";\n    \"~*DuckAssistBot\"  \"ai-search\";\n    \"~*DuckDuckBot\"  \"search\";\n    \"~*FacebookBot\"  \"ai-training\";\n    \"~*facebookexternalhit\"  \"preview\";\n    \"~*FirecrawlAgent\"  \"tool\";\n    \"~*Google\\-CloudVertexBot\"  \"ai-search\";\n    \"~*Google\\-InspectionTool\"  \"tool\";\n    \"~*Googlebot\"  \"search\";\n    \"~*Googlebot\\-Image\"  \"search\";\n    \"~*Googlebot\\-News\"  \"search\";\n    \"~*GoogleOther\"  \"ai-training\";\n    \"~*GPTBot\"  \"ai-training\";\n    \"~*ia_archiver\"  \"archive\";\n    \"~*ImagesiftBot\"  \"dataset\";\n    \"~*img2dataset\"  \"dataset\";\n    \"~*meta\\-externalagent\"  \"ai-training\";\n    \"~*meta\\-externalfetcher\"  \"user-fetch\";\n    \"~*MistralAI\\-User\"  \"user-fetch\";\n    \"~*OAI\\-SearchBot\"  \"ai-search\";\n    \"~*omgili\"  \"dataset\";\n    \"~*omgilibot\"  \"dataset\";\n    \"~*Perplexity\\-User\"  \"user-fetch\";\n    \"~*PerplexityBot\"  \"ai-search\";\n    \"~*PetalBot\"  \"search\";\n    \"~*Scrapy\"  \"tool\";\n    \"~*SemrushBot\"  \"seo\";\n    \"~*SemrushBot\\-OCOB\"  \"ai-training\";\n    \"~*SeznamBot\"  \"search\";\n    \"~*Storebot\\-Google\"  \"search\";\n    \"~*TikTokSpider\"  \"ai-training\";\n    \"~*Timpibot\"  \"search\";\n    \"~*Webzio\\-Extended\"  \"ai-training\";\n    \"~*YandexBot\"  \"search\";\n    \"~*Yeti\"  \"search\";\n    \"~*YouBot\"  \"ai-search\";\n}\n\n# Then, in a server block: log it, rate-limit it, or refuse it.\n#   access_log /var/log/nginx/ai.log combined if=$ai_crawler;\n#   if ($ai_crawler = \"disputed\") { return 403; }\n"
}