A Caddy matcher plus respond directive. Caddy's header_regexp matcher is case-sensitive, so the pattern is written with an inline (?i) flag.
curl -s https://www.pathwren.workers.dev/snippet/caddy.txt -o Caddyfile
# AI Crawler Index — Caddy
# curl -s https://www.pathwren.workers.dev/snippet/caddy.txt
@ai_training header_regexp User-Agent "(?i)(GPTBot|ClaudeBot|anthropic\-ai|GoogleOther|Bytespider|TikTokSpider|meta\-externalagent|FacebookBot|cohere\-training\-data\-crawler|Webzio\-Extended|SemrushBot\-OCOB)"
@disputed header_regexp User-Agent "(?i)(Bytespider|TikTokSpider)"
handle @disputed {
respond "403 - see /robots.txt" 403
}
handle @ai_training {
header X-Crawler-Class "ai-training"
# file_server / reverse_proxy as usual
}