# AI Crawler Index — Caddy
# curl -s https://www.pathwren.workers.dev/snippet/caddy.txt
@ai_training header_regexp User-Agent "(?i)(GPTBot|ClaudeBot|anthropic\-ai|GoogleOther|Bytespider|TikTokSpider|meta\-externalagent|FacebookBot|cohere\-training\-data\-crawler|Webzio\-Extended|SemrushBot\-OCOB)"
@disputed    header_regexp User-Agent "(?i)(Bytespider|TikTokSpider)"

handle @disputed {
	respond "403 - see /robots.txt" 403
}
handle @ai_training {
	header X-Crawler-Class "ai-training"
	# file_server / reverse_proxy as usual
}