Fetches this index's regex list at the edge and tags every request with a category header, so your origin can log AI traffic without shipping a bot list to every app.
curl -s https://www.pathwren.workers.dev/snippet/cloudflare-worker.txt -o worker.js
// AI Crawler Index — Cloudflare Worker: tag every request with a crawler class.
// curl -s https://www.pathwren.workers.dev/snippet/cloudflare-worker.txt
// Pulls the regex list from this index once per hour and caches it at the edge,
// so the bot list lives in one place instead of in every app.
const INDEX = "https://www.pathwren.workers.dev/data/ua-regex.json";
let cache = null, cachedAt = 0;
async function patterns() {
if (cache && Date.now() - cachedAt < 3600_000) return cache;
const r = await fetch(INDEX, { cf: { cacheTtl: 3600 } });
const d = await r.json();
cache = Object.entries(d.by_category).map(([k, v]) => [k, new RegExp(v, "i")]);
cachedAt = Date.now();
return cache;
}
export default {
async fetch(request) {
const ua = request.headers.get("user-agent") || "";
let klass = "human";
for (const [name, re] of await patterns()) if (re.test(ua)) { klass = name; break; }
const req = new Request(request);
req.headers.set("X-Crawler-Class", klass);
return fetch(req);
}
};