{
 "slug": "python-classify",
 "title": "Python: classify a request",
 "language": "python",
 "suggested_filename": "classify.py",
 "description": "Forty lines that load agents.json and return the category, operator and robots token for a user-agent string. The shape most log pipelines need.",
 "url": "https://www.pathwren.workers.dev/snippet/python-classify.txt",
 "body": "#!/usr/bin/env python3\n\"\"\"AI Crawler Index — classify a user-agent string.\ncurl -s https://www.pathwren.workers.dev/snippet/python-classify.txt -o classify.py\n\nLoads the index once, answers in O(number of crawlers) per call, no dependencies.\nRefresh agents.json on whatever schedule you like; the shape never changes.\n\"\"\"\nimport json, re, urllib.request\n\nINDEX = \"https://www.pathwren.workers.dev/data/agents.json\"\n\n\ndef load(url=INDEX):\n    with urllib.request.urlopen(url, timeout=20) as r:\n        return json.load(r)[\"crawlers\"]\n\n\nclass Classifier:\n    def __init__(self, crawlers=None):\n        self.crawlers = crawlers or load()\n        self.rx = [(re.compile(re.escape(c[\"user_agent_substring\"]), re.I), c)\n                   for c in self.crawlers\n                   if not c[\"user_agent_substring\"].startswith(\"(\")]\n\n    def __call__(self, ua: str):\n        \"\"\"Return the matching record, or None for anything unrecognised.\"\"\"\n        for rx, c in self.rx:\n            if rx.search(ua or \"\"):\n                return c\n        return None\n\n\nif __name__ == \"__main__\":\n    import sys\n    c = Classifier()\n    for line in (sys.argv[1:] or sys.stdin):\n        hit = c(line.strip())\n        print(f\"{hit['name']}\\t{hit['category']}\\t{hit['operator']}\" if hit\n              else \"-\\tunknown\\t-\")\n"
}