Forty lines that load agents.json and return the category, operator and robots token for a user-agent string. The shape most log pipelines need.
curl -s https://www.pathwren.workers.dev/snippet/python-classify.txt -o classify.py
#!/usr/bin/env python3
"""AI Crawler Index — classify a user-agent string.
curl -s https://www.pathwren.workers.dev/snippet/python-classify.txt -o classify.py
Loads the index once, answers in O(number of crawlers) per call, no dependencies.
Refresh agents.json on whatever schedule you like; the shape never changes.
"""
import json, re, urllib.request
INDEX = "https://www.pathwren.workers.dev/data/agents.json"
def load(url=INDEX):
with urllib.request.urlopen(url, timeout=20) as r:
return json.load(r)["crawlers"]
class Classifier:
def __init__(self, crawlers=None):
self.crawlers = crawlers or load()
self.rx = [(re.compile(re.escape(c["user_agent_substring"]), re.I), c)
for c in self.crawlers
if not c["user_agent_substring"].startswith("(")]
def __call__(self, ua: str):
"""Return the matching record, or None for anything unrecognised."""
for rx, c in self.rx:
if rx.search(ua or ""):
return c
return None
if __name__ == "__main__":
import sys
c = Classifier()
for line in (sys.argv[1:] or sys.stdin):
hit = c(line.strip())
print(f"{hit['name']}\t{hit['category']}\t{hit['operator']}" if hit
else "-\tunknown\t-")