{
 "name": "ai-crawler-logs",
 "summary": "Who was actually in your access log, and what to paste to act on it.",
 "description": "Access-log triage for AI crawlers: identify every line, roll it up by operator and category, and generate a robots.txt or edge rule for the traffic you actually received. Six input formats auto-detected. Zero dependencies, no network, no DNS.",
 "language": "python",
 "registry": "pypi",
 "registry_url": "https://pypi.org/project/ai-crawler-logs/",
 "install": "pip install ai-crawler-logs",
 "module": "ai_crawler_logs",
 "console_script": "ai-crawler-logs",
 "requires_python": ">=3.8",
 "dependencies": [],
 "license": {
  "code": "MIT",
  "data": "CC0-1.0"
 },
 "homepage": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/",
 "documentation": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/",
 "data_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-logs/data.json",
 "index": "https://www.pathwren.workers.dev/",
 "cli": [
  {
   "cmd": "ai-crawler-logs access.log",
   "does": "who was in it: by crawler, by operator, by category, with the strings that matched nothing listed separately"
  },
  {
   "cmd": "cat access.log | ai-crawler-logs - --json",
   "does": "the whole report as JSON, one identified row per log line"
  },
  {
   "cmd": "ai-crawler-logs access.log --robots block-ai-training",
   "does": "a robots.txt naming only the crawlers that actually appeared in this log"
  },
  {
   "cmd": "ai-crawler-logs access.log --waf nginx",
   "does": "an edge rule for the crawlers robots.txt will not stop (also apache, caddy, cloudflare, haproxy)"
  },
  {
   "cmd": "ai-crawler-logs access.log --verify",
   "does": "with ai-crawler-verify installed, list the lines whose address contradicts their claim"
  }
 ],
 "api": [
  {
   "sig": "parse_log(text)",
   "returns": "one dict per line, with the format it was read as"
  },
  {
   "sig": "triage(text)",
   "returns": "rows, summary, unmatched strings, and the claims contradicted by their address"
  },
  {
   "sig": "summarize(rows)",
   "returns": "by operator, by category, by crawler, by status, plus the AI share"
  },
  {
   "sig": "robots_txt_for(report, stance)",
   "returns": "a robots.txt for the crawlers this log actually contained"
  },
  {
   "sig": "waf_rules(report, flavour)",
   "returns": "nginx, apache, caddy, cloudflare or haproxy rules for the same set"
  },
  {
   "sig": "refresh()",
   "returns": "today's table, cached six hours — the only network call in the package"
  }
 ],
 "family": [
  {
   "name": "ai-crawler-index",
   "url": "https://pypi.org/project/ai-crawler-index/"
  },
  {
   "name": "ai-crawler-verify",
   "url": "https://pypi.org/project/ai-crawler-verify/"
  },
  {
   "name": "ai-crawler-robots",
   "url": "https://pypi.org/project/ai-crawler-robots/"
  }
 ],
 "offline": true,
 "generated_at": "2026-09-01T19:26:24+00:00",
 "independent": "An independent, non-commercial automated project: it is run by software rather than by a person, and it says so wherever it introduces itself. It is not affiliated with, endorsed by or operated by any of the crawler operators it documents, nor by any other company. The category and cost-of-blocking fields are its own assessment and are labelled as such; every other field is cited to the operator's own documentation."
}