{
 "name": "ai-crawler-robots",
 "summary": "Does your robots.txt block the crawlers you think it blocks?",
 "description": "Lint a robots.txt, evaluate any path against RFC 9309, and audit the file against every AI crawler this index knows. Zero dependencies, no network, no URL fetching.",
 "language": "python",
 "registry": "pypi",
 "registry_url": "https://pypi.org/project/ai-crawler-robots/",
 "install": "pip install ai-crawler-robots",
 "module": "ai_crawler_robots",
 "console_script": "ai-crawler-robots",
 "requires_python": ">=3.8",
 "dependencies": [],
 "license": {
  "code": "MIT",
  "data": "CC0-1.0"
 },
 "homepage": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/",
 "documentation": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/",
 "data_url": "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/data.json",
 "index": "https://www.pathwren.workers.dev/",
 "cli": [
  {
   "cmd": "ai-crawler-robots lint robots.txt",
   "does": "the faults that make a file do nothing: a UA string where a product token belongs, a misspelled directive, Noindex:, a shadowed group, a path without a leading slash"
  },
  {
   "cmd": "ai-crawler-robots audit robots.txt",
   "does": "every AI crawler this index knows, split into blocked / allowed / says-no-but-will-not-obey"
  },
  {
   "cmd": "ai-crawler-robots check robots.txt --agent GPTBot --path /blog/",
   "does": "allowed or not, and the line number that decided it"
  },
  {
   "cmd": "ai-crawler-robots diff old.txt new.txt",
   "does": "what changed by EFFECT, not by text — a reformat is not a change"
  },
  {
   "cmd": "ai-crawler-robots policy block-ai-training --merge robots.txt",
   "does": "add a ready-made stance without touching the rules already in the file"
  }
 ],
 "api": [
  {
   "sig": "lint(text)",
   "returns": "findings with level, line, message and a suggested fix"
  },
  {
   "sig": "is_allowed(text, agent, path)",
   "returns": "the RFC 9309 verdict, the winning rule, its line, and whether two conformant parsers would disagree"
  },
  {
   "sig": "audit(text, path='/')",
   "returns": "blocked / allowed / unenforceable per crawler, plus the tokens in the file that match nothing"
  },
  {
   "sig": "diff(old, new)",
   "returns": "only the crawlers whose verdict flipped"
  },
  {
   "sig": "merge_policy(text, stance)",
   "returns": "the file with a stance added and its existing groups untouched"
  },
  {
   "sig": "refresh()",
   "returns": "today's table, cached six hours — the only network call in the package"
  }
 ],
 "family": [
  {
   "name": "ai-crawler-index",
   "url": "https://pypi.org/project/ai-crawler-index/"
  },
  {
   "name": "ai-crawler-verify",
   "url": "https://pypi.org/project/ai-crawler-verify/"
  },
  {
   "name": "ai-crawler-logs",
   "url": "https://pypi.org/project/ai-crawler-logs/"
  }
 ],
 "offline": true,
 "generated_at": "2026-09-01T19:26:24+00:00",
 "independent": "An independent, non-commercial automated project: it is run by software rather than by a person, and it says so wherever it introduces itself. It is not affiliated with, endorsed by or operated by any of the crawler operators it documents, nor by any other company. The category and cost-of-blocking fields are its own assessment and are labelled as such; every other field is cited to the operator's own documentation."
}