{
 "id": "robots-allowed",
 "endpoint": "https://www.pathwren.workers.dev/tools/robots-allowed",
 "title": "Would this crawler fetch this path?",
 "what": "Evaluate a pasted robots.txt for one crawler and one or more paths under RFC 9309 — longest token match for the group, longest pattern for the rule, Allow breaking a tie, * and $ supported — and get allowed/disallowed per path with the exact line that decided it, plus a flag on the cases where a merge-groups parser and a first-group-wins parser would disagree.",
 "auth": "none — no account, no key, no handshake, no session",
 "method": "GET, or POST the file as the raw body",
 "cors": "*",
 "cache_seconds": 3600,
 "parameters": [
  {
   "name": "robots_txt",
   "aliases": [
    "robots",
    "text"
   ],
   "required": true,
   "repeatable": false,
   "what": "The contents of the robots.txt file, URL-encoded, or the raw POST body.",
   "becomes": "robots_txt"
  },
  {
   "name": "ua",
   "aliases": [
    "user_agent",
    "token"
   ],
   "required": true,
   "repeatable": false,
   "what": "The crawler's product token — GPTBot, Googlebot, ClaudeBot — not a whole User-Agent header.",
   "becomes": "user_agent"
  },
  {
   "name": "path",
   "aliases": [
    "paths"
   ],
   "required": false,
   "repeatable": true,
   "what": "Path to test. Repeat for several, or send a comma separated list. Defaults to /. Max 200.",
   "becomes": "paths"
  }
 ],
 "example": {
  "url": "https://www.pathwren.workers.dev/tools/robots-allowed?robots_txt=User-agent%3A%20*%0ADisallow%3A%20%2F%0AAllow%3A%20%2Fblog%2F%0A&ua=GPTBot&path=/blog/post&path=/private&s=client-dossiers",
  "curl": "curl -s 'https://www.pathwren.workers.dev/tools/robots-allowed?robots_txt=User-agent%3A%20*%0ADisallow%3A%20%2F%0AAllow%3A%20%2Fblog%2F%0A&ua=GPTBot&path=/blog/post&path=/private&s=client-dossiers'",
  "what_it_answers": "One file, one crawler, two paths: /blog/post allowed by the longer Allow, /private disallowed, each with the line number that decided it."
 },
 "same_answer_over_mcp": {
  "server": "robots-policy-lint",
  "endpoint": "https://www.pathwren.workers.dev/mcp/robots",
  "method": "tools/call",
  "tool": "check_path_allowed",
  "note": "The GET endpoint calls this tool in-process through the MCP server's own JSON-RPC envelope. One implementation, two doors."
 },
 "caveats": [
  "robots.txt is a request, not an enforcement mechanism. This says what a compliant crawler would do; it does not say what an arbitrary client will do."
 ],
 "docs": "https://www.pathwren.workers.dev/tools/robots-allowed.html?s=client-dossiers",
 "catalogue": "https://www.pathwren.workers.dev/tools/index.json?s=client-dossiers",
 "license": "CC0-1.0",
 "independent": true
}