{
 "aid": "www.pathwren.workers.dev",
 "name": "AI Crawler Index",
 "type": "Index",
 "description": "Every AI crawler on the web, what it is for, what blocking it costs you, and the IP ranges its operator publishes — as JSON, CSV, robots.txt and regex. 56 crawlers from 30 operators, 1887 IPv4 and 1056 IPv6 prefixes mirrored from 12 operator-published endpoints and refreshed every six hours. Read-only static files: no key, no signup, no rate limit, CORS open, CC0.",
 "image": "https://www.pathwren.workers.dev/icon.png",
 "url": "https://www.pathwren.workers.dev/apis.json",
 "specificationVersion": "1.0",
 "created": "2026-08-31",
 "modified": "2026-09-01",
 "tags": [
  "AI crawlers",
  "web crawlers",
  "robots.txt",
  "user agents",
  "bot detection",
  "GPTBot",
  "ClaudeBot",
  "crawler IP ranges",
  "llms.txt",
  "open data"
 ],
 "maintainers": [
  {
   "FN": "Pathwren",
   "email": "pathwren@tutamail.com",
   "url": "https://www.pathwren.workers.dev/about.html"
  }
 ],
 "apis": [
  {
   "aid": "www.pathwren.workers.dev:ai-crawler-index",
   "name": "AI Crawler Index",
   "description": "Which crawler a User-Agent header belongs to, who operates it, whether it documents obeying robots.txt, how to verify it is genuine against the operator's own published IP ranges, and what blocking it costs you. Every endpoint is a GET returning JSON, CSV or plain text.",
   "image": "https://www.pathwren.workers.dev/icon.png",
   "humanURL": "https://www.pathwren.workers.dev/api.html?s=ai-crawler-index&c=apis-json",
   "baseURL": "https://www.pathwren.workers.dev",
   "tags": [
    "AI crawlers",
    "web crawlers",
    "robots.txt",
    "user agents",
    "bot detection",
    "GPTBot",
    "ClaudeBot",
    "crawler IP ranges",
    "llms.txt",
    "open data"
   ],
   "properties": [
    {
     "type": "OpenAPI",
     "url": "https://www.pathwren.workers.dev/openapi.json"
    },
    {
     "type": "x-openapi-yaml",
     "url": "https://www.pathwren.workers.dev/openapi.yaml"
    },
    {
     "type": "Swagger",
     "url": "https://www.pathwren.workers.dev/swagger.json"
    },
    {
     "type": "Documentation",
     "url": "https://www.pathwren.workers.dev/api.html"
    },
    {
     "type": "Feed",
     "url": "https://www.pathwren.workers.dev/feed.xml"
    },
    {
     "type": "Feed",
     "url": "https://www.pathwren.workers.dev/feed.json"
    },
    {
     "type": "x-mcp-server",
     "url": "https://www.pathwren.workers.dev/mcp"
    },
    {
     "type": "x-mcp-manifest",
     "url": "https://www.pathwren.workers.dev/.well-known/mcp.json"
    },
    {
     "type": "x-onboarding",
     "url": "https://www.pathwren.workers.dev/.well-known/api-onboarding"
    },
    {
     "type": "x-plugin-manifest",
     "url": "https://www.pathwren.workers.dev/.well-known/ai-plugin.json"
    },
    {
     "type": "x-api-catalog",
     "url": "https://www.pathwren.workers.dev/.well-known/api-catalog"
    },
    {
     "type": "x-llms-txt",
     "url": "https://www.pathwren.workers.dev/llms.txt"
    },
    {
     "type": "x-bulk-data",
     "url": "https://www.pathwren.workers.dev/data/agents.json"
    },
    {
     "type": "x-ip-ranges",
     "url": "https://www.pathwren.workers.dev/ip-ranges/all.json"
    },
    {
     "type": "x-security-txt",
     "url": "https://www.pathwren.workers.dev/.well-known/security.txt"
    },
    {
     "type": "x-sitemap",
     "url": "https://www.pathwren.workers.dev/sitemap.xml"
    },
    {
     "type": "StatusPage",
     "url": "https://www.pathwren.workers.dev/status.html"
    },
    {
     "type": "License",
     "url": "https://creativecommons.org/publicdomain/zero/1.0/"
    }
   ],
   "contact": [
    {
     "FN": "Pathwren",
     "email": "pathwren@tutamail.com",
     "url": "https://www.pathwren.workers.dev/about.html"
    }
   ]
  }
 ]
}