# AI Crawler Index — APIs.json 1.0 record, as YAML
# The same document as https://www.pathwren.workers.dev/apis.json, serialised as YAML by the same
# build from the same object. If they ever disagree, this file is the bug.
# `url:` below points at the JSON form on purpose: one record, one home.
# APIs.json is a JSON *or* YAML format — see 3.3.1 of the specification at
# https://apisjson.org/format/apisjson_0.23.txt, which also gives /apis.yml
# (3.1) and /apis.yaml (3.7) as the access paths for the YAML form.
# Generated 2026-09-07T08:42:05+00:00. No key, no rate limit, CORS open, CC0.
aid: "www.pathwren.workers.dev"
name: "AI Crawler Index"
type: "Index"
description: "Every AI crawler on the web, what it is for, what blocking it costs you, and the IP ranges its operator publishes — as JSON, CSV, robots.txt and regex. 150 crawlers from 74 operators, 1987 IPv4 and 1062 IPv6 prefixes mirrored from 15 operator-published endpoints and refreshed every six hours. Read-only static files: no key, no signup, no rate limit, CORS open, CC0."
image: "https://www.pathwren.workers.dev/icon.png"
url: "https://www.pathwren.workers.dev/apis.json"
specificationVersion: "1.0"
created: "2026-08-31"
modified: "2026-09-07"
tags:
  - "AI crawlers"
  - "web crawlers"
  - "robots.txt"
  - "user agents"
  - "bot detection"
  - "GPTBot"
  - "ClaudeBot"
  - "crawler IP ranges"
  - "llms.txt"
  - "open data"
maintainers:
  - FN: "Pathwren"
    email: "pathwren@tutamail.com"
    url: "https://www.pathwren.workers.dev/about.html"
apis:
  - aid: "www.pathwren.workers.dev:ai-crawler-index"
    name: "AI Crawler Index"
    description: "Which crawler a User-Agent header belongs to, who operates it, whether it documents obeying robots.txt, how to verify it is genuine against the operator's own published IP ranges, and what blocking it costs you. Every endpoint is a GET returning JSON, CSV or plain text."
    image: "https://www.pathwren.workers.dev/icon.png"
    humanURL: "https://www.pathwren.workers.dev/api.html?s=ai-crawler-index&c=apis-json"
    baseURL: "https://www.pathwren.workers.dev"
    tags:
      - "AI crawlers"
      - "web crawlers"
      - "robots.txt"
      - "user agents"
      - "bot detection"
      - "GPTBot"
      - "ClaudeBot"
      - "crawler IP ranges"
      - "llms.txt"
      - "open data"
    properties:
      - type: "OpenAPI"
        url: "https://www.pathwren.workers.dev/openapi.json"
      - type: "x-openapi-yaml"
        url: "https://www.pathwren.workers.dev/openapi.yaml"
      - type: "x-well-known-openapi"
        url: "https://www.pathwren.workers.dev/.well-known/openapi.json"
      - type: "x-well-known-openapi-yaml"
        url: "https://www.pathwren.workers.dev/.well-known/openapi.yaml"
      - type: "AsyncAPI"
        url: "https://www.pathwren.workers.dev/asyncapi.json"
      - type: "x-asyncapi-yaml"
        url: "https://www.pathwren.workers.dev/asyncapi.yaml"
      - type: "x-apis-yaml"
        url: "https://www.pathwren.workers.dev/apis.yml"
      - type: "Documentation"
        url: "https://www.pathwren.workers.dev/api.html"
      - type: "Feed"
        url: "https://www.pathwren.workers.dev/feed.xml"
      - type: "Feed"
        url: "https://www.pathwren.workers.dev/feed.json"
      - type: "x-mcp-server"
        url: "https://www.pathwren.workers.dev/mcp"
      - type: "x-mcp-manifest"
        url: "https://www.pathwren.workers.dev/.well-known/mcp.json"
      - type: "x-onboarding"
        url: "https://www.pathwren.workers.dev/.well-known/api-onboarding"
      - type: "SignUp"
        url: "https://www.pathwren.workers.dev/register"
      - type: "x-access"
        url: "https://www.pathwren.workers.dev/register.json"
      - type: "x-plugin-manifest"
        url: "https://www.pathwren.workers.dev/.well-known/ai-plugin.json"
      - type: "x-api-catalog"
        url: "https://www.pathwren.workers.dev/.well-known/api-catalog"
      - type: "x-llms-txt"
        url: "https://www.pathwren.workers.dev/llms.txt"
      - type: "x-changes-feed"
        url: "https://www.pathwren.workers.dev/changes.json?since=0"
      - type: "x-bulk-data"
        url: "https://www.pathwren.workers.dev/data/agents.json"
      - type: "x-ip-ranges"
        url: "https://www.pathwren.workers.dev/ip-ranges/all.json"
      - type: "x-security-txt"
        url: "https://www.pathwren.workers.dev/.well-known/security.txt"
      - type: "x-sitemap"
        url: "https://www.pathwren.workers.dev/sitemap.xml"
      - type: "StatusPage"
        url: "https://www.pathwren.workers.dev/status.html"
      - type: "TermsOfService"
        url: "https://www.pathwren.workers.dev/terms.html"
      - type: "PrivacyPolicy"
        url: "https://www.pathwren.workers.dev/privacy.html"
      - type: "License"
        url: "https://creativecommons.org/publicdomain/zero/1.0/"
    contact:
      - FN: "Pathwren"
        email: "pathwren@tutamail.com"
        url: "https://www.pathwren.workers.dev/about.html"
changed_at: "2026-09-07T02:39:55+00:00"
recent_changes:
  - "2026-09-07T02:39Z - changed: modified"
  - "2026-09-06T01:25Z - changed: modified"
  - "2026-09-05T00:45Z - changed: modified"
