{ "name": "Crawler IP Verifier — AI Crawler Index", "description": "Verify at volume that an address really belongs to the crawler operator it claims. Hand it up to 500 addresses and it says, for each, which operator prefix contains it, from which published source and when that source was last mirrored; hand it a CIDR and it reports every published prefix that contains, is contained by or overlaps it; ask it for an allowlist and it returns a paste-ready nginx, apache, haproxy, Cloudflare, ipset, caddy or plain-CIDR config. It also says which operators can be verified by prefix at all and which need the two-step reverse-DNS check — which it hands you as a command and never runs, because it makes no outbound request of any kind. Absence from a range list is reported as a miss, never as proof of a forgery. Deterministic and read-only: there is no model behind it — every answer comes from a public dataset rebuilt every six hours from each operator's own published documentation and IP ranges, and the same skills are also available as MCP tools at https://www.pathwren.workers.dev/mcp/netcheck. No key, no signup, no quota. Independent and unaffiliated with any operator it documents.", "supportedInterfaces": [ { "url": "https://www.pathwren.workers.dev/a2a/netcheck", "protocolBinding": "JSONRPC", "protocolVersion": "1.0" } ], "url": "https://www.pathwren.workers.dev/a2a/netcheck", "preferredTransport": "JSONRPC", "protocolVersion": "1.0", "provider": { "organization": "Pathwren", "url": "https://www.pathwren.workers.dev" }, "version": "1.0.0", "documentationUrl": "https://www.pathwren.workers.dev/a2a.html", "iconUrl": "https://www.pathwren.workers.dev/icon.png", "capabilities": { "streaming": false, "pushNotifications": false, "extendedAgentCard": false, "extensions": [ { "uri": "https://www.pathwren.workers.dev/changes.json", "description": "Since-cursor change feed over everything this agent answers from: GET /changes.json?since= returns only what moved — operator IP-range lists that gained or lost prefixes, upstreams that failed or recovered, crawler records added or edited. Read `cursor` from the answer and send it back next time; it advances only on a real change, so an unchanged answer is proof and costs a few hundred bytes. The same feed is the changes_since skill on this endpoint.", "required": false, "params": { "cursorParameter": "since", "transport": "https-get", "minPollSeconds": 21600, "skill": "changes_since", "siblingDocument": "https://www.pathwren.workers.dev/data/agents.json" } } ] }, "defaultInputModes": [ "application/json", "text/plain" ], "defaultOutputModes": [ "application/json", "text/plain" ], "skills": [ { "id": "verify_batch", "name": "Verify many addresses at once", "description": "Check up to 500 addresses against every mirrored operator prefix list in one call. Returns, per address, the matching prefix, the source that published it and when it was mirrored, plus a rollup by operator. Accepts {ip, claim} pairs to test a claimed identity against the ranges, and says plainly that a miss is not evidence of a fake.", "tags": [ "ip ranges", "verification", "bot detection", "batch" ], "examples": [ "{\"skill\":\"verify_batch\",\"addresses\":[\"20.171.206.5\",\"66.249.66.1\",\"203.0.113.9\"]}", "is 20.171.206.5 or 66.249.66.1 a published crawler address" ], "inputModes": [ "application/json", "text/plain" ], "outputModes": [ "application/json", "text/plain" ] }, { "id": "lookup_prefix", "name": "Prefix arithmetic against published ranges", "description": "Give a CIDR and get every published operator prefix that contains it, is contained by it, or partially overlaps it — the check for 'does my network collide with a crawler range' and for auditing an allowlist somebody handed you. Give an operator or source slug instead and get everything that source publishes.", "tags": [ "cidr", "ip ranges", "network", "lookup" ], "examples": [ "{\"skill\":\"lookup_prefix\",\"cidr\":\"66.249.64.0/19\"}", "lookup_prefix 20.171.206.0/24" ], "inputModes": [ "application/json", "text/plain" ], "outputModes": [ "application/json", "text/plain" ] }, { "id": "export_ip_acl", "name": "Paste-ready allowlist or denylist", "description": "Turn a set of operators into a config you can paste: nginx geo, nginx allow/deny, Apache, HAProxy, a Cloudflare firewall expression, an ipset script, a Caddy matcher, or a plain CIDR list. Every export carries a provenance header naming each source URL and the mirror time, and reports the rule cost.", "tags": [ "firewall", "nginx", "allowlist", "configuration" ], "examples": [ "{\"skill\":\"export_ip_acl\",\"operators\":[\"openai\"],\"format\":\"nginx-allow-deny\",\"action\":\"allow\"}" ], "inputModes": [ "application/json", "text/plain" ], "outputModes": [ "application/json", "text/plain" ] }, { "id": "verification_methods", "name": "Can this crawler be verified at all, and how?", "description": "Per crawler: whether the operator publishes IP ranges, documents reverse DNS, or publishes no verification method at all — with the exact source URL or the exact dig commands, and how fresh this mirror is. Call it before reading anything into an address that matched nothing. Reverse DNS is never run here; the command is handed back instead.", "tags": [ "verification", "reverse dns", "operators", "method" ], "examples": [ "{\"skill\":\"verification_methods\",\"crawler\":\"gptbot\"}", "how do I verify claudebot" ], "inputModes": [ "application/json", "text/plain" ], "outputModes": [ "application/json", "text/plain" ] }, { "id": "range_stats", "name": "The state of the mirror", "description": "Per source: prefix counts, total addresses and /24-equivalents, smallest and largest prefix, the publishing URL and when it was last fetched; plus prefixes published by more than one operator. This is the sizing and freshness information an allowlist decision needs.", "tags": [ "freshness", "monitoring", "data quality", "mirror" ], "examples": [ "{\"skill\":\"range_stats\"}", "how fresh is the ip range mirror" ], "inputModes": [ "application/json", "text/plain" ], "outputModes": [ "application/json", "text/plain" ] } ] }