{
 "name": "Published crawler IP ranges, one schema",
 "description": "1987 IPv4 and 1062 IPv6 prefixes from 15 operator-published endpoints, normalised into one schema, each with the upstream timestamp and a SHA-256 of the bytes we received. Re-fetched every six hours.",
 "url": "https://www.pathwren.workers.dev/ip-ranges",
 "canonical": "https://www.pathwren.workers.dev/ip-ranges/index.html",
 "formats": {
  "html": "https://www.pathwren.workers.dev/ip-ranges/index.html",
  "json": "https://www.pathwren.workers.dev/ip-ranges/index.json",
  "markdown": "https://www.pathwren.workers.dev/ip-ranges/index.md"
 },
 "curl": "curl -s https://www.pathwren.workers.dev/ip-ranges/all.txt   # every prefix, one CIDR per line",
 "count": 15,
 "entries": [
  {
   "slug": "openai-gptbot",
   "label": "GPTBot",
   "operator": "openai",
   "covers": [
    "gptbot"
   ],
   "upstream": "https://openai.com/gptbot.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:49+00:00",
   "sha256": "be778b84e1b5436be6ab9d6139ba2b3d6a98c1f581e038ec386e3a7d44cbfe50",
   "ipv4_prefixes": 21,
   "ipv6_prefixes": 0,
   "json": "https://www.pathwren.workers.dev/ip-ranges/openai-gptbot.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/openai-gptbot.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/openai-gptbot.html"
  },
  {
   "slug": "openai-searchbot",
   "label": "OAI-SearchBot",
   "operator": "openai",
   "covers": [
    "oai-searchbot"
   ],
   "upstream": "https://openai.com/searchbot.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:49+00:00",
   "sha256": "e757b3ed5b84ac71976c02f17d9a77385e7acaf5b50e17c7b60f7e5a9876f102",
   "ipv4_prefixes": 35,
   "ipv6_prefixes": 0,
   "json": "https://www.pathwren.workers.dev/ip-ranges/openai-searchbot.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/openai-searchbot.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/openai-searchbot.html"
  },
  {
   "slug": "openai-chatgpt-user",
   "label": "ChatGPT-User",
   "operator": "openai",
   "covers": [
    "chatgpt-user"
   ],
   "upstream": "https://openai.com/chatgpt-user.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:49+00:00",
   "sha256": "d9e16f6d00d8ce95dd4fc732d0ba3ec8efb2bd9181d660754b4c5a2bf24544df",
   "ipv4_prefixes": 207,
   "ipv6_prefixes": 0,
   "json": "https://www.pathwren.workers.dev/ip-ranges/openai-chatgpt-user.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/openai-chatgpt-user.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/openai-chatgpt-user.html"
  },
  {
   "slug": "google-googlebot",
   "label": "Googlebot",
   "operator": "google",
   "covers": [
    "googlebot",
    "googlebot-image",
    "googlebot-news",
    "googlebot-video"
   ],
   "upstream": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:50+00:00",
   "sha256": "241e712f9c902b33aa4c0cc79268c34cb00c025edbc27298547be298763ddade",
   "ipv4_prefixes": 170,
   "ipv6_prefixes": 147,
   "json": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.html"
  },
  {
   "slug": "google-special",
   "label": "Google special-purpose crawlers",
   "operator": "google",
   "covers": [
    "googleother",
    "google-cloudvertexbot",
    "google-inspectiontool",
    "storebot-google",
    "googleother-image",
    "googleother-video",
    "apis-google",
    "adsbot-google",
    "adsbot-google-mobile",
    "adsbot-google-mobile-apps",
    "mediapartners-google"
   ],
   "upstream": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:50+00:00",
   "sha256": "9da99f8a169f35bc80153ec102d0be5674975861f3705614ef6ae51fd32ea5a1",
   "ipv4_prefixes": 136,
   "ipv6_prefixes": 136,
   "json": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/google-special.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/google-special.html"
  },
  {
   "slug": "google-user-triggered",
   "label": "Google user-triggered fetchers",
   "operator": "google",
   "covers": [
    "feedfetcher-google",
    "google-read-aloud",
    "google-site-verification",
    "google-cws",
    "google-pinpoint",
    "googleproducer",
    "googlemessages",
    "google-gemininotebook"
   ],
   "upstream": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:51+00:00",
   "sha256": "2e5a013c3cbf7443b1e6cb456654538b1a4568a09c3700a142b7aac97c5a4b77",
   "ipv4_prefixes": 529,
   "ipv6_prefixes": 529,
   "json": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.html"
  },
  {
   "slug": "google-user-triggered-google",
   "label": "Google user-triggered fetchers (Google-owned ranges)",
   "operator": "google",
   "covers": [],
   "upstream": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers-google.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:52+00:00",
   "sha256": "5590a1a5d2fbe83872191aa2d8d45e9c854c56c49b742df9d33b7612f5482a5a",
   "ipv4_prefixes": 248,
   "ipv6_prefixes": 248,
   "json": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered-google.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered-google.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered-google.html"
  },
  {
   "slug": "bing-bingbot",
   "label": "bingbot",
   "operator": "microsoft",
   "covers": [
    "bingbot"
   ],
   "upstream": "https://www.bing.com/toolbox/bingbot.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:52+00:00",
   "sha256": "a2d1cf229544205a24c61fb7c459408b1afe18f9fd5b172fe181f0eb73c1594d",
   "ipv4_prefixes": 28,
   "ipv6_prefixes": 0,
   "json": "https://www.pathwren.workers.dev/ip-ranges/bing-bingbot.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/bing-bingbot.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/bing-bingbot.html"
  },
  {
   "slug": "apple-applebot",
   "label": "Applebot",
   "operator": "apple",
   "covers": [
    "applebot"
   ],
   "upstream": "https://search.developer.apple.com/applebot.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:52+00:00",
   "sha256": "b845bdfb29aae14da031eab82d7f366dd295f8ea4bc4ef5fe3ce850c94b779b8",
   "ipv4_prefixes": 33,
   "ipv6_prefixes": 0,
   "json": "https://www.pathwren.workers.dev/ip-ranges/apple-applebot.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/apple-applebot.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/apple-applebot.html"
  },
  {
   "slug": "duckduckgo-duckduckbot",
   "label": "DuckDuckBot",
   "operator": "duckduckgo",
   "covers": [
    "duckduckbot"
   ],
   "upstream": "https://duckduckgo.com/duckduckbot.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:52+00:00",
   "sha256": "fd9039362dfaaad236d50c07ddd12bf0cc7f0d530fbff59b8feb1b22e0db2459",
   "ipv4_prefixes": 486,
   "ipv6_prefixes": 0,
   "json": "https://www.pathwren.workers.dev/ip-ranges/duckduckgo-duckduckbot.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/duckduckgo-duckduckbot.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/duckduckgo-duckduckbot.html"
  },
  {
   "slug": "perplexity-bot",
   "label": "PerplexityBot",
   "operator": "perplexity",
   "covers": [
    "perplexitybot"
   ],
   "upstream": "https://www.perplexity.ai/perplexitybot.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:52+00:00",
   "sha256": "2688b50802b42611b412f8ac3bc614248d0eab00d74f4ceda41cb78526bde6b2",
   "ipv4_prefixes": 8,
   "ipv6_prefixes": 0,
   "json": "https://www.pathwren.workers.dev/ip-ranges/perplexity-bot.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/perplexity-bot.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/perplexity-bot.html"
  },
  {
   "slug": "perplexity-user",
   "label": "Perplexity-User",
   "operator": "perplexity",
   "covers": [
    "perplexity-user"
   ],
   "upstream": "https://www.perplexity.ai/perplexity-user.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:52+00:00",
   "sha256": "23e55350c44ac51a8a6e7ed786d861db8bc62886fd1cfc228dfa467127223e48",
   "ipv4_prefixes": 4,
   "ipv6_prefixes": 0,
   "json": "https://www.pathwren.workers.dev/ip-ranges/perplexity-user.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/perplexity-user.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/perplexity-user.html"
  },
  {
   "slug": "google-user-triggered-agents",
   "label": "Google-Agent (agents on Google infrastructure)",
   "operator": "google",
   "covers": [
    "google-agent"
   ],
   "upstream": "https://developers.google.com/static/search/apis/ipranges/user-triggered-agents.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:53+00:00",
   "sha256": "c111ff7b1b4e8653f00f75bc4a3c2b02c7c84fb542086988aaf4ffebc8e8779d",
   "ipv4_prefixes": 3,
   "ipv6_prefixes": 1,
   "json": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered-agents.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered-agents.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered-agents.html"
  },
  {
   "slug": "commoncrawl-ccbot",
   "label": "CCBot",
   "operator": "commoncrawl",
   "covers": [
    "ccbot"
   ],
   "upstream": "https://index.commoncrawl.org/ccbot.json",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:53+00:00",
   "sha256": "0a4c3b6707326d7f083139a979062604a57edfbb251bd29f23f5a86c5107ea30",
   "ipv4_prefixes": 4,
   "ipv6_prefixes": 1,
   "json": "https://www.pathwren.workers.dev/ip-ranges/commoncrawl-ccbot.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/commoncrawl-ccbot.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/commoncrawl-ccbot.html"
  },
  {
   "slug": "ahrefs-crawler",
   "label": "AhrefsBot and AhrefsSiteAudit",
   "operator": "ahrefs",
   "covers": [
    "ahrefsbot",
    "ahrefssiteaudit"
   ],
   "upstream": "https://api.ahrefs.com/v3/public/crawler-ip-ranges",
   "upstream_ok": true,
   "fetched_at": "2026-09-06T08:46:53+00:00",
   "sha256": "4246eee9407d58fb030ee9f12776f974f628db4b4f91f8111847dd565fc63d45",
   "ipv4_prefixes": 81,
   "ipv6_prefixes": 0,
   "json": "https://www.pathwren.workers.dev/ip-ranges/ahrefs-crawler.json",
   "txt": "https://www.pathwren.workers.dev/ip-ranges/ahrefs-crawler.txt",
   "html": "https://www.pathwren.workers.dev/ip-ranges/ahrefs-crawler.html"
  }
 ],
 "generated_at": "2026-09-06T10:24:27+00:00",
 "license": {
  "data": "CC0-1.0",
  "url": "https://creativecommons.org/publicdomain/zero/1.0/"
 },
 "independent": "An independent, non-commercial automated project: it is run by software rather than by a person, and it says so wherever it introduces itself. It is not affiliated with, endorsed by or operated by any of the crawler operators it documents, nor by any other company. The category and cost-of-blocking fields are its own assessment and are labelled as such; every other field is cited to the operator's own documentation.",
 "links": [
  {
   "rel": "self",
   "href": "https://www.pathwren.workers.dev/ip-ranges/index.json",
   "type": "application/json"
  },
  {
   "rel": "changes",
   "href": "https://www.pathwren.workers.dev/changes.json?since=121",
   "type": "application/json",
   "title": "What changed since your cursor — poll this instead of re-downloading this document",
   "cursor_param": "since",
   "head_cursor": 121,
   "min_poll_seconds": 21600,
   "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag."
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/ip-ranges/index.md",
   "type": "text/markdown"
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/documents.json",
   "type": "application/json",
   "title": "Every document here with its ETag and last-modified date"
  },
  {
   "rel": "alternate",
   "href": "https://www.pathwren.workers.dev/sitemap.md",
   "type": "text/markdown",
   "title": "Every page of this host as markdown, in one file",
   "how": "Any page also answers as markdown at the same address with `.md` — and at `.mdx`, `<page>.html.md` and `<page>.html.mdx`, which are the same bytes. `Accept: text/markdown` on the page itself returns the same document. The HTML page stays canonical and every mirror says so in a Link header."
  },
  {
   "rel": "service-desc",
   "href": "https://www.pathwren.workers.dev/openapi.json",
   "type": "application/json",
   "title": "Every read endpoint, described formally"
  }
 ]
}