{
 "operator": "Meta",
 "slug": "meta",
 "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
 "crawler_count": 4,
 "crawlers": [
  {
   "slug": "facebookbot",
   "name": "FacebookBot",
   "operator": "Meta",
   "operator_slug": "meta",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "FacebookBot",
   "user_agent_substring": "FacebookBot",
   "user_agent_example": "FacebookBot/1.0 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Meta's older speech- and language-corpus crawler, largely superseded by meta-externalagent but still listed as a valid robots token.",
   "cost_of_blocking": "Negligible today. Keep the rule; expect little traffic.",
   "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/facebookbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/facebookbot.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "facebookexternalhit",
   "name": "facebookexternalhit",
   "operator": "Meta",
   "operator_slug": "meta",
   "category": "preview",
   "category_label": "Link preview fetchers",
   "robots_token": "facebookexternalhit",
   "user_agent_substring": "facebookexternalhit",
   "user_agent_example": "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The link unfurler: it reads your Open Graph tags when somebody shares your URL on a Meta property.",
   "cost_of_blocking": "Severe and usually accidental. Your links share as bare grey boxes with no title, image or description across Facebook, Instagram, Messenger and WhatsApp. Almost nobody means to block this.",
   "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/facebookexternalhit.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/facebookexternalhit.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "meta-externalagent",
   "name": "meta-externalagent",
   "operator": "Meta",
   "operator_slug": "meta",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "meta-externalagent",
   "user_agent_substring": "meta-externalagent",
   "user_agent_example": "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Meta's AI crawler, gathering training data for Llama and Meta AI. It replaced the older FacebookBot name for this purpose.",
   "cost_of_blocking": "Excluded from Meta AI training. Link previews on Facebook, Instagram and WhatsApp are unaffected — those are a different bot.",
   "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/meta-externalagent.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/meta-externalagent.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "meta-externalfetcher",
   "name": "meta-externalfetcher",
   "operator": "Meta",
   "operator_slug": "meta",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "meta-externalfetcher",
   "user_agent_substring": "meta-externalfetcher",
   "user_agent_example": "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a page when a Meta AI user asks about a specific link.",
   "cost_of_blocking": "Meta AI cannot read pages users hand it.",
   "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.json",
   "last_reviewed": "2026-09-01"
  }
 ],
 "ip_range_endpoints": []
}