{
 "operator": "ByteDance",
 "slug": "bytedance",
 "docs": "https://www.bytespider.net/",
 "crawler_count": 2,
 "crawlers": [
  {
   "slug": "bytespider",
   "name": "Bytespider",
   "operator": "ByteDance",
   "operator_slug": "bytedance",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "Bytespider",
   "user_agent_substring": "Bytespider",
   "user_agent_example": "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)",
   "respects_robots_txt": "disputed",
   "respects_robots_txt_label": "compliance disputed",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "ByteDance's crawler, associated with training data collection for Doubao and related models. Repeatedly reported by CDNs and site operators as the highest-volume AI crawler on the web and as inconsistent about robots.txt.",
   "cost_of_blocking": "Little to lose. If you want it gone, expect to block by user-agent at the edge rather than to ask politely in robots.txt.",
   "operator_docs": "https://www.bytespider.net/",
   "html_url": "https://www.pathwren.workers.dev/crawler/bytespider.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/bytespider.json",
   "last_reviewed": "2026-09-01"
  },
  {
   "slug": "tiktokspider",
   "name": "TikTokSpider",
   "operator": "ByteDance",
   "operator_slug": "bytedance",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "TikTokSpider",
   "user_agent_substring": "TikTokSpider",
   "user_agent_example": "Mozilla/5.0 (compatible; TikTokSpider; ttspider-feedback@tiktok.com)",
   "respects_robots_txt": "disputed",
   "respects_robots_txt_label": "compliance disputed",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "A second ByteDance crawler identifying with TikTok, collecting page content for the same family of models.",
   "cost_of_blocking": "Little to lose unless TikTok search referral matters to you.",
   "operator_docs": "https://www.bytespider.net/",
   "html_url": "https://www.pathwren.workers.dev/crawler/tiktokspider.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/tiktokspider.json",
   "last_reviewed": "2026-09-01"
  }
 ],
 "ip_range_endpoints": []
}