{
  "schema_version": 1,
  "updated": "2026-10-12",
  "description": "Bundled directory of known AI agents, crawlers and automation markers for Agent Lane. ua_patterns are case-insensitive substrings of the User-Agent header. signature_agent is the Web Bot Auth Signature-Agent origin (compared by host). ip_list_url is kept for a future opt-in fetch and is not fetched in v1. Entries with a note are unconfirmed or have known caveats.",
  "categories": {
    "assistant": "Fetches pages on behalf of a chat user in real time",
    "crawler": "Bulk crawler for training or indexing",
    "search": "Search engine or AI search index crawler",
    "browser-agent": "Agent that drives a real browser; usually looks like Chrome",
    "headless": "Headless browser or browser automation framework marker",
    "http-client": "Scripted HTTP client library marker"
  },
  "notes": [
    "Agentic browsers (Perplexity Comet, OpenAI ChatGPT Atlas, Claude in Chrome, Dia) use the user's own Chrome-like user agent and carry no marker. They can only be caught by behavior signals or by an honest purpose declaration.",
    "Google-Extended and Applebot-Extended are robots.txt tokens, not user agent strings; patterns are kept only for completeness.",
    "Cloudflare's signed-agents program (Web Bot Auth) lists Browserbase, Anchor Browser and Manus as participants; their Signature-Agent origins were not confirmed from primary sources and are left null."
  ],
  "agents": [
    {
      "id": "openai-chatgpt-agent",
      "name": "ChatGPT agent (OpenAI)",
      "operator": "OpenAI",
      "ua_patterns": [],
      "signature_agent": "https://chatgpt.com",
      "category": "browser-agent",
      "default_policy": "queue",
      "verified_via": "signature",
      "ip_list_url": "https://openai.com/chatgpt-user.json",
      "docs_url": "https://platform.openai.com/docs/bots",
      "note": "Agent mode drives a Chrome-like browser; the UA has no stable marker. Identify by the Signature-Agent header and Web Bot Auth signature. IP list shared with ChatGPT-User is an assumption."
    },
    {
      "id": "openai-chatgpt-user",
      "name": "ChatGPT-User (OpenAI)",
      "operator": "OpenAI",
      "ua_patterns": ["ChatGPT-User"],
      "signature_agent": null,
      "category": "assistant",
      "default_policy": "queue",
      "verified_via": "ip_list",
      "ip_list_url": "https://openai.com/chatgpt-user.json",
      "docs_url": "https://platform.openai.com/docs/bots",
      "note": null
    },
    {
      "id": "openai-gptbot",
      "name": "GPTBot (OpenAI)",
      "operator": "OpenAI",
      "ua_patterns": ["GPTBot"],
      "signature_agent": null,
      "category": "crawler",
      "default_policy": "queue",
      "verified_via": "ip_list",
      "ip_list_url": "https://openai.com/gptbot.json",
      "docs_url": "https://platform.openai.com/docs/bots",
      "note": null
    },
    {
      "id": "openai-searchbot",
      "name": "OAI-SearchBot (OpenAI)",
      "operator": "OpenAI",
      "ua_patterns": ["OAI-SearchBot"],
      "signature_agent": null,
      "category": "search",
      "default_policy": "queue",
      "verified_via": "ip_list",
      "ip_list_url": "https://openai.com/searchbot.json",
      "docs_url": "https://platform.openai.com/docs/bots",
      "note": null
    },
    {
      "id": "anthropic-claudebot",
      "name": "ClaudeBot (Anthropic)",
      "operator": "Anthropic",
      "ua_patterns": ["ClaudeBot", "anthropic-ai", "Claude-Web"],
      "signature_agent": null,
      "category": "crawler",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "note": "No public crawler IP list known as of 2026-10; anthropic-ai and Claude-Web are legacy tokens."
    },
    {
      "id": "anthropic-claude-user",
      "name": "Claude-User (Anthropic)",
      "operator": "Anthropic",
      "ua_patterns": ["Claude-User"],
      "signature_agent": null,
      "category": "assistant",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "note": "No public IP list known as of 2026-10."
    },
    {
      "id": "anthropic-claude-searchbot",
      "name": "Claude-SearchBot (Anthropic)",
      "operator": "Anthropic",
      "ua_patterns": ["Claude-SearchBot"],
      "signature_agent": null,
      "category": "search",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "note": "No public IP list known as of 2026-10."
    },
    {
      "id": "anthropic-claude-in-chrome",
      "name": "Claude in Chrome (Anthropic)",
      "operator": "Anthropic",
      "ua_patterns": [],
      "signature_agent": null,
      "category": "browser-agent",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": null,
      "note": "Runs inside the user's own Chrome; no UA marker and no signature known. Listed so the owner sees it in the policy table; matching relies on behavior signals and the purpose field."
    },
    {
      "id": "google-agent",
      "name": "Google-Agent / Mariner (Google)",
      "operator": "Google",
      "ua_patterns": ["GoogleAgent-Mariner", "Google-Agent"],
      "signature_agent": "https://agent.bot.goog",
      "category": "browser-agent",
      "default_policy": "queue",
      "verified_via": "signature",
      "ip_list_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
      "docs_url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
      "note": "Signature-Agent directory host agent.bot.goog per research 2026-09. UA tokens from Google's crawler list; also check user-triggered-fetchers.json for IP ranges."
    },
    {
      "id": "google-googlebot",
      "name": "Googlebot (Google)",
      "operator": "Google",
      "ua_patterns": ["Googlebot"],
      "signature_agent": null,
      "category": "search",
      "default_policy": "queue",
      "verified_via": "ip_list",
      "ip_list_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
      "docs_url": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot",
      "note": null
    },
    {
      "id": "google-extended",
      "name": "Google-Extended (Google)",
      "operator": "Google",
      "ua_patterns": ["Google-Extended"],
      "signature_agent": null,
      "category": "crawler",
      "default_policy": "queue",
      "verified_via": "ip_list",
      "ip_list_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
      "docs_url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
      "note": "robots.txt token only; Google does not send it as a user agent. Pattern kept for completeness."
    },
    {
      "id": "perplexity-bot",
      "name": "PerplexityBot (Perplexity)",
      "operator": "Perplexity",
      "ua_patterns": ["PerplexityBot"],
      "signature_agent": null,
      "category": "search",
      "default_policy": "queue",
      "verified_via": "ip_list",
      "ip_list_url": "https://www.perplexity.com/perplexitybot.json",
      "docs_url": "https://docs.perplexity.ai/guides/bots",
      "note": null
    },
    {
      "id": "perplexity-user",
      "name": "Perplexity-User (Perplexity)",
      "operator": "Perplexity",
      "ua_patterns": ["Perplexity-User"],
      "signature_agent": null,
      "category": "assistant",
      "default_policy": "queue",
      "verified_via": "ip_list",
      "ip_list_url": "https://www.perplexity.com/perplexity-user.json",
      "docs_url": "https://docs.perplexity.ai/guides/bots",
      "note": null
    },
    {
      "id": "perplexity-comet",
      "name": "Comet browser (Perplexity)",
      "operator": "Perplexity",
      "ua_patterns": [],
      "signature_agent": null,
      "category": "browser-agent",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": null,
      "note": "Agentic Chromium browser; sends a normal Chrome UA. No marker known as of 2026-10."
    },
    {
      "id": "meta-externalagent",
      "name": "Meta-ExternalAgent (Meta)",
      "operator": "Meta",
      "ua_patterns": ["meta-externalagent", "meta-externalfetcher", "FacebookBot"],
      "signature_agent": null,
      "category": "crawler",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "note": "Meta publishes IP ranges via whois on AS32934, not as a JSON file."
    },
    {
      "id": "meta-muse",
      "name": "Muse (Meta)",
      "operator": "Meta",
      "ua_patterns": [],
      "signature_agent": null,
      "category": "browser-agent",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": null,
      "note": "User agent unknown as of 2026-10; placeholder record so the policy table can list it. Add ua_patterns when Meta documents them."
    },
    {
      "id": "microsoft-bingbot",
      "name": "Bingbot (Microsoft)",
      "operator": "Microsoft",
      "ua_patterns": ["bingbot"],
      "signature_agent": null,
      "category": "search",
      "default_policy": "queue",
      "verified_via": "ip_list",
      "ip_list_url": "https://www.bing.com/toolbox/bingbot.json",
      "docs_url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "note": null
    },
    {
      "id": "microsoft-copilot",
      "name": "Copilot (Microsoft)",
      "operator": "Microsoft",
      "ua_patterns": ["Microsoft-Copilot", "MicrosoftPreview"],
      "signature_agent": null,
      "category": "assistant",
      "default_policy": "queue",
      "verified_via": "ip_list",
      "ip_list_url": "https://www.bing.com/toolbox/bingbot.json",
      "docs_url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "note": "Copilot mostly fetches through bingbot. The Microsoft-Copilot token is unconfirmed; MicrosoftPreview is Bing's link-preview fetcher."
    },
    {
      "id": "apple-applebot",
      "name": "Applebot (Apple)",
      "operator": "Apple",
      "ua_patterns": ["Applebot"],
      "signature_agent": null,
      "category": "search",
      "default_policy": "queue",
      "verified_via": "ip_list",
      "ip_list_url": "https://search.developer.apple.com/applebot.json",
      "docs_url": "https://support.apple.com/en-us/119829",
      "note": "Applebot-Extended is a robots.txt token only and is covered by this pattern."
    },
    {
      "id": "amazon-amazonbot",
      "name": "Amazonbot (Amazon)",
      "operator": "Amazon",
      "ua_patterns": ["Amazonbot"],
      "signature_agent": null,
      "category": "crawler",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": "https://developer.amazon.com/amazonbot",
      "note": "Amazon lists IP ranges on the docs page; no machine-readable URL confirmed."
    },
    {
      "id": "bytedance-bytespider",
      "name": "Bytespider (ByteDance)",
      "operator": "ByteDance",
      "ua_patterns": ["Bytespider", "TikTokSpider"],
      "signature_agent": null,
      "category": "crawler",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": null,
      "note": "Known to ignore robots.txt; no published IP list."
    },
    {
      "id": "commoncrawl-ccbot",
      "name": "CCBot (Common Crawl)",
      "operator": "Common Crawl",
      "ua_patterns": ["CCBot"],
      "signature_agent": null,
      "category": "crawler",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": "https://commoncrawl.org/ccbot",
      "note": null
    },
    {
      "id": "mistral-user",
      "name": "MistralAI-User (Mistral)",
      "operator": "Mistral AI",
      "ua_patterns": ["MistralAI-User"],
      "signature_agent": null,
      "category": "assistant",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": "https://docs.mistral.ai/robots/",
      "note": null
    },
    {
      "id": "duckduckgo-duckassistbot",
      "name": "DuckAssistBot (DuckDuckGo)",
      "operator": "DuckDuckGo",
      "ua_patterns": ["DuckAssistBot"],
      "signature_agent": null,
      "category": "assistant",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/",
      "note": null
    },
    {
      "id": "you-youbot",
      "name": "YouBot (You.com)",
      "operator": "You.com",
      "ua_patterns": ["YouBot"],
      "signature_agent": null,
      "category": "search",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": "https://about.you.com/youbot/",
      "note": null
    },
    {
      "id": "browserbase",
      "name": "Browserbase",
      "operator": "Browserbase",
      "ua_patterns": ["Browserbase"],
      "signature_agent": null,
      "category": "browser-agent",
      "default_policy": "queue",
      "verified_via": "signature",
      "ip_list_url": null,
      "docs_url": "https://docs.browserbase.com",
      "note": "Hosted headless Chrome for agents (Stagehand). Default UA is plain Chrome; the Browserbase token appears only when customers keep the identifying UA. Signs requests under Cloudflare's signed-agents program; Signature-Agent origin unconfirmed."
    },
    {
      "id": "anchor-browser",
      "name": "Anchor Browser",
      "operator": "Anchor",
      "ua_patterns": ["AnchorBrowser"],
      "signature_agent": null,
      "category": "browser-agent",
      "default_policy": "queue",
      "verified_via": "signature",
      "ip_list_url": null,
      "docs_url": "https://docs.anchorbrowser.io",
      "note": "Hosted browser for agents. UA token and Signature-Agent origin unconfirmed."
    },
    {
      "id": "manus",
      "name": "Manus",
      "operator": "Manus (Butterfly Effect)",
      "ua_patterns": ["Manus-Agent"],
      "signature_agent": null,
      "category": "browser-agent",
      "default_policy": "queue",
      "verified_via": "signature",
      "ip_list_url": null,
      "docs_url": "https://manus.im",
      "note": "General-purpose agent that browses in a cloud sandbox. UA token and Signature-Agent origin unconfirmed; pattern is a guess kept narrow to avoid false positives."
    },
    {
      "id": "headless-browser",
      "name": "Headless browser",
      "operator": null,
      "ua_patterns": ["HeadlessChrome", "PhantomJS", "Puppeteer", "Playwright", "Selenium", "SlimerJS", "CasperJS", "Nightmare", "HtmlUnit", "Chrome-Lighthouse"],
      "signature_agent": null,
      "category": "headless",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": null,
      "note": "Marker list. Puppeteer and Playwright normally hide behind a plain Chrome UA; these tokens catch only careless setups. Class is bot, so the block threshold decides."
    },
    {
      "id": "scripted-http-client",
      "name": "Scripted HTTP client",
      "operator": null,
      "ua_patterns": ["python-requests", "python-urllib", "python-httpx", "aiohttp", "curl/", "Wget/", "Go-http-client", "node-fetch", "undici", "axios/", "okhttp", "Java/", "Apache-HttpClient", "libwww-perl", "Scrapy", "Guzzle", "PostmanRuntime", "HTTPie"],
      "signature_agent": null,
      "category": "http-client",
      "default_policy": "queue",
      "verified_via": "none",
      "ip_list_url": null,
      "docs_url": null,
      "note": "Library default user agents. A form POST from any of these is not a person in a browser. Class is bot, so the block threshold decides."
    }
  ]
}
