{
    "schema_version": 1,
    "updated_at": "2026-09-04",
    "crawlers": [
        {
            "id": "gptbot",
            "vendor": "OpenAI",
            "ua_pattern": "GPTBot",
            "category": "training",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://openai.com/gptbot.json"
            },
            "docs_url": "https://developers.openai.com/api/docs/bots"
        },
        {
            "id": "oai-searchbot",
            "vendor": "OpenAI",
            "ua_pattern": "OAI-SearchBot",
            "category": "search_index",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://openai.com/searchbot.json"
            },
            "docs_url": "https://developers.openai.com/api/docs/bots"
        },
        {
            "id": "chatgpt-user",
            "vendor": "OpenAI",
            "ua_pattern": "ChatGPT-User",
            "category": "user_fetch",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://openai.com/chatgpt-user.json"
            },
            "docs_url": "https://developers.openai.com/api/docs/bots"
        },
        {
            "id": "claudebot",
            "vendor": "Anthropic",
            "ua_pattern": "ClaudeBot",
            "category": "training",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://claude.com/crawling/bots.json"
            },
            "docs_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
            "note": "Anthropicは2026-08-18に方針を変更し、3クローラー共通のIPレンジを公開した(bot別の分割はないため、検証は「Anthropic由来である」ことまでを保証する)。rDNSドメインは非公表"
        },
        {
            "id": "claude-searchbot",
            "vendor": "Anthropic",
            "ua_pattern": "Claude-SearchBot",
            "category": "search_index",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://claude.com/crawling/bots.json"
            },
            "docs_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
            "note": "Anthropicは2026-08-18に方針を変更し、3クローラー共通のIPレンジを公開した(bot別の分割はないため、検証は「Anthropic由来である」ことまでを保証する)。rDNSドメインは非公表"
        },
        {
            "id": "claude-user",
            "vendor": "Anthropic",
            "ua_pattern": "Claude-User",
            "category": "user_fetch",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://claude.com/crawling/bots.json"
            },
            "docs_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
            "note": "Anthropicは2026-08-18に方針を変更し、3クローラー共通のIPレンジを公開した(bot別の分割はないため、検証は「Anthropic由来である」ことまでを保証する)。rDNSドメインは非公表"
        },
        {
            "id": "perplexitybot",
            "vendor": "Perplexity",
            "ua_pattern": "PerplexityBot",
            "category": "search_index",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://www.perplexity.com/perplexitybot.json"
            },
            "docs_url": "https://docs.perplexity.ai/guides/bots"
        },
        {
            "id": "perplexity-user",
            "vendor": "Perplexity",
            "ua_pattern": "Perplexity-User",
            "category": "user_fetch",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://www.perplexity.com/perplexity-user.json"
            },
            "docs_url": "https://docs.perplexity.ai/guides/bots",
            "note": "ユーザー起点の取得のためrobots.txtを原則無視すると公式が明言"
        },
        {
            "id": "googleother",
            "vendor": "Google",
            "ua_pattern": "GoogleOther",
            "category": "training",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
                "domains": ["googlebot.com", "google.com"]
            },
            "docs_url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
            "note": "Google社の研究開発用途を含む汎用クローラー。AI学習用途を含むが特定は不可。GoogleOther-Image / GoogleOther-Video も本パターンで一致する"
        },
        {
            "id": "applebot",
            "vendor": "Apple",
            "ua_pattern": "Applebot",
            "category": "search_index",
            "verify": {
                "method": "rdns",
                "domains": ["applebot.apple.com"]
            },
            "docs_url": "https://support.apple.com/en-us/119829",
            "note": "Siri・Spotlight等の検索向けクローラー。取得済みコンテンツのAI学習利用の可否は robots.txt の Applebot-Extended トークンで制御される"
        },
        {
            "id": "meta-externalagent",
            "vendor": "Meta",
            "ua_pattern": "meta-externalagent",
            "category": "training",
            "verify": {
                "method": "none"
            },
            "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
            "note": "Metaは検証用のrDNS・IPレンジを公表していない"
        },
        {
            "id": "meta-webindexer",
            "vendor": "Meta",
            "ua_pattern": "meta-webindexer",
            "category": "search_index",
            "verify": {
                "method": "none"
            },
            "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
            "note": "Meta AI検索の結果品質向上用クローラー(2026年時点の公式ドキュメントで確認)"
        },
        {
            "id": "meta-externalfetcher",
            "vendor": "Meta",
            "ua_pattern": "meta-externalfetcher",
            "category": "user_fetch",
            "verify": {
                "method": "none"
            },
            "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
            "note": "エージェント的なタスク実行を含むユーザー起点フェッチ"
        },
        {
            "id": "amazonbot",
            "vendor": "Amazon",
            "ua_pattern": "Amazonbot",
            "category": "training",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://developer.amazon.com/amazonbot/ip-addresses/"
            },
            "docs_url": "https://developer.amazon.com/amazonbot",
            "note": "公式が「Amazon AIモデルの学習に使われることがある」と明記(検索専用は Amzn-SearchBot)。IPリストのフォーマットはJSONでない可能性があり実装時に要確認"
        },
        {
            "id": "amzn-searchbot",
            "vendor": "Amazon",
            "ua_pattern": "Amzn-SearchBot",
            "category": "search_index",
            "verify": {
                "method": "none"
            },
            "docs_url": "https://developer.amazon.com/amazonbot",
            "note": "検索インデックス専用と公式が明記(生成AI学習には不使用)。2026年時点の新クローラーで検証手段は実装時に要確認"
        },
        {
            "id": "amzn-user",
            "vendor": "Amazon",
            "ua_pattern": "Amzn-User",
            "category": "user_fetch",
            "verify": {
                "method": "none"
            },
            "docs_url": "https://developer.amazon.com/amazonbot",
            "note": "ユーザー起点のリアルタイム取得。2026年時点の新クローラーで検証手段は実装時に要確認"
        },
        {
            "id": "ccbot",
            "vendor": "Common Crawl",
            "ua_pattern": "CCBot",
            "category": "training",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://index.commoncrawl.org/ccbot.json",
                "domains": [
                    "crawl.commoncrawl.org"
                ]
            },
            "docs_url": "https://commoncrawl.org/ccbot",
            "note": "非営利のWebアーカイブ。多くのAI企業が学習データとして利用。2026-08-11にIPレンジを公開(公式はFCrDNSの併用も推奨)"
        },
        {
            "id": "bytespider",
            "vendor": "ByteDance",
            "ua_pattern": "Bytespider",
            "category": "training",
            "verify": {
                "method": "none"
            },
            "docs_url": "",
            "note": "公式ドキュメントが存在せず、robots.txt遵守も不確実との報告が多い。検証手段なし"
        },
        {
            "id": "duckassistbot",
            "vendor": "DuckDuckGo",
            "ua_pattern": "DuckAssistBot",
            "category": "user_fetch",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://duckduckgo.com/duckassistbot.json"
            },
            "docs_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/",
            "note": "AI回答(DuckAssist)用のリアルタイム取得。学習には不使用と公式が明言"
        },
        {
            "id": "mistralai-user",
            "vendor": "Mistral",
            "ua_pattern": "MistralAI-User",
            "category": "user_fetch",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://mistral.ai/mistralai-user-ips.json"
            },
            "docs_url": "https://docs.mistral.ai/robots"
        },
        {
            "id": "mistralai-index",
            "vendor": "Mistral",
            "ua_pattern": "MistralAI-Index",
            "category": "search_index",
            "verify": {
                "method": "ip_list",
                "ip_list_url": "https://mistral.ai/mistralai-index-ips.json"
            },
            "docs_url": "https://docs.mistral.ai/robots",
            "note": "Mistral検索用インデックスクローラー(学習には不使用と公式が明言)"
        }
    ],
    "robots_txt_tokens": [
        {
            "id": "google-extended",
            "vendor": "Google",
            "token": "Google-Extended",
            "purpose": "training",
            "docs_url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
            "note": "UAとしては現れないrobots.txt制御トークン。Gemini等のAI学習利用可否を制御する"
        },
        {
            "id": "applebot-extended",
            "vendor": "Apple",
            "token": "Applebot-Extended",
            "purpose": "training",
            "docs_url": "https://support.apple.com/en-us/119829",
            "note": "UAとしては現れないrobots.txt制御トークン。Apple Intelligence等の学習利用可否を制御する"
        }
    ]
}