{
	"schema": 1,
	"version": "2026-08-16",
	"notes": {
		"match": "How the agent can be detected. 'ua' = a real crawler that sends this token in its User-Agent header, so it can be matched by .htaccess and PHP. 'robots' = a robots.txt-only control token that is NEVER sent as a User-Agent; it can only be honoured in robots.txt. Emitting a 'robots' token into a UA-matching layer produces a rule that can never fire.",
		"search_impact": "What the site loses if this agent is blocked. 'none' = no search visibility effect. 'ai-search' = removed from that vendor's AI search/answer surface. 'web-search' = affects classic web search indexing. Anything other than 'none' requires explicit user confirmation before it can be set to block.",
		"legacy": "Token no longer documented by its vendor. Kept because it is widely present in existing robots.txt files and blocking it is harmless, but it should not be presented as currently enforced by the vendor."
	},
	"groups": [
		{
			"id": "openai",
			"label": "OpenAI",
			"docs": "https://developers.openai.com/api/docs/bots",
			"agents": [
				{
					"ua": "GPTBot",
					"purpose": "training",
					"match": "ua",
					"search_impact": "none",
					"description": "Crawls content that may be used to train OpenAI's generative AI foundation models."
				},
				{
					"ua": "OAI-SearchBot",
					"purpose": "search",
					"match": "ua",
					"search_impact": "ai-search",
					"description": "Surfaces sites in ChatGPT's search features. Blocking it removes the site from ChatGPT search answers."
				},
				{
					"ua": "ChatGPT-User",
					"purpose": "user-fetch",
					"match": "ua",
					"search_impact": "none",
					"description": "Fetches a page because a ChatGPT user asked for it. Not training. Blocking it means people cannot ask ChatGPT to read your page."
				}
			]
		},
		{
			"id": "anthropic",
			"label": "Anthropic",
			"docs": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
			"agents": [
				{
					"ua": "ClaudeBot",
					"purpose": "training",
					"match": "ua",
					"search_impact": "none",
					"description": "Collects web content that may contribute to training Anthropic's models."
				},
				{
					"ua": "Claude-User",
					"purpose": "user-fetch",
					"match": "ua",
					"search_impact": "none",
					"description": "Fetches a page because a Claude user asked a question about it. Not training."
				},
				{
					"ua": "Claude-SearchBot",
					"purpose": "search",
					"match": "ua",
					"search_impact": "ai-search",
					"description": "Improves search result quality for Claude users. Blocking it affects how the site appears in Claude's search responses."
				},
				{
					"ua": "anthropic-ai",
					"purpose": "training",
					"match": "ua",
					"search_impact": "none",
					"legacy": true,
					"description": "Historical token, no longer listed in Anthropic's current documentation. Harmless to keep; do not present it as actively honoured."
				}
			]
		},
		{
			"id": "perplexity",
			"label": "Perplexity",
			"agents": [
				{
					"ua": "PerplexityBot",
					"purpose": "search",
					"match": "ua",
					"search_impact": "ai-search",
					"description": "Indexes pages for Perplexity's answer engine."
				},
				{
					"ua": "Perplexity-User",
					"purpose": "user-fetch",
					"match": "ua",
					"search_impact": "none",
					"description": "Fetches a page at a Perplexity user's direction."
				}
			]
		},
		{
			"id": "google",
			"label": "Google",
			"docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
			"agents": [
				{
					"ua": "Google-Extended",
					"purpose": "training",
					"match": "robots",
					"search_impact": "none",
					"description": "Controls whether content trains Gemini. Google states it 'does not impact a site's inclusion in Google Search nor is it used as a ranking signal'. It has no user-agent string of its own, so it can only be honoured in robots.txt."
				},
				{
					"ua": "GoogleOther",
					"purpose": "dataset",
					"match": "ua",
					"search_impact": "none",
					"description": "Generic crawler used by Google product teams for one-off fetches and internal R&D. Distinct from Googlebot, which this plugin never blocks."
				}
			]
		},
		{
			"id": "meta",
			"label": "Meta",
			"agents": [
				{
					"ua": "meta-externalagent",
					"purpose": "training",
					"match": "ua",
					"search_impact": "none",
					"description": "Crawls content used to train Meta's AI models."
				},
				{
					"ua": "FacebookBot",
					"purpose": "training",
					"match": "ua",
					"search_impact": "none",
					"description": "Meta crawler used for speech and language model training. Not the link-preview scraper."
				}
			]
		},
		{
			"id": "bytedance",
			"label": "ByteDance",
			"agents": [
				{
					"ua": "Bytespider",
					"purpose": "training",
					"match": "ua",
					"search_impact": "none",
					"description": "ByteDance crawler widely reported to collect training data."
				},
				{
					"ua": "TikTokSpider",
					"purpose": "training",
					"match": "ua",
					"search_impact": "none",
					"description": "ByteDance/TikTok crawler."
				}
			]
		},
		{
			"id": "amazon",
			"label": "Amazon",
			"agents": [
				{
					"ua": "Amazonbot",
					"purpose": "training",
					"match": "ua",
					"search_impact": "none",
					"description": "Amazon crawler used to improve Alexa and Amazon services."
				}
			]
		},
		{
			"id": "apple",
			"label": "Apple",
			"docs": "https://support.apple.com/en-us/119829",
			"agents": [
				{
					"ua": "Applebot-Extended",
					"purpose": "training",
					"match": "robots",
					"search_impact": "none",
					"description": "Opts content out of training Apple's foundation models. Apple states it 'does not crawl webpages' and that pages disallowing it 'can still be included in search results'. It has no user-agent string of its own, so it can only be honoured in robots.txt."
				}
			]
		},
		{
			"id": "datasets",
			"label": "Datasets and other",
			"agents": [
				{
					"ua": "CCBot",
					"purpose": "dataset",
					"match": "ua",
					"search_impact": "none",
					"description": "Common Crawl. Its archives are a primary training corpus for many AI models."
				},
				{
					"ua": "Diffbot",
					"purpose": "dataset",
					"match": "ua",
					"search_impact": "none",
					"description": "Commercial structured-data extraction crawler."
				},
				{
					"ua": "ImagesiftBot",
					"purpose": "dataset",
					"match": "ua",
					"search_impact": "none",
					"description": "Crawls images at scale. Relevant to photographers and illustrators."
				},
				{
					"ua": "Omgilibot",
					"purpose": "dataset",
					"match": "ua",
					"search_impact": "none",
					"description": "Web data reseller crawler."
				},
				{
					"ua": "Timpibot",
					"purpose": "dataset",
					"match": "ua",
					"search_impact": "none",
					"description": "Crawler for the Timpi decentralised index."
				},
				{
					"ua": "cohere-ai",
					"purpose": "training",
					"match": "ua",
					"search_impact": "none",
					"description": "Cohere crawler."
				},
				{
					"ua": "YouBot",
					"purpose": "search",
					"match": "ua",
					"search_impact": "ai-search",
					"description": "Crawler for the You.com answer engine."
				},
				{
					"ua": "AI2Bot",
					"purpose": "dataset",
					"match": "ua",
					"search_impact": "none",
					"description": "Allen Institute for AI crawler, used to build open training corpora."
				},
				{
					"ua": "Scrapy",
					"purpose": "scraping-tool",
					"match": "ua",
					"search_impact": "none",
					"description": "Default user agent of the Scrapy scraping framework. Identifies unconfigured scrapers only; anyone who changes the default will not be caught."
				}
			]
		}
	],
	"protected": {
		"notes": "User-agent fragments that must NEVER be blocked by any layer this plugin writes. Every generated rule is tested against this list and the write is refused if one matches. Getting a site de-indexed is this plugin's catastrophic failure mode.",
		"agents": [
			"Googlebot",
			"Googlebot-Image",
			"Googlebot-News",
			"Googlebot-Video",
			"Storebot-Google",
			"Google-InspectionTool",
			"AdsBot-Google",
			"Mediapartners-Google",
			"Bingbot",
			"BingPreview",
			"Slurp",
			"DuckDuckBot",
			"Baiduspider",
			"YandexBot",
			"Applebot",
			"facebookexternalhit",
			"Twitterbot",
			"LinkedInBot"
		]
	}
}