##\
# This file is a part of the CIDRAM package.
# Homepage: https://cidram.github.io/
#
# CIDRAM COPYRIGHT 2016 and beyond by Caleb Mazalevskis (Maikuolan).
#
# License: GNU/GPLv2
# @see LICENSE.txt
#
# This file: Verification data for search engines and social media (last modified: 2021.10.01).
##/

# Additional references: https://udger.com/resources/ua-list/crawlers
Search Engine Verification:
 # https://discussions.apple.com/thread/7090135
 # https://support.apple.com/en-gb/HT204683
 Applebot:
  User Agent: "applebot"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".applebot.apple.com"

 # https://help.baidu.com/question?prod_en=master&class=Baiduspider
 # https://help.baidu.com/question?prod_id=99&class=476&id=3001
 Baiduspider/百度:
  Bypass flag: "Flag-Bypass-Baidu-Check"
  User Agent: "baiduspider"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".baidu.com"
   - ".baidu.jp"

 # https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot
 # https://github.com/CIDRAM/CIDRAM/issues/242
 Bingbot:
  Bypass flag: "Flag-Bypass-Bingbot-Check"
  User Agent Pattern: "~(?:msn|bing)bot|bingpreview~"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".search.msn.com"
  Single hit bypass: true

 # https://duckduckgo.com/duckduckbot
 DuckDuckBot:
  User Agent Pattern: "~duckduck(?:go-favicons-)?bot~"
  Closure: "UA-IP-Match"
  Valid domains:
   - "20.191.45.212"
   - "23.21.227.69"
   - "40.88.21.235"
   - "50.16.241.113"
   - "50.16.241.114"
   - "50.16.241.117"
   - "50.16.247.234"
   - "52.5.190.19"
   - "52.204.97.54"
   - "54.197.234.188"
   - "54.208.100.253"
   - "54.208.102.37"
   - "107.21.1.8"
  Single hit bypass: true

 # https://support.google.com/webmasters/answer/80553?hl=en
 Googlebot:
  Bypass flag: "Flag-Bypass-Googlebot-Check"
  User Agent: "googlebot"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".google.com"
   - ".googlebot.com"

 # https://www.mojeek.com/bot.html
 MojeekBot:
  User Agent: "mojeekbot"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".mojeek.com"

 # https://aspiegel.com/petalbot
 # https://github.com/CIDRAM/CIDRAM/issues/254
 # https://webmaster.petalsearch.com/site/petalbot
 PetalBot:
  Bypass flag: "Flag-Bypass-PetalBot-Check"
  User Agent: "petalbot"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".aspiegel.com"
   - ".petalsearch.com"

 # https://help.qwant.com/bot/
 Qwantify/Bleriot:
  User Agent: "qwantify/bleriot"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - "search.qwant.com"

 # https://napoveda.seznam.cz/en/full-text-search/seznambot-crawler/
 SeznamBot:
  User Agent: "seznambot"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".seznam.cz"

 # https://www.sogou.com/docs/help/webmasters.htm#07
 Sogou/搜狗:
  Bypass flag: "Flag-Bypass-Sogou-Check"
  User Agent: "sogou"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".sogou.com"

 # https://help.yahoo.com/help/us/ysearch/slurp
 Yahoo/Slurp:
  User Agent: "slurp"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".crawl.yahoo.net"
   - ".yse.yahoo.net"

 # https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml
 Yandex/Яндекс:
  Bypass flag: "Flag-Bypass-Yandex-Check"
  User Agent: "yandex"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".yandex.com"
   - ".yandex.net"
   - ".yandex.ru"

 # https://udger.com/resources/ua-list/bot-detail?bot=YoudaoBot#id1507
 YoudaoBot:
  User Agent: "youdaobot"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".crawler.youdao.com"

Social Media Verification:
 # https://udger.com/resources/ua-list/bot-detail?bot=Embedly#id22674
 Embedly:
  User Agent: "embedly"
  Closure: "DNS-Reverse-Forward"
  Reverse only: true
  Can modify trackable: false
  Valid domains:
   - "embed.ly"

 # https://developers.facebook.com/docs/sharing/webmasters/crawler/
 # https://github.com/CIDRAM/CIDRAM/issues/170
 Facebook external hit:
  User Agent: "facebookexternalhit/"
  Closure: "UA-ASN-Match"
  Valid domains:
   - "32934"

 # https://github.com/CIDRAM/CIDRAM/issues/253
 # https://help.pinterest.com/en/articles/about-pinterest-crawler-0
 Pinterest:
  User Agent: "pinterest"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".pinterest.com"
   - ".pinterestcrawler.com"

 # https://udger.com/resources/ua-list/bot-detail?bot=Twitterbot#id6168
 Twitterbot:
  User Agent: "twitterbot"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".twttr.com"

Other Verification:
 # https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers
 # https://github.com/CIDRAM/CIDRAM/issues/233
 AdSense:
  User Agent: "mediapartners-google"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".google.com"
   - ".googlebot.com"

 # https://adbot.amazon.com/index.html
 # https://github.com/CIDRAM/CIDRAM/issues/260
 AmazonAdBot:
  Bypass flag: "Flag-Bypass-AmazonAdBot-Check"
  User Agent Pattern: "~amazonadbot|adbot\.amazon\.com~"
  Closure: "DNS-Reverse-Forward"
  Valid domains:
   - ".amazonadbot.com"
  Single hit bypass: true

 # https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html
 Oracle Data Cloud Crawler:
  User Agent: "grapeshot"
  Closure: "UA-CIDR-Match"
  Valid domains:
   - "132.145.9.5/32"
   - "132.145.11.125/32"
   - "132.145.14.70/32"
   - "132.145.15.209/32"
   - "132.145.64.33/32"
   - "132.145.66.116/32"
   - "132.145.66.156/32"
   - "132.145.67.248/32"
   - "140.238.81.78/32"
   - "140.238.83.181/32"
   - "140.238.94.137/32"
   - "140.238.95.47/32"
   - "140.238.95.199/32"
   - "148.64.56.64/28"
   - "148.64.56.112/28"
   - "152.67.128.219/32"
   - "152.67.137.35/32"
   - "152.67.138.180/32"