Module: BotVerification::BotPatterns

Included in:: Service

Defined in:: lib/bot_verification/bot_patterns.rb

Constant Summary collapse

SEARCH_BOT_PATTERNS = User agent patterns for search engine bots

{
  google: /Googlebot|Google-Extended|Mediapartners-Google|AdsBot-Google|APIs-Google/i,
  bing: /Bingbot|msnbot|BingPreview/i,
  apple: /Applebot/i,
  yandex: /YandexBot/i,
  baidu: /Baiduspider/i
}.freeze

SEARCH_BOT_DNS_SUFFIXES = Valid reverse DNS suffixes for search engine bots

{
  google: %w[.googlebot.com .google.com .googleusercontent.com],
  bing: %w[.search.msn.com],
  apple: %w[.applebot.apple.com],
  yandex: %w[.yandex.ru .yandex.net .yandex.com],
  baidu: %w[.crawl.baidu.com .crawl.baidu.jp]
}.freeze

SEARCH_BOTS_WITH_IP_RANGES = Search engine bots with IP ranges in database

i[google bing].freeze

SEARCH_BOTS_WITH_DNS = Search engine bots that support reverse DNS verification

i[google bing apple yandex baidu].freeze

AI_BOT_PATTERNS = User agent patterns for AI bots

{
  openai_gptbot: /GPTBot/i,
  openai_chatgpt: /ChatGPT-User/i,
  openai_searchbot: /OAI-SearchBot/i,
  anthropic: /ClaudeBot|Claude-Web|anthropic-ai/i,
  perplexity: /PerplexityBot|Perplexity-User/i,
  amazon: /Amazonbot/i,
  cohere: /cohere-ai/i,
  meta: /meta-externalagent/i,
  bytedance: /Bytespider/i
}.freeze

AI_BOTS_WITH_IP_RANGES = AI bots with officially published IP ranges (can be verified)

i[openai_gptbot openai_chatgpt openai_searchbot perplexity amazon].freeze

AI_BOTS_WITHOUT_VERIFICATION = AI bots without official IP ranges (cannot be reliably verified)

i[anthropic cohere meta bytedance].freeze

SOCIAL_BOT_PATTERNS = ============================================================================= SOCIAL/OTHER BOTS (no verification available)

{
  duckduckgo: /DuckDuckBot/i,
  facebook: /facebookexternalhit|Facebot/i,
  twitter: /Twitterbot/i,
  linkedin: /LinkedInBot/i,
  slack: /Slackbot/i,
  discord: /Discordbot/i,
  telegram: /TelegramBot/i
}.freeze

ALL_BOT_PATTERNS = ============================================================================= COMBINED

SEARCH_BOT_PATTERNS.merge(AI_BOT_PATTERNS).merge(SOCIAL_BOT_PATTERNS).freeze

ALL_BOTS_WITH_IP_RANGES =

(SEARCH_BOTS_WITH_IP_RANGES + AI_BOTS_WITH_IP_RANGES).freeze

SEARCH_BOT_TYPES = Search engine bot types (as strings for database)

%w[google bing].freeze

AI_BOT_TYPES = AI bot types with IP ranges (as strings for database)

%w[openai_gptbot openai_chatgpt openai_searchbot perplexity amazon].freeze

VALID_BOT_TYPES = All bot types that have IP ranges

(SEARCH_BOT_TYPES + AI_BOT_TYPES).freeze

SEARCH_ENGINE_SOURCES = ============================================================================= IP RANGE SOURCES

{
  google: [
    { url: "https://developers.google.com/static/search/apis/ipranges/googlebot.json", name: "Googlebot" },
    { url: "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", name: "Google Special" }
  ],
  bing: [
    { url: "https://www.bing.com/toolbox/bingbot.json", name: "Bingbot" }
  ]
}.freeze

AI_BOT_SOURCES =

{
  openai_gptbot: [
    { url: "https://openai.com/gptbot.json", name: "GPTBot" }
  ],
  openai_chatgpt: [
    { url: "https://openai.com/chatgpt-user.json", name: "ChatGPT-User" }
  ],
  openai_searchbot: [
    { url: "https://openai.com/searchbot.json", name: "OAI-SearchBot" }
  ],
  perplexity: [
    { url: "https://www.perplexity.ai/perplexitybot.json", name: "PerplexityBot" },
    { url: "https://www.perplexity.ai/perplexity-user.json", name: "Perplexity-User" }
  ],
  amazon: [
    { url: "https://developer.amazon.com/amazonbot/ip-addresses/", name: "Amazonbot" }
  ]
}.freeze

IP_RANGE_SOURCES =

SEARCH_ENGINE_SOURCES.merge(AI_BOT_SOURCES).freeze