Module: BotDetection

Included in:
ActionController::Base
Defined in:
lib/bot_detection.rb,
lib/bot_detection/version.rb

Defined Under Namespace

Modules: InstanceMethods

Constant Summary collapse

VERSION = 
"1.0.6"
BotUserAgents = 
[
  "FacebookExternalHit/1.1",
  "FacebookExternalHit/1.0",
  "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)",
  "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
  "facebookplatform/1.0 (+http://developers.facebook.com)",
  "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)",
  "Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)",
  "ShowyouBot (http://showyou.com/crawler)",
  "Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)",
  "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)",
  "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11",
  "ia_archiver (+http://www.alexa.com/site/help/webmasters; [email protected])",
  "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)",
  "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)",
  "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)",
  "Twitterbot/1.0",
  "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)",
  "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)",
  "Mozilla/5.0 (compatible; SearchmetricsBot; http://www.searchmetrics.com/en/searchmetrics-bot/)",
  "Pinterest/0.1 +http://pinterest.com/",
  "Googlebot (gocrawl v0.4)",
  "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; [email protected])",
  "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)", # perhaps a search engine?
  "ContextAd Bot 1.0",
  "WeSEE:Ads/PageBot (http://www.wesee.com/bot/)",
  "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)",
  "Mozilla/5.0 (compatible; grapeFX/0.9; [email protected]",
  "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)",
  "CodeGator Crawler v1.0",
  "German Wikipedia Broken Weblinks Bot; contact: [email protected]",
  "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)",
  "Mozilla/5.0 (compatible; TwitterCrawler)",
  "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, [email protected])",
  "crawler4j (http://code.google.com/p/crawler4j/)",
  "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)",
  "LivelapBot/0.2 (http://site.livelap.com/crawler)",
  "Clicapack Bot",
  "CRAZYWEBCRAWLER 0.9.1, http://www.crazywebcrawler.com",
  "Mozilla/4.0 (compatible; MSIE 5.01; Windows 95; MSIECrawler)",
  "bitlybot",
  "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0",
  "Mozilla/5.0 (Linux; U; Android 4.2.2; bg-bg; CUBOT X6 Build/JDQ39) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
  "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
  "AppEngine-Google; (+http://code.google.com/appengine; appid: asburyisgay)",
  "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)",
  "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, [email protected])",
]
GOOGLE_USER_AGENTS = 
[
  "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
  "Googlebot/2.1 (+http://www.googlebot.com/bot.html)",
  "Googlebot/2.1 (+http://www.google.com/bot.html)",
  "Googlebot-News",
  "Googlebot-Image/1.0",
  "Googlebot-Video/1.0",
  "(compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
  "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
  "(compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)",
  "Mediapartners-Google",
  "AdsBot-Google (+http://www.google.com/adsbot.html)",
  "Googlebot",
  "Googlebot-News (Googlebot)",
  "Googlebot-Image (Googlebot)",
  "Googlebot-Video (Googlebot)",
  "Googlebot-Mobile",
  "Mediapartners-Google",
  "Mediapartners (Googlebot)",
  "Mediapartners-Google Mediapartners (Googlebot)",
  "AdsBot-Google"
]
SearchEngineUserAgents = 
[
  "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
  "Baiduspider+(+http://www.baidu.com/search/spider.htm)",
  "Baiduspider+(+http://www.baidu.com/search/spider_jp.html)",
  "BaiDuSpider",
  "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
  "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexPagechecker/1.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexDirect/2.0; Dyatel; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexNews/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexCatalog/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexZakladki/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexMarket/1.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
  "Mozilla/5.0 (compatible; Yahoo Slurp;   http://help.yahoo.com/help/us/ysearch/slurp)",
  "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)",
  "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
  "Mozilla/5.0 (compatible; bingbot/2.0 +http://www.bing.com/bingbot.htm)",
  "msnbot/2.1",
  "msnbot/2.0b",
  "msnbot/1.1 (+http://search.msn.com/msnbot.htm)",
  "msnbot/1.1",
  "msnbot/1.0 (+http://search.msn.com/msnbot.htm)",
  "msnbot/0.9 (+http://search.msn.com/msnbot.htm)",
  "msnbot/0.11 ( http://search.msn.com/msnbot.htm)",
  "MSNBOT/0.1 (http://search.msn.com/msnbot.htm)",
]

Class Method Summary collapse

Class Method Details

.included(target) ⇒ Object 



5
6
7
 
# File 'lib/bot_detection.rb', line 5

def self.included(target)
  target.send(:include, BotDetection::InstanceMethods)
end