Module: BotDetection

Included in:: ActionController::Base

Defined in:: lib/bot_detection.rb,
lib/bot_detection/version.rb

Defined Under Namespace

Constant Summary collapse

VERSION =

"1.0.9"

BotUserAgents =

[
  "FacebookExternalHit/1.1",
  "FacebookExternalHit/1.0",
  "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)",
  "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
  "facebookplatform/1.0 (+http://developers.facebook.com)",
  "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)",
  "Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)",
  "ShowyouBot (http://showyou.com/crawler)",
  "Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)",
  "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)",
  "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11",
  "ia_archiver (+http://www.alexa.com/site/help/webmasters; [email protected])",
  "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)",
  "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)",
  "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)",
  "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)",
  "Twitterbot/1.0",
  "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)",
  "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)",
  "Mozilla/5.0 (compatible; SearchmetricsBot; http://www.searchmetrics.com/en/searchmetrics-bot/)",
  "Pinterest/0.1 +http://pinterest.com/",
  "Googlebot (gocrawl v0.4)",
  "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; [email protected])",
  "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)", # perhaps a search engine?
  "ContextAd Bot 1.0",
  "WeSEE:Ads/PageBot (http://www.wesee.com/bot/)",
  "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)",
  "Mozilla/5.0 (compatible; grapeFX/0.9; [email protected]",
  "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)",
  "CodeGator Crawler v1.0",
  "German Wikipedia Broken Weblinks Bot; contact: [email protected]",
  "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)",
  "Mozilla/5.0 (compatible; TwitterCrawler)",
  "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, [email protected])",
  "crawler4j (http://code.google.com/p/crawler4j/)",
  "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)",
  "LivelapBot/0.2 (http://site.livelap.com/crawler)",
  "Clicapack Bot",
  "CRAZYWEBCRAWLER 0.9.1, http://www.crazywebcrawler.com",
  "Mozilla/4.0 (compatible; MSIE 5.01; Windows 95; MSIECrawler)",
  "bitlybot",
  "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0",
  "Mozilla/5.0 (Linux; U; Android 4.2.2; bg-bg; CUBOT X6 Build/JDQ39) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
  "Mozilla/5.0 (Linux; U; Android 4.2.2; de-de; CUBOT S108 Build/CUBOT_S108) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
  "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
  "AppEngine-Google; (+http://code.google.com/appengine; appid: asburyisgay)",
  "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)",
  "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, [email protected])",
  "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
  "MaxPointCrawler/Nutch-1.1 (maxpoint.crawler at maxpointinteractive dot com)",
  "woobot/2.0",
  "techmixx Spider v1 (http://techmixx.com/)",
  "WordPress/4.0.1; http://novarabota.org",
  "yacybot (/global; amd64 Windows 7 6.1; java 1.8.0_25; Europe/en) http://yacy.net/bot.html",
  "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Fast/2.0; +http://go.mail.ru/help/robots)",
  "QuerySeekerSpider ( http://queryseeker.com/bot.html )",
  "Mozilla/5.0 (compatible; LinkpadBot/1.06; +http://www.linkpad.ru)",
  "Mozilla/5.0 (compatible; MixrankBot; [email protected])",
  "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, [email protected])",
  "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)",
  "Superarama.com - BOT/v.0.1",
  "Mozilla/5.0 (compatible; memoryBot/1.21.14 +http://mignify.com/bot.html)",
  "Elmer, the Thinglink ImageBot (http://www.thinglink.com/help/ThinglinkImageBot)",
  "Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)",
  "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)",
  "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; CrystalSemanticsBot http://www.crystalsemantics.com/service-navigation/imprint/useragent/)",
  "Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)",
  "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)",
  "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",
  "Wotbox/2.01 (+http://www.wotbox.com/bot/)",
  "Wotbox/2.0 ([email protected]; http://www.wotbox.com)",
  "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; [email protected] http://everyonesocial.com/)",
  "Jabse.com/2.0 (+http://www.jabse.com/bot.php)",
  "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)",
  "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)",
  "CCBot/2.0 (http://commoncrawl.org/faq/)",
  "Mozilla/5.0 (compatible; Kraken/0.1; http://linkfluence.net/; [email protected])",
  "Mozilla/5.0 (compatible; uMBot-FC/1.0; mailto: [email protected])",
  "Mozilla/5.0 (compatible; memoryBot/1.20.311 +http://internetmemory.org/en/)",
  "Mozilla/5.0 (compatible; uMBot-LN/1.0; mailto: [email protected])",
  "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)",
  "German Wikipedia Broken Weblinks Bot; contact: [email protected]",
  "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)",
  "Mozilla/5.0 (compatible; TwitterCrawler)",
  "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, [email protected])",
  "crawler4j (http://code.google.com/p/crawler4j/)",
  "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)",
  "Mozilla/5.0 (compatible; Optimizer)",
  "Disqus/1.0",
  "facebookexternalhit/1.1",
  "Nutch/2.2.1 (page scorer; http://integralads.com/site-indexing-policy/)",
  "R6_CommentReader(www.radian6.com/crawler)",
  "R6_FeedFetcher(www.radian6.com/crawler)",
  "tbot-nutch/Nutch-1.10",
  "ADmantX Platform Semantic Analyzer - Turn - ADmantX Inc. - www.admantx.com - [email protected]"
]

GOOGLE_USER_AGENTS =

[
  "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
  "Googlebot/2.1 (+http://www.googlebot.com/bot.html)",
  "Googlebot/2.1 (+http://www.google.com/bot.html)",
  "Googlebot-News",
  "Googlebot-Image/1.0",
  "Googlebot-Video/1.0",
  "(compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
  "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
  "(compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)",
  "Mediapartners-Google",
  "AdsBot-Google (+http://www.google.com/adsbot.html)",
  "Googlebot",
  "Googlebot-News (Googlebot)",
  "Googlebot-Image (Googlebot)",
  "Googlebot-Video (Googlebot)",
  "Googlebot-Mobile",
  "Mediapartners-Google",
  "Mediapartners (Googlebot)",
  "Mediapartners-Google Mediapartners (Googlebot)",
  "AdsBot-Google"
]

SearchEngineUserAgents =

[
  "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
  "Baiduspider+(+http://www.baidu.com/search/spider.htm)",
  "Baiduspider+(+http://www.baidu.com/search/spider_jp.html)",
  "BaiDuSpider",
  "Mozilla/5.0+(compatible;+Baiduspider-cpro;++http://www.baidu.com/search/spider.html)",
  "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
  "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexPagechecker/1.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexDirect/2.0; Dyatel; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexNews/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexCatalog/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexZakladki/3.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; YandexMarket/1.0; +http://yandex.com/bots)",
  "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
  "Mozilla/5.0 (compatible; Yahoo Slurp;   http://help.yahoo.com/help/us/ysearch/slurp)",
  "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)",
  "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
  "Mozilla/5.0 (compatible; bingbot/2.0 +http://www.bing.com/bingbot.htm)",
  "msnbot/2.1",
  "msnbot/2.0b",
  "msnbot/2.0b (+http://search.msn.com/msnbot.htm)",
  "msnbot/1.1 (+http://search.msn.com/msnbot.htm)",
  "msnbot/1.1",
  "msnbot/1.0 (+http://search.msn.com/msnbot.htm)",
  "msnbot/0.9 (+http://search.msn.com/msnbot.htm)",
  "msnbot/0.11 ( http://search.msn.com/msnbot.htm)",
  "MSNBOT/0.1 (http://search.msn.com/msnbot.htm)",
  "Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/robots/)",
]

Class Method Summary collapse

.included(target) ⇒ Object

Class Method Details

.included(target) ⇒ `Object`



5
6
7

# File 'lib/bot_detection.rb', line 5

def self.included(target)
  target.send(:include, BotDetection::InstanceMethods)
end

Module: BotDetection

Defined Under Namespace

Constant Summary collapse

Class Method Summary collapse

Class Method Details

.included(target) ⇒ Object

.included(target) ⇒ `Object`