Class: CMSScanner::Browser

Inherits:
Object
  • Object
show all
Extended by:
Actions
Defined in:
lib/cms_scanner/browser.rb,
lib/cms_scanner/browser/actions.rb,
lib/cms_scanner/browser/options.rb

Overview

Options available in the Browser

Defined Under Namespace

Modules: Actions

Constant Summary collapse

OPTIONS = 
%i[
  cache_ttl
  cookie_jar
  cookie_string
  connect_timeout
  disable_tls_checks
  headers
  http_auth
  max_threads
  proxy
  proxy_auth
  random_user_agent
  request_timeout
  throttle
  user_agent
  user_agents_list
  vhost
].freeze

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Actions

get, get_and_follow_location, head, post

Constructor Details

#initialize(parsed_options = {}) ⇒ Void

Parameters:

  • parsed_options (Hash) (defaults to: {}) 


12
13
14
15
16
 
# File 'lib/cms_scanner/browser.rb', line 12

def initialize(parsed_options = {})
  self.throttle = 0

  load_options(parsed_options)
end

Class Method Details

.instance(parsed_options = {}) ⇒ Browser

Returns The instance.

Parameters:

  • parsed_options (Hash) (defaults to: {})

Returns:



23
24
25
 
# File 'lib/cms_scanner/browser.rb', line 23

def self.instance(parsed_options = {})
  @@instance ||= new(parsed_options)
end

.reset ⇒ Object 



27
28
29
 
# File 'lib/cms_scanner/browser.rb', line 27

def self.reset
  @@instance = nil
end

Instance Method Details

#default_request_params ⇒ Hash

Returns:

  • (Hash) 


47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/cms_scanner/browser.rb', line 47

def default_request_params
  params = {
    headers: { 'User-Agent' => user_agent }.merge(headers || {}),
    accept_encoding: 'gzip, deflate',
    method: :get
  }

  if disable_tls_checks
    # See http://curl.haxx.se/libcurl/c/CURLOPT_SSL_VERIFYHOST.html
    params[:ssl_verifypeer] = false
    params[:ssl_verifyhost] = 0
  end

  typhoeus_to_browser_opts.each do |typhoeus_opt, browser_opt|
    attr_value = public_send(browser_opt)
    params[typhoeus_opt] = attr_value unless attr_value.nil?
  end

  params[:proxyuserpwd] = "#{proxy_auth[:username]}:#{proxy_auth[:password]}" if proxy_auth
  params[:userpwd] = "#{http_auth[:username]}:#{http_auth[:password]}" if http_auth

  params[:headers]['Host'] = vhost if vhost

  params
end

#default_user_agent ⇒ String

Returns:

  • (String) 


26
27
28
 
# File 'lib/cms_scanner/browser/options.rb', line 26

def default_user_agent
  "#{NS} v#{NS::VERSION}"
end

#forge_request(url, params = {}) ⇒ Typhoeus::Request

Parameters:

  • url (String)
  • params (Hash) (defaults to: {})

Returns:

  • (Typhoeus::Request) 


35
36
37
 
# File 'lib/cms_scanner/browser.rb', line 35

def forge_request(url, params = {})
  Typhoeus::Request.new(url, request_params(params))
end

#hydra ⇒ Object 



30
31
32
 
# File 'lib/cms_scanner/browser/options.rb', line 30

def hydra
  @hydra ||= Typhoeus::Hydra.new(max_concurrency: max_threads || 1)
end

#load_options(options = {}) ⇒ Object

Parameters:

  • options (Hash) (defaults to: {}) 


35
36
37
38
39
 
# File 'lib/cms_scanner/browser/options.rb', line 35

def load_options(options = {})
  OPTIONS.each do |sym|
    send("#{sym}=", options[sym]) if options.key?(sym)
  end
end

#max_threads=(number) ⇒ Object

Set the threads attribute and update the max_concurrency of Typhoeus::Hydra

If the throttle attribute is > 0, max_threads will be forced to 1

Parameters:

  • number (Integer) 


47
48
49
50
51
 
# File 'lib/cms_scanner/browser/options.rb', line 47

def max_threads=(number)
  @max_threads = number.to_i.positive? && throttle.zero? ? number.to_i : 1

  hydra.max_concurrency = @max_threads
end

#request_params(params = {}) ⇒ Hash

Parameters:

  • params (Hash) (defaults to: {})

Returns:

  • (Hash) 


76
77
78
79
80
 
# File 'lib/cms_scanner/browser.rb', line 76

def request_params(params = {})
  default_request_params.merge(params) do |key, oldval, newval|
    key == :headers ? oldval.merge(newval) : newval
  end
end

#throttle=(value) ⇒ Object

if value > 0, the max_threads will be set to 1

Parameters:

  • The (value) —

    throttle time in milliseconds



81
82
83
84
85
 
# File 'lib/cms_scanner/browser/options.rb', line 81

def throttle=(value)
  @throttle = value.to_i.abs / 1000.0

  self.max_threads = 1 if @throttle.positive?
end

#trottle! ⇒ Object 



87
88
89
 
# File 'lib/cms_scanner/browser/options.rb', line 87

def trottle!
  sleep(throttle) if throttle.positive?
end

#typhoeus_to_browser_opts ⇒ Hash

Returns:

  • (Hash) 


40
41
42
43
44
 
# File 'lib/cms_scanner/browser.rb', line 40

def typhoeus_to_browser_opts
  { connecttimeout: :connect_timeout, cache_ttl: :cache_ttl,
    proxy: :proxy, timeout: :request_timeout, cookiejar: :cookie_jar,
    cookiefile: :cookie_jar, cookie: :cookie_string }
end

#user_agent ⇒ String

Returns The user agent.

Returns:

  • (String) —

    The user agent



54
55
56
 
# File 'lib/cms_scanner/browser/options.rb', line 54

def user_agent
  @user_agent ||= random_user_agent ? user_agents.sample : default_user_agent
end

#user_agents ⇒ Array<String>

Returns:

  • (Array<String>) 


59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/cms_scanner/browser/options.rb', line 59

def user_agents
  return @user_agents if @user_agents

  @user_agents = []

  File.open(user_agents_list).each do |line|
    next if line == "\n" || line[0, 1] == '#'

    @user_agents << line.chomp
  end

  @user_agents
end

#user_agents_list ⇒ String

Returns The path to the user agents list.

Returns:

  • (String) —

    The path to the user agents list



74
75
76
 
# File 'lib/cms_scanner/browser/options.rb', line 74

def user_agents_list
  @user_agents_list ||= File.join(APP_DIR, 'user_agents.txt')
end