Class: CMSScanner::Browser

Inherits:
Object
  • Object
show all
Extended by:
Actions
Defined in:
lib/cms_scanner/browser.rb,
lib/cms_scanner/browser/actions.rb,
lib/cms_scanner/browser/options.rb

Overview

Options available in the Browser

Defined Under Namespace

Modules: Actions

Constant Summary collapse

OPTIONS = 
%i[
  cache_ttl
  cookie_jar
  cookie_string
  connect_timeout
  disable_tls_checks
  headers
  http_auth
  max_threads
  proxy
  proxy_auth
  random_user_agent
  request_timeout
  throttle
  url
  user_agent
  user_agents_list
  vhost
].freeze

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Actions

get, get_and_follow_location, head, post

Constructor Details

#initialize(parsed_options = {}) ⇒ Void

Parameters:

  • parsed_options (Hash) (defaults to: {}) 


14
15
16
17
18
 
# File 'lib/cms_scanner/browser.rb', line 14

def initialize(parsed_options = {})
  self.throttle = 0

  load_options(parsed_options.dup)
end

Class Method Details

.instance(parsed_options = {}) ⇒ Browser

Returns The instance.

Parameters:

  • parsed_options (Hash) (defaults to: {})

Returns:



25
26
27
 
# File 'lib/cms_scanner/browser.rb', line 25

def self.instance(parsed_options = {})
  @@instance ||= new(parsed_options)
end

.reset ⇒ Object 



29
30
31
 
# File 'lib/cms_scanner/browser.rb', line 29

def self.reset
  @@instance = nil
end

Instance Method Details

#default_request_params ⇒ Hash

Returns:

  • (Hash) 


49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/cms_scanner/browser.rb', line 49

def default_request_params
  params = {
    headers: { 'User-Agent' => user_agent, 'Referer' => url }.merge(headers || {}),
    accept_encoding: 'gzip, deflate',
    method: :get
  }

  if disable_tls_checks
    # See http://curl.haxx.se/libcurl/c/CURLOPT_SSL_VERIFYHOST.html
    params[:ssl_verifypeer] = false
    params[:ssl_verifyhost] = 0
    # TLSv1.0 and plus, allows to use a protocol potentially lower than the OS default
    params[:sslversion] = :tlsv1
  end

  typhoeus_to_browser_opts.each do |typhoeus_opt, browser_opt|
    attr_value = public_send(browser_opt)
    params[typhoeus_opt] = attr_value unless attr_value.nil?
  end

  params[:proxyuserpwd] = "#{proxy_auth[:username]}:#{proxy_auth[:password]}" if proxy_auth
  params[:userpwd] = "#{http_auth[:username]}:#{http_auth[:password]}" if http_auth

  params[:headers]['Host'] = vhost if vhost

  params
end

#default_user_agent ⇒ String

Returns:

  • (String) 


29
30
31
 
# File 'lib/cms_scanner/browser/options.rb', line 29

def default_user_agent
  "#{NS} v#{NS::VERSION}"
end

#forge_request(url, params = {}) ⇒ Typhoeus::Request

Parameters:

  • url (String)
  • params (Hash) (defaults to: {})

Returns:

  • (Typhoeus::Request) 


37
38
39
 
# File 'lib/cms_scanner/browser.rb', line 37

def forge_request(url, params = {})
  Typhoeus::Request.new(url, request_params(params))
end

#hydra ⇒ Typhoeus::Hydra

Returns:



34
35
36
 
# File 'lib/cms_scanner/browser/options.rb', line 34

def hydra
  @hydra ||= Typhoeus::Hydra.new(max_concurrency: max_threads || 1)
end

#load_options(options = {}) ⇒ Object

Parameters:

  • options (Hash) (defaults to: {}) 


39
40
41
42
43
 
# File 'lib/cms_scanner/browser/options.rb', line 39

def load_options(options = {})
  OPTIONS.each do |sym|
    send("#{sym}=", options[sym]) if options.key?(sym)
  end
end

#max_threads=(number) ⇒ Object

Set the threads attribute and update hydra accordinly If the throttle attribute is > 0, max_threads will be forced to 1

Parameters:

  • number (Integer) 


49
50
51
52
53
 
# File 'lib/cms_scanner/browser/options.rb', line 49

def max_threads=(number)
  @max_threads = number.to_i.positive? && throttle.zero? ? number.to_i : 1

  hydra.max_concurrency = @max_threads
end

#request_params(params = {}) ⇒ Hash

Parameters:

  • params (Hash) (defaults to: {})

Returns:

  • (Hash) 


80
81
82
83
84
 
# File 'lib/cms_scanner/browser.rb', line 80

def request_params(params = {})
  default_request_params.merge(params) do |key, oldval, newval|
    key == :headers ? oldval.merge(newval) : newval
  end
end

#throttle=(value) ⇒ Object

if value > 0, the max_threads will be set to 1

Parameters:

  • The (value) —

    throttle time in milliseconds



80
81
82
83
84
 
# File 'lib/cms_scanner/browser/options.rb', line 80

def throttle=(value)
  @throttle = value.to_i.abs / 1000.0

  self.max_threads = 1 if @throttle.positive?
end

#trottle! ⇒ Object 



86
87
88
 
# File 'lib/cms_scanner/browser/options.rb', line 86

def trottle!
  sleep(throttle) if throttle.positive?
end

#typhoeus_to_browser_opts ⇒ Hash

Returns:

  • (Hash) 


42
43
44
45
46
 
# File 'lib/cms_scanner/browser.rb', line 42

def typhoeus_to_browser_opts
  { connecttimeout: :connect_timeout, cache_ttl: :cache_ttl,
    proxy: :proxy, timeout: :request_timeout, cookiejar: :cookie_jar,
    cookiefile: :cookie_jar, cookie: :cookie_string }
end

#user_agent ⇒ String

Returns The user agent.

Returns:

  • (String) —

    The user agent



56
57
58
 
# File 'lib/cms_scanner/browser/options.rb', line 56

def user_agent
  @user_agent ||= random_user_agent ? user_agents.sample : default_user_agent
end

#user_agents ⇒ Array<String>

Returns:

  • (Array<String>) 


61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/cms_scanner/browser/options.rb', line 61

def user_agents
  return @user_agents if @user_agents

  @user_agents = []

  # The user_agents_list is managed by the CLI options, with the default being
  # APP_DIR/user_agents.txt
  File.open(user_agents_list).each do |line|
    next if line == "\n" || line[0, 1] == '#'

    @user_agents << line.chomp
  end

  @user_agents
end