Class: RequestManager

Inherits:
Object
  • Object
show all
Defined in:
lib/requestmanager.rb

Instance Method Summary collapse

Constructor Details

#initialize(proxy_list, request_interval, browser_num) ⇒ RequestManager

Returns a new instance of RequestManager.



7
8
9
10
11
12
13
14
 
# File 'lib/requestmanager.rb', line 7

def initialize(proxy_list, request_interval, browser_num)
  @proxy_list = parse_proxy_list(proxy_list)
  @request_interval = request_interval
  @used_proxies = Array.new
  @browser_num = browser_num
  @browsers = Hash.new
  open_n_browsers
end

Instance Method Details

#close_all_browsers ⇒ Object

Close all the browsers



68
69
70
71
72
 
# File 'lib/requestmanager.rb', line 68

def close_all_browsers
  @browsers.each do |browser|
    browser[1][0].quit
  end
end

#gen_driver(chosen_proxy) ⇒ Object

Generate driver for searches



95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/requestmanager.rb', line 95

def gen_driver(chosen_proxy)
  # Profile settings
  profile = Selenium::WebDriver::Firefox::Profile.new
  profile['intl.accept_languages'] = 'en'

  # Set proxy if proxy list, otherwise sleep
  if chosen_proxy
    proxy = Selenium::WebDriver::Proxy.new(http: chosen_proxy, ssl: chosen_proxy)
    profile.proxy = proxy
  else
    sleep(rand(@request_interval[0]..@request_interval[1]))
  end
  
  return Selenium::WebDriver.for :firefox, profile: profile
end

#get_least_recent_browser ⇒ Object

Get the least recently used browser



42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/requestmanager.rb', line 42

def get_least_recent_browser
  least_recent = @browsers.first
  @browsers.each do |browser|
      if browser[1][1] < least_recent[1][1]
        least_recent = browser
      end
  end
  
  # Update the usage time
  @browsers[least_recent[0]] = [least_recent[1][0], Time.now]
  return least_recent[1][0]
end

#get_most_recent_browser ⇒ Object

Get the most recently used browser



30
31
32
33
34
35
36
37
38
39
 
# File 'lib/requestmanager.rb', line 30

def get_most_recent_browser
  most_recent = @browsers.first
  @browsers.each do |browser|
    if browser[1][1] > most_recent[1][1]
      most_recent = browser
    end
  end

  return most_recent
end

#get_page(url, form_input = nil) ⇒ Object

Get the page requested



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/requestmanager.rb', line 75

def get_page(url, form_input = nil)
  # Get the page
  browser = get_least_recent_browser
  browser.navigate.to url
  puts "Getting page " + url

  # Handle form input if there is any
  if form_input
    element = driver.find_element(name: "q")
    element.send_keys form_input
    element.submit
  end

  # Sleep while things load then save output
  sleep(rand(@request_interval[0]..@request_interval[1]))
  page_html = browser.page_source
  return page_html
end

#get_random_proxy ⇒ Object

Choose a random proxy that hasn’t been used recently



112
113
114
115
116
117
118
119
120
121
122
123
124
125
 
# File 'lib/requestmanager.rb', line 112

def get_random_proxy
  max = @proxy_list.length
  chosen = @proxy_list[Random.rand(max)]
  chosen_proxy = chosen[0]+":"+chosen[1]
  
  # Only use proxy if it hasn't been used in last n seconds on same host
  if !@used_proxies.include?(chosen_proxy)
    @used_proxies.push(chosen_proxy)
    return chosen_proxy
  else
    sleep(0.005)
    get_random_proxy
  end
end

#open_browser ⇒ Object

Open the browser with a random proxy



24
25
26
27
 
# File 'lib/requestmanager.rb', line 24

def open_browser
  chosen_proxy = @proxy_list != nil ? get_random_proxy : nil
  @browsers[chosen_proxy] = [gen_driver(chosen_proxy), Time.now]
end

#open_n_browsers ⇒ Object

Open the specified number of browsers



17
18
19
20
21
 
# File 'lib/requestmanager.rb', line 17

def open_n_browsers
  (1..@browser_num).each do |i|
    open_browser
  end
end

#parse_proxy_list(proxy_file) ⇒ Object

Parse the proxy list



128
129
130
131
132
 
# File 'lib/requestmanager.rb', line 128

def parse_proxy_list(proxy_file)
  if proxy_file
    return IO.readlines(proxy_file).map{ |proxy| proxy.strip.split(":")}
  end
end

#restart_browser ⇒ Object

Restart the browser and open new one



56
57
58
59
60
61
62
63
64
65
 
# File 'lib/requestmanager.rb', line 56

def restart_browser
  # Get most recently used browser and close it
  close_browser = get_most_recent_browser
  close_browser[1][0].quit

  # Remove it from lists of used browsers and start new
  @browsers.delete(close_browser[0])
  open_browser
  @used_proxies.delete(close_browser[0])
end