Module: ProxyRotator

Defined in:
lib/proxy_rotator.rb,
lib/proxy_rotator/rotator.rb,
lib/proxy_rotator/version.rb,
lib/proxy_rotator/configuration.rb

Defined Under Namespace

Classes: Configuration

Constant Summary collapse

VERSION = 
"0.1.0"

Class Method Summary collapse

Class Method Details

.configuration ⇒ Object 



48
49
50
 
# File 'lib/proxy_rotator/configuration.rb', line 48

def self.configuration
  @configuration ||= initialize_configuration!
end

.configure {|configuration| ... } ⇒ Object

Yields:



52
53
54
 
# File 'lib/proxy_rotator/configuration.rb', line 52

def self.configure
  yield(configuration)
end

.initialize_configuration! ⇒ Object 



56
57
58
 
# File 'lib/proxy_rotator/configuration.rb', line 56

def self.initialize_configuration!
  @configuration = Configuration.new
end

.load_file(filepath) ⇒ Object 



9
10
11
12
13
14
15
16
17
18
 
# File 'lib/proxy_rotator/rotator.rb', line 9

def self.load_file(filepath)
  @start_index=0
  File.open(filepath, "r") do |f|
    f.each_line do |line|
      unless validate_url(line).nil? || @proxies.include?(line)
        @proxies << line.strip
      end
    end
  end
end

.remote_first_then_rotate(check_first = false) ⇒ Object 



68
69
70
71
72
73
 
# File 'lib/proxy_rotator/rotator.rb', line 68

def self.remote_first_then_rotate(check_first=false)
  result = self.rotate_remote
  result = self.rotate(check_first) unless !result.nil?

  result
end

.reset_rotate(check_first = false) ⇒ Object 



20
21
22
23
 
# File 'lib/proxy_rotator/rotator.rb', line 20

def self.reset_rotate(check_first=false)
  @start_index = 0
  self.rotate(check_first)
end

.rotate(check_first = false) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/proxy_rotator/rotator.rb', line 25

def self.rotate(check_first=false)
  return nil unless @proxies.count > 0
  proxy = @proxies[@start_index]
  @start_index=@start_index+1

  if check_first
    begin
      RestClient::Request.new(
          :method => :get,
          :url => ProxyRotator.configuration.default_test_url,
          :proxy => proxy,
          :timeout => ProxyRotator.configuration.default_timeout
      ).execute

      return proxy
    rescue Exception => e
      p "Proxy #{proxy} failed : #{e.message}"
      @failed_proxies << proxy unless @failed_proxies.include? proxy
      return nil unless @proxies.count > 1 || @proxies.count == @failed_proxies.count
      self.rotate(true)
    end
  else
    return proxy
  end
end

.rotate_first_then_remote(check_first = false) ⇒ Object 



61
62
63
64
65
66
 
# File 'lib/proxy_rotator/rotator.rb', line 61

def self.rotate_first_then_remote(check_first=false)
  result = self.rotate(check_first)
  result = self.rotate_remote unless !result.nil?

  result
end

.rotate_remote ⇒ Object 



51
52
53
54
55
56
57
58
59
 
# File 'lib/proxy_rotator/rotator.rb', line 51

def self.rotate_remote
  url = ProxyRotator.configuration.base_url + "?apiKeys=" + ProxyRotator.configuration.api_key
  response = RestClient.get(url)
  json = JSON.parse(response.body)
  proxy = json["proxy"]
  return "http://#{proxy}" unless proxy.nil?

  return nil
end

.validate_url(url) ⇒ Object 



75
76
77
 
# File 'lib/proxy_rotator/rotator.rb', line 75

def self.validate_url(url)
  url =~ URI::regexp(%w(http https))
end