Class: ProxyService

Inherits:
Object
  • Object
show all
Defined in:
lib/proxy_service.rb,
lib/proxy_service/version.rb

Defined Under Namespace

Classes: MechanizeAgent, NullWorker, Proxy, Worker

Constant Summary collapse

VERSION = 
'1.1.3'

Class Attribute Summary collapse

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source, options = {}) ⇒ ProxyService

Create a new proxy service with for a specific ODS

Parameters:

  • source (String, Symbol) —

    name of the ODS (e.g. :trip_advisor), will look for a queue with that name “proxy/##source”

  • options (Hash) (defaults to: {})

Options Hash (options):

  • :proxies_enabled (Boolean) —

    override the class configuration

  • :failure_limit (Integer) —

    before blocking the proxy

  • :failure_codes (Array) —

    that indicate a proxy was blocked by the site



23
24
25
26
27
28
 
# File 'lib/proxy_service.rb', line 23

def initialize(source, options = {})
  @source = source
  @proxies_enabled = options.fetch(:proxies_enabled, !!self.class.proxies_enabled)
  @failure_limit = options.fetch(:failure_limit, self.class.failure_limit || 3)
  @failure_codes = options.fetch(:failure_codes, self.class.failure_codes || %w[403])
end

Class Attribute Details

.failure_codes ⇒ Object

Returns the value of attribute failure_codes.



6
7
8
 
# File 'lib/proxy_service.rb', line 6

def failure_codes
  @failure_codes
end

.failure_limit ⇒ Object

Returns the value of attribute failure_limit.



6
7
8
 
# File 'lib/proxy_service.rb', line 6

def failure_limit
  @failure_limit
end

.password ⇒ Object

Returns the value of attribute password.



6
7
8
 
# File 'lib/proxy_service.rb', line 6

def password
  @password
end

.proxies_enabled ⇒ Object

Returns the value of attribute proxies_enabled.



6
7
8
 
# File 'lib/proxy_service.rb', line 6

def proxies_enabled
  @proxies_enabled
end

.username ⇒ Object

Returns the value of attribute username.



6
7
8
 
# File 'lib/proxy_service.rb', line 6

def username
  @username
end

Instance Attribute Details

#failure_codes ⇒ Object

Returns the value of attribute failure_codes.



13
14
15
 
# File 'lib/proxy_service.rb', line 13

def failure_codes
  @failure_codes
end

#failure_limit ⇒ Object

Returns the value of attribute failure_limit.



13
14
15
 
# File 'lib/proxy_service.rb', line 13

def failure_limit
  @failure_limit
end

#proxies_enabled=(value) ⇒ Object (writeonly)

Sets the attribute proxies_enabled

Parameters:

  • value —

    the value to set the attribute proxies_enabled to.



14
15
16
 
# File 'lib/proxy_service.rb', line 14

def proxies_enabled=(value)
  @proxies_enabled = value
end

#source ⇒ Object

Returns the value of attribute source.



13
14
15
 
# File 'lib/proxy_service.rb', line 13

def source
  @source
end

Class Method Details

.configure {|_self| ... } ⇒ Object

Yields:

  • (_self)

Yield Parameters:

  • _self (ProxyService) —

    the object that the method was called on



8
9
10
 
# File 'lib/proxy_service.rb', line 8

def configure
  yield self
end

Instance Method Details

#new_worker ⇒ Object 



72
73
74
75
76
77
78
 
# File 'lib/proxy_service.rb', line 72

def new_worker
  if proxies_enabled?
    Worker.new("proxy/#{source}")
  else
    NullWorker.new
  end
end

#proxies_enabled? ⇒ Boolean

Private

Returns:

  • (Boolean) 


53
54
55
 
# File 'lib/proxy_service.rb', line 53

def proxies_enabled?
  @proxies_enabled
end

#reserve_proxy ⇒ Proxy

Sleeps until the worker receives a proxy (message)

Returns:

  • (Proxy) —

    a new proxy object



60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/proxy_service.rb', line 60

def reserve_proxy
  worker = new_worker
  worker.subscribe
  loop do
    if worker.ready?
      return Proxy.new(worker)
    else
      sleep(1)
    end
  end
end

#with_mechanize {|agent| ... } ⇒ Object

Yields:

  • (agent) —

    Passes a [proxied] Mechanize agent to the block



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/proxy_service.rb', line 31

def with_mechanize
  proxy = reserve_proxy
  agent = MechanizeAgent.new
  agent.set_proxy(proxy)
  yield agent
  proxy.reset_failures
rescue Mechanize::ResponseCodeError => e
  if failure_codes.include?(e.response_code)
    if proxy.failures >= failure_limit
      proxy.blocked!
    else
      proxy.increment_failures
    end
  end
ensure
  proxy.release
end