Module: HttpUtilities::Proxies::Nosql::ProxyModule::ClassMethods

Defined in:
lib/http_utilities/proxies/nosql/proxy_module.rb

Instance Method Summary collapse

Instance Method Details

#format_proxy_address(proxy_host, proxy_port = 80, include_http = false) ⇒ Object 



63
64
65
66
67
68
 
# File 'lib/http_utilities/proxies/nosql/proxy_module.rb', line 63

def format_proxy_address(proxy_host, proxy_port = 80, include_http = false)
  proxy_address = "#{proxy_host}:#{proxy_port}"
  proxy_address.insert(0, "http://") if (include_http && !proxy_address.start_with?("http://"))
    
  return proxy_address
end

#format_proxy_credentials(username, password) ⇒ Object 



70
71
72
 
# File 'lib/http_utilities/proxies/nosql/proxy_module.rb', line 70

def format_proxy_credentials(username, password)
  return "#{username}:#{password}"
end

#get_proxies_for_protocol_and_proxy_type(protocol, proxy_type) ⇒ Object 



55
56
57
58
59
60
61
 
# File 'lib/http_utilities/proxies/nosql/proxy_module.rb', line 55

def get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
  proxies     =   ::Proxy.where(nil)
  proxies     =   proxies.where(protocol: protocol)     if (protocol && !protocol.downcase.to_sym.eql?(:all))
  proxies     =   proxies.where(proxy_type: proxy_type) if (proxy_type && !proxy_type.downcase.to_sym.eql?(:all))
    
  return proxies
end

#get_random_proxy(protocol: :all, proxy_type: :all, maximum_failed_attempts: nil, retries: 3) ⇒ Object 



40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/http_utilities/proxies/nosql/proxy_module.rb', line 40

def get_random_proxy(protocol: :all, proxy_type: :all, maximum_failed_attempts: nil, retries: 3)
  proxies     =   get_valid_proxies(protocol: protocol, proxy_type: proxy_type, maximum_failed_attempts: maximum_failed_attempts)
  proxy       =   nil
  
  begin
    proxy     =   proxies.skip(rand(proxies.count)).first
  
  rescue StandardError
    retries  -=   1
    retry if retries > 0
  end
  
  return proxy
end

#get_valid_proxies(protocol: :all, proxy_type: :all, maximum_failed_attempts: nil, retries: 3) ⇒ Object 



34
35
36
37
38
 
# File 'lib/http_utilities/proxies/nosql/proxy_module.rb', line 34

def get_valid_proxies(protocol: :all, proxy_type: :all, maximum_failed_attempts: nil, retries: 3)
  proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
  proxies     =   proxies.where(valid_proxy: true)
  proxies     =   proxies.where(:failed_attempts.lte => maximum_failed_attempts) if maximum_failed_attempts
end

#should_be_checked(protocol: :all, proxy_type: :all, date: Time.now, limit: 10, maximum_failed_attempts: 10) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/http_utilities/proxies/nosql/proxy_module.rb', line 13

def should_be_checked(protocol: :all, proxy_type: :all, date: Time.now, limit: 10, maximum_failed_attempts: 10)
  proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
    
  proxies     =   proxies.any_of(
    {:last_checked_at.exists => false},
    {:last_checked_at.ne => nil},
    {:last_checked_at.exists => true, :last_checked_at.ne => nil, :last_checked_at.lt => date}
  )
    
  proxies     =   proxies.any_of(
    {:failed_attempts.exists => false},
    {:failed_attempts.in => ["", nil]},
    {:failed_attempts.exists => true, :failed_attempts.nin => ["", nil], :failed_attempts.lte => maximum_failed_attempts}
  )
    
  proxies     =   proxies.order_by([[:valid_proxy, :asc], [:failed_attempts, :asc], [:last_checked_at, :asc]])
  proxies     =   proxies.limit(limit)
    
  return proxies
end