Class: ProxyFetcher::Manager

Inherits:
Object
  • Object
show all
Defined in:
lib/proxy_fetcher/manager.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(refresh: true, validate: false, filters: {}) ⇒ Manager

refresh: true - load proxy list from the remote server on initialization refresh: false - just initialize the class, proxy list will be empty ([])



7
8
9
10
11
12
13
14
15
 
# File 'lib/proxy_fetcher/manager.rb', line 7

def initialize(refresh: true, validate: false, filters: {})
  if refresh
    refresh_list!(filters)
  else
    @proxies = []
  end

  cleanup! if validate
end

Instance Attribute Details

#proxies ⇒ Object (readonly)

Returns the value of attribute proxies.



3
4
5
 
# File 'lib/proxy_fetcher/manager.rb', line 3

def proxies
  @proxies
end

Instance Method Details

#cleanup! ⇒ Object Also known as: validate!

Clean current proxy list from dead proxies (that doesn’t respond by timeout)



60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/proxy_fetcher/manager.rb', line 60

def cleanup!
  lock = Mutex.new

  proxies.dup.each_slice(ProxyFetcher.config.pool_size) do |proxy_group|
    threads = proxy_group.map do |group_proxy|
      Thread.new(group_proxy, proxies) do |proxy, proxies|
        lock.synchronize { proxies.delete(proxy) } unless proxy.connectable?
      end
    end

    threads.each(&:join)
  end

  @proxies
end

#get ⇒ Object Also known as: pop

Pop just first proxy (and back it to the end of the proxy list)



32
33
34
35
36
37
38
39
 
# File 'lib/proxy_fetcher/manager.rb', line 32

def get
  return if @proxies.empty?

  first_proxy = @proxies.shift
  @proxies << first_proxy

  first_proxy
end

#get! ⇒ Object Also known as: pop!

Pop first valid proxy (and back it to the end of the proxy list) Invalid proxies will be removed from the list



45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/proxy_fetcher/manager.rb', line 45

def get!
  index = proxies.find_index(&:connectable?)
  return if index.nil?

  proxy = proxies.delete_at(index)
  tail = proxies[index..-1]

  @proxies = tail << proxy

  proxy
end

#inspect ⇒ Object

No need to put all the attr_readers



91
92
93
 
# File 'lib/proxy_fetcher/manager.rb', line 91

def inspect
  to_s
end

#random_proxy ⇒ Object Also known as: random

Return random proxy



79
80
81
 
# File 'lib/proxy_fetcher/manager.rb', line 79

def random_proxy
  proxies.sample
end

#raw_proxies ⇒ Object

Returns array of proxy URLs (just schema + host + port)



86
87
88
 
# File 'lib/proxy_fetcher/manager.rb', line 86

def raw_proxies
  proxies.map(&:url)
end

#refresh_list!(filters = nil) ⇒ Object Also known as: fetch!

Update current proxy list from the provider



18
19
20
21
22
23
24
25
26
27
 
# File 'lib/proxy_fetcher/manager.rb', line 18

def refresh_list!(filters = nil)
  @proxies = []

  ProxyFetcher.config.providers.each do |provider_name|
    provider = ProxyFetcher::Configuration.providers_registry.class_for(provider_name)
    provider_filters = filters && filters.fetch(provider_name.to_sym, filters)

    @proxies.concat(provider.fetch_proxies!(provider_filters))
  end
end