Class: ProxyFetcher::Manager

Inherits:
Object
  • Object
show all
Defined in:
lib/proxy_fetcher/manager.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(refresh: true, filters: {}) ⇒ Manager

refresh: true - load proxy list from the remote server on initialization refresh: false - just initialize the class, proxy list will be empty ([])



7
8
9
10
11
12
13
14
15
 
# File 'lib/proxy_fetcher/manager.rb', line 7

def initialize(refresh: true, filters: {})
  @filters = filters

  if refresh
    refresh_list!
  else
    @proxies = []
  end
end

Instance Attribute Details

#filters ⇒ Object (readonly)

Returns the value of attribute filters.



3
4
5
 
# File 'lib/proxy_fetcher/manager.rb', line 3

def filters
  @filters
end

#proxies ⇒ Object (readonly)

Returns the value of attribute proxies.



3
4
5
 
# File 'lib/proxy_fetcher/manager.rb', line 3

def proxies
  @proxies
end

Instance Method Details

#cleanup!(pool_size = 10) ⇒ Object Also known as: validate!

Clean current proxy list from dead proxies (that doesn’t respond by timeout)



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/proxy_fetcher/manager.rb', line 53

def cleanup!(pool_size = 10)
  lock = Mutex.new

  proxies.dup.each_slice(pool_size) do |proxy_group|
    threads = proxy_group.map do |group_proxy|
      Thread.new(group_proxy, proxies) do |proxy, proxies|
        lock.synchronize { proxies.delete(proxy) } unless proxy.connectable?
      end
    end

    threads.each(&:join)
  end

  @proxies
end

#get ⇒ Object Also known as: pop

Pop just first proxy (and back it to the end of the proxy list)



25
26
27
28
29
30
31
32
 
# File 'lib/proxy_fetcher/manager.rb', line 25

def get
  return if @proxies.empty?

  first_proxy = @proxies.shift
  @proxies << first_proxy

  first_proxy
end

#get! ⇒ Object Also known as: pop!

Pop first valid proxy (and back it to the end of the proxy list) Invalid proxies will be removed from the list



38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/proxy_fetcher/manager.rb', line 38

def get!
  index = proxies.find_index(&:connectable?)
  return if index.nil?

  proxy = proxies.delete_at(index)
  tail = proxies[index..-1]

  @proxies = tail << proxy

  proxy
end

#inspect ⇒ Object

No need to put all the attr_readers



84
85
86
 
# File 'lib/proxy_fetcher/manager.rb', line 84

def inspect
  to_s
end

#random_proxy ⇒ Object Also known as: random

Return random proxy



72
73
74
 
# File 'lib/proxy_fetcher/manager.rb', line 72

def random_proxy
  proxies.sample
end

#raw_proxies ⇒ Object

Returns array of proxy URLs (just schema + host + port)



79
80
81
 
# File 'lib/proxy_fetcher/manager.rb', line 79

def raw_proxies
  proxies.map(&:url)
end

#refresh_list! ⇒ Object Also known as: fetch!

Update current proxy list from the provider



18
19
20
 
# File 'lib/proxy_fetcher/manager.rb', line 18

def refresh_list!
  @proxies = ProxyFetcher.config.provider.fetch_proxies!(filters)
end