Class: ProxyFetcher::Manager

Inherits:
Object
  • Object
show all
Defined in:
lib/proxy_fetcher/manager.rb

Overview

ProxyFetcher Manager class for interacting with proxy lists from various providers.

Constant Summary collapse

REFRESHER_LOCK = 
Mutex.new

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(**options) ⇒ Manager

Initialize ProxyFetcher Manager instance for managing proxies

refresh: true - load proxy list from the remote server on initialization refresh: false - just initialize the class, proxy list will be empty ([])



27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/proxy_fetcher/manager.rb', line 27

def initialize(**options)
  if options.fetch(:refresh, true)
    refresh_list!(options.fetch(:filters, {}))
  else
    @proxies = []
  end

  files = Array(options.fetch(:file, options.fetch(:files, [])))
  load_proxies_from_files!(files) if files&.any?

  cleanup! if options.fetch(:validate, false)
end

Instance Attribute Details

#proxies ⇒ Object (readonly)

Returns the value of attribute proxies.



18
19
20
 
# File 'lib/proxy_fetcher/manager.rb', line 18

def proxies
  @proxies
end

Class Method Details

.from_files(files, **options) ⇒ Object Also known as: from_file 



9
10
11
 
# File 'lib/proxy_fetcher/manager.rb', line 9

def from_files(files, **options)
  new(**options.merge(files: Array(files)))
end

Instance Method Details

#cleanup! ⇒ Array<ProxyFetcher::Proxy> Also known as: validate!

Clean current proxy list from dead proxies (that doesn’t respond by timeout)

Returns:



129
130
131
132
 
# File 'lib/proxy_fetcher/manager.rb', line 129

def cleanup!
  valid_proxies = ProxyListValidator.new(@proxies).validate
  @proxies &= valid_proxies
end

#get ⇒ ProxyFetcher::Proxy, NilClass Also known as: pop

Pop just first proxy (and back it to the end of the proxy list).

Returns:



74
75
76
77
78
79
80
81
 
# File 'lib/proxy_fetcher/manager.rb', line 74

def get
  return if @proxies.empty?

  first_proxy = @proxies.shift
  @proxies << first_proxy

  first_proxy
end

#get! ⇒ ProxyFetcher::Proxy, NilClass Also known as: pop!

Pop first valid proxy (and back it to the end of the proxy list) Invalid proxies will be removed from the list

Returns:



91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/proxy_fetcher/manager.rb', line 91

def get!
  index = proxies.find_index(&:connectable?)
  return if index.nil?

  proxy = proxies.delete_at(index)
  tail = proxies[index..-1]

  @proxies = tail << proxy

  proxy
end

#inspect ⇒ Object 



157
158
159
 
# File 'lib/proxy_fetcher/manager.rb', line 157

def inspect
  to_s
end

#load_proxies_from_files!(proxy_files) ⇒ Object

Loads proxies from files.

Parameters:

  • proxy_files (String, Array<String,Pathname>) —

    file path of list of files to load



110
111
112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/proxy_fetcher/manager.rb', line 110

def load_proxies_from_files!(proxy_files)
  proxy_files = Array(proxy_files)
  return if proxy_files.empty?

  proxy_files.each do |proxy_file|
    File.foreach(proxy_file, chomp: true) do |proxy_string|
      addr, port = proxy_string.split(":", 2)
      port = Integer(port) if port
      @proxies << Proxy.new(addr: addr, port: port)
    end
  end

  @proxies.uniq!
end

#random_proxy ⇒ Proxy Also known as: random

Returns random proxy

Returns:

  • (Proxy) —

    random proxy from the loaded list



141
142
143
 
# File 'lib/proxy_fetcher/manager.rb', line 141

def random_proxy
  proxies.sample
end

#raw_proxies ⇒ Array<String>

Returns array of proxy URLs (just schema + host + port)

Returns:

  • (Array<String>) —

    collection of proxies



152
153
154
 
# File 'lib/proxy_fetcher/manager.rb', line 152

def raw_proxies
  proxies.map(&:url)
end

#refresh_list!(filters = nil) ⇒ Object Also known as: fetch!

Update current proxy list using configured providers.

Parameters:

  • filters (Hash) (defaults to: nil) —

    providers filters



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'lib/proxy_fetcher/manager.rb', line 44

def refresh_list!(filters = nil)
  @proxies = []
  threads = []

  ProxyFetcher.config.providers.each do |provider_name|
    threads << Thread.new do
      Thread.current.report_on_exception = false

      provider = ProxyFetcher::Configuration.providers_registry.class_for(provider_name)
      provider_filters = filters && filters.fetch(provider_name.to_sym, filters)
      provider_proxies = provider.fetch_proxies!(provider_filters)

      REFRESHER_LOCK.synchronize do
        @proxies.concat(provider_proxies)
      end
    end
  end

  threads.each(&:join)

  @proxies
end