Class: ProxyFetcher::Configuration

Inherits:
Object
  • Object
show all
Defined in:
lib/proxy_fetcher/configuration.rb

Overview

ProxyFetcher configuration. Stores all the options for dealing with HTTP requests, adapters, custom classes.

Constant Summary collapse

DEFAULT_USER_AGENT =

User-Agent string that will be used by the ProxyFetcher HTTP client (to send requests via proxy) and to fetch proxy lists from the sources.

Default is Google Chrome 60, but can be changed in ProxyFetcher.config.

"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 " \
"(KHTML, like Gecko) Chrome/60.0.3112 Safari/537.36"
DEFAULT_ADAPTER =

HTML parser adapter name.

Default is Nokogiri, but can be changed in ProxyFetcher.config.

:nokogiri

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ ProxyFetcher::Configuration

Initialize ProxyFetcher configuration with default options.



111
112
113
 
# File 'lib/proxy_fetcher/configuration.rb', line 111

def initialize
  reset!
end

Instance Attribute Details

#adapter ⇒ Object

Returns the value of attribute adapter.



43
44
45
 
# File 'lib/proxy_fetcher/configuration.rb', line 43

def adapter
  @adapter
end

#client_timeout ⇒ Integer Also known as: timeout

Returns HTTP request timeout (connect / open) for [ProxyFetcher::Client].

Returns:

  • (Integer) —

    HTTP request timeout (connect / open) for [ProxyFetcher::Client]



11
12
13
 
# File 'lib/proxy_fetcher/configuration.rb', line 11

def client_timeout
  @client_timeout
end

#http_client ⇒ Object

Returns the value of attribute http_client.



47
48
49
 
# File 'lib/proxy_fetcher/configuration.rb', line 47

def http_client
  @http_client
end

#logger ⇒ Logger

Returns Logger object.

Returns:

  • (Logger) —

    Logger object



39
40
41
 
# File 'lib/proxy_fetcher/configuration.rb', line 39

def logger
  @logger
end

#pool_size ⇒ Integer

Returns proxy validator pool size (max number of threads).

Returns:

  • (Integer) —

    proxy validator pool size (max number of threads)



31
32
33
 
# File 'lib/proxy_fetcher/configuration.rb', line 31

def pool_size
  @pool_size
end

#provider_proxies_load_timeout ⇒ Integer

Returns HTTP request timeout (connect / open) for loading of proxies list by provider.

Returns:

  • (Integer) —

    HTTP request timeout (connect / open) for loading of proxies list by provider



17
18
19
 
# File 'lib/proxy_fetcher/configuration.rb', line 17

def provider_proxies_load_timeout
  @provider_proxies_load_timeout
end

#providers ⇒ Object Also known as: provider

Returns the value of attribute providers.



55
56
57
 
# File 'lib/proxy_fetcher/configuration.rb', line 55

def providers
  @providers
end

#proxy_validation_timeout ⇒ Integer

Returns HTTP request timeout (connect / open) for proxy validation with [ProxyFetcher::ProxyValidator].

Returns:

  • (Integer) —

    HTTP request timeout (connect / open) for proxy validation with [ProxyFetcher::ProxyValidator]



23
24
25
 
# File 'lib/proxy_fetcher/configuration.rb', line 23

def proxy_validation_timeout
  @proxy_validation_timeout
end

#proxy_validator ⇒ Object

Returns the value of attribute proxy_validator.



51
52
53
 
# File 'lib/proxy_fetcher/configuration.rb', line 51

def proxy_validator
  @proxy_validator
end

#user_agent ⇒ String

Returns User-Agent string.

Returns:

  • (String) —

    User-Agent string



35
36
37
 
# File 'lib/proxy_fetcher/configuration.rb', line 35

def user_agent
  @user_agent
end

Class Method Details

.providers_registry ⇒ ProxyFetcher::ProvidersRegistry

Registry for handling proxy providers.

Returns:



79
80
81
 
# File 'lib/proxy_fetcher/configuration.rb', line 79

def providers_registry
  @providers_registry ||= ProvidersRegistry.new
end

.register_provider(name, klass) ⇒ Object

Register new proxy provider. Requires provider name and class that will process proxy list.

Parameters:

  • name (String, Symbol) —

    name of the provider

  • klass (Class) —

    Class that will fetch and process proxy list



92
93
94
 
# File 'lib/proxy_fetcher/configuration.rb', line 92

def register_provider(name, klass)
  providers_registry.register(name, klass)
end

.registered_providers ⇒ Array<String, Symbol>

Returns registered providers names.

Returns:

  • (Array<String, Symbol>) —

    registered providers names



101
102
103
 
# File 'lib/proxy_fetcher/configuration.rb', line 101

def registered_providers
  providers_registry.providers.keys
end

Instance Method Details

#adapter_class ⇒ Object 



135
136
137
138
139
140
141
142
143
 
# File 'lib/proxy_fetcher/configuration.rb', line 135

def adapter_class
  self.class.instance_variable_get(:@__adapter_lock__).synchronize do
    return @adapter_class if defined?(@adapter_class)

    @adapter_class = ProxyFetcher::Document::Adapters.lookup(adapter)
    @adapter_class.setup!
    @adapter_class
  end
end

#reset! ⇒ Object

Sets default configuration options



116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/proxy_fetcher/configuration.rb', line 116

def reset!
  @logger = Logger.new($stdout)
  @user_agent = DEFAULT_USER_AGENT
  @pool_size = 10
  @client_timeout = 3
  @provider_proxies_load_timeout = 30
  @proxy_validation_timeout = 3

  @http_client = HTTPClient
  @proxy_validator = ProxyValidator

  self.providers = self.class.registered_providers
end