Class: ProxyFetcher::HTTPClient

Inherits:
Object
  • Object
show all
Defined in:
lib/proxy_fetcher/utils/http_client.rb

Overview

Default ProxyFetcher HTTP client used to fetch proxy lists from the different providers. Uses ProxyFetcher configuration options for sending HTTP requests to providers URLs.

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(url, method: :get, params: {}, headers: {}) ⇒ HTTPClient

Initialize HTTP client instance



52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 52

def initialize(url, method: :get, params: {}, headers: {})
  @url = url.to_s
  @method = method
  @params = params
  @headers = headers

  @http = HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout)
  @timeout = ProxyFetcher.config.provider_proxies_load_timeout

  @ssl_ctx = OpenSSL::SSL::SSLContext.new
  @ssl_ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
end

Instance Attribute Details

#headers ⇒ Object (readonly)

Returns the value of attribute headers.



22
23
24
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 22

def headers
  @headers
end

#http ⇒ Object (readonly)

Returns the value of attribute http.



26
27
28
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 26

def http
  @http
end

#HTTP headers(headers) ⇒ Hash (readonly)

Returns headers.

Returns:

  • (Hash) —

    headers



22
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 22

attr_reader :headers

#HTTP method(method) ⇒ String (readonly)

Returns HTTP method verb.

Returns:

  • (String) —

    HTTP method verb



14
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 14

attr_reader :method

#HTTP params(params) ⇒ Hash (readonly)

Returns params.

Returns:

  • (Hash) —

    params



18
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 18

attr_reader :params

#method ⇒ Object (readonly)

Returns the value of attribute method.



14
15
16
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 14

def method
  @method
end

#params ⇒ Object (readonly)

Returns the value of attribute params.



18
19
20
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 18

def params
  @params
end

#ssl_ctx ⇒ Object (readonly)

Returns the value of attribute ssl_ctx.



30
31
32
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 30

def ssl_ctx
  @ssl_ctx
end

#timeout ⇒ Object (readonly)

Returns the value of attribute timeout.



34
35
36
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 34

def timeout
  @timeout
end

#url ⇒ Object (readonly)

Returns the value of attribute url.



10
11
12
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 10

def url
  @url
end

Class Method Details

.fetch(*args) ⇒ String

Fetches resource content by sending HTTP request to it. Synthetic sugar to simplify URIes fetching.

Parameters:

  • url (String) —

    URL

Returns:

  • (String) —

    resource content



44
45
46
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 44

def self.fetch(*args)
  new(*args).fetch
end

Instance Method Details

#fetch ⇒ String

Fetches resource content by sending HTTP request to it.

Returns:

  • (String) —

    response body



70
71
72
73
74
75
76
 
# File 'lib/proxy_fetcher/utils/http_client.rb', line 70

def fetch
  response = process_http_request
  response.body.to_s
rescue StandardError => e
  ProxyFetcher.config.logger.warn("Failed to process request to #{url} (#{e.message})")
  ""
end