Class: Proxy::Base

Inherits:
Object
  • Object
show all
Defined in:
lib/proxy/base.rb

Direct Known Subclasses

BrightData, CrawlBase, ScraperApi

Instance Method Summary collapse

Constructor Details

#initialize(params = {}) ⇒ Base

Returns a new instance of Base.



12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/proxy/base.rb', line 12

def initialize(params = {})
  @country = params[:country]
  @headers = params[:headers] || {}
  @query_params = params[:query_params] || {}
  @cookies = params[:cookies] || []
  @timeout = params[:timeout] || 60
  @tls_verify = params.fetch(:tls_verify, true)
  @retry = params[:retry_attr] || {}
  @enable_js = params.fetch(:enable_js, false)
  @logger = Logger.new($stdout)
end

Instance Method Details

#format_response(response) ⇒ Object 



37
38
39
40
41
42
43
44
 
# File 'lib/proxy/base.rb', line 37

def format_response(response)
  Response.new(
    code: response.code.to_i,
    body: response.body,
    headers: response.to_hash,
    cookies: response.get_fields('set-cookie')
  )
end

#prepare_request(uri, proxy_uri = nil) ⇒ Object 



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/proxy/base.rb', line 46

def prepare_request(uri, proxy_uri = nil)
  http = if proxy_uri.nil?
           Net::HTTP.new(uri.host, uri.port)
         else
           Net::HTTP.new(uri.host, uri.port, proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password)
         end
  if uri.scheme == 'https'
    http.use_ssl = true
    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
  end

  http.read_timeout = @timeout
  http.open_timeout = @timeout
  http
end

#with_retry ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/proxy/base.rb', line 24

def with_retry
  attempts = 0
  begin
    attempts += 1
    yield
  rescue StandardError => e
    raise e unless attempts <= @retry[:count].to_i

    sleep(@retry[:wait_time] || 5)
    retry
  end
end