Class: Proxy::ScraperApi

Inherits:
Base
  • Object
show all
Defined in:
lib/proxy/scraper_api.rb

Instance Method Summary collapse

Methods inherited from Base

#format_response, #get_headers, #initialize, #prepare_request, #with_retry

Constructor Details

This class inherits a constructor from Proxy::Base

Instance Method Details

#fetch(url, proxy_from_server) ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/proxy/scraper_api.rb', line 7

def fetch(url, proxy_from_server)
  uri = URI.parse(proxy_from_server['proxyHost'])

  uri.query = URI.encode_www_form({
    'api_key' => proxy_from_server['proxyToken'],
    'render' => @enable_js.to_s,
    proxy_from_server['proxyCountryKey'] => @country,
    'url' => url
  }.merge(@query_params))

  request = Net::HTTP::Get.new(uri, @headers)
  http = prepare_request(uri)

  @cookies.each do |cookie|
    request.add_field('Cookie', "#{cookie[:name]}=#{cookie[:value]}")
  end

  response = with_retry do
    http.request(request)
  end

  format_response(response)
rescue StandardError => e
  @logger.error("Request failed error: #{e.message}")
  nil
end