Module: NewsScraper::ExtractorsHelpers

Included in:
NewsScraper::Extractors::Article, NewsScraper::Extractors::GoogleNewsRss
Defined in:
lib/news_scraper/extractors_helpers.rb

Instance Method Summary collapse

Instance Method Details

#http_request(url) ⇒ Object

Perform an HTTP request with a standardized response

Params

  • url: the url on which to perform a get request

Raises:



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/news_scraper/extractors_helpers.rb', line 8

def http_request(url)
  url = URIParser.new(url).with_scheme

  CLI.put_header(url)
  CLI.log "Beginning HTTP request for #{url}"
  response = HTTParty.get(url, headers: { "User-Agent" => "news-scraper-#{NewsScraper::VERSION}" })

  raise ResponseError.new(
    error_code: response.code,
    message: response.message,
    url: url
  ) unless response.code == 200

  CLI.log "#{response.code} - #{response.message}. Request successful for #{url}"
  CLI.put_footer

  if block_given?
    yield response
  else
    response
  end
end