Class: UrlFetcher

Inherits:
Object
  • Object
show all
Defined in:
lib/fundamentus_data/util/url_fetcher.rb

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ UrlFetcher

Returns a new instance of UrlFetcher.



5
6
7
8
9
 
# File 'lib/fundamentus_data/util/url_fetcher.rb', line 5

def initialize(options = {})
  if options.has_key?(:verbose)
    @verbose = options[:verbose]
  end
end

Instance Method Details

#download(url) ⇒ Object 



23
24
25
26
27
 
# File 'lib/fundamentus_data/util/url_fetcher.rb', line 23

def download(url)
  URI.open(url, "User-Agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0") do |f|
    f.read
  end
end

#fetch(urls) ⇒ Object 



11
12
13
14
15
16
17
18
19
20
21
 
# File 'lib/fundamentus_data/util/url_fetcher.rb', line 11

def fetch(urls)
  responses = {}
  urls.each do |key, url|
    if @verbose
      puts "Downloading #{key} from #{url}"
    end
    responses[key] = download(url)
    sleep(0.5)
  end
  responses
end