Class: ActiveScraper::Fetcher

Inherits:
Object
  • Object
show all
Defined in:
lib/active_scraper/fetcher.rb

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.build_response_object(obj) ⇒ Object

returns an OpenStruct that Response can use



60
61
62
 
# File 'lib/active_scraper/fetcher.rb', line 60

def self.build_response_object(obj)
  return AgnosticResponseObject.new(obj)
end

Instance Method Details

#build_response_object(obj) ⇒ Object 



55
56
57
 
# File 'lib/active_scraper/fetcher.rb', line 55

def build_response_object(obj)
  self.class.build_response_object(obj)
end

#convert_uri_object(u) ⇒ Object

u can either be a Request object, a String, or Addressable::URI returns an Addressable::URI



45
46
47
48
49
50
51
52
53
 
# File 'lib/active_scraper/fetcher.rb', line 45

def convert_uri_object(u)
  if u.is_a?(ActiveScraper::Request)
    x = u.uri
  else
    x = Addressable::URI.parse(u)
  end

  return x
end

#fetch(u, opts = {}) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
 
# File 'lib/active_scraper/fetcher.rb', line 6

def fetch(u, opts={})
  url = convert_uri_object(u)
  force_fresh = opts.delete :fresh

  if force_fresh != true && (record = fetch_from_cache(url, opts))
    resp_obj = record
  else
    resp_obj = fetch_fresh(url, opts)
  end

  build_response_object(resp_obj)
end

#fetch_fresh(url, opts = {}) ⇒ Object 



20
21
22
23
24
25
26
27
 
# File 'lib/active_scraper/fetcher.rb', line 20

def fetch_fresh(url, opts={})
  opts = opts.stringify_keys

  url = url.to_s
  verb = opts.fetch('verb'){ 'get' }

  resp = HTTParty.send(verb, url)
end

#fetch_from_cache(uri, opts = {}) ⇒ Object

returns:

single ScrapeCache if a valid ActiveScraper::Request exists



33
34
35
 
# File 'lib/active_scraper/fetcher.rb', line 33

def fetch_from_cache(uri, opts={})

end

#has_cache?(uri, opts = {}) ⇒ Boolean

true or false if ActiveScraper::Request with these parameters exist

Returns:

  • (Boolean) 


38
39
40
 
# File 'lib/active_scraper/fetcher.rb', line 38

def has_cache?(uri, opts={})

end