Module: ActiveScraper

Defined in:
lib/active_scraper.rb,
lib/active_scraper/engine.rb,
lib/active_scraper/version.rb,
lib/active_scraper/response_object.rb,
app/models/active_scraper/cached_request.rb,
lib/active_scraper/response_object/basic.rb,
app/models/active_scraper/cached_response.rb,
lib/active_scraper/fake_http_party_response.rb,
app/helpers/active_scraper/application_helper.rb,
app/controllers/active_scraper/application_controller.rb

Defined Under Namespace

Modules: ApplicationHelper, ResponseObject Classes: ApplicationController, CachedRequest, CachedResponse, Engine, FakeHTTPartyResponse

Constant Summary collapse

VERSION = 
"0.0.2"

Class Method Summary collapse

Class Method Details

.build_usable_response(request, response) ⇒ Object

Returns an object compatible with HTTParty, i.e. an ActiveScraper::FakeHTTPartyResponse to be deprecated



63
64
65
 
# File 'lib/active_scraper.rb', line 63

def self.build_usable_response(request, response)
  ActiveScraper::FakeHTTPartyResponse.new(request, response)
end

.create_request_and_fetch_response(uri, opts = {}) ⇒ Object 



46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/active_scraper.rb', line 46

def self.create_request_and_fetch_response(uri, opts={})
  opts = normalize_hash(opts)
  # first, find or build the request
  request = find_or_build_request(uri, opts)
  # then find or build a matching response
  response = find_or_build_response(request, opts)
  # associate and save the two
  request.responses << response
  request.save
   
  obj = Hashie::Mash.new(request: request, response: response)
  
  return obj
end

.fetch_fresh(url, opts = {}) ⇒ Object 



69
70
71
72
73
 
# File 'lib/active_scraper.rb', line 69

def self.fetch_fresh(url, opts={})
   resp = HTTParty.get(url, opts)

   return ActiveScraper::ResponseObject.factory(resp)
end

.find_or_build_request(req, opts = {}) ⇒ Object

delegates to CachedRequest::find_or_build_from_uri

req (URI or String). If CachedRequest, is idempotent

returns a new or existing CachedRequest



23
24
25
 
# File 'lib/active_scraper.rb', line 23

def self.find_or_build_request(req, opts={})
  CachedRequest.find_or_build_from_uri(req, opts)
end

.find_or_build_response(cached_request, opts = {}) ⇒ Object

cached_request (CachedRequest) => the request to find a response for

returns a new or existing CachedResponse

Raises:

  • (ArgumentError) 


31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/active_scraper.rb', line 31

def self.find_or_build_response(cached_request, opts={})
  raise ArgumentError, "Only accepted CachedRequest, but was passed in a #{cached_request.class}" unless cached_request.is_a?(CachedRequest)
  opts = normalize_hash(opts)

  response = CachedResponse.find_cache_for_cached_request(cached_request, opts)

  if response.blank?
    fetched_obj = fetch_fresh(cached_request.uri, opts)
    response = CachedResponse.build_from_response_object(fetched_obj)
  end

  return response
end

.get(uri, options = {}) ⇒ Object

returns a ActiveScraper::CachedResponse



11
12
13
14
15
 
# File 'lib/active_scraper.rb', line 11

def self.get(uri, options={})
  o = create_request_and_fetch_response(uri, options)

  return o.response
end

.normalize_hash(hsh) ⇒ Object 



78
79
80
81
82
83
84
 
# File 'lib/active_scraper.rb', line 78

def self.normalize_hash(hsh)
  unless hsh.is_a?(HashWithIndifferentAccess)
    hsh = HashWithIndifferentAccess.new(hsh) 
  end

  return hsh
end