Class: ActiveScraper::CachedRequest

Inherits:
ActiveRecord::Base
  • Object
show all
Defined in:
app/models/active_scraper/cached_request.rb

Constant Summary collapse

QUERY_NORMALIZER = 
HTTParty::Request::NON_RAILS_QUERY_STRING_NORMALIZER

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#unobfuscated_query ⇒ Object

Returns the value of attribute unobfuscated_query.



11
12
13
 
# File 'app/models/active_scraper/cached_request.rb', line 11

def unobfuscated_query
  @unobfuscated_query
end

Class Method Details

.build_from_uri(uri, opts = {}) ⇒ Object 



106
107
108
109
110
111
 
# File 'app/models/active_scraper/cached_request.rb', line 106

def self.build_from_uri(uri, opts={})
  request_params = build_request_params(uri, opts)
  request_obj = CachedRequest.new(request_params)

  return request_obj
end

.build_request_params(uri, opts = {}) ⇒ Object

Returns a Hash with symbolized keys



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'app/models/active_scraper/cached_request.rb', line 87

def self.build_request_params(uri, opts={})
  u = Addressable::URI.parse(uri)
  hsh = {scheme: u.normalized_scheme, host: u.normalized_host, path: u.normalized_path, query: u.normalized_query , extname: u.extname}
  # deal with query separately
  unless opts[:normalize_query] == false
    hsh[:query] = normalize_query_params(hsh[:query])        
  end

  hsh[:unobfuscated_query] = hsh[:query]
  if ob_keys = opts[:obfuscate_query]
    hsh[:query] = obfuscate_query_params(hsh[:query], ob_keys)
    hsh[:is_obfuscated] = true
  else
    hsh[:is_obfuscated] = false
  end

  return hsh
end

.build_validating_params(uri, opts = {}) ⇒ Object 



75
76
77
78
79
 
# File 'app/models/active_scraper/cached_request.rb', line 75

def self.build_validating_params(uri, opts={})
  h = build_request_params(uri, opts)

  h.slice(:scheme, :host, :path, :query)
end

.create_from_uri(uri, opts = {}) ⇒ Object 



118
119
120
121
122
123
 
# File 'app/models/active_scraper/cached_request.rb', line 118

def self.create_from_uri(uri, opts={})
  req = build_from_uri(uri, opts)
  req.save

  return req
end

.find_or_build_from_uri(uri, opts = {}) ⇒ Object 



113
114
115
 
# File 'app/models/active_scraper/cached_request.rb', line 113

def self.find_or_build_from_uri(uri, opts={})
  self.matching_request(uri, opts).first || self.build_from_uri(uri, opts)
end

.normalize_query_params(q) ⇒ Object

:q is a query String or Hash e.g. ‘z=hello&b=world&a=dog’

or: {z: ['hello', 'world'], a: 'dog'}

returns: (String) “a=dog&z=hello&z=world”



133
134
135
136
137
138
139
140
 
# File 'app/models/active_scraper/cached_request.rb', line 133

def self.normalize_query_params(q)
  return q if q.blank?

  params_hash = CGI.parse(q)
  params_str = QUERY_NORMALIZER[params_hash]

  return params_str
end

Instance Method Details

#latest_response_fetched_after(time) ⇒ Object 



35
36
37
38
39
40
41
 
# File 'app/models/active_scraper/cached_request.rb', line 35

def latest_response_fetched_after(time)
  if latest_response.present?
    return latest_response if latest_response.created_at > time
  end
  
  nil      
end

#obfuscated? ⇒ Boolean

Returns:

  • (Boolean) 


54
55
56
 
# File 'app/models/active_scraper/cached_request.rb', line 54

def obfuscated?
  is_obfuscated == true
end

#standard_uri ⇒ Object

to follow HTTParty conventions



59
60
61
 
# File 'app/models/active_scraper/cached_request.rb', line 59

def standard_uri
  URI.parse(uri)
end

#to_fake_party_hash ⇒ Object 



43
44
45
46
47
48
49
50
 
# File 'app/models/active_scraper/cached_request.rb', line 43

def to_fake_party_hash
  h = Hashie::Mash.new(self.attributes.symbolize_keys.slice(:scheme, :host, :path, :query))
  h[:uri] = self.standard_uri
  h[:options] ||= {}
  h[:headers] ||= {}

  return h
end

#to_uri ⇒ Object

during a fresh query, we need to actually use the unobfuscated_query



68
69
70
71
72
73
 
# File 'app/models/active_scraper/cached_request.rb', line 68

def to_uri
  h = self.attributes.symbolize_keys.slice(:scheme, :host, :path)
  h[:query] = self.unobfuscated_query || self.query

  return Addressable::URI.new(h)
end

#uri ⇒ Object 



63
64
65
 
# File 'app/models/active_scraper/cached_request.rb', line 63

def uri
  to_uri
end