Class: ActiveScraper::Request

Inherits:
ActiveRecord::Base
  • Object
show all
Defined in:
app/models/active_scraper/request.rb

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.build_from_uri(uri, opts = {}) ⇒ Object 



56
57
58
59
60
61
 
# File 'app/models/active_scraper/request.rb', line 56

def self.build_from_uri(uri, opts={})
  request_params = build_request_params(uri, opts)
  request_obj = Request.new(request_params)

  return request_obj
end

.build_request_params(uri, opts = {}) ⇒ Object

Returns a Hash with symbolized keys



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'app/models/active_scraper/request.rb', line 31

def self.build_request_params(uri, opts={})
  u = Addressable::URI.parse(uri)
  hsh = {scheme: u.normalized_scheme, host: u.normalized_host, path: u.normalized_path, query: u.normalized_query, extname: u.extname}

  if ob_keys = opts.delete(:obfuscate_query)
    Array(ob_keys).each do |key|
      a = Array(key)

      key_to_omit = Regexp.escape(a[0].to_s)
      char_num = a[1] || 0
      
      if val_to_omit = hsh[:query].match(/(?<=#{key_to_omit}=)(.*?)(?=&|$)/)
        val = val_to_omit[1]
        hsh[:query].sub!( val, "__OMIT__#{val[-char_num, char_num]}")
      end
    end

    hsh[:is_obfuscated] = true
  else
    hsh[:is_obfuscated] = false
  end

  return hsh
end

.build_validating_params(uri, opts = {}) ⇒ Object 



24
25
26
27
28
 
# File 'app/models/active_scraper/request.rb', line 24

def self.build_validating_params(uri, opts={})
  h = build_request_params(uri, opts)

  h.slice(:scheme, :host, :path, :query)
end

.create_and_fetch_response(uri, opts = {}, fetcher = nil) ⇒ Object 



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'app/models/active_scraper/request.rb', line 76

def self.create_and_fetch_response(uri, opts={}, fetcher = nil)
  request = find_or_build_from_uri(uri, opts)
  fetcher = fetcher || Fetcher.new

  if request.id.nil? 
    # this request is new
    # so skip to the fresh
    resp = fetcher.fetch request, fresh: true 
  else 
    # will check the cache and the fresh
    resp = fetcher.fetch request
  end

  # build the response
  response = request.responses.build(resp)
  # theoretically, response will be saved too
  request.save

  return request
end

.create_from_uri(uri, opts = {}) ⇒ Object 



68
69
70
71
72
73
 
# File 'app/models/active_scraper/request.rb', line 68

def self.create_from_uri(uri, opts={})
  req = build_from_uri(uri, opts)
  req.save

  return req
end

.find_or_build_from_uri(uri, opts = {}) ⇒ Object 



63
64
65
 
# File 'app/models/active_scraper/request.rb', line 63

def self.find_or_build_from_uri(uri, opts={})
  self.with_url(uri).first || self.build_from_uri(uri, opts)
end

Instance Method Details

#obfuscated? ⇒ Boolean

Returns:

  • (Boolean) 


14
15
16
 
# File 'app/models/active_scraper/request.rb', line 14

def obfuscated?
  is_obfuscated == true
end

#uri ⇒ Object 



18
19
20
21
22
 
# File 'app/models/active_scraper/request.rb', line 18

def uri
  Addressable::URI.new(
    self.attributes.symbolize_keys.slice(:scheme, :host, :path, :query)
  )
end