Class: Popularity::Crawler

Inherits:
Object
  • Object
show all
Defined in:
lib/popularity/crawler.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(url) ⇒ Crawler

Returns a new instance of Crawler.



26
27
28
 
# File 'lib/popularity/crawler.rb', line 26

def initialize(url)
  @url = url
end

Instance Attribute Details

#url ⇒ Object (readonly)

Returns the value of attribute url.



9
10
11
 
# File 'lib/popularity/crawler.rb', line 9

def url
  @url
end

Class Method Details

.property_names ⇒ Object 



18
19
20
 
# File 'lib/popularity/crawler.rb', line 18

def self.property_names
  @property_names
end

.stats(*args) ⇒ Object 



11
12
13
14
15
16
 
# File 'lib/popularity/crawler.rb', line 11

def self.stats(*args)
  @property_names ||= []
  args.each do |name|
    @property_names << name
  end
end

Instance Method Details

#as_json(options = {}) ⇒ Object 



67
68
69
70
71
72
73
74
75
76
 
# File 'lib/popularity/crawler.rb', line 67

def as_json(options = {})
  json = {}

  self.class.property_names.each do |name|
    json[name.to_s] = self.send(name.to_sym)
  end

  json["total"] = total
  json
end

#async_done? ⇒ Boolean

Returns:

  • (Boolean) 


51
52
53
 
# File 'lib/popularity/crawler.rb', line 51

def async_done?
  @async_done
end

#fetch ⇒ Object 



88
89
90
91
92
93
94
95
96
97
 
# File 'lib/popularity/crawler.rb', line 88

def fetch
  return false unless valid?

  begin
    response = Unirest.get(request_url)
    @response = response.raw_body
  rescue OpenURI::HTTPError, Timeout::Error, SocketError
    nil
  end
end

#fetch_async(&block) ⇒ Object 



78
79
80
81
82
83
84
85
86
 
# File 'lib/popularity/crawler.rb', line 78

def fetch_async(&block)
  return false unless valid?

  Unirest.get(request_url) do |response|
    @async_done = true
    @response = response.raw_body
    block.call(response.code, response.raw_body) if block_given?
  end
end

#has_response? ⇒ Boolean

Returns:

  • (Boolean) 


38
39
40
41
42
43
44
45
 
# File 'lib/popularity/crawler.rb', line 38

def has_response?
  response #fetch it

  return false if response.nil?
  return false if response.empty?

  true
end

#host ⇒ Object 



55
56
57
 
# File 'lib/popularity/crawler.rb', line 55

def host
  URI.parse(@url).host.gsub('www.', '')
end

#name ⇒ Object 



63
64
65
 
# File 'lib/popularity/crawler.rb', line 63

def name
  self.class.to_s.split('::').last.gsub(/(.)([A-Z])/,'\1_\2').downcase
end

#request_url ⇒ Object

Raises:

  • (NotImplemented) 


30
31
32
 
# File 'lib/popularity/crawler.rb', line 30

def request_url
  raise NotImplemented
end

#response ⇒ Object 



34
35
36
 
# File 'lib/popularity/crawler.rb', line 34

def response
  @response ||= fetch
end

#response_json ⇒ Object 



59
60
61
 
# File 'lib/popularity/crawler.rb', line 59

def response_json
  @json ||= JSON.parse(response)
end

#total ⇒ Object 



22
23
24
 
# File 'lib/popularity/crawler.rb', line 22

def total
  self.class.property_names.uniq.collect { |n| self.send(n.to_sym) }.select { |t| t.class == Fixnum }.compact.reduce(:+)
end

#valid? ⇒ Boolean

Returns:

  • (Boolean) 


47
48
49
 
# File 'lib/popularity/crawler.rb', line 47

def valid?
  true # to be overridden in subclasses
end