Class: Digger::Page

Inherits:
Object
  • Object
show all
Defined in:
lib/digger/page.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(url, params = {}) ⇒ Page

Create a new page



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/digger/page.rb', line 43

def initialize(url, params = {})
  @url = URI(url)
  @code = params[:code]
  @headers = params[:headers] || {}
  @headers['content-type'] ||= ['']
  @aliases = Array(params[:aka]).compact
  @referer = params[:referer]
  @depth = params[:depth] || 0
  @redirect_to = to_absolute(params[:redirect_to])
  @response_time = params[:response_time]
  @body = params[:body]
  @error = params[:error]
  @fetched = !params[:code].nil?
  @user_data = OpenStruct.new
  @domain_aliases = params[:domain_aliases] ||= []
  @storable = true
  @fetched_at = params[:fetched_at]
end

Instance Attribute Details

#aliases ⇒ Object

Returns the value of attribute aliases.



30
31
32
 
# File 'lib/digger/page.rb', line 30

def aliases
  @aliases
end

#body ⇒ Object (readonly)

The raw HTTP response body of the page



12
13
14
 
# File 'lib/digger/page.rb', line 12

def body
  @body
end

#code ⇒ Object

Integer response code of the page



20
21
22
 
# File 'lib/digger/page.rb', line 20

def code
  @code
end

#depth ⇒ Object

Depth of this page from the root of the crawl.



22
23
24
 
# File 'lib/digger/page.rb', line 22

def depth
  @depth
end

#domain_aliases ⇒ Object

Returns the value of attribute domain_aliases.



32
33
34
 
# File 'lib/digger/page.rb', line 32

def domain_aliases
  @domain_aliases
end

#error ⇒ Object (readonly)

Exception object, if one was raised during HTTP#fetch_page



18
19
20
 
# File 'lib/digger/page.rb', line 18

def error
  @error
end

#fetched_at ⇒ Object

Returns the value of attribute fetched_at.



38
39
40
 
# File 'lib/digger/page.rb', line 38

def fetched_at
  @fetched_at
end

#headers ⇒ Object (readonly)

Headers of the HTTP response



14
15
16
 
# File 'lib/digger/page.rb', line 14

def headers
  @headers
end

#redirect_to ⇒ Object (readonly)

URL of the page this one redirected to, if any



16
17
18
 
# File 'lib/digger/page.rb', line 16

def redirect_to
  @redirect_to
end

#referer ⇒ Object

URL of the page that brought us to this page



24
25
26
 
# File 'lib/digger/page.rb', line 24

def referer
  @referer
end

#response_time ⇒ Object

Response time of the request for this page in milliseconds



26
27
28
 
# File 'lib/digger/page.rb', line 26

def response_time
  @response_time
end

#storable ⇒ Object

Whether the current page should be stored Default: true



36
37
38
 
# File 'lib/digger/page.rb', line 36

def storable
  @storable
end

#url ⇒ Object (readonly)

Returns the value of attribute url.



10
11
12
 
# File 'lib/digger/page.rb', line 10

def url
  @url
end

#user_data ⇒ Object

OpenStruct it holds users defined data



28
29
30
 
# File 'lib/digger/page.rb', line 28

def user_data
  @user_data
end

Class Method Details

.from_hash(hash) ⇒ Object 



253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
 
# File 'lib/digger/page.rb', line 253

def self.from_hash(hash)
  page = new(URI(hash['url']))
  {
    '@headers'       => hash['headers'] && !hash['headers'].empty? ? Marshal.load(hash['headers']) : { 'content-type' => [''] },
    '@body'          => hash['body'],
    '@links'         => hash['links'] ? hash['links'].map { |link| URI(link) } : [],
    '@code'          => hash['code'].to_i,
    '@depth'         => hash['depth'].to_i,
    '@referer'       => hash['referer'],
    '@redirect_to'   => (hash['redirect_to'] && !hash['redirect_to'].empty?) ? URI(hash['redirect_to']) : nil,
    '@response_time' => hash['response_time'].to_i,
    '@fetched'       => hash['fetched'],
    '@user_data'     => hash['user_data'] ? OpenStruct.new(hash['user_data']) : nil,
    '@fetched_at'    => hash['fetched_at'],
    '@error'         => hash['error']
  }.each do |var, value|
    page.instance_variable_set(var, value)
  end
  page
end

.from_json(json) ⇒ Object 



274
275
276
277
 
# File 'lib/digger/page.rb', line 274

def self.from_json(json)
  hash = JSON.parse json
  from_hash hash
end

Instance Method Details

#base ⇒ Object

Base URI from the HTML doc head element http://www.w3.org/TR/html4/struct/links.html#edef-BASE



166
167
168
169
170
171
172
173
174
 
# File 'lib/digger/page.rb', line 166

def base
  @base = if doc
            href = doc.search('//head/base/@href')
            URI(href.to_s) unless href.nil? rescue nil
          end unless @base

  return nil if @base && @base.to_s.empty?
  @base
end

#content_type ⇒ Object

The content-type returned by the HTTP request for this page



126
127
128
 
# File 'lib/digger/page.rb', line 126

def content_type
  headers['content-type'].first
end

#discard_doc! ⇒ Object

Delete the Nokogiri document and response body to conserve memory



110
111
112
113
 
# File 'lib/digger/page.rb', line 110

def discard_doc!
  links # force parsing of page links before we trash the document
  @doc = @body = nil
end

#discard_links! ⇒ Object

Discard links, a next call of page.links will return an empty array



103
104
105
 
# File 'lib/digger/page.rb', line 103

def discard_links!
  @links = []
end

#doc ⇒ Object

Nokogiri document for the HTML body



87
88
89
90
91
92
93
94
95
96
 
# File 'lib/digger/page.rb', line 87

def doc
  # return @doc if @doc
  # @body ||= ''
  # @body = @body.encode('utf-8', 'binary', :invalid => :replace,
  #                      :undef => :replace, :replace => '')
  # @doc = Nokogiri::HTML(@body.toutf8, nil, 'utf-8') if @body && html?
  @doc ||= begin
    Nokogiri::HTML(body) if !body.nil? && html? rescue nil
  end
end

#expired?(ttl) ⇒ Boolean

Returns:

  • (Boolean) 


248
249
250
251
 
# File 'lib/digger/page.rb', line 248

def expired?(ttl)
  return false if fetched_at.nil?
  (Time.now.to_i - ttl) > fetched_at
end

#fetched? ⇒ Boolean

Was the page successfully fetched? true if the page was fetched with no error, false otherwise.

Returns:

  • (Boolean) 


119
120
121
 
# File 'lib/digger/page.rb', line 119

def fetched?
  @fetched
end

#html? ⇒ Boolean

Returns true if the page is a HTML document, returns false otherwise.

Returns:

  • (Boolean) 


134
135
136
 
# File 'lib/digger/page.rb', line 134

def html?
  content_type =~ %r{^(text/html|application/xhtml+xml)\b}
end

#in_domain?(uri) ⇒ Boolean

Returns true if uri is in the same domain as the page, returns false otherwise

Returns:

  • (Boolean) 


209
210
211
212
 
# File 'lib/digger/page.rb', line 209

def in_domain?(uri)
  @domain_aliases ||= []
  uri.host == @url.host || @domain_aliases.include?(uri.host)
end

#links ⇒ Object

Array of distinct A tag HREFs from the page



69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/digger/page.rb', line 69

def links
  unless @links.nil?
    @links = Set.new
    return [] unless doc

    doc.search('//a[@href]').each do |a|
      u = a['href']
      next if u.nil? || u.empty?
      abs = to_absolute(u) rescue next
      @links << abs if abs && in_domain?(abs)
    end
  end
  @links.to_a
end

#not_found? ⇒ Boolean

Returns true if the page was not found (returned 404 code), returns false otherwise.

Returns:

  • (Boolean) 


158
159
160
 
# File 'lib/digger/page.rb', line 158

def not_found?
  404 == @code
end

#redirect? ⇒ Boolean

Returns true if the page is a HTTP redirect, returns false otherwise.

Returns:

  • (Boolean) 


142
143
144
 
# File 'lib/digger/page.rb', line 142

def redirect?
  (300...400).include?(@code)
end

#storable? ⇒ Boolean

Returns true if page is marked as storeable false otherwise Default is true

Returns:

  • (Boolean) 


244
245
246
 
# File 'lib/digger/page.rb', line 244

def storable?
  @storable
end

#success? ⇒ Boolean

Returns true if the page is a HTTP success, returns false otherwise.

Returns:

  • (Boolean) 


150
151
152
 
# File 'lib/digger/page.rb', line 150

def success?
  (200..206).include?(@code)
end

#title ⇒ Object 



62
63
64
 
# File 'lib/digger/page.rb', line 62

def title
  doc.title if doc
end

#to_absolute(link) ⇒ Object

Converts relative URL link into an absolute URL based on the location of the page



180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
 
# File 'lib/digger/page.rb', line 180

def to_absolute(link)
  return nil if link.nil?

  # link = link.to_s.encode('utf-8', 'binary', :invalid => :replace, :undef => :replace, :replace => '')

  # remove anchor
  link =
    begin
      URI.encode(URI.decode(link.gsub(/#[a-zA-Z0-9_-]*$/, '')))
    rescue URI::Error
      return nil
    end

  relative = begin
               URI(link)
             rescue URI::Error
               return nil
             end
  absolute = base ? base.merge(relative) : @url.merge(relative)

  absolute.path = '/' if absolute.path.empty?

  absolute
end

#to_hash ⇒ Object 



214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
 
# File 'lib/digger/page.rb', line 214

def to_hash
  {
    'url'           => @url.to_s,
    'headers'       => Marshal.dump(@headers),
    'body'          => @body,
    'links'         => links.map(&:to_s),
    'code'          => @code,
    'depth'         => @depth,
    'referer'       => @referer.to_s,
    'redirect_to'   => @redirect_to.to_s,
    'response_time' => @response_time,
    'fetched'       => @fetched,
    'user_data'     => @user_data.nil? ? {} : @user_data.marshal_dump,
    'fetched_at'    => @fetched_at,
    'error'         => @error.to_s
  }
end

#to_json ⇒ Object 



232
233
234
235
236
237
 
# File 'lib/digger/page.rb', line 232

def to_json
  th = to_hash.dup
  th.each { |k, v| th.delete(k) if v.nil? || (v.respond_to?(:empty?) && v.empty?) }
  th.delete('headers') if content_type.empty?
  th.to_json
end