Class: Mechanize::Page

Inherits:
File
  • Object
show all
Extended by:
Forwardable, ElementMatcher
Defined in:
lib/mechanize/page.rb

Overview

This class encapsulates an HTML page. If Mechanize finds a content type of ‘text/html’, this class will be instantiated and returned.

Example:

require 'mechanize'

agent = Mechanize.new
agent.get('http://google.com/').class # => Mechanize::Page

Defined Under Namespace

Classes: Base, Frame, Image, Label, Link, MetaRefresh

Constant Summary collapse

DEFAULT_RESPONSE = 
{
  'content-type' => 'text/html',
}.freeze

Constants included from Parser

Mechanize::Parser::SPECIAL_FILENAMES

Instance Attribute Summary collapse

Attributes inherited from File

#body, #filename

Attributes included from Parser

#code, #response, #uri

Class Method Summary collapse

Instance Method Summary collapse

Methods included from ElementMatcher

elements_with

Methods inherited from File

#save, #save!

Methods included from Parser

#extract_filename, #fill_header, #find_free_name

Constructor Details

#initialize(uri = nil, response = nil, body = nil, code = nil, mech = nil) ⇒ Page

Returns a new instance of Page.



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/mechanize/page.rb', line 27

def initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil)
  response ||= DEFAULT_RESPONSE

  @meta_content_type = nil
  @encoding = nil
  @encodings = [nil]
  raise 'no' if mech and not Mechanize === mech
  @mech = mech

  reset

  @encodings << Mechanize::Util.detect_charset(body) if body

  @encodings.concat self.class.response_header_charset(response)

  if body
    # Force the encoding to be 8BIT so we can perform regular expressions.
    # We'll set it to the detected encoding later
    body.force_encoding 'ASCII-8BIT' if body.respond_to? :force_encoding

    @encodings.concat self.class.meta_charset body

    meta_content_type = self.class.meta_content_type body
    @meta_content_type = meta_content_type if meta_content_type
  end

  @encodings << mech.default_encoding if mech and mech.default_encoding

  super uri, response, body, code
end

Instance Attribute Details

#encodings ⇒ Object (readonly)

Possible encodings for this page based on HTTP headers and meta elements



25
26
27
 
# File 'lib/mechanize/page.rb', line 25

def encodings
  @encodings
end

#mech ⇒ Object

Returns the value of attribute mech.



20
21
22
 
# File 'lib/mechanize/page.rb', line 20

def mech
  @mech
end

Class Method Details

.charset(content_type) ⇒ Object Also known as: charset_from_content_type 



468
469
470
471
472
 
# File 'lib/mechanize/page.rb', line 468

def charset content_type
  charset = content_type[/;(?:\s*,)?\s*charset\s*=\s*([^()<>@,;:\\\"\/\[\]?={}\s]+)/i, 1]
  return nil if charset == 'none'
  charset
end

.meta_charset(body) ⇒ Object

Retrieves all charsets from meta tags in body



490
491
492
493
494
495
496
497
498
499
500
501
502
503
 
# File 'lib/mechanize/page.rb', line 490

def self.meta_charset body
  # HACK use .map
  body.scan(/<meta .*?>/i).map do |meta|
    if meta =~ /charset\s*=\s*(["'])?\s*(.+)\s*\1/i then
      $2
    elsif meta =~ /http-equiv\s*=\s*(["'])?content-type\1/i then
      meta =~ /content\s*=\s*(["'])?(.*?)\1/i

      m_charset = charset $2 if $2

      m_charset if m_charset
    end
  end.compact
end

.meta_content_type(body) ⇒ Object

Retrieves the last content-type set by a meta tag in body



508
509
510
511
512
513
514
515
516
517
518
 
# File 'lib/mechanize/page.rb', line 508

def self.meta_content_type body
  body.scan(/<meta .*?>/i).reverse.map do |meta|
    if meta =~ /http-equiv\s*=\s*(["'])?content-type\1/i then
      meta =~ /content=(["'])?(.*?)\1/i

      return $2
    end
  end

  nil
end

.response_header_charset(response) ⇒ Object 



477
478
479
480
481
482
483
484
485
 
# File 'lib/mechanize/page.rb', line 477

def self.response_header_charset response
  charsets = []
  response.each do |header, value|
    next unless header == 'content-type'
    next unless value =~ /charset/i
    charsets << charset(value)
  end
  charsets
end

Instance Method Details

#base ⇒ Object

:method: bases_with

:call-seq: bases_with(criteria)

Find all base tags matching criteria. Example:

page.bases_with(:href => /foo/).each do |base|
  puts base.href
end



300
 
# File 'lib/mechanize/page.rb', line 300

elements_with :base

#bases ⇒ Object

Return a list of all base tags



419
420
421
422
 
# File 'lib/mechanize/page.rb', line 419

def bases
  @bases ||=
    search('base').map { |node| Base.new(node, @mech, self) }
end

#canonical_uri ⇒ Object

Return the canonical URI for the page if there is a link tag with href=“canonical”.



177
178
179
180
181
182
183
184
185
 
# File 'lib/mechanize/page.rb', line 177

def canonical_uri
  link = at('link[@rel="canonical"][@href]')
  return unless link
  href = link['href']

  URI href
rescue URI::InvalidURIError
  URI Mechanize::Util.uri_escape href
end

#content_type ⇒ Object

Get the content type



188
189
190
 
# File 'lib/mechanize/page.rb', line 188

def content_type
  @meta_content_type || response['content-type']
end

#detected_encoding ⇒ Object 



74
75
76
 
# File 'lib/mechanize/page.rb', line 74

def detected_encoding
  Mechanize::Util.detect_charset(body)
end

#encoding ⇒ Object 



94
95
96
 
# File 'lib/mechanize/page.rb', line 94

def encoding
  parser.respond_to?(:encoding) ? parser.encoding : nil
end

#encoding=(encoding) ⇒ Object 



78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/mechanize/page.rb', line 78

def encoding=(encoding)
  reset

  @encoding = encoding

  if @parser
    parser_encoding = @parser.encoding
    if parser_encoding && encoding && parser_encoding.casecmp(encoding) != 0
      # lazy reinitialize the parser with the new encoding
      @parser = nil
    end
  end

  encoding
end

#encoding_error?(parser = nil) ⇒ Boolean

Return whether parser result has errors related to encoding or not. false indicates just parser has no encoding errors, not encoding is vaild.

Returns:

  • (Boolean) 


100
101
102
103
104
105
106
107
108
 
# File 'lib/mechanize/page.rb', line 100

def encoding_error?(parser=nil)
  parser = self.parser unless parser
  return false if parser.errors.empty?
  parser.errors.any? do |error|
    error.message =~ /(indicate\ encoding)|
                      (Invalid\ char)|
                      (input\ conversion\ failed)/x
  end
end

#form ⇒ Object

:method: forms_with

:call-seq: forms_with(criteria)

Find all forms form matching criteria. Example:

page.forms_with(:action => '/post/login.php').each do |f|
  ...
end



243
 
# File 'lib/mechanize/page.rb', line 243

elements_with :form

#forms ⇒ Object

Return a list of all form tags



398
399
400
401
402
403
404
 
# File 'lib/mechanize/page.rb', line 398

def forms
  @forms ||= search('form').map do |html_form|
    form = Mechanize::Form.new(html_form, @mech, self)
    form.action ||= @uri.to_s
    form
  end
end

#frame ⇒ Object

:method: frames_with

:call-seq: frames_with(criteria)

Find all frame tags matching criteria. Example:

page.frames_with(:src => /foo/).each do |frame|
  p frame.src
end



328
 
# File 'lib/mechanize/page.rb', line 328

elements_with :frame

#frames ⇒ Object

Return a list of all frame tags



426
427
428
429
 
# File 'lib/mechanize/page.rb', line 426

def frames
  @frames ||=
    search('frame').map { |node| Frame.new(node, @mech, self) }
end

#iframe ⇒ Object

:method: iframes_with

:call-seq: iframes_with(criteria)

Find all iframe tags matching criteria. Example:

page.iframes_with(:src => /foo/).each do |iframe|
  p iframe.src
end



356
 
# File 'lib/mechanize/page.rb', line 356

elements_with :iframe

#iframes ⇒ Object

Return a list of all iframe tags



433
434
435
436
 
# File 'lib/mechanize/page.rb', line 433

def iframes
  @iframes ||=
    search('iframe').map { |node| Frame.new(node, @mech, self) }
end

#image ⇒ Object

:method: images_with

:call-seq: images_with(criteria)

Find all images matching criteria. Example:

page.images_with(:src => /jpg\Z/).each do |img|
  img.fetch.save
end



384
 
# File 'lib/mechanize/page.rb', line 384

elements_with :image

#image_urls ⇒ Object 



445
446
447
 
# File 'lib/mechanize/page.rb', line 445

def image_urls
  @image_urls ||= images.map(&:url).uniq
end

#images ⇒ Object

Return a list of all img tags



440
441
442
443
 
# File 'lib/mechanize/page.rb', line 440

def images
  @images ||=
    search('img').map { |node| Image.new(node, self) }
end

#labels ⇒ Object

Return a list of all label tags



451
452
453
454
 
# File 'lib/mechanize/page.rb', line 451

def labels
  @labels ||=
    search('label').map { |node| Label.new(node, self) }
end

#labels_hash ⇒ Object 



456
457
458
459
460
461
462
463
464
465
 
# File 'lib/mechanize/page.rb', line 456

def labels_hash
  unless @labels_hash
    hash = {}
    labels.each do |label|
      hash[label.node['for']] = label if label.for
    end
    @labels_hash = hash
  end
  return @labels_hash
end

#link ⇒ Object

:method: links_with

:call-seq:

links_with(criteria)

Find all links matching criteria. Example:

page.links_with(:href => /foo/).each do |link|
  puts link.href
end



272
 
# File 'lib/mechanize/page.rb', line 272

elements_with :link

#links ⇒ Object

Return a list of all link and area tags



388
389
390
391
392
393
394
 
# File 'lib/mechanize/page.rb', line 388

def links
  @links ||= %w{ a area }.map do |tag|
    search(tag).map do |node|
      Link.new(node, @mech, self)
    end
  end.flatten
end

#meta_charset ⇒ Object 



70
71
72
 
# File 'lib/mechanize/page.rb', line 70

def meta_charset
  self.class.meta_charset(body)
end

#meta_refresh ⇒ Object

Return a list of all meta refresh elements



409
410
411
412
413
414
415
 
# File 'lib/mechanize/page.rb', line 409

def meta_refresh
  query = @mech.follow_meta_refresh == :anywhere ? 'meta' : 'head > meta'

  @meta_refresh ||= search(query).map do |node|
    MetaRefresh.from_node node, self
  end.compact
end

#parser ⇒ Object Also known as: root

:method: at

Search through the page for path under namespace using Nokogiri’s #at. The path may be either a CSS or XPath expression.

See also Nokogiri::XML::Node#at



200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/mechanize/page.rb', line 200

def parser
  return @parser if @parser
  return nil unless @body

  if @encoding then
    @parser = @mech.html_parser.parse html_body, nil, @encoding
  elsif mech.force_default_encoding then
    @parser = @mech.html_parser.parse html_body, nil, @mech.default_encoding
  else
    @encodings.reverse_each do |encoding|
      @parser = @mech.html_parser.parse html_body, nil, encoding

      break unless encoding_error? @parser
    end
  end

  @parser
end

#pretty_print(q) ⇒ Object

:nodoc:



131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'lib/mechanize/page.rb', line 131

def pretty_print(q) # :nodoc:
  q.object_group(self) {
    q.breakable
    q.group(1, '{url', '}') {q.breakable; q.pp uri }
    q.breakable
    q.group(1, '{meta_refresh', '}') {
      meta_refresh.each { |link| q.breakable; q.pp link }
    }
    q.breakable
    q.group(1, '{title', '}') { q.breakable; q.pp title }
    q.breakable
    q.group(1, '{iframes', '}') {
      iframes.each { |link| q.breakable; q.pp link }
    }
    q.breakable
    q.group(1, '{frames', '}') {
      frames.each { |link| q.breakable; q.pp link }
    }
    q.breakable
    q.group(1, '{links', '}') {
      links.each { |link| q.breakable; q.pp link }
    }
    q.breakable
    q.group(1, '{forms', '}') {
      forms.each { |form| q.breakable; q.pp form }
    }
  }
end

#reset ⇒ Object 



162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/mechanize/page.rb', line 162

def reset
  @bases = nil
  @forms = nil
  @frames = nil
  @iframes = nil
  @links = nil
  @labels = nil
  @labels_hash = nil
  @meta_refresh = nil
  @parser = nil
  @title = nil
end

#response_header_charset ⇒ Object 



66
67
68
 
# File 'lib/mechanize/page.rb', line 66

def response_header_charset
  self.class.response_header_charset(response)
end

#title ⇒ Object 



58
59
60
61
62
63
64
 
# File 'lib/mechanize/page.rb', line 58

def title
  @title ||=
    if doc = parser
      title = doc.search('title').inner_text
      title.empty? ? nil : title
    end
end