Class: Retriever::Page

Inherits:
Object
  • Object
show all
Defined in:
lib/retriever/page.rb

Constant Summary collapse

HREF_CONTENTS_RE = 
Regexp.new(/\shref=['|"]([^\s][a-z0-9\.\/\:\-\%\+\?\!\=\&\,\:\;\~\_]+)['|"][\s|\W]/ix).freeze
NONPAGE_EXT_RE = 
Regexp.new(/\.(?:css|js|png|gif|jpg|mp4|wmv|flv|mp3|wav|doc|txt|ico|xml)/ix).freeze
HTTP_RE = 
Regexp.new(/^http/i).freeze
DUB_DUB_DUB_DOT_RE = 
Regexp.new(/^www\./i).freeze
TITLE_RE = 
Regexp.new(/<title>(.*)<\/title>/i).freeze
DESC_RE = 
Regexp.new(/<meta[^>]*name=[\"|\']description[\"|\'][^>]*content=[\"]([^\"]*)[\"][^>]*>/i).freeze
H1_RE = 
Regexp.new(/<h1>(.*)<\/h1>/i).freeze
H2_RE = 
Regexp.new(/<h2>(.*)<\/h2>/i).freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source, t) ⇒ Page

Returns a new instance of Page.



16
17
18
19
20
 
# File 'lib/retriever/page.rb', line 16

def initialize(source, t)
  @t = t
  @source = source.encode('UTF-8', :invalid => :replace, :undef => :replace)
  @links = nil
end

Instance Attribute Details

recieves page source as string returns array of unique href links



24
25
26
 
# File 'lib/retriever/page.rb', line 24

def links
  @links
end

#source ⇒ Object (readonly)

Returns the value of attribute source.



14
15
16
 
# File 'lib/retriever/page.rb', line 14

def source
  @source
end

#t ⇒ Object (readonly)

Returns the value of attribute t.



14
15
16
 
# File 'lib/retriever/page.rb', line 14

def t
  @t
end

Instance Method Details

#desc ⇒ Object 



51
52
53
 
# File 'lib/retriever/page.rb', line 51

def desc
  DESC_RE =~ @source ? @source.match(DESC_RE)[1] : ''
end

#h1 ⇒ Object 



55
56
57
 
# File 'lib/retriever/page.rb', line 55

def h1
  H1_RE =~ @source ? @source.match(H1_RE)[1] : ''
end

#h2 ⇒ Object 



59
60
61
 
# File 'lib/retriever/page.rb', line 59

def h2
  H2_RE =~ @source ? @source.match(H2_RE)[1] : ''
end

#parse_files ⇒ Object 



43
44
45
 
# File 'lib/retriever/page.rb', line 43

def parse_files
  links.select { |linky| (@t.file_re =~ linky) }
end

#parse_internal ⇒ Object 



35
36
37
 
# File 'lib/retriever/page.rb', line 35

def parse_internal
  links.select { |linky| (@t.host_re =~ linky) }
end

#parse_internal_visitable ⇒ Object 



39
40
41
 
# File 'lib/retriever/page.rb', line 39

def parse_internal_visitable
  parse_internal.select { |linky| (!(NONPAGE_EXT_RE =~ linky)) }
end

#parse_seo ⇒ Object 



63
64
65
 
# File 'lib/retriever/page.rb', line 63

def parse_seo
  [title, desc, h1, h2]
end

#title ⇒ Object 



47
48
49
 
# File 'lib/retriever/page.rb', line 47

def title
  TITLE_RE =~ @source ? @source.match(TITLE_RE)[1] : ''
end