Class: Retriever::Page

Inherits:
Object
  • Object
show all
Defined in:
lib/retriever/page.rb

Constant Summary collapse

HREF_CONTENTS_RE = 
Regexp.new(/\shref=['|"]([^\s][a-z0-9\.\/\:\-\%\+\?\!\=\&\,\:\;\~\_]+)['|"][\s|\W]/ix).freeze
NONPAGE_EXT_RE = 
Regexp.new(/\.(?:css|js|png|gif|jpg|mp4|wmv|flv|mp3|wav|doc|txt|ico|xml)/ix).freeze
HTTP_RE = 
Regexp.new(/^http/i).freeze
DUB_DUB_DUB_DOT_RE = 
Regexp.new(/^www\./i).freeze
TITLE_RE = 
Regexp.new(/<title>(.*)<\/title>/i).freeze
DESC_RE = 
Regexp.new(/<meta[^>]*name=[\"|\']description[\"|\'][^>]*content=[\"]([^\"]*)[\"][^>]*>/i).freeze
H1_RE = 
Regexp.new(/<h1>(.*)<\/h1>/i).freeze
H2_RE = 
Regexp.new(/<h2>(.*)<\/h2>/i).freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source, t) ⇒ Page

Returns a new instance of Page.



17
18
19
20
21
 
# File 'lib/retriever/page.rb', line 17

def initialize(source,t)
  @t = t
  @source = source.encode('UTF-8', :invalid => :replace, :undef => :replace)
  @links = nil
end

Instance Attribute Details

recieves page source as string returns array of unique href links



25
26
27
 
# File 'lib/retriever/page.rb', line 25

def links
  @links
end

#source ⇒ Object (readonly)

Returns the value of attribute source.



15
16
17
 
# File 'lib/retriever/page.rb', line 15

def source
  @source
end

#t ⇒ Object (readonly)

Returns the value of attribute t.



15
16
17
 
# File 'lib/retriever/page.rb', line 15

def t
  @t
end

Instance Method Details

#desc ⇒ Object 



50
51
52
 
# File 'lib/retriever/page.rb', line 50

def desc
  DESC_RE =~ @source ? @source.match(DESC_RE)[1] : ""
end

#h1 ⇒ Object 



54
55
56
 
# File 'lib/retriever/page.rb', line 54

def h1
  H1_RE =~ @source ? @source.match(H1_RE)[1] : ""
end

#h2 ⇒ Object 



58
59
60
 
# File 'lib/retriever/page.rb', line 58

def h2
  H2_RE =~ @source ? @source.match(H2_RE)[1] : ""
end

#parseFiles ⇒ Object 



42
43
44
 
# File 'lib/retriever/page.rb', line 42

def parseFiles
  links.select{ |linky| (@t.file_re =~ linky)}
end

#parseInternal ⇒ Object 



34
35
36
 
# File 'lib/retriever/page.rb', line 34

def parseInternal
  links.select{ |linky| (@t.host_re =~ linky) }
end

#parseInternalVisitable ⇒ Object 



38
39
40
 
# File 'lib/retriever/page.rb', line 38

def parseInternalVisitable
  parseInternal.select{ |linky| (!(NONPAGE_EXT_RE =~linky)) }
end

#parseSEO ⇒ Object 



62
63
64
 
# File 'lib/retriever/page.rb', line 62

def parseSEO
  return [title,desc,h1,h2]
end

#title ⇒ Object 



46
47
48
 
# File 'lib/retriever/page.rb', line 46

def title
  TITLE_RE =~ @source ? @source.match(TITLE_RE)[1] : ""
end