Class: StaticImageDownloader::Parser

Inherits:
Object
  • Object
show all
Defined in:
lib/static_image_download/parser.rb

Constant Summary collapse

PARSER_OPTIONS = 
{
  'URI_EXTRACT'    =>  :img_parse_uri_extract,
  'NOKOGIRI'     =>  :img_parse_nokogiri,
  'HPRICOT'      => :img_parse_hpricot
}
@@DEFAULTPARSEOPTION =

also you can use one ‘NOKOGIRI’ or ‘HPRICOT’

'URI_EXTRACT'
@@DEFAULTUSERAGENT = 
'Mozilla/5.0'
@@DEFAULTPATH = 
"./"
@@DEFAULTSITE = 
'http://feed.informer.com'
@@DEFAULTTIMEOUT = 
15

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(url = @@DEFAULTSITE, path = @@DEFAULTPATH, parse_option = @@DEFAULTPARSEOPTION, timeout = @@DEFAULTTIMEOUT, user_agent = @@DEFAULTUSERAGENT, h = {}) ⇒ Parser

Returns a new instance of Parser.



21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/static_image_download/parser.rb', line 21

def initialize(url=@@DEFAULTSITE, path=@@DEFAULTPATH, parse_option=@@DEFAULTPARSEOPTION, timeout=@@DEFAULTTIMEOUT, user_agent=@@DEFAULTUSERAGENT, h={})
  @url         = url.nil? ? @@DEFAULTSITE : url
  @user_agent    = user_agent.nil? ? @@DEFAULTUSERAGENT : user_agent
  @path        = path.nil? ? @@DEFAULTPATH : path
  @timeout       = timeout.nil? ? @@DEFAULTTIMEOUT : timeout
  @parse_option    = parse_option.nil? ? @@DEFAULTPARSEOPTION : parse_option
  @images      = []
  @extracted_links   = []
  @rgxp_img_uri    = Regexp.new(/^(http|https|ftp)\:\/\/([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)?((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.[a-zA-Z]{2,4})(\:[0-9]+)?(\/[^\/][a-zA-Z0-9\.\,\?\'\\\/\+&%\$#\=~_\-@]*)\.(#{Images::IMAGE_EXT.join('|')})/i)
  #@rgxp_img_uri     = Regexp.new(/^(((http|https|ftp)\:\/\/)|www|(\/\/))([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)?((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.[a-zA-Z]{2,4})(\:[0-9]+)?(\/[^\/][a-zA-Z0-9\.\,\?\'\\\/\+&%\$#\=~_\-@]*)\.(#{Images::IMAGE_EXT.join('|')})/i)
  @domain      = URI.parse(url).host
  @content     = nil
end

Instance Attribute Details

#content ⇒ Object

Returns the value of attribute content.



6
7
8
 
# File 'lib/static_image_download/parser.rb', line 6

def content
  @content
end

#extracted_links ⇒ Object

Returns the value of attribute extracted_links.



6
7
8
 
# File 'lib/static_image_download/parser.rb', line 6

def extracted_links
  @extracted_links
end

#images ⇒ Object

Returns the value of attribute images.



6
7
8
 
# File 'lib/static_image_download/parser.rb', line 6

def images
  @images
end

#parse_option ⇒ Object

Returns the value of attribute parse_option.



6
7
8
 
# File 'lib/static_image_download/parser.rb', line 6

def parse_option
  @parse_option
end

#url ⇒ Object

Returns the value of attribute url.



6
7
8
 
# File 'lib/static_image_download/parser.rb', line 6

def url
  @url
end

#user_agent ⇒ Object

Returns the value of attribute user_agent.



6
7
8
 
# File 'lib/static_image_download/parser.rb', line 6

def user_agent
  @user_agent
end

Class Method Details

.default_parse_option ⇒ Object 



36
37
38
 
# File 'lib/static_image_download/parser.rb', line 36

def default_parse_option
  @@DEFAULTPARSEOPTION
end

.default_path ⇒ Object 



44
45
46
 
# File 'lib/static_image_download/parser.rb', line 44

def default_path
  @@DEFAULTPATH
end

.default_timeout ⇒ Object 



48
49
50
 
# File 'lib/static_image_download/parser.rb', line 48

def default_timeout
  @@DEFAULTTIMEOUT
end

.default_user_agent ⇒ Object 



40
41
42
 
# File 'lib/static_image_download/parser.rb', line 40

def default_user_agent
  @@DEFAULTUSERAGENT
end

Instance Method Details

#collect_images ⇒ Object 



116
117
118
119
120
 
# File 'lib/static_image_download/parser.rb', line 116

def collect_images
  @extracted_links.each do |link|
    self.push_image(link)
  end
end

#get_content_raw ⇒ Object 



69
70
71
72
73
 
# File 'lib/static_image_download/parser.rb', line 69

def get_content_raw
  @content = self.get_url.read
  @content.gsub!(/[\n\r\t]+/,' ')
  #p @content if $debug_option
end

#get_extracted_links(links) ⇒ Object 



93
94
95
96
97
98
99
100
101
 
# File 'lib/static_image_download/parser.rb', line 93

def get_extracted_links(links)
  return false unless links 
  links.each do |link|
    p "link= #{link}" if $debug_option
    link = link[:src].to_s unless link.is_a?(String)
    @extracted_links << link.match(@rgxp_img_uri)[0] if link.match(@rgxp_img_uri) and !@extracted_links.include?(link.match(@rgxp_img_uri)[0])
  end
  #p "extracted_links= #{@extracted_links}" if $debug_option
end

#get_url ⇒ Object 



75
76
77
 
# File 'lib/static_image_download/parser.rb', line 75

def get_url
  open(self.url, 'User-Agent' => self.user_agent)
end

#img_parse_hpricot(h = {}) ⇒ Object 



84
85
86
87
 
# File 'lib/static_image_download/parser.rb', line 84

def img_parse_hpricot(h={})
  doc = Hpricot(@content)
  get_extracted_links(doc.search("//img"))
end

#img_parse_nokogiri(h = {}) ⇒ Object 



79
80
81
82
 
# File 'lib/static_image_download/parser.rb', line 79

def img_parse_nokogiri(h={})
  doc = Nokogiri::HTML(@content)
  get_extracted_links(doc.search("//img"))
end

#img_parse_uri_extract(h = {}) ⇒ Object 



89
90
91
 
# File 'lib/static_image_download/parser.rb', line 89

def img_parse_uri_extract(h={})
  get_extracted_links(URI.extract(@content).select{ |l| l[/#{@rgxp_img_uri}/] })
end

#method_to_value(option, h = {}) ⇒ Object 



57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/static_image_download/parser.rb', line 57

def method_to_value(option, h={})
  method = option_to_method(option)
  p "method= #{method}" if $debug_option
  begin
    response = send(method, h) || ""
    return response
  rescue => error
    p "method_to_value.error = #{error}"
    nil
  end
end

#option_to_method(option) ⇒ Object 



53
54
55
 
# File 'lib/static_image_download/parser.rb', line 53

def option_to_method(option)
  opt = PARSER_OPTIONS[option]
end

#parse_images(h = {}) ⇒ Object 



103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/static_image_download/parser.rb', line 103

def parse_images(h={})
  begin
    response = nil
    status = Timeout::timeout(@timeout) {
      response = method_to_value(self.parse_option, h)
      collect_images
    }
  rescue => error
    p "#{error}"
    nil
  end
end

#push_image(src) ⇒ Object 



122
123
124
 
# File 'lib/static_image_download/parser.rb', line 122

def push_image(src)
  self.images.push Images.new(src, @path, Images.default_download_option)
end