Class: WebStat::Fetch

Inherits:
Object
  • Object
show all
Defined in:
lib/web_stat/fetch.rb

Direct Known Subclasses

FetchAsHtml, FetchAsWeb

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#html ⇒ Object

Returns the value of attribute html.



3
4
5
 
# File 'lib/web_stat/fetch.rb', line 3

def html
  @html
end

#nokogiri ⇒ Object

Returns the value of attribute nokogiri.



3
4
5
 
# File 'lib/web_stat/fetch.rb', line 3

def nokogiri
  @nokogiri
end

#url ⇒ Object

Returns the value of attribute url.



3
4
5
 
# File 'lib/web_stat/fetch.rb', line 3

def url
  @url
end

#userdic ⇒ Object

Returns the value of attribute userdic.



3
4
5
 
# File 'lib/web_stat/fetch.rb', line 3

def userdic
  @userdic
end

Instance Method Details

#content ⇒ Object

Get main section



30
31
32
 
# File 'lib/web_stat/fetch.rb', line 30

def content
  Sanitize.clean(Readability::Document.new(@nokogiri.at('body')).content)
end

#eyecatch_image_path ⇒ Object

Get temporary path of image



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/web_stat/fetch.rb', line 35

def eyecatch_image_path
  # Reuse `path` in this method
  path = nil
  WebStat::Configure.get["eyecatch_image_xpaths"].each do |xpath|
    if @nokogiri.xpath(xpath).first.respond_to?(:value)
      path = @nokogiri.xpath(xpath).first.value
      break
    end
  end
  if ! path.nil? && path.match(/^\//)
    "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}#{path}"
  else
    path
  end
end

#get_url(url) ⇒ Object

Get url

Parameters:

  • url (String) 


70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/web_stat/fetch.rb', line 70

def get_url(url)
  agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
  # Enable to read Robots.txt
  agent.robots = true
  document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
  if document.class == Mechanize::File
    document.body
  else
    document.body.encode('UTF-8', document.encoding)
  end
end

#save_local_path(url) ⇒ Object

Get local path to save url

Parameters:

  • url (String) 


53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/web_stat/fetch.rb', line 53

def save_local_path(url)
  return nil if url.nil?
  tmp_file = "/tmp/#{Digest::SHA1.hexdigest(url)}"
  agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
  image = agent.get(url)
  File.open(tmp_file, "w+b") do |_file|
    if image.class == Mechanize::File
      _file.puts(image.body)
    else
      _file.puts(image.body_io.read)
    end
  end
  tmp_file
end

#site_name ⇒ Object

Get name of domain



20
21
22
23
24
25
26
27
 
# File 'lib/web_stat/fetch.rb', line 20

def site_name
  begin
    site_name = @nokogiri.title.split(/#{WebStat::Configure.get["regex_to_sprit_title"]}/, 2).last
  rescue
    site_name = @nokogiri.title
  end
  site_name.strip
end

#stat(userdics: nil) ⇒ Object

Get the informations of @url

Parameters:

  • Specify (Hash) —

    a dictionary for each language code. example ) /***/**.dic, "other": /***/***.dic



84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/web_stat/fetch.rb', line 84

def stat(userdics: nil)
  clean_content = content.scrub('').gsub(/[\n\t\r　]/, "").gsub(/\s{2,}/, "\s")
  language_code = CLD.detect_language(clean_content)[:code]
  if userdics && userdics.has_key?(language_code)
    tag = WebStat::Tag.new("#{title} #{content}", userdic: userdics[language_code])
  elsif userdics && userdics.has_key?("other")
    tag = WebStat::Tag.new("#{title} #{content}", userdic: userdics["other"])
  else
    tag = WebStat::Tag.new("#{title} #{content}", userdic: WebStat::Configure.get["userdic"])
  end
  {
    title: title,
    site_name: site_name,
    content: clean_content,
    language_code: language_code,
    url: @url,
    eyecatch_image_path: save_local_path(eyecatch_image_path),
    tags: tag.nouns
  }
end

#title ⇒ String

Get title

Returns:

  • (String) —

    title



7
8
9
10
11
12
13
14
15
16
17
 
# File 'lib/web_stat/fetch.rb', line 7

def title
  begin
    title = @nokogiri.title.split(/#{WebStat::Configure.get["regex_to_sprit_title"]}/, 2).first
    if title.length < WebStat::Configure.get["min_length_of_meta_title"]
      title = @nokogiri.css("h1").first.content
    end
  rescue
    title = @nokogiri.title
  end
  title.strip
end