Class: WebStat::Fetch

Inherits:
Object
  • Object
show all
Defined in:
lib/web_stat/fetch.rb

Direct Known Subclasses

FetchAsHtml, FetchAsWeb

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#html ⇒ Object

Returns the value of attribute html.



3
4
5
 
# File 'lib/web_stat/fetch.rb', line 3

def html
  @html
end

#nokogiri ⇒ Object

Returns the value of attribute nokogiri.



3
4
5
 
# File 'lib/web_stat/fetch.rb', line 3

def nokogiri
  @nokogiri
end

#url ⇒ Object

Returns the value of attribute url.



3
4
5
 
# File 'lib/web_stat/fetch.rb', line 3

def url
  @url
end

#userdic ⇒ Object

Returns the value of attribute userdic.



3
4
5
 
# File 'lib/web_stat/fetch.rb', line 3

def userdic
  @userdic
end

Instance Method Details

#content ⇒ Object

Get main section



30
31
32
 
# File 'lib/web_stat/fetch.rb', line 30

def content
  Sanitize.clean(Readability::Document.new(@html).content)
end

#eyecatch_image_path ⇒ Object

Get temporary path of image



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/web_stat/fetch.rb', line 35

def eyecatch_image_path
  # Reuse `path` in this method
  path = nil
  WebStat::Configure.get["eyecatch_image_xpaths"].each do |xpath|
    if @nokogiri.xpath(xpath).first.respond_to?(:value)
      path = @nokogiri.xpath(xpath).first.value
      break
    end
  end
  if path.match(/^\//)
    "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}#{path}"
  else
    path
  end
end

#get_url(url) ⇒ Object

Get url

Parameters:

  • url (String) 


66
67
68
69
70
71
 
# File 'lib/web_stat/fetch.rb', line 66

def get_url(url)
  agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
  # Enable to read Robots.txt
  agent.robots = true
  agent.get(url, [], nil, { 'Accept-Language' => 'ja'}).body
end

#save_local_path(url) ⇒ Object

Get local path to save url

Parameters:

  • url (String) 


53
54
55
56
57
58
59
60
61
62
 
# File 'lib/web_stat/fetch.rb', line 53

def save_local_path(url)
  return nil if url.nil?
  tmp_file = "/tmp/#{Digest::SHA1.hexdigest(url)}"
  URI.open(original_url(url)) do |remote_file|
    File.open(tmp_file, "w+b") do |_file|
      _file.puts(remote_file.read)
    end
  end
  tmp_file
end

#site_name ⇒ Object

Get name of domain



20
21
22
23
24
25
26
27
 
# File 'lib/web_stat/fetch.rb', line 20

def site_name
  begin
    site_name = @nokogiri.title.split(/#{WebStat::Configure.get["regex_to_sprit_title"]}/, 2).last
  rescue
    site_name = @nokogiri.title
  end
  site_name.strip
end

#stat ⇒ Object

Get the informations of @url



74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/web_stat/fetch.rb', line 74

def stat
  content.gsub!(/(\n|\t|\s|　)/, "")
  tag = WebStat::Tag.new(content, userdic: WebStat::Configure.get["userdic"])
  {
    title: title,
    site_name: site_name,
    content: content,
    language_code: CLD.detect_language(content)[:code],
    url: @url,
    eyecatch_image_path: save_local_path(eyecatch_image_path),
    tags: tag.nouns
  }
end

#title ⇒ String

Get title

Returns:

  • (String) —

    title



7
8
9
10
11
12
13
14
15
16
17
 
# File 'lib/web_stat/fetch.rb', line 7

def title
  begin
    title = @nokogiri.title.split(/#{WebStat::Configure.get["regex_to_sprit_title"]}/, 2).first
    if title.length < WebStat::Configure.get["min_length_of_meta_title"]
      title = @nokogiri.css("h1").first.content
    end
  rescue
    title = @nokogiri.title
  end
  title.strip
end