Module: Onebox::Helpers

Defined in:
lib/onebox/helpers.rb

Defined Under Namespace

Classes: DownloadTooLarge

Class Method Summary collapse

Class Method Details

.blank?(value) ⇒ Boolean

Returns:

  • (Boolean) 


161
162
163
164
165
166
167
168
169
 
# File 'lib/onebox/helpers.rb', line 161

def self.blank?(value)
  if value.nil?
    true
  elsif String === value
    value.empty? || !(/[[:^space:]]/ === value)
  else
    value.respond_to?(:empty?) ? !!value.empty? : !value
  end
end

.clean(html) ⇒ Object 



17
18
19
 
# File 'lib/onebox/helpers.rb', line 17

def self.clean(html)
  html.gsub(/<[^>]+>/, ' ').gsub(/\n/, '')
end

.click_to_scroll_div(width = 690, height = 400) ⇒ Object 



157
158
159
 
# File 'lib/onebox/helpers.rb', line 157

def self.click_to_scroll_div(width = 690, height = 400)
  "<div style=\"background:transparent;position:relative;width:#{width}px;height:#{height}px;top:#{height}px;margin-top:-#{height}px;\" onClick=\"style.pointerEvents='none'\"></div>"
end

.extract_opengraph(doc) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/onebox/helpers.rb', line 21

def self.extract_opengraph(doc)
  return {} unless doc

  og = {}

  doc.css('meta').each do |m|
    if (m["property"] && m["property"][/^(?:og|article):(.+)$/i]) || (m["name"] && m["name"][/^(?:og|article):(.+)$/i])
      value = (m["content"] || m["value"]).to_s
      og[$1.tr('-:', '_').to_sym] ||= value unless Onebox::Helpers::blank?(value)
    end
  end

  # Attempt to retrieve the title from the meta tag
  title_element = doc.at_css('title')
  if title_element && title_element.text
    og[:title] ||= title_element.text unless Onebox::Helpers.blank?(title_element.text)
  end

  og
end

.fetch_content_length(location) ⇒ Object 



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'lib/onebox/helpers.rb', line 118

def self.fetch_content_length(location)
  uri = URI(location)

  Net::HTTP.start(uri.host, uri.port, use_ssl: uri.is_a?(URI::HTTPS)) do |http|
    http.open_timeout = Onebox.options.connect_timeout
    http.read_timeout = Onebox.options.timeout
    if uri.is_a?(URI::HTTPS)
      http.use_ssl = true
      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
    end

    http.request_head([uri.path, uri.query].join("?")) do |response|
      code = response.code.to_i
      unless code === 200 || Onebox::Helpers.blank?(response.header['content-length'])
        return nil
      end
      return response.header['content-length']
    end
  end
end

.fetch_html_doc(url, headers = nil) ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/onebox/helpers.rb', line 42

def self.fetch_html_doc(url, headers = nil)
  response = (fetch_response(url, nil, nil, headers) rescue nil)
  doc = Nokogiri::HTML(response)

  ignore_canonical = doc.at('meta[property="og:ignore_canonical"]')
  unless ignore_canonical && ignore_canonical['content'].to_s == 'true'
    # prefer canonical link
    canonical_link = doc.at('//link[@rel="canonical"]/@href')
    if canonical_link && "#{URI(canonical_link).host}#{URI(canonical_link).path}" != "#{URI(url).host}#{URI(url).path}"
      response = (fetch_response(canonical_link, nil, nil, headers) rescue nil)
      doc = Nokogiri::HTML(response) if response
    end
  end

  doc
end

.fetch_response(location, limit = nil, domain = nil, headers = nil) ⇒ Object

Raises:

  • (Net::HTTPError) 


59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/onebox/helpers.rb', line 59

def self.fetch_response(location, limit = nil, domain = nil, headers = nil)

  limit ||= 5
  limit = Onebox.options.redirect_limit if limit > Onebox.options.redirect_limit

  raise Net::HTTPError.new('HTTP redirect too deep', location) if limit == 0

  uri = URI(location)
  uri = URI("#{domain}#{location}") if !uri.host

  result = StringIO.new
  Net::HTTP.start(uri.host, uri.port, use_ssl: uri.is_a?(URI::HTTPS)) do |http|
    http.open_timeout = Onebox.options.connect_timeout
    http.read_timeout = Onebox.options.timeout
    if uri.is_a?(URI::HTTPS)
      http.use_ssl = true
      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
    end

    headers ||= {}

    if Onebox.options.user_agent && !headers['User-Agent']
      headers['User-Agent'] = Onebox.options.user_agent
    end

    request = Net::HTTP::Get.new(uri.request_uri, headers)
    start_time = Time.now

    size_bytes = Onebox.options.max_download_kb * 1024
    http.request(request) do |response|

      if cookie = response.get_fields('set-cookie')
        header = { 'Cookie' => cookie.join }
      end

      header = nil unless header.is_a? Hash

      code = response.code.to_i
      unless code === 200
        response.error! unless [301, 302].include?(code)
        return fetch_response(
          response['location'],
          limit - 1,
          "#{uri.scheme}://#{uri.host}",
          header
        )
      end

      response.read_body do |chunk|
        result.write(chunk)
        raise DownloadTooLarge.new if result.size > size_bytes
        raise Timeout::Error.new if (Time.now - start_time) > Onebox.options.timeout
      end

      return result.string
    end
  end
end

.normalize_url_for_output(url) ⇒ Object 



179
180
181
182
183
184
185
186
187
 
# File 'lib/onebox/helpers.rb', line 179

def self.normalize_url_for_output(url)
  return "" unless url
  url = url.dup
  # expect properly encoded url, remove any unsafe chars
  url.gsub!("'", "&apos;")
  url.gsub!('"', "&quot;")
  url.gsub!(/[^\w\-`.~:\/?#\[\]@!$&'\(\)*+,;=%]/, "")
  url
end

.pretty_filesize(size) ⇒ Object 



139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/onebox/helpers.rb', line 139

def self.pretty_filesize(size)
  conv = [ 'B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB' ];
  scale = 1024;

  ndx = 1
  if (size < 2 * (scale**ndx)) then
    return "#{(size)} #{conv[ndx - 1]}"
  end
  size = size.to_f
  [2, 3, 4, 5, 6, 7].each do |i|
    if (size < 2 * (scale**i)) then
      return "#{'%.2f' % (size / (scale**(i - 1)))} #{conv[i - 1]}"
    end
  end
  ndx = 7
  return "#{'%.2f' % (size / (scale**(ndx - 1)))} #{conv[ndx - 1]}"
end

.symbolize_keys(hash) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
 
# File 'lib/onebox/helpers.rb', line 6

def self.symbolize_keys(hash)
  return {} if hash.nil?

  hash.inject({}) do |result, (key, value)|
    new_key = key.is_a?(String) ? key.to_sym : key
    new_value = value.is_a?(Hash) ? symbolize_keys(value) : value
    result[new_key] = new_value
    result
  end
end

.title_attr(meta) ⇒ Object 



175
176
177
 
# File 'lib/onebox/helpers.rb', line 175

def self.title_attr(meta)
  (meta && !blank?(meta[:title])) ? "title='#{meta[:title]}'" : ""
end

.truncate(string, length = 50) ⇒ Object 



171
172
173
 
# File 'lib/onebox/helpers.rb', line 171

def self.truncate(string, length = 50)
  string.size > length ? string[0...(string.rindex(" ", length) || length)] + "..." : string
end