Module: NetHTTPUtils

Defined in:
lib/nethttputils.rb

Defined Under Namespace

Classes: Error

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.logger ⇒ Object

Returns the value of attribute logger.



9
10
11
 
# File 'lib/nethttputils.rb', line 9

def logger
  @logger
end

Class Method Details

.remove_tags(str) ⇒ Object 



27
28
29
30
31
 
# File 'lib/nethttputils.rb', line 27

def remove_tags str
  str.gsub(/<script( [a-z]+="[^"]*")*>.*?<\/script>/m, "").
      gsub(/<style( [a-z]+="[^"]*")*>.*?<\/style>/m, "").
      gsub(/<[^>]*>/, "").split(?\n).map(&:strip).reject(&:empty?).join(?\n)
end

.request_data(http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30, max_start_http_retry_delay: 3600, max_read_retry_delay: 3600, patch_request: nil, &block) ⇒ Object 



285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
 
# File 'lib/nethttputils.rb', line 285

def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30,
    max_start_http_retry_delay: 3600,
    max_read_retry_delay: 3600,
    patch_request: nil, &block
  http = start_http http, max_start_http_retry_delay, timeout unless http.is_a? Net::HTTP
  path = http.instance_variable_get(:@uri).path

  check_code = lambda do |body|
    fail unless code = body.instance_variable_get(:@last_response).code
    case code
      # TODO: raise on 405
      when /\A(20\d|3\d\d|405)\z/
        nil
      else
        ct = body.instance_variable_get(:@last_response).to_hash["content-type"]
        raise Error.new(
          (ct == ["image/png"] ? "<#{ct.first}>" : body),
          code.to_i
        )
    end
  end
  if mtd == :GET && !@@_405.include?(http.address)
    body = begin
      request_data http, :HEAD, form: form, header: header, auth: auth,
        max_start_http_retry_delay: max_start_http_retry_delay,
        max_read_retry_delay: max_read_retry_delay
    rescue NetHTTPUtils::Error => e
      raise unless e.code == 400
    end
    if !body || "405" == body.instance_variable_get(:@last_response).code
      @@_405.add http.address
    else
      check_code.call body
    end
  end
  body = read http, mtd, type, form: form, header: header, auth: auth, timeout: timeout,
    max_read_retry_delay: max_read_retry_delay,
    patch_request: patch_request, &block
  check_code.call body

  last_response = body.instance_variable_get :@last_response
  if last_response.to_hash["content-encoding"] == "gzip"
    Zlib::GzipReader.new(StringIO.new(body)).read
  else
    body
  end.tap do |string|
  end
# ensure
#   response.instance_variable_get("@nethttputils_close").call if response
end

.start_http(url, max_start_http_retry_delay = 3600, timeout = 30) ⇒ Object 



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/nethttputils.rb', line 33

def start_http url, max_start_http_retry_delay = 3600, timeout = 30
  uri = url
  uri = URI.parse begin
    URI url
    url
  rescue URI::InvalidURIError
    URI.escape url
  end unless url.is_a? URI::HTTP
  delay = 5
  begin
    Net::HTTP.start(
      uri.host, uri.port,
      use_ssl: uri.scheme == "https",
      verify_mode: OpenSSL::SSL::VERIFY_NONE,
      **({open_timeout: timeout}), #  if timeout
      **({read_timeout: timeout}), #  if timeout
    ) do |http|
      # http.open_timeout = timeout   # seems like when opening hangs, this line in unreachable
      # http.read_timeout = timeout
      http.set_debug_output( Object.new.tap do |obj|
        obj.instance_eval do
          def << msg
            @@buffer ||= "[Net::HTTP debug] "
            @@buffer.concat msg
            @@buffer = @@buffer[0...997] + "..." if @@buffer.size > 500
            return unless @@buffer.end_with? ?\n
            NetHTTPUtils.logger.debug @@buffer.sub ?\n, "  "
            @@buffer = nil
          end
        end
      end ) if logger.level == Logger::DEBUG # use `logger.debug?`?
      http
    end
  rescue Errno::ECONNREFUSED => e
    if max_start_http_retry_delay < delay *= 2
      e.message.concat " to #{uri}"
      raise
    end
    logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}'"
    sleep delay
    retry
  rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
    logger.warn "retrying in 5 seconds because of #{e.class} '#{e.message}'"
    sleep 5
    retry
  rescue SocketError => e
    if max_start_http_retry_delay < delay *= 2
      e.message.concat " to #{uri}"
      raise e
    end
    logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
    sleep delay
    retry
  rescue Errno::ETIMEDOUT, Net::OpenTimeout => e
    raise if max_start_http_retry_delay < delay *= 2
    logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
    sleep delay
    retry
  rescue OpenSSL::SSL::SSLError => e
    raise if max_start_http_retry_delay < delay *= 2
    logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
    sleep delay
    retry
  end.tap do |http|
    http.instance_variable_set :@uri, uri
    http.instance_variable_set :@max_start_http_retry_delay, max_start_http_retry_delay
  end
end