Module: NetHTTPUtils

Defined in:
lib/nethttputils.rb

Defined Under Namespace

Classes: EOFError_from_rbuf_fill, Error

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.logger ⇒ Object

Returns the value of attribute logger.



9
10
11
 
# File 'lib/nethttputils.rb', line 9

def logger
  @logger
end

Class Method Details

.remove_tags(str) ⇒ Object 



30
31
32
33
34
 
# File 'lib/nethttputils.rb', line 30

def remove_tags str
  str.gsub(/<script( [a-z-]+="[^"]*")*>.*?<\/script>/m, "").
      gsub(/<style( [a-z-]+="[^"]*")*>.*?<\/style>/m, "").
      gsub(/<[^>]*>/, "").split(?\n).map(&:strip).reject(&:empty?).join(?\n)
end

.request_data(http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil, force_post: false, timeout: nil, no_redirect: false, max_start_http_retry_delay: 3600, max_read_retry_delay: 3600, patch_request: nil, &block) ⇒ Object 



311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
 
# File 'lib/nethttputils.rb', line 311

def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil, force_post: false,
    timeout: nil, no_redirect: false,
    max_start_http_retry_delay: 3600,
    max_read_retry_delay: 3600,
    patch_request: nil, &block
  timeout ||= 30
  http = start_http http, max_start_http_retry_delay, timeout, no_redirect, *proxy unless http.is_a? Net::HTTP
  path = http.instance_variable_get(:@uri).path

  check_code = lambda do |body|
    fail unless code = body.instance_variable_get(:@last_response).code
    case code
      # TODO: raise on 405
      when /\A(20\d|3\d\d|405)\z/
        nil
      else
        ct = body.instance_variable_get(:@last_response).to_hash["content-type"]
        raise Error.new(
          (ct == ["image/png"] ? "<#{ct.first}>" : body),
          code.to_i
        )
    end
  end
  if mtd == :GET && !@@_405.include?(http.address)
    body = begin
      request_data http, :HEAD, form: form, header: header, auth: auth,
        max_start_http_retry_delay: max_start_http_retry_delay,
        max_read_retry_delay: max_read_retry_delay
    rescue NetHTTPUtils::Error => e
      raise unless e.code == 400
    end
    if !body || "405" == body.instance_variable_get(:@last_response).code
      @@_405.add http.address
    else
      check_code.call body
    end
  end
  body = read http, mtd, type, form: form, header: header, auth: auth, force_post: force_post,
    timeout: timeout, no_redirect: no_redirect,
    max_read_retry_delay: max_read_retry_delay,
    patch_request: patch_request, &block
  check_code.call body

  last_response = body.instance_variable_get :@last_response
  if last_response.to_hash["content-encoding"] == "gzip"
    Zlib::GzipReader.new(StringIO.new(body)).read
  else
    body
  end
# ensure
#   response.instance_variable_get("@nethttputils_close").call if response
end

.start_http(url, max_start_http_retry_delay = 3600, timeout = nil, no_redirect = false, proxy = nil) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/nethttputils.rb', line 36

def start_http url, max_start_http_retry_delay = 3600, timeout = nil, no_redirect = false, proxy = nil
  timeout ||= 30
  uri = url
  uri = URI.parse begin
    URI url
    url
  rescue URI::InvalidURIError
    URI.escape url
  end unless url.is_a? URI::HTTP
  delay = 5
  begin
    Net::HTTP.start(
      uri.host, uri.port,
      *(proxy.split ?: if proxy),
      use_ssl: uri.scheme == "https",
      verify_mode: OpenSSL::SSL::VERIFY_NONE,
      **({open_timeout: timeout}), #  if timeout
      **({read_timeout: timeout}), #  if timeout
    ) do |http|
      # http.open_timeout = timeout   # seems like when opening hangs, this line in unreachable
      # http.read_timeout = timeout
      http.set_debug_output( Object.new.tap do |obj|
        obj.instance_eval do
          def << msg
            @@buffer ||= "[Net::HTTP debug] "
            @@buffer.concat msg
            @@buffer = @@buffer[0...997] + "..." if @@buffer.size > 1000
            return unless @@buffer.end_with? ?\n
            NetHTTPUtils.logger.debug @@buffer.sub ?\n, "  "
            @@buffer = nil
          end
        end
      end ) if logger.level == Logger::DEBUG # use `logger.debug?`?
      http
    end
  rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
    if max_start_http_retry_delay < delay *= 2
      e.message.concat " to #{uri}"
      raise
    end
    logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}'"
    sleep delay
    retry
  rescue SocketError => e
    if max_start_http_retry_delay < delay *= 2
      e.message.concat " to #{uri}"
      raise e
    end
    logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
    sleep delay
    retry
  rescue Errno::ETIMEDOUT, Net::OpenTimeout => e
    raise if max_start_http_retry_delay < delay *= 2
    logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
    sleep delay
    retry
  rescue OpenSSL::SSL::SSLError => e
    raise if max_start_http_retry_delay < delay *= 2
    logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
    sleep delay
    retry
  end.tap do |http|
    http.instance_variable_set :@uri, uri
    http.instance_variable_set :@max_start_http_retry_delay, max_start_http_retry_delay
  end
end