Module: NetHTTPUtils

Defined in:
lib/nethttputils.rb

Defined Under Namespace

Classes: EOFError_from_rbuf_fill, Error

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.logger ⇒ Object

Returns the value of attribute logger.



9
10
11
 
# File 'lib/nethttputils.rb', line 9

def logger
  @logger
end

Class Method Details

.remove_tags(str) ⇒ Object 



33
34
35
36
37
 
# File 'lib/nethttputils.rb', line 33

def remove_tags str
  str.gsub(/<script( [a-z-]+="[^"]*")*>.*?<\/script>/m, "").
      gsub(/<style( [a-z-]+="[^"]*")*>.*?<\/style>/m, "").
      gsub(/<[^>]*>/, "").split(?\n).map(&:strip).reject(&:empty?).join(?\n)
end

.request_data(http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil, force_post: false, no_redirect: false, head: false, timeout: nil, max_start_http_retry_delay: 3600, max_read_retry_delay: 3600, patch_request: nil, &block) ⇒ Object 



316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
 
# File 'lib/nethttputils.rb', line 316

def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil, force_post: false, no_redirect: false, head: false,
    timeout: nil, max_start_http_retry_delay: 3600, max_read_retry_delay: 3600,
    patch_request: nil, &block
  timeout ||= 30
  http = start_http http, max_start_http_retry_delay, timeout, no_redirect, *proxy unless http.is_a? Net::HTTP
  path = http.instance_variable_get(:@uri).path

  check_code = lambda do |body|
    fail unless code = body.instance_variable_get(:@last_response).code
    case code
      # TODO: raise on 405
      when /\A(20\d|3\d\d|405)\z/
        nil
      else
        ct = body.instance_variable_get(:@last_response).to_hash["content-type"]
        raise Error.new(
          (ct == ["image/png"] ? "<#{ct.first}>" : body),
          code.to_i
        )
    end
  end
  if head && mtd == :GET && !@@_405.include?(http.address)
    body = begin
      request_data http, :HEAD, form: form, header: header, auth: auth,
        max_start_http_retry_delay: max_start_http_retry_delay,
        max_read_retry_delay: max_read_retry_delay
    rescue NetHTTPUtils::Error => e
      raise unless e.code == 400
    end
    if !body || "405" == body.instance_variable_get(:@last_response).code
      @@_405.add http.address
    else
      check_code.call body
    end
  end
  body = read http, mtd, type, form: form, header: header, auth: auth, force_post: force_post,
    timeout: timeout, no_redirect: no_redirect,
    max_read_retry_delay: max_read_retry_delay,
    patch_request: patch_request, &block
  check_code.call body

  last_response = body.instance_variable_get :@last_response
  if last_response.to_hash["content-encoding"] == "gzip"
    Zlib::GzipReader.new(StringIO.new(body)).read
  else
    body
  end
# ensure
#   response.instance_variable_get("@nethttputils_close").call if response
end

.start_http(url, max_start_http_retry_delay = 3600, timeout = nil, no_redirect = false, proxy = nil) ⇒ Object

Raises:



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/nethttputils.rb', line 39

def start_http url, max_start_http_retry_delay = 3600, timeout = nil, no_redirect = false, proxy = nil
  timeout ||= 30
  uri = url

  uri = begin
    URI url
  rescue URI::InvalidURIError
    URI Addressable::URI.escape url
  end unless url.is_a? URI::HTTP
  raise Error, "can't parse host" unless uri.host

  delay = 5
  begin
    Net::HTTP.start(
      uri.host, uri.port,
      *(proxy.split ?: if proxy),
      use_ssl: uri.scheme == "https",
      verify_mode: OpenSSL::SSL::VERIFY_NONE,
      **({open_timeout: timeout}), #  if timeout
      **({read_timeout: timeout}), #  if timeout
    ) do |http|
      # http.open_timeout = timeout   # seems like when opening hangs, this line in unreachable
      # http.read_timeout = timeout
      http.set_debug_output( Object.new.tap do |obj|
        obj.instance_eval do
          def << msg
            @@buffer ||= "[Net::HTTP debug] "
            @@buffer.concat msg
            @@buffer = @@buffer[0...997] + "..." if @@buffer.size > 1000
            return unless @@buffer.end_with? ?\n
            NetHTTPUtils.logger.debug @@buffer.sub ?\n, "  "
            @@buffer = nil
          end
        end
      end ) if logger.level == Logger::DEBUG # use `logger.debug?`?
      http
    end
  rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
    if max_start_http_retry_delay < delay *= 2
      e.message.concat " to #{uri}"
      raise
    end
    logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}'"
    sleep delay
    retry
  rescue SocketError => e
    if max_start_http_retry_delay < delay *= 2
      e.message.concat " to #{uri}"
      raise e
    end
    logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
    sleep delay
    retry
  rescue Errno::ETIMEDOUT, Net::OpenTimeout => e
    raise if max_start_http_retry_delay < delay *= 2
    logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
    sleep delay
    retry
  rescue OpenSSL::SSL::SSLError => e
    raise if max_start_http_retry_delay < delay *= 2
    logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
    sleep delay
    retry
  end.tap do |http|
    http.instance_variable_set :@uri, uri
    http.instance_variable_set :@max_start_http_retry_delay, max_start_http_retry_delay
  end
end