Module: NetHTTPUtils

Defined in:
lib/nethttputils.rb

Defined Under Namespace

Classes: Error

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.logger ⇒ Object

Returns the value of attribute logger.



9
10
11
 
# File 'lib/nethttputils.rb', line 9

def logger
  @logger
end

Class Method Details

.get_response(url, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30, max_timeout_retry_delay: 3600, max_sslerror_retry_delay: 3600, max_read_retry_delay: 3600, max_econnrefused_retry_delay: 3600, max_socketerror_retry_delay: 3600, patch_request: nil, &block) ⇒ Object

TODO: make it private?



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
 
# File 'lib/nethttputils.rb', line 32

def get_response url, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30, max_timeout_retry_delay: 3600, max_sslerror_retry_delay: 3600, max_read_retry_delay: 3600, max_econnrefused_retry_delay: 3600, max_socketerror_retry_delay: 3600, patch_request: nil, &block
  uri = URI.parse begin
    URI url
    url
  rescue URI::InvalidURIError
    URI.escape url
  end unless uri.is_a? URI::HTTP


  logger.warn "Warning: query params included in `url` argument are discarded because `:form` isn't empty" if uri.query && !form.empty?
  # we can't just merge because URI fails to parse such queries as "/?1"

  uri.query = URI.encode_www_form form if :GET == (mtd = mtd.upcase) && !form.empty?
  cookies = {}
  prepare_request = lambda do |uri|
    case mtd.upcase
      when :GET    ; Net::HTTP::Get
      when :POST   ; Net::HTTP::Post
      when :PUT    ; Net::HTTP::Put
      when :DELETE ; Net::HTTP::Delete
      when :PATCH  ; Net::HTTP::Patch
      else         ; raise "unknown method '#{mtd}'"
    end.new(uri).tap do |request| # somehow Get eats even raw url, not URI object
      patch_request.call uri, form, request if patch_request
      request.basic_auth *auth if auth
      request["cookie"] = [*request["cookie"], cookies.map{ |k, v| "#{k}=#{v}" }].join "; " unless cookies.empty?
      # pp Object.instance_method(:method).bind(request).call(:set_form).source_location
      if (mtd == :POST || mtd == :PATCH) && !form.empty?
        case type
          when :form ; if form.any?{ |k, v| v.respond_to? :to_path }
                         request.set_form form, "multipart/form-data"
                       else
                         request.set_form_data form
                         request.content_type = "application/x-www-form-urlencoded;charset=UTF-8"
                       end
          when :json ; request.body = JSON.dump form
                       request.content_type = "application/json"
          else       ; raise "unknown content-type '#{type}'"
        end
      end
      header.each{ |k, v| request[k.to_s] = v }

      logger.info "> #{request} #{request.path}"
      next unless logger.debug?
      logger.debug "curl -vsSL -o /dev/null #{request.each_header.map{ |k, v| "-H \"#{k}: #{v}\" " unless k == "host" }.join}#{url.gsub "&", "\\\\&"}"
      logger.debug "> header: #{request.each_header.to_a}"
      logger.debug "> body: #{request.body.inspect.tap{ |body| body[100..-1] = "..." if body.size > 100 }}"
      stack = caller.reverse.map do |level|
        /((?:[^\/:]+\/)?[^\/:]+):([^:]+)/.match(level).captures
      end.chunk(&:first).map do |file, group|
        "#{file}:#{group.map(&:last).chunk{|_|_}.map(&:first).join(",")}"
      end
      logger.debug stack.join " -> "
    end
  end
  start_http = lambda do |uri|
    delay = 5
    begin
      Net::HTTP.start(
        uri.host, uri.port,
        use_ssl: uri.scheme == "https",
        verify_mode: OpenSSL::SSL::VERIFY_NONE,
        **({open_timeout: timeout}), #  if timeout
        **({read_timeout: timeout}), #  if timeout
      ) do |http|
        # http.open_timeout = timeout   # seems like when opening hangs, this line in unreachable
        # http.read_timeout = timeout
        http.set_debug_output STDERR if logger.level == Logger::DEBUG # use `logger.debug?`?
        http
      end
    rescue Errno::ECONNREFUSED => e
      if max_econnrefused_retry_delay < delay *= 2
        e.message.concat " to #{uri}"
        raise
      end
      logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}'"
      sleep delay
      retry
    rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
      logger.warn "retrying in 5 seconds because of #{e.class} '#{e.message}'"
      sleep 5
      retry
    rescue SocketError => e
      if max_socketerror_retry_delay < delay *= 2
        e.message.concat " to #{uri}"
        raise e
      end
      logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
      sleep delay
      retry
    rescue Errno::ETIMEDOUT, Net::OpenTimeout => e
      raise if max_timeout_retry_delay < delay *= 2
      logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
      sleep delay
      retry
    rescue OpenSSL::SSL::SSLError => e
      raise if max_sslerror_retry_delay < delay *= 2
      logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
      sleep delay
      retry
    end
  end
  http = start_http[uri]
  do_request = lambda do |request|
    delay = 5
    response = begin
      http.request request, &block
    rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
      raise if max_read_retry_delay < delay *= 2
      logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{request.uri}"
      sleep delay
      retry
    end
    # response.instance_variable_set "@nethttputils_close", http.method(:finish)
    # response.singleton_class.instance_eval{ attr_accessor :nethttputils_socket_to_close }

    if response.key? "x-ratelimit-userremaining"
      c = response.fetch("x-ratelimit-userremaining").to_i
      logger.debug "x-ratelimit-userremaining: #{c}"
      t = response.fetch("x-ratelimit-clientremaining").to_i
      logger.debug "x-ratelimit-clientremaining: #{t}"
      unless 100 < c
        a = response.fetch("x-timer")[/\d+/].to_i
        b = response.fetch("x-ratelimit-userreset").to_i
        t = (b - a + 1).fdiv c
        logger.warn "x-ratelimit sleep #{t} seconds"
        sleep t
      end
    end

    response.to_hash.fetch("set-cookie", []).each{ |c| k, v = c.split(?=); cookies[k] = v[/[^;]+/] }
    case response.code
    when /\A3\d\d\z/
      logger.info "redirect: #{response["location"]}"
      new_uri = URI.join request.uri, URI.escape(response["location"])
      new_host = new_uri.host
      if http.address != new_host ||
         http.port != new_uri.port ||
         http.use_ssl? != (new_uri.scheme == "https")
        logger.debug "changing host from '#{http.address}' to '#{new_host}'"
        # http.finish
        http = start_http[new_uri]
      end
      do_request.call prepare_request[new_uri]
    when "404"
      logger.error "404 at #{request.method} #{request.uri} with body: #{
        response.body.is_a?(Net::ReadAdapter) ? "impossible to reread Net::ReadAdapter -- check the IO you've used in block form" : response.body.tap do |body|
          body.replace remove_tags body if body[/<html[> ]/]
        end.inspect
      }"
      response
    when "429"
      logger.error "429 at #{request.method} #{request.uri} with body: #{response.body.inspect}"
      response
    when /\A50\d\z/
      logger.error "#{response.code} at #{request.method} #{request.uri} with body: #{
        response.body.tap do |body|
          body.replace remove_tags body if body[/<html[> ]/]
        end.inspect
      }"
      response
    when /\A20/
      response
    else
      logger.warn "code #{response.code} at #{request.method} #{request.uri}#{
        " and so #{url}" if request.uri.to_s != url
      } from #{
        [__FILE__, caller.map{ |i| i[/(?<=:)\d+/] }].join ?:
      }"
      logger.debug "< header: #{response.to_hash}"
      logger.debug "< body: #{
        response.body.tap do |body|
          body.replace remove_tags body if body[/<html[> ]/]
        end.inspect
      }"
      response
    end
  end
  do_request[prepare_request[uri]].tap do |response|
    cookies.each{ |k, v| response.add_field "Set-Cookie", "#{k}=#{v};" }
    logger.debug response.to_hash
  end
end

.remove_tags(str) ⇒ Object 



27
28
29
 
# File 'lib/nethttputils.rb', line 27

def remove_tags str
  str.gsub(/<script( type="text\/javascript"| src="[^"]+")?>.*?<\/script>/m, "").gsub(/<[^>]*>/, "").strip
end

.request_data(*args, &block) ⇒ Object 



216
217
218
219
220
221
222
223
224
225
226
227
228
 
# File 'lib/nethttputils.rb', line 216

def request_data *args, &block
  response = get_response *args, &block
  raise Error.new response.body, response.code.to_i unless response.code[/\A(20\d|3\d\d)\z/]
  if response["content-encoding"] == "gzip"
    Zlib::GzipReader.new(StringIO.new(response.body)).read
  else
    response.body
  end.tap do |string|
    string.instance_variable_set :@uri_path, response.uri.path
  end
# ensure
#   response.instance_variable_get("@nethttputils_close").call if response
end