Class: PersistentHTTP

Inherits:
Object
  • Object
show all
Defined in:
lib/persistent_http.rb

Overview

Persistent connections for Net::HTTP

PersistentHTTP maintains a connection pool of Net::HTTP persistent connections. When connections fail due to resets or bad responses, the connection is renewed and the request is retried per RFC 2616 (POST requests will only get retried if the :force_retry option is set to true).

Example:

@@persistent_http = PersistentHTTP.new(
  :name         => 'MyHTTPClient',
  :logger       => Rails.logger,
  :pool_size    => 10,
  :warn_timeout => 0.25,
  :force_retry  => true,
  :url          => 'https://www.example.com/echo/foo'  # equivalent to :use_ssl => true, :host => 'www.example.com', :default_path => '/echo/foo'
)

def send_get_message
  response = @@persistent_http.request
  ... Handle response as you would a normal Net::HTTPResponse ...
end

def send_post_message
  request = Net::HTTP::Post.new('/perform_service)
  ... Modify request as needed ...
  response = @@persistent_http.request(request)
  ... Handle response as you would a normal Net::HTTPResponse ...
end

Defined Under Namespace

Classes: Error

Constant Summary collapse

VERSION =

The version of PersistentHTTP use are using

'1.0.0'

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ PersistentHTTP

Creates a new PersistentHTTP.

Set name to keep your connections apart from everybody else’s. Not required currently, but highly recommended. Your library name should be good enough. This parameter will be required in a future version.

proxy may be set to a URI::HTTP or :ENV to pick up proxy options from the environment. See proxy_from_env for details.

In order to use a URI for the proxy you’ll need to do some extra work beyond URI.parse:

proxy = URI.parse 'http://proxy.example'
proxy.user     = 'AzureDiamond'
proxy.password = 'hunter2'



173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
 
# File 'lib/persistent_http.rb', line 173

def initialize(options={})
  @name            = options[:name]            || 'PersistentHTTP'
  @ca_file         = options[:ca_file]
  @certificate     = options[:certificate]
  @debug_output    = options[:debug_output]
  @default_path    = options[:default_path]
  @force_retry     = options[:force_retry]
  @headers         = options[:header]          || {}
  @host            = options[:host]
  @idle_timeout    = options[:idle_timeout]    || 10
  @keep_alive      = options[:keep_alive]      || 30
  @logger          = options[:logger]
  @pool_timeout    = options[:pool_timeout]
  @open_timeout    = options[:open_timeout]
  @pool_size       = options[:pool_size]       || 1
  @port            = options[:port]
  @private_key     = options[:private_key]
  @read_timeout    = options[:read_timeout]
  @use_ssl         = options[:use_ssl]
  @verify_callback = options[:verify_callback]
  @verify_mode     = options[:verify_mode]
  @warn_timeout    = options[:warn_timeout]    || 0.5
  
  url              = options[:url]
  if url
    url = URI.parse(url) if url.kind_of? String
    @default_path ||= url.request_uri
    @host         ||= url.host
    @port         ||= url.port
    @use_ssl      ||= url.scheme == 'https'          
  end
  
  @port ||= (@use_ssl ? 443 : 80)

  # Hash containing the request counts based on the connection
  @count_hash = Hash.new(0)

  raise 'host not set' unless @host
  net_http_args = [@host, @port]
  connection_id = net_http_args.join ':'

  proxy = options[:proxy]

  @proxy_uri = case proxy
               when :ENV      then proxy_from_env
               when URI::HTTP then proxy
               when nil       then # ignore
               else raise ArgumentError, 'proxy must be :ENV or a URI::HTTP'
               end

  if @proxy_uri then
    @proxy_args = [
      @proxy_uri.host,
      @proxy_uri.port,
      @proxy_uri.user,
      @proxy_uri.password,
    ]

    @proxy_connection_id = [nil, *@proxy_args].join ':'

    connection_id << @proxy_connection_id
    net_http_args.concat @proxy_args
  end

  @pool = GenePool.new(:name         => name + '-' + connection_id,
                       :pool_size    => @pool_size,
                       :timeout      => @pool_timeout,
                       :warn_timeout => @warn_timeout,
                       :idle_timeout => @idle_timeout,
                       :close_proc   => nil,
                       :logger       => @logger) do
    begin
      @logger.debug { "#{name}: Creating connection" } if @logger
      connection = Net::HTTP.new(*net_http_args)
      connection.set_debug_output @debug_output if @debug_output
      connection.open_timeout = @open_timeout if @open_timeout
      connection.read_timeout = @read_timeout if @read_timeout

      ssl connection if @use_ssl

      connection.start
      @logger.debug { "#{name} #{connection}: Connection created" } if @logger
      connection
    rescue Errno::ECONNREFUSED
      raise Error, "connection refused: #{connection.address}:#{connection.port}"
    rescue Errno::EHOSTDOWN
      raise Error, "host down: #{connection.address}:#{connection.port}"
    end
  end
end

Instance Attribute Details

#ca_file ⇒ Object

An SSL certificate authority. Setting this will set verify_mode to VERIFY_PEER.



53
54
55
 
# File 'lib/persistent_http.rb', line 53

def ca_file
  @ca_file
end

#certificate ⇒ Object

This client’s OpenSSL::X509::Certificate



57
58
59
 
# File 'lib/persistent_http.rb', line 57

def certificate
  @certificate
end

#debug_output ⇒ Object

Sends debug_output to this IO via Net::HTTP#set_debug_output.

Never use this method in production code, it causes a serious security hole.



64
65
66
 
# File 'lib/persistent_http.rb', line 64

def debug_output
  @debug_output
end

#default_path ⇒ Object

Default path for the request



68
69
70
 
# File 'lib/persistent_http.rb', line 68

def default_path
  @default_path
end

#force_retry ⇒ Object

Retry even for non-idempotent (POST) requests.



72
73
74
 
# File 'lib/persistent_http.rb', line 72

def force_retry
  @force_retry
end

#headers ⇒ Object

Headers that are added to every request



76
77
78
 
# File 'lib/persistent_http.rb', line 76

def headers
  @headers
end

#host ⇒ Object (readonly)

Host for the Net:HTTP connection



80
81
82
 
# File 'lib/persistent_http.rb', line 80

def host
  @host
end

#http_version ⇒ Object (readonly)

HTTP version to enable version specific features.



84
85
86
 
# File 'lib/persistent_http.rb', line 84

def http_version
  @http_version
end

#idle_timeout ⇒ Object (readonly)

Connection will be renewed if it hasn’t been used in this amount of time. Defaults to 10 seconds.



88
89
90
 
# File 'lib/persistent_http.rb', line 88

def idle_timeout
  @idle_timeout
end

#keep_alive ⇒ Object

The value sent in the Keep-Alive header. Defaults to 30. Not needed for HTTP/1.1 servers.

This may not work correctly for HTTP/1.0 servers

This method may be removed in a future version as RFC 2616 does not require this header.



98
99
100
 
# File 'lib/persistent_http.rb', line 98

def keep_alive
  @keep_alive
end

#logger ⇒ Object

Logger for message logging.



102
103
104
 
# File 'lib/persistent_http.rb', line 102

def logger
  @logger
end

#name ⇒ Object (readonly)

A name for this connection. Allows you to keep your connections apart from everybody else’s.



107
108
109
 
# File 'lib/persistent_http.rb', line 107

def name
  @name
end

#open_timeout ⇒ Object

Seconds to wait until a connection is opened. See Net::HTTP#open_timeout



114
115
116
 
# File 'lib/persistent_http.rb', line 114

def open_timeout
  @open_timeout
end

#pool_size ⇒ Object

Return the size of the connection pool



118
119
120
 
# File 'lib/persistent_http.rb', line 118

def pool_size
  @pool_size
end

#pool_timeout ⇒ Object

Seconds to wait for an available connection before a Timeout::Error is raised



111
112
113
 
# File 'lib/persistent_http.rb', line 111

def pool_timeout
  @pool_timeout
end

#port ⇒ Object (readonly)

Port for the Net:HTTP connection



122
123
124
 
# File 'lib/persistent_http.rb', line 122

def port
  @port
end

#private_key ⇒ Object

This client’s SSL private key



126
127
128
 
# File 'lib/persistent_http.rb', line 126

def private_key
  @private_key
end

#proxy_uri ⇒ Object (readonly)

The URL through which requests will be proxied



130
131
132
 
# File 'lib/persistent_http.rb', line 130

def proxy_uri
  @proxy_uri
end

#read_timeout ⇒ Object

Seconds to wait until reading one block. See Net::HTTP#read_timeout



134
135
136
 
# File 'lib/persistent_http.rb', line 134

def read_timeout
  @read_timeout
end

#use_ssl ⇒ Object (readonly)

Use ssl if set



138
139
140
 
# File 'lib/persistent_http.rb', line 138

def use_ssl
  @use_ssl
end

#verify_callback ⇒ Object

SSL verification callback. Used when ca_file is set.



142
143
144
 
# File 'lib/persistent_http.rb', line 142

def verify_callback
  @verify_callback
end

#verify_mode ⇒ Object

HTTPS verify mode. Defaults to OpenSSL::SSL::VERIFY_NONE which ignores certificate problems.

You can use verify_mode to override any default values.



149
150
151
 
# File 'lib/persistent_http.rb', line 149

def verify_mode
  @verify_mode
end

#warn_timeout ⇒ Object (readonly)

The threshold in seconds for checking out a connection at which a warning will be logged via the logger



154
155
156
 
# File 'lib/persistent_http.rb', line 154

def warn_timeout
  @warn_timeout
end

Instance Method Details

#request(req = nil, options = {}, &block) ⇒ Object

Makes a request per req. If req is nil a Net::HTTP::Get is performed against default_path.

If a block is passed #request behaves like Net::HTTP#request (the body of the response will not have been read).

req must be a Net::HTTPRequest subclass (see Net::HTTP for a list).

If there is an error and the request is idempontent according to RFC 2616 it will be retried automatically.



286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
 
# File 'lib/persistent_http.rb', line 286

def request(req = nil, options = {}, &block)
  retried      = false
  bad_response = false

  req = Net::HTTP::Get.new @default_path unless req

  headers.each do |pair|
    req.add_field(*pair)
  end

  req.add_field 'Connection', 'keep-alive'
  req.add_field 'Keep-Alive', @keep_alive

  @pool.with_connection do |connection|
    begin
      options.each do |key, value|
        connection.send("#{key}=", value)
      end
      response = connection.request req, &block
      @http_version ||= response.http_version
      @count_hash[connection.object_id] += 1
      return response

    rescue Timeout::Error => e
      due_to = "(due to #{e.message} - #{e.class})"
      message = error_message connection
      @logger.info "#{name}: Removing connection #{due_to} #{message}" if @logger
      remove connection
      raise
      
    rescue Net::HTTPBadResponse => e
      message = error_message connection
      if bad_response or not (idempotent? req or @force_retry)
        @logger.info "#{name}: Removing connection because of too many bad responses #{message}" if @logger
        remove connection
        raise Error, "too many bad responses #{message}"
      else
        bad_response = true
        @logger.info "#{name}: Renewing connection because of bad response #{message}" if @logger
        connection = renew connection
        retry
      end

    rescue IOError, EOFError, Errno::ECONNABORTED, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EPIPE => e
      due_to = "(due to #{e.message} - #{e.class})"
      message = error_message connection
      if retried or not (idempotent? req or @force_retry)
        @logger.info "#{name}: Removing connection #{due_to} #{message}" if @logger
        remove connection
        raise Error, "too many connection resets #{due_to} #{message}"
      else
        retried = true
        @logger.info "#{name}: Renewing connection #{due_to} #{message}" if @logger
        connection = renew connection
        retry
      end
    end
  end
end

#shutdown(timeout = 10) ⇒ Object

Shuts down all connections.



348
349
350
 
# File 'lib/persistent_http.rb', line 348

def shutdown(timeout=10)
  @pool.close(timeout)
end