Class: Aws::HttpConnection

Inherits:
Object show all
Defined in:
lib/awsbase/http_connection.rb,
test/test_helper.rb

Overview

A HttpConnection will retry a request a certain number of times (as defined by HTTP_CONNNECTION_RETRY_COUNT). If all the retries fail, an exception is thrown and all HttpConnections associated with a server enter a probationary period defined by HTTP_CONNECTION_RETRY_DELAY. If the user makes a new request subsequent to entering probation, the request will fail immediately with the same exception thrown on probation entry. This is so that if the HTTP server has gone down, not every subsequent request must wait for a connect timeout before failing. After the probation period expires, the internal state of the HttpConnection is reset and subsequent requests have the full number of potential reconnects and retries available to them.

Constant Summary collapse

HTTP_CONNECTION_RETRY_COUNT =

Number of times to retry the request after encountering the first error

3
HTTP_CONNECTION_OPEN_TIMEOUT =

Throw a Timeout::Error if a connection isn’t established within this number of seconds

5
HTTP_CONNECTION_READ_TIMEOUT =

Throw a Timeout::Error if no data have been read on this connnection within this number of seconds

120
HTTP_CONNECTION_RETRY_DELAY =

Length of the post-error probationary period during which all requests will fail

15
@@params =

class methods

{}
@@response_stack = 
[]

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(params = {}) ⇒ HttpConnection

Params hash:

:user_agent => 'www.HostName.com'    # String to report as HTTP User agent
:ca_file    => 'path_to_file'        # A path of a CA certification file in PEM format. The file can contain several CA certificates.
:logger     => Logger object         # If omitted, HttpConnection logs to STDOUT
:exception  => Exception to raise    # The type of exception to raise if a request repeatedly fails. RuntimeError is raised if this parameter is omitted.
:http_connection_retry_count         # by default == Aws::HttpConnection.params[:http_connection_retry_count]
:http_connection_open_timeout        # by default == Aws::HttpConnection.params[:http_connection_open_timeout]
:http_connection_read_timeout        # by default == Aws::HttpConnection.params[:http_connection_read_timeout]
:http_connection_retry_delay         # by default == Aws::HttpConnection.params[:http_connection_retry_delay]



124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/awsbase/http_connection.rb', line 124

def initialize(params={})
  @params = params
  @params[:http_connection_retry_count]  ||= @@params[:http_connection_retry_count]
  @params[:http_connection_open_timeout] ||= @@params[:http_connection_open_timeout]
  @params[:http_connection_read_timeout] ||= @@params[:http_connection_read_timeout]
  @params[:http_connection_retry_delay]  ||= @@params[:http_connection_retry_delay]
  @http   = nil
  @server = nil
  @logger = get_param(:logger) ||
    (RAILS_DEFAULT_LOGGER if defined?(RAILS_DEFAULT_LOGGER)) ||
    Logger.new(STDOUT)
end

Instance Attribute Details

#http ⇒ Object

instance methods



109
110
111
 
# File 'lib/awsbase/http_connection.rb', line 109

def http
  @http
end

#logger ⇒ Object

Returns the value of attribute logger.



112
113
114
 
# File 'lib/awsbase/http_connection.rb', line 112

def logger
  @logger
end

#params ⇒ Object

see @@params



111
112
113
 
# File 'lib/awsbase/http_connection.rb', line 111

def params
  @params
end

#server ⇒ Object

Returns the value of attribute server.



110
111
112
 
# File 'lib/awsbase/http_connection.rb', line 110

def server
  @server
end

Class Method Details

.length ⇒ Object 



61
62
63
 
# File 'test/test_helper.rb', line 61

def self.length
  @@response_stack.length
end

.params ⇒ Object

Query the global (class-level) parameters:

:user_agent => 'www.HostName.com'    # String to report as HTTP User agent
:ca_file    => 'path_to_file'        # Path to a CA certification file in PEM format. The file can contain several CA certificates.  If this parameter isn't set, HTTPS certs won't be verified.
:logger     => Logger object         # If omitted, HttpConnection logs to STDOUT
:exception  => Exception to raise    # The type of exception to raise
                                     # if a request repeatedly fails. RuntimeError is raised if this parameter is omitted.
:http_connection_retry_count         # by default == Aws::HttpConnection::HTTP_CONNECTION_RETRY_COUNT
:http_connection_open_timeout        # by default == Aws::HttpConnection::HTTP_CONNECTION_OPEN_TIMEOUT
:http_connection_read_timeout        # by default == Aws::HttpConnection::HTTP_CONNECTION_READ_TIMEOUT
:http_connection_retry_delay         # by default == Aws::HttpConnection::HTTP_CONNECTION_RETRY_DELAY



97
98
99
 
# File 'lib/awsbase/http_connection.rb', line 97

def self.params
  @@params
end

.params=(params) ⇒ Object

Set the global (class-level) parameters



102
103
104
 
# File 'lib/awsbase/http_connection.rb', line 102

def self.params=(params)
  @@params = params
end

.pop ⇒ Object 



57
58
59
 
# File 'test/test_helper.rb', line 57

def self.pop
  @@response_stack.pop
end

.push(code, msg = nil) ⇒ Object 



52
53
54
55
 
# File 'test/test_helper.rb', line 52

def self.push(code, msg=nil)
  response = {:code => code, :msg => msg}
  @@response_stack << response
end

.reset ⇒ Object 



48
49
50
 
# File 'test/test_helper.rb', line 48

def self.reset
  @@response_stack = []
end

Instance Method Details

#finish(reason = '') ⇒ Object 



400
401
402
403
404
405
406
 
# File 'lib/awsbase/http_connection.rb', line 400

def finish(reason = '')
  if @http && @http.started?
    reason = ", reason: '#{reason}'" unless reason.blank?
    @logger.info("Closing #{@http.use_ssl? ? 'HTTPS' : 'HTTP'} connection to #{@http.address}:#{@http.port}#{reason}")
    @http.finish
  end
end

#get_param(name) ⇒ Object 



137
138
139
 
# File 'lib/awsbase/http_connection.rb', line 137

def get_param(name)
  @params[name] || @@params[name]
end

#local_read_size=(newsize) ⇒ Object

Set the maximum size (in bytes) of a single read from local data sources like files. This can be used to tune the performance of, for example, a streaming PUT of a large buffer.



165
166
167
 
# File 'lib/awsbase/http_connection.rb', line 165

def local_read_size=(newsize)
  Net::HTTPGenericRequest.local_read_size=(newsize)
end

#local_read_size? ⇒ Boolean

Query for the maximum size (in bytes) of a single read from local data sources like files. This is important, for example, in a streaming PUT of a large buffer.

Returns:

  • (Boolean) 


158
159
160
 
# File 'lib/awsbase/http_connection.rb', line 158

def local_read_size?
  Net::HTTPGenericRequest.local_read_size?
end

#real_request ⇒ Object

Raises RuntimeError, Interrupt, and params (if specified in new).



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'test/test_helper.rb', line 28

def request(request_params, &block)
  # We save the offset here so that if we need to retry, we can return the file pointer to its initial position
  mypos = get_fileptr_offset(request_params)
  loop do
    # if we are inside a delay between retries: no requests this time!
    if error_count > @params[:http_connection_retry_count] &&
        error_time + @params[:http_connection_retry_delay] > Time.now
      # store the message (otherwise it will be lost after error_reset and
      # we will raise an exception with an empty text)
      banana_message_text = banana_message
      @logger.warn("#{err_header} re-raising same error: #{banana_message_text} " +
                   "-- error count: #{error_count}, error age: #{Time.now.to_i - error_time.to_i}")
      exception = get_param(:exception) || RuntimeError
      raise exception.new(banana_message_text)
    end

    # try to connect server(if connection does not exist) and get response data
    begin
      request_params[:protocol] ||= (request_params[:port] == 443 ? 'https' : 'http')

      request = request_params[:request]
      request['User-Agent'] = get_param(:user_agent) || ''

      # (re)open connection to server if none exists or params has changed
      unless @http          &&
          @http.started? &&
          @server   == request_params[:server] &&
          @port     == request_params[:port]   &&
          @protocol == request_params[:protocol]
        start(request_params)
      end

      # Detect if the body is a streamable object like a file or socket.  If so, stream that
      # bad boy.
      setup_streaming(request)
      response = @http.request(request, &block)

      error_reset
      eof_reset
      return response

      # We treat EOF errors and the timeout/network errors differently.  Both
      # are tracked in different statistics blocks.  Note below that EOF
      # errors will sleep for a certain (exponentially increasing) period.
      # Other errors don't sleep because there is already an inherent delay
      # in them; connect and read timeouts (for example) have already
      # 'slept'.  It is still not clear which way we should treat errors
      # like RST and resolution failures.  For now, there is no additional
      # delay for these errors although this may change in the future.

      # EOFError means the server closed the connection on us.
    rescue EOFError => e
      @logger.debug("#{err_header} server #{@server} closed connection")
      @http = nil

      # if we have waited long enough - raise an exception...
      if raise_on_eof_exception?
        exception = get_param(:exception) || RuntimeError
        @logger.warn("#{err_header} raising #{exception} due to permanent EOF being received from #{@server}, error age: #{Time.now.to_i - eof_time.to_i}")
        raise exception.new("Permanent EOF is being received from #{@server}.")
      else
        # ... else just sleep a bit before new retry
        sleep(add_eof)
        # We will be retrying the request, so reset the file pointer
        reset_fileptr_offset(request, mypos)
      end
    rescue Exception => e  # See comment at bottom for the list of errors seen...
      @http = nil
      # if ctrl+c is pressed - we have to reraise exception to terminate proggy
      if e.is_a?(Interrupt) && !( e.is_a?(Errno::ETIMEDOUT) || e.is_a?(Timeout::Error))
        @logger.debug( "#{err_header} request to server #{@server} interrupted by ctrl-c")
        raise
      elsif e.is_a?(ArgumentError) && e.message.include?('wrong number of arguments (5 for 4)')
        # seems our net_fix patch was overriden...
        exception = get_param(:exception) || RuntimeError
        raise exception.new('incompatible Net::HTTP monkey-patch')
      end
      # oops - we got a banana: log it
      error_add(e.message)
      @logger.warn("#{err_header} request failure count: #{error_count}, exception: #{e.inspect}")

      # We will be retrying the request, so reset the file pointer
      reset_fileptr_offset(request, mypos)

    end
  end
end

#request(request_params, &block) ⇒ Object

Raises RuntimeError, Interrupt, and params (if specified in new).



312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
 
# File 'lib/awsbase/http_connection.rb', line 312

def request(request_params, &block)
  # We save the offset here so that if we need to retry, we can return the file pointer to its initial position
  mypos = get_fileptr_offset(request_params)
  loop do
    # if we are inside a delay between retries: no requests this time!
    if error_count > @params[:http_connection_retry_count] &&
        error_time + @params[:http_connection_retry_delay] > Time.now
      # store the message (otherwise it will be lost after error_reset and
      # we will raise an exception with an empty text)
      banana_message_text = banana_message
      @logger.warn("#{err_header} re-raising same error: #{banana_message_text} " +
                   "-- error count: #{error_count}, error age: #{Time.now.to_i - error_time.to_i}")
      exception = get_param(:exception) || RuntimeError
      raise exception.new(banana_message_text)
    end

    # try to connect server(if connection does not exist) and get response data
    begin
      request_params[:protocol] ||= (request_params[:port] == 443 ? 'https' : 'http')

      request = request_params[:request]
      request['User-Agent'] = get_param(:user_agent) || ''

      # (re)open connection to server if none exists or params has changed
      unless @http          &&
          @http.started? &&
          @server   == request_params[:server] &&
          @port     == request_params[:port]   &&
          @protocol == request_params[:protocol]
        start(request_params)
      end

      # Detect if the body is a streamable object like a file or socket.  If so, stream that
      # bad boy.
      setup_streaming(request)
      response = @http.request(request, &block)

      error_reset
      eof_reset
      return response

      # We treat EOF errors and the timeout/network errors differently.  Both
      # are tracked in different statistics blocks.  Note below that EOF
      # errors will sleep for a certain (exponentially increasing) period.
      # Other errors don't sleep because there is already an inherent delay
      # in them; connect and read timeouts (for example) have already
      # 'slept'.  It is still not clear which way we should treat errors
      # like RST and resolution failures.  For now, there is no additional
      # delay for these errors although this may change in the future.

      # EOFError means the server closed the connection on us.
    rescue EOFError => e
      @logger.debug("#{err_header} server #{@server} closed connection")
      @http = nil

      # if we have waited long enough - raise an exception...
      if raise_on_eof_exception?
        exception = get_param(:exception) || RuntimeError
        @logger.warn("#{err_header} raising #{exception} due to permanent EOF being received from #{@server}, error age: #{Time.now.to_i - eof_time.to_i}")
        raise exception.new("Permanent EOF is being received from #{@server}.")
      else
        # ... else just sleep a bit before new retry
        sleep(add_eof)
        # We will be retrying the request, so reset the file pointer
        reset_fileptr_offset(request, mypos)
      end
    rescue Exception => e  # See comment at bottom for the list of errors seen...
      @http = nil
      # if ctrl+c is pressed - we have to reraise exception to terminate proggy
      if e.is_a?(Interrupt) && !( e.is_a?(Errno::ETIMEDOUT) || e.is_a?(Timeout::Error))
        @logger.debug( "#{err_header} request to server #{@server} interrupted by ctrl-c")
        raise
      elsif e.is_a?(ArgumentError) && e.message.include?('wrong number of arguments (5 for 4)')
        # seems our net_fix patch was overriden...
        exception = get_param(:exception) || RuntimeError
        raise exception.new('incompatible Net::HTTP monkey-patch')
      end
      # oops - we got a banana: log it
      error_add(e.message)
      @logger.warn("#{err_header} request failure count: #{error_count}, exception: #{e.inspect}")

      # We will be retrying the request, so reset the file pointer
      reset_fileptr_offset(request, mypos)

    end
  end
end

#socket_read_size=(newsize) ⇒ Object

Set the maximum size (in bytes) of a single read from the underlying socket. For bulk transfer, especially over fast links, this is value is critical to performance.



151
152
153
 
# File 'lib/awsbase/http_connection.rb', line 151

def socket_read_size=(newsize)
  Net::BufferedIO.socket_read_size=(newsize)
end

#socket_read_size? ⇒ Boolean

Query for the maximum size (in bytes) of a single read from the underlying socket. For bulk transfer, especially over fast links, this is value is critical to performance.

Returns:

  • (Boolean) 


144
145
146
 
# File 'lib/awsbase/http_connection.rb', line 144

def socket_read_size?
  Net::BufferedIO.socket_read_size?
end