Method: NetSuite::Utilities#backoff

Defined in:
lib/netsuite/utilities.rb

#backoff(options = {}) ⇒ Object 



94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
 
# File 'lib/netsuite/utilities.rb', line 94

def backoff(options = {})
  # TODO the default backoff attempts should be customizable the global config
  options[:attempts] ||= 8

  count = 0

  begin
    count += 1
    yield
  rescue StandardError => e
    exceptions_to_retry = [
      Errno::ECONNRESET,
      Errno::ETIMEDOUT,
      Errno::EHOSTUNREACH,
      EOFError,
      Wasabi::Resolver::HTTPError,
      Savon::SOAPFault,
      Savon::InvalidResponseError,
      Zlib::BufError,
      Savon::HTTPError,
      SocketError,
      Net::OpenTimeout
    ]

    # available in ruby > 1.9
    if defined?(Net::ReadTimeout)
      exceptions_to_retry << Net::ReadTimeout
    end

    # available in ruby > 2.2.0
    exceptions_to_retry << IO::EINPROGRESSWaitWritable if defined?(IO::EINPROGRESSWaitWritable)
    exceptions_to_retry << OpenSSL::SSL::SSLErrorWaitReadable if defined?(OpenSSL::SSL::SSLErrorWaitReadable)

    # depends on the http library chosen
    exceptions_to_retry << HTTPI::SSLError if defined?(HTTPI::SSLError)
    exceptions_to_retry << HTTPI::TimeoutError if defined?(HTTPI::TimeoutError)
    exceptions_to_retry << HTTPClient::TimeoutError if defined?(HTTPClient::TimeoutError)
    exceptions_to_retry << HTTPClient::ConnectTimeoutError if defined?(HTTPClient::ConnectTimeoutError)
    exceptions_to_retry << HTTPClient::ReceiveTimeoutError if defined?(HTTPClient::ReceiveTimeoutError)
    exceptions_to_retry << HTTPClient::SendTimeoutError if defined?(HTTPClient::SendTimeoutError)
    exceptions_to_retry << Excon::Error::Timeout if defined?(Excon::Error::Timeout)
    exceptions_to_retry << Excon::Error::Socket if defined?(Excon::Error::Socket)

    if !exceptions_to_retry.include?(e.class)
      raise
    end

    # whitelist certain SOAPFaults; all other network errors should automatically retry
    if e.is_a?(Savon::SOAPFault)
      # https://github.com/stripe/stripe-netsuite/issues/815
      if !e.message.include?("Only one request may be made against a session at a time") &&
        !e.message.include?('java.util.ConcurrentModificationException') &&
        !e.message.include?('java.lang.NullPointerException') &&
        !e.message.include?('java.lang.IllegalStateException') &&
        !e.message.include?('java.lang.reflect.InvocationTargetException') &&
        !e.message.include?('com.netledger.common.exceptions.NLDatabaseOfflineException') &&
        !e.message.include?('com.netledger.database.NLConnectionUtil$NoCompanyDbsOnlineException') &&
        !e.message.include?('com.netledger.cache.CacheUnavailableException') &&
        !e.message.include?('java.lang.IllegalStateException') &&
        !e.message.include?('An unexpected error occurred.') &&
        !e.message.include?('An unexpected error has occurred.  Technical Support has been alerted to this problem.') &&
        !e.message.include?('Session invalidation is in progress with different thread') &&
        !e.message.include?('[missing resource APP:ERRORMESSAGE:WS_AN_UNEXPECTED_ERROR_OCCURRED] [missing resource APP:ERRORMESSAGE:ERROR_ID_1]') &&
        !e.message.include?('SuiteTalk concurrent request limit exceeded. Request blocked.') &&
        # maintenance is the new outage: this message is being used for intermittent errors
        !e.message.include?('The account you are trying to access is currently unavailable while we undergo our regularly scheduled maintenance.') &&
        !e.message.include?('The Connection Pool is not intialized.') &&
        # it looks like NetSuite mispelled their error message...
        !e.message.include?('The Connection Pool is not intiialized.')
        raise
      end
    end

    if count >= options[:attempts]
      raise
    end

    # log.warn("concurrent request failure", sleep: count, attempt: count)
    sleep(count)

    retry
  end
end