Module: MongoHA::MongoClient::InstanceMethods

Defined in:
lib/mongo_ha/mongo_client.rb

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.included(base) ⇒ Object

Add retry logic to MongoClient



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/mongo_ha/mongo_client.rb', line 36

def self.included(base)
  base.class_eval do
    # Give MongoClient a class-specific logger if SemanticLogger V2.12 or above is available
    # to give better logging information during a connection recovery scenario
    if defined?(SemanticLogger::DebugAsTraceLogger)
      # Map Debug level calls to trace to reduce log file clutter
      @@logger = SemanticLogger::DebugAsTraceLogger.new(self)

      def self.logger
        @@logger
      end

      def logger
        self.class.logger
      end
    end

    alias_method :receive_message_original, :receive_message
    alias_method :connect_original, :connect
    alias_method :valid_opts_original, :valid_opts
    alias_method :setup_original, :setup

    attr_accessor *CONNECTION_RETRY_OPTS

    # Prevent multiple threads from trying to reconnect at the same time during
    # connection failures
    @@failover_mutex = Mutex.new
    # Wrap internal networking calls with retry logic

    # Do not stub out :send_message_with_gle or :send_message
    # It modifies the message, see CollectionWriter#send_write_operation

    def receive_message(*args)
      retry_on_connection_failure do
        receive_message_original *args
      end
    end

    def connect(*args)
      retry_on_connection_failure do
        connect_original *args
      end
    end

    protected

    def valid_opts(*args)
      valid_opts_original(*args) + CONNECTION_RETRY_OPTS
    end

    def setup(opts)
      self.reconnect_attempts          = (opts.delete(:reconnect_attempts) || 53).to_i
      self.reconnect_retry_seconds     = (opts.delete(:reconnect_retry_seconds) || 0.1).to_f
      self.reconnect_retry_multiplier  = (opts.delete(:reconnect_retry_multiplier) || 2).to_f
      self.reconnect_max_retry_seconds = (opts.delete(:reconnect_max_retry_seconds) || 5).to_f
      setup_original(opts)
    end

  end
end

Instance Method Details

#_reconnect ⇒ Object

Call this method whenever a Mongo::ConnectionFailure Exception has been raised to re-establish the connection

This method is thread-safe and ensure that only one thread at a time per connection will attempt to re-establish the connection

Returns whether the connection is connected again



157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
 
# File 'lib/mongo_ha/mongo_client.rb', line 157

def _reconnect
  logger.debug "Going to reconnect"

  # Prevent other threads from invoking reconnect logic at the same time
  @@failover_mutex.synchronize do
    # Another thread may have already failed over the connection by the
    # time this threads gets in
    if active?
      logger.info "Connected to: #{primary.inspect}"
      return true
    end

    # Close all sockets that are not checked out so that other threads not
    # currently waiting on Mongo, don't get bad connections and have to
    # retry each one in turn
    @primary_pool.close if @primary_pool

    if reconnect_attempts > 0
      # Wait for other threads to finish working on their sockets
      retries = 1
      retry_seconds = reconnect_retry_seconds
      begin
        logger.warn "Connection unavailable. Waiting: #{retry_seconds} seconds before retrying"
        sleep retry_seconds
        # Call original connect method since it is already within a retry block
        connect_original
      rescue Mongo::ConnectionFailure => exc
        if retries < reconnect_attempts
          retries += 1
          retry_seconds *=  reconnect_retry_multiplier
          retry_seconds = reconnect_max_retry_seconds if retry_seconds > reconnect_max_retry_seconds
          retry
        end

        logger.error "Auto-reconnect giving up after #{retries} reconnect attempts"
        raise exc
      end
      logger.info "Successfully reconnected to: #{primary.inspect}"
    end
    connected?
  end

end

#retry_on_connection_failure(&block) ⇒ Object

Retry the supplied block when a Mongo::ConnectionFailure occurs

Note: Check for Duplicate Key on inserts

Returns the result of the block

Example:

connection.retry_on_connection_failure { |retried| connection.ping }



105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/mongo_ha/mongo_client.rb', line 105

def retry_on_connection_failure(&block)
  raise "Missing mandatory block parameter on call to Mongo::Connection#retry_on_connection_failure" unless block
  retried = false
  mongos_retries = 0
  begin
    result = block.call(retried)
    retried = false
    result
  rescue Mongo::ConnectionFailure => exc
    # Retry if reconnected, but only once to prevent an infinite loop
    logger.warn "Connection Failure: '#{exc.message}' [#{exc.error_code}]"
    if !retried && _reconnect
      retried = true
      # TODO There has to be a way to flush the connection pool of all inactive connections
      retry
    end
    raise exc
  rescue Mongo::OperationFailure => exc
    # Workaround not master issue. Disconnect connection when we get a not master
    # error message. Master checks for an exact match on "not master", whereas
    # it sometimes gets: "not master and slaveok=false"
    if exc.result
      error = exc.result['err'] || exc.result['errmsg']
      close if error && error.include?("not master")
    end

    # These get returned when connected to a local mongos router when it in turn
    # has connection failures talking to the remote shards. All we do is retry the same operation
    # since it's connections to multiple remote shards may have failed.
    # Disconnecting the current connection will not help since it is just to the mongos router
    # First make sure it is connected to the mongos router
    raise exc unless (MONGOS_CONNECTION_ERRORS.any? { |err| exc.message.include?(err) }) || (exc.message.strip == ':')

    mongos_retries += 1
    if mongos_retries <= 60
      retried = true
      Kernel.sleep(0.5)
      logger.warn "[#{primary.inspect}] Router Connection Failure. Retry ##{mongos_retries}. Exc: '#{exc.message}' [#{exc.error_code}]"
      # TODO Is there a way to flush the connection pool of all inactive connections
      retry
    end
    raise exc
  end
end