Class: Gitlab::SidekiqMiddleware::MemoryKiller

Inherits:
Object
  • Object
show all
Defined in:
lib/gitlab/sidekiq_middleware/memory_killer.rb

Constant Summary collapse

MAX_RSS =

Default the RSS limit to 0, meaning the MemoryKiller is disabled

(ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i
GRACE_TIME =

Give Sidekiq 15 minutes of grace time after exceeding the RSS limit

(ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
SHUTDOWN_WAIT =

Wait 30 seconds for running jobs to finish during graceful shutdown

(ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
SHUTDOWN_SIGNAL = 
(ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_SIGNAL'] || 'SIGKILL').to_s
MUTEX =

Create a mutex used to ensure there will be only one thread waiting to shut Sidekiq down

Mutex.new

Instance Method Summary collapse

Instance Method Details

#call(worker, job, queue) ⇒ Object 


16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/gitlab/sidekiq_middleware/memory_killer.rb', line 16

def call(worker, job, queue)
  yield
  current_rss = get_rss

  return unless MAX_RSS > 0 && current_rss > MAX_RSS

  Thread.new do
    # Return if another thread is already waiting to shut Sidekiq down
    return unless MUTEX.try_lock

    Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\
      "#{MAX_RSS}"
    Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} "\
      "in #{GRACE_TIME} seconds"
    sleep(GRACE_TIME)

    Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}"
    Process.kill('SIGTERM', Process.pid)

    Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\
      "#{SHUTDOWN_SIGNAL} to PID #{Process.pid}"
    sleep(SHUTDOWN_WAIT)

    Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid}"
    Process.kill(SHUTDOWN_SIGNAL, Process.pid)
  end
end