Class: Amigo::Autoscaler

Inherits:
Object
  • Object
show all
Defined in:
lib/amigo/autoscaler.rb

Defined Under Namespace

Classes: InvalidHandler

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(poll_interval: 20, latency_threshold: 5, hostname_regex: /^web\.1$/, handlers: ["log"], alert_interval: 120) ⇒ Autoscaler

Returns a new instance of Autoscaler.



54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/amigo/autoscaler.rb', line 54

def initialize(
  poll_interval: 20,
  latency_threshold: 5,
  hostname_regex: /^web\.1$/,
  handlers: ["log"],
  alert_interval: 120
)

  @poll_interval = poll_interval
  @latency_threshold = latency_threshold
  @hostname_regex = hostname_regex
  @handlers = handlers
  @alert_interval = alert_interval
end

Instance Attribute Details

#alert_interval ⇒ Integer (readonly)

Only alert this often. For example, with poll_interval of 10 seconds and alert_interval of 200 seconds, we’d alert once and then 210 seconds later.

Returns:

  • (Integer) 


52
53
54
 
# File 'lib/amigo/autoscaler.rb', line 52

def alert_interval
  @alert_interval
end

#handlers ⇒ Array<String,Proc> (readonly)

Methods to call when alerting. Valid values are ‘log’ and ‘sentry’ (requires Sentry to be required already). Anything that responds to call will be invoked with a hash of ‘name => latency in seconds`.

Returns:

  • (Array<String,Proc>) 


46
47
48
 
# File 'lib/amigo/autoscaler.rb', line 46

def handlers
  @handlers
end

#hostname_regex ⇒ Regexp (readonly)

What hosts/processes should this run on? Look at ENV and Socket.gethostname. Default to only run on ‘web.1’, which is the first Heroku web dyno. We run on the web, not worker, dyno, so we report backed up queues in case we, say, turn off all workers (broken web processes are generally easier to find).

Returns:

  • (Regexp) 


40
41
42
 
# File 'lib/amigo/autoscaler.rb', line 40

def hostname_regex
  @hostname_regex
end

#latency_threshold ⇒ Integer (readonly)

What latency should we alert on?

Returns:

  • (Integer) 


32
33
34
 
# File 'lib/amigo/autoscaler.rb', line 32

def latency_threshold
  @latency_threshold
end

#poll_interval ⇒ Integer (readonly)

How often should Autoscaler check for latency?

Returns:

  • (Integer) 


29
30
31
 
# File 'lib/amigo/autoscaler.rb', line 29

def poll_interval
  @poll_interval
end

Instance Method Details

#alert_log(names_and_latencies) ⇒ Object 



137
138
139
 
# File 'lib/amigo/autoscaler.rb', line 137

def alert_log(names_and_latencies)
  self.log(:warn, "high_latency_queues", queues: names_and_latencies)
end

#alert_sentry(names_and_latencies) ⇒ Object 



129
130
131
132
133
134
135
 
# File 'lib/amigo/autoscaler.rb', line 129

def alert_sentry(names_and_latencies)
  Sentry.with_scope do |scope|
    scope.set_extras(high_latency_queues: names_and_latencies)
    names = names_and_latencies.map(&:first).sort.join(", ")
    Sentry.capture_message("Some queues have a high latency: #{names}")
  end
end

#alert_test(_names_and_latencies) ⇒ Object 



141
 
# File 'lib/amigo/autoscaler.rb', line 141

def alert_test(_names_and_latencies); end

#check ⇒ Object 



110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/amigo/autoscaler.rb', line 110

def check
  now = Time.now
  skip_check = now < (@last_alerted + self.poll_interval)
  if skip_check
    self.log(:debug, "async_autoscaler_skip_check")
    return
  end
  self.log(:info, "async_autoscaler_check")
  high_latency_queues = Sidekiq::Queue.all.
    map { |q| [q.name, q.latency] }.
    select { |(_, latency)| latency > self.latency_threshold }.
    to_h
  return if high_latency_queues.empty?
  @alert_methods.each do |m|
    m.respond_to?(:call) ? m.call(high_latency_queues) : self.send(m, high_latency_queues)
  end
  @last_alerted = now
end

#polling_thread ⇒ Object 



69
70
71
 
# File 'lib/amigo/autoscaler.rb', line 69

def polling_thread
  return @polling_thread
end

#setup ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/amigo/autoscaler.rb', line 73

def setup
  # Store these as strings OR procs, rather than grabbing self.method here.
  # It gets extremely hard ot test if we capture the method here.
  @alert_methods = self.handlers.map do |a|
    if a.respond_to?(:call)
      a
    else
      method_name = meth = "alert_#{a.strip}".to_sym
      raise InvalidHandler, a.inspect unless self.method(method_name)
      meth
    end
  end
  @last_alerted = Time.at(0)
  @stop = false
end

#start ⇒ Object 



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/amigo/autoscaler.rb', line 89

def start
  raise "already started" unless @polling_thread.nil?

  hostname = ENV.fetch("DYNO") { Socket.gethostname }
  return false unless self.hostname_regex.match?(hostname)

  self.log(:info, "async_autoscaler_starting")
  self.setup
  @polling_thread = Thread.new do
    until @stop
      Kernel.sleep(self.poll_interval)
      self.check unless @stop
    end
  end
  return true
end

#stop ⇒ Object 



106
107
108
 
# File 'lib/amigo/autoscaler.rb', line 106

def stop
  @stop = true
end