Class: Amigo::Autoscaler

Inherits:
Object
  • Object
show all
Defined in:
lib/amigo/autoscaler.rb,
lib/amigo/autoscaler/handlers/log.rb,
lib/amigo/autoscaler/checkers/fake.rb,
lib/amigo/autoscaler/handlers/fake.rb,
lib/amigo/autoscaler/checkers/chain.rb,
lib/amigo/autoscaler/handlers/chain.rb,
lib/amigo/autoscaler/handlers/heroku.rb,
lib/amigo/autoscaler/handlers/sentry.rb,
lib/amigo/autoscaler/checkers/sidekiq.rb,
lib/amigo/autoscaler/checkers/web_latency.rb,
lib/amigo/autoscaler/checkers/puma_pool_usage.rb

Defined Under Namespace

Modules: Checkers, Handlers Classes: Checker, Handler, Persisted

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(handler:, checker:, poll_interval: 20, latency_threshold: 5, usage_threshold: 1, hostname_regex: /^web\.1$/, alert_interval: 120, latency_restored_threshold: latency_threshold, on_unhandled_exception: nil, namespace: "amigo/autoscaler") ⇒ Autoscaler

Returns a new instance of Autoscaler.

Raises:

  • (ArgumentError) 


101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/amigo/autoscaler.rb', line 101

def initialize(
  handler:,
  checker:,
  poll_interval: 20,
  latency_threshold: 5,
  usage_threshold: 1,
  hostname_regex: /^web\.1$/,
  alert_interval: 120,
  latency_restored_threshold: latency_threshold,
  on_unhandled_exception: nil,
  namespace: "amigo/autoscaler"
)
  raise ArgumentError, "latency_threshold must be > 0" if
    latency_threshold <= 0
  raise ArgumentError, "latency_restored_threshold must be >= 0" if
    latency_restored_threshold.negative?
  raise ArgumentError, "latency_restored_threshold must be <= latency_threshold" if
    latency_restored_threshold > latency_threshold
  @handler = handler
  @checker = checker
  @poll_interval = poll_interval
  @latency_threshold = latency_threshold
  @usage_threshold = usage_threshold
  @hostname_regex = hostname_regex
  @alert_interval = alert_interval
  @latency_restored_threshold = latency_restored_threshold
  @on_unhandled_exception = on_unhandled_exception
  @namespace = namespace
end

Instance Attribute Details

#alert_interval ⇒ Integer (readonly)

Only alert this often. For example, with poll_interval of 10 seconds and alert_interval of 200 seconds, we’d alert once and then 210 seconds later.

Returns:

  • (Integer) 


72
73
74
 
# File 'lib/amigo/autoscaler.rb', line 72

def alert_interval
  @alert_interval
end

#checker ⇒ Amigo::Autoscaler::Checker (readonly)

Returns:



84
85
86
 
# File 'lib/amigo/autoscaler.rb', line 84

def checker
  @checker
end

#handler ⇒ Amigo::Autoscaler::Handler (readonly)

Returns:



86
87
88
 
# File 'lib/amigo/autoscaler.rb', line 86

def handler
  @handler
end

#hostname_regex ⇒ Regexp (readonly)

What hosts/processes should this run on? Looks at ENV and Socket.gethostname for a match. Default to only run on ‘web.1’, which is the first Heroku web dyno. We run on the web, not worker, dyno, so we report backed up queues in case we, say, turn off all workers (broken web processes are generally easier to find).

Returns:

  • (Regexp) 


66
67
68
 
# File 'lib/amigo/autoscaler.rb', line 66

def hostname_regex
  @hostname_regex
end

#latency_restored_threshold ⇒ Object (readonly)

After an alert happens, what latency should be considered “back to normal” and scale_down will be called? In most cases this should be the same as (and defaults to) latency_threshold so that we’re ‘back to normal’ once we’re below the threshold. It may also commonly be 0, so that the callback is fired when the queue is entirely clear. Note that, if latency_restored_threshold is less than latency_threshold, while the latency is between the two, no alerts will fire.



81
82
83
 
# File 'lib/amigo/autoscaler.rb', line 81

def latency_restored_threshold
  @latency_restored_threshold
end

#latency_threshold ⇒ Numeric (readonly)

The latency, in seconds, that triggers an alert.

Returns:

  • (Numeric) 


52
53
54
 
# File 'lib/amigo/autoscaler.rb', line 52

def latency_threshold
  @latency_threshold
end

#namespace ⇒ Object (readonly)

Store autoscaler keys in this Redis namespace. Note that if you are running multiple autoscalers for different services (web, worker), you will need different namespaces.



91
92
93
 
# File 'lib/amigo/autoscaler.rb', line 91

def namespace
  @namespace
end

#on_unhandled_exception ⇒ Object (readonly)

Proc called with an exception that occurs while the thread is running. If the handler returns true, then the thread will keep going. All other values will kill the thread, which breaks autoscaling. Note that Amigo automatically logs unhandled exceptions at :error level. If you use an error reporter like Sentry, you can pass in something like:

-> (e) { Sentry.capture_exception(e) }



99
100
101
 
# File 'lib/amigo/autoscaler.rb', line 99

def on_unhandled_exception
  @on_unhandled_exception
end

#poll_interval ⇒ Integer (readonly)

How often the Autoscaler checks for latency/usage statistics.

Returns:

  • (Integer) 


48
49
50
 
# File 'lib/amigo/autoscaler.rb', line 48

def poll_interval
  @poll_interval
end

#usage_threshold ⇒ Object (readonly)

The pool usage, as a float between 0 and 1 (or above), that triggers an alert. Note that usage-based autoscaling should generally not be used for background jobs. It is much more useful for web autoscaling, since it is more responsive than latency.



57
58
59
 
# File 'lib/amigo/autoscaler.rb', line 57

def usage_threshold
  @usage_threshold
end

Instance Method Details

#_check ⇒ Object 



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
 
# File 'lib/amigo/autoscaler.rb', line 203

def _check
  now = Time.now
  skip_check = now < (@last_alerted + self.alert_interval)
  if skip_check
    self._debug(:debug, "async_autoscaler_skip_check")
    return
  end
  self._debug(:info, "async_autoscaler_check")
  high_latency_queues = self.checker.get_latencies.
    select { |_, latency| latency > self.latency_threshold }
  high_pool_usage = !(pu = self.checker.get_pool_usage).nil? && pu > self.usage_threshold
  if high_latency_queues.empty? && !high_pool_usage
    # Whenever we are in a latency event, we have a depth > 0. So a depth of 0 means
    # we're not in a latency event, and still have no latency, so can noop.
    return if @depth.zero?
    # We WERE in a latency event, and now we're not, so report on it.
    self.handler.scale_down(depth: @depth, duration: (Time.now - @latency_event_started).to_f)
    # Reset back to 0 depth so we know we're not in a latency event.
    @depth = 0
    @latency_event_started = Time.at(0)
    @last_alerted = now
    self.persist
    return
  end
  if @depth.positive?
    # We have already alerted, so increment the depth and when the latency started.
    @depth += 1
    duration = (Time.now - @latency_event_started).to_f
  else
    # Indicate we are starting a high latency event.
    @depth = 1
    @latency_event_started = Time.now
    duration = 0.0
  end
  @handler.scale_up(high_latencies: high_latency_queues, depth: @depth, duration: duration, pool_usage: pu)
  @last_alerted = now
  self.persist
end

#_debug(lvl, msg, **kw) ⇒ Object 



242
243
244
245
 
# File 'lib/amigo/autoscaler.rb', line 242

def _debug(lvl, msg, **kw)
  return unless ENV["DEBUG"]
  Amigo.log(nil, lvl, msg, kw)
end

#check ⇒ Object 



195
196
197
198
199
200
201
 
# File 'lib/amigo/autoscaler.rb', line 195

def check
  self._check
rescue StandardError => e
  self._debug(:error, "async_autoscaler_unhandled_error", exception: e)
  handled = self.on_unhandled_exception&.call(e)
  raise e unless handled.eql?(true)
end

#fetch_persisted ⇒ Object 



144
145
146
147
148
149
150
151
152
 
# File 'lib/amigo/autoscaler.rb', line 144

def fetch_persisted
  return Sidekiq.redis do |r|
    Persisted.new(
      Time.at((r.get("#{namespace}/last_alerted") || 0).to_f),
      (r.get("#{namespace}/depth") || 0).to_i,
      Time.at((r.get("#{namespace}/latency_event_started") || 0).to_f),
    )
  end
end

#polling_thread ⇒ Thread

Returns:

  • (Thread) 


132
133
134
 
# File 'lib/amigo/autoscaler.rb', line 132

def polling_thread
  return @polling_thread
end

#setup ⇒ Object 



136
137
138
139
140
141
142
 
# File 'lib/amigo/autoscaler.rb', line 136

def setup
  @thr_event = ThreadingEvent.new
  persisted = self.fetch_persisted
  @last_alerted = persisted.last_alerted_at
  @depth = persisted.depth
  @latency_event_started = persisted.latency_event_started_at
end

#start ⇒ Object 



173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
 
# File 'lib/amigo/autoscaler.rb', line 173

def start
  raise "already started" unless @polling_thread.nil?

  hostname = ENV.fetch("DYNO") { Socket.gethostname }
  return false unless self.hostname_regex.match?(hostname)

  self._debug(:info, "async_autoscaler_starting")
  self.setup
  @polling_thread = Thread.new do
    loop do
      @thr_event.wait(self.poll_interval)
      break if @thr_event.set?
      self.check
    end
  end
  return true
end

#stop ⇒ Object 



191
192
193
 
# File 'lib/amigo/autoscaler.rb', line 191

def stop
  @thr_event.set
end

#unpersist ⇒ Object

Delete all the keys that Autoscaler stores. Can be used in extreme cases where things need to be cleaned up, but should not be normally used.



165
166
167
168
169
170
171
 
# File 'lib/amigo/autoscaler.rb', line 165

def unpersist
  Sidekiq.redis do |r|
    r.del("#{namespace}/last_alerted")
    r.del("#{namespace}/depth")
    r.del("#{namespace}/latency_event_started")
  end
end