Class: Cron::Server

Inherits:
Object
  • Object
show all
Includes:
ServerProcessAble, StandardModel
Defined in:
lib/app/jobs/cron/server.rb

Overview

Handle the coordination of which server should be running the cron jobs

Constant Summary collapse

STATE_PRIMARY =

Constants

'primary'
STATE_SECONDARY = 
'secondary'
ALL_STATES = 
[STATE_PRIMARY, STATE_SECONDARY].freeze

Class Method Summary collapse

Instance Method Summary collapse

Methods included from ServerProcessAble

#check_in, included, #start, #stop

Methods included from StandardModel

#audit_action, #auto_strip_attributes, #capture_user_info, #clear_cache, #created_by_display_name, #delete_and_log, #destroy_and_log, included, #last_modified_by_display_name, #log_change, #log_deletion, #remove_blank_secure_fields, #save_and_log, #save_and_log!, #secure_fields, #update, #update!, #update_and_log, #update_and_log!

Methods included from App47Logger

clean_params, #clean_params, delete_parameter_keys, #log_controller_error, log_debug, #log_debug, log_error, #log_error, log_exception, #log_message, log_message, #log_warn, log_warn, mask_parameter_keys, #update_flash_messages

Class Method Details

.auto_scaling_configured? ⇒ Boolean

Test if autoscaling is configured, return false if there is an error

Returns:

  • (Boolean) 


189
190
191
192
193
 
# File 'lib/app/jobs/cron/server.rb', line 189

def self.auto_scaling_configured?
  SystemConfiguration.aws_auto_scaling_configured?
rescue StandardError
  false
end

.find_or_create_server ⇒ Object

Find a record for this server



77
78
79
 
# File 'lib/app/jobs/cron/server.rb', line 77

def self.find_or_create_server
  Cron::Server.find_or_create_by!(host_name: Socket.gethostname, pid: Process.pid)
end

.primary_server ⇒ Object

Find a the current master



84
85
86
 
# File 'lib/app/jobs/cron/server.rb', line 84

def self.primary_server
  Cron::Server.where(state: STATE_PRIMARY).first
end

.warm_up_server ⇒ Object

Warm up a server on the next evaluation



91
92
93
94
95
 
# File 'lib/app/jobs/cron/server.rb', line 91

def self.warm_up_server
  return unless auto_scaling_configured?

  primary_server.auto_scale([primary_server.desired_server_count + 1, 10].min)
end

Instance Method Details

#active_count ⇒ Object

Returns the count of active servers



289
290
291
 
# File 'lib/app/jobs/cron/server.rb', line 289

def active_count
  current_server_count
end

#alive? ⇒ Boolean

Return true if I’ve reported in the last two minutes

Returns:

  • (Boolean) 


140
141
142
 
# File 'lib/app/jobs/cron/server.rb', line 140

def alive?
  last_check_in_at >= 90.seconds.ago.utc
end

#auto_scale(desired_count = 0) ⇒ Object

Sets the desired and minimum number of EC2 instances to run



264
265
266
267
268
269
270
271
272
273
274
 
# File 'lib/app/jobs/cron/server.rb', line 264

def auto_scale(desired_count = 0)
  set desired_server_count: desired_count
  # Make sure we don't remove any workers with assigned jobs by accident
  return if desired_count.positive? && desired_count <= current_desired_capacity

  client.update_auto_scaling_group(auto_scaling_group_name: sys_config.aws_auto_scaling_group_name,
                                   min_size: desired_count,
                                   desired_capacity: desired_count)
rescue StandardError => error
  App47Logger.log_error "Unable to set auto scaler to #{desired_count}", error
end

#auto_scaling_configured? ⇒ Boolean

Test if autoscaling is configured, return false if there is an error

Returns:

  • (Boolean) 


180
181
182
183
184
 
# File 'lib/app/jobs/cron/server.rb', line 180

def auto_scaling_configured?
  @auto_scaling_configured ||= sys_config.aws_auto_scaling_configured?
rescue StandardError
  false
end

#auto_scaling_group ⇒ Object

Returns the AutoScalingGroup associated with the account



198
199
200
201
 
# File 'lib/app/jobs/cron/server.rb', line 198

def auto_scaling_group
  filter = { auto_scaling_group_names: [sys_config.aws_auto_scaling_group_name] }
  @auto_scaling_group ||= client.describe_auto_scaling_groups(filter).auto_scaling_groups.first
end

#become_primary ⇒ Object

Become primary, making others secondary



100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/app/jobs/cron/server.rb', line 100

def become_primary
  Cron::Server.each(&:become_secondary)
  # sleep a small amount of time to randomize a new primary
  sleep rand(1..15)
  # Check to see if another node already became primary
  primary = Cron::Server.primary_server
  return if primary.present? && primary.alive?

  # no one else is in, so become primary
  update! state: STATE_PRIMARY, last_check_in_at: Time.now.utc
end

#become_secondary(user = nil) ⇒ Object

Become secondary node



115
116
117
118
119
120
121
 
# File 'lib/app/jobs/cron/server.rb', line 115

def become_secondary(user = nil)
  if user.present?
    update_and_log! user, state: STATE_SECONDARY
  else
    update! state: STATE_SECONDARY
  end
end

#check_auto_scale ⇒ Object

Auto scale environment



154
155
156
157
158
159
160
161
162
 
# File 'lib/app/jobs/cron/server.rb', line 154

def check_auto_scale
  return unless auto_scaling_configured?

  if delayed_jobs_count.eql?(0)
    handle_zero_job_count
  else
    handle_auto_scale_jobs
  end
end

#client ⇒ Object

Returns the AWS AutoScaling Client



167
168
169
170
171
 
# File 'lib/app/jobs/cron/server.rb', line 167

def client
  @client ||= Aws::AutoScaling::Client.new(access_key_id: sys_config.aws_access_key_id,
                                           secret_access_key: sys_config.aws_secret_access_key,
                                           region: sys_config.aws_region)
end

#current_desired_capacity ⇒ Object

Returns the current value of ‘desired capacity’ for the AutoScalingGroup



213
214
215
216
217
218
219
 
# File 'lib/app/jobs/cron/server.rb', line 213

def current_desired_capacity
  current = auto_scaling_group.desired_capacity
  set current_server_count: current
  current
rescue StandardError
  0
end

#dead? ⇒ Boolean

Is the server dead, meaning is it not reporting within the last two minutes

Returns:

  • (Boolean) 


147
148
149
 
# File 'lib/app/jobs/cron/server.rb', line 147

def dead?
  !alive?
end

#delayed_jobs_count ⇒ Object

Returns a count of the Delayed Jobs in queue that have not failed



206
207
208
 
# File 'lib/app/jobs/cron/server.rb', line 206

def delayed_jobs_count
  @delayed_jobs_count ||= Delayed::Backend::Mongoid::Job.where(failed_at: nil).read(mode: :primary).count
end

#ensure_last_check_in ⇒ Object 



300
301
302
 
# File 'lib/app/jobs/cron/server.rb', line 300

def ensure_last_check_in
  self.last_check_in_at ||= Time.now.utc
end

#execute ⇒ Object

Go through the logic once a minute



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/app/jobs/cron/server.rb', line 35

def execute
  if primary?
    run_cron_jobs
  else
    primary = Cron::Server.where(state: STATE_PRIMARY).first
    if primary.blank? || primary.dead?
      become_primary
      run_cron_jobs
    end
  end
  time_to_next_run
rescue StandardError => error
  App47Logger.log_error 'Unable to run cron server', error
  time_to_next_run
ensure
  check_in
  GC.start
end

#handle_auto_scale_jobs ⇒ Object

Calls the ‘auto_scale’ method with a variable ‘desired_count’ based on how many jobs are running We don’t need any more workers if the job count is less than 1,000



234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
 
# File 'lib/app/jobs/cron/server.rb', line 234

def handle_auto_scale_jobs
  return if delayed_jobs_count < 50

  case delayed_jobs_count
  when 50..250
    auto_scale(1)
  when 251..500
    auto_scale(2)
  when 501..1_000
    auto_scale(3)
  when 1_001..2_000
    auto_scale(4)
  when 2_001..3_999
    auto_scale(4)
  when 4_000..7_999
    auto_scale(5)
  when 8_000..10_999
    auto_scale(5)
  when 11_000..13_999
    auto_scale(6)
  when 14_000..17_999
    auto_scale(6)
  else
    auto_scale(7)
  end
end

#handle_zero_job_count ⇒ Object

Calls the ‘auto_scale’ method with a ‘desired_count’ of 0 unless the capacity is already at 0



224
225
226
227
228
 
# File 'lib/app/jobs/cron/server.rb', line 224

def handle_zero_job_count
  return if current_desired_capacity.eql?(0)

  auto_scale
end

#high_lander ⇒ Object

Look to make sure there is only one primary



279
280
281
282
283
284
 
# File 'lib/app/jobs/cron/server.rb', line 279

def high_lander
  return if secondary? # Don't need to check if not primary

  primary = Cron::Server.where(state: STATE_PRIMARY).first
  errors.add(:state, 'there can only be one primary') unless primary.blank? || primary.eql?(self)
end

#inactive_count ⇒ Object

Returns the count of inactive servers



296
297
298
 
# File 'lib/app/jobs/cron/server.rb', line 296

def inactive_count
  desired_server_count
end

#primary? ⇒ Boolean

Am I the primary server

Returns:

  • (Boolean) 


126
127
128
 
# File 'lib/app/jobs/cron/server.rb', line 126

def primary?
  alive? && STATE_PRIMARY.eql?(state)
end

#run_cron_jobs ⇒ Object 



54
55
56
57
 
# File 'lib/app/jobs/cron/server.rb', line 54

def run_cron_jobs
  run_jobs
  check_auto_scale
end

#run_jobs ⇒ Object

Run all cron tab jobs



62
63
64
65
 
# File 'lib/app/jobs/cron/server.rb', line 62

def run_jobs
  now = Time.now.utc
  Cron::Tab.all.each { |tab| tab.run if tab.time_to_run?(now) }
end

#secondary? ⇒ Boolean

Am I a secondary server

Returns:

  • (Boolean) 


133
134
135
 
# File 'lib/app/jobs/cron/server.rb', line 133

def secondary?
  STATE_SECONDARY.eql?(state)
end

#sys_config ⇒ Object 



173
174
175
 
# File 'lib/app/jobs/cron/server.rb', line 173

def sys_config
  @sys_config ||= SystemConfiguration.configuration
end

#time_to_next_run ⇒ Object

Determine the next minute to run,



70
71
72
 
# File 'lib/app/jobs/cron/server.rb', line 70

def time_to_next_run
  60 - Time.now.utc.to_i % 60
end