Class: Cron::Server

Inherits:
Object show all
Includes:
ServerProcessAble, StandardModel
Defined in:
lib/app/jobs/cron/server.rb

Overview

Handle the coordination of which server should be running the cron jobs

Constant Summary collapse

STATE_PRIMARY =

Constants

'primary'
STATE_SECONDARY = 
'secondary'
ALL_STATES = 
[STATE_PRIMARY, STATE_SECONDARY].freeze

Constants included from StandardModel

StandardModel::STANDARD_FIELDS

Class Method Summary collapse

Instance Method Summary collapse

Methods included from ServerProcessAble

#check_in, included, #start, #stop

Methods included from StandardModel

#audit_action, #auto_strip_attributes, #capture_user_info, #clear_cache, #created_by_display_name, #delete_able?, #delete_and_log, #destroy_and_log, #edit_able?, included, #last_modified_by_display_name, #log_change, #log_deletion, #remove_blank_secure_fields, #save_and_log, #save_and_log!, #secure_fields, #update, #update!, #update_and_log, #update_and_log!

Methods included from App47Logger

clean_params, #clean_params, delete_parameter_keys, #log_controller_error, log_debug, #log_debug, log_error, #log_error, log_exception, #log_message, log_message, #log_warn, log_warn, mask_parameter_keys, matches_filter?, #update_flash_messages

Class Method Details

.auto_scaling_configured? ⇒ Boolean

Test if autoscaling is configured, return false if there is an error

Returns:

  • (Boolean) 


180
181
182
183
184
 
# File 'lib/app/jobs/cron/server.rb', line 180

def self.auto_scaling_configured?
  SystemConfiguration.aws_auto_scaling_configured?
rescue StandardError
  false
end

.find_or_create_server ⇒ Object

Find a record for this server



77
78
79
 
# File 'lib/app/jobs/cron/server.rb', line 77

def self.find_or_create_server
  Cron::Server.find_or_create_by!(host_name: Socket.gethostname, pid: Process.pid)
end

.primary_server ⇒ Object

Find a the current master



84
85
86
 
# File 'lib/app/jobs/cron/server.rb', line 84

def self.primary_server
  Cron::Server.where(state: STATE_PRIMARY).first
end

.warm_up_server ⇒ Object

Warm up a server on the next evaluation



91
92
93
94
95
 
# File 'lib/app/jobs/cron/server.rb', line 91

def self.warm_up_server
  return unless auto_scaling_configured?

  primary_server.auto_scale([primary_server.desired_server_count + 1, 10].min)
end

Instance Method Details

#active_count ⇒ Object

Returns the count of active servers



282
283
284
 
# File 'lib/app/jobs/cron/server.rb', line 282

def active_count
  current_server_count
end

#alive? ⇒ Boolean

Return true if I’ve reported in the last two minutes

Returns:

  • (Boolean) 


140
141
142
 
# File 'lib/app/jobs/cron/server.rb', line 140

def alive?
  last_check_in_at >= 90.seconds.ago.utc
end

#auto_scale(desired_count = 0) ⇒ Object

Sets the desired and minimum number of EC2 instances to run



257
258
259
260
261
262
263
264
265
266
267
 
# File 'lib/app/jobs/cron/server.rb', line 257

def auto_scale(desired_count = 0)
  set desired_server_count: desired_count
  # Make sure we don't remove any workers with assigned jobs by accident
  return if desired_count.positive? && desired_count <= current_desired_capacity

  ecs_client.update_service(cluster: sys_config.aws_ecs_cluster_worker_name,
                            service: sys_config.aws_ecs_service_worker_name,
                            desired_count: desired_count)
rescue StandardError => error
  App47Logger.log_error "Unable to set ECS service auto scaler to #{desired_count}", error
end

#auto_scaling_configured? ⇒ Boolean

Test if autoscaling is configured, return false if there is an error

Returns:

  • (Boolean) 


171
172
173
174
175
 
# File 'lib/app/jobs/cron/server.rb', line 171

def auto_scaling_configured?
  @auto_scaling_configured ||= sys_config.aws_auto_scaling_configured?
rescue StandardError
  false
end

#become_primary ⇒ Object

Become primary, making others secondary



100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/app/jobs/cron/server.rb', line 100

def become_primary
  Cron::Server.each(&:become_secondary)
  # sleep a small amount of time to randomize a new primary
  sleep rand(1..15)
  # Check to see if another node already became primary
  primary = Cron::Server.primary_server
  return if primary.present? && primary.alive?

  # no one else is in, so become primary
  update! state: STATE_PRIMARY, last_check_in_at: Time.now.utc
end

#become_secondary(user = nil) ⇒ Object

Become secondary node



115
116
117
118
119
120
121
 
# File 'lib/app/jobs/cron/server.rb', line 115

def become_secondary(user = nil)
  if user.present?
    update_and_log! user, state: STATE_SECONDARY
  else
    update! state: STATE_SECONDARY
  end
end

#check_auto_scale ⇒ Object

Auto scale environment



154
155
156
157
158
159
160
161
162
 
# File 'lib/app/jobs/cron/server.rb', line 154

def check_auto_scale
  return unless auto_scaling_configured?

  if delayed_jobs_count.zero?
    handle_zero_job_count
  else
    handle_auto_scale_jobs
  end
end

#current_desired_capacity ⇒ Integer

This method is abstract.

Returns the current value of ‘desired capacity’ for the ECS Service

Returns:

  • (Integer) 


201
202
203
204
205
206
207
208
 
# File 'lib/app/jobs/cron/server.rb', line 201

def current_desired_capacity
  current = ecs_service.desired_count
  set current_server_count: current
  current
rescue StandardError
  puts 'error on desirect capacy'
  0
end

#dead? ⇒ Boolean

Is the server dead, meaning is it not reporting within the last two minutes

Returns:

  • (Boolean) 


147
148
149
 
# File 'lib/app/jobs/cron/server.rb', line 147

def dead?
  !alive?
end

#delayed_jobs_count ⇒ Integer

This method is abstract.

Returns a count of the Delayed Jobs in queue that have not failed

Returns:

  • (Integer) 


195
196
197
 
# File 'lib/app/jobs/cron/server.rb', line 195

def delayed_jobs_count
  @delayed_jobs_count ||= Delayed::Backend::Mongoid::Job.where(failed_at: nil).read(mode: :primary).count
end

#ecs_client ⇒ Aws::ECS::Client

This method is abstract.

Returns the AWS ECS client

Returns:

  • (Aws::ECS::Client) 


299
300
301
 
# File 'lib/app/jobs/cron/server.rb', line 299

def ecs_client
  @ecs_client ||= sys_config.aws_ecs_client
end

#ecs_service ⇒ Aws::ECS::Types::Service

This method is abstract.

Returns the AWS ECS service

Returns:

  • (Aws::ECS::Types::Service) 


188
189
190
191
 
# File 'lib/app/jobs/cron/server.rb', line 188

def ecs_service
  filter = { cluster: sys_config.aws_ecs_cluster_worker_name, services: [sys_config.aws_ecs_service_worker_name] }
  @ecs_service ||= ecs_client.describe_services(filter).services.first
end

#ensure_last_check_in ⇒ Object 



293
294
295
 
# File 'lib/app/jobs/cron/server.rb', line 293

def ensure_last_check_in
  self.last_check_in_at ||= Time.now.utc
end

#execute ⇒ Object

Go through the logic once a minute



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/app/jobs/cron/server.rb', line 35

def execute
  if primary?
    run_cron_jobs
  else
    primary = Cron::Server.where(state: STATE_PRIMARY).first
    if primary.blank? || primary.dead?
      become_primary
      run_cron_jobs
    end
  end
  time_to_next_run
rescue StandardError => error
  App47Logger.log_error 'Unable to run cron server', error
  time_to_next_run
ensure
  check_in
end

#handle_auto_scale_jobs ⇒ Object

Calls the ‘auto_scale’ method with a variable ‘desired_count’ based on how many jobs are running We don’t need any more workers if the job count is less than 1,000



223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/app/jobs/cron/server.rb', line 223

def handle_auto_scale_jobs
  return if delayed_jobs_count < sys_config.aws_min_worker_job_count

  case delayed_jobs_count
  when 1..200
    auto_scale(1)
  when 201..500
    auto_scale(2)
  when 501..800
    auto_scale(3)
  when 801..1_200
    auto_scale(4)
  when 1_201..2_000
    auto_scale(4)
  when 2_001..3_000
    auto_scale(5)
  when 3_001..4_000
    auto_scale(5)
  when 4_001..6_000
    auto_scale(6)
  when 6_001..8_000
    auto_scale(7)
  when 8_001..10_000
    auto_scale(8)
  when 10_001..13_000
    auto_scale(9)
  else
    auto_scale(10)
  end
end

#handle_zero_job_count ⇒ Object

Calls the ‘auto_scale’ method with a ‘desired_count’ of 0 unless the capacity is already at 0



213
214
215
216
217
 
# File 'lib/app/jobs/cron/server.rb', line 213

def handle_zero_job_count
  return if current_desired_capacity.zero?

  auto_scale
end

#high_lander ⇒ Object

Look to make sure there is only one primary



272
273
274
275
276
277
 
# File 'lib/app/jobs/cron/server.rb', line 272

def high_lander
  return if secondary? # Don't need to check if not primary

  primary = Cron::Server.where(state: STATE_PRIMARY).first
  errors.add(:state, 'there can only be one primary') unless primary.blank? || primary.eql?(self)
end

#inactive_count ⇒ Object

Returns the count of inactive servers



289
290
291
 
# File 'lib/app/jobs/cron/server.rb', line 289

def inactive_count
  desired_server_count
end

#primary? ⇒ Boolean

Am I the primary server

Returns:

  • (Boolean) 


126
127
128
 
# File 'lib/app/jobs/cron/server.rb', line 126

def primary?
  alive? && STATE_PRIMARY.eql?(state)
end

#run_cron_jobs ⇒ Object 



53
54
55
56
 
# File 'lib/app/jobs/cron/server.rb', line 53

def run_cron_jobs
  run_jobs
  check_auto_scale
end

#run_jobs ⇒ Object

This method is abstract.

Run all cron tab and emitter jobs



59
60
61
62
63
64
65
 
# File 'lib/app/jobs/cron/server.rb', line 59

def run_jobs
  now = Time.now.utc
  Cron::Tab.all.each { |tab| tab.run if tab.time_to_run?(now) }
  Cron::Emitter.descendants&.each(&:run)
rescue StandardError => error
  App47Logger.log_error 'Unable to run jobs', error
end

#secondary? ⇒ Boolean

Am I a secondary server

Returns:

  • (Boolean) 


133
134
135
 
# File 'lib/app/jobs/cron/server.rb', line 133

def secondary?
  STATE_SECONDARY.eql?(state)
end

#sys_config ⇒ Object 



164
165
166
 
# File 'lib/app/jobs/cron/server.rb', line 164

def sys_config
  @sys_config ||= SystemConfiguration.configuration
end

#time_to_next_run ⇒ Object

Determine the next minute to run,



70
71
72
 
# File 'lib/app/jobs/cron/server.rb', line 70

def time_to_next_run
  60 - Time.now.utc.to_i % 60
end