Class: Cron::Server

Inherits:
Object
  • Object
show all
Includes:
StandardModel
Defined in:
lib/app/jobs/cron/server.rb

Constant Summary collapse

STATE_PRIMARY =

Constants

'primary'.freeze
STATE_SECONDARY = 
'secondary'.freeze
ALL_STATES = 
[STATE_PRIMARY, STATE_SECONDARY].freeze

Class Method Summary collapse

Instance Method Summary collapse

Methods included from StandardModel

#clear_cache, included, #remove_blank_secure_fields, #secure_fields, #update, #update!

Methods included from App47Logger

#clean_params, #delete_parameter_keys, #log_controller_error, #log_debug, log_debug, log_error, #log_error, log_exception, #log_message, log_message, #log_warn, log_warn, #mask_parameter_keys, #update_flash_messages

Class Method Details

.find_or_create_server ⇒ Object

Find a record for this server



74
75
76
 
# File 'lib/app/jobs/cron/server.rb', line 74

def self.find_or_create_server
  Cron::Server.find_or_create_by!(host_name: Socket.gethostname, pid: Process.pid)
end

.primary_server ⇒ Object

Find a the current master



81
82
83
 
# File 'lib/app/jobs/cron/server.rb', line 81

def self.primary_server
  Cron::Server.where(state: STATE_PRIMARY).first
end

.warm_up_server ⇒ Object

Warm up a server on the next evaluation



88
89
90
91
92
 
# File 'lib/app/jobs/cron/server.rb', line 88

def self.warm_up_server
  return unless SystemConfiguration.aws_auto_scaling_configured?

  primary_server.auto_scale([primary_server.desired_server_count + 1, 10].min)
end

Instance Method Details

#active_count ⇒ Object

Returns the count of active servers



274
275
276
 
# File 'lib/app/jobs/cron/server.rb', line 274

def active_count
  current_server_count
end

#alive? ⇒ Boolean

Return true if I’ve reported in the last two minutes

Returns:

  • (Boolean) 


138
139
140
 
# File 'lib/app/jobs/cron/server.rb', line 138

def alive?
  last_check_in_at >= 90.seconds.ago.utc
end

#auto_scale(desired_count = 0) ⇒ Object

Sets the desired and minimum number of EC2 instances to run



251
252
253
254
255
256
257
258
259
 
# File 'lib/app/jobs/cron/server.rb', line 251

def auto_scale(desired_count = 0)
  set desired_server_count: desired_count
  # Make sure we don't remove any workers with assigned jobs by accident
  return if desired_count.positive? && desired_count <= current_desired_capacity

  client.update_auto_scaling_group(auto_scaling_group_name: sys_config.aws_auto_scaling_group_name,
                                   min_size: desired_count,
                                   desired_capacity: desired_count)
end

#auto_scaling_group ⇒ Object

Returns the AutoScalingGroup associated with the account



185
186
187
188
 
# File 'lib/app/jobs/cron/server.rb', line 185

def auto_scaling_group
  filter = { auto_scaling_group_names: [sys_config.auto_scaling_group_name] }
  @auto_scaling_group ||= client.describe_auto_scaling_groups(filter).auto_scaling_groups.first
end

#become_primary ⇒ Object

Become primary, making others secondary



97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/app/jobs/cron/server.rb', line 97

def become_primary
  Cron::Server.each(&:become_secondary)
  # sleep a small amount of time to randomize a new primary
  sleep rand(1..15)
  # Check to see if another node already became primary
  primary = Cron::Server.primary_server
  return if primary.present? && primary.alive?

  # no one else is in, so become primary
  update! state: STATE_PRIMARY, last_check_in_at: Time.now.utc
end

#become_secondary(user = nil) ⇒ Object

Become secondary node



112
113
114
115
116
117
118
119
 
# File 'lib/app/jobs/cron/server.rb', line 112

def become_secondary(user = nil)
  if user.present?
    # update_attributes_and_log! user, state: STATE_SECONDARY
    update! state: STATE_SECONDARY
  else
    update! state: STATE_SECONDARY
  end
end

#check_auto_scale ⇒ Object

Auto scale environment



159
160
161
162
163
164
165
166
167
 
# File 'lib/app/jobs/cron/server.rb', line 159

def check_auto_scale
  return unless SystemConfiguration.aws_auto_scaling_configured?

  if delayed_jobs_count.eql?(0)
    handle_zero_job_count
  else
    handle_auto_scale_jobs
  end
end

#check_in ⇒ Object

Perform a check in for the server



152
153
154
 
# File 'lib/app/jobs/cron/server.rb', line 152

def check_in
  set last_check_in_at: Time.now.utc
end

#client ⇒ Object

Returns the AWS AutoScaling Client



172
173
174
175
176
 
# File 'lib/app/jobs/cron/server.rb', line 172

def client
  @client ||= Aws::AutoScaling::Client.new(access_key_id: sys_config.aws_access_key_id,
                                           secret_access_key: sys_config.aws_secret_access_key,
                                           region: sys_config.aws_region)
end

#current_desired_capacity ⇒ Object

Returns the current value of ‘desired capacity’ for the AutoScalingGroup



200
201
202
203
204
205
206
 
# File 'lib/app/jobs/cron/server.rb', line 200

def current_desired_capacity
  current = auto_scaling_group.desired_capacity
  set current_server_count: current
  current
rescue StandardError
  0
end

#dead? ⇒ Boolean

Is the server dead, meaning is it not reporting within the last two minutes

Returns:

  • (Boolean) 


145
146
147
 
# File 'lib/app/jobs/cron/server.rb', line 145

def dead?
  !alive?
end

#delayed_jobs_count ⇒ Object

Returns a count of the Delayed Jobs in queue that have not failed



193
194
195
 
# File 'lib/app/jobs/cron/server.rb', line 193

def delayed_jobs_count
  @delayed_jobs_count ||= Delayed::Backend::Mongoid::Job.where(failed_at: nil).read(mode: :primary).count
end

#execute ⇒ Object

Go through the logic once a minute



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/app/jobs/cron/server.rb', line 33

def execute
  if primary?
    run_cron_jobs
  else
    primary = Cron::Server.where(state: STATE_PRIMARY).first
    if primary.blank? || primary.dead?
      become_primary
      run_cron_jobs
    end
  end
  time_to_next_run
rescue StandardError => error
  App47Logger.log_error 'Unable to run cron server', error
  time_to_next_run
ensure
  check_in
end

#handle_auto_scale_jobs ⇒ Object

Calls the ‘auto_scale’ method with a variable ‘desired_count’ based on how many jobs are running We don’t need any more workers if the job count is less than 1,000



221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
 
# File 'lib/app/jobs/cron/server.rb', line 221

def handle_auto_scale_jobs
  return if delayed_jobs_count < 50

  case delayed_jobs_count
  when 50..250
    auto_scale(1)
  when 251..500
    auto_scale(2)
  when 501..1_000
    auto_scale(3)
  when 1_001..2_000
    auto_scale(4)
  when 2_001..3_999
    auto_scale(4)
  when 4_000..7_999
    auto_scale(5)
  when 8_000..10_999
    auto_scale(5)
  when 11_000..13_999
    auto_scale(6)
  when 14_000..17_999
    auto_scale(6)
  else
    auto_scale(7)
  end
end

#handle_zero_job_count ⇒ Object

Calls the ‘auto_scale’ method with a ‘desired_count’ of 0 unless the capacity is already at 0



211
212
213
214
215
 
# File 'lib/app/jobs/cron/server.rb', line 211

def handle_zero_job_count
  return if current_desired_capacity.eql?(0)

  auto_scale
end

#high_lander ⇒ Object

Look to make sure there is only one primary



264
265
266
267
268
269
 
# File 'lib/app/jobs/cron/server.rb', line 264

def high_lander
  return if secondary? # Don't need to check if not primary

  primary = Cron::Server.where(state: STATE_PRIMARY).first
  errors.add(:state, 'there can only be one primary') unless primary.blank? || primary.eql?(self)
end

#inactive_count ⇒ Object

Returns the count of inactive servers



281
282
283
 
# File 'lib/app/jobs/cron/server.rb', line 281

def inactive_count
  desired_server_count
end

#primary? ⇒ Boolean

Am I the primary server

Returns:

  • (Boolean) 


124
125
126
 
# File 'lib/app/jobs/cron/server.rb', line 124

def primary?
  alive? && STATE_PRIMARY.eql?(state)
end

#run_cron_jobs ⇒ Object 



51
52
53
54
 
# File 'lib/app/jobs/cron/server.rb', line 51

def run_cron_jobs
  run_jobs
  check_auto_scale
end

#run_jobs ⇒ Object

Run all cron tab jobs



59
60
61
62
 
# File 'lib/app/jobs/cron/server.rb', line 59

def run_jobs
  now = Time.now.utc
  Cron::Tab.all.each { |tab| tab.run if tab.time_to_run?(now) }
end

#secondary? ⇒ Boolean

Am I a secondary server

Returns:

  • (Boolean) 


131
132
133
 
# File 'lib/app/jobs/cron/server.rb', line 131

def secondary?
  STATE_SECONDARY.eql?(state)
end

#sys_config ⇒ Object 



178
179
180
 
# File 'lib/app/jobs/cron/server.rb', line 178

def sys_config
  @sys_config ||= SystemConfiguration.configuration
end

#time_to_next_run ⇒ Object

Determine the next minute to run,



67
68
69
 
# File 'lib/app/jobs/cron/server.rb', line 67

def time_to_next_run
  60 - Time.now.utc.to_i % 60
end