Class: Flor::Scheduler

Inherits:
Object
  • Object
show all
Defined in:
lib/flor/unit/scheduler.rb

Constant Summary collapse

RETURN_KEYS = 
%w[ exid nid payload tasker cause ].freeze
DUMP_KEYS = 
%w[ timestamp executions timers traps pointers ].freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(conf = {}, over_conf = {}) ⇒ Scheduler

Returns a new instance of Scheduler.



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/flor/unit/scheduler.rb', line 17

def initialize(conf={}, over_conf={})

  @conf = Flor::Conf.prepare(conf, over_conf)

  fail ArgumentError.new(
    "invalid domain name #{@conf['domain']}"
  ) if @conf['domain'] && ! Flor.potential_domain_name?(@conf['domain'])

  @env = @conf['env'] ||= 'dev'

  @env = (Kernel.const_get(@env) rescue @env) if @env.match(/\A[A-Z]+\z/)
    # when env is "RAILS_ENV" for example...

  @loader =
    (Flor::Conf.get_class(@conf, 'loader') || Flor::Loader).new(self)
  @caller =
    (Flor::Conf.get_class(@conf, 'caller') || Flor::Caller).new(self)
  @hooker =
    (Flor::Conf.get_class(@conf, 'hooker') || Flor::Hooker).new(self)
  @storage =
    (Flor::Conf.get_class(@conf, 'storage') || Flor::Storage).new(self)
  @ganger =
    (Flor::Conf.get_class(@conf, 'ganger') || Flor::Ganger).new(self)

  @logger =
    (Flor::Conf.get_class(@conf, 'logger') || Flor::Logger).new(self)
  @wlist =
    (Flor::Conf.get_class(@conf, 'wlist') || Flor::WaitList).new(self)

  @spooler =
    (Flor::Conf.get_class(@conf, 'spooler') || Flor::Spooler).new(self)

  @heart_rate = @conf['sch_heart_rate'] || 0.3
  @reload_after = @conf['sch_reload_after'] || 60
    #
  @wake_up = true
  @next_time = nil
  @reloaded_at = Time.now

  @msg_max_res_time = @conf['sch_msg_max_res_time'] || 10 * 60

  @idle_count = 0

  @max_executor_count =
    @conf['sch_max_executor_count'] ||
    @conf['sch_max_executors'] ||
    7

  @executors = []

  c = @conf['constant']
    #
  Kernel.const_set(c, self) if c

  @archive = @conf['archive'] ? {} : nil # used, so far, only for testing
end

Instance Attribute Details

#archive ⇒ Object (readonly)

Returns the value of attribute archive.



15
16
17
 
# File 'lib/flor/unit/scheduler.rb', line 15

def archive
  @archive
end

#caller ⇒ Object (readonly)

Returns the value of attribute caller.



9
10
11
 
# File 'lib/flor/unit/scheduler.rb', line 9

def caller
  @caller
end

#conf ⇒ Object (readonly)

Returns the value of attribute conf.



7
8
9
 
# File 'lib/flor/unit/scheduler.rb', line 7

def conf
  @conf
end

#env ⇒ Object (readonly)

Returns the value of attribute env.



7
8
9
 
# File 'lib/flor/unit/scheduler.rb', line 7

def env
  @env
end

#ganger ⇒ Object (readonly)

Returns the value of attribute ganger.



9
10
11
 
# File 'lib/flor/unit/scheduler.rb', line 9

def ganger
  @ganger
end

#hooker ⇒ Object (readonly)

Returns the value of attribute hooker.



9
10
11
 
# File 'lib/flor/unit/scheduler.rb', line 9

def hooker
  @hooker
end

#last_queued_message_id ⇒ Object (readonly)

Returns the value of attribute last_queued_message_id.



13
14
15
 
# File 'lib/flor/unit/scheduler.rb', line 13

def last_queued_message_id
  @last_queued_message_id
end

#loader ⇒ Object (readonly)

Returns the value of attribute loader.



9
10
11
 
# File 'lib/flor/unit/scheduler.rb', line 9

def loader
  @loader
end

#logger ⇒ Object (readonly)

Returns the value of attribute logger.



10
11
12
 
# File 'lib/flor/unit/scheduler.rb', line 10

def logger
  @logger
end

#storage ⇒ Object (readonly)

Returns the value of attribute storage.



9
10
11
 
# File 'lib/flor/unit/scheduler.rb', line 9

def storage
  @storage
end

#thread_status ⇒ Object (readonly)

Returns the value of attribute thread_status.



12
13
14
 
# File 'lib/flor/unit/scheduler.rb', line 12

def thread_status
  @thread_status
end

Instance Method Details

#add_branches(exid, *as) ⇒ Object Also known as: add_branch 



285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
 
# File 'lib/flor/unit/scheduler.rb', line 285

def add_branches(exid, *as)

  msg, opts = prepare_message('add-branches', [ exid, *as ])

  msg['point'] = 'add'
  msg['trees'] = prepare_trees(opts)

  msg['tnid'] = tnid =
    opts.delete(:tnid) || msg.delete('nid')
  msg['nid'] =
    msg.delete('nid') || opts.delete(:pnid) || Flor.parent_nid(tnid)

  exe = @storage.executions[exid: msg['exid']]
  pnid = msg['nid']
  ptree = exe.lookup_tree(pnid)

  fail ArgumentError.new(
    "parent #{pnid} is a leaf, cannot add branch at #{tnid}"
  ) unless ptree[1].is_a?(Array)
    #
    # not likely to happen, since leaves reply immediately

  size = ptree[1].size
  tnid = (msg['tnid'] ||= Flor.make_child_nid(pnid, size))

  cid = Flor.child_id(tnid)

  tide, tcid = nil
    (0..size - 1).reverse_each do |i|
      tcid = Flor.make_child_nid(pnid, i)
      next unless exe.nodes[tcid]
      tide = i; break
    end

  fail ArgumentError.new(
    "target #{tnid} too low, execution has already reached #{tcid}"
  ) if tide && cid < tide

  fail ArgumentError.new(
    "target #{tnid} is off by #{cid - size}, " +
    "node #{pnid} has #{size} branch#{size == 1 ? '' : 'es'}"
  ) if cid > size

  queue(msg, opts)
end

#add_iterations(exid, *as) ⇒ Object Also known as: add_iteration 



332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
 
# File 'lib/flor/unit/scheduler.rb', line 332

def add_iterations(exid, *as)

  msg, opts = prepare_message('add-iterations', [ exid, *as ])

  msg['point'] = 'add'
  msg['elements'] = prepare_elements(opts)
  msg['nid'] = msg.delete('nid') || opts.delete(:pnid)

  exe = @storage.executions[exid: msg['exid']]
  nid = msg['nid']

  fail ArgumentError.new(
    "cannot add iteration to missing execution #{msg['exid'].inspect}"
  ) unless exe

  fail ArgumentError.new(
    "missing nid: or pnid:"
  ) unless nid

  fail ArgumentError.new(
    "cannot add iteration to node #{nid.inspect} not present in tree"
  ) unless exe.lookup_tree(nid)

  fail ArgumentError.new(
    "cannot add iteration to node #{nid.inspect} not present in execution"
  ) unless exe.nodes.has_key?(nid)

  queue(msg, opts)
end

#archive_node(exid, node) ⇒ Object 



396
397
398
399
 
# File 'lib/flor/unit/scheduler.rb', line 396

def archive_node(exid, node)

  (@archive[exid] ||= {})[node['nid']] = Flor.dup(node) if @archive
end

#archived_node(exid, nid) ⇒ Object 



401
402
403
404
 
# File 'lib/flor/unit/scheduler.rb', line 401

def archived_node(exid, nid)

  (@archive[exid] || {})[nid]
end

#cancel(exid, *as) ⇒ Object 



243
244
245
246
247
248
249
 
# File 'lib/flor/unit/scheduler.rb', line 243

def cancel(exid, *as)

  msg, opts = prepare_message('cancel', [ exid, *as ])
  msg['nid'] ||= '0'

  queue(msg, opts)
end

#check_migration_version ⇒ Object 



143
144
145
146
147
148
149
150
 
# File 'lib/flor/unit/scheduler.rb', line 143

def check_migration_version

  fail(
    "database not ready, " +
    "db ver: #{@storage.db_version.inspect}, " +
    "mig ver: #{@storage.migration_version}"
  ) if !! @conf['sto_migration_check'] && @storage.ready?
end

#dump(io = nil, opts = nil, &block) ⇒ Object

Dumps all or some of the executions to a JSON string. See Scheduler#load for importing.

unit.dump -> string # returns a JSON string of all executions unit.dump(io) -> io # dumps the JSON to the given IO instance

unit.dump(exid: i) # dumps only the given execution unit.dump(exids: [ i0, i1 ]) # dumps only the givens executions unit.dump(domain: d) # dumps exes from domains, unit.dump(domains: [ d0, d1 ]) # and their subdomains unit.dump(sdomain: d) # dumps strictly from given domains, unit.dump(sdomains: [ d0, d1 ]) # doesn’t look at subdomains

unit.dump() { |h| … } # modify the has right before it’s turned to JSON

unit.dump(hash: true) # returns the hash (instead of JSONing it)



438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
 
# File 'lib/flor/unit/scheduler.rb', line 438

def dump(io=nil, opts=nil, &block)

  io, opts = nil, io if io.is_a?(Hash)
  opts ||= {}

  exis, doms, sdms = extract_dump_and_load_filters(opts)
    #
  filter = lambda { |q|
    q = q.where(
      exid: exis) if exis
    q = q.where {
      Sequel.|(*doms
        .inject([]) { |a, d|
          a.concat([
            { domain: d },
            Sequel.like(:domain, d + '.%') ]) }) } if doms
    q = q.where(
      domain: sdms) if sdms
    q }

  hash =
    storage.db.transaction {

      h = {
        timestamp: Flor.tstamp,
        executions: filter[executions].collect(&:to_h),
        timers: filter[timers].collect(&:to_h),
        traps: filter[traps].collect(&:to_h),
        pointers: filter[pointers].collect(&:to_h) }

      block.call(h) if block

      h }

  return hash if opts[:hash] || opts[:h]

  o = io ? io : StringIO.new

  JSON.dump(hash, o)

  io ? io : o.string
end

#execution(exid) ⇒ Object

Given an exid, returns the execution, if currently executing.



413
414
415
416
417
 
# File 'lib/flor/unit/scheduler.rb', line 413

def execution(exid)

  ex = executor(exid)
  ex ? ex.execution : nil
end

#executor(exid) ⇒ Object 



406
407
408
409
 
# File 'lib/flor/unit/scheduler.rb', line 406

def executor(exid)

  @executors.find { |x| x.exid == exid }
end

#has_tasker?(exid, tname) ⇒ Boolean

Returns:

  • (Boolean) 


102
103
104
105
 
# File 'lib/flor/unit/scheduler.rb', line 102

def has_tasker?(exid, tname)

  @ganger.has_tasker?(exid, tname)
end

#identifier ⇒ Object 



84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/flor/unit/scheduler.rb', line 84

def identifier

  @identifier ||=
    begin
      ai =
        Socket.ip_address_list.find { |a| a.ipv4_private? } ||
        Socket.ip_address_list.find { |a| a.ip_address != '::1' }
      ip =
        (ai ? ai.ip_address : '::1').split('%').first
      [
        'sch', self.name,
        'i' + ip,
        'p' + Process.pid.to_s,
        'o' + (self.object_id % 100_000).to_s(32)
      ].join('-')
    end
end

#join ⇒ Object 



160
161
162
163
 
# File 'lib/flor/unit/scheduler.rb', line 160

def join

  @thread.join
end

#kill(exid, *as) ⇒ Object 



251
252
253
254
255
256
257
258
259
260
 
# File 'lib/flor/unit/scheduler.rb', line 251

def kill(exid, *as)

  msg, opts = prepare_message('kill', [ exid, *as ])

  msg['point'] = 'cancel'
  msg['flavour'] = 'kill'
  msg['nid'] ||= '0'

  queue(msg, opts)
end

#launch(source_or_path, opts = {}) ⇒ Object 



165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/flor/unit/scheduler.rb', line 165

def launch(source_or_path, opts={})

  source, domain, flow_name =
    if df = Flor.split_flow_name(source_or_path)
      [ source_or_path, opts[:domain] || df[0], df[1] ]
    else
      [ source_or_path, opts[:domain] || @conf['domain'] || 'domain0', nil ]
    end

  fail ArgumentError.new(
    "invalid domain name #{domain.inspect}"
  ) unless Flor.potential_domain_name?(domain)

  if flow_name

    source_path, source = @loader.library(source_or_path)

    opts[:fname] = source_path

    # TODO variables
    #        loaded as needed, via the loader
    # TODO payload
    #        yes, still has to be done
  end

  fail ArgumentError.new(
    "flow not found in #{Flor.truncate_string(source_or_path, 35).inspect}"
  ) unless source # will anyway fail badly if src is a tree (array of ...)

  @archive ||= {} if opts[:archive]
    # all subsequent launches will be `archive: true` ...

  @logger.log_src(source, opts)

  unit = opts[:unit] || self.name

  exid = Flor.generate_exid(domain, unit)
  msg = Flor.make_launch_msg(exid, source, opts)

  @logger.log_tree(msg['tree'])

  return [ msg, opts ] if opts[:nolaunch]
    # for testing purposes

  queue(msg, opts)
end

#load(string_or_io, opts = {}, &block) ⇒ Object

Read a previous JSON dump and loads it into the storage. Can be useful when testing, dumping once and reloading multiple times to test variants.

load(string) -> h # load all executions from given JSON string

# returns object inserted stat hash

load(io) # load all executions from the given IO load(io, close: true) # load from the given IO and close it after read

load(x, exid: i) # load only given executions, load(x, exids: [ i0, i1 ]) # ignore the rest of the data in the source load(x, domain: d) # load only exes from given domains, load(x, domains: [ d0, d1 ]) # and their subdomains load(x, sdomain: d) # load only exes from strict domains, load(x, sdomains: [ d0, d1 ]) # ignores exes in their subdomains



497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
 
# File 'lib/flor/unit/scheduler.rb', line 497

def load(string_or_io, opts={}, &block)

  s = string_or_io
  s = s.read if s.respond_to?(:read)
  string_or_io.close if string_or_io.respond_to?(:close) && opts[:close]
  h = JSON.load(s)

  mks = DUMP_KEYS - h.keys
  fail Flor::FlorError.new("missing keys #{mks.inspect}") if mks.any?

  exis, doms, sdms = extract_dump_and_load_filters(opts)
  doms = doms.collect { |d| /\A#{d}(\.#{Flor::NAME_REX})*\z/ } if doms

  counts = { executions: 0, timers: 0, traps: 0, pointers: 0, total: 0 }

  storage.db.transaction do

    (DUMP_KEYS - %w[ timestamp ]).each do |k|

      y = k.to_sym
      cla = storage.send(k)
      cols = cla.columns

      rows = h[k]
        .inject([]) { |a, hh|

          next a if exis && ! exis.include?(hh['exid'])
          next a if doms && ! doms.find { |d| d.match(hh['domain']) }
          next a if sdms && ! sdms.include?(hh['domain'])

          counts[y] += 1
          counts[:total] += 1

          vals = cla.from_h(hh)
          a << cols.collect { |c| vals[c] } }

      cla.import(cols, rows) if rows.any?
    end

    block.call(h, counts) if block
  end

  counts
end

#name ⇒ Object 



74
75
76
77
 
# File 'lib/flor/unit/scheduler.rb', line 74

def name

  @conf['unit'] || @conf['uni_name'] || 'u0'
end

#notify(executor, message) ⇒ Object 



373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
 
# File 'lib/flor/unit/scheduler.rb', line 373

def notify(executor, message)

  if executor
    @hooker.notify(executor, message)
  else
    @hooker.wlist.notify(nil, message)
  end

  # NO, let it fail...
  #
#rescue => err
#  puts '-sch' * 19
#  puts "+ error in #{self.class}#notify"
#  p err
#  puts err.backtrace
#  puts ('-sch' * 19) + ' .'
end

#queue(message, opts = {}) ⇒ Object 



212
213
214
215
216
217
218
219
220
221
222
223
224
225
 
# File 'lib/flor/unit/scheduler.rb', line 212

def queue(message, opts={})

  @last_queued_message_id =
    @storage.put_message(message)

  # Nota bene:
  # the #wait method is added to the Scheduler by Flor::WaitList

  if opts[:wait]
    wait(message['exid'], opts)
  else
    message['exid']
  end
end

#re_apply(exid, *as) ⇒ Object Also known as: reapply 



275
276
277
278
279
280
281
282
 
# File 'lib/flor/unit/scheduler.rb', line 275

def re_apply(exid, *as)

  msg, opts = prepare_message('cancel', [ exid, *as ])

  msg['on_receive_last'] = prepare_re_apply_messages(msg, opts)

  queue(msg, opts)
end

#return(message) ⇒ Object 



229
230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/flor/unit/scheduler.rb', line 229

def return(message)

  queue(
    if message['point'] == 'failed'
      message
    else
      message
        .select { |k, _| RETURN_KEYS.include?(k) }
        .merge!('point' => 'return')
    end)

  nil
end

#running? ⇒ Boolean

Returns:

  • (Boolean) 


157
 
# File 'lib/flor/unit/scheduler.rb', line 157

def running?; @thread_status == :running; end

#schedule(message) ⇒ Object 



363
364
365
366
 
# File 'lib/flor/unit/scheduler.rb', line 363

def schedule(message)

  @storage.put_timer(message)
end

#shutdown ⇒ Object 



107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/flor/unit/scheduler.rb', line 107

def shutdown

  @thread_status = :shutdown
  @thread = nil

  @executors.each(&:shutdown)

  @hooker.shutdown
  @storage.shutdown
  @ganger.shutdown
end

#signal(name, h = {}) ⇒ Object 



262
263
264
265
266
267
268
269
270
271
272
273
 
# File 'lib/flor/unit/scheduler.rb', line 262

def signal(name, h={})

  h[:payload] ||= {}
  h[:name] ||= name

  msg, opts = prepare_message('signal', [ h ])

  fail ArgumentError.new('missing :name string key') \
    unless msg['name'].is_a?(String)

  queue(msg, opts)
end

#start ⇒ Object 



119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/flor/unit/scheduler.rb', line 119

def start

  # TODO heartbeat, every x minutes, when idle, log something

  check_migration_version

  @thread_status = :running

  @thread =
    if defined?(@thread) && @thread
      @thread.run
    else
      Thread.new do
        loop do
          Thread.stop if @thread_status == :stop
          break if @thread_status == :shutdown
          tick
        end
      end
    end

  self
end

#stop ⇒ Object 



152
153
154
155
 
# File 'lib/flor/unit/scheduler.rb', line 152

def stop

  @thread_status = :stop
end

#stopped? ⇒ Boolean

Returns:

  • (Boolean) 


158
 
# File 'lib/flor/unit/scheduler.rb', line 158

def stopped?; ! running?; end

#storage_mutex ⇒ Object 



79
80
81
82
 
# File 'lib/flor/unit/scheduler.rb', line 79

def storage_mutex

  @storage.mutex
end

#trap(node, tra) ⇒ Object 



391
392
393
394
 
# File 'lib/flor/unit/scheduler.rb', line 391

def trap(node, tra)

  @storage.put_trap(node, tra)
end

#wake_up ⇒ Object 



368
369
370
371
 
# File 'lib/flor/unit/scheduler.rb', line 368

def wake_up

  @wake_up = true
end