Class: Flor::Scheduler

Inherits:
Object
  • Object
show all
Defined in:
lib/flor/unit/scheduler.rb

Constant Summary collapse

RETURN_KEYS = 
%w[ exid nid payload tasker cause ].freeze
DUMP_KEYS = 
%w[ timestamp executions timers traps pointers ].freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(conf = {}, over_conf = {}) ⇒ Scheduler

Returns a new instance of Scheduler.



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/flor/unit/scheduler.rb', line 17

def initialize(conf={}, over_conf={})

  @conf = Flor::Conf.prepare(conf, over_conf)

  fail ArgumentError.new(
    "invalid domain name #{@conf['domain']}"
  ) if @conf['domain'] && ! Flor.potential_domain_name?(@conf['domain'])

  @env = @conf['env'] ||= 'dev'

  @env = (Kernel.const_get(@env) rescue @env) if @env.match(/\A[A-Z]+\z/)
    # when env is "RAILS_ENV" for example...

  @loader =
    (Flor::Conf.get_class(@conf, 'loader') || Flor::Loader).new(self)
  @caller =
    (Flor::Conf.get_class(@conf, 'caller') || Flor::Caller).new(self)
  @hooker =
    (Flor::Conf.get_class(@conf, 'hooker') || Flor::Hooker).new(self)
  @storage =
    (Flor::Conf.get_class(@conf, 'storage') || Flor::Storage).new(self)
  @ganger =
    (Flor::Conf.get_class(@conf, 'ganger') || Flor::Ganger).new(self)

  @logger =
    (Flor::Conf.get_class(@conf, 'logger') || Flor::Logger).new(self)
  @wlist =
    (Flor::Conf.get_class(@conf, 'wlist') || Flor::WaitList).new(self)

  @spooler =
    (Flor::Conf.get_class(@conf, 'spooler') || Flor::Spooler).new(self)

  @heart_rate = @conf[:sch_heart_rate] || 0.3
  @reload_after = @conf[:sch_reload_after] || 60
    #
  @wake_up = true
  @next_time = nil
  @reloaded_at = Time.now

  @msg_max_res_time = @conf[:sch_msg_max_res_time] || 10 * 60

  @idle_count = 0

  @max_executors = @conf[:sch_max_executors] || 1
    #
  @executors = []

  c = @conf['constant']
    #
  Kernel.const_set(c, self) if c

  @archive = @conf['archive'] ? {} : nil # used, so far, only for testing
end

Instance Attribute Details

#archive ⇒ Object (readonly)

Returns the value of attribute archive.



15
16
17
 
# File 'lib/flor/unit/scheduler.rb', line 15

def archive
  @archive
end

#caller ⇒ Object (readonly)

Returns the value of attribute caller.



9
10
11
 
# File 'lib/flor/unit/scheduler.rb', line 9

def caller
  @caller
end

#conf ⇒ Object (readonly)

Returns the value of attribute conf.



7
8
9
 
# File 'lib/flor/unit/scheduler.rb', line 7

def conf
  @conf
end

#env ⇒ Object (readonly)

Returns the value of attribute env.



7
8
9
 
# File 'lib/flor/unit/scheduler.rb', line 7

def env
  @env
end

#ganger ⇒ Object (readonly)

Returns the value of attribute ganger.



9
10
11
 
# File 'lib/flor/unit/scheduler.rb', line 9

def ganger
  @ganger
end

#hooker ⇒ Object (readonly)

Returns the value of attribute hooker.



9
10
11
 
# File 'lib/flor/unit/scheduler.rb', line 9

def hooker
  @hooker
end

#last_queued_message_id ⇒ Object (readonly)

Returns the value of attribute last_queued_message_id.



13
14
15
 
# File 'lib/flor/unit/scheduler.rb', line 13

def last_queued_message_id
  @last_queued_message_id
end

#loader ⇒ Object (readonly)

Returns the value of attribute loader.



9
10
11
 
# File 'lib/flor/unit/scheduler.rb', line 9

def loader
  @loader
end

#logger ⇒ Object (readonly)

Returns the value of attribute logger.



10
11
12
 
# File 'lib/flor/unit/scheduler.rb', line 10

def logger
  @logger
end

#storage ⇒ Object (readonly)

Returns the value of attribute storage.



9
10
11
 
# File 'lib/flor/unit/scheduler.rb', line 9

def storage
  @storage
end

#thread_status ⇒ Object (readonly)

Returns the value of attribute thread_status.



12
13
14
 
# File 'lib/flor/unit/scheduler.rb', line 12

def thread_status
  @thread_status
end

Instance Method Details

#add_branches(exid, *as) ⇒ Object Also known as: add_branch 



282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
 
# File 'lib/flor/unit/scheduler.rb', line 282

def add_branches(exid, *as)

  msg, opts = prepare_message('add-branches', [ exid, *as ])

  msg['point'] = 'add'
  msg['trees'] = prepare_trees(opts)

  msg['tnid'] = tnid =
    opts.delete(:tnid) || msg.delete('nid')
  msg['nid'] =
    msg.delete('nid') || opts.delete(:pnid) || Flor.parent_nid(tnid)

  exe = @storage.executions[exid: msg['exid']]
  pnid = msg['nid']
  ptree = exe.lookup_tree(pnid)

  fail ArgumentError.new(
    "parent #{pnid} is a leaf, cannot add branch at #{tnid}"
  ) unless ptree[1].is_a?(Array)
    #
    # not likely to happen, since leaves reply immediately

  size = ptree[1].size
  tnid = (msg['tnid'] ||= Flor.make_child_nid(pnid, size))

  cid = Flor.child_id(tnid)

  tide, tcid = nil
    (0..size - 1).reverse_each do |i|
      tcid = Flor.make_child_nid(pnid, i)
      next unless exe.nodes[tcid]
      tide = i; break
    end

  fail ArgumentError.new(
    "target #{tnid} too low, execution has already reached #{tcid}"
  ) if tide && cid < tide

  fail ArgumentError.new(
    "target #{tnid} is off by #{cid - size}, " +
    "node #{pnid} has #{size} branch#{size == 1 ? '' : 'es'}"
  ) if cid > size

  queue(msg, opts)
end

#add_iterations(exid, *as) ⇒ Object Also known as: add_iteration 



329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
 
# File 'lib/flor/unit/scheduler.rb', line 329

def add_iterations(exid, *as)

  msg, opts = prepare_message('add-iterations', [ exid, *as ])

  msg['point'] = 'add'
  msg['elements'] = prepare_elements(opts)
  msg['nid'] = msg.delete('nid') || opts.delete(:pnid)

  exe = @storage.executions[exid: msg['exid']]
  nid = msg['nid']

  fail ArgumentError.new(
    "cannot add iteration to missing execution #{msg['exid'].inspect}"
  ) unless exe

  fail ArgumentError.new(
    "missing nid: or pnid:"
  ) unless nid

  fail ArgumentError.new(
    "cannot add iteration to node #{nid.inspect} not present in tree"
  ) unless exe.lookup_tree(nid)

  fail ArgumentError.new(
    "cannot add iteration to node #{nid.inspect} not present in execution"
  ) unless exe.nodes.has_key?(nid)

  queue(msg, opts)
end

#archive_node(exid, node) ⇒ Object 



391
392
393
394
 
# File 'lib/flor/unit/scheduler.rb', line 391

def archive_node(exid, node)

  (@archive[exid] ||= {})[node['nid']] = Flor.dup(node) if @archive
end

#archived_node(exid, nid) ⇒ Object 



396
397
398
399
 
# File 'lib/flor/unit/scheduler.rb', line 396

def archived_node(exid, nid)

  (@archive[exid] || {})[nid]
end

#cancel(exid, *as) ⇒ Object 



240
241
242
243
244
245
246
 
# File 'lib/flor/unit/scheduler.rb', line 240

def cancel(exid, *as)

  msg, opts = prepare_message('cancel', [ exid, *as ])
  msg['nid'] ||= '0'

  queue(msg, opts)
end

#check_migration_version ⇒ Object 



140
141
142
143
144
145
146
147
 
# File 'lib/flor/unit/scheduler.rb', line 140

def check_migration_version

  fail(
    "database not ready, " +
    "db ver: #{@storage.db_version.inspect}, " +
    "mig ver: #{@storage.migration_version}"
  ) if !! @conf['sto_migration_check'] && @storage.ready?
end

#dump(io = nil, opts = nil, &block) ⇒ Object

Dumps all or some of the executions to a JSON string. See Scheduler#load for importing.

unit.dump -> string # returns a JSON string of all executions unit.dump(io) -> io # dumps the JSON to the given IO instance

unit.dump(exid: i) # dumps only the given execution unit.dump(exids: [ i0, i1 ]) # dumps only the givens executions unit.dump(domain: d) # dumps exes from domains, unit.dump(domains: [ d0, d1 ]) # and their subdomains unit.dump(sdomain: d) # dumps strictly from given domains, unit.dump(sdomains: [ d0, d1 ]) # doesn’t look at subdomains

unit.dump() { |h| … } # modify the has right before it’s turned to JSON

unit.dump(hash: true) # returns the hash (instead of JSONing it)



433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
 
# File 'lib/flor/unit/scheduler.rb', line 433

def dump(io=nil, opts=nil, &block)

  io, opts = nil, io if io.is_a?(Hash)
  opts ||= {}

  exis, doms, sdms = extract_dump_and_load_filters(opts)
    #
  filter = lambda { |q|
    q = q.where(
      exid: exis) if exis
    q = q.where {
      Sequel.|(*doms
        .inject([]) { |a, d|
          a.concat([
            { domain: d },
            Sequel.like(:domain, d + '.%') ]) }) } if doms
    q = q.where(
      domain: sdms) if sdms
    q }

  hash =
    storage.db.transaction {

      h = {
        timestamp: Flor.tstamp,
        executions: filter[executions].collect(&:to_h),
        timers: filter[timers].collect(&:to_h),
        traps: filter[traps].collect(&:to_h),
        pointers: filter[pointers].collect(&:to_h) }

      block.call(h) if block

      h }

  return hash if opts[:hash] || opts[:h]

  o = io ? io : StringIO.new

  JSON.dump(hash, o)

  io ? io : o.string
end

#execution(exid) ⇒ Object

Given an exid, returns the execution, if currently executing.



408
409
410
411
412
 
# File 'lib/flor/unit/scheduler.rb', line 408

def execution(exid)

  ex = executor(exid)
  ex ? ex.execution : nil
end

#executor(exid) ⇒ Object 



401
402
403
404
 
# File 'lib/flor/unit/scheduler.rb', line 401

def executor(exid)

  @executors.find { |x| x.exid == exid }
end

#has_tasker?(exid, tname) ⇒ Boolean

Returns:

  • (Boolean) 


99
100
101
102
 
# File 'lib/flor/unit/scheduler.rb', line 99

def has_tasker?(exid, tname)

  @ganger.has_tasker?(exid, tname)
end

#identifier ⇒ Object 



81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/flor/unit/scheduler.rb', line 81

def identifier

  @identifier ||=
    begin
      ai =
        Socket.ip_address_list.find { |a| a.ipv4_private? } ||
        Socket.ip_address_list.find { |a| a.ip_address != '::1' }
      ip =
        (ai ? ai.ip_address : '::1').split('%').first
      [
        'sch', self.name,
        'i' + ip,
        'p' + Process.pid.to_s,
        'o' + (self.object_id % 100_000).to_s(32)
      ].join('-')
    end
end

#join ⇒ Object 



157
158
159
160
 
# File 'lib/flor/unit/scheduler.rb', line 157

def join

  @thread.join
end

#kill(exid, *as) ⇒ Object 



248
249
250
251
252
253
254
255
256
257
 
# File 'lib/flor/unit/scheduler.rb', line 248

def kill(exid, *as)

  msg, opts = prepare_message('kill', [ exid, *as ])

  msg['point'] = 'cancel'
  msg['flavour'] = 'kill'
  msg['nid'] ||= '0'

  queue(msg, opts)
end

#launch(source_or_path, opts = {}) ⇒ Object 



162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
 
# File 'lib/flor/unit/scheduler.rb', line 162

def launch(source_or_path, opts={})

  source, domain, flow_name =
    if df = Flor.split_flow_name(source_or_path)
      [ source_or_path, opts[:domain] || df[0], df[1] ]
    else
      [ source_or_path, opts[:domain] || @conf['domain'] || 'domain0', nil ]
    end

  fail ArgumentError.new(
    "invalid domain name #{domain.inspect}"
  ) unless Flor.potential_domain_name?(domain)

  if flow_name

    source_path, source = @loader.library(source_or_path)

    opts[:fname] = source_path

    # TODO variables
    #        loaded as needed, via the loader
    # TODO payload
    #        yes, still has to be done
  end

  fail ArgumentError.new(
    "flow not found in #{Flor.truncate_string(source_or_path, 35).inspect}"
  ) unless source # will anyway fail badly if src is a tree (array of ...)

  @archive ||= {} if opts[:archive]
    # all subsequent launches will be `archive: true` ...

  @logger.log_src(source, opts)

  unit = opts[:unit] || self.name

  exid = Flor.generate_exid(domain, unit)
  msg = Flor.make_launch_msg(exid, source, opts)

  @logger.log_tree(msg['tree'])

  return [ msg, opts ] if opts[:nolaunch]
    # for testing purposes

  queue(msg, opts)
end

#load(string_or_io, opts = {}, &block) ⇒ Object

Read a previous JSON dump and loads it into the storage. Can be useful when testing, dumping once and reloading multiple times to test variants.

load(string) -> h # load all executions from given JSON string

# returns object inserted stat hash

load(io) # load all executions from the given IO load(io, close: true) # load from the given IO and close it after read

load(x, exid: i) # load only given executions, load(x, exids: [ i0, i1 ]) # ignore the rest of the data in the source load(x, domain: d) # load only exes from given domains, load(x, domains: [ d0, d1 ]) # and their subdomains load(x, sdomain: d) # load only exes from strict domains, load(x, sdomains: [ d0, d1 ]) # ignores exes in their subdomains



492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
 
# File 'lib/flor/unit/scheduler.rb', line 492

def load(string_or_io, opts={}, &block)

  s = string_or_io
  s = s.read if s.respond_to?(:read)
  string_or_io.close if string_or_io.respond_to?(:close) && opts[:close]
  h = JSON.load(s)

  mks = DUMP_KEYS - h.keys
  fail Flor::FlorError.new("missing keys #{mks.inspect}") if mks.any?

  exis, doms, sdms = extract_dump_and_load_filters(opts)
  doms = doms.collect { |d| /\A#{d}(\.#{Flor::NAME_REX})*\z/ } if doms

  counts = { executions: 0, timers: 0, traps: 0, pointers: 0, total: 0 }

  storage.db.transaction do

    (DUMP_KEYS - %w[ timestamp ]).each do |k|

      y = k.to_sym
      cla = storage.send(k)
      cols = cla.columns

      rows = h[k]
        .inject([]) { |a, hh|

          next a if exis && ! exis.include?(hh['exid'])
          next a if doms && ! doms.find { |d| d.match(hh['domain']) }
          next a if sdms && ! sdms.include?(hh['domain'])

          counts[y] += 1
          counts[:total] += 1

          vals = cla.from_h(hh)
          a << cols.collect { |c| vals[c] } }

      cla.import(cols, rows) if rows.any?
    end

    block.call(h, counts) if block
  end

  counts
end

#name ⇒ Object 



71
72
73
74
 
# File 'lib/flor/unit/scheduler.rb', line 71

def name

  @conf['unit'] || @conf['uni_name'] || 'u0'
end

#notify(executor, o) ⇒ Object 



370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
 
# File 'lib/flor/unit/scheduler.rb', line 370

def notify(executor, o)

  if executor
    @hooker.notify(executor, o)
  else
    @hooker.wlist.notify(nil, o)
  end

rescue => err
  puts '-sch' * 19
  puts "+ error in #{self.class}#notify"
  p err
  puts err.backtrace
  puts ('-sch' * 19) + ' .'
end

#queue(message, opts = {}) ⇒ Object 



209
210
211
212
213
214
215
216
217
218
219
220
221
222
 
# File 'lib/flor/unit/scheduler.rb', line 209

def queue(message, opts={})

  @last_queued_message_id =
    @storage.put_message(message)

  # Nota bene:
  # the #wait method is added to the Scheduler by Flor::WaitList

  if opts[:wait]
    wait(message['exid'], opts)
  else
    message['exid']
  end
end

#re_apply(exid, *as) ⇒ Object Also known as: reapply 



272
273
274
275
276
277
278
279
 
# File 'lib/flor/unit/scheduler.rb', line 272

def re_apply(exid, *as)

  msg, opts = prepare_message('cancel', [ exid, *as ])

  msg['on_receive_last'] = prepare_re_apply_messages(msg, opts)

  queue(msg, opts)
end

#return(message) ⇒ Object 



226
227
228
229
230
231
232
233
234
235
236
237
238
 
# File 'lib/flor/unit/scheduler.rb', line 226

def return(message)

  queue(
    if message['point'] == 'failed'
      message
    else
      message
        .select { |k, _| RETURN_KEYS.include?(k) }
        .merge!('point' => 'return')
    end)

  nil
end

#running? ⇒ Boolean

Returns:

  • (Boolean) 


154
 
# File 'lib/flor/unit/scheduler.rb', line 154

def running?; @thread_status == :running; end

#schedule(message) ⇒ Object 



360
361
362
363
 
# File 'lib/flor/unit/scheduler.rb', line 360

def schedule(message)

  @storage.put_timer(message)
end

#shutdown ⇒ Object 



104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/flor/unit/scheduler.rb', line 104

def shutdown

  @thread_status = :shutdown
  @thread = nil

  @executors.each(&:shutdown)

  @hooker.shutdown
  @storage.shutdown
  @ganger.shutdown
end

#signal(name, h = {}) ⇒ Object 



259
260
261
262
263
264
265
266
267
268
269
270
 
# File 'lib/flor/unit/scheduler.rb', line 259

def signal(name, h={})

  h[:payload] ||= {}
  h[:name] ||= name

  msg, opts = prepare_message('signal', [ h ])

  fail ArgumentError.new('missing :name string key') \
    unless msg['name'].is_a?(String)

  queue(msg, opts)
end

#start ⇒ Object 



116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/flor/unit/scheduler.rb', line 116

def start

  # TODO heartbeat, every x minutes, when idle, log something

  check_migration_version

  @thread_status = :running

  @thread =
    if defined?(@thread) && @thread
      @thread.run
    else
      Thread.new do
        loop do
          Thread.stop if @thread_status == :stop
          break if @thread_status == :shutdown
          tick
        end
      end
    end

  self
end

#stop ⇒ Object 



149
150
151
152
 
# File 'lib/flor/unit/scheduler.rb', line 149

def stop

  @thread_status = :stop
end

#stopped? ⇒ Boolean

Returns:

  • (Boolean) 


155
 
# File 'lib/flor/unit/scheduler.rb', line 155

def stopped?; ! running?; end

#storage_mutex ⇒ Object 



76
77
78
79
 
# File 'lib/flor/unit/scheduler.rb', line 76

def storage_mutex

  @storage.mutex
end

#trap(node, tra) ⇒ Object 



386
387
388
389
 
# File 'lib/flor/unit/scheduler.rb', line 386

def trap(node, tra)

  @storage.put_trap(node, tra)
end

#wake_up ⇒ Object 



365
366
367
368
 
# File 'lib/flor/unit/scheduler.rb', line 365

def wake_up

  @wake_up = true
end