Class: BaseChip::Project

Inherits:
Object
  • Object
show all
Includes:
Base, Dsl, Hierarchy, TaskMaster
Defined in:
lib/base_chip/project.rb

Constant Summary collapse

DEFINED_SUBCOMMANDS =

@block_names = Dir.glob(“#@directory/*”) @block_names.map{|path| path.split(///).pop} @block_names.delete ‘settings.yaml’ @block_names

%W{all gate}

Constants included from TaskMaster

TaskMaster::TASKER

Instance Attribute Summary collapse

Attributes included from Dsl

#modes

Instance Method Summary collapse

Methods included from Hierarchy

included

Methods included from Base

included

Methods included from Dsl

#add_child_mode_as_child, included, #inherit, #mode, #mode?, #type_plural

Methods included from TaskMaster

included

Constructor Details

#initialize ⇒ Project

Returns a new instance of Project.



48
49
50
51
52
 
# File 'lib/base_chip/project.rb', line 48

def initialize()
  super
  @project   = self
  @pwd       = Dir.pwd
end

Instance Attribute Details

#registered_modes ⇒ Object (readonly)

Returns the value of attribute registered_modes.



377
378
379
 
# File 'lib/base_chip/project.rb', line 377

def registered_modes
  @registered_modes
end

#workload ⇒ Object

Returns the value of attribute workload.



46
47
48
 
# File 'lib/base_chip/project.rb', line 46

def workload
  @workload
end

Instance Method Details



267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
 
# File 'lib/base_chip/project.rb', line 267

def banner

  banner_line      '--------------------------------- ------------ ------------ ------------ --------'
  banner_line      '                                 | PASS #   % | FAIL #   % | CNCL #   % |  TOTAL '
  if @results['Action']
    banner_line  ( ' ACTIONS                         |' + results_line('Action','all',true))
    @results['Action'].each_key do |block|
      banner_line(('   %-29s |' % block)                + results_line('Action',block,true))
    end
  end
  if @results['Test']
    banner_line  ( ' TESTS                           |' + results_line('Test','all',true))
    @results['Test'].each_key do |block|
      banner_line(('   %-29s |' % block)                + results_line('Test',block,true))
    end
  end
  banner_line      '--------------------------------- ------------ ------------ ------------ --------'
  @report_file.close

  puts "Full report available here: #{@report_name}"
  if @first_error
    fault("This workload had #{@failing} error(s).\n" + 
          (@first_error['Action'] ? "  The first action related error encountered was:\n    #{@first_error['Action']}" : '') +
          (@first_error['Test'  ] ? "  The first test related error encountered was:\n    #{  @first_error['Test'  ]}" : '') ,
          @first_error['Action'] || @first_error['Test']
         )
  end
end



263
264
265
266
 
# File 'lib/base_chip/project.rb', line 263

def banner_line(str)
  @report_file.puts str.gsub(/\[\d+m/,'')
               puts str
end

#block_dereference(name, names, passive) ⇒ Object 



166
167
168
169
170
171
172
173
 
# File 'lib/base_chip/project.rb', line 166

def block_dereference(name,names,passive)
  if block = @blocks[name.to_sym]
    block.configure
    block.dereference_workload(names,passive)
  else
    fault "Could not find block #{name} in project #{@name}" # FIXME say who wanted it, and if shortcut occurred
  end
end

#block_names ⇒ Object

fault “‘base_chip’ should be run from a block’s work directory or the top level of the project.”



79
80
81
82
83
84
85
 
# File 'lib/base_chip/project.rb', line 79

def block_names
  return @block_names if @block_names
  # \@block_names = Dir.glob("#\{@directory}/*")
  # \@block_names.map!{|path| path.split(/\//).pop}
  # \@block_names.delete 'settings.yaml'
  # \@block_names
end

#configure ⇒ Object 



385
386
387
388
389
390
391
392
393
394
395
396
397
 
# File 'lib/base_chip/project.rb', line 385

def configure
  return if @configured
  @directory          = BaseChip.root
  @modes             += BaseChip.options.modes.split(/,/) if BaseChip.options.modes
  super
  @modes.each do |name|
    fault "Attempted to call mode '#{name}', which isn't registered with the project.  Call register_mode(#{name}) in project.rb to allow.  Valid modes are #{project.registered_modes}" unless project.registered_modes.include? name.to_s
  end
  if self.blocks
    @directory_prefix = figure_directory_prefix
    @block_names      = self.blocks.keys
  end
end

#default_modes ⇒ Object 



382
383
384
 
# File 'lib/base_chip/project.rb', line 382

def default_modes
  %w{ coverage gates fast debug profiling }.each { |m| register_mode m }
end

#dereference_clusters ⇒ Object 



398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
 
# File 'lib/base_chip/project.rb', line 398

def dereference_clusters
  return nil unless cluster_names = BaseChip.options.jobs
  configure
  fault "Cluster submission attempted, but no cluster types are configured." unless @cluster_types
  @dereferenced = []
  cluster_names = cluster_names.split(/,/)
  cluster_names.each do |cn|
    type     = nil
    cluster  = nil
    jobs     = nil
    cn_orig  = cn
    clusters = []

    if cn.sub!(/^(\d+)(?::|$)/,'')
      jobs = $1.to_i
    end
    if cn.sub!(/:(\d+)$/,'')
      fault "Multiple counts found in '#{cn_orig}'.  Separate multiple cluster specifiers with commas." if jobs
      jobs = $1.to_i
    end

    case cn
    when /^([^:]+):([^:]+)$/
      cluster = $2
      if $1 == 'all' || $1 == ''
        cluster_types.each_value do |cluster_type|
          clusters += cluster_type.dereference(cluster,jobs,true)
        end
      elsif cluster_type = @cluster_types[$1.to_sym]
        clusters += cluster_type.dereference(cluster,jobs,false)
      else
        fault "Could not find cluster type '#{$1}' in cluster specifier '#{cn_orig}'." if jobs
      end
    when /^([^:]+)$/
      if cluster_type = @cluster_types[$1.to_sym]
        clusters += cluster_type.dereference('all',jobs,true)
      else
        cluster_types.each_value do |cluster_type|
          clusters += cluster_type.dereference('all',jobs,true)
        end
      end
    when ''
      cluster_types.each_value do |cluster_type|
        clusters += cluster_type.dereference('all',jobs,true)
      end
    else
      fault "Could not parse the cluster specifier '#{cn_orig}'." if jobs
    end
    fault "No clusters could be determined for the cluster specifier '#{cn_orig}'." if clusters.empty?

    if cluster_order
      cluster_order.each do |name|
        tmp = clusters.delete_if {|c| c.full_name == "#{@name}:#{name}"}
        try_add_clusters tmp, jobs
      end
    end
    try_add_clusters clusters, jobs
  end
  @dereferenced
end

#dereference_workload(targets) ⇒ Object 



174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
 
# File 'lib/base_chip/project.rb', line 174

def dereference_workload(targets)
  configure
  fault "project '#{name}' has no blocks defined" unless @blocks
  targets ||= ['all']
  out = []

  workload_hash = Hash.new

  targets.each do |t|
    case t 
    when 'gate', 'diffgate', 'all'
      self.block_names.each do |b|
        out += block_dereference(b,[t],true)
      end
    when /^all:(.*)$/
      self.block_names.each do |b|
        out += block_dereference(b,[$1],true)
      end
    when /:/
      tmp = t.split(/:/)
      b = tmp.shift
      out += block_dereference(b,[tmp.join(':')],true)
    else
      fault "Cannot determine block in order to run #{t}."
    end
  end
  out.uniq!
  out
end

#diffgate(targets) ⇒ Object 



118
119
120
121
122
123
 
# File 'lib/base_chip/project.rb', line 118

def diffgate(targets)
  targets.each do |t|
    return false unless t =~ /\bdiffgate$/
  end
  true
end

#discover_blocks ⇒ Object 



368
369
 
# File 'lib/base_chip/project.rb', line 368

def discover_blocks         ; file_glob("#{@directory}/*/base_chip/block.rb"         , /(\w+)\/base_chip\/block\.rb$/                , :block         )
file_glob("#{@directory}/base_chip/block/*.rb"         ,        /base_chip\/blocks\/(\w+)\.rb$/        , :block         ) end

#discover_cluster_types ⇒ Object 



371
 
# File 'lib/base_chip/project.rb', line 371

def discover_cluster_types  ; file_glob("#{@directory}/base_chip/cluster_types/*.rb" ,        /base_chip\/cluster_types\/(\w+)\.rb$/ , :cluster_type  ) end

#discover_configurations ⇒ Object 



370
 
# File 'lib/base_chip/project.rb', line 370

def discover_configurations ; file_glob("#{@directory}/base_chip/configurations/*.rb",        /base_chip\/configurations\/(\w+)\.rb$/, :configuration ) end

#discover_subprojects ⇒ Object 



366
367
 
# File 'lib/base_chip/project.rb', line 366

def discover_subprojects    ; file_glob("#{@directory}/*/base_chip/project.rb"       , /(\w+)\/base_chip\/project\.rb$/              , :subproject    )
file_glob("#{@directory}/base_chip/subprojects/*.rb"   ,        /base_chip\/subprojects\/(\w+)\.rb$/   , :subproject    ) end

#figure_directory_prefix ⇒ Object

def configure_blocks(block_names)

block_names.each do |name|
  self.blocks[name.to_sym].configure
end

end



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/base_chip/project.rb', line 58

def figure_directory_prefix
  # if @directory == @pwd
  #   return nil
  # elsif @pwd.length > @directory.length
    configure
    if @blocks
      @blocks.each do |bname,block|
        block.configure
        if @pwd =~ /^#{block.directory}\b/
          block.configurations.each do |cname,configuration|
            configuration.configure
            return "#{bname}:#{cname}" if @pwd =~ /^#{configuration.directory}\b/
          end
          return bname.to_s
        end
      end
    end
  # end
  nil
  # fault "'base_chip' should be run from a block's work directory or the top level of the project."
end

#find_action(t_name) ⇒ Object 



245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
 
# File 'lib/base_chip/project.rb', line 245

def find_action(t_name)
  address     = t_name.split(/:/)

  block_name  = address.shift.to_sym
  block       = self.blocks[block_name]           or fault("Couldn't find block '#{block_name}'")

  config_name = address.shift.to_sym
  config      = block.configurations[config_name] or fault("Couldn't find configuration '#{config_name}' in block '#{block_name}")

  thing       = address.shift.to_sym
  action      = config.actions[thing]
  return action if action

  test_list   = config.test_lists[thing]          or fault("Could not find action or test list '#{thing}' in block:configuration '#{block_name}:#{config_name}'")
  test_name   = address.shift.to_sym
  test        = test_list.tests[test_name]        or fault("Could not find test '#{test_name}' in block:configuration:list '#{block_name}:#{config_name}:#{thing}'")
  return test
end

#ready(targets = nil) ⇒ Object

,available_only=false)



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/base_chip/project.rb', line 87

def ready(targets=nil) #,available_only=false)
  configure
  @orig_targets = targets
  if @directory_prefix
    # no map! here because @orig_targets needs to stay true to the original command
    targets = targets.map { |t| "#{@directory_prefix}:#{t}" }
  end
  @workload = dereference_workload targets
  @workload.delete_if {|w|w.nil?}
  if @workload.empty?
    if diffgate(targets)
      puts  "diffgate returned no targets needing to run"
      exit
    else
      fault "Could not determine anything to run based on the targets #{targets}.  Did you mean to run this from another directory?"
    end
  end

  w_names = @workload.map{|w|w.name}
  @workload.each do |w|
    w.configure
    if w.depends
      w.deep_depends.each do |d|
        if @workload.include? d
          w.wait_count += 1
          d.next_tasks << w
        end
      end
    end
  end
end

#register_mode(name) ⇒ Object 



378
379
380
381
 
# File 'lib/base_chip/project.rb', line 378

def register_mode(name)
  @registered_modes ||= []
  @registered_modes << name.to_s
end

#results_line(type, block, total) ⇒ Object 



223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
 
# File 'lib/base_chip/project.rb', line 223

def results_line(type,block,total)
  blocks = block == 'all' ? @results[type].keys : [block]
  if total
    pass_fail_cancel = [0,0,0,0,0,0,0]
    blocks.each do |b|
      res_hash = @results[type][b]
      pass_fail_cancel[0] += res_hash['pass'].size
      pass_fail_cancel[2] += res_hash['fail'].size
      pass_fail_cancel[4] += res_hash[:cancel].size
    end
    pass_fail_cancel[6] = pass_fail_cancel[0] + pass_fail_cancel[2] + pass_fail_cancel[4]
    pass_fail_cancel[1] = (100 * pass_fail_cancel[0]/pass_fail_cancel[6]).round
    pass_fail_cancel[3] = (100 * pass_fail_cancel[2]/pass_fail_cancel[6]).round
    pass_fail_cancel[5] = (100 * pass_fail_cancel[4]/pass_fail_cancel[6]).round
    str  = pass_fail_cancel[0] > 0 ? ' %6d %3d |' : ' %6d %3d |'
    str += pass_fail_cancel[3] > 0 ? ' %6d %3d |' : ' %6d %3d |'
    str += pass_fail_cancel[5] > 0 ? ' %6d %3d |' : ' %6d %3d |'
    return "#{str} %6d" % pass_fail_cancel
  else
    fault 'Internal error: detailed result reporting not yet supported'
  end
end

#run_all_readied ⇒ Object 



124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/base_chip/project.rb', line 124

def run_all_readied
  # @report_hash = {}
  FileUtils.mkdir_p "#{@directory}/reports"
  @report_name    = "#{@directory}/reports/workload.#{BaseChip.random_string}"
  @report_file    = File.new(@report_name,'w')
  if @tasker.clusters
    @tasker.ready
  else
    @foreground = @workload.size == 1
  end
  begin
    @tasker.submit_workload_chain workload_chain
    @tasker.run
  rescue Interrupt
    @tasker.finish
  end
end

#shortcut(name, array) ⇒ Object

def enable_tracking(opc,opi)

# puts "enabled tracking(#{opc},#{opi})"
init_results_sockets(:client)
@opc = opc
@opi = opi

end def track_complete(state,first_error = nil)

return unless @opc
msg = Message.new
msg.project = 'gandalf'
msg.type    = 'complete'
msg.state   = state
msg.data = {
  'opc'         => @opc  ,
  'opi'         => @opi  ,
  'first_error' => first_error
}
send_work_message(msg)

end def track_results(test_action,state,problem,totals)

return unless @opc
msg = Message.new
msg.project = 'gandalf'
msg.type    = 'test_action'
msg.state   = state
msg.data = {
  'name'        => test_action.bundle_name,
  'block'       => test_action.block.name,
  'opc'         => @opc,
  'opi'         => @opi,
  'first_error' => problem && problem.signature,
  'file'        => problem && problem.file,
  'bundle'      => problem && problem.bundle,
  'totals'      => totals
}
# puts "sending totals #{test_action.totals.inspect}"
puts "sending message #{msg.data.inspect}"
if test_action.class_string == 'Action'
  msg.data['action'   ] = test_action.name
else
  msg.data['test'     ] = test_action.name
  msg.data['test_list'] = test_action.test_list.name
end
send_work_message(msg)

end



361
362
363
364
 
# File 'lib/base_chip/project.rb', line 361

def shortcut(name, array)
  @shortcuts ||= {}
  @shortcuts[name] = array
end

#tasker_finish ⇒ Object 



295
296
297
 
# File 'lib/base_chip/project.rb', line 295

def tasker_finish
  banner
end

#tasker_handle_results(task, result, problem_or_directory = nil, totals = {}) ⇒ Object 



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
 
# File 'lib/base_chip/project.rb', line 203

def tasker_handle_results(task, result, problem_or_directory=nil, totals={})
  action = @workload.select{|w|w.full_name == task.task_name}.first
  # track_results(action, result, problem, totals)
  if result == 'fail'
    message = (problem_or_directory || "#{action.class_string} '#{action.name}' failed without an error signature").to_s
    begin
      error message
    rescue
    end
    @failing ||= 0
    @failing  += 1
    @first_error = {}
    @first_error[action.class_string] ||= message
  end
  @results                                         ||= {}
  @results[action.class_string]                    ||= {}
  @results[action.class_string][action.block.name] ||= {'pass' => [], 'fail' => [], :cancel =>[]}
  @results[action.class_string][action.block.name][result] << action
  @report_file.puts "#{result} #{action.class_string} #{task.task_name} #{problem_or_directory}"
end

#tasker_run_task(t_obj) ⇒ Object 



298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
 
# File 'lib/base_chip/project.rb', line 298

def tasker_run_task(t_obj)
  t = find_action(t_obj.task_name)
  begin
    t.foreground = @foreground
    t.run
  rescue ReportingError => e
    # FIXME reuse test error states here
    t.state = 'fail'
  rescue Exception => e
    t.problem = Problem.new
    t.problem.signature = "#{t.full_name} #{e.class} --- #{e.message}"
    puts e.backtrace
    # FIXME reuse test error states here
    t.state = 'fail'
  end
            #track_results(t    ,t.state,t.problem               ,t.totals)
  @tasker.register_results(t_obj,t.state,t.problem || t.directory,t.totals)
end

#try_add_clusters(clusters, jobs) ⇒ Object 



458
459
460
461
462
463
464
465
466
467
468
469
470
 
# File 'lib/base_chip/project.rb', line 458

def try_add_clusters(clusters, jobs)
  clusters.each do |c|
    next if @dereferenced.include? c
    c.configure
    fault "could not determine number of jobs to run in cluster '#{c.full_name}'.  Consider setting a default for this cluster." unless (c.default_jobs || c.maximum_jobs || jobs)    
    if    (c.default_jobs || c.maximum_jobs) == nil; c.slots = jobs
    elsif jobs.nil?                                ; c.slots = c.default_jobs || c.maximum_jobs
    elsif jobs > c.maximum_jobs                    ; c.slots = c.maximum_jobs
    else                                             c.slots = jobs
    end
    @dereferenced << c
  end
end

#use_tool(name, version) ⇒ Object 



372
373
374
375
376
 
# File 'lib/base_chip/project.rb', line 372

def use_tool(name,version)
  tool :name do |t|
    t.select_version version
  end
end

#workload_chain ⇒ Object 



141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
 
# File 'lib/base_chip/project.rb', line 141

def workload_chain
  @workload.each do |w|
    w.task           = t = Task.new
    t.task_name      = w.full_name
    t.foreground     = @foreground
    t.worker_command = if @tasker.clusters
                         lambda { |client_id|
                           # The "@drb_uri ||=" is necessary because Drb.uri can change for some reason, and a command compare happens later
                           "#{$0} #{@orig_targets.join(' ')} --client-of #{@drb_uri ||= DRb.uri} --client-id #{client_id} --work-dir #{@pwd} #{
                             '--modes ' + BaseChip.options.modes                        if BaseChip.options.modes             } #{
                             (['--' ]   + BaseChip.options. append_arguments).join(' ') if BaseChip.options. append_arguments } #{
                             (['---']   + BaseChip.options.replace_arguments).join(' ') if BaseChip.options.replace_arguments }" }
                       else
                         nil
                       end
  end
  @workload.map do |w|
    t = w.task
    t.next_tasks = w.next_tasks.map { |nt| nt.task }
    t.wait_count = w.wait_count
    w.next_tasks.uniq!
    t
  end
end