Class: HammerCLIImport::BaseCommand

Inherits:
HammerCLI::Apipie::Command
  • Object
show all
Extended by:
AsyncTasksReactor::Extend, ImportTools::ImportLogging::Extend, PersistentMap::Extend
Includes:
AsyncTasksReactor::Include, ImportTools::Exceptional::Include, ImportTools::ImportLogging::Include, ImportTools::Task::Include, PersistentMap::Include
Defined in:
lib/hammer_cli_import/base.rb

Class Attribute Summary collapse

Attributes included from PersistentMap::Extend

#map_description, #map_target_entity, #maps

Class Method Summary collapse

Instance Method Summary collapse

Methods included from PersistentMap::Extend

persistent_map, persistent_maps

Methods included from ImportTools::ImportLogging::Extend

add_logging_options

Methods included from AsyncTasksReactor::Extend

add_async_tasks_reactor_options

Methods included from AsyncTasksReactor::Include

#atr_exit, #atr_init, #postpone_till, #wait_for

Methods included from ImportTools::Exceptional::Include

#handle_missing_and_supress, #silently

Methods included from ImportTools::Task::Include

#annotate_tasks

Methods included from ImportTools::ImportLogging::Include

#debug, #error, #fatal, #info, #log, #logtrace, #progress, #setup_logging, #warn

Methods included from PersistentMap::Include

#load_persistent_maps, #map_target_entity, #maps, #prune_persistent_maps, #save_persistent_maps

Constructor Details

#initialize(*list) ⇒ BaseCommand

Returns a new instance of BaseCommand.



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/hammer_cli_import/base.rb', line 45

def initialize(*list)
  super(*list)

  # wrap API parameters into extra hash
  @wrap_out = {
    :users => :user,
    :template_snippets => :config_template
  }
  # APIs return objects encapsulated in extra hash
  #@wrap_in = {:organizations => 'organization'}
  @wrap_in = {}
  # entities that needs organization to be listed
  @prerequisite = {
    :activation_keys => :organizations,
    :content_views => :organizations,
    :content_view_versions => :organizations,
    :host_collections => :organizations,
    :products => :organizations,
    :repositories => :organizations,
    :repository_sets => :products,
    :systems => :organizations
  }
  # cache imported objects (created/lookuped)
  @cache = {}
  class << @cache
    def []=(key, val)
      raise "@cache: #{val.inspect} is not a hash!" unless val.is_a? Hash
      super
    end
  end
  @summary = {}
  # Initialize AsyncTaskReactor
  atr_init
end

Class Attribute Details

.reportname ⇒ Object

Returns the value of attribute reportname.



81
82
83
 
# File 'lib/hammer_cli_import/base.rb', line 81

def reportname
  @reportname
end

Class Method Details

.api_call(resource, action, params = {}, headers = {}, dbg = false) ⇒ Object

Call API. Ideally accessed via api_call instance method. This is supposed to be the only way to access @api.



123
124
125
126
127
128
129
130
131
132
 
# File 'lib/hammer_cli_import/base.rb', line 123

def api_call(resource, action, params = {}, headers = {}, dbg = false)
  if resource == :organizations && action == :create
    params[:organization] ||= {}
    params[:organization][:name] = params[:name]
  end
  @api.resource(resource).call(action, params, headers)
rescue
  error("Error on api.resource(#{resource.inspect}).call(#{action.inspect}, #{params.inspect}):") if dbg
  raise
end

.api_init ⇒ Object

Initialize API. Needed to be called before any api_call calls. If used in shell, it may be called multiple times



116
117
118
119
 
# File 'lib/hammer_cli_import/base.rb', line 116

def api_init
  @api = HammerCLIForeman.foreman_api_connection.api
  nil
end

.csv_columns(*list) ⇒ Object

Which columns have to be be present in CSV.



106
107
108
109
110
 
# File 'lib/hammer_cli_import/base.rb', line 106

def csv_columns(*list)
  return @csv_columns if list.empty?
  raise 'set more than once' if @csv_columns
  @csv_columns = list
end

Instance Method Details

#_compare_hash(entity_hash, search_hash) ⇒ Object 



184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/hammer_cli_import/base.rb', line 184

def _compare_hash(entity_hash, search_hash)
  equal = nil
  search_hash.each do |key, value|
    if value.is_a? Hash
      equal = _compare_hash(entity_hash[key], search_hash[key])
    else
      equal = entity_hash[key] == value
    end
    return false unless equal
  end
  return true
end

#_create_entity(entity_type, entity_hash, original_id) ⇒ Object

Use create_entity instead.



422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
 
# File 'lib/hammer_cli_import/base.rb', line 422

def _create_entity(entity_type, entity_hash, original_id)
  type = to_singular(entity_type)
  if @pm[entity_type][original_id]
    info type.capitalize + ' [' + original_id.to_s + '->' + @pm[entity_type][original_id].to_s + '] already imported.'
    report_summary :found, entity_type
    return get_cache(entity_type)[@pm[entity_type][original_id]]
  else
    info 'Creating new ' + type + ': ' + entity_hash.values_at(:name, :label, :login).compact[0]
    entity_hash = {@wrap_out[entity_type] => entity_hash} if @wrap_out[entity_type]
    debug "entity_hash: #{entity_hash.inspect}"
    entity = mapped_api_call(entity_type, :create, entity_hash)
    debug "created entity: #{entity.inspect}"
    entity = entity[@wrap_in[entity_type]] if @wrap_in[entity_type]
    # workaround for Bug
    entity['id'] = entity['uuid'] if entity_type == :systems
    @pm[entity_type][original_id] = entity['id']
    get_cache(entity_type)[entity['id']] = entity
    debug "@pm[#{entity_type}]: #{@pm[entity_type].inspect}"
    report_summary :created, entity_type
    return entity
  end
end

#api_call(*list) ⇒ Object

Call API. Convenience method for calling api_call class method.



136
137
138
 
# File 'lib/hammer_cli_import/base.rb', line 136

def api_call(*list)
  self.class.api_call(*list)
end

#create_entity(entity_type, entity_hash, original_id, recover = nil, retries = 2) ⇒ Object

Create entity, with recovery strategy.

  • :map - Use existing entity

  • :rename - Change name

  • nil - Fail

Raises:



381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
 
# File 'lib/hammer_cli_import/base.rb', line 381

def create_entity(entity_type, entity_hash, original_id, recover = nil, retries = 2)
  raise ImportRecoveryError, "Creation of #{entity_type} not recovered by " \
    "'#{recover || option_recover.to_sym}' strategy" if retries < 0
  uniq = nil
  begin
    return _create_entity(entity_type, entity_hash, original_id)
  rescue RestClient::UnprocessableEntity => ue
    error " Creation of #{to_singular(entity_type)} failed."
    uniq = nil
    err = JSON.parse(ue.response)
    err = err['error'] if err.key?('error')
    if found_errors(err)
      uniq = process_error(err, entity_hash)
    end
    raise ue unless uniq
  end

  uniq = uniq.to_sym

  case recover || option_recover.to_sym
  when :rename
    entity_hash[uniq] = original_id.to_s + '-' + entity_hash[uniq]
    info " Recovering by renaming to: \"#{uniq}\"=\"#{entity_hash[uniq]}\""
    return create_entity(entity_type, entity_hash, original_id, recover, retries - 1)
  when :map
    entity = lookup_entity_in_cache(entity_type, {uniq.to_s => entity_hash[uniq]})
    if entity
      info " Recovering by remapping to: #{entity['id']}"
      return map_entity(entity_type, original_id, entity['id'])
    else
      warn "Creation of #{entity_type} not recovered by \'#{recover}\' strategy."
      raise ImportRecoveryError, "Creation of #{entity_type} not recovered by \'#{recover}\' strategy."
    end
  else
    fatal 'No recover strategy.'
    raise ue
  end
  nil
end

#cvs_iterate(filename, action) ⇒ Object 



524
525
526
527
528
529
530
 
# File 'lib/hammer_cli_import/base.rb', line 524

def cvs_iterate(filename, action)
  CSVHelper.csv_each filename, self.class.csv_columns do |data|
    handle_missing_and_supress "processing CSV line:\n#{data.inspect}" do
      action.call(data)
    end
  end
end

#data_dir ⇒ Object 



145
146
147
 
# File 'lib/hammer_cli_import/base.rb', line 145

def data_dir
  File.join(File.expand_path('~'), '.transition_data')
end

#delete(filename) ⇒ Object 



544
545
546
 
# File 'lib/hammer_cli_import/base.rb', line 544

def delete(filename)
  cvs_iterate(filename, (method :delete_single_row))
end

#delete_entity(entity_type, original_id) ⇒ Object

Delete entity by original (Sat5) id



451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
 
# File 'lib/hammer_cli_import/base.rb', line 451

def delete_entity(entity_type, original_id)
  type = to_singular(entity_type)
  unless @pm[entity_type][original_id]
    error 'Unknown ' + type + ' to delete [' + original_id.to_s + '].'
    return nil
  end
  info 'Deleting imported ' + type + ' [' + original_id.to_s + '->' + @pm[entity_type][original_id].to_s + '].'
  begin
    mapped_api_call(entity_type, :destroy, {:id => @pm[entity_type][original_id]})
    # delete from cache
    get_cache(entity_type).delete(@pm[entity_type][original_id])
    # delete from pm
    unmap_entity(entity_type, @pm[entity_type][original_id])
    report_summary :deleted, entity_type
  rescue => e
    warn "Delete of #{to_singular(entity_type)} [#{original_id}] failed with #{e.class}: #{e.message}"
    report_summary :failed, entity_type
  end
end

#delete_entity_by_import_id(entity_type, import_id, delete_key = 'id') ⇒ Object

Delete entity by target (Sat6) id



472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
 
# File 'lib/hammer_cli_import/base.rb', line 472

def delete_entity_by_import_id(entity_type, import_id, delete_key = 'id')
  type = to_singular(entity_type)
  original_id = get_original_id(entity_type, import_id)
  if original_id.nil?
    error 'Unknown imported ' + type + ' to delete [' + import_id.to_s + '].'
    return nil
  end
  info "Deleting imported #{type} [#{original_id}->#{@pm[entity_type][original_id]}]."
  if delete_key == 'id'
    delete_id = import_id
  else
    delete_id = get_cache(entity_type)[import_id][delete_key]
  end
  begin
    mapped_api_call(entity_type, :destroy, {:id => delete_id})
    # delete from cache
    get_cache(entity_type).delete(import_id)
    # delete from pm
    @pm[entity_type].delete original_id
    report_summary :deleted, entity_type
  rescue => e
    warn "Delete of #{to_singular(entity_type)} [#{delete_id}] failed with #{e.class}: #{e.message}"
    report_summary :failed, entity_type
  end
end

#delete_single_row(_row) ⇒ Object

This method is called to process single CSV line when deleting



157
158
159
 
# File 'lib/hammer_cli_import/base.rb', line 157

def delete_single_row(_row)
  error 'Delete not implemented.'
end

#execute ⇒ Object 



548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
 
# File 'lib/hammer_cli_import/base.rb', line 548

def execute
  # Get set up to do logging as soon as reasonably possible
  setup_logging
  # create a storage directory if not exists yet
  Dir.mkdir data_dir unless File.directory? data_dir

  # initialize apipie binding
  self.class.api_init
  load_persistent_maps
  load_cache
  prune_persistent_maps @cache
  # TODO: This big ugly thing might need some cleanup
  begin
    if option_delete?
      info "Deleting from #{option_csv_file}"
      delete option_csv_file
      handle_missing_and_supress 'post_delete' do
        post_delete option_csv_file
      end
    else
      info "Importing from #{option_csv_file}"
      import option_csv_file
      handle_missing_and_supress 'post_import' do
        post_import option_csv_file
      end
    end
    atr_exit
  rescue StandardError, SystemExit, Interrupt => e
    error "Exiting: #{e}"
    logtrace e
  end
  save_persistent_maps
  print_summary
  HammerCLI::EX_OK
end

#find_uniq(arr) ⇒ Object 



348
349
350
351
352
353
354
355
 
# File 'lib/hammer_cli_import/base.rb', line 348

def find_uniq(arr)
  uniq = nil
  uniq = arr[0] if arr[1].is_a?(Array) &&
                   (arr[1][0] =~ /has already been taken/ ||
                    arr[1][0] =~ /already exists/ ||
                    arr[1][0] =~ /must be unique within one organization/)
  return uniq
end

#found_errors(err) ⇒ Object 



357
358
359
 
# File 'lib/hammer_cli_import/base.rb', line 357

def found_errors(err)
  return err && err['errors'] && err['errors'].respond_to?(:each)
end

#get_cache(entity_type) ⇒ Object 



161
162
163
 
# File 'lib/hammer_cli_import/base.rb', line 161

def get_cache(entity_type)
  @cache[map_target_entity[entity_type]]
end

#get_original_id(entity_type, import_id) ⇒ Object

this method returns a first found original_id (since we’re able to map several organizations into one)



273
274
275
276
277
278
279
280
281
282
283
 
# File 'lib/hammer_cli_import/base.rb', line 273

def get_original_id(entity_type, import_id)
  if was_translated(entity_type, import_id)
    # find original_ids
    @pm[entity_type].to_hash.each do |key, value|
      return key if value == import_id
    end
  else
    debug "Unknown imported #{to_singular(entity_type)} [#{import_id}]."
  end
  return nil
end

#get_translated_id(entity_type, entity_id) ⇒ Object

Raises:



263
264
265
266
267
268
269
 
# File 'lib/hammer_cli_import/base.rb', line 263

def get_translated_id(entity_type, entity_id)
  if @pm[entity_type] && @pm[entity_type][entity_id]
    return @pm[entity_type][entity_id]
  end
  raise MissingObjectError, 'Unable to import, first import ' + to_singular(entity_type) + \
    ' with id ' + entity_id.inspect
end

#import(filename) ⇒ Object 



532
533
534
 
# File 'lib/hammer_cli_import/base.rb', line 532

def import(filename)
  cvs_iterate(filename, (method :import_single_row))
end

#import_single_row(_row) ⇒ Object

This method is called to process single CSV line when importing.



151
152
153
 
# File 'lib/hammer_cli_import/base.rb', line 151

def import_single_row(_row)
  error 'Import not implemented.'
end

#last_in_cache?(entity_type, id) ⇒ Boolean

Returns:

  • (Boolean) 


212
213
214
 
# File 'lib/hammer_cli_import/base.rb', line 212

def last_in_cache?(entity_type, id)
  return get_cache(entity_type).size == 1 && get_cache(entity_type).first[0] == id
end

#list_server_entities(entity_type, extra_hash = {}, use_cache = false) ⇒ Object 



285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
 
# File 'lib/hammer_cli_import/base.rb', line 285

def list_server_entities(entity_type, extra_hash = {}, use_cache = false)
  if @prerequisite[entity_type]
    list_server_entities(@prerequisite[entity_type]) unless @cache[@prerequisite[entity_type]]
  end

  @cache[entity_type] ||= {}
  results = []

  if !extra_hash.empty? || @prerequisite[entity_type].nil?
    if use_cache
      @list_cache ||= {}
      if @list_cache[entity_type]
        return @list_cache[entity_type][extra_hash] if @list_cache[entity_type][extra_hash]
      else
        @list_cache[entity_type] ||= {}
      end
    end
    entities = api_call(entity_type, :index, {'per_page' => 999999}.merge(extra_hash))
    results = entities['results']
    @list_cache[entity_type][extra_hash] = results if use_cache
  elsif @prerequisite[entity_type] == :organizations
    # check only entities in imported orgs (not all of them)
    @pm[:organizations].to_hash.values.each do |org_id|
      entities = api_call(entity_type, :index, {'per_page' => 999999, 'organization_id' => org_id})
      results += entities['results']
    end
  else
    @cache[@prerequisite[entity_type]].each do |pre_id, _|
      entities = api_call(
        entity_type,
        :index,
        {
          'per_page' => 999999,
          @prerequisite[entity_type].to_s.sub(/s$/, '_id').to_sym => pre_id
        })
      results += entities['results']
    end
  end

  results.each do |entity|
    entity['id'] = entity['uuid'] if entity_type == :systems
    @cache[entity_type][entity['id']] = entity
  end
end

#load_cache ⇒ Object 



165
166
167
168
169
 
# File 'lib/hammer_cli_import/base.rb', line 165

def load_cache
  maps.collect { |map_sym| map_target_entity[map_sym] } .uniq.each do |entity_type|
    list_server_entities entity_type
  end
end

#lookup_entity(entity_type, entity_id, online_lookup = false) ⇒ Object 



171
172
173
174
175
176
177
178
 
# File 'lib/hammer_cli_import/base.rb', line 171

def lookup_entity(entity_type, entity_id, online_lookup = false)
  if (!get_cache(entity_type)[entity_id] || online_lookup)
    get_cache(entity_type)[entity_id] = mapped_api_call(entity_type, :show, {'id' => entity_id})
  else
    debug "#{to_singular(entity_type).capitalize} #{entity_id} taken from cache."
  end
  return get_cache(entity_type)[entity_id]
end

#lookup_entity_in_array(array, search_hash) ⇒ Object 



204
205
206
207
208
209
210
 
# File 'lib/hammer_cli_import/base.rb', line 204

def lookup_entity_in_array(array, search_hash)
  return nil if array.nil?
  array.each do |entity_hash|
    return entity_hash if _compare_hash(entity_hash, search_hash)
  end
  return nil
end

#lookup_entity_in_cache(entity_type, search_hash) ⇒ Object 



197
198
199
200
201
202
 
# File 'lib/hammer_cli_import/base.rb', line 197

def lookup_entity_in_cache(entity_type, search_hash)
  get_cache(entity_type).each do |_entity_id, entity_hash|
    return entity_hash if _compare_hash(entity_hash, search_hash)
  end
  return nil
end

#map_entity(entity_type, original_id, id) ⇒ Object 



330
331
332
333
334
335
336
337
338
339
340
341
 
# File 'lib/hammer_cli_import/base.rb', line 330

def map_entity(entity_type, original_id, id)
  if @pm[entity_type][original_id]
    info "#{to_singular(entity_type).capitalize} [#{original_id}->#{@pm[entity_type][original_id]}] already mapped. " \
      'Skipping.'
    report_summary :found, entity_type
    return
  end
  info "Mapping #{to_singular(entity_type)} [#{original_id}->#{id}]."
  @pm[entity_type][original_id] = id
  report_summary :mapped, entity_type
  return get_cache(entity_type)[id]
end

#mapped_api_call(entity_type, *list) ⇒ Object

Call API on corresponding resource (defined by map_target_entity).



141
142
143
 
# File 'lib/hammer_cli_import/base.rb', line 141

def mapped_api_call(entity_type, *list)
  api_call(map_target_entity[entity_type], *list)
end

#post_delete(_csv_file) ⇒ Object 



540
541
542
 
# File 'lib/hammer_cli_import/base.rb', line 540

def post_delete(_csv_file)
  # empty by default
end

#post_import(_csv_file) ⇒ Object 



536
537
538
 
# File 'lib/hammer_cli_import/base.rb', line 536

def post_import(_csv_file)
  # empty by default
end

#print_summary ⇒ Object 



243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
 
# File 'lib/hammer_cli_import/base.rb', line 243

def print_summary
  progress 'Summary'
  @summary.each do |verb, what|
    what.each do |entity, count|
      noun = if count == 1
               to_singular entity
             else
               entity
             end
      report = "  #{verb.to_s.capitalize} #{count} #{noun}."
      if verb == :found
        info report
      else
        progress report
      end
    end
  end
  progress '  No action taken.' if (@summary.keys - [:found]).empty?
end

#process_error(err, entity_hash) ⇒ Object 



365
366
367
368
369
370
371
372
373
374
 
# File 'lib/hammer_cli_import/base.rb', line 365

def process_error(err, entity_hash)
  uniq = nil
  err['errors'].each do |arr|
    next unless recognizable_error(arr)
    uniq = find_uniq(arr)
    break if uniq && entity_hash.key?(uniq.to_sym)
    uniq = nil # otherwise uniq is not usable
  end
  return uniq
end

#recognizable_error(arr) ⇒ Object 



361
362
363
 
# File 'lib/hammer_cli_import/base.rb', line 361

def recognizable_error(arr)
  return arr.is_a?(Array) && arr.size >= 2
end

#report_summary(verb, item) ⇒ Object

Method to call when you have created/deleted/found/mapped… something. Collected data used for summary reporting.

:found is used for situation, when you want to create something, but you found out, it is already created.



236
237
238
239
240
241
 
# File 'lib/hammer_cli_import/base.rb', line 236

def report_summary(verb, item)
  raise "Not summary supported action: #{verb}" unless
    [:created, :deleted, :found, :mapped, :skipped, :uploaded, :wrote, :failed].include? verb
  @summary[verb] ||= {}
  @summary[verb][item] = @summary[verb].fetch(item, 0) + 1
end

#split_multival(multival, convert_to_int = true, separator = ';') ⇒ Object 



225
226
227
228
229
 
# File 'lib/hammer_cli_import/base.rb', line 225

def split_multival(multival, convert_to_int = true, separator = ';')
  arr = (multival || '').split(separator).delete_if { |v| v == 'None' }
  arr.map!(&:to_i) if convert_to_int
  return arr
end

#to_singular(plural) ⇒ Object

Method for use when writing messages to user.

> to_singular(:contentveiws)
"contentview"
> to_singular(:repositories)
"repository"



221
222
223
 
# File 'lib/hammer_cli_import/base.rb', line 221

def to_singular(plural)
  return plural.to_s.gsub(/_/, ' ').sub(/s$/, '').sub(/ie$/, 'y')
end

#unmap_entity(entity_type, target_id) ⇒ Object 



343
344
345
346
 
# File 'lib/hammer_cli_import/base.rb', line 343

def unmap_entity(entity_type, target_id)
  deleted = @pm[entity_type].delete_value(target_id)
  info " Unmapped #{to_singular(entity_type)} with id #{target_id}: #{deleted}x" if deleted > 1
end

#update_entity(entity_type, id, entity_hash) ⇒ Object 



445
446
447
448
 
# File 'lib/hammer_cli_import/base.rb', line 445

def update_entity(entity_type, id, entity_hash)
  info "Updating #{to_singular(entity_type)} with id: #{id}"
  mapped_api_call(entity_type, :update, {:id => id}.merge!(entity_hash))
end

#wait_for_task(uuid, start_wait = 0, delta_wait = 1, max_wait = 10) ⇒ Object

Wait for asynchronous task.

  • uuid - UUID of async task.

  • start_wait - Seconds to wait before first check.

  • delta_wait - How much longer will every next wait be (unless max_wait is reached).

  • max_wait - Maximum time to wait between two checks.



504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
 
# File 'lib/hammer_cli_import/base.rb', line 504

def wait_for_task(uuid, start_wait = 0, delta_wait = 1, max_wait = 10)
  wait_time = start_wait
  if option_quiet?
    info "Waiting for the task [#{uuid}] "
  else
    print "Waiting for the task [#{uuid}] "
  end

  loop do
    sleep wait_time
    wait_time = [wait_time + delta_wait, max_wait].min
    print '.' unless option_quiet?
    STDOUT.flush unless option_quiet?
    task = api_call(:foreman_tasks, :show, {:id => uuid})
    next unless task['state'] == 'stopped'
    print "\n" unless option_quiet?
    return task['return'] == 'success'
  end
end

#was_translated(entity_type, import_id) ⇒ Object 



180
181
182
 
# File 'lib/hammer_cli_import/base.rb', line 180

def was_translated(entity_type, import_id)
  return @pm[entity_type].to_hash.value?(import_id)
end