Class: HammerCLIImport::ImportCommand::ConfigFileImportCommand

Inherits:
BaseCommand
  • Object
show all
Includes:
Open3
Defined in:
lib/hammer_cli_import/configfile.rb

Class Attribute Summary collapse

Attributes included from PersistentMap::Extend

#map_description, #map_target_entity, #maps

Instance Method Summary collapse

Methods inherited from BaseCommand

#_compare_hash, #_create_entity, #api_call, api_call, api_init, #create_entity, csv_columns, #cvs_iterate, #data_dir, #delete, #delete_entity, #delete_entity_by_import_id, #execute, #find_uniq, #found_errors, #get_cache, #get_original_id, #get_translated_id, #import, #initialize, #last_in_cache?, #list_server_entities, #load_cache, #lookup_entity, #lookup_entity_in_array, #lookup_entity_in_cache, #map_entity, #mapped_api_call, #post_delete, #print_summary, #process_error, #recognizable_error, #report_summary, #split_multival, #to_singular, #unmap_entity, #update_entity, #wait_for_task, #was_translated

Methods included from PersistentMap::Extend

#persistent_map, #persistent_maps

Methods included from ImportTools::ImportLogging::Extend

#add_logging_options

Methods included from AsyncTasksReactor::Extend

#add_async_tasks_reactor_options

Methods included from AsyncTasksReactor::Include

#atr_exit, #atr_init, #postpone_till, #wait_for

Methods included from ImportTools::Exceptional::Include

#handle_missing_and_supress, #silently

Methods included from ImportTools::Task::Include

#annotate_tasks

Methods included from ImportTools::ImportLogging::Include

#debug, #error, #fatal, #info, #log, #logtrace, #progress, #setup_logging, #warn

Methods included from PersistentMap::Include

#load_persistent_maps, #map_target_entity, #maps, #prune_persistent_maps, #save_persistent_maps

Constructor Details

This class inherits a constructor from HammerCLIImport::BaseCommand

Class Attribute Details

.interview_questions ⇒ Object

Returns the value of attribute interview_questions.



58
59
60
 
# File 'lib/hammer_cli_import/configfile.rb', line 58

def interview_questions
  @interview_questions
end

Instance Method Details

#build_and_upload ⇒ Object

We’re going to build a product-per-org, with a repo-per-channel and upload the built-puppet-module, one-per-repo

We’re using the hammer-repository-upload subcommand to do this, because the direct-API-route is ‘touchy’ and repo-upload already does all the Right Stuff



368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
 
# File 'lib/hammer_cli_import/configfile.rb', line 368

def build_and_upload
  progress 'Building and uploading puppet modules'
  prod_name = 'Imported Satellite5 Configuration Files'
  @modules.each do |mname, files|
    data = files[0]

    # Build the puppet-module for upload
    module_dir = build_puppet_module(mname)
    next if module_dir.nil?

    # Build/find the product
    product_hash = mk_product_hash(data, prod_name)
    composite_id = [data['org_id'].to_i, prod_name]
    product_id = create_entity(:products, product_hash, composite_id)['id']

    # Build the repo
    repo_hash = mk_repo_hash data, product_id
    # Try creating a repo in the product, skip if it fails
    repo = create_entity(:puppet_repositories, repo_hash,
                         [data['org_id'].to_i, data['channel_id'].to_i])

    # Find the built-module .tar.gz
    built_module_path = File.join('pkg',
                                  "#{mname}-#{@interview_answers['version']}.tar.gz")
    info "Uploading #{built_module_path}"

    # Ask hammer repository upload to Do Its Thing
    require 'hammer_cli_katello/repository'
    ucc = HammerCLIKatello::Repository::UploadContentCommand.new('', context)
    rc = ucc.run(%W(--id #{repo['id']} --path #{built_module_path}))

    # If hammer fails us, record it and move on
    if rc == 0
      report_summary :uploaded, :puppet_modules
    else
      report_summary :failed, :puppet_modules
    end
  end
end

#build_module_name(data) ⇒ Object

puppet-module-names are username-classname usernames can only be alphanumeric classnames can only be alphanumeric and ‘_’



85
86
87
88
89
90
91
92
93
94
 
# File 'lib/hammer_cli_import/configfile.rb', line 85

def build_module_name(data)
  owning_org = lookup_entity_in_cache(:organizations,
                                      {'id' => get_translated_id(:organizations, data['org_id'].to_i)})
  org_name = owning_org['name'].gsub(/[^0-9a-zA-Z]*/, '').downcase
  if org_name.empty?
    org_name = 'orgid' + owning_org['id'].to_s
  end
  chan_name = data['channel'].gsub(/[^0-9a-zA-Z_]/, '_').downcase
  return org_name + '-' + chan_name
end

#build_puppet_module(module_name) ⇒ Object 



182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/hammer_cli_import/configfile.rb', line 182

def build_puppet_module(module_name)
  module_dir = File.join(option_working_directory, module_name)
  return nil unless File.exist? module_dir
  Dir.chdir(module_dir)
  gen_cmd = 'env -i bash -l -c "puppet module build"'
  Open3.popen3(gen_cmd) do |_stdin, stdout, _stderr|
    rd = ''
    begin
      rd = stdout.readline while rd
      debug rd
    rescue EOFError
      debug 'Done reading'
    end
  end
  return module_dir
end

#clean_module(name) ⇒ Object

If module ‘name’ has been generated, throw away it filesystem existence



128
129
130
131
132
 
# File 'lib/hammer_cli_import/configfile.rb', line 128

def clean_module(name)
  path = File.join(option_working_directory, name)
  debug "Removing #{path}"
  system("rm -rf #{path}")
end

#delete_single_row(data) ⇒ Object 



270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
 
# File 'lib/hammer_cli_import/configfile.rb', line 270

def delete_single_row(data)
  # repo maps to channel_id
  composite_id = [data['org_id'].to_i, data['channel_id'].to_i]
  unless @pm[:puppet_repositories][composite_id]
    info "#{to_singular(:puppet_repositories).capitalize} with id #{composite_id} wasn't imported. Skipping deletion."
    return
  end

  # find out product id
  repo_id = get_translated_id(:puppet_repositories, composite_id)
  product_id = lookup_entity(:puppet_repositories, repo_id)['product']['id']
  # delete repo
  delete_entity(:puppet_repositories, composite_id)
  # delete its product, if it's not associated with any other repositories
  product = lookup_entity(:products, product_id, true)

  delete_entity_by_import_id(:products, product_id) if product['repository_count'] == 0
end

#export_files ⇒ Object

For each module, write file-content to <module>/files or <module>/templates, and fill <module>/manifests/init.pp with appropriate metadata



297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
 
# File 'lib/hammer_cli_import/configfile.rb', line 297

def export_files
  progress 'Writing converted files'
  @modules.each do |mname, files|
    module_dir = File.join(option_working_directory, mname)
    next unless File.exist? module_dir
    info "Found module #{mname}"
    dsl = ''

    fdir = File.join(module_dir, 'files')
    Dir.mkdir(fdir)
    tdir = File.join(module_dir, 'templates')
    Dir.mkdir(tdir)
    class_name = mname.partition('-').last

    files.each do |a_file|
      debug "...file #{a_file['name']}"

      dsl += "file { '#{a_file['name']}':\n"
      dsl += "  path => '#{a_file['path']}',\n"

      case a_file['file_type']
      when 'file'
        write_file(fdir, a_file['name'], a_file['contents'])
        dsl += "  source => 'puppet:///modules/#{mname}/#{a_file['name']}',\n"
        dsl += "  group => '#{a_file['groupname']}',\n"
        dsl += "  owner => '#{a_file['username']}',\n"
        dsl += "  ensure => 'file',\n"
        dsl += "  mode => '#{a_file['filemode']}',\n"
        dsl += "}\n\n"
      when 'template'
        write_file(tdir, a_file['name'] + '.erb', a_file['contents'])
        dsl += "  group => '#{a_file['groupname']}',\n"
        dsl += "  owner => '#{a_file['username']}',\n"
        dsl += "  ensure => 'file',\n"
        dsl += "  mode => '#{a_file['filemode']}',\n"
        dsl += "  content => template('#{mname}/#{a_file['name']}.erb'),\n"
        dsl += "}\n\n"
      when 'directory'
        dsl += "  group => '#{a_file['groupname']}',\n"
        dsl += "  owner => '#{a_file['username']}',\n"
        dsl += "  ensure => 'directory',\n"
        dsl += "  mode => '#{a_file['filemode']}',\n"
        dsl += "}\n\n"
      when'symlink'
        dsl += "  target => '#{a_file['symbolic_link']}',\n"
        dsl += "  ensure => 'link',\n"
        dsl += "}\n\n"
      end
      report_summary :wrote, :puppet_files
    end
    export_manifest(mname, class_name, dsl)
  end
end

#export_manifest(mname, channel_name, dsl) ⇒ Object 



351
352
353
354
355
356
357
358
359
360
 
# File 'lib/hammer_cli_import/configfile.rb', line 351

def export_manifest(mname, channel_name, dsl)
  debug "Exporting manifest #{option_working_directory}/#{mname}/manifests/init.pp"
  module_dir = File.join(option_working_directory, mname)
  mdir = File.join(module_dir, 'manifests')
  File.open(File.join(mdir, 'init.pp'), 'w') do |f|
    f.puts "class #{channel_name} {"
    f.puts dsl
    f.puts '}'
  end
end

#file_data(data) ⇒ Object 



209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/hammer_cli_import/configfile.rb', line 209

def file_data(data)
  # Everybody gets a name, which is 'path' with '/' chgd to '_'
  data['name'] = data['path'].gsub('/', '_')

  # UTF8 names are Issues, for Pulp - downgrade
  data['name'] = data['name'].gsub(/[^0-9a-zA-Z+\-\.]*/, '')
  if data['name'].empty?
     data['name'] = 'file_id' + data['file_id']
  end

  # If we're not type='file', done - return data
  return data unless data['file_type'] == 'file'

  # If we're not a binary-file, check for macros
  if data['is_binary'] == 'N'
    sdelim = data['delim_start']
    edelim = data['delim_end']
    cstr = data['contents']
    matched = false
    data['contents'] = cstr.gsub(/(#{Regexp.escape(sdelim)})(.*)(#{Regexp.escape(edelim)})/) do |_match|
      matched = true
      "<%= #{map_macro Regexp.last_match[2].strip!} %>"
    end if cstr
    # If we replaced any macros, we're now type='template'
    data['file_type'] = 'template' if matched
  else
    # If we're binary, base64-decode contents
    debug 'decoding'
    data['contents'] = data['contents'].unpack('m')
  end

  return data
end

#first_time_only ⇒ Object

Load the macro-mapping and interview-answers ONLY once-per-run



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/hammer_cli_import/configfile.rb', line 62

def first_time_only
  unless option_delete?
    # Load interview-answers
    @interview_answers = YAML.load_file(option_answers_file)
    @interview_answers['Y'] = 'Y'

    # Load macro-mappings
    if File.exist? option_macro_mapping
      @macros = YAML.load_file(option_macro_mapping)
    else
      @macros = {}
      warn "Macro-mapping file #{option_macro_mapping} not found, no puppet-facts will be assigned"
    end

    # Create the puppet-working-directory
    Dir.mkdir option_working_directory unless File.directory? option_working_directory
  end
  return 'loaded'
end

#generate_module(module_name) ⇒ Object

If we haven’t seen this module-name before, arrange to do ‘puppet generate module’ for it



201
202
203
204
205
206
207
 
# File 'lib/hammer_cli_import/configfile.rb', line 201

def generate_module(module_name)
  return if @modules.key? module_name

  @modules[module_name] = []
  clean_module(module_name)
  generate_module_template_for(module_name)
end

#generate_module_template_for(name) ⇒ Object

Create a puppet module-template on the filesystem, inside of working-directory



137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/hammer_cli_import/configfile.rb', line 137

def generate_module_template_for(name)
  module_name = name
  pwd = Dir.pwd
  Dir.chdir(option_working_directory)
  gen_cmd = "env -i bash -l -c '/usr/bin/puppet module generate #{name}'"
  Open3.popen3(gen_cmd) do |stdin, stdout, _stderr|
    begin
      stdout.sync = true
      ConfigFileImportCommand.interview_questions.each do |q|
        rd = ''
        until rd.include? '?'
          rd = stdout.readline
          debug "Read #{rd}"
        end
        answer = @interview_answers[q].gsub('#{module_name}', module_name)
        stdin.puts(answer)
      end
    rescue EOFError => e
      debug 'Done reading'
      break
    end
    begin
      rd = ''
      rd = stdout.readline while rd
    rescue EOFError
      debug 'Done reading'
    end
  end
  Dir.chdir(pwd)

  # Now that we have generated the module, add a 'description' to the
  # metadata.json file found at option_working_dir/<name>/metadata.json
  metadata_path = File.join(File.join(option_working_directory, name), 'metadata.json')
  if File.exist? metadata_path
    answer = @interview_answers['description'].gsub('#{module_name}', module_name)
    sed_cmd = "sed -i '\/\"summary\":\/a \\ \\ \"description\": \"#{answer}\",' #{metadata_path}"
    debug "About to issue #{sed_cmd}"
    system sed_cmd
    report_summary :wrote, :puppet_modules
  else
    info "Failed puppet module #{name}: metadata.json not created"
    report_summary :failed, :puppet_modules
  end
end

#import_single_row(data) ⇒ Object

Store all files into a hash keyed by module-name



259
260
261
262
263
264
265
266
267
268
 
# File 'lib/hammer_cli_import/configfile.rb', line 259

def import_single_row(data)
  @first_time ||= first_time_only
  @modules ||= {}

  mname = build_module_name(data)
  generate_module(mname)
  file_hash = file_data(data)
  debug "name #{data['name']}, path #{file_hash['path']}, type #{file_hash['file_type']}"
  @modules[mname] << file_hash
end

#map_macro(macro) ⇒ Object

Return a mapped puppet-fact for a macro, if there is one Otherwise, leave the macro in place

NOTE: rhn.system.net_interface* macros are special - they have the form rhn.system.net_interface.THING(INTERFACE) (eg, rhn.system.net_interface.netmask(eth0)) We need to look them up from config_macros.yml as rhn.system.net_interface.THING(eth_device),

and in the matching value (if there is one), replace "{NETWORK_INTERFACE}" with the
specified INTERFACE.

Ew. TODO: Make this less hard-coded, then beg for forgiveness.



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/hammer_cli_import/configfile.rb', line 106

def map_macro(macro)
  debug ">>> macro #{macro}"
  if @macros.key? macro
    return @macros[macro]
  elsif /^rhn\.system\.net_interface\.(.*)\((.*)\)/.match(macro)
    # Magic net_interface assumptions - hold onto your hats...
    net_interface_key = "rhn.system.net_interface.#{Regexp.last_match[1]}(eth_device)"
    # If the constructed key can't be found, shrug and move along
    return macro unless @macros.key? net_interface_key
    # We found a key we can use
    puppet_value = @macros[net_interface_key]
    # Bolt if we don't have a value
    return puppet_value if puppet_value.nil?
    # Return the value with the Magic String replaced with what the user specified
    return puppet_value.sub('{NETWORK INTERFACE}', Regexp.last_match[2])
  else
    return macro
  end
end

#mk_product_hash(data, product_name) ⇒ Object 



243
244
245
246
247
248
 
# File 'lib/hammer_cli_import/configfile.rb', line 243

def mk_product_hash(data, product_name)
  {
    :name => product_name,
    :organization_id => get_translated_id(:organizations, data['org_id'].to_i)
  }
end

#mk_repo_hash(data, product_id) ⇒ Object 



250
251
252
253
254
255
256
 
# File 'lib/hammer_cli_import/configfile.rb', line 250

def mk_repo_hash(data, product_id)
  {
    :name => data['channel'],
    :product_id => product_id,
    :content_type => 'puppet'
  }
end

#post_import(_csv) ⇒ Object 



408
409
410
411
412
 
# File 'lib/hammer_cli_import/configfile.rb', line 408

def post_import(_csv)
  return unless @modules
  export_files
  build_and_upload unless option_generate_only?
end

#write_file(dir, name, content) ⇒ Object 



289
290
291
292
293
 
# File 'lib/hammer_cli_import/configfile.rb', line 289

def write_file(dir, name, content)
  File.open(File.join(dir, name), 'w') do |f|
    f.syswrite(content)
  end
end