Class: HammerCLIImport::ImportCommand::ConfigFileImportCommand

Inherits:
BaseCommand
  • Object
show all
Includes:
Open3
Defined in:
lib/hammer_cli_import/configfile.rb

Class Attribute Summary collapse

Attributes included from PersistentMap::Extend

#map_description, #map_target_entity, #maps

Instance Method Summary collapse

Methods inherited from BaseCommand

#_compare_hash, #_create_entity, #api_call, api_call, api_init, #create_entity, csv_columns, #cvs_iterate, #data_dir, #delete, #delete_entity, #delete_entity_by_import_id, #execute, #find_uniq, #found_errors, #get_cache, #get_original_id, #get_translated_id, #import, #initialize, #last_in_cache?, #list_server_entities, #load_cache, #lookup_entity, #lookup_entity_in_array, #lookup_entity_in_cache, #map_entity, #mapped_api_call, #post_delete, #print_summary, #process_error, #recognizable_error, #report_summary, #split_multival, #to_singular, #unmap_entity, #update_entity, #wait_for_task, #was_translated

Methods included from PersistentMap::Extend

#persistent_map, #persistent_maps

Methods included from ImportTools::ImportLogging::Extend

#add_logging_options

Methods included from AsyncTasksReactor::Extend

#add_async_tasks_reactor_options

Methods included from AsyncTasksReactor::Include

#atr_exit, #atr_init, #postpone_till, #wait_for

Methods included from ImportTools::Exceptional::Include

#handle_missing_and_supress, #silently

Methods included from ImportTools::Task::Include

#annotate_tasks

Methods included from ImportTools::ImportLogging::Include

#debug, #error, #fatal, #info, #log, #logtrace, #progress, #setup_logging, #warn

Methods included from PersistentMap::Include

#load_persistent_maps, #map_target_entity, #maps, #prune_persistent_maps, #save_persistent_maps

Constructor Details

This class inherits a constructor from HammerCLIImport::BaseCommand

Class Attribute Details

.interview_questions ⇒ Object

Returns the value of attribute interview_questions.



58
59
60
 
# File 'lib/hammer_cli_import/configfile.rb', line 58

def interview_questions
  @interview_questions
end

Instance Method Details

#build_and_upload ⇒ Object

We’re going to build a product-per-org, with a repo-per-channel and upload the built-puppet-module, one-per-repo

We’re using the hammer-repository-upload subcommand to do this, because the direct-API-route is ‘touchy’ and repo-upload already does all the Right Stuff



354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
 
# File 'lib/hammer_cli_import/configfile.rb', line 354

def build_and_upload
  progress 'Building and uploading puppet modules'
  prod_name = 'Imported Satellite5 Configuration Files'
  @modules.each do |mname, files|
    data = files[0]

    # Build the puppet-module for upload
    module_dir = build_puppet_module(mname)

    # Build/find the product
    product_hash = mk_product_hash(data, prod_name)
    composite_id = [data['org_id'].to_i, prod_name]
    product_id = create_entity(:products, product_hash, composite_id)['id']

    # Build the repo
    repo_hash = mk_repo_hash data, product_id
    # Try creating a repo in the product, skip if it fails
    repo = create_entity(:puppet_repositories, repo_hash,
                         [data['org_id'].to_i, data['channel_id'].to_i])

    # Find the built-module .tar.gz
    built_module_path = File.join(File.join(module_dir, 'pkg'),
                                  "#{mname}-#{@interview_answers['version']}.tar.gz")
    info "Uploading #{built_module_path}"

    # Ask hammer repository upload to Do Its Thing
    require 'hammer_cli_katello/repository'
    ucc = HammerCLIKatello::Repository::UploadContentCommand.new('', context)
    rc = ucc.run(%W(--id #{repo['id']} --path #{built_module_path}))

    # If hammer fails us, record it and move on
    if rc == 0
      report_summary :uploaded, :puppet_modules
    else
      report_summary :failed, :puppet_modules
    end
  end
end

#build_module_name(data) ⇒ Object

puppet-module-names are username-classname usernames can only be alphanumeric classnames can only be alphanumeric and ‘_’



85
86
87
88
89
90
91
92
93
94
 
# File 'lib/hammer_cli_import/configfile.rb', line 85

def build_module_name(data)
  owning_org = lookup_entity_in_cache(:organizations,
                                      {'id' => get_translated_id(:organizations, data['org_id'].to_i)})
  org_name = owning_org['name'].gsub(/[^0-9a-zA-Z]*/, '').downcase
  if org_name.empty?
    org_name = 'orgid' + owning_org['id'].to_s
  end
  chan_name = data['channel'].gsub(/[^0-9a-zA-Z_]/, '_').downcase
  return org_name + '-' + chan_name
end

#build_puppet_module(module_name) ⇒ Object 



170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
 
# File 'lib/hammer_cli_import/configfile.rb', line 170

def build_puppet_module(module_name)
  module_dir = File.join(option_working_directory, module_name)
  Dir.chdir(module_dir)
  gen_cmd = 'puppet module build'
  Open3.popen3(gen_cmd) do |_stdin, stdout, _stderr|
    rd = ''
    begin
      rd = stdout.readline while rd
      debug rd
    rescue EOFError
      debug 'Done reading'
    end
  end
  return module_dir
end

#clean_module(name) ⇒ Object

If module ‘name’ has been generated, throw away it filesystem existence



128
129
130
131
132
 
# File 'lib/hammer_cli_import/configfile.rb', line 128

def clean_module(name)
  path = File.join(option_working_directory, name)
  debug "Removing #{path}"
  system("rm -rf #{path}")
end

#delete_single_row(data) ⇒ Object 



257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
 
# File 'lib/hammer_cli_import/configfile.rb', line 257

def delete_single_row(data)
  # repo maps to channel_id
  composite_id = [data['org_id'].to_i, data['channel_id'].to_i]
  unless @pm[:puppet_repositories][composite_id]
    info "#{to_singular(:puppet_repositories).capitalize} with id #{composite_id} wasn't imported. Skipping deletion."
    return
  end

  # find out product id
  repo_id = get_translated_id(:puppet_repositories, composite_id)
  product_id = lookup_entity(:puppet_repositories, repo_id)['product']['id']
  # delete repo
  delete_entity(:puppet_repositories, composite_id)
  # delete its product, if it's not associated with any other repositories
  product = lookup_entity(:products, product_id, true)

  delete_entity_by_import_id(:products, product_id) if product['repository_count'] == 0
end

#export_files ⇒ Object

For each module, write file-content to <module>/files or <module>/templates, and fill <module>/manifests/init.pp with appropriate metadata



284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
 
# File 'lib/hammer_cli_import/configfile.rb', line 284

def export_files
  progress 'Writing converted files'
  @modules.each do |mname, files|
    info "Found module #{mname}"
    dsl = ''

    module_dir = File.join(option_working_directory, mname)
    fdir = File.join(module_dir, 'files')
    Dir.mkdir(fdir)
    tdir = File.join(module_dir, 'templates')
    Dir.mkdir(tdir)
    class_name = mname.partition('-').last

    files.each do |a_file|
      debug "...file #{a_file['name']}"

      dsl += "file { '#{a_file['name']}':\n"
      dsl += "  path => '#{a_file['path']}',\n"

      case a_file['file_type']
      when 'file'
        write_file(fdir, a_file['name'], a_file['contents'])
        dsl += "  source => 'puppet:///modules/#{mname}/#{a_file['name']}',\n"
        dsl += "  group => '#{a_file['groupname']}',\n"
        dsl += "  owner => '#{a_file['username']}',\n"
        dsl += "  ensure => 'file',\n"
        dsl += "  mode => '#{a_file['filemode']}',\n"
        dsl += "}\n\n"
      when 'template'
        write_file(tdir, a_file['name'] + '.erb', a_file['contents'])
        dsl += "  group => '#{a_file['groupname']}',\n"
        dsl += "  owner => '#{a_file['username']}',\n"
        dsl += "  ensure => 'file',\n"
        dsl += "  mode => '#{a_file['filemode']}',\n"
        dsl += "  content => template('#{mname}/#{a_file['name']}.erb'),\n"
        dsl += "}\n\n"
      when 'directory'
        dsl += "  group => '#{a_file['groupname']}',\n"
        dsl += "  owner => '#{a_file['username']}',\n"
        dsl += "  ensure => 'directory',\n"
        dsl += "  mode => '#{a_file['filemode']}',\n"
        dsl += "}\n\n"
      when'symlink'
        dsl += "  target => '#{a_file['symbolic_link']}',\n"
        dsl += "  ensure => 'link',\n"
        dsl += "}\n\n"
      end
      report_summary :wrote, :puppet_files
    end
    export_manifest(mname, class_name, dsl)
  end
end

#export_manifest(mname, channel_name, dsl) ⇒ Object 



337
338
339
340
341
342
343
344
345
346
 
# File 'lib/hammer_cli_import/configfile.rb', line 337

def export_manifest(mname, channel_name, dsl)
  debug "Exporting manifest #{option_working_directory}/#{mname}/manifests/init.pp"
  module_dir = File.join(option_working_directory, mname)
  mdir = File.join(module_dir, 'manifests')
  File.open(File.join(mdir, 'init.pp'), 'w') do |f|
    f.puts "class #{channel_name} {"
    f.puts dsl
    f.puts '}'
  end
end

#file_data(data) ⇒ Object 



196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
 
# File 'lib/hammer_cli_import/configfile.rb', line 196

def file_data(data)
  # Everybody gets a name, which is 'path' with '/' chgd to '_'
  data['name'] = data['path'].gsub('/', '_')

  # UTF8 names are Issues, for Pulp - downgrade
  data['name'] = data['name'].gsub(/[^0-9a-zA-Z+\-\.]*/, '')
  if data['name'].empty?
     data['name'] = 'file_id' + data['file_id']
  end

  # If we're not type='file', done - return data
  return data unless data['file_type'] == 'file'

  # If we're not a binary-file, check for macros
  if data['is_binary'] == 'N'
    sdelim = data['delim_start']
    edelim = data['delim_end']
    cstr = data['contents']
    matched = false
    data['contents'] = cstr.gsub(/(#{Regexp.escape(sdelim)})(.*)(#{Regexp.escape(edelim)})/) do |_match|
      matched = true
      "<%= #{map_macro Regexp.last_match[2].strip!} %>"
    end if cstr
    # If we replaced any macros, we're now type='template'
    data['file_type'] = 'template' if matched
  else
    # If we're binary, base64-decode contents
    debug 'decoding'
    data['contents'] = data['contents'].unpack('m')
  end

  return data
end

#first_time_only ⇒ Object

Load the macro-mapping and interview-answers ONLY once-per-run



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/hammer_cli_import/configfile.rb', line 62

def first_time_only
  unless option_delete?
    # Load interview-answers
    @interview_answers = YAML.load_file(option_answers_file)
    @interview_answers['Y'] = 'Y'

    # Load macro-mappings
    if File.exist? option_macro_mapping
      @macros = YAML.load_file(option_macro_mapping)
    else
      @macros = {}
      warn "Macro-mapping file #{option_macro_mapping} not found, no puppet-facts will be assigned"
    end

    # Create the puppet-working-directory
    Dir.mkdir option_working_directory unless File.directory? option_working_directory
  end
  return 'loaded'
end

#generate_module(module_name) ⇒ Object

If we haven’t seen this module-name before, arrange to do ‘puppet generate module’ for it



188
189
190
191
192
193
194
 
# File 'lib/hammer_cli_import/configfile.rb', line 188

def generate_module(module_name)
  return if @modules.key? module_name

  @modules[module_name] = []
  clean_module(module_name)
  generate_module_template_for(module_name)
end

#generate_module_template_for(name) ⇒ Object

Create a puppet module-template on the filesystem, inside of working-directory



137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
 
# File 'lib/hammer_cli_import/configfile.rb', line 137

def generate_module_template_for(name)
  module_name = name
  Dir.chdir(option_working_directory)
  gen_cmd = "puppet module generate #{name}"
  Open3.popen3(gen_cmd) do |stdin, stdout, _stderr|
    stdout.sync = true
    ConfigFileImportCommand.interview_questions.each do |q|
      rd = ''
      until rd.include? '?'
        rd = stdout.readline
        #debug "Read #{rd}"
      end
      answer = @interview_answers[q].gsub('#{module_name}', module_name)
      stdin.puts(answer)
    end
    rd = ''
    begin
      rd = stdout.readline while rd
    rescue EOFError
      debug 'Done reading'
    end
  end

  # Now that we have generated the module, add a 'description' to the
  # metadata.json file found at option_working_dir/<name>/metadata.json
  metadata_path = File.join(File.join(option_working_directory, name), 'metadata.json')
  answer = @interview_answers['description'].gsub('#{module_name}', module_name)
  sed_cmd = "sed -i '\/\"summary\":\/a \\ \\ \"description\": \"#{answer}\",' #{metadata_path}"
  debug "About to issue #{sed_cmd}"
  system sed_cmd
  report_summary :wrote, :puppet_modules
end

#import_single_row(data) ⇒ Object

Store all files into a hash keyed by module-name



246
247
248
249
250
251
252
253
254
255
 
# File 'lib/hammer_cli_import/configfile.rb', line 246

def import_single_row(data)
  @first_time ||= first_time_only
  @modules ||= {}

  mname = build_module_name(data)
  generate_module(mname)
  file_hash = file_data(data)
  debug "name #{data['name']}, path #{file_hash['path']}, type #{file_hash['file_type']}"
  @modules[mname] << file_hash
end

#map_macro(macro) ⇒ Object

Return a mapped puppet-fact for a macro, if there is one Otherwise, leave the macro in place

NOTE: rhn.system.net_interface* macros are special - they have the form rhn.system.net_interface.THING(INTERFACE) (eg, rhn.system.net_interface.netmask(eth0)) We need to look them up from config_macros.yml as rhn.system.net_interface.THING(eth_device),

and in the matching value (if there is one), replace "{NETWORK_INTERFACE}" with the
specified INTERFACE.

Ew. TODO: Make this less hard-coded, then beg for forgiveness.



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/hammer_cli_import/configfile.rb', line 106

def map_macro(macro)
  debug ">>> macro #{macro}"
  if @macros.key? macro
    return @macros[macro]
  elsif /^rhn\.system\.net_interface\.(.*)\((.*)\)/.match(macro)
    # Magic net_interface assumptions - hold onto your hats...
    net_interface_key = "rhn.system.net_interface.#{Regexp.last_match[1]}(eth_device)"
    # If the constructed key can't be found, shrug and move along
    return macro unless @macros.key? net_interface_key
    # We found a key we can use
    puppet_value = @macros[net_interface_key]
    # Bolt if we don't have a value
    return puppet_value if puppet_value.nil?
    # Return the value with the Magic String replaced with what the user specified
    return puppet_value.sub('{NETWORK INTERFACE}', Regexp.last_match[2])
  else
    return macro
  end
end

#mk_product_hash(data, product_name) ⇒ Object 



230
231
232
233
234
235
 
# File 'lib/hammer_cli_import/configfile.rb', line 230

def mk_product_hash(data, product_name)
  {
    :name => product_name,
    :organization_id => get_translated_id(:organizations, data['org_id'].to_i)
  }
end

#mk_repo_hash(data, product_id) ⇒ Object 



237
238
239
240
241
242
243
 
# File 'lib/hammer_cli_import/configfile.rb', line 237

def mk_repo_hash(data, product_id)
  {
    :name => data['channel'],
    :product_id => product_id,
    :content_type => 'puppet'
  }
end

#post_import(_csv) ⇒ Object 



393
394
395
396
397
 
# File 'lib/hammer_cli_import/configfile.rb', line 393

def post_import(_csv)
  return unless @modules
  export_files
  build_and_upload unless option_generate_only?
end

#write_file(dir, name, content) ⇒ Object 



276
277
278
279
280
 
# File 'lib/hammer_cli_import/configfile.rb', line 276

def write_file(dir, name, content)
  File.open(File.join(dir, name), 'w') do |f|
    f.syswrite(content)
  end
end