Module: PuppetLitmus::RakeHelper

Included in:
PuppetLitmus
Defined in:
lib/puppet_litmus/rake_helper.rb

Overview

helper methods for the litmus rake tasks

Defined Under Namespace

Classes: LitmusTimeoutError

Instance Method Summary collapse

Instance Method Details

#build_module(module_dir = nil, target_dir = nil) ⇒ String

Build the module in ‘module_dir` and put the resulting compressed tarball into `target_dir`.

Parameters:

  • opts —

    Hash of options to build the module

  • module_dir (String) (defaults to: nil) —

    The path of the module to build. If missing defaults to Dir.pwd

  • target_dir (String) (defaults to: nil) —

    The path the module will be built into. The default is <module_dir>/pkg

Returns:

  • (String) —

    The path to the built module



251
252
253
254
255
256
257
258
259
260
261
262
263
264
 
# File 'lib/puppet_litmus/rake_helper.rb', line 251

def build_module(module_dir = nil, target_dir = nil)
  require 'puppet/modulebuilder'

  module_dir ||= Dir.pwd
  target_dir ||= File.join(source_dir, 'pkg')

  puts "Building '#{module_dir}' into '#{target_dir}'"
  builder = Puppet::Modulebuilder::Builder.new(module_dir, target_dir, nil)

  # Force the metadata to be read. Raises if metadata could not be found
  _metadata = builder.metadata

  builder.build
end

#build_modules_in_dir(source_dir, target_dir = nil) ⇒ Array

Builds all the modules in a specified directory

Parameters:

  • source_dir (String) —

    the directory to get the modules from

  • target_dir (String) (defaults to: nil) —

    temporary location to store tarballs before uploading. This directory will be cleaned before use. The default is <source_dir>/pkg

Returns:

  • (Array) —

    an array of module tars’ filenames



271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
 
# File 'lib/puppet_litmus/rake_helper.rb', line 271

def build_modules_in_dir(source_dir, target_dir = nil)
  target_dir ||= File.join(Dir.pwd, 'pkg')
  # remove old build dir if exists, before we build afresh
  FileUtils.rm_rf(target_dir) if File.directory?(target_dir)

  module_tars = Dir.entries(source_dir).map do |entry|
    next if ['.', '..'].include? entry

    module_dir = File.join(source_dir, entry)
    next unless File.directory? module_dir

    build_module(module_dir, target_dir)
  end
  module_tars.compact
end

#build_modules_in_folder(source_folder) ⇒ Object

Deprecated.

Use ‘build_modules_in_dir` instead



288
289
290
 
# File 'lib/puppet_litmus/rake_helper.rb', line 288

def build_modules_in_folder(source_folder)
  build_modules_in_dir(source_folder)
end

#check_bolt_errors(result_set) ⇒ Hash

Parse out errors messages in result set returned by Bolt command.

Parameters:

  • result_set (Array) —

    result set returned by Bolt command.

Returns:

  • (Hash) —

    Errors grouped by target.



400
401
402
403
404
405
406
407
408
409
410
411
412
413
 
# File 'lib/puppet_litmus/rake_helper.rb', line 400

def check_bolt_errors(result_set)
  errors = {}
  # iterate through each error
  result_set.each do |target_result|
    status = target_result['status']
    # jump to the next one when there is not fail
    next if status != 'failure'

    target = target_result['target']
    # get some info from error
    errors[target] = target_result['value']
  end
  errors
end

#check_connectivity?(inventory_hash, target_node_name) ⇒ Boolean

Returns:

  • (Boolean) 


357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
 
# File 'lib/puppet_litmus/rake_helper.rb', line 357

def check_connectivity?(inventory_hash, target_node_name)
  Honeycomb.start_span(name: 'litmus.check_connectivity') do |span|
    ENV['HONEYCOMB_TRACE'] = span.to_trace_header
    # if we're only checking connectivity for a single node
    if target_node_name
      span.add_field('litmus.target_node_name', target_node_name)
      add_platform_field(inventory_hash, target_node_name)
    end

    include ::BoltSpec::Run
    target_nodes = find_targets(inventory_hash, target_node_name)
    puts "Checking connectivity for #{target_nodes.inspect}"
    span.add_field('litmus.target_nodes', target_nodes)

    results = run_command('cd .', target_nodes, config: nil, inventory: inventory_hash)
    span.add_field('litmus.bolt_result', results)
    failed = []
    results.reject { |r| r['status'] == 'success' }.each do |result|
      puts "Failure connecting to #{result['target']}:\n#{result.inspect}"
      failed.push(result['target'])
    end
    span.add_field('litmus.connectivity_success', results.select { |r| r['status'] == 'success' })
    span.add_field('litmus.connectivity_failure', results.reject { |r| r['status'] == 'success' })
    raise "Connectivity has failed on: #{failed}" unless failed.length.zero?

    puts 'Connectivity check PASSED.'
    true
  end
end

#configure_path(inventory_hash) ⇒ Object 



231
232
233
234
235
236
237
238
239
240
241
242
243
 
# File 'lib/puppet_litmus/rake_helper.rb', line 231

def configure_path(inventory_hash)
  results = []
  # fix the path on ssh_nodes
  unless inventory_hash['groups'].select { |group| group['name'] == 'ssh_nodes' && !group['targets'].empty? }.size.zero?
    results << run_command('echo PATH="$PATH:/opt/puppetlabs/puppet/bin" > /etc/environment',
                           'ssh_nodes', config: nil, inventory: inventory_hash)
  end
  unless inventory_hash['groups'].select { |group| group['name'] == 'winrm_nodes' && !group['targets'].empty? }.size.zero?
    results << run_command('[Environment]::SetEnvironmentVariable("Path", $env:Path + ";C:\Program Files\Puppet Labs\Puppet\bin;C:\Program Files (x86)\Puppet Labs\Puppet\bin", "Machine")',
                           'winrm_nodes', config: nil, inventory: inventory_hash)
  end
  results
end

#get_metadata_operating_systems(metadata) ⇒ String

Gets a string representing the operating system and version.

Parameters:

  • metadata (Hash) —

    metadata to parse for operating system info

Returns:

  • (String) —

    the operating system string with version info for use in provisioning.



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/puppet_litmus/rake_helper.rb', line 64

def get_metadata_operating_systems(metadata)
  return unless metadata.is_a?(Hash)
  return unless metadata['operatingsystem_support'].is_a?(Array)

  metadata['operatingsystem_support'].each do |os_info|
    next unless os_info['operatingsystem'] && os_info['operatingsystemrelease']

    os_name = case os_info['operatingsystem']
              when 'Amazon', 'Archlinux', 'AIX', 'OSX'
                next
              when 'OracleLinux'
                'oracle'
              when 'Windows'
                'win'
              else
                os_info['operatingsystem'].downcase
              end

    os_info['operatingsystemrelease'].each do |release|
      version = case os_name
                when 'ubuntu', 'osx'
                  release.sub('.', '')
                when 'sles'
                  release.gsub(%r{ SP[14]}, '')
                when 'win'
                  release = release.delete('.') if release.include? '8.1'
                  release.sub('Server', '').sub('10', '10-pro')
                else
                  release
                end

      yield "#{os_name}-#{version.downcase}-x86_64".delete(' ')
    end
  end
end

#install_agent(collection, targets, inventory_hash) ⇒ Object 



209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
 
# File 'lib/puppet_litmus/rake_helper.rb', line 209

def install_agent(collection, targets, inventory_hash)
  Honeycomb.start_span(name: 'litmus.install_agent') do |span|
    ENV['HONEYCOMB_TRACE'] = span.to_trace_header
    span.add_field('litmus.collection', collection)
    span.add_field('litmus.targets', targets)

    include ::BoltSpec::Run
    params = if collection.nil?
               {}
             else
               { 'collection' => collection }
             end
    raise "puppet_agent was not found in #{DEFAULT_CONFIG_DATA['modulepath']}, please amend the .fixtures.yml file" \
     unless File.directory?(File.join(DEFAULT_CONFIG_DATA['modulepath'], 'puppet_agent'))

    # using boltspec, when the runner is called it changes the inventory_hash dropping the version field. The clone works around this
    bolt_result = run_task('puppet_agent::install', targets, params, config: DEFAULT_CONFIG_DATA, inventory: inventory_hash.clone)
    raise_bolt_errors(bolt_result, 'Installation of agent failed.')
    bolt_result
  end
end

#install_module(inventory_hash, target_node_name, module_tar, module_repository = nil, ignore_dependencies = false) ⇒ Object

Install a specific module tarball to the specified target. This method installs dependencies using a forge repository.

Parameters:

  • inventory_hash (Hash) —

    the pre-loaded inventory

  • target_node_name (String) —

    the name of the target where the module should be installed

  • module_tar (String) —

    the filename of the module tarball to upload

  • module_repository (String) (defaults to: nil) —

    the URL for the forge to use for downloading modules. Defaults to the public Forge API.

  • ignore_dependencies (Boolean) (defaults to: false) —

    flag used to ignore module dependencies defaults to false.

Returns:

  • a bolt result



301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
 
# File 'lib/puppet_litmus/rake_helper.rb', line 301

def install_module(inventory_hash, target_node_name, module_tar, module_repository = nil, ignore_dependencies = false) # rubocop:disable Style/OptionalBooleanParameter
  Honeycomb.start_span(name: 'install_module') do |span|
    ENV['HONEYCOMB_TRACE'] = span.to_trace_header
    span.add_field('litmus.target_node_name', target_node_name)
    span.add_field('litmus.module_tar', module_tar)

    # make sure the module to install is not installed
    # otherwise `puppet module install` might silently skip it
    module_name = File.basename(module_tar, '.tar.gz').split('-', 3)[0..1].join('-')
    uninstall_module(inventory_hash.clone, target_node_name, module_name, force: true)

    include ::BoltSpec::Run

    target_nodes = find_targets(inventory_hash, target_node_name)
    span.add_field('litmus.target_nodes', target_nodes)
    bolt_result = upload_file(module_tar, File.basename(module_tar), target_nodes, options: {}, config: nil, inventory: inventory_hash.clone)
    raise_bolt_errors(bolt_result, 'Failed to upload module.')

    module_repository_opts = "--module_repository '#{module_repository}'" unless module_repository.nil?
    install_module_command = "puppet module install #{module_repository_opts} #{File.basename(module_tar)}"
    install_module_command += ' --ignore-dependencies --force' if ignore_dependencies.to_s.downcase == 'true'
    span.add_field('litmus.install_module_command', install_module_command)

    bolt_result = run_command(install_module_command, target_nodes, config: nil, inventory: inventory_hash.clone)
    raise_bolt_errors(bolt_result, "Installation of package #{File.basename(module_tar)} failed.")
    bolt_result
  end
end

#metadata_module_name ⇒ Object 



330
331
332
333
334
335
336
337
338
 
# File 'lib/puppet_litmus/rake_helper.rb', line 330

def metadata_module_name
  require 'json'
  raise 'Could not find metadata.json' unless File.exist?(File.join(Dir.pwd, 'metadata.json'))

  metadata = JSON.parse(File.read(File.join(Dir.pwd, 'metadata.json')))
  raise 'Could not read module name from metadata.json' if metadata['name'].nil?

  metadata['name']
end

#provision(provisioner, platform, inventory_vars) ⇒ Object 



119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
 
# File 'lib/puppet_litmus/rake_helper.rb', line 119

def provision(provisioner, platform, inventory_vars)
  include ::BoltSpec::Run
  raise "the provision module was not found in #{DEFAULT_CONFIG_DATA['modulepath']}, please amend the .fixtures.yml file" unless
    File.directory?(File.join(DEFAULT_CONFIG_DATA['modulepath'], 'provision'))

  params = { 'action' => 'provision', 'platform' => platform, 'inventory' => Dir.pwd }
  params['vars'] = inventory_vars unless inventory_vars.nil?

  Honeycomb.add_field_to_trace('litmus.provisioner', provisioner)
  Honeycomb.start_span(name: 'litmus.provision') do |span|
    ENV['HONEYCOMB_TRACE'] = span.to_trace_header
    span.add_field('litmus.platform', platform)

    task_name = provisioner_task(provisioner)
    span.add_field('litmus.task_name', task_name)
    span.add_field('litmus.params', params)
    span.add_field('litmus.config', DEFAULT_CONFIG_DATA)

    bolt_result = run_task(task_name, 'localhost', params, config: DEFAULT_CONFIG_DATA, inventory: nil)
    span.add_field('litmus.result', bolt_result)
    span.add_field('litmus.node_name', bolt_result&.first&.dig('value', 'node_name'))

    raise_bolt_errors(bolt_result, "provisioning of #{platform} failed.")

    bolt_result
  end
end

#provision_list(provision_hash, key) ⇒ Object 



147
148
149
150
151
152
153
154
155
156
157
158
159
 
# File 'lib/puppet_litmus/rake_helper.rb', line 147

def provision_list(provision_hash, key)
  provisioner = provision_hash[key]['provisioner']
  inventory_vars = provision_hash[key]['vars']
  # Splat the params into environment variables to pass to the provision task but only in this runspace
  provision_hash[key]['params']&.each { |k, value| ENV[k.upcase] = value.to_s }
  results = []

  Honeycomb.current_span.add_field('litmus.images', provision_hash[key]['images'])
  provision_hash[key]['images'].each do |image|
    results << provision(provisioner, image, inventory_vars)
  end
  results
end

#provisioner_task(provisioner) ⇒ Object 



387
388
389
390
391
392
393
394
 
# File 'lib/puppet_litmus/rake_helper.rb', line 387

def provisioner_task(provisioner)
  if SUPPORTED_PROVISIONERS.include?(provisioner)
    "provision::#{provisioner}"
  else
    warn "WARNING: Unsuported provisioner '#{provisioner}', try #{SUPPORTED_PROVISIONERS.join('/')}"
    provisioner.to_s
  end
end

#raise_bolt_errors(result_set, error_msg) ⇒ Object

Parse out errors messages in result set returned by Bolt command. If there are errors, raise them.

Parameters:

  • result_set (Array) —

    result set returned by Bolt command.

  • error_msg (String) —

    error message to raise when errors are detected. The actual errors will be appended.



419
420
421
422
423
424
425
426
427
428
 
# File 'lib/puppet_litmus/rake_helper.rb', line 419

def raise_bolt_errors(result_set, error_msg)
  errors = check_bolt_errors(result_set)

  unless errors.empty?
    formatted_results = errors.map { |k, v| "  #{k}: #{v.inspect}" }.join("\n")
    raise "#{error_msg}\nResults:\n#{formatted_results}}"
  end

  nil
end

#run_local_command(command) ⇒ Object

Executes a command on the test runner.

Parameters:

  • command (String) —

    command to execute.

Returns:

  • (Object) —

    the standard out stream.



104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/puppet_litmus/rake_helper.rb', line 104

def run_local_command(command)
  Honeycomb.start_span(name: 'litmus.run_local_command') do |span|
    ENV['HONEYCOMB_TRACE'] = span.to_trace_header
    span.add_field('litmus.command', command)

    require 'open3'
    stdout, stderr, status = Open3.capture3(command)
    error_message = "Attempted to run\ncommand:'#{command}'\nstdout:#{stdout}\nstderr:#{stderr}"

    raise error_message unless status.to_i.zero?

    stdout
  end
end

#start_spinner(message) ⇒ Object 



430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
 
# File 'lib/puppet_litmus/rake_helper.rb', line 430

def (message)
  if (ENV['CI'] || '').downcase == 'true'
    puts message
     = Thread.new do
      # CI systems are strange beasts, we only output a '.' every wee while to keep the terminal alive.
      loop do
        printf '.'
        sleep(10)
      end
    end
  else
    require 'tty-spinner'
     = TTY::Spinner.new("[:spinner] #{message}")
    .auto_spin
  end
  
end

#stop_spinner(spinner) ⇒ Object 



448
449
450
451
452
453
454
 
# File 'lib/puppet_litmus/rake_helper.rb', line 448

def ()
  if (ENV['CI'] || '').downcase == 'true'
    Thread.kill()
  else
    .success
  end
end

#tear_down(node_name, inventory_hash) ⇒ Object 



193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
 
# File 'lib/puppet_litmus/rake_helper.rb', line 193

def tear_down(node_name, inventory_hash)
  Honeycomb.start_span(name: 'litmus.tear_down') do |span|
    ENV['HONEYCOMB_TRACE'] = span.to_trace_header
    # how do we know what provisioner to use

    span.add_field('litmus.node_name', node_name)
    add_platform_field(inventory_hash, node_name)

    params = { 'action' => 'tear_down', 'node_name' => node_name, 'inventory' => Dir.pwd }
    node_facts = facts_from_node(inventory_hash, node_name)
    bolt_result = run_task(provisioner_task(node_facts['provisioner']), 'localhost', params, config: DEFAULT_CONFIG_DATA, inventory: nil)
    raise_bolt_errors(bolt_result, "tear_down of #{node_name} failed.")
    bolt_result
  end
end

#tear_down_nodes(targets, inventory_hash) ⇒ Object 



161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
 
# File 'lib/puppet_litmus/rake_helper.rb', line 161

def tear_down_nodes(targets, inventory_hash)
  Honeycomb.start_span(name: 'litmus.tear_down_nodes') do |span|
    ENV['HONEYCOMB_TRACE'] = span.to_trace_header
    span.add_field('litmus.targets', targets)

    include ::BoltSpec::Run
    config_data = { 'modulepath' => File.join(Dir.pwd, 'spec', 'fixtures', 'modules') }
    raise "the provision module was not found in #{config_data['modulepath']}, please amend the .fixtures.yml file" unless File.directory?(File.join(config_data['modulepath'], 'provision'))

    results = {}
    targets.each do |node_name|
      next if node_name == 'litmus_localhost'

      result = tear_down(node_name, inventory_hash)
      # Some provisioners tear_down targets that were created as a batch job.
      # These provisioners should return the list of additional targets
      # removed so that we do not attempt to process them.
      if result != [] && result[0]['value'].key?('removed')
        removed_targets = result[0]['value']['removed']
        result[0]['value'].delete('removed')
        removed_targets.each do |removed_target|
          targets.delete(removed_target)
          results[removed_target] = result
        end
      end

      results[node_name] = result unless result == []
    end
    results
  end
end

#uninstall_module(inventory_hash, target_node_name, module_to_remove = nil, **opts) ⇒ Object

Uninstall a module from a specified target

Parameters:

  • inventory_hash (Hash) —

    the pre-loaded inventory

  • target_node_name (String) —

    the name of the target where the module should be uninstalled

  • module_to_remove (String) (defaults to: nil) —

    the name of the module to remove. Defaults to the module under test.

  • opts (Hash) —

    additional options to pass on to ‘puppet module uninstall`



345
346
347
348
349
350
351
352
353
354
355
 
# File 'lib/puppet_litmus/rake_helper.rb', line 345

def uninstall_module(inventory_hash, target_node_name, module_to_remove = nil, **opts)
  include ::BoltSpec::Run
  module_name = module_to_remove || metadata_module_name
  target_nodes = find_targets(inventory_hash, target_node_name)
  install_module_command = "puppet module uninstall #{module_name}"
  install_module_command += ' --force' if opts[:force]
  bolt_result = run_command(install_module_command, target_nodes, config: nil, inventory: inventory_hash)
  # `puppet module uninstall --force` fails if the module is not installed. Ignore errors when force is set
  raise_bolt_errors(bolt_result, "uninstalling #{module_name} failed.") unless opts[:force]
  bolt_result
end

#with_retries(options: { tries: Float::INFINITY }, max_wait_minutes: 8) ⇒ Object 



467
468
469
470
471
472
473
474
 
# File 'lib/puppet_litmus/rake_helper.rb', line 467

def with_retries(options: { tries: Float::INFINITY }, max_wait_minutes: 8)
  stop = Time.now + (max_wait_minutes * 60)
  Retryable.retryable(options.merge(not: [LitmusTimeoutError])) do
    raise LitmusTimeoutError if Time.now > stop

    yield
  end
end