Class: HybridPlatformsConductor::Deployer

Inherits:
Object
  • Object
show all
Includes:
LoggerHelpers, SafeMerge
Defined in:
lib/hybrid_platforms_conductor/deployer.rb

Overview

Gives ways to deploy on several nodes

Defined Under Namespace

Modules: ConfigDSLExtension

Constant Summary collapse

PACKAGING_FUTEX_FILE =

String: File used as a Futex for packaging

"#{Dir.tmpdir}/hpc_packaging"

Constants included from LoggerHelpers

LoggerHelpers::LEVELS_MODIFIERS, LoggerHelpers::LEVELS_TO_STDERR

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from SafeMerge

#safe_merge

Methods included from LoggerHelpers

#err, #init_loggers, #log_component=, #log_debug?, #log_level=, #out, #section, #set_loggers_format, #stderr_device, #stderr_device=, #stderr_displayed?, #stdout_device, #stdout_device=, #stdout_displayed?, #stdouts_to_s, #with_progress_bar

Constructor Details

#initialize(logger: Logger.new($stdout), logger_stderr: Logger.new($stderr), config: Config.new, cmd_runner: CmdRunner.new, nodes_handler: NodesHandler.new, actions_executor: ActionsExecutor.new, services_handler: ServicesHandler.new) ⇒ Deployer

Constructor

Parameters

  • logger (Logger): Logger to be used [default: Logger.new(STDOUT)]

  • logger_stderr (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]

  • config (Config): Config to be used. [default: Config.new]

  • cmd_runner (CmdRunner): Command executor to be used. [default: CmdRunner.new]

  • nodes_handler (NodesHandler): Nodes handler to be used. [default: NodesHandler.new]

  • actions_executor (ActionsExecutor): Actions Executor to be used. [default: ActionsExecutor.new]

  • services_handler (ServicesHandler): Services Handler to be used. [default: ServicesHandler.new]



143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 143

def initialize(
  logger: Logger.new($stdout),
  logger_stderr: Logger.new($stderr),
  config: Config.new,
  cmd_runner: CmdRunner.new,
  nodes_handler: NodesHandler.new,
  actions_executor: ActionsExecutor.new,
  services_handler: ServicesHandler.new
)
  init_loggers(logger, logger_stderr)
  @config = config
  @cmd_runner = cmd_runner
  @nodes_handler = nodes_handler
  @actions_executor = actions_executor
  @services_handler = services_handler
  @override_secrets = nil
  @secrets_readers = Plugins.new(
    :secrets_reader,
    logger: @logger,
    logger_stderr: @logger_stderr,
    init_plugin: proc do |plugin_class|
      plugin_class.new(
        logger: @logger,
        logger_stderr: @logger_stderr,
        config: @config,
        cmd_runner: @cmd_runner,
        nodes_handler: @nodes_handler
      )
    end
  )
  @provisioners = Plugins.new(:provisioner, logger: @logger, logger_stderr: @logger_stderr)
  @log_plugins = Plugins.new(
    :log,
    logger: @logger,
    logger_stderr: @logger_stderr,
    init_plugin: proc do |plugin_class|
      plugin_class.new(
        logger: @logger,
        logger_stderr: @logger_stderr,
        config: @config,
        cmd_runner: @cmd_runner,
        nodes_handler: @nodes_handler,
        actions_executor: @actions_executor
      )
    end
  )
  # Default values
  @use_why_run = false
  @timeout = nil
  @concurrent_execution = false
  @local_environment = false
  @nbr_retries_on_error = 0
end

Instance Attribute Details

#concurrent_execution ⇒ Object

Concurrent execution of the deployment? [default = false]

Boolean



123
124
125
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 123

def concurrent_execution
  @concurrent_execution
end

#local_environment ⇒ Object

Are we deploying in a local environment?

Boolean



127
128
129
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 127

def local_environment
  @local_environment
end

#nbr_retries_on_error ⇒ Object

Number of retries to do in case of non-deterministic errors during deployment

Integer



131
132
133
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 131

def nbr_retries_on_error
  @nbr_retries_on_error
end

#timeout ⇒ Object

Timeout (in seconds) to be used for each deployment, or nil for no timeout [default = nil]

Integer or nil



119
120
121
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 119

def timeout
  @timeout
end

#use_why_run ⇒ Object

Do we use why-run mode while deploying? [default = false]

Boolean



115
116
117
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 115

def use_why_run
  @use_why_run
end

Instance Method Details

#deploy_on(*nodes_selectors) ⇒ Object

Deploy on a given list of nodes selectors. The workflow is the following:

  1. Package the services to be deployed, considering the nodes, services and context (options, secrets, environment…)

  2. Deploy on the nodes (once per node to be deployed)

  3. Save deployment logs (in case of real deployment)

Parameters

  • nodes_selectors (Array<Object>): The list of nodes selectors we will deploy to.

Result

  • Hash<String, [Integer or Symbol, String, String]>: Exit status code (or Symbol in case of error or dry run), standard output and error for each node that has been deployed.



263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 263

def deploy_on(*nodes_selectors)
  # Get the sorted list of services to be deployed, per node
  # Hash<String, Array<String> >
  services_to_deploy = @nodes_handler.select_nodes(nodes_selectors.flatten).to_h do |node|
    [node, @nodes_handler.get_services_of(node)]
  end

  # Get the secrets to be deployed
  secrets = {}
  if @override_secrets
    secrets = @override_secrets
  else
    services_to_deploy.each do |node, services|
      # If there is no config for secrets, just use cli
      (@config.secrets_readers.empty? ? [{ secrets_readers: i[cli] }] : @nodes_handler.select_confs_for_node(node, @config.secrets_readers)).inject([]) do |secrets_readers, secrets_readers_info|
        secrets_readers + secrets_readers_info[:secrets_readers]
      end.sort.uniq.each do |secrets_reader|
        services.each do |service|
          node_secrets = @secrets_readers[secrets_reader].secrets_for(node, service)
          conflicting_path = safe_merge(secrets, node_secrets)
          raise "Secret set at path #{conflicting_path.join('->')} by #{secrets_reader} for service #{service} on node #{node} has conflicting values (#{log_debug? ? "#{node_secrets.dig(*conflicting_path)} != #{secrets.dig(*conflicting_path)}" : 'set debug for value details'})." unless conflicting_path.nil?
        end
      end
    end
  end

  # Check that we are allowed to deploy
  unless @use_why_run
    reason_for_interdiction = @services_handler.deploy_allowed?(
      services: services_to_deploy,
      local_environment: @local_environment
    )
    raise "Deployment not allowed: #{reason_for_interdiction}" unless reason_for_interdiction.nil?
  end

  # Package the deployment
  # Protect packaging by a Futex
  Futex.new(PACKAGING_FUTEX_FILE, timeout: @config.packaging_timeout_secs).open do
    section 'Packaging deployment' do
      @services_handler.package(
        services: services_to_deploy,
        secrets: secrets,
        local_environment: @local_environment
      )
    end
  end

  # Prepare the deployment as a whole, before getting individual deployment actions.
  # Do this after packaging, this way we ensure that services packaging cannot depend on the way deployment will be performed.
  @services_handler.prepare_for_deploy(
    services: services_to_deploy,
    secrets: secrets,
    local_environment: @local_environment,
    why_run: @use_why_run
  )

  # Launch deployment processes
  results = {}

  section "#{@use_why_run ? 'Checking' : 'Deploying'} on #{services_to_deploy.keys.size} nodes" do
    # Prepare all the control masters here, as they will be reused for the whole process, including mutexes, deployment and logs saving
    @actions_executor.with_connections_prepared_to(services_to_deploy.keys, no_exception: true) do

      nbr_retries = @nbr_retries_on_error
      remaining_nodes_to_deploy = services_to_deploy.keys
      while nbr_retries >= 0 && !remaining_nodes_to_deploy.empty?
        last_deploy_results = deploy(services_to_deploy.slice(*remaining_nodes_to_deploy))
        if nbr_retries.positive?
          # Check if we need to retry deployment on some nodes
          # Only parse the last deployment attempt logs
          retriable_nodes = remaining_nodes_to_deploy.
            map do |node|
              exit_status, stdout, stderr = last_deploy_results[node]
              if exit_status.zero?
                nil
              else
                retriable_errors = retriable_errors_from(node, exit_status, stdout, stderr)
                if retriable_errors.empty?
                  nil
                else
                  # Log the issue in the stderr of the deployment
                  stderr << "!!! #{retriable_errors.size} retriable errors detected in this deployment:\n#{retriable_errors.map { |error| "* #{error}" }.join("\n")}\n"
                  [node, retriable_errors]
                end
              end
            end.
            compact.
            to_h
          unless retriable_nodes.empty?
            log_warn "              Retry deployment for \#{retriable_nodes.size} nodes as they got non-deterministic errors (\#{nbr_retries} retries remaining):\n              \#{retriable_nodes.map { |node, retriable_errors| \"  * \#{node}:\\n\#{retriable_errors.map { |error| \"    - \#{error}\" }.join(\"\\n\")}\" }.join(\"\\n\")}\n            EO_LOG\n          end\n          remaining_nodes_to_deploy = retriable_nodes.keys\n        end\n        # Merge deployment results\n        results.merge!(last_deploy_results) do |_node, (exit_status_1, stdout_1, stderr_1), (exit_status_2, stdout_2, stderr_2)|\n          [\n            exit_status_2,\n            <<~EO_STDOUT,\n              \#{stdout_1}\n              Deployment exit status code: \#{exit_status_1}\n              !!! Retry deployment due to non-deterministic error (\#{nbr_retries} remaining attempts)...\n              \#{stdout_2}\n            EO_STDOUT\n            <<~EO_STDERR\n              \#{stderr_1}\n              !!! Retry deployment due to non-deterministic error (\#{nbr_retries} remaining attempts)...\n              \#{stderr_2}\n            EO_STDERR\n          ]\n        end\n        nbr_retries -= 1\n      end\n\n    end\n  end\n  results\nend\n".strip

#deployment_info_from(*nodes) ⇒ Object

Get deployment info from a list of nodes selectors

Parameters

  • nodes (Array<String>): Nodes to get info from

Result

  • Hash<String, Hash<Symbol,Object>: The deployed info, per node name.

    • error (String): Error string in case deployment logs could not be retrieved. If set then further properties will be ignored. [optional]

    • services (Array<String>): List of services deployed on the node

    • deployment_info (Hash<Symbol,Object>): Deployment metadata

    • exit_status (Integer or Symbol): Deployment exit status

    • stdout (String): Deployment stdout

    • stderr (String): Deployment stderr



464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 464

def deployment_info_from(*nodes)
  nodes = nodes.flatten
  @actions_executor.max_threads = 64
  read_actions_results = @actions_executor.execute_actions(
    nodes.map do |node|
      master_log_plugin = @log_plugins[log_plugins_for(node).first]
      master_log_plugin.respond_to?(:actions_to_read_logs) ? [node, master_log_plugin.actions_to_read_logs(node)] : nil
    end.compact.to_h,
    log_to_stdout: false,
    concurrent: true,
    timeout: 10,
    progress_name: 'Read deployment logs'
  )
  nodes.to_h do |node|
    exit_code, stdout, stderr = read_actions_results[node] || [nil, nil, nil]
    [
      node,
      @log_plugins[log_plugins_for(node).first].logs_for(
        node,
        exit_code,
        stdout&.force_encoding(Encoding::UTF_8),
        stderr&.force_encoding(Encoding::UTF_8)
      )
    ]
  end
end

#options_parse(options_parser, parallel_switch: true, why_run_switch: false, timeout_options: true) ⇒ Object

Complete an option parser with options meant to control this Deployer

Parameters

  • options_parser (OptionParser): The option parser to complete

  • parallel_switch (Boolean): Do we allow parallel execution to be switched? [default = true]

  • why_run_switch (Boolean): Do we allow the why-run mode to be switched? [default = false]

  • timeout_options (Boolean): Do we allow timeout options? [default = true]



204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 204

def options_parse(options_parser, parallel_switch: true, why_run_switch: false, timeout_options: true)
  options_parser.separator ''
  options_parser.separator 'Deployer options:'
  if parallel_switch
    options_parser.on('-p', '--parallel', 'Execute the commands in parallel (put the standard output in files <hybrid-platforms-dir>/run_logs/*.stdout)') do
      @concurrent_execution = true
    end
  end
  if timeout_options
    options_parser.on('-t', '--timeout SECS', "Timeout in seconds to wait for each chef run. Only used in why-run mode. (defaults to #{@timeout.nil? ? 'no timeout' : @timeout})") do |nbr_secs|
      @timeout = nbr_secs.to_i
    end
  end
  if why_run_switch
    options_parser.on('-W', '--why-run', 'Use the why-run mode to see what would be the result of the deploy instead of deploying it for real.') do
      @use_why_run = true
    end
  end
  options_parser.on('--retries-on-error NBR', "Number of retries in case of non-deterministic errors (defaults to #{@nbr_retries_on_error})") do |nbr_retries|
    @nbr_retries_on_error = nbr_retries.to_i
  end
  # Display options secrets readers might have
  @secrets_readers.each do |secret_reader_name, secret_reader|
    next unless secret_reader.respond_to?(:options_parse)

    options_parser.separator ''
    options_parser.separator "Secrets reader #{secret_reader_name} options:"
    secret_reader.options_parse(options_parser)
  end
end

#override_secrets(secrets) ⇒ Object

Override the secrets with a given JSON. When using this method with a secrets Hash, further deployments will not query secrets readers, but will use those secrets directly. Useful to override secrets in test conditions when using dummy secrets for example.

Parameters

  • secrets (Hash or nil): Secrets to take into account in place of secrets readers, or nil to cancel a previous overriding and use secrets readers instead.



249
250
251
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 249

def override_secrets(secrets)
  @override_secrets = secrets
end

#parse_deploy_output(_node, stdout, stderr) ⇒ Object

Parse stdout and stderr of a given deploy run and get the list of tasks with their status

Parameters

  • node (String): Node for which this deploy run has been done.

  • stdout (String): stdout to be parsed.

  • stderr (String): stderr to be parsed.

Result

  • Array< Hash<Symbol,Object> >: List of task properties. The following properties should be returned, among free ones:

    • name (String): Task name

    • status (Symbol): Task status. Should be on of:

      • :changed: The task has been changed

      • :identical: The task has not been changed

    • diffs (String): Differences, if any



504
505
506
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 504

def parse_deploy_output(_node, stdout, stderr)
  @services_handler.parse_deploy_output(stdout, stderr).map { |deploy_info| deploy_info[:tasks] }.flatten
end

#validate_params ⇒ Object

Validate that parsed parameters are valid



236
237
238
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 236

def validate_params
  raise 'Can\'t have a timeout unless why-run mode. Please don\'t use --timeout without --why-run.' if !@timeout.nil? && !@use_why_run
end

#with_test_provisioned_instance(provisioner_id, node, environment:, reuse_instance: false) ⇒ Object

Provision a test instance for a given node.

Parameters

  • provisioner_id (Symbol): The provisioner ID to be used

  • node (String): The node for which we want the image

  • environment (String): An ID to differentiate different running instances for the same node

  • reuse_instance (Boolean): Do we reuse an eventual existing instance? [default: false]

  • Proc: Code called when the container is ready. The container will be stopped at the end of execution.

    • Parameters

      • deployer (Deployer): A new Deployer configured to override access to the node through the Docker container

      • instance (Provisioner): The provisioned instance



395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
 
# File 'lib/hybrid_platforms_conductor/deployer.rb', line 395

def with_test_provisioned_instance(provisioner_id, node, environment:, reuse_instance: false)
  # Add the user to the environment to better track belongings on shared provisioners
  environment = "#{@cmd_runner.whoami}_#{environment}"
  # Add PID, TID and a random number to the ID to make sure other containers used by other runs are not being reused.
  environment << "_#{Process.pid}_#{Thread.current.object_id}_#{SecureRandom.hex(8)}" unless reuse_instance
  # Create different NodesHandler and Deployer to handle this Docker container in place of the real node.
  sub_logger, sub_logger_stderr =
    if log_debug?
      [@logger, @logger_stderr]
    else
      [Logger.new(StringIO.new, level: :info), Logger.new(StringIO.new, level: :info)]
    end
  begin
    sub_executable = Executable.new(logger: sub_logger, logger_stderr: sub_logger_stderr)
    instance = @provisioners[provisioner_id].new(
      node,
      environment: environment,
      logger: @logger,
      logger_stderr: @logger_stderr,
      config: sub_executable.config,
      cmd_runner: @cmd_runner,
      # Here we use the NodesHandler that will be bound to the sub-Deployer only, as the node's metadata might be modified by the Provisioner.
      nodes_handler: sub_executable.nodes_handler,
      actions_executor: @actions_executor
    )
    instance.with_running_instance(stop_on_exit: true, destroy_on_exit: !reuse_instance, port: 22) do
      # Test-provisioned nodes have SSH Session Exec capabilities and are not local
      sub_executable.nodes_handler.override_metadata_of node, :ssh_session_exec, true
      sub_executable.nodes_handler.override_metadata_of node, :local_node, false
      # Test-provisioned nodes use default sudo
      sub_executable.config.sudo_procs.replace(sub_executable.config.sudo_procs.map do |sudo_proc_info|
        {
          nodes_selectors_stack: sudo_proc_info[:nodes_selectors_stack].map do |nodes_selector|
            @nodes_handler.select_nodes(nodes_selector).reject { |selected_node| selected_node == node }
          end,
          sudo_proc: sudo_proc_info[:sudo_proc]
        }
      end)
      actions_executor = sub_executable.actions_executor
      deployer = sub_executable.deployer
      # Setup test environment for this container
      actions_executor.connector(:ssh).ssh_user = 'root'
      actions_executor.connector(:ssh).passwords[node] = 'root_pwd'
      deployer.local_environment = true
      # Ignore secrets that might have been given: in Docker containers we always use dummy secrets
      dummy_secrets_file = "#{@config.hybrid_platforms_dir}/dummy_secrets.json"
      deployer.override_secrets(File.exist?(dummy_secrets_file) ? JSON.parse(File.read(dummy_secrets_file)) : {})
      yield deployer, instance
    end
  rescue
    # Make sure Docker logs are being output to better investigate errors if we were not already outputing them in debug mode
    stdouts = sub_executable.stdouts_to_s
    log_error "[ #{node}/#{environment} ] - Encountered unhandled exception #{$ERROR_INFO}\n#{$ERROR_INFO.backtrace.join("\n")}\n-----\n#{stdouts}" unless stdouts.nil?
    raise
  end
end