Class: Gitlab::BackgroundMigration::MoveCiBuildsMetadata

Inherits:
BatchedMigrationJob show all
Defined in:
lib/gitlab/background_migration/move_ci_builds_metadata.rb

Direct Known Subclasses

MoveCiBuildsMetadataSelfManaged

Defined Under Namespace

Classes: BulkJobDefinitionsCreator, EnvConfig, JobPresenter, JobsDefinitionsAssigner, JobsFilter, JobsPreloader

Constant Summary collapse

DEFS_UNIQ_ATTRS =

Each job definition is unique by these attributes.

[:project_id, :partition_id, :checksum].freeze

Constants inherited from BatchedMigrationJob

BatchedMigrationJob::DEFAULT_FEATURE_CATEGORY, BatchedMigrationJob::MINIMUM_PAUSE_MS

Constants included from Database::DynamicModelHelpers

Database::DynamicModelHelpers::BATCH_SIZE

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from BatchedMigrationJob

#batch_metrics, cursor, cursor?, cursor_columns, feature_category, #filter_batch, generic_instance, health_context_tables, #initialize, job_arguments, operation_name, scope_to, tables_to_check_for_vacuum

Methods included from Database::DynamicModelHelpers

define_batchable_model, #each_batch, #each_batch_range

Constructor Details

This class inherits a constructor from Gitlab::BackgroundMigration::BatchedMigrationJob

Class Method Details

.job_arguments_count ⇒ Object 



80
81
82
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 80

def self.job_arguments_count
  2
end

Instance Method Details

#bulk_insert_job_environments(attributes) ⇒ Object 



589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 589

def bulk_insert_job_environments(attributes)
  values_list = Arel::Nodes::ValuesList.new(attributes.map(&:values)).to_sql
  command = <<~SQL.squish
    WITH ci_job_attributes (project_id, ci_job_id, ci_pipeline_id, expanded_environment_name, options) AS (#{values_list})
    INSERT INTO job_environments (project_id, environment_id, ci_job_id, ci_pipeline_id, deployment_id, expanded_environment_name, options)
    SELECT
      ci_job_attributes.project_id,
      environments.id,
      ci_job_id,
      ci_pipeline_id,
      deployments.id,
      expanded_environment_name,
      options::jsonb
    FROM
      ci_job_attributes
      INNER JOIN environments ON environments.project_id = ci_job_attributes.project_id
        AND environments.name = ci_job_attributes.expanded_environment_name
      LEFT JOIN deployments ON deployments.deployable_id = ci_job_attributes.ci_job_id
        AND deployments.deployable_type = 'CommitStatus'
      ON CONFLICT DO NOTHING;
  SQL

  ApplicationRecord.connection.execute(command)
end

#ci_model(table_name, primary_key: :id) ⇒ Object 



535
536
537
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 535

def ci_model(table_name, primary_key: :id)
  define_batchable_model(table_name, connection: connection, primary_key: primary_key)
end

#copy_environments(jobs_batch) ⇒ Object

We used to store the job environments data in the p_ci_builds_metadata table, but now this is kept in a new table in the main database.



542
543
544
545
546
547
548
549
550
551
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 542

def copy_environments(jobs_batch)
  metadata_records = metadata_model
    .where("(p_ci_builds_metadata.build_id, p_ci_builds_metadata.partition_id) IN (?)",
      jobs_batch.select(:id, :partition_id))

  job_environment_attributes = fetch_environment_attributes(metadata_records)
  return if job_environment_attributes.empty?

  bulk_insert_job_environments(job_environment_attributes)
end

#definition_instance_model ⇒ Object 



512
513
514
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 512

def definition_instance_model
  @definition_instance_model ||= ci_model(:p_ci_job_definition_instances, primary_key: :job_id)
end

#definition_model ⇒ Object

Dynamic ActiveRecord models to interact with the database tables.



477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 477

def definition_model
  @definition_model ||= ci_model(:p_ci_job_definitions).tap do |klass|
    # Define instance methods, like `global_identifier` that returns an
    # array containing the unique attributes values.
    klass.class_eval do
      def global_identifier
        attributes.values_at(*DEFS_UNIQ_ATTRS.map(&:to_s))
      end
    end

    # Define class methods
    klass.instance_eval do
      # Similar to the fabricate method in app/models/ci/job_definition.rb#L44-74
      def initialize_from(job)
        config = job.definition_config

        attrs = {
          project_id: job.project_id,
          partition_id: job.partition_id,
          config: config,
          checksum: compute_checksum(config),
          created_at: Time.current,
          interruptible: config.fetch(:interruptible, false)
        }

        new(attrs)
      end

      def compute_checksum(config)
        Digest::SHA256.hexdigest(Gitlab::Json.dump(config))
      end
    end
  end
end

#extract_environment_attributes(metadata) ⇒ Object 



575
576
577
578
579
580
581
582
583
584
585
586
587
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 575

def extract_environment_attributes(metadata)
  attributes = metadata.attributes.slice(
    'project_id', 'ci_job_id', 'ci_pipeline_id', 'expanded_environment_name', 'options'
  )

  options = attributes['options'] || {}
  kubernetes_options = options['kubernetes']&.slice('namespace')
  options = options.slice('action', 'deployment_tier')
  options['kubernetes'] = kubernetes_options if kubernetes_options.present?

  attributes['options'] = options.to_json
  attributes
end

#fetch_environment_attributes(relation) ⇒ Object 



553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 553

def fetch_environment_attributes(relation)
  join_sql = <<~SQL.squish
    INNER JOIN p_ci_builds
      ON p_ci_builds.partition_id = p_ci_builds_metadata.partition_id
      AND p_ci_builds.id = p_ci_builds_metadata.build_id
  SQL

  select_sql = <<~SQL.squish
    p_ci_builds_metadata.project_id,
    p_ci_builds_metadata.build_id AS ci_job_id,
    p_ci_builds_metadata.expanded_environment_name,
    p_ci_builds_metadata.config_options -> 'environment' AS options,
    p_ci_builds.commit_id AS ci_pipeline_id
  SQL

  relation
    .where.not(expanded_environment_name: nil)
    .joins(join_sql)
    .select(select_sql)
    .map { |metadata| extract_environment_attributes(metadata) }
end

#job_artifact_model ⇒ Object 



531
532
533
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 531

def job_artifact_model
  @job_artifact_model ||= ci_model(:p_ci_job_artifacts)
end

#job_model ⇒ Object 



520
521
522
523
524
525
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 520

def job_model
  @job_model ||= ci_model(:p_ci_builds).tap do |model|
    model.serialize :options
    model.serialize :yaml_variables
  end
end

#job_taggings_model ⇒ Object 



516
517
518
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 516

def job_taggings_model
  @job_taggings_model ||= ci_model(:p_ci_build_tags)
end

#metadata_model ⇒ Object 



527
528
529
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 527

def metadata_model
  @metadata_model ||= ci_model(:p_ci_builds_metadata)
end

#perform ⇒ Object 



394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 394

def perform
  config = EnvConfig.new

  Gitlab::AppJsonLogger.info(message: "Migration cutoff config", class: self.class.name.to_s,
    migration_cutoff: config.migration_cutoff, processing_data_cutoff: config.processing_data_cutoff)

  each_sub_batch do |sub_batch|
    context = JobsFilter.new(self, sub_batch, config)

    update_jobs(context.job_ids_filter)
    update_job_artifacts(context.job_ids_filter)
    setup_definitions(context.jobs_for_definitions)
    copy_environments(context.jobs_for_environments)
  end
end

#setup_definitions(available_jobs) ⇒ Object 



410
411
412
413
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 410

def setup_definitions(available_jobs)
  jobs = JobsPreloader.new(self, available_jobs).execute
  JobsDefinitionsAssigner.new(self, jobs).execute
end

#update_job_artifacts(metadata_filters) ⇒ Object

This copies the data from p_ci_builds_metadata to p_ci_job_artifacts without overriding it. This data is shown in in the MR widget as exposed artifacts.



443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 443

def update_job_artifacts(metadata_filters)
  return if metadata_filters.empty?

  scoped_metadata_sql = <<~SQL.squish
    p_ci_job_artifacts.job_id = p_ci_builds_metadata.build_id
      AND p_ci_job_artifacts.partition_id = p_ci_builds_metadata.partition_id
  SQL

  update_sql = <<~SQL.squish
    exposed_as = COALESCE(
      p_ci_job_artifacts.exposed_as,
      p_ci_builds_metadata.config_options->'artifacts'->>'expose_as'
    ),
    exposed_paths = COALESCE(
      p_ci_job_artifacts.exposed_paths,
      CASE
        WHEN jsonb_typeof(p_ci_builds_metadata.config_options->'artifacts'->'paths') = 'array'
        THEN ARRAY(
          SELECT jsonb_array_elements_text(p_ci_builds_metadata.config_options->'artifacts'->'paths')
        )
        ELSE NULL
      END
    )
    FROM p_ci_builds_metadata
  SQL

  job_artifact_model
    .where(file_type: 2) # metadata
    .where(scoped_metadata_sql)
    .where([:job_id, :partition_id] => metadata_filters)
    .update_all(update_sql)
end

#update_jobs(metadata_filters) ⇒ Object

This copies the data from p_ci_builds_metadata to p_ci_builds without overriding it if it already exists in the destination table. This is usually showed in the job’s log page.



418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
 
# File 'lib/gitlab/background_migration/move_ci_builds_metadata.rb', line 418

def update_jobs(metadata_filters)
  return if metadata_filters.empty?

  scoped_metadata_sql = <<~SQL.squish
    p_ci_builds.id = p_ci_builds_metadata.build_id
      AND p_ci_builds.partition_id = p_ci_builds_metadata.partition_id
  SQL

  update_sql = <<~SQL
    scoped_user_id = COALESCE(p_ci_builds.scoped_user_id, (p_ci_builds_metadata.config_options->>'scoped_user_id')::bigint),
    timeout = COALESCE(p_ci_builds.timeout, p_ci_builds_metadata.timeout),
    timeout_source = COALESCE(p_ci_builds.timeout_source, p_ci_builds_metadata.timeout_source::smallint),
    exit_code = COALESCE(p_ci_builds.exit_code, p_ci_builds_metadata.exit_code),
    debug_trace_enabled = COALESCE(p_ci_builds.debug_trace_enabled, p_ci_builds_metadata.debug_trace_enabled)
    FROM p_ci_builds_metadata
  SQL

  job_model
    .where(scoped_metadata_sql)
    .where([:id, :partition_id] => metadata_filters)
    .update_all(update_sql)
end