Module: Gitlab::BackgroundMigration

Defined in:
lib/gitlab/background_migration.rb,
lib/gitlab/background_migration/logger.rb,
lib/gitlab/background_migration/copy_column.rb,
lib/gitlab/background_migration/reset_merge_status.rb,
lib/gitlab/background_migration/legacy_upload_mover.rb,
lib/gitlab/background_migration/calculate_wiki_sizes.rb,
lib/gitlab/background_migration/migrate_stage_status.rb,
lib/gitlab/background_migration/sync_issues_state_id.rb,
lib/gitlab/background_migration/fix_pages_access_level.rb,
lib/gitlab/background_migration/migrate_pages_metadata.rb,
lib/gitlab/background_migration/migrate_security_scans.rb,
lib/gitlab/background_migration/legacy_uploads_migrator.rb,
lib/gitlab/background_migration/fix_user_namespace_names.rb,
lib/gitlab/background_migration/mailers/unconfirm_mailer.rb,
lib/gitlab/background_migration/migrate_legacy_artifacts.rb,
lib/gitlab/background_migration/backfill_project_settings.rb,
lib/gitlab/background_migration/cleanup_concurrent_rename.rb,
lib/gitlab/background_migration/link_lfs_objects_projects.rb,
lib/gitlab/background_migration/populate_canonical_emails.rb,
lib/gitlab/background_migration/prune_orphaned_geo_events.rb,
lib/gitlab/background_migration/user_mentions/models/epic.rb,
lib/gitlab/background_migration/user_mentions/models/note.rb,
lib/gitlab/background_migration/fix_orphan_promoted_issues.rb,
lib/gitlab/background_migration/backfill_namespace_settings.rb,
lib/gitlab/background_migration/user_mentions/models/commit.rb,
lib/gitlab/background_migration/fix_user_project_route_names.rb,
lib/gitlab/background_migration/move_epic_issues_after_epics.rb,
lib/gitlab/background_migration/remove_duplicate_cs_findings.rb,
lib/gitlab/background_migration/sync_merge_requests_state_id.rb,
lib/gitlab/background_migration/backfill_project_repositories.rb,
lib/gitlab/background_migration/backfill_snippet_repositories.rb,
lib/gitlab/background_migration/generate_gitlab_subscriptions.rb,
lib/gitlab/background_migration/cleanup_concurrent_type_change.rb,
lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb,
lib/gitlab/background_migration/update_vulnerability_confidence.rb,
lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb,
lib/gitlab/background_migration/cleanup_optimistic_locking_nulls.rb,
lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb,
lib/gitlab/background_migration/fix_promoted_epics_discussion_ids.rb,
lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb,
lib/gitlab/background_migration/populate_user_highest_roles_table.rb,
lib/gitlab/background_migration/backfill_designs_relative_position.rb,
lib/gitlab/background_migration/backfill_push_rules_id_in_projects.rb,
lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb,
lib/gitlab/background_migration/recalculate_project_authorizations.rb,
lib/gitlab/background_migration/set_merge_request_diff_files_count.rb,
lib/gitlab/background_migration/user_mentions/models/merge_request.rb,
lib/gitlab/background_migration/populate_project_snippet_statistics.rb,
lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb,
lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb,
lib/gitlab/background_migration/backfill_legacy_project_repositories.rb,
lib/gitlab/background_migration/fix_projects_without_project_feature.rb,
lib/gitlab/background_migration/populate_personal_snippet_statistics.rb,
lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb,
lib/gitlab/background_migration/migrate_null_private_profile_to_false.rb,
lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb,
lib/gitlab/background_migration/populate_merge_request_assignees_table.rb,
lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb,
lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb,
lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb,
lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb,
lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb,
lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb,
lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb,
lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb,
lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb,
lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb,
lib/gitlab/background_migration/set_null_external_diff_store_to_local_value.rb,
lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb,
lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb,
lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb,
lib/gitlab/background_migration/backfill_deployment_clusters_from_deployments.rb,
lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb,
lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb,
lib/gitlab/background_migration/user_mentions/models/design_management/design.rb,
lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb,
lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb,
lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb,
lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb,
lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb,
lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb,
lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb,
lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb,
lib/gitlab/background_migration/recalculate_project_authorizations_with_min_max_user_id.rb,
lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb,
lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb,
lib/gitlab/background_migration/user_mentions/models/concerns/mentionable_migration_methods.rb,
lib/gitlab/background_migration/update_existing_subgroup_to_match_visibility_level_of_parent.rb

Defined Under Namespace

Modules: Mailers, UserMentions Classes: AddMergeRequestDiffCommitsCount, BackfillDeploymentClustersFromDeployments, BackfillDesignsRelativePosition, BackfillEnvironmentIdDeploymentMergeRequests, BackfillLegacyProjectRepositories, BackfillNamespaceSettings, BackfillProjectRepositories, BackfillProjectSettings, BackfillPushRulesIdInProjects, BackfillSnippetRepositories, BackfillVersionDataFromGitaly, CalculateWikiSizes, CleanupConcurrentRename, CleanupConcurrentSchemaChange, CleanupConcurrentTypeChange, CleanupOptimisticLockingNulls, CopyColumn, CopyMergeRequestTargetProjectToMergeRequestMetrics, FillValidTimeForPagesDomainCertificate, FixOrphanPromotedIssues, FixPagesAccessLevel, FixProjectsWithoutProjectFeature, FixProjectsWithoutPrometheusService, FixPromotedEpicsDiscussionIds, FixRubyObjectInAuditEvents, FixUserNamespaceNames, FixUserProjectRouteNames, GenerateGitlabSubscriptions, LegacyUploadMover, LegacyUploadsMigrator, LinkLfsObjectsProjects, Logger, MergeRequestAssigneesMigrationProgressCheck, MigrateApproverToApprovalRules, MigrateApproverToApprovalRulesCheckProgress, MigrateApproverToApprovalRulesInBatch, MigrateFingerprintSha256WithinKeys, MigrateIssueTrackersSensitiveData, MigrateLegacyArtifacts, MigrateNullPrivateProfileToFalse, MigratePagesMetadata, MigrateSecurityScans, MigrateStageStatus, MigrateUsersBioToUserDetails, MoveEpicIssuesAfterEpics, PopulateAnyApprovalRuleForMergeRequests, PopulateAnyApprovalRuleForProjects, PopulateCanonicalEmails, PopulateMergeRequestAssigneesTable, PopulatePersonalSnippetStatistics, PopulateProjectSnippetStatistics, PopulateResolvedOnDefaultBranchColumn, PopulateUserHighestRolesTable, PopulateVulnerabilityHistoricalStatistics, PruneOrphanedGeoEvents, RecalculateProjectAuthorizations, RecalculateProjectAuthorizationsWithMinMaxUserId, RemoveDuplicateCsFindings, RemoveUndefinedOccurrenceConfidenceLevel, RemoveUndefinedOccurrenceSeverityLevel, RemoveUndefinedVulnerabilityConfidenceLevel, RemoveUndefinedVulnerabilitySeverityLevel, ResetMergeStatus, SetMergeRequestDiffFilesCount, SetNullExternalDiffStoreToLocalValue, SetNullPackageFilesFileStoreToLocalValue, SyncIssuesStateId, SyncMergeRequestsStateId, UpdateExistingSubgroupToMatchVisibilityLevelOfParent, UpdateLocationFingerprintForContainerScanningFindings, UpdateVulnerabilitiesFromDismissalFeedback, UpdateVulnerabilitiesToDismissed, UpdateVulnerabilityConfidence, WrongfullyConfirmedEmailUnconfirmer

Class Method Summary collapse

Class Method Details

.dead_jobs?(migration_class) ⇒ Boolean

Returns:

  • (Boolean) 

77
78
79
80
81
 
# File 'lib/gitlab/background_migration.rb', line 77

def self.dead_jobs?(migration_class)
  dead_set = Sidekiq::DeadSet.new

  enqueued_job?([dead_set], migration_class)
end

.enqueued_job?(queues, migration_class) ⇒ Boolean

Returns:

  • (Boolean) 

107
108
109
110
111
112
113
114
115
 
# File 'lib/gitlab/background_migration.rb', line 107

def self.enqueued_job?(queues, migration_class)
  queues.each do |queue|
    queue.each do |job|
      return true if job.queue == self.queue && job.args.first == migration_class
    end
  end

  false
end

.exists?(migration_class, additional_queues = []) ⇒ Boolean

Returns:

  • (Boolean) 

70
71
72
73
74
75
 
# File 'lib/gitlab/background_migration.rb', line 70

def self.exists?(migration_class, additional_queues = [])
  enqueued = Sidekiq::Queue.new(self.queue)
  scheduled = Sidekiq::ScheduledSet.new

  enqueued_job?([enqueued, scheduled], migration_class)
end

.migration_class_for(class_name) ⇒ Object 


89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/gitlab/background_migration.rb', line 89

def self.migration_class_for(class_name)
  # We don't pass class name with Gitlab::BackgroundMigration:: prefix anymore
  # but some jobs could be already spawned so we need to have some backward compatibility period.
  # Can be removed since 13.x
  full_class_name_prefix_regexp = /\A(::)?Gitlab::BackgroundMigration::/

  if class_name.match(full_class_name_prefix_regexp)
    Gitlab::ErrorTracking.track_and_raise_for_dev_exception(
      StandardError.new("Full class name is used"),
      class_name: class_name
    )

    class_name = class_name.sub(full_class_name_prefix_regexp, '')
  end

  const_get(class_name, false)
end

.perform(class_name, arguments) ⇒ Object

Performs a background migration.

class_name - The name of the background migration class as defined in the

Gitlab::BackgroundMigration namespace.

arguments - The arguments to pass to the background migration's “perform”

method.


58
59
60
 
# File 'lib/gitlab/background_migration.rb', line 58

def self.perform(class_name, arguments)
  migration_class_for(class_name).new.perform(*arguments)
end

.queue ⇒ Object 


5
6
7
 
# File 'lib/gitlab/background_migration.rb', line 5

def self.queue
  @queue ||= BackgroundMigrationWorker.sidekiq_options['queue']
end

.remaining ⇒ Object 


62
63
64
65
66
67
68
 
# File 'lib/gitlab/background_migration.rb', line 62

def self.remaining
  scheduled = Sidekiq::ScheduledSet.new.count do |job|
    job.queue == self.queue
  end

  scheduled + Sidekiq::Queue.new(self.queue).size
end

.retrying_jobs?(migration_class) ⇒ Boolean

Returns:

  • (Boolean) 

83
84
85
86
87
 
# File 'lib/gitlab/background_migration.rb', line 83

def self.retrying_jobs?(migration_class)
  retry_set = Sidekiq::RetrySet.new

  enqueued_job?([retry_set], migration_class)
end

.steal(steal_class, retry_dead_jobs: false) ⇒ Object

Begins stealing jobs from the background migrations queue, blocking the caller until all jobs have been completed.

When a migration raises a StandardError it is going to retry up to three times, for example, to recover from a deadlock.

When Exception is being raised, it enqueues the migration again, and re-raises the exception.

steal_class - The name of the class for which to steal jobs.


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/gitlab/background_migration.rb', line 19

def self.steal(steal_class, retry_dead_jobs: false)
  queues = [
    Sidekiq::ScheduledSet.new,
    Sidekiq::Queue.new(self.queue)
  ]

  if retry_dead_jobs
    queues << Sidekiq::RetrySet.new
    queues << Sidekiq::DeadSet.new
  end

  queues.each do |queue|
    queue.each do |job|
      migration_class, migration_args = job.args

      next unless job.queue == self.queue
      next unless migration_class == steal_class
      next if block_given? && !(yield migration_args)

      begin
        perform(migration_class, migration_args) if job.delete
      rescue Exception # rubocop:disable Lint/RescueException
        BackgroundMigrationWorker # enqueue this migration again
          .perform_async(migration_class, migration_args)

        raise
      end
    end
  end
end