Class: Gitlab::BackgroundMigration::BatchedMigrationJob

Inherits:

Object

• Object
• Gitlab::BackgroundMigration::BatchedMigrationJob

Includes:

ClassAttributes, Database::DynamicModelHelpers

Defined in:

lib/gitlab/background_migration/batched_migration_job.rb

Overview

Base class for batched background migrations. Subclasses should implement the ‘#perform` method as the entry point for the job’s execution.

Job arguments needed must be defined explicitly, see docs.gitlab.com/ee/development/database/batched_background_migrations.html#use-job-arguments. rubocop:disable Metrics/ClassLength rubocop:disable Metrics/ParameterLists

Direct Known Subclasses

AlterWebhookDeletedAuditEvent, BackfillAdminModeScopeForPersonalAccessTokens, BackfillArchivedAndTraversalIdsToVulnerabilityReads, BackfillBranchProtectionNamespaceSetting, BackfillCatalogResourceVersionSemVer, BackfillCatalogResourceVersionsReleasedAt, BackfillComplianceViolationNullTargetProjectIds, BackfillDefaultBranchProtectionNamespaceSetting, BackfillDefaultBranchProtectionSettings, BackfillDefaultOrganizationOwners, BackfillDefaultOrganizationOwnersAgain, BackfillDefaultOrganizationUsers, BackfillDeployTokensShardingKey, BackfillDesignManagementRepositories, BackfillDesiredShardingKeyJob, BackfillDetectedAtFromCreatedAtColumn, BackfillDismissalReasonInVulnerabilityReads, BackfillEpicBasicFieldsToWorkItemRecord, BackfillEpicDatesToWorkItemDatesSources, BackfillEpicIssuesIntoWorkItemParentLinks, BackfillEventsShardingKey, BackfillFindingIdInVulnerabilities, BackfillFindingInitialPipelineId, BackfillFreeSharedRunnersMinutesLimit, BackfillHasIssuesForExternalIssueLinks, BackfillHasRemediationsOfVulnerabilityReads, BackfillHasVulnerabilityResolution, BackfillIdentifierNamesOfVulnerabilityReads, BackfillImportedIssueSearchData, BackfillIssueLinkIdOnRelatedEpicLinks, BackfillIssueSearchDataNamespaceId, BackfillIssuesCorrectWorkItemTypeId, BackfillIssuesDatesWithWorkItemDatesSources, BackfillJiraTrackerDataProjectKeys, BackfillMembersRequestAcceptedAt, BackfillMergeRequestDiffsProjectId, BackfillMissingCiCdSettings, BackfillMissingVulnerabilityDismissalDetails, BackfillNamespaceIdOfVulnerabilityReads, BackfillNewAuditEventTables, BackfillNugetNormalizedVersion, BackfillOnboardingStatusStepUrl, BackfillOrDropCiPipelineOnProjectId, BackfillPCiPipelinesTriggerId, BackfillPackagesDependenciesProjectId, BackfillPackagesTagsProjectId, BackfillPartitionIdCiDailyBuildGroupReportResult, BackfillPartitionIdCiPipelineArtifact, BackfillPartitionIdCiPipelineChatData, BackfillPartitionIdCiPipelineConfig, BackfillPartitionIdCiPipelineMessage, BackfillPartitionIdCiPipelineMetadata, BackfillPartitionedTable, BackfillPersonalAccessTokenSevenDaysNotificationSent, BackfillProjectFeaturePackageRegistryAccessLevel, BackfillProjectIdToDependencyListExports, BackfillProjectIdToSecurityScans, BackfillProjectMemberNamespaceId, BackfillProjectNamespaceOnIssues, BackfillRelatedEpicLinksToIssueLinks, BackfillReservedStorageBytes, BackfillRootNamespaceClusterAgentMappings, BackfillRunnerTypeAndShardingKeyIdOnCiRunnerManagers, BackfillSeatAssignmentsTable, BackfillSecurityPolicies, BackfillShardingKeyIdOnCiRunners, BackfillSoftwareLicenseSpdxIdentifierForSoftwareLicensePolicies, BackfillSubscriptionAddOnPurchasesStartedAt, BackfillUpstreamPipelinePartitionIdOnPCiBuilds, BackfillUserDetails, BackfillUserPreferencesWithDefaults, BackfillUsersColorModeId, BackfillUsersWithDefaults, BackfillVsCodeSettingsUuid, BackfillVsCodeSettingsVersion, BackfillVulnerabilityNamespaceHistoricalStatistics, BackfillVulnerabilityReadsClusterAgent, BackfillWorkItemHierarchyForEpics, BackfillWorkspacePersonalAccessToken, BackfillZoektReplicas, BackupAndRemoveNotesWithNullNoteableType, CleanupOrphanedRoutes, ConvertCreditCardValidationDataToHashes, CopyColumnUsingBackgroundMigrationJob, CopyTaggingsToPCiBuildTags, CreateComplianceStandardsAdherence, CreateJiraCloudAppIntegrationForJiraConnectSubscription, CreateMissingExternalLinksForVulnerabilities, CreateMissingTerraformModuleMetadata, CreateVulnerabilityLinks, DeduplicateLfsObjectsProjects, DeleteDuplicateIssuableResourceLinks, DeleteInvalidEpicIssues, DeleteInvalidProtectedBranchMergeAccessLevels, DeleteInvalidProtectedBranchPushAccessLevels, DeleteInvalidProtectedTagCreateAccessLevels, DeleteOrphanedBuildRecords, DeleteOrphanedDeployTokens, DeleteOrphanedGroups, DeleteOrphanedPackagesDependencies, DeleteOrphanedPartitionedCiRunnerMachineRecords, DeleteOrphanedPipelineVariableRecords, DeleteOrphanedStageRecords, DeleteOrphanedTransferredProjectApprovalRules, DeleteOrphansApprovalMergeRequestRules2, DeleteOrphansApprovalProjectRules2, DeleteStalePackagesNpmMetadataCaches, DestroyInvalidGroupMembers, DestroyInvalidMembers, DestroyInvalidProjectMembers, DisableAllowRunnerRegistrationOnNamespaceSettingsForGitlabCom, DisableLegacyOpenSourceLicenseForInactivePublicProjects, DisableLegacyOpenSourceLicenseForNoIssuesNoRepoProjects, DisableLegacyOpenSourceLicenseForOneMemberNoRepoProjects, DropVulnerabilitiesWithoutFindingId, EncryptCiTriggerToken, ExpireOAuthTokens, FixAllowDescendantsOverrideDisabledSharedRunners, FixCorruptedScannerIdsOfVulnerabilityReads, FixNamespaceIdForWorkItemParentLinks, FixPickUpAtCiDeletedObject, FixProjectSettingsHasVulnerabilities, FixSyncedEpicWorkItemParentLinks, MakeSecurityPolicyBotsPrivate, MarkDuplicateIssuableResourceLinks, MarkSingleIssuableResourceLinks, MigrateHumanUserType, MigrateLinksForVulnerabilityFindings, MigrateOsSbomOccurrencesToComponentsWithoutPrefix, MigrateRemainingSoftwareLicenseWithoutSpdxIdentifierToCustomLicenses, MigrateSharedVulnerabilityScanners, MigrateVulnerabilitiesFeedbackToVulnerabilitiesStateTransition, NullifyBuildsAutoCanceledById, NullifyOrganizationIdForSnippets, PopulateDenormalizedColumnsForSbomOccurrences, PopulateDetumbledEmailInEmails, PopulateOperationVisibilityPermissionsFromOperations, PopulateTopicsSlugColumn, PopulateVulnerabilityDismissalFields, ProjectBotUserDetailsBotNamespaceMigration, PurgeSecurityScansWithEmptyFindingData, QueueBackfillAutocancelPartitionIdOnCiPipelines, RecalculateDoraDeploymentFrequencyScore, RecoverDeletedMlModelVersionPackages, RemoveNamespaceFromOsTypeSbomComponents, RemoveProjectGroupLinkWithMissingGroups, ResolveVulnerabilitiesForRemovedAnalyzers, RestoreOptInToGitlabCom, ResyncHasVulnerabilities, SanitizeConfidentialTodos, SetCorrectVulnerabilityState, SetLegacyOpenSourceLicenseAvailableForNonPublicProjects, SetTotalNumberOfVulnerabilitiesForExistingProjects, SyncScanResultPolicies, UpdateDelayedProjectRemovalToNullForUserNamespaces, UpdateJiraTrackerDataDeploymentTypeBasedOnUrl, UpdateOwaspTop10DefaultOfVulnerabilityReads, UpdateSbomComponentsNameBasedOnPep503, UpdateSbomOccurrencesComponentNameBasedOnPep503, UpdateUsersSetExternalIfServiceAccount, UpdateWorkspacesConfigVersion, UpdateWorkspacesConfigVersion3

Constant Summary

DEFAULT_FEATURE_CATEGORY =

:database

Constants included from Database::DynamicModelHelpers

Database::DynamicModelHelpers::BATCH_SIZE

Class Method Summary

• .cursor(*args) ⇒ Object
• .cursor? ⇒ Boolean
• .cursor_columns ⇒ Object
• .feature_category(feature_category_name = nil) ⇒ Object
• .generic_instance(batch_table:, batch_column:, connection:, job_arguments: []) ⇒ Object
• .job_arguments(*args) ⇒ Object
• .job_arguments_count ⇒ Object
• .operation_name(operation) ⇒ Object
• .scope_to(scope) ⇒ Object

Instance Method Summary

• #batch_metrics ⇒ Object
• #filter_batch(relation) ⇒ Object
• #initialize(batch_table:, batch_column:, sub_batch_size:, pause_ms:, connection:, job_arguments: [], start_id: nil, end_id: nil, start_cursor: nil, end_cursor: nil, sub_batch_exception: nil) ⇒ BatchedMigrationJob constructor

  A new instance of BatchedMigrationJob.

• #perform ⇒ Object

Methods included from Database::DynamicModelHelpers

#define_batchable_model, #each_batch, #each_batch_range

Constructor Details

#initialize(batch_table:, batch_column:, sub_batch_size:, pause_ms:, connection:, job_arguments: [], start_id: nil, end_id: nil, start_cursor: nil, end_cursor: nil, sub_batch_exception: nil) ⇒ BatchedMigrationJob

Returns a new instance of BatchedMigrationJob.

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 78

def initialize(
  batch_table:, batch_column:, sub_batch_size:, pause_ms:, connection:, job_arguments: [],
  start_id: nil, end_id: nil, start_cursor: nil, end_cursor: nil, sub_batch_exception: nil
)

  @start_id = start_id
  @end_id = end_id
  @start_cursor = start_cursor
  @end_cursor = end_cursor
  @batch_table = batch_table
  @batch_column = batch_column
  @sub_batch_size = sub_batch_size
  @pause_ms = pause_ms
  @job_arguments = job_arguments
  @connection = connection
  @sub_batch_exception = sub_batch_exception
end

Class Method Details

.cursor(*args) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 71

def cursor(*args)
  define_singleton_method(:cursor_columns) do
    args
  end
end

.cursor? ⇒ Boolean

Returns:

• (Boolean)

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 67

def cursor?
  cursor_columns.count > 1
end

.cursor_columns ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 63

def cursor_columns
  []
end

.feature_category(feature_category_name = nil) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 55

def feature_category(feature_category_name = nil)
  if feature_category_name.present?
    set_class_attribute(:feature_category, feature_category_name)
  else
    get_class_attribute(:feature_category) || DEFAULT_FEATURE_CATEGORY
  end
end

.generic_instance(batch_table:, batch_column:, connection:, job_arguments: []) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 19

def generic_instance(batch_table:, batch_column:, connection:, job_arguments: [])
  new(
    batch_table: batch_table, batch_column: batch_column,
    job_arguments: job_arguments, connection: connection,
    start_id: 0, end_id: 0, sub_batch_size: 0, pause_ms: 0
  )
end

.job_arguments(*args) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 37

def job_arguments(*args)
  args.each.with_index do |arg, index|
    define_method(arg) do
      @job_arguments[index]
    end
  end

  define_singleton_method(:job_arguments_count) do
    args.count
  end
end

.job_arguments_count ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 27

def job_arguments_count
  0
end

.operation_name(operation) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 31

def operation_name(operation)
  define_method(:operation_name) do
    operation
  end
end

.scope_to(scope) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 49

def scope_to(scope)
  define_method(:filter_batch) do |relation|
    instance_exec(relation, &scope)
  end
end

Instance Method Details

#batch_metrics ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 104

def batch_metrics
  @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new
end

#filter_batch(relation) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 96

def filter_batch(relation)
  relation
end

#perform ⇒ Object

Raises:

• (NotImplementedError)

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 100

def perform
  raise NotImplementedError, "subclasses of #{self.class.name} must implement #{__method__}"
end