Class: Gitlab::BackgroundMigration::BatchedMigrationJob

Inherits:

Object

• Object
• Gitlab::BackgroundMigration::BatchedMigrationJob

Includes:

ClassAttributes, Database::DynamicModelHelpers

Defined in:

lib/gitlab/background_migration/batched_migration_job.rb

Overview

Base class for batched background migrations. Subclasses should implement the ‘#perform` method as the entry point for the job’s execution.

Job arguments needed must be defined explicitly, see docs.gitlab.com/ee/development/database/batched_background_migrations.html#use-job-arguments. rubocop:disable Metrics/ClassLength rubocop:disable Metrics/ParameterLists

Direct Known Subclasses

BackfillAmazonGroupAuditEventDestinations, BackfillAmazonGroupAuditEventDestinationsFixed, BackfillAmazonInstanceAuditEventDestinations, BackfillAmazonInstanceAuditEventDestinationsFixed, BackfillAnalyzerProjectStatuses, BackfillAnalyzerProjectStatusesFromProjectSecuritySettings, BackfillApprovalPolicyRuleIds, BackfillArchivedAndTraversalIdsToVulnerabilityStatistics, BackfillAwardEmojiShardingKey, BackfillBulkImportBatchTrackersShardingKey, BackfillBulkImportTrackersShardingKey, BackfillClustersShardingKey, BackfillCommitUserMentionsNamespaceId, BackfillComplianceFrameworkSecurityPolicyId, BackfillComplianceViolationNullTargetProjectIds, BackfillDeploymentMergeRequestsProjectId, BackfillDescriptionVersionsNamespace, BackfillDesignManagementDesignsProjectNamespaceId, BackfillDesignManagementRepositoriesProjectNamespaceId, BackfillDesiredShardingKeyJob, BackfillDiffNotePositionsNamespaceId, BackfillDismissalReasonInVulnerabilityReads, BackfillEmptyProjectsInSecurityInventoryFilters, BackfillEpicIssuesWorkItemParentLinkId, BackfillEpicsWorkItemParentLinkId, BackfillExternalGroupAuditEventDestinations, BackfillExternalGroupAuditEventDestinationsFixed, BackfillExternalInstanceAuditEventDestinations, BackfillExternalInstanceAuditEventDestinationsFixed, BackfillFindingInitialPipelineId, BackfillGoogleGroupAuditEventDestinations, BackfillGoogleGroupAuditEventDestinationsFixed, BackfillGoogleInstanceAuditEventDestinations, BackfillGoogleInstanceAuditEventDestinationsFixed, BackfillGroupIdAndUserTypeForHumanUsersPersonalAccessTokens, BackfillGroupIdAndUserTypeForNonhumanAccessTokens, BackfillGroupPushRulesFromPushRulesWithIds, BackfillHasVulnerabilityResolutionCwe78Cwe89, BackfillImportedIssueSearchData, BackfillIncidentManagementPendingIssueEscalationsNamespaceId, BackfillIssueAssigneesNamespaceId, BackfillIssueTrackerDataShardingKey, BackfillJiraTrackerDataShardingKey, BackfillLicensesOutsideSpdx, BackfillLicensesOutsideSpdxCatalogue, BackfillListsShardingKey, BackfillMergeRequestContextCommitDiffFilesProjectId, BackfillMissingNamespaceDetails, BackfillMissingNamespaceIdOnNotes, BackfillMissingOrganizationIdOnCiRunnerMachines, BackfillNamespaceDetailsDescriptionFields, BackfillNamespaceTraversalIdsOnIssues, BackfillNamespacesRedirectRoutesNamespaceId, BackfillNoteDiffFilesNamespaceId, BackfillNoteMetadataNamespaceId, BackfillOccurrenceIdToVulnerabilityAssociations, BackfillOidOnLfsObjectsProjects, BackfillOnboardingStatusSetupForCompany, BackfillOrganizationIdKeys, BackfillOrganizationIdOnCiRunnerMachines, BackfillOrganizationIdOnCiRunnerTaggings, BackfillOrganizationIdOnCiRunners, BackfillPackagesComposerPackagesTable, BackfillPackagesProtectionRules, BackfillPartitionedTable, BackfillPartitionedUploads, BackfillPipelineExecutionPoliciesMetadata, BackfillPipelineSdAndCsAnalyzerProjectStatuses, BackfillPlaceholderUsersDetailsFromSourceUsers, BackfillProjectFeaturePackageRegistryAccessLevel, BackfillProjectIdForProjectsWithPipelineVariables, BackfillProjectIdOnCiBuildNeeds, BackfillProjectMemberNamespaceId, BackfillProjectNamespaceDetails, BackfillProjectNamespaceOnIssues, BackfillProjectsRedirectRoutesNamespaceId, BackfillProtectedBranchPushAccessLevelsFields, BackfillResourceIterationEventsNamespaceId, BackfillResourceLabelEventsNamespaceIdAndCleanInvalid, BackfillResourceMilestoneEventsNamespace, BackfillResourceStateEventsNamespaceId, BackfillRolledUpWeightForWorkItems, BackfillSeatAssignmentsTable, BackfillSecurityInventoryFilters, BackfillSentNotificationsAfterPartition, BackfillShardingKeyAndCleanLabelLinksTable, BackfillSlackIntegrationsShardingKey, BackfillSoftwareLicensePolicies, BackfillSolutionToVulnerabilities, BackfillSubscriptionUserAddOnAssignmentVersions, BackfillSuggestionsNamespaceId, BackfillSystemNoteMetadataNamespaceId, BackfillTagNameOnCiRunnerTaggings, BackfillTerraformModulesMetadataWithSemver, BackfillTimelogsNamespace, BackfillTodosWithMultipleColumnShardingKey, BackfillUserDetailOnboardingUrl, BackfillUserDetailsCompany, BackfillUserGroupMemberRoles, BackfillUserGroupMemberRolesForGroupLinks, BackfillVulnerabilityFindingRiskScores, BackfillVulnerabilityNamespaceHistoricalStatistics, BackfillWorkItemTransitions, BackfillWorkspaceAgentkStates, BackfillX509TablesProjectId, CleanSubscriptionsUserIdColumn, CleanupOrphanedRoutes, CleanupTerminatedBulkImportConfigs, ClearResolvedAtForNonResolvedVulnerabilities, CopyColumnUsingBackgroundMigrationJob, CreateMissingExternalLinksForVulnerabilities, CreateMissingNugetSymbolFiles, CreateMissingTerraformModuleMetadata, CreateVulnerabilityLinks, DeleteExpiredVulnerabilityExports, DeleteOrphanRedirectRoutesRows, DeleteOrphanedCiRunnerMachineRecordsOnSelfManaged, DeleteOrphanedPartitionedCiRunnerMachineRecords, DeleteOrphanedProjectNamespaces, DeleteOrphanedRelationExportUploads, DeleteOrphanedRoutes, DeleteOrphanedSecurityPolicyBotUsers, DeleteTwitterIdentities, DestroyInvalidGroupMembers, DestroyInvalidProjectMembers, DisableLegacyOpenSourceLicenseForInactivePublicProjects, DisableLegacyOpenSourceLicenseForNoIssuesNoRepoProjects, EncryptMissedCiRunnerTokens, ExpireOAuthTokens, FixIncompleteExternalAuditDestinations, FixIncompleteGroupExternalAuditDestinationsV2, FixIncompleteInstanceExternalAuditDestinations, FixIncompleteInstanceExternalAuditDestinationsV2, FixNonExistingTimelogUsers, FixOutOfRangeEpicDates, FixOutOfRangeWorkItemDates, FixPSentNotificationsRecordsRelatedToDesignManagement, FixSecretTokensForHttpDestinations, FixStringConfigHashesGroupStreamingDestinations, FixStringConfigHashesInstanceStreamingDestinations, FixUsernamespaceAuditEvents, FixVulnerabilitiesTransitionedFromDismissedToResolved, LimitNamespaceVisibilityByOrganizationVisibility, MarkPackagesHelmMetadataCachesPendingDestruction, MigrateAiCodeSuggestionEvents, MigrateCiTriggersTokenToTokenEncrypted, MigrateDuplicatePushRulesToProjectPushRules, MigrateEpicLabelLinksToWorkItems, MigrateEvidencesForVulnerabilityFindings, MigrateNonDuplicatePushRulesToProjectPushRules, MigrateRemediationsForVulnerabilityFindings, MigrateScimIdentities, MigrateScimTokens, MigrateSharedVulnerabilityScanners, MigrateVulnerabilitiesFeedbackToVulnerabilitiesStateTransition, MoveCiBuildsMetadata, RecalculateTwoFactorEnforcementOnUsers, ReindexProjectElasticZoektData, RemoveExperimentsFromUserDetailsOnboardingStatus, RemoveInvalidOrganizationUsers, RemoveOldJobTokens, RemoveOrphanedVulnerabilityNotesBatchedMigration, RemoveProjectGroupLinkWithMissingGroups, RemoveRowsWithDeletedUserFromIdentities, ResetAutoDuoCodeReviewFalseValues, ResetDuoRemoteFlowsEnabledFalseValues, ResetNamespaceSettingsDuoFoundationalFlowsEnabled, ResetProjectSettingsDuoFoundationalFlowsEnabled, ResyncApprovalPolicies, ResyncEpicDatesToWorkItemsDatesSources, SanitizeConfidentialTodos, SyncScanResultPolicies, SyncUnlinkedSecurityPolicyProjectLinks, UpdateBadgesRowsWithMulticolumnShardingKeyColumns, UpdateClosedMergedMrs, UpdateDelayedProjectRemovalToNullForUserNamespaces, UpdateJiraTrackerDataDeploymentTypeBasedOnUrl, UpdateRequireDpopForManageApiEndpointsToFalse, UpdateSoftwareLicensePoliciesWithCustomLicenses, UpdateStatusForDeprecatedNpmPackages, UpdateUsersSetExternalIfServiceAccount

Constant Summary

DEFAULT_FEATURE_CATEGORY =

:database

MINIMUM_PAUSE_MS =

100

Constants included from Database::DynamicModelHelpers

Database::DynamicModelHelpers::BATCH_SIZE

Class Method Summary

• .cursor(*args) ⇒ Object
• .cursor? ⇒ Boolean
• .cursor_columns ⇒ Object
• .feature_category(feature_category_name = nil) ⇒ Object
• .generic_instance(batch_table:, batch_column:, connection:, job_arguments: []) ⇒ Object
• .health_context_tables ⇒ Object
• .job_arguments(*args) ⇒ Object
• .job_arguments_count ⇒ Object
• .operation_name(operation) ⇒ Object
• .scope_to(scope) ⇒ Object
• .tables_to_check_for_vacuum(*args) ⇒ Object

Instance Method Summary

• #batch_metrics ⇒ Object
• #filter_batch(relation) ⇒ Object
• #initialize(batch_table:, batch_column:, sub_batch_size:, pause_ms:, connection:, job_arguments: [], start_id: nil, end_id: nil, start_cursor: nil, end_cursor: nil, sub_batch_exception: nil) ⇒ BatchedMigrationJob constructor

  A new instance of BatchedMigrationJob.

• #perform ⇒ Object

Methods included from Database::DynamicModelHelpers

define_batchable_model, #each_batch, #each_batch_range

Constructor Details

#initialize(batch_table:, batch_column:, sub_batch_size:, pause_ms:, connection:, job_arguments: [], start_id: nil, end_id: nil, start_cursor: nil, end_cursor: nil, sub_batch_exception: nil) ⇒ BatchedMigrationJob

Returns a new instance of BatchedMigrationJob.

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 89

def initialize(
  batch_table:, batch_column:, sub_batch_size:, pause_ms:, connection:, job_arguments: [],
  start_id: nil, end_id: nil, start_cursor: nil, end_cursor: nil, sub_batch_exception: nil
)
  @start_id = start_id
  @end_id = end_id
  @start_cursor = start_cursor
  @end_cursor = end_cursor
  @batch_table = batch_table
  @batch_column = batch_column
  @sub_batch_size = sub_batch_size
  @pause_ms = [pause_ms, self.class::MINIMUM_PAUSE_MS].max
  @job_arguments = job_arguments
  @connection = connection
  @sub_batch_exception = sub_batch_exception
end

Class Method Details

.cursor(*args) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 72

def cursor(*args)
  define_singleton_method(:cursor_columns) do
    args
  end
end

.cursor? ⇒ Boolean

Returns:

• (Boolean)

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 68

def cursor?
  cursor_columns.count > 1
end

.cursor_columns ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 64

def cursor_columns
  []
end

.feature_category(feature_category_name = nil) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 56

def feature_category(feature_category_name = nil)
  if feature_category_name.present?
    set_class_attribute(:feature_category, feature_category_name)
  else
    get_class_attribute(:feature_category) || DEFAULT_FEATURE_CATEGORY
  end
end

.generic_instance(batch_table:, batch_column:, connection:, job_arguments: []) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 20

def generic_instance(batch_table:, batch_column:, connection:, job_arguments: [])
  new(
    batch_table: batch_table, batch_column: batch_column,
    job_arguments: job_arguments, connection: connection,
    start_id: 0, end_id: 0, sub_batch_size: 0, pause_ms: MINIMUM_PAUSE_MS
  )
end

.health_context_tables ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 78

def health_context_tables
  []
end

.job_arguments(*args) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 38

def job_arguments(*args)
  args.each.with_index do |arg, index|
    define_method(arg) do
      @job_arguments[index]
    end
  end

  define_singleton_method(:job_arguments_count) do
    args.count
  end
end

.job_arguments_count ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 28

def job_arguments_count
  0
end

.operation_name(operation) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 32

def operation_name(operation)
  define_method(:operation_name) do
    operation
  end
end

.scope_to(scope) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 50

def scope_to(scope)
  define_method(:filter_batch) do |relation|
    instance_exec(relation, &scope)
  end
end

.tables_to_check_for_vacuum(*args) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 82

def tables_to_check_for_vacuum(*args)
  define_singleton_method(:health_context_tables) do
    args.map(&:to_s)
  end
end

Instance Method Details

#batch_metrics ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 114

def batch_metrics
  @batch_metrics ||= Gitlab::Database::Batch::Metrics.new
end

#filter_batch(relation) ⇒ Object

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 106

def filter_batch(relation)
  relation
end

#perform ⇒ Object

Raises:

• (NotImplementedError)

# File 'lib/gitlab/background_migration/batched_migration_job.rb', line 110

def perform
  raise NotImplementedError, "subclasses of #{self.class.name} must implement #{__method__}"
end