Class: BulkImports::PipelineWorker

Inherits:
Object
  • Object
show all
Includes:
ApplicationWorker, ExclusiveLeaseGuard, Gitlab::Utils::StrongMemoize, Sidekiq::InterruptionsExhausted
Defined in:
app/workers/bulk_imports/pipeline_worker.rb

Constant Summary collapse

FILE_EXTRACTION_PIPELINE_PERFORM_DELAY = 
10.seconds
LimitedBatches = 
Struct.new(:numbers, :final?, keyword_init: true).freeze
DEFER_ON_HEALTH_DELAY = 
5.minutes

Constants included from ApplicationWorker

ApplicationWorker::LOGGING_EXTRA_KEY, ApplicationWorker::SAFE_PUSH_BULK_LIMIT

Constants included from Gitlab::Loggable

Gitlab::Loggable::ANONYMOUS

Constants included from WorkerAttributes

WorkerAttributes::DEFAULT_DATA_CONSISTENCY, WorkerAttributes::DEFAULT_DATA_CONSISTENCY_PER_DB, WorkerAttributes::DEFAULT_DEFER_DELAY, WorkerAttributes::LOAD_BALANCED_DATA_CONSISTENCIES, WorkerAttributes::NAMESPACE_WEIGHTS, WorkerAttributes::VALID_DATA_CONSISTENCIES, WorkerAttributes::VALID_RESOURCE_BOUNDARIES, WorkerAttributes::VALID_URGENCIES

Class Method Summary collapse

Instance Method Summary collapse

Methods included from ExclusiveLeaseGuard

#exclusive_lease, #lease_release?, #lease_taken_log_level, #lease_taken_message, #log_lease_taken, #release_lease, #renew_lease!, #try_obtain_lease

Methods included from Gitlab::Loggable

#build_structured_payload

Methods included from Gitlab::SidekiqVersioning::Worker

#job_version

Methods included from WorkerContext

#with_context

Class Method Details

.defer_on_database_health_signal? ⇒ Boolean

Returns:

  • (Boolean) 


50
51
52
 
# File 'app/workers/bulk_imports/pipeline_worker.rb', line 50

def self.defer_on_database_health_signal?
  Feature.enabled?(:bulk_import_deferred_workers)
end

Instance Method Details

#perform(pipeline_tracker_id, _stage, entity_id) ⇒ Object

Keep _stage parameter for backwards compatibility.



55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'app/workers/bulk_imports/pipeline_worker.rb', line 55

def perform(pipeline_tracker_id, _stage, entity_id)
  @entity = ::BulkImports::Entity.find(entity_id)
  @pipeline_tracker = ::BulkImports::Tracker.find(pipeline_tracker_id)

  log_extra_metadata_on_done(:pipeline_class, @pipeline_tracker.pipeline_name)

  try_obtain_lease do
    if pipeline_tracker.enqueued? || pipeline_tracker.started?
      logger.info(log_attributes(message: 'Pipeline starting'))
      run
    end
  end
end

#perform_failure(pipeline_tracker_id, entity_id, exception) ⇒ Object 



69
70
71
72
73
74
 
# File 'app/workers/bulk_imports/pipeline_worker.rb', line 69

def perform_failure(pipeline_tracker_id, entity_id, exception)
  @entity = ::BulkImports::Entity.find(entity_id)
  @pipeline_tracker = ::BulkImports::Tracker.find(pipeline_tracker_id)

  fail_pipeline(exception)
end