Class: Gitlab::Database::BackgroundMigration::BatchedJob

Inherits:
SharedModel
  • Object
show all
Includes:
EachBatch, FromUnion
Defined in:
lib/gitlab/database/background_migration/batched_job.rb

Constant Summary collapse

MINIMUM_BATCH_SIZE = 
1
BATCH_SIZE_DIVISOR = 
2
MINIMUM_PAUSE_MS = 
100
MAX_ATTEMPTS = 
3
MAX_SIDEKIQ_SHUTDOWN_FAILURES = 
15
MIN_BATCH_SIZE = 
1
SUB_BATCH_SIZE_REDUCE_FACTOR = 
0.75
SUB_BATCH_SIZE_THRESHOLD = 
65
STUCK_JOBS_TIMEOUT = 
1.hour.freeze
TIMEOUT_EXCEPTIONS = 
[ActiveRecord::StatementTimeout, ActiveRecord::ConnectionTimeoutError,
ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout,
ActiveRecord::QueryCanceled].freeze

Constants inherited from SharedModel

SharedModel::SHARED_SCHEMAS

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from SharedModel

connection, #connection_db_config, connection_pool, ensure_connection_set!, using_connection

Class Method Details

.extract_transition_options(args) ⇒ Object 



117
118
119
120
121
122
123
124
125
126
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 117

def self.extract_transition_options(args)
  error_hash = args.find { |arg| arg[:error].present? }

  return [] unless error_hash

  exception = error_hash.fetch(:error)
  from_sub_batch = error_hash[:from_sub_batch]

  [exception, from_sub_batch]
end

Instance Method Details

#can_reduce_sub_batch_size? ⇒ Boolean

Returns:

  • (Boolean) 


162
163
164
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 162

def can_reduce_sub_batch_size?
  still_retryable? && within_batch_size_boundaries?
end

#can_split?(exception) ⇒ Boolean

Returns:

  • (Boolean) 


156
157
158
159
160
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 156

def can_split?(exception)
  return if still_retryable?

  exception.class.in?(TIMEOUT_EXCEPTIONS) && within_batch_size_boundaries?
end

#handle_sidekiq_shutdown_failure?(exception) ⇒ Boolean

Returns:

  • (Boolean) 


232
233
234
235
236
237
238
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 232

def handle_sidekiq_shutdown_failure?(exception)
  return false unless exception.is_a?(Sidekiq::Shutdown)
  return false unless attempts > 1
  return false unless sidekiq_shutdown_failures_count <= MAX_SIDEKIQ_SHUTDOWN_FAILURES

  true
end

#job_attributes ⇒ Object 



128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 128

def job_attributes
  {
    batch_table: migration_table_name,
    batch_column: migration_column_name,
    sub_batch_size: sub_batch_size,
    pause_ms: pause_ms,
    job_arguments: migration_job_arguments
  }.tap do |attributes|
    if migration_job_class.cursor?
      attributes[:start_cursor] = min_cursor
      attributes[:end_cursor] = max_cursor
    else
      attributes[:start_id] = min_value
      attributes[:end_id] = max_value
    end
  end
end

#reduce_sub_batch_size! ⇒ Object

It reduces the size of sub_batch_size by 25%

Raises:



184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 184

def reduce_sub_batch_size!
  raise ReduceSubBatchSizeError, 'Only sub_batch_size of failed jobs can be reduced' unless failed?

  return if sub_batch_exceeds_threshold?

  with_lock do
    actual_sub_batch_size = sub_batch_size
    reduced_sub_batch_size = (sub_batch_size * SUB_BATCH_SIZE_REDUCE_FACTOR).to_i.clamp(1, batch_size)

    update!(sub_batch_size: reduced_sub_batch_size)

    Gitlab::AppLogger.warn(
      message: 'Sub batch size reduced due to timeout',
      batched_job_id: id,
      sub_batch_size: actual_sub_batch_size,
      reduced_sub_batch_size: reduced_sub_batch_size,
      attempts: attempts,
      batched_migration_id: batched_migration.id,
      job_class_name: migration_job_class_name,
      job_arguments: migration_job_arguments
    )
  end
end

#sidekiq_shutdown_failures_count ⇒ Object 



228
229
230
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 228

def sidekiq_shutdown_failures_count
  sidekiq_shutdown_failures.count
end

#split_and_retry! ⇒ Object 



166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 166

def split_and_retry!
  with_lock do
    validate_split_preconditions!

    new_batch_size = batch_size / BATCH_SIZE_DIVISOR
    next update!(attempts: 0) if new_batch_size < MINIMUM_BATCH_SIZE

    midpoint = calculate_midpoint(new_batch_size)

    if midpoint >= max_value
      update!(batch_size: new_batch_size, attempts: 0)
    else
      split_job_at_midpoint(midpoint, new_batch_size)
    end
  end
end

#still_retryable? ⇒ Boolean

Returns:

  • (Boolean) 


208
209
210
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 208

def still_retryable?
  attempts < MAX_ATTEMPTS
end

#sub_batch_exceeds_threshold? ⇒ Boolean

It doesn’t allow sub-batch size to be reduced lower than the threshold

Returns:

  • (Boolean) 


220
221
222
223
224
225
226
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 220

def sub_batch_exceeds_threshold?
  initial_sub_batch_size = batched_migration.sub_batch_size
  reduced_sub_batch_size = (sub_batch_size * SUB_BATCH_SIZE_REDUCE_FACTOR).to_i
  diff = initial_sub_batch_size - reduced_sub_batch_size

  (1.0 * diff / initial_sub_batch_size * 100).round(2) > SUB_BATCH_SIZE_THRESHOLD
end

#time_efficiency ⇒ Object 



146
147
148
149
150
151
152
153
154
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 146

def time_efficiency
  return unless succeeded?
  return unless finished_at && started_at

  duration = finished_at - started_at

  # TODO: Switch to individual job interval (prereq: https://gitlab.com/gitlab-org/gitlab/-/issues/328801)
  duration.to_f / batched_migration.interval
end

#within_batch_size_boundaries? ⇒ Boolean

Returns:

  • (Boolean) 


212
213
214
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 212

def within_batch_size_boundaries?
  batch_size > MIN_BATCH_SIZE && batch_size > sub_batch_size
end