Class: Gitlab::Database::BackgroundMigration::BatchedJob

Inherits:
SharedModel
  • Object
show all
Includes:
EachBatch, FromUnion
Defined in:
lib/gitlab/database/background_migration/batched_job.rb

Constant Summary collapse

MAX_ATTEMPTS = 
3
MIN_BATCH_SIZE = 
1
SUB_BATCH_SIZE_REDUCE_FACTOR = 
0.75
SUB_BATCH_SIZE_THRESHOLD = 
65
STUCK_JOBS_TIMEOUT = 
1.hour.freeze
TIMEOUT_EXCEPTIONS = 
[ActiveRecord::StatementTimeout, ActiveRecord::ConnectionTimeoutError,
ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout,
ActiveRecord::QueryCanceled].freeze

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from SharedModel

connection, #connection_db_config, connection_pool, using_connection

Class Method Details

.extract_transition_options(args) ⇒ Object 



105
106
107
108
109
110
111
112
113
114
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 105

def self.extract_transition_options(args)
  error_hash = args.find { |arg| arg[:error].present? }

  return [] unless error_hash

  exception = error_hash.fetch(:error)
  from_sub_batch = error_hash[:from_sub_batch]

  [exception, from_sub_batch]
end

Instance Method Details

#can_reduce_sub_batch_size? ⇒ Boolean

Returns:

  • (Boolean) 


132
133
134
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 132

def can_reduce_sub_batch_size?
  still_retryable? && within_batch_size_boundaries?
end

#can_split?(exception) ⇒ Boolean

Returns:

  • (Boolean) 


126
127
128
129
130
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 126

def can_split?(exception)
  return if still_retryable?

  exception.class.in?(TIMEOUT_EXCEPTIONS) && within_batch_size_boundaries?
end

#reduce_sub_batch_size! ⇒ Object

It reduces the size of sub_batch_size by 25%

Raises:



184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 184

def reduce_sub_batch_size!
  raise ReduceSubBatchSizeError, 'Only sub_batch_size of failed jobs can be reduced' unless failed?

  return if sub_batch_exceeds_threshold?

  with_lock do
    actual_sub_batch_size = sub_batch_size
    reduced_sub_batch_size = (sub_batch_size * SUB_BATCH_SIZE_REDUCE_FACTOR).to_i.clamp(1, batch_size)

    update!(sub_batch_size: reduced_sub_batch_size)

    Gitlab::AppLogger.warn(
      message: 'Sub batch size reduced due to timeout',
      batched_job_id: id,
      sub_batch_size: actual_sub_batch_size,
      reduced_sub_batch_size: reduced_sub_batch_size,
      attempts: attempts,
      batched_migration_id: batched_migration.id,
      job_class_name: migration_job_class_name,
      job_arguments: migration_job_arguments
    )
  end
end

#split_and_retry! ⇒ Object 



136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 136

def split_and_retry!
  with_lock do
    raise SplitAndRetryError, 'Only failed jobs can be split' unless failed?

    new_batch_size = batch_size / 2

    next update!(attempts: 0) if new_batch_size < 1

    batching_strategy = batched_migration.batch_class.new(connection: self.class.connection)
    next_batch_bounds = batching_strategy.next_batch(
      batched_migration.table_name,
      batched_migration.column_name,
      batch_min_value: min_value,
      batch_size: new_batch_size,
      job_arguments: batched_migration.job_arguments,
      job_class: batched_migration.job_class
    )
    midpoint = next_batch_bounds.last

    # We don't want the midpoint to go over the existing max_value because
    # those IDs would already be in the next batched migration job.
    # This could happen when a lot of records in the current batch are deleted.
    #
    # In this case, we just lower the batch size so that future calls to this
    # method could eventually split the job if it continues to fail.
    if midpoint >= max_value
      update!(batch_size: new_batch_size, attempts: 0)
    else
      old_max_value = max_value

      update!(
        batch_size: new_batch_size,
        max_value: midpoint,
        attempts: 0,
        started_at: nil,
        finished_at: nil,
        metrics: {}
      )

      new_record = dup
      new_record.min_value = midpoint.next
      new_record.max_value = old_max_value
      new_record.save!
    end
  end
end

#still_retryable? ⇒ Boolean

Returns:

  • (Boolean) 


208
209
210
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 208

def still_retryable?
  attempts < MAX_ATTEMPTS
end

#sub_batch_exceeds_threshold? ⇒ Boolean

It doesn’t allow sub-batch size to be reduced lower than the threshold

Returns:

  • (Boolean) 


220
221
222
223
224
225
226
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 220

def sub_batch_exceeds_threshold?
  initial_sub_batch_size = batched_migration.sub_batch_size
  reduced_sub_batch_size = (sub_batch_size * SUB_BATCH_SIZE_REDUCE_FACTOR).to_i
  diff = initial_sub_batch_size - reduced_sub_batch_size

  (1.0 * diff / initial_sub_batch_size * 100).round(2) > SUB_BATCH_SIZE_THRESHOLD
end

#time_efficiency ⇒ Object 



116
117
118
119
120
121
122
123
124
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 116

def time_efficiency
  return unless succeeded?
  return unless finished_at && started_at

  duration = finished_at - started_at

  # TODO: Switch to individual job interval (prereq: https://gitlab.com/gitlab-org/gitlab/-/issues/328801)
  duration.to_f / batched_migration.interval
end

#within_batch_size_boundaries? ⇒ Boolean

Returns:

  • (Boolean) 


212
213
214
 
# File 'lib/gitlab/database/background_migration/batched_job.rb', line 212

def within_batch_size_boundaries?
  batch_size > MIN_BATCH_SIZE && batch_size > sub_batch_size
end