Class: Gitlab::Database::BackgroundMigration::BatchedMigration

Inherits:
SharedModel
  • Object
show all
Defined in:
lib/gitlab/database/background_migration/batched_migration.rb

Constant Summary collapse

JOB_CLASS_MODULE =
'Gitlab::BackgroundMigration'
BATCH_CLASS_MODULE =
"#{JOB_CLASS_MODULE}::BatchingStrategies"
MAXIMUM_FAILED_RATIO =
0.5
MINIMUM_JOBS =
50

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from SharedModel

connection, #connection_db_config, using_connection

Class Method Details

.active_migration(connection:) ⇒ Object


94
95
96
97
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 94

def self.active_migration(connection:)
  for_gitlab_schema(Gitlab::Database.gitlab_schemas_for_connection(connection))
    .executable.queue_order.first
end

.find_for_configuration(gitlab_schema, job_class_name, table_name, column_name, job_arguments) ⇒ Object


90
91
92
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 90

def self.find_for_configuration(gitlab_schema, job_class_name, table_name, column_name, job_arguments)
  for_configuration(gitlab_schema, job_class_name, table_name, column_name, job_arguments).first
end

.gitlab_schema_column_exists?Boolean

Returns:

  • (Boolean)

44
45
46
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 44

def self.gitlab_schema_column_exists?
  column_names.include?('gitlab_schema')
end

.successful_rows_counts(migrations) ⇒ Object


99
100
101
102
103
104
105
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 99

def self.successful_rows_counts(migrations)
  BatchedJob
    .with_status(:succeeded)
    .where(batched_background_migration_id: migrations)
    .group(:batched_background_migration_id)
    .sum(:batch_size)
end

.valid_statusObject


86
87
88
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 86

def self.valid_status
  state_machine.states.map(&:name)
end

Instance Method Details

#batch_classObject


161
162
163
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 161

def batch_class
  "#{BATCH_CLASS_MODULE}::#{batch_class_name}".constantize
end

#batch_class_name=(class_name) ⇒ Object


169
170
171
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 169

def batch_class_name=(class_name)
  write_attribute(:batch_class_name, class_name.delete_prefix("::"))
end

#create_batched_job!(min, max) ⇒ Object


120
121
122
123
124
125
126
127
128
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 120

def create_batched_job!(min, max)
  batched_jobs.create!(
    min_value: min,
    max_value: max,
    batch_size: batch_size,
    sub_batch_size: sub_batch_size,
    pause_ms: pause_ms
  )
end

#health_contextObject


208
209
210
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 208

def health_context
  HealthStatus::Context.new([table_name])
end

#hold!(until_time: 10.minutes.from_now) ⇒ Object


212
213
214
215
216
217
218
219
220
221
222
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 212

def hold!(until_time: 10.minutes.from_now)
  duration_s = (until_time - Time.current).round
  Gitlab::AppLogger.info(
    message: "#{self} put on hold until #{until_time}",
    migration_id: id,
    job_class_name: job_class_name,
    duration_s: duration_s
  )

  update!(on_hold_until: until_time)
end

#interval_elapsed?(variance: 0) ⇒ Boolean

Returns:

  • (Boolean)

113
114
115
116
117
118
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 113

def interval_elapsed?(variance: 0)
  return true unless last_job

  interval_with_variance = interval - variance
  last_job.created_at <= Time.current - interval_with_variance
end

#job_classObject


157
158
159
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 157

def job_class
  "#{JOB_CLASS_MODULE}::#{job_class_name}".constantize
end

#job_class_name=(class_name) ⇒ Object


165
166
167
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 165

def job_class_name=(class_name)
  write_attribute(:job_class_name, class_name.delete_prefix("::"))
end

#migrated_tuple_countObject


173
174
175
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 173

def migrated_tuple_count
  batched_jobs.with_status(:succeeded).sum(:batch_size)
end

#next_min_valueObject


153
154
155
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 153

def next_min_value
  last_job&.max_value&.next || min_value
end

#on_hold?Boolean

Returns:

  • (Boolean)

224
225
226
227
228
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 224

def on_hold?
  return false unless on_hold_until

  on_hold_until > Time.zone.now
end

#optimize!Object


204
205
206
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 204

def optimize!
  BatchOptimizer.new(self).optimize!
end

#prometheus_labelsObject


177
178
179
180
181
182
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 177

def prometheus_labels
  @prometheus_labels ||= {
    migration_id: id,
    migration_identifier: "%s/%s.%s" % [job_class_name, table_name, column_name]
  }
end

#reset_attempts_of_blocked_jobs!Object


107
108
109
110
111
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 107

def reset_attempts_of_blocked_jobs!
  batched_jobs.blocked_by_max_attempts.each_batch(of: 100) do |batch|
    batch.update_all(attempts: 0)
  end
end

#retry_failed_jobs!Object


130
131
132
133
134
135
136
137
138
139
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 130

def retry_failed_jobs!
  batched_jobs.with_status(:failed).each_batch(of: 100) do |batch|
    self.class.transaction do
      batch.lock.each(&:split_and_retry!)
      self.execute!
    end
  end

  self.execute!
end

#should_stop?Boolean

Returns:

  • (Boolean)

141
142
143
144
145
146
147
148
149
150
151
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 141

def should_stop?
  return unless started_at

  total_jobs = batched_jobs.created_since(started_at).count

  return if total_jobs < MINIMUM_JOBS

  failed_jobs = batched_jobs.with_status(:failed).created_since(started_at).count

  failed_jobs.fdiv(total_jobs) > MAXIMUM_FAILED_RATIO
end

#smoothed_time_efficiency(number_of_jobs: 10, alpha: 0.2) ⇒ Object


184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 184

def smoothed_time_efficiency(number_of_jobs: 10, alpha: 0.2)
  jobs = batched_jobs.successful_in_execution_order.reverse_order.limit(number_of_jobs).with_preloads

  return if jobs.size < number_of_jobs

  efficiencies = jobs.map(&:time_efficiency).reject(&:nil?).each_with_index

  dividend = efficiencies.reduce(0) do |total, (job_eff, i)|
    total + job_eff * (1 - alpha)**i
  end

  divisor = efficiencies.reduce(0) do |total, (job_eff, i)|
    total + (1 - alpha)**i
  end

  return if divisor == 0

  (dividend / divisor).round(2)
end

#to_sObject


230
231
232
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 230

def to_s
  "BatchedMigration[id: #{id}]"
end