Class: ContainerRepository

Inherits:
ApplicationRecord show all
Includes:
AfterCommitQueue, EachBatch, Gitlab::SQL::Pattern, Gitlab::Utils::StrongMemoize, Sortable
Defined in:
app/models/container_repository.rb

Constant Summary collapse

WAITING_CLEANUP_STATUSES = 
%i[cleanup_scheduled cleanup_unfinished].freeze
REQUIRING_CLEANUP_STATUSES = 
%i[cleanup_unscheduled cleanup_scheduled].freeze
IDLE_MIGRATION_STATES = 
%w[default pre_import_done import_done import_aborted import_skipped].freeze
ACTIVE_MIGRATION_STATES = 
%w[pre_importing importing].freeze
MIGRATION_STATES = 
(IDLE_MIGRATION_STATES + ACTIVE_MIGRATION_STATES).freeze
ABORTABLE_MIGRATION_STATES = 
(ACTIVE_MIGRATION_STATES + %w[pre_import_done default]).freeze
SKIPPABLE_MIGRATION_STATES = 
(ABORTABLE_MIGRATION_STATES + %w[import_aborted]).freeze
MIGRATION_PHASE_1_STARTED_AT = 
Date.new(2021, 11, 4).freeze
MIGRATION_PHASE_1_ENDED_AT = 
Date.new(2022, 01, 23).freeze
TooManyImportsError = 
Class.new(StandardError)

Constants included from Gitlab::SQL::Pattern

Gitlab::SQL::Pattern::MIN_CHARS_FOR_PARTIAL_MATCHING, Gitlab::SQL::Pattern::REGEX_QUOTED_WORD

Constants inherited from ApplicationRecord

ApplicationRecord::MAX_PLUCK

Class Method Summary collapse

Instance Method Summary collapse

Methods included from AfterCommitQueue

#run_after_commit, #run_after_commit_or_now

Methods included from Gitlab::Utils::StrongMemoize

#clear_memoization, #strong_memoize, #strong_memoized?

Methods inherited from ApplicationRecord

cached_column_list, #create_or_load_association, declarative_enum, default_select_columns, id_in, id_not_in, iid_in, pluck_primary_key, primary_key_in, #readable_by?, safe_ensure_unique, safe_find_or_create_by, safe_find_or_create_by!, #to_ability_name, underscore, where_exists, where_not_exists, with_fast_read_statement_timeout, without_order

Methods included from SensitiveSerializableHash

#serializable_hash

Class Method Details

.all_migrated? ⇒ Boolean

Returns:

  • (Boolean) 

232
233
234
235
236
237
 
# File 'app/models/container_repository.rb', line 232

def self.all_migrated?
  # check that the set of non migrated repositories is empty
  where(created_at: ...MIGRATION_PHASE_1_ENDED_AT)
    .where.not(migration_state: 'import_done')
    .empty?
end

.build_from_path(path) ⇒ Object 


527
528
529
530
 
# File 'app/models/container_repository.rb', line 527

def self.build_from_path(path)
  self.new(project: path.repository_project,
           name: path.repository_name)
end

.build_root_repository(project) ⇒ Object 


542
543
544
 
# File 'app/models/container_repository.rb', line 542

def self.build_root_repository(project)
  self.new(project: project, name: '')
end

.exists_by_path?(path) ⇒ Boolean

Returns:

  • (Boolean) 

225
226
227
228
229
230
 
# File 'app/models/container_repository.rb', line 225

def self.exists_by_path?(path)
  where(
    project: path.repository_project,
    name: path.repository_name
  ).exists?
end

.find_by_path(path) ⇒ Object 


551
552
553
554
 
# File 'app/models/container_repository.rb', line 551

def self.find_by_path(path)
  self.find_by(project: path.repository_project,
                name: path.repository_name)
end

.find_by_path!(path) ⇒ Object 


546
547
548
549
 
# File 'app/models/container_repository.rb', line 546

def self.find_by_path!(path)
  self.find_by!(project: path.repository_project,
                name: path.repository_name)
end

.find_or_create_from_path(path) ⇒ Object 


532
533
534
535
536
537
538
539
540
 
# File 'app/models/container_repository.rb', line 532

def self.find_or_create_from_path(path)
  repository = safe_find_or_create_by(
    project: path.repository_project,
    name: path.repository_name
  )
  return repository if repository.persisted?

  find_by_path!(path)
end

.requiring_cleanup ⇒ Object 


244
245
246
247
248
249
 
# File 'app/models/container_repository.rb', line 244

def self.requiring_cleanup
  with_enabled_policy
    .where(container_repositories: { expiration_policy_cleanup_status: REQUIRING_CLEANUP_STATUSES })
    .where('container_repositories.expiration_policy_started_at IS NULL OR container_repositories.expiration_policy_started_at < container_expiration_policies.next_run_at')
    .where('container_expiration_policies.next_run_at < ?', Time.zone.now)
end

.with_enabled_policy ⇒ Object 


239
240
241
242
 
# File 'app/models/container_repository.rb', line 239

def self.with_enabled_policy
  joins('INNER JOIN container_expiration_policies ON container_repositories.project_id = container_expiration_policies.project_id')
    .where(container_expiration_policies: { enabled: true })
end

.with_stale_migration(before_timestamp) ⇒ Object 


255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
 
# File 'app/models/container_repository.rb', line 255

def self.with_stale_migration(before_timestamp)
  stale_pre_importing = with_migration_states(:pre_importing)
                          .with_migration_pre_import_started_at_nil_or_before(before_timestamp)
  stale_pre_import_done = with_migration_states(:pre_import_done)
                            .with_migration_pre_import_done_at_nil_or_before(before_timestamp)
  stale_importing = with_migration_states(:importing)
                      .with_migration_import_started_at_nil_or_before(before_timestamp)

  union = ::Gitlab::SQL::Union.new([
        stale_pre_importing,
        stale_pre_import_done,
        stale_importing
      ])
  from("(#{union.to_sql}) #{ContainerRepository.table_name}")
end

.with_target_import_tier ⇒ Object 


271
272
273
274
275
276
277
 
# File 'app/models/container_repository.rb', line 271

def self.with_target_import_tier
  # overridden in ee
  #
  # Repositories are being migrated by tier on Saas, so we need to
  # filter by plan/subscription which is not available in FOSS
  all
end

.with_unfinished_cleanup ⇒ Object 


251
252
253
 
# File 'app/models/container_repository.rb', line 251

def self.with_unfinished_cleanup
  with_enabled_policy.cleanup_unfinished
end

Instance Method Details

#blob(config) ⇒ Object 


436
437
438
 
# File 'app/models/container_repository.rb', line 436

def blob(config)
  ContainerRegistry::Blob.new(self, config)
end

#delete_tag_by_digest(digest) ⇒ Object 


456
457
458
 
# File 'app/models/container_repository.rb', line 456

def delete_tag_by_digest(digest)
  client.delete_repository_tag_by_digest(self.path, digest)
end

#delete_tag_by_name(name) ⇒ Object 


460
461
462
 
# File 'app/models/container_repository.rb', line 460

def delete_tag_by_name(name)
  client.delete_repository_tag_by_name(self.path, name)
end

#delete_tags! ⇒ Object 


448
449
450
451
452
453
454
 
# File 'app/models/container_repository.rb', line 448

def delete_tags!
  return unless has_tags?

  digests = tags.map { |tag| tag.digest }.compact.to_set

  digests.map(&method(:delete_tag_by_digest)).all?
end

#external_import_status ⇒ Object 


384
385
386
387
388
 
# File 'app/models/container_repository.rb', line 384

def external_import_status
  strong_memoize(:import_status) do
    gitlab_api_client.import_status(self.path)
  end
end

#finish_pre_import_and_start_import ⇒ Object 


303
304
305
306
 
# File 'app/models/container_repository.rb', line 303

def finish_pre_import_and_start_import
  # nothing to do between those two transitions for now.
  finish_pre_import && start_import
end

#force_migration_cancel ⇒ Object

This method is not meant for consumption by the code It is meant for manual use in the case that a migration needs to be cancelled by an admin or SRE


517
518
519
520
521
522
523
524
525
 
# File 'app/models/container_repository.rb', line 517

def force_migration_cancel
  return :error unless gitlab_api_client.supports_gitlab_api?

  response = gitlab_api_client.cancel_repository_import(self.path, force: true)

  skip_import(reason: :migration_forced_canceled) if response[:status] == :ok

  response
end

#has_tags? ⇒ Boolean

Returns:

  • (Boolean) 

440
441
442
 
# File 'app/models/container_repository.rb', line 440

def has_tags?
  tags.any?
end

#last_import_step_done_at ⇒ Object 


380
381
382
 
# File 'app/models/container_repository.rb', line 380

def last_import_step_done_at
  [migration_pre_import_done_at, migration_import_done_at, migration_aborted_at, migration_skipped_at].compact.max
end

#location ⇒ Object 


408
409
410
 
# File 'app/models/container_repository.rb', line 408

def location
  File.join(registry.path, path)
end

#manifest ⇒ Object 


416
417
418
 
# File 'app/models/container_repository.rb', line 416

def manifest
  @manifest ||= client.repository_tags(path)
end

#migration_cancel ⇒ Object 


508
509
510
511
512
 
# File 'app/models/container_repository.rb', line 508

def migration_cancel
  return :error unless gitlab_api_client.supports_gitlab_api?

  gitlab_api_client.cancel_repository_import(self.path)
end

#migration_import ⇒ Object

Raises:


499
500
501
502
503
504
505
506
 
# File 'app/models/container_repository.rb', line 499

def migration_import
  return :error unless gitlab_api_client.supports_gitlab_api?

  response = gitlab_api_client.import_repository(self.path)
  raise TooManyImportsError if response == :too_many_imports

  response
end

#migration_importing? ⇒ Boolean

Returns:

  • (Boolean) 

482
483
484
 
# File 'app/models/container_repository.rb', line 482

def migration_importing?
  migration_state == 'importing'
end

#migration_in_active_state? ⇒ Boolean

Returns:

  • (Boolean) 

478
479
480
 
# File 'app/models/container_repository.rb', line 478

def migration_in_active_state?
  migration_state.in?(ACTIVE_MIGRATION_STATES)
end

#migration_pre_import ⇒ Object

Raises:


490
491
492
493
494
495
496
497
 
# File 'app/models/container_repository.rb', line 490

def migration_pre_import
  return :error unless gitlab_api_client.supports_gitlab_api?

  response = gitlab_api_client.pre_import_repository(self.path)
  raise TooManyImportsError if response == :too_many_imports

  response
end

#migration_pre_importing? ⇒ Boolean

Returns:

  • (Boolean) 

486
487
488
 
# File 'app/models/container_repository.rb', line 486

def migration_pre_importing?
  migration_state == 'pre_importing'
end

#nearing_or_exceeded_retry_limit? ⇒ Boolean

Returns:

  • (Boolean) 

376
377
378
 
# File 'app/models/container_repository.rb', line 376

def nearing_or_exceeded_retry_limit?
  migration_retries_count >= ContainerRegistry::Migration.max_retries - 1
end

#path ⇒ Object

rubocop: enable CodeReuse/ServiceClass


403
404
405
406
 
# File 'app/models/container_repository.rb', line 403

def path
  @path ||= [project.full_path, name]
    .select(&:present?).join('/').downcase
end

#reconcile_import_status(status) ⇒ Object 


317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
 
# File 'app/models/container_repository.rb', line 317

def reconcile_import_status(status)
  case status
  when 'native'
    finish_import_as(:native_import)
  when 'pre_import_in_progress'
    return if pre_importing?

    start_pre_import(forced: true)
  when 'import_in_progress'
    return if importing?

    start_import(forced: true)
  when 'import_complete'
    finish_import
  when 'import_failed', 'import_canceled'
    retry_import
  when 'pre_import_complete'
    finish_pre_import_and_start_import
  when 'pre_import_failed', 'pre_import_canceled'
    retry_pre_import
  else
    yield
  end
end

#registry ⇒ Object

rubocop: disable CodeReuse/ServiceClass


391
392
393
394
395
396
397
398
399
400
 
# File 'app/models/container_repository.rb', line 391

def registry
  @registry ||= begin
    token = Auth::ContainerRegistryAuthenticationService.full_access_token(path)

    url = Gitlab.config.registry.api_url
    host_port = Gitlab.config.registry.host_port

    ContainerRegistry::Registry.new(url, token: token, path: host_port)
  end
end

#retried_too_many_times? ⇒ Boolean

Returns:

  • (Boolean) 

372
373
374
 
# File 'app/models/container_repository.rb', line 372

def retried_too_many_times?
  migration_retries_count >= ContainerRegistry::Migration.max_retries
end

#retry_aborted_migration ⇒ Object 


308
309
310
311
312
313
314
315
 
# File 'app/models/container_repository.rb', line 308

def retry_aborted_migration
  return unless migration_state == 'import_aborted'

  reconcile_import_status(external_import_status) do
    # If the import_status request fails, use the timestamp to guess current state
    migration_pre_import_done_at ? retry_import : retry_pre_import
  end
end

#retry_import ⇒ Object 


297
298
299
300
301
 
# File 'app/models/container_repository.rb', line 297

def retry_import
  return false unless ContainerRegistry::Migration.enabled?

  super
end

#retry_pre_import ⇒ Object 


291
292
293
294
295
 
# File 'app/models/container_repository.rb', line 291

def retry_pre_import
  return false unless ContainerRegistry::Migration.enabled?

  super
end

#root_repository? ⇒ Boolean

Returns:

  • (Boolean) 

444
445
446
 
# File 'app/models/container_repository.rb', line 444

def root_repository?
  name.empty?
end

#size ⇒ Object 


468
469
470
471
472
473
474
475
476
 
# File 'app/models/container_repository.rb', line 468

def size
  strong_memoize(:size) do
    next unless Gitlab.com?
    next if self.created_at.before?(MIGRATION_PHASE_1_STARTED_AT)
    next unless gitlab_api_client.supports_gitlab_api?

    gitlab_api_client.repository_details(self.path, sizing: :self)['size_bytes']
  end
end

#skip_import(reason:) ⇒ Object 


279
280
281
282
283
 
# File 'app/models/container_repository.rb', line 279

def skip_import(reason:)
  self.migration_skipped_reason = reason

  super
end

#start_expiration_policy! ⇒ Object 


464
465
466
 
# File 'app/models/container_repository.rb', line 464

def start_expiration_policy!
  update!(expiration_policy_started_at: Time.zone.now, last_cleanup_deleted_tags_count: nil)
end

#start_pre_import(*args) ⇒ Object 


285
286
287
288
289
 
# File 'app/models/container_repository.rb', line 285

def start_pre_import(*args)
  return false unless ContainerRegistry::Migration.enabled?

  super(*args)
end

#tag(tag) ⇒ Object 


412
413
414
 
# File 'app/models/container_repository.rb', line 412

def tag(tag)
  ContainerRegistry::Tag.new(self, tag)
end

#tags ⇒ Object 


420
421
422
423
424
425
426
427
428
 
# File 'app/models/container_repository.rb', line 420

def tags
  return [] unless manifest && manifest['tags']

  strong_memoize(:tags) do
    manifest['tags'].sort.map do |tag|
      ContainerRegistry::Tag.new(self, tag)
    end
  end
end

#tags_count ⇒ Object 


430
431
432
433
434
 
# File 'app/models/container_repository.rb', line 430

def tags_count
  return 0 unless manifest && manifest['tags']

  manifest['tags'].size
end

#try_import ⇒ Object

Raises:

  • (ArgumentError) 

342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
 
# File 'app/models/container_repository.rb', line 342

def try_import
  raise ArgumentError, 'block not given' unless block_given?

  try_count = 0
  begin
    try_count += 1

    case yield
    when :ok
      return true
    when :not_found
      finish_import_as(:not_found)
    when :already_imported
      finish_import_as(:native_import)
    else
      abort_import
    end

    false
  rescue TooManyImportsError
    if try_count <= ::ContainerRegistry::Migration.start_max_retries
      sleep 0.1 * try_count
      retry
    else
      abort_import
      false
    end
  end
end