Module: Gitlab::BitbucketImport::ParallelScheduling

Includes:
ErrorTracking, Loggable
Included in:
Importers::IssueNotesImporter, Importers::IssuesImporter, Importers::IssuesNotesImporter, Importers::LfsObjectsImporter, Importers::PullRequestNotesImporter, Importers::PullRequestsImporter, Importers::PullRequestsNotesImporter
Defined in:
lib/gitlab/bitbucket_import/parallel_scheduling.rb

Constant Summary collapse

ALREADY_ENQUEUED_CACHE_KEY =

The base cache key to use for tracking already enqueued objects.

'bitbucket-importer/already-enqueued/%{project}/%{collection}'
JOB_WAITER_CACHE_KEY =

The base cache key to use for storing job waiter key

'bitbucket-importer/job-waiter/%{project}/%{collection}'
JOB_WAITER_REMAINING_CACHE_KEY =

The base cache key to use for storing job waiter remaining jobs

'bitbucket-importer/job-waiter-remaining/%{project}/%{collection}'

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from ErrorTracking

#track_import_failure!

Methods included from Loggable

#log_debug, #log_error, #log_info, #log_warn, #metrics

Instance Attribute Details

#already_enqueued_cache_key ⇒ Object (readonly)

Returns the value of attribute already_enqueued_cache_key.



9
10
11
 
# File 'lib/gitlab/bitbucket_import/parallel_scheduling.rb', line 9

def already_enqueued_cache_key
  @already_enqueued_cache_key
end

#job_waiter_cache_key ⇒ Object (readonly)

Returns the value of attribute job_waiter_cache_key.



9
10
11
 
# File 'lib/gitlab/bitbucket_import/parallel_scheduling.rb', line 9

def job_waiter_cache_key
  @job_waiter_cache_key
end

#job_waiter_remaining_cache_key ⇒ Object (readonly)

Returns the value of attribute job_waiter_remaining_cache_key.



9
10
11
 
# File 'lib/gitlab/bitbucket_import/parallel_scheduling.rb', line 9

def job_waiter_remaining_cache_key
  @job_waiter_remaining_cache_key
end

#page_keyset ⇒ Object (readonly)

Returns the value of attribute page_keyset.



9
10
11
 
# File 'lib/gitlab/bitbucket_import/parallel_scheduling.rb', line 9

def page_keyset
  @page_keyset
end

#project ⇒ Object (readonly)

Returns the value of attribute project.



9
10
11
 
# File 'lib/gitlab/bitbucket_import/parallel_scheduling.rb', line 9

def project
  @project
end

Instance Method Details

#each_object_to_import ⇒ Object

The method that will be called for traversing through all the objects to import, yielding them to the supplied block.



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/gitlab/bitbucket_import/parallel_scheduling.rb', line 39

def each_object_to_import
  repo = project.import_source

  options = collection_options.merge(next_url: page_keyset.current)

  client.each_page(collection_method, representation_type, repo, options) do |page|
    page.items.each do |object|
      job_waiter.jobs_remaining = Gitlab::Cache::Import::Caching.increment(job_waiter_remaining_cache_key)

      object = object.to_hash

      next if already_enqueued?(object)

      yield object

      # We mark the object as imported immediately so we don't end up
      # scheduling it multiple times.
      mark_as_enqueued(object)
    end

    page_keyset.set(page.next) if page.next?
  end
end

#initialize(project) ⇒ Object

project - An instance of ‘Project`.



25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/gitlab/bitbucket_import/parallel_scheduling.rb', line 25

def initialize(project)
  @project = project

  @already_enqueued_cache_key =
    format(ALREADY_ENQUEUED_CACHE_KEY, project: project.id, collection: collection_method)
  @job_waiter_cache_key =
    format(JOB_WAITER_CACHE_KEY, project: project.id, collection: collection_method)
  @job_waiter_remaining_cache_key = format(JOB_WAITER_REMAINING_CACHE_KEY, project: project.id,
    collection: collection_method)
  @page_keyset = Gitlab::Import::PageKeyset.new(project, collection_method, 'bitbucket-importer')
end