Class: Ci::ArchiveTraceService

Inherits:
Object
  • Object
show all
Includes:
Gitlab::ExclusiveLeaseHelpers
Defined in:
app/services/ci/archive_trace_service.rb

Constant Summary collapse

EXCLUSIVE_LOCK_KEY = 
'archive_trace_service:batch_execute:lock'
LOCK_TIMEOUT = 
56.minutes
LOOP_TIMEOUT = 
55.minutes
LOOP_LIMIT = 
2000
BATCH_SIZE = 
100

Constants included from Gitlab::ExclusiveLeaseHelpers

Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError

Instance Method Summary collapse

Methods included from Gitlab::ExclusiveLeaseHelpers

#in_lock

Instance Method Details

#batch_execute(worker_name:) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'app/services/ci/archive_trace_service.rb', line 14

def batch_execute(worker_name:)
  start_time = Time.current
  in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
    Ci::Build.with_stale_live_trace.find_each(batch_size: BATCH_SIZE).with_index do |build, index|
      break if Time.current - start_time > LOOP_TIMEOUT

      if index > LOOP_LIMIT
        Sidekiq.logger.warn(class: worker_name, message: 'Loop limit reached.', job_id: build.id)
        break
      end

      begin
        execute(build, worker_name: worker_name)
      rescue StandardError
        next
      end
    end
  end
end

#execute(job, worker_name:) ⇒ Object

rubocop: enable CodeReuse/ActiveRecord



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'app/services/ci/archive_trace_service.rb', line 35

def execute(job, worker_name:)
  unless job.trace.archival_attempts_available?
    Sidekiq.logger.warn(class: worker_name, message: 'The job is out of archival attempts.', job_id: job.id)

    job.trace.attempt_archive_cleanup!
    return
  end

  unless job.trace.can_attempt_archival_now?
    Sidekiq.logger.warn(class: worker_name, message: 'The job can not be archived right now.', job_id: job.id)
    return
  end

  job.trace.archive!
  job.remove_pending_state!

  if job.job_artifacts_trace.present?
    job.project.execute_integrations(Gitlab::DataBuilder::ArchiveTrace.build(job), :archive_trace_hooks)
  end
rescue ::Gitlab::Ci::Trace::AlreadyArchivedError
  # It's already archived, thus we can safely ignore this exception.
rescue StandardError => e
  job.trace.increment_archival_attempts!

  # Tracks this error with application logs, Sentry, and Prometheus.
  # If `archive!` keeps failing for over a week, that could incur data loss.
  # (See more https://docs.gitlab.com/ee/administration/job_logs.html#new-incremental-logging-architecture)
  # In order to avoid interrupting the system, we do not raise an exception here.
  archive_error(e, job, worker_name)
end