Class: Namespaces::EnableDescendantsCacheCronWorker

Inherits:
Object
  • Object
show all
Includes:
ApplicationWorker, CronjobQueue
Defined in:
app/workers/namespaces/enable_descendants_cache_cron_worker.rb

Constant Summary collapse

GROUP_BATCH_SIZE = 
5000
NAMESPACE_BATCH_SIZE = 
500
PERSIST_SLICE_SIZE = 
100
CACHE_THRESHOLD =

Covers the top 3000 namespaces on .com

700
CURSOR_KEY = 
'enable_namespace_descendants_cron_worker'
MAX_RUNTIME = 
45.seconds

Constants included from ApplicationWorker

ApplicationWorker::LOGGING_EXTRA_KEY, ApplicationWorker::SAFE_PUSH_BULK_LIMIT

Constants included from Gitlab::Loggable

Gitlab::Loggable::ANONYMOUS

Constants included from WorkerAttributes

WorkerAttributes::DEFAULT_CONCURRENCY_LIMIT_PERCENTAGE_BY_URGENCY, WorkerAttributes::DEFAULT_DATA_CONSISTENCY, WorkerAttributes::DEFAULT_DATA_CONSISTENCY_PER_DB, WorkerAttributes::DEFAULT_DEFER_DELAY, WorkerAttributes::LOAD_BALANCED_DATA_CONSISTENCIES, WorkerAttributes::NAMESPACE_WEIGHTS, WorkerAttributes::VALID_DATA_CONSISTENCIES, WorkerAttributes::VALID_RESOURCE_BOUNDARIES, WorkerAttributes::VALID_URGENCIES

Instance Method Summary collapse

Methods included from Gitlab::Loggable

#build_structured_payload

Methods included from Gitlab::SidekiqVersioning::Worker

#job_version

Methods included from WorkerContext

#with_context

Instance Method Details

#get_last_id ⇒ Object 



70
71
72
73
74
75
 
# File 'app/workers/namespaces/enable_descendants_cache_cron_worker.rb', line 70

def get_last_id
  value = Gitlab::Redis::SharedState.with { |redis| redis.get(CURSOR_KEY) }
  return if value.nil?

  Integer(value)
end

#perform ⇒ Object

rubocop: disable CodeReuse/ActiveRecord – Batching over groups.



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'app/workers/namespaces/enable_descendants_cache_cron_worker.rb', line 26

def perform
  limiter = Gitlab::Metrics::RuntimeLimiter.new(MAX_RUNTIME)
  ids_to_cache = Set.new
  last_id = get_last_id

  # 1. Iterate over groups.
  # 2. For each group, start counting the descendants.
  # 3. When CACHE_THRESHOLD count is reached, stop the counting.
  Group.where('id > ?', last_id || 0).each_batch(of: GROUP_BATCH_SIZE) do |relation|
    relation.select(:id).each do |group|
      cursor = { current_id: group.id, depth: [group.id] }
      iterator = Gitlab::Database::NamespaceEachBatch.new(namespace_class: Namespace, cursor: cursor)
      count = 0
      iterator.each_batch(of: NAMESPACE_BATCH_SIZE) do |ids|
        count += ids.size

        break if count >= CACHE_THRESHOLD || limiter.over_time?
      end

      ids_to_cache << group.id if count >= CACHE_THRESHOLD

      break if limiter.was_over_time?

      last_id = group.id
    end
    break if limiter.was_over_time?
  end

  last_id = nil unless limiter.was_over_time?

  persist(ids_to_cache)
  set_last_id(last_id)

  log_extra_metadata_on_done(:result,
    { over_time: limiter.was_over_time?, last_id: last_id, cache_count: ids_to_cache.size })
end

#persist(ids_to_cache) ⇒ Object

rubocop: enable CodeReuse/ActiveRecord



64
65
66
67
68
 
# File 'app/workers/namespaces/enable_descendants_cache_cron_worker.rb', line 64

def persist(ids_to_cache)
  ids_to_cache.each_slice(PERSIST_SLICE_SIZE) do |slice|
    Namespaces::Descendants.upsert_all(slice.map { |id| { namespace_id: id, outdated_at: Time.current } })
  end
end

#set_last_id(last_id) ⇒ Object 



77
78
79
80
81
82
83
 
# File 'app/workers/namespaces/enable_descendants_cache_cron_worker.rb', line 77

def set_last_id(last_id)
  if last_id.nil?
    Gitlab::Redis::SharedState.with { |redis| redis.del(CURSOR_KEY) }
  else
    Gitlab::Redis::SharedState.with { |redis| redis.set(CURSOR_KEY, last_id, ex: 1.day) }
  end
end