Module: Gitlab::SidekiqConfig

Extended by:
CliMethods
Defined in:
lib/gitlab/sidekiq_config.rb,
lib/gitlab/sidekiq_config/worker.rb,
lib/gitlab/sidekiq_config/cli_methods.rb,
lib/gitlab/sidekiq_config/dummy_worker.rb,
lib/gitlab/sidekiq_config/worker_router.rb,
lib/gitlab/sidekiq_config/worker_matcher.rb

Defined Under Namespace

Modules: CliMethods Classes: DummyWorker, Worker, WorkerMatcher, WorkerRouter

Constant Summary collapse

FOSS_QUEUE_CONFIG_PATH = 
'app/workers/all_queues.yml'
EE_QUEUE_CONFIG_PATH = 
'ee/app/workers/all_queues.yml'
JH_QUEUE_CONFIG_PATH = 
'jh/app/workers/all_queues.yml'
SIDEKIQ_QUEUES_PATH = 
'config/sidekiq_queues.yml'
JH_SIDEKIQ_QUEUES_PATH = 
'jh/config/sidekiq_queues.yml'
QUEUE_CONFIG_PATHS = 
[
  FOSS_QUEUE_CONFIG_PATH,
  (EE_QUEUE_CONFIG_PATH if Gitlab.ee?),
  (JH_QUEUE_CONFIG_PATH if Gitlab.jh?)
].compact.freeze
DEFAULT_WORKERS =

This maps workers not in our application code to queues. We need these queues in our YAML files to ensure we don’t accidentally miss jobs from these queues.

The default queue should be unused, which is why it maps to an invalid class name. We keep it in the YAML file for safety, just in case anything does get scheduled to run there.

{
  '_' => DummyWorker.new(
    queue: 'default',
    weight: 1,
    tags: []
  ),
  'ActionMailer::MailDeliveryJob' => DummyWorker.new(
    name: 'ActionMailer::MailDeliveryJob',
    queue: 'mailers',
    urgency: 'low',
    weight: 2,
    tags: []
  )
}.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false, jh: false) }.freeze

Class Method Summary collapse

Methods included from CliMethods

clear_memoization!, expand_queues, query_queues, worker_metadatas, worker_queues

Class Method Details

.all_queues_yml_outdated? ⇒ Boolean

YAML.load_file is OK here as we control the file contents

Returns:

  • (Boolean) 


115
116
117
118
119
120
121
122
123
 
# File 'lib/gitlab/sidekiq_config.rb', line 115

def all_queues_yml_outdated?
  foss_workers, ee_workers, jh_workers = workers_for_all_queues_yml

  return true if foss_workers != YAML.load_file(FOSS_QUEUE_CONFIG_PATH)

  return true if Gitlab.ee? && ee_workers != YAML.load_file(EE_QUEUE_CONFIG_PATH)

  Gitlab.jh? && File.exist?(JH_QUEUE_CONFIG_PATH) && jh_workers != YAML.load_file(JH_QUEUE_CONFIG_PATH)
end

.config_queues ⇒ Object 



49
50
51
52
53
54
 
# File 'lib/gitlab/sidekiq_config.rb', line 49

def config_queues
  @config_queues ||= begin
    config = YAML.load_file(Rails.root.join(SIDEKIQ_QUEUES_PATH))
    config[:queues].map(&:first)
  end
end

.cron_jobs ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/gitlab/sidekiq_config.rb', line 56

def cron_jobs
  @cron_jobs ||= begin
    Gitlab.config.load_dynamic_cron_schedules!

    jobs = Gitlab.config.cron_jobs.to_hash

    jobs.delete('poll_interval') # Would be interpreted as a job otherwise

    # Settingslogic (former gem used for yaml configuration) didn't allow 'class' key
    # Therefore, we configure cron jobs with `job_class` as a workaround.
    required_keys = %w[job_class cron]
    jobs.each do |k, v|
      if jobs[k] && required_keys.all? { |s| jobs[k].key?(s) }
        jobs[k]['class'] = jobs[k].delete('job_class')
      else
        jobs.delete(k)
        Gitlab::AppLogger.error("Invalid cron_jobs config key: '#{k}'. Check your gitlab config file.")
      end
    end

    jobs
  end
end

.cron_workers ⇒ Object 



80
81
82
 
# File 'lib/gitlab/sidekiq_config.rb', line 80

def cron_workers
  @cron_workers ||= cron_jobs.map { |job_name, options| options['class'].constantize }
end

.current_worker_queue_mappings ⇒ Object

Like worker_queue_mappings, but only for the queues running in the current Sidekiq process



162
163
164
165
166
 
# File 'lib/gitlab/sidekiq_config.rb', line 162

def current_worker_queue_mappings
  worker_queue_mappings
    .select { |worker, queue| Sidekiq[:queues].include?(queue) }
    .to_h
end

.jh_queues_for_sidekiq_queues_yml ⇒ Object

Override in JH repo



142
143
144
 
# File 'lib/gitlab/sidekiq_config.rb', line 142

def jh_queues_for_sidekiq_queues_yml
  []
end

.queues_for_sidekiq_queues_yml ⇒ Object 



125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/gitlab/sidekiq_config.rb', line 125

def queues_for_sidekiq_queues_yml
  namespaces_with_equal_weights =
    workers
      .reject { |worker| worker.jh? }
      .group_by(&:queue_namespace)
      .map(&:last)
      .select { |workers| workers.map(&:get_weight).uniq.count == 1 }
      .map(&:first)

  namespaces = namespaces_with_equal_weights.map(&:queue_namespace).to_set
  remaining_queues = workers.reject { |worker| worker.jh? }.reject { |worker| namespaces.include?(worker.queue_namespace) }

  (namespaces_with_equal_weights.map(&:namespace_and_weight) +
   remaining_queues.map(&:queue_and_weight)).sort
end

.redis_queues ⇒ Object 



44
45
46
47
 
# File 'lib/gitlab/sidekiq_config.rb', line 44

def redis_queues
  # Not memoized, because this can change during the life of the application
  Sidekiq::Queue.all.map(&:name)
end

.routing_queues ⇒ Object

Get the list of queues from all available workers following queue routing rules. Sidekiq::Queue.all fetches the list of queues from Redis. It may contain some redundant, obsolete queues from previous iterations of GitLab.



172
173
174
175
176
177
178
179
180
 
# File 'lib/gitlab/sidekiq_config.rb', line 172

def routing_queues
  @routing_queues ||= workers.map do |worker|
    if worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker)
      worker.queue
    else
      ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)
    end
  end.uniq.sort
end

.sidekiq_queues_yml_outdated? ⇒ Boolean

YAML.load_file is OK here as we control the file contents

Returns:

  • (Boolean) 


147
148
149
150
151
 
# File 'lib/gitlab/sidekiq_config.rb', line 147

def sidekiq_queues_yml_outdated?
  config_queues = YAML.load_file(SIDEKIQ_QUEUES_PATH)[:queues]

  queues_for_sidekiq_queues_yml != config_queues
end

.worker_queue_mappings ⇒ Object

Returns a hash of worker class name => mapped queue name



154
155
156
157
158
 
# File 'lib/gitlab/sidekiq_config.rb', line 154

def worker_queue_mappings
  workers
    .reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) }
    .to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] }
end

.workers ⇒ Object 



84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/gitlab/sidekiq_config.rb', line 84

def workers
  @workers ||= begin
    result = []
    result.concat(DEFAULT_WORKERS.values)
    result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false, jh: false))

    if Gitlab.ee?
      result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'), ee: true, jh: false))
    end

    if Gitlab.jh?
      result.concat(find_workers(Rails.root.join('jh', 'app', 'workers'), ee: false, jh: true))
    end

    result
  end
end

.workers_for_all_queues_yml ⇒ Object 



102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/gitlab/sidekiq_config.rb', line 102

def workers_for_all_queues_yml
  workers.each_with_object([[], [], []]) do |worker, array|
    if worker.jh?
      array[2].push(worker)
    elsif worker.ee?
      array[1].push(worker)
    else
      array[0].push(worker)
    end
  end.map(&:sort)
end