Module: Gitlab::SidekiqConfig

Extended by:
CliMethods
Defined in:
lib/gitlab/sidekiq_config.rb,
lib/gitlab/sidekiq_config/worker.rb,
lib/gitlab/sidekiq_config/cli_methods.rb,
lib/gitlab/sidekiq_config/dummy_worker.rb,
lib/gitlab/sidekiq_config/worker_router.rb,
lib/gitlab/sidekiq_config/worker_matcher.rb

Defined Under Namespace

Modules: CliMethods Classes: DummyWorker, Worker, WorkerMatcher, WorkerRouter

Constant Summary collapse

FOSS_QUEUE_CONFIG_PATH =
'app/workers/all_queues.yml'
EE_QUEUE_CONFIG_PATH =
'ee/app/workers/all_queues.yml'
JH_QUEUE_CONFIG_PATH =
'jh/app/workers/all_queues.yml'
SIDEKIQ_QUEUES_PATH =
'config/sidekiq_queues.yml'
JH_SIDEKIQ_QUEUES_PATH =
'jh/config/sidekiq_queues.yml'
QUEUE_CONFIG_PATHS =
[
  FOSS_QUEUE_CONFIG_PATH,
  (EE_QUEUE_CONFIG_PATH if Gitlab.ee?),
  (JH_QUEUE_CONFIG_PATH if Gitlab.jh?)
].compact.freeze
DEFAULT_WORKERS =

This maps workers not in our application code to queues. We need these queues in our YAML files to ensure we don't accidentally miss jobs from these queues.

The default queue should be unused, which is why it maps to an invalid class name. We keep it in the YAML file for safety, just in case anything does get scheduled to run there.

{
  '_' => DummyWorker.new(
    queue: 'default',
    weight: 1,
    tags: []
  ),
  'ActionMailer::MailDeliveryJob' => DummyWorker.new(
    name: 'ActionMailer::MailDeliveryJob',
    queue: 'mailers',
    urgency: 'low',
    weight: 2,
    tags: []
  )
}.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false, jh: false) }.freeze

Class Method Summary collapse

Methods included from CliMethods

clear_memoization!, expand_queues, query_queues, worker_metadatas, worker_queues

Class Method Details

.all_queues_yml_outdated?Boolean

YAML.load_file is OK here as we control the file contents

Returns:

  • (Boolean)

91
92
93
94
95
96
97
98
99
# File 'lib/gitlab/sidekiq_config.rb', line 91

def all_queues_yml_outdated?
  foss_workers, ee_workers, jh_workers = workers_for_all_queues_yml

  return true if foss_workers != YAML.load_file(FOSS_QUEUE_CONFIG_PATH)

  return true if Gitlab.ee? && ee_workers != YAML.load_file(EE_QUEUE_CONFIG_PATH)

  Gitlab.jh? && File.exist?(JH_QUEUE_CONFIG_PATH) && jh_workers != YAML.load_file(JH_QUEUE_CONFIG_PATH)
end

.config_queuesObject


49
50
51
52
53
54
# File 'lib/gitlab/sidekiq_config.rb', line 49

def config_queues
  @config_queues ||= begin
    config = YAML.load_file(Rails.root.join(SIDEKIQ_QUEUES_PATH))
    config[:queues].map(&:first)
  end
end

.cron_workersObject


56
57
58
# File 'lib/gitlab/sidekiq_config.rb', line 56

def cron_workers
  @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize }
end

.current_worker_queue_mappingsObject

Like worker_queue_mappings, but only for the queues running in the current Sidekiq process


138
139
140
141
142
# File 'lib/gitlab/sidekiq_config.rb', line 138

def current_worker_queue_mappings
  worker_queue_mappings
    .select { |worker, queue| Sidekiq.options[:queues].include?(queue) }
    .to_h
end

.jh_queues_for_sidekiq_queues_ymlObject

Override in JH repo


118
119
120
# File 'lib/gitlab/sidekiq_config.rb', line 118

def jh_queues_for_sidekiq_queues_yml
  []
end

.queues_for_sidekiq_queues_ymlObject


101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
# File 'lib/gitlab/sidekiq_config.rb', line 101

def queues_for_sidekiq_queues_yml
  namespaces_with_equal_weights =
    workers
      .reject { |worker| worker.jh? }
      .group_by(&:queue_namespace)
      .map(&:last)
      .select { |workers| workers.map(&:get_weight).uniq.count == 1 }
      .map(&:first)

  namespaces = namespaces_with_equal_weights.map(&:queue_namespace).to_set
  remaining_queues = workers.reject { |worker| worker.jh? }.reject { |worker| namespaces.include?(worker.queue_namespace) }

  (namespaces_with_equal_weights.map(&:namespace_and_weight) +
   remaining_queues.map(&:queue_and_weight)).sort
end

.redis_queuesObject


44
45
46
47
# File 'lib/gitlab/sidekiq_config.rb', line 44

def redis_queues
  # Not memoized, because this can change during the life of the application
  Sidekiq::Queue.all.map(&:name)
end

.sidekiq_queues_yml_outdated?Boolean

YAML.load_file is OK here as we control the file contents

Returns:

  • (Boolean)

123
124
125
126
127
# File 'lib/gitlab/sidekiq_config.rb', line 123

def sidekiq_queues_yml_outdated?
  config_queues = YAML.load_file(SIDEKIQ_QUEUES_PATH)[:queues]

  queues_for_sidekiq_queues_yml != config_queues
end

.worker_queue_mappingsObject

Returns a hash of worker class name => mapped queue name


130
131
132
133
134
# File 'lib/gitlab/sidekiq_config.rb', line 130

def worker_queue_mappings
  workers
    .reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) }
    .to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] }
end

.workersObject


60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
# File 'lib/gitlab/sidekiq_config.rb', line 60

def workers
  @workers ||= begin
    result = []
    result.concat(DEFAULT_WORKERS.values)
    result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false, jh: false))

    if Gitlab.ee?
      result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'), ee: true, jh: false))
    end

    if Gitlab.jh?
      result.concat(find_workers(Rails.root.join('jh', 'app', 'workers'), ee: false, jh: true))
    end

    result
  end
end

.workers_for_all_queues_ymlObject


78
79
80
81
82
83
84
85
86
87
88
# File 'lib/gitlab/sidekiq_config.rb', line 78

def workers_for_all_queues_yml
  workers.each_with_object([[], [], []]) do |worker, array|
    if worker.jh?
      array[2].push(worker)
    elsif worker.ee?
      array[1].push(worker)
    else
      array[0].push(worker)
    end
  end.map(&:sort)
end