Class: Gitlab::SidekiqMiddleware::ServerMetrics

Inherits:
Object
  • Object
show all
Extended by:
MetricsHelper
Includes:
MetricsHelper
Defined in:
lib/gitlab/sidekiq_middleware/server_metrics.rb

Constant Summary collapse

SIDEKIQ_LATENCY_BUCKETS =

SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq timeframes than the DEFAULT_BUCKET definition. Defined in seconds. This information is better viewed in logs, but these buckets cover most of the durations for cpu, gitaly, db and elasticsearch

[0.1, 0.5, 1, 2.5].freeze
SIDEKIQ_JOB_DURATION_BUCKETS =

These buckets are only available on self-managed. We have replaced with Application SLIs on GitLab.com. gitlab.com/groups/gitlab-com/gl-infra/-/epics/700

[10, 300].freeze
SIDEKIQ_QUEUE_DURATION_BUCKETS = 
[10, 60].freeze
SIDEKIQ_SLI_LABELS =

These labels from Gitlab::SidekiqMiddleware::MetricsHelper are included in SLI metrics

[:worker, :feature_category, :urgency, :external_dependencies, :queue].freeze

Constants included from MetricsHelper

MetricsHelper::FALSE_LABEL, MetricsHelper::TRUE_LABEL

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ ServerMetrics

Returns a new instance of ServerMetrics.



82
83
84
85
 
# File 'lib/gitlab/sidekiq_middleware/server_metrics.rb', line 82

def initialize
  @metrics = self.class.metrics
  @metrics[:sidekiq_load_balancing_count] = ::Gitlab::Metrics.counter(:sidekiq_load_balancing_count, 'Sidekiq jobs with load balancing')
end

Class Method Details

.initialize_process_metrics ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/gitlab/sidekiq_middleware/server_metrics.rb', line 54

def initialize_process_metrics
  metrics = self.metrics

  metrics[:sidekiq_concurrency].set({}, Sidekiq[:concurrency].to_i)

  return unless ::Feature.enabled?(:sidekiq_job_completion_metric_initialize)

  possible_sli_labels = []
  ::Gitlab::SidekiqConfig.current_worker_queue_mappings.each do |worker, queue|
    worker_class = worker.safe_constantize

    next unless worker_class

    base_labels = create_labels(worker_class, queue, {})
    possible_sli_labels << base_labels.slice(*SIDEKIQ_SLI_LABELS)

    next unless Feature.enabled?(:emit_sidekiq_histogram_metrics, type: :ops)

    %w[done fail].each do |status|
      metrics[:sidekiq_jobs_completion_seconds].get(base_labels.merge(job_status: status))
    end
  end

  Gitlab::Metrics::SidekiqSlis.initialize_execution_slis!(possible_sli_labels)
  Gitlab::Metrics::SidekiqSlis.initialize_queueing_slis!(possible_sli_labels)
end

.metrics ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/gitlab/sidekiq_middleware/server_metrics.rb', line 26

def metrics
  metrics = {
    sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS),
    sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
    sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
    sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS),
    sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS),
    sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'),
    sidekiq_jobs_interrupted_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_interrupted_total, 'Sidekiq jobs interrupted'),
    sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'),
    sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'),
    sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all),
    sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all),
    sidekiq_mem_total_bytes: ::Gitlab::Metrics.gauge(:sidekiq_mem_total_bytes, 'Number of bytes allocated for both objects consuming an object slot and objects that required a malloc', {}, :all)
  }

  if Feature.enabled?(:emit_sidekiq_histogram_metrics, type: :ops)
    metrics[:sidekiq_jobs_completion_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_JOB_DURATION_BUCKETS)
    metrics[:sidekiq_jobs_queue_duration_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_QUEUE_DURATION_BUCKETS)
    metrics[:sidekiq_jobs_failed_total] = ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed')
  else
    # The sum metric is still used in GitLab.com for dashboards
    metrics[:sidekiq_jobs_completion_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_completion_seconds_sum, 'Total of seconds to complete Sidekiq job')
  end

  metrics
end

Instance Method Details

#call(worker, job, queue) ⇒ Object 



87
88
89
90
91
92
93
94
95
96
 
# File 'lib/gitlab/sidekiq_middleware/server_metrics.rb', line 87

def call(worker, job, queue)
  # This gives all the sidekiq worker threads a name, so we can recognize them
  # in metrics and can use them in the `ThreadsSampler` for setting a label
  Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME

  labels = create_labels(worker.class, queue, job)
  instrument(job, labels) do
    yield
  end
end