Class: Gitlab::UsageData

Inherits:
Object
  • Object
show all
Extended by:
Gitlab::Utils::StrongMemoize, Gitlab::Utils::UsageData
Defined in:
lib/gitlab/usage_data.rb,
lib/gitlab/usage_data/topology.rb

Direct Known Subclasses

UsageDataQueries

Defined Under Namespace

Classes: Topology

Constant Summary

Constants included from Gitlab::Utils::UsageData

Gitlab::Utils::UsageData::FALLBACK

Class Method Summary collapse

Methods included from Gitlab::Utils::StrongMemoize

clear_memoization, strong_memoize, strong_memoized?

Methods included from Gitlab::Utils::UsageData

alt_usage_data, count, distinct_count, measure_duration, redis_usage_data, track_usage_event, with_finished_at, with_prometheus_client

Class Method Details

.action_monthly_active_users(time_period) ⇒ Object 


674
675
676
677
678
679
 
# File 'lib/gitlab/usage_data.rb', line 674

def action_monthly_active_users(time_period)
  date_range = { date_from: time_period[:created_at].first, date_to: time_period[:created_at].last }

  event_monthly_active_users(date_range)
    .merge!(ide_monthly_active_users(date_range))
end

.analytics_unique_visits_data ⇒ Object 


642
643
644
645
646
647
648
649
650
 
# File 'lib/gitlab/usage_data.rb', line 642

def analytics_unique_visits_data
  results = ::Gitlab::Analytics::UniqueVisits.analytics_events.each_with_object({}) do |target, hash|
    hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) }
  end
  results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics) }
  results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, start_date: 4.weeks.ago.to_date, end_date: Date.current) }

  { analytics_unique_visits: results }
end

.compliance_unique_visits_data ⇒ Object 


652
653
654
655
656
657
658
659
660
 
# File 'lib/gitlab/usage_data.rb', line 652

def compliance_unique_visits_data
  results = ::Gitlab::Analytics::UniqueVisits.compliance_events.each_with_object({}) do |target, hash|
    hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) }
  end
  results['compliance_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance) }
  results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, start_date: 4.weeks.ago.to_date, end_date: Date.current) }

  { compliance_unique_visits: results }
end

.components_usage_data ⇒ Object 


271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
 
# File 'lib/gitlab/usage_data.rb', line 271

def components_usage_data
  {
    git: { version: alt_usage_data(fallback: { major: -1 }) { Gitlab::Git.version } },
    gitaly: {
      version: alt_usage_data { Gitaly::Server.all.first.server_version },
      servers: alt_usage_data { Gitaly::Server.count },
      clusters: alt_usage_data { Gitaly::Server.gitaly_clusters },
      filesystems: alt_usage_data(fallback: ["-1"]) { Gitaly::Server.filesystems }
    },
    gitlab_pages: {
      enabled: alt_usage_data(fallback: nil) { Gitlab.config.pages.enabled },
      version: alt_usage_data { Gitlab::Pages::VERSION }
    },
    container_registry_server: {
      vendor: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_vendor },
      version: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_version }
    },
    database: {
      adapter: alt_usage_data { Gitlab::Database.adapter_name },
      version: alt_usage_data { Gitlab::Database.version }
    },
    mail: {
      smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] }
    }
  }
end

.container_expiration_policies_usage ⇒ Object

rubocop: disable CodeReuse/ActiveRecord


360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
 
# File 'lib/gitlab/usage_data.rb', line 360

def container_expiration_policies_usage
  results = {}
  start = ::Project.minimum(:id)
  finish = ::Project.maximum(:id)

  results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish)
  # rubocop: disable UsageData/LargeTable
  base = ::ContainerExpirationPolicy.active
  # rubocop: enable UsageData/LargeTable
  results[:projects_with_expiration_policy_enabled] = distinct_count(base, :project_id, start: start, finish: finish)

  # rubocop: disable UsageData/LargeTable
  %i[keep_n cadence older_than].each do |option|
    ::ContainerExpirationPolicy.public_send("#{option}_options").keys.each do |value| # rubocop: disable GitlabSecurity/PublicSend
      results["projects_with_expiration_policy_enabled_with_#{option}_set_to_#{value}".to_sym] = distinct_count(base.where(option => value), :project_id, start: start, finish: finish)
    end
  end
  # rubocop: enable UsageData/LargeTable

  results[:projects_with_expiration_policy_enabled_with_keep_n_unset] = distinct_count(base.where(keep_n: nil), :project_id, start: start, finish: finish)
  results[:projects_with_expiration_policy_enabled_with_older_than_unset] = distinct_count(base.where(older_than: nil), :project_id, start: start, finish: finish)

  results
end

.cycle_analytics_usage_data ⇒ Object

rubocop: enable CodeReuse/ActiveRecord


211
212
213
214
215
 
# File 'lib/gitlab/usage_data.rb', line 211

def cycle_analytics_usage_data
  Gitlab::CycleAnalytics::UsageData.new.to_json
rescue ActiveRecord::StatementInvalid
  { avg_cycle_analytics: {} }
end

.data(force_refresh: false) ⇒ Object 


19
20
21
22
23
 
# File 'lib/gitlab/usage_data.rb', line 19

def data(force_refresh: false)
  Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do
    uncached_data
  end
end

.features_usage_data ⇒ Object

rubocop: enable CodeReuse/ActiveRecord


225
226
227
 
# File 'lib/gitlab/usage_data.rb', line 225

def features_usage_data
  features_usage_data_ce
end

.features_usage_data_ce ⇒ Object 


229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
 
# File 'lib/gitlab/usage_data.rb', line 229

def features_usage_data_ce
  {
    instance_auto_devops_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.auto_devops_enabled? },
    container_registry_enabled: alt_usage_data(fallback: nil) { Gitlab.config.registry.enabled },
    dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled,
    gitlab_shared_runners_enabled: alt_usage_data(fallback: nil) { Gitlab.config.gitlab_ci.shared_runners_enabled },
    gravatar_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gravatar_enabled? },
    ldap_enabled: alt_usage_data(fallback: nil) { Gitlab.config.ldap.enabled },
    mattermost_enabled: alt_usage_data(fallback: nil) { Gitlab.config.mattermost.enabled },
    omniauth_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth.omniauth_enabled? },
    prometheus_enabled: alt_usage_data(fallback: nil) { Gitlab::Prometheus::Internal.prometheus_enabled? },
    prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? },
    reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::IncomingEmail.enabled? },
    signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? },
    web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? },
    ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity),
    grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }
  }
end

.grafana_embed_usage_data ⇒ Object

rubocop:disable CodeReuse/ActiveRecord


218
219
220
221
222
 
# File 'lib/gitlab/usage_data.rb', line 218

def grafana_embed_usage_data
  count(Issue.joins('JOIN grafana_integrations USING (project_id)')
    .where("issues.description LIKE '%' || grafana_integrations.grafana_url || '%'")
    .where(grafana_integrations: { enabled: true }))
end

.ingress_modsecurity_usage ⇒ Object

rubocop: disable UsageData/DistinctCountByLargeForeignKey


337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
 
# File 'lib/gitlab/usage_data.rb', line 337

def ingress_modsecurity_usage
  ##
  # This method measures usage of the Modsecurity Web Application Firewall across the entire
  # instance's deployed environments.
  #
  # NOTE: this service is an approximation as it does not yet take into account if environment
  # is enabled and only measures applications installed using GitLab Managed Apps (disregards
  # CI-based managed apps).
  #
  # More details: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28331#note_318621786
  ##

  column = ::Deployment.arel_table[:environment_id]
  {
    ingress_modsecurity_logging: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_enabled.logging), column),
    ingress_modsecurity_blocking: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_enabled.blocking), column),
    ingress_modsecurity_disabled: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_disabled), column),
    ingress_modsecurity_not_installed: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_not_installed), column)
  }
end

.installation_type ⇒ Object 


463
464
465
466
467
468
469
 
# File 'lib/gitlab/usage_data.rb', line 463

def installation_type
  if Rails.env.production?
    Gitlab::INSTALLATION_TYPE
  else
    "gitlab-development-kit"
  end
end

.jira_import_usage ⇒ Object

rubocop: enable CodeReuse/ActiveRecord


424
425
426
427
428
429
430
431
432
433
434
 
# File 'lib/gitlab/usage_data.rb', line 424

def jira_import_usage
  # rubocop: disable UsageData/LargeTable
  finished_jira_imports = JiraImportState.finished

  {
    jira_imports_total_imported_count: count(finished_jira_imports),
    jira_imports_projects_count: distinct_count(finished_jira_imports, :project_id),
    jira_imports_total_imported_issues_count: alt_usage_data { JiraImportState.finished_imports_count }
  }
  # rubocop: enable UsageData/LargeTable
end

.jira_usage ⇒ Object 


396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
 
# File 'lib/gitlab/usage_data.rb', line 396

def jira_usage
  # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999
  # so we can just check for subdomains of atlassian.net
  results = {
    projects_jira_server_active: 0,
    projects_jira_cloud_active: 0,
    projects_jira_dvcs_cloud_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled),
    projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false))
  }

  # rubocop: disable UsageData/LargeTable:
  JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services|
    counts = services.group_by do |service|
      # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404
      service_url = service.data_fields&.url || (service.properties && service.properties['url'])
      service_url&.include?('.atlassian.net') ? :cloud : :server
    end

    results[:projects_jira_server_active] += counts[:server].size if counts[:server]
    results[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud]
  end
  # rubocop: enable UsageData/LargeTable:
  results
rescue ActiveRecord::StatementInvalid
  { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK }
end

.last_28_days_time_period(column: :created_at) ⇒ Object 


471
472
473
 
# File 'lib/gitlab/usage_data.rb', line 471

def last_28_days_time_period(column: :created_at)
  { column => 28.days.ago..Time.current }
end

.license_usage_data ⇒ Object 


50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/gitlab/usage_data.rb', line 50

def license_usage_data
  {
    recorded_at: recorded_at,
    uuid: alt_usage_data { Gitlab::CurrentSettings.uuid },
    hostname: alt_usage_data { Gitlab.config.gitlab.host },
    version: alt_usage_data { Gitlab::VERSION },
    installation_type: alt_usage_data { installation_type },
    active_user_count: count(User.active),
    edition: 'CE'
  }
end

.merge_requests_users(time_period) ⇒ Object 


451
452
453
454
455
456
457
458
459
460
461
 
# File 'lib/gitlab/usage_data.rb', line 451

def merge_requests_users(time_period)
  counter = Gitlab::UsageDataCounters::TrackUniqueEvents

  redis_usage_data do
    counter.count_unique_events(
      event_action: Gitlab::UsageDataCounters::TrackUniqueEvents::MERGE_REQUEST_ACTION,
      date_from: time_period[:created_at].first,
      date_to: time_period[:created_at].last
    )
  end
end

.object_store_config(component) ⇒ Object 


298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
 
# File 'lib/gitlab/usage_data.rb', line 298

def object_store_config(component)
  config = alt_usage_data(fallback: nil) do
    Settings[component]['object_store']
  end

  if config
    {
      enabled: alt_usage_data { Settings[component]['enabled'] },
      object_store: {
        enabled: alt_usage_data { config['enabled'] },
        direct_upload: alt_usage_data { config['direct_upload'] },
        background_upload: alt_usage_data { config['background_upload'] },
        provider: alt_usage_data { config['connection']['provider'] }
      }
    }
  else
    {
      enabled: alt_usage_data { Settings[component]['enabled'] }
    }
  end
end

.object_store_usage_data ⇒ Object 


320
321
322
323
324
325
326
327
328
329
330
 
# File 'lib/gitlab/usage_data.rb', line 320

def object_store_usage_data
  {
    object_store: {
      artifacts: object_store_config('artifacts'),
      external_diffs: object_store_config('external_diffs'),
      lfs: object_store_config('lfs'),
      uploads: object_store_config('uploads'),
      packages: object_store_config('packages')
    }
  }
end

.recorded_at ⇒ Object 


62
63
64
 
# File 'lib/gitlab/usage_data.rb', line 62

def recorded_at
  Time.current
end

.redis_hll_counters ⇒ Object 


638
639
640
 
# File 'lib/gitlab/usage_data.rb', line 638

def redis_hll_counters
  { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data }
end

.search_unique_visits_data ⇒ Object 


662
663
664
665
666
667
668
669
670
671
672
 
# File 'lib/gitlab/usage_data.rb', line 662

def search_unique_visits_data
  events = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('search')
  results = events.each_with_object({}) do |event, hash|
    hash[event] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) }
  end

  results['search_unique_visits_for_any_target_weekly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: 7.days.ago.to_date, end_date: Date.current) }
  results['search_unique_visits_for_any_target_monthly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: 4.weeks.ago.to_date, end_date: Date.current) }

  { search_unique_visits: results }
end

.services_usage ⇒ Object 


385
386
387
388
389
390
391
392
393
394
 
# File 'lib/gitlab/usage_data.rb', line 385

def services_usage
  # rubocop: disable UsageData/LargeTable:
  Service.available_services_names.each_with_object({}) do |service_name, response|
    response["projects_#{service_name}_active".to_sym] = count(Service.active.where(template: false, instance: false, type: "#{service_name}_service".camelize))
    response["templates_#{service_name}_active".to_sym] = count(Service.active.where(template: true, type: "#{service_name}_service".camelize))
    response["instances_#{service_name}_active".to_sym] = count(Service.active.where(instance: true, type: "#{service_name}_service".camelize))
    response["projects_inheriting_instance_#{service_name}_active".to_sym] = count(Service.active.where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize))
  end.merge(jira_usage, jira_import_usage)
  # rubocop: enable UsageData/LargeTable:
end

.snowplow_event_counts(time_period: {}) ⇒ Object

rubocop: enable Metrics/AbcSize


178
179
180
181
182
183
184
185
186
187
188
189
 
# File 'lib/gitlab/usage_data.rb', line 178

def snowplow_event_counts(time_period: {})
  return {} unless report_snowplow_events?

  {
    promoted_issues: count(
      self_monitoring_project
        .product_analytics_events
        .by_category_and_action('epics', 'promote')
        .where(time_period)
    )
  }
end

.successful_deployments_with_cluster(scope) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord rubocop: disable UsageData/LargeTable


438
439
440
441
442
443
 
# File 'lib/gitlab/usage_data.rb', line 438

def successful_deployments_with_cluster(scope)
  scope
    .joins(cluster: :deployments)
    .merge(::Clusters::Cluster.enabled)
    .merge(Deployment.success)
end

.system_usage_data ⇒ Object

rubocop: disable Metrics/AbcSize rubocop: disable CodeReuse/ActiveRecord


68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/gitlab/usage_data.rb', line 68

def system_usage_data
  issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id)

  {
    counts: {
      assignee_lists: count(List.assignee),
      boards: count(Board),
      ci_builds: count(::Ci::Build),
      ci_internal_pipelines: count(::Ci::Pipeline.internal),
      ci_external_pipelines: count(::Ci::Pipeline.external),
      ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source),
      ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source),
      ci_runners: count(::Ci::Runner),
      ci_triggers: count(::Ci::Trigger),
      ci_pipeline_schedules: count(::Ci::PipelineSchedule),
      auto_devops_enabled: count(::ProjectAutoDevops.enabled),
      auto_devops_disabled: count(::ProjectAutoDevops.disabled),
      deploy_keys: count(DeployKey),
      # rubocop: disable UsageData/LargeTable:
      deployments: deployment_count(Deployment),
      successful_deployments: deployment_count(Deployment.success),
      failed_deployments: deployment_count(Deployment.failed),
      # rubocop: enable UsageData/LargeTable:
      environments: count(::Environment),
      clusters: count(::Clusters::Cluster),
      clusters_enabled: count(::Clusters::Cluster.enabled),
      project_clusters_enabled: count(::Clusters::Cluster.enabled.project_type),
      group_clusters_enabled: count(::Clusters::Cluster.enabled.group_type),
      instance_clusters_enabled: count(::Clusters::Cluster.enabled.instance_type),
      clusters_disabled: count(::Clusters::Cluster.disabled),
      project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type),
      group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type),
      instance_clusters_disabled: count(::Clusters::Cluster.disabled.instance_type),
      clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled),
      clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled),
      clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled),
      clusters_applications_helm: count(::Clusters::Applications::Helm.available),
      clusters_applications_ingress: count(::Clusters::Applications::Ingress.available),
      clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.available),
      clusters_applications_crossplane: count(::Clusters::Applications::Crossplane.available),
      clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.available),
      clusters_applications_runner: count(::Clusters::Applications::Runner.available),
      clusters_applications_knative: count(::Clusters::Applications::Knative.available),
      clusters_applications_elastic_stack: count(::Clusters::Applications::ElasticStack.available),
      clusters_applications_jupyter: count(::Clusters::Applications::Jupyter.available),
      clusters_applications_cilium: count(::Clusters::Applications::Cilium.available),
      clusters_management_project: count(::Clusters::Cluster.with_management_project),
      kubernetes_agents: count(::Clusters::Agent),
      kubernetes_agents_with_token: distinct_count(::Clusters::AgentToken, :agent_id),
      in_review_folder: count(::Environment.in_review_folder),
      grafana_integrated_projects: count(GrafanaIntegration.enabled),
      groups: count(Group),
      issues: count(Issue, start: issue_minimum_id, finish: issue_maximum_id),
      issues_created_from_gitlab_error_tracking_ui: count(SentryIssue),
      issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
      issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
      issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data,
      issues_created_from_alerts: total_alert_issues,
      issues_created_gitlab_alerts: issues_created_manually_from_alerts,
      issues_created_manually_from_alerts: issues_created_manually_from_alerts,
      incident_issues: count(::Issue.incident, start: issue_minimum_id, finish: issue_maximum_id),
      alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id),
      incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: issue_minimum_id, finish: issue_maximum_id),
      keys: count(Key),
      label_lists: count(List.label),
      lfs_objects: count(LfsObject),
      milestone_lists: count(List.milestone),
      milestones: count(Milestone),
      projects_with_packages: distinct_count(::Packages::Package, :project_id),
      packages: count(::Packages::Package),
      pages_domains: count(PagesDomain),
      pool_repositories: count(PoolRepository),
      projects: count(Project),
      projects_imported_from_github: count(Project.where(import_type: 'github')),
      projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)),
      projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)),
      projects_with_alerts_service_enabled: count(AlertsService.active),
      projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id),
      projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id),
      projects_with_terraform_states: distinct_count(::Terraform::State, :project_id),
      protected_branches: count(ProtectedBranch),
      protected_branches_except_default: count(ProtectedBranch.where.not(name: ['main', 'master', Gitlab::CurrentSettings.default_branch_name])),
      releases: count(Release),
      remote_mirrors: count(RemoteMirror),
      personal_snippets: count(PersonalSnippet),
      project_snippets: count(ProjectSnippet),
      suggestions: count(Suggestion),
      terraform_reports: count(::Ci::JobArtifact.terraform_reports),
      terraform_states: count(::Terraform::State),
      todos: count(Todo),
      uploads: count(Upload),
      web_hooks: count(WebHook),
      labels: count(Label),
      merge_requests: count(MergeRequest),
      notes: count(Note)
    }.merge(
      services_usage,
      usage_counters,
      user_preferences_usage,
      ingress_modsecurity_usage,
      container_expiration_policies_usage,
      service_desk_counts,
      snowplow_event_counts
    ).tap do |data|
      data[:snippets] = data[:personal_snippets] + data[:project_snippets]
    end
  }
end

.system_usage_data_monthly ⇒ Object 


191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
 
# File 'lib/gitlab/usage_data.rb', line 191

def system_usage_data_monthly
  {
    counts_monthly: {
      # rubocop: disable UsageData/LargeTable:
      deployments: deployment_count(Deployment.where(last_28_days_time_period)),
      successful_deployments: deployment_count(Deployment.success.where(last_28_days_time_period)),
      failed_deployments: deployment_count(Deployment.failed.where(last_28_days_time_period)),
      # rubocop: enable UsageData/LargeTable:
      packages: count(::Packages::Package.where(last_28_days_time_period)),
      personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)),
      project_snippets: count(ProjectSnippet.where(last_28_days_time_period))
    }.merge(
      snowplow_event_counts(time_period: last_28_days_time_period(column: :collector_tstamp))
    ).tap do |data|
      data[:snippets] = data[:personal_snippets] + data[:project_snippets]
    end
  }
end

.to_json(force_refresh: false) ⇒ Object 


46
47
48
 
# File 'lib/gitlab/usage_data.rb', line 46

def to_json(force_refresh: false)
  data(force_refresh: force_refresh).to_json
end

.topology_usage_data ⇒ Object 


332
333
334
 
# File 'lib/gitlab/usage_data.rb', line 332

def topology_usage_data
  Gitlab::UsageData::Topology.new.topology_usage_data
end

.uncached_data ⇒ Object 


25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/gitlab/usage_data.rb', line 25

def uncached_data
  clear_memoized

  with_finished_at(:recording_ce_finished_at) do
    license_usage_data
      .merge(system_usage_data)
      .merge(system_usage_data_monthly)
      .merge(features_usage_data)
      .merge(components_usage_data)
      .merge(cycle_analytics_usage_data)
      .merge(object_store_usage_data)
      .merge(topology_usage_data)
      .merge(usage_activity_by_stage)
      .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, last_28_days_time_period))
      .merge(analytics_unique_visits_data)
      .merge(compliance_unique_visits_data)
      .merge(search_unique_visits_data)
      .merge(redis_hll_counters)
  end
end

.usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {}) ⇒ Object

Source: gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv


476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
 
# File 'lib/gitlab/usage_data.rb', line 476

def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {})
  {
    key => {
      configure: usage_activity_by_stage_configure(time_period),
      create: usage_activity_by_stage_create(time_period),
      manage: usage_activity_by_stage_manage(time_period),
      monitor: usage_activity_by_stage_monitor(time_period),
      package: usage_activity_by_stage_package(time_period),
      plan: usage_activity_by_stage_plan(time_period),
      release: usage_activity_by_stage_release(time_period),
      secure: usage_activity_by_stage_secure(time_period),
      verify: usage_activity_by_stage_verify(time_period)
    }
  }
end

.usage_activity_by_stage_configure(time_period) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord rubocop: disable UsageData/LargeTable


494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
 
# File 'lib/gitlab/usage_data.rb', line 494

def usage_activity_by_stage_configure(time_period)
  {
    clusters_applications_cert_managers: cluster_applications_user_distinct_count(::Clusters::Applications::CertManager, time_period),
    clusters_applications_helm: cluster_applications_user_distinct_count(::Clusters::Applications::Helm, time_period),
    clusters_applications_ingress: cluster_applications_user_distinct_count(::Clusters::Applications::Ingress, time_period),
    clusters_applications_knative: cluster_applications_user_distinct_count(::Clusters::Applications::Knative, time_period),
    clusters_management_project: clusters_user_distinct_count(::Clusters::Cluster.with_management_project, time_period),
    clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled, time_period),
    clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled, time_period),
    clusters_platforms_gke: clusters_user_distinct_count(::Clusters::Cluster.gcp_installed.enabled, time_period),
    clusters_platforms_eks: clusters_user_distinct_count(::Clusters::Cluster.aws_installed.enabled, time_period),
    clusters_platforms_user: clusters_user_distinct_count(::Clusters::Cluster.user_provided.enabled, time_period),
    instance_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.instance_type, time_period),
    instance_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.instance_type, time_period),
    group_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.group_type, time_period),
    group_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.group_type, time_period),
    project_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.project_type, time_period),
    project_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.project_type, time_period)
  }
end

.usage_activity_by_stage_create(time_period) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord


518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
 
# File 'lib/gitlab/usage_data.rb', line 518

def usage_activity_by_stage_create(time_period)
  {
    deploy_keys: distinct_count(::DeployKey.where(time_period), :user_id),
    keys: distinct_count(::Key.regular_keys.where(time_period), :user_id),
    merge_requests: distinct_count(::MergeRequest.where(time_period), :author_id),
    projects_with_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: true))),
    projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))),
    remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id),
    snippets: distinct_count(::Snippet.where(time_period), :author_id)
  }.tap do |h|
    if time_period.present?
      h[:merge_requests_users] = merge_requests_users(time_period)
      h.merge!(action_monthly_active_users(time_period))
    end
  end
end

.usage_activity_by_stage_manage(time_period) ⇒ Object

Omitted because no user, creator or author associated: `campaigns_imported_from_github`, `ldap_group_links` rubocop: disable CodeReuse/ActiveRecord


538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
 
# File 'lib/gitlab/usage_data.rb', line 538

def usage_activity_by_stage_manage(time_period)
  {
    events: distinct_count(::Event.where(time_period), :author_id),
    groups: distinct_count(::GroupMember.where(time_period), :user_id),
    users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id),
    omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' },
    projects_imported: {
      gitlab_project: projects_imported_count('gitlab_project', time_period),
      gitlab: projects_imported_count('gitlab', time_period),
      github: projects_imported_count('github', time_period),
      bitbucket: projects_imported_count('bitbucket', time_period),
      bitbucket_server: projects_imported_count('bitbucket_server', time_period),
      gitea: projects_imported_count('gitea', time_period),
      git: projects_imported_count('git', time_period),
      manifest: projects_imported_count('manifest', time_period)
    },
    issues_imported: {
      jira: distinct_count(::JiraImportState.where(time_period), :user_id),
      fogbugz: projects_imported_count('fogbugz', time_period),
      phabricator: projects_imported_count('phabricator', time_period)
    }
  }
end

.usage_activity_by_stage_monitor(time_period) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord


564
565
566
567
568
569
570
571
572
 
# File 'lib/gitlab/usage_data.rb', line 564

def usage_activity_by_stage_monitor(time_period)
  {
    clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id),
    clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period),
    operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period),
                                                  start: user_minimum_id,
                                                  finish: user_maximum_id)
  }
end

.usage_activity_by_stage_package(time_period) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord


576
577
578
579
580
 
# File 'lib/gitlab/usage_data.rb', line 576

def usage_activity_by_stage_package(time_period)
  {
    projects_with_packages: distinct_count(::Project.with_packages.where(time_period), :creator_id)
  }
end

.usage_activity_by_stage_plan(time_period) ⇒ Object

Omitted because no user, creator or author associated: `boards`, `labels`, `milestones`, `uploads` Omitted because too expensive: `epics_deepest_relationship_level` Omitted because of encrypted properties: `projects_jira_cloud_active`, `projects_jira_server_active` rubocop: disable CodeReuse/ActiveRecord


587
588
589
590
591
592
593
594
595
596
597
598
599
 
# File 'lib/gitlab/usage_data.rb', line 587

def usage_activity_by_stage_plan(time_period)
  {
    issues: distinct_count(::Issue.where(time_period), :author_id),
    notes: distinct_count(::Note.where(time_period), :author_id),
    projects: distinct_count(::Project.where(time_period), :creator_id),
    todos: distinct_count(::Todo.where(time_period), :author_id),
    service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period),
    service_desk_issues: count(::Issue.service_desk.where(time_period)),
    projects_jira_active: distinct_count(::Project.with_active_jira_services.where(time_period), :creator_id),
    projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_jira_services.with_jira_dvcs_cloud.where(time_period), :creator_id),
    projects_jira_dvcs_server_active: distinct_count(::Project.with_active_jira_services.with_jira_dvcs_server.where(time_period), :creator_id)
  }
end

.usage_activity_by_stage_release(time_period) ⇒ Object

Omitted because no user, creator or author associated: `environments`, `feature_flags`, `in_review_folder`, `pages_domains` rubocop: disable CodeReuse/ActiveRecord


604
605
606
607
608
609
610
611
 
# File 'lib/gitlab/usage_data.rb', line 604

def usage_activity_by_stage_release(time_period)
  {
    deployments: distinct_count(::Deployment.where(time_period), :user_id),
    failed_deployments: distinct_count(::Deployment.failed.where(time_period), :user_id),
    releases: distinct_count(::Release.where(time_period), :author_id),
    successful_deployments: distinct_count(::Deployment.success.where(time_period), :user_id)
  }
end

.usage_activity_by_stage_secure(time_period) ⇒ Object

Currently too complicated and to get reliable counts for these stats: container_scanning_jobs, dast_jobs, dependency_scanning_jobs, license_management_jobs, sast_jobs, secret_detection_jobs Once gitlab.com/gitlab-org/gitlab/merge_requests/17568 is merged, this might be doable


634
635
636
 
# File 'lib/gitlab/usage_data.rb', line 634

def usage_activity_by_stage_secure(time_period)
  {}
end

.usage_activity_by_stage_verify(time_period) ⇒ Object

Omitted because no user, creator or author associated: `ci_runners` rubocop: disable CodeReuse/ActiveRecord


616
617
618
619
620
621
622
623
624
625
626
627
628
 
# File 'lib/gitlab/usage_data.rb', line 616

def usage_activity_by_stage_verify(time_period)
  {
    ci_builds: distinct_count(::Ci::Build.where(time_period), :user_id),
    ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
    ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
    ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
    ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
    ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id),
    ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
    ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id),
    clusters_applications_runner: cluster_applications_user_distinct_count(::Clusters::Applications::Runner, time_period)
  }
end

.usage_counters ⇒ Hash<Symbol, Integer>

Returns:

  • (Hash<Symbol, Integer>) 

250
251
252
 
# File 'lib/gitlab/usage_data.rb', line 250

def usage_counters
  usage_data_counters.map { |counter| redis_usage_data(counter) }.reduce({}, :merge)
end

.usage_data_counters ⇒ Array<#totals>

Returns An array of objects that respond to `#totals`.

Returns:

  • (Array<#totals>) —

    An array of objects that respond to `#totals`


255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
 
# File 'lib/gitlab/usage_data.rb', line 255

def usage_data_counters
  [
    Gitlab::UsageDataCounters::WikiPageCounter,
    Gitlab::UsageDataCounters::WebIdeCounter,
    Gitlab::UsageDataCounters::NoteCounter,
    Gitlab::UsageDataCounters::SnippetCounter,
    Gitlab::UsageDataCounters::SearchCounter,
    Gitlab::UsageDataCounters::CycleAnalyticsCounter,
    Gitlab::UsageDataCounters::ProductivityAnalyticsCounter,
    Gitlab::UsageDataCounters::SourceCodeCounter,
    Gitlab::UsageDataCounters::MergeRequestCounter,
    Gitlab::UsageDataCounters::DesignsCounter,
    Gitlab::UsageDataCounters::KubernetesAgentCounter
  ]
end

.user_preferences_usage ⇒ Object

rubocop: enable UsageData/LargeTable rubocop: enable CodeReuse/ActiveRecord


447
448
449
 
# File 'lib/gitlab/usage_data.rb', line 447

def user_preferences_usage
  {} # augmented in EE
end