Module: Gitlab::UsageDataCounters::HLLRedisCounter

Extended by:
Gitlab::Usage::TimeFrame, Gitlab::Utils::UsageData
Defined in:
lib/gitlab/usage_data_counters/hll_redis_counter.rb

Constant Summary collapse

DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH =
6.weeks
DEFAULT_DAILY_KEY_EXPIRY_LENGTH =
29.days
DEFAULT_REDIS_SLOT =
''
EventError =
Class.new(StandardError)
UnknownEvent =
Class.new(EventError)
UnknownAggregation =
Class.new(EventError)
AggregationMismatch =
Class.new(EventError)
SlotMismatch =
Class.new(EventError)
CategoryMismatch =
Class.new(EventError)
InvalidContext =
Class.new(EventError)
KNOWN_EVENTS_PATH =
File.expand_path('known_events/*.yml', __dir__)
ALLOWED_AGGREGATIONS =
%i(daily weekly).freeze
CATEGORIES_FOR_TOTALS =
%w[
  analytics
  code_review
  compliance
  deploy_token_packages
  ecosystem
  epic_boards_usage
  epics_usage
  error_tracking
  ide_edit
  incident_management
  issues_edit
  pipeline_authoring
  quickactions
  search
  testing
  user_packages
].freeze

Constants included from Gitlab::Usage::TimeFrame

Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME, Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME, Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME

Constants included from Gitlab::Utils::UsageData

Gitlab::Utils::UsageData::DISTRIBUTED_HLL_FALLBACK, Gitlab::Utils::UsageData::FALLBACK, Gitlab::Utils::UsageData::HISTOGRAM_FALLBACK, Gitlab::Utils::UsageData::MAX_BUCKET_SIZE

Class Method Summary collapse

Methods included from Gitlab::Usage::TimeFrame

monthly_time_range, monthly_time_range_db_params, weekly_time_range

Methods included from Gitlab::Utils::UsageData

add, add_metric, alt_usage_data, count, distinct_count, epics_deepest_relationship_level, estimate_batch_distinct_count, histogram, jira_integration_data, maximum_id, measure_duration, minimum_id, redis_usage_data, sum, track_usage_event, with_duration, with_finished_at, with_prometheus_client

Methods included from Gitlab::Utils::StrongMemoize

#clear_memoization, #strong_memoize, #strong_memoized?

Class Method Details

.calculate_events_union(event_names:, start_date:, end_date:) ⇒ Object


142
143
144
145
146
147
# File 'lib/gitlab/usage_data_counters/hll_redis_counter.rb', line 142

def calculate_events_union(event_names:, start_date:, end_date:)
  count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events|
    raise SlotMismatch, events unless events_in_same_slot?(events)
    raise AggregationMismatch, events unless events_same_aggregation?(events)
  end
end

.categoriesObject


99
100
101
# File 'lib/gitlab/usage_data_counters/hll_redis_counter.rb', line 99

def categories
  @categories ||= known_events.map { |event| event[:category] }.uniq
end

.events_for_category(category) ⇒ Array<String>

Returns list of event names for given category.

Parameters:

  • category (String)

    the category name

Returns:

  • (Array<String>)

    list of event names for given category


105
106
107
# File 'lib/gitlab/usage_data_counters/hll_redis_counter.rb', line 105

def events_for_category(category)
  known_events.select { |event| event[:category] == category.to_s }.map { |event| event[:name] }
end

.known_event?(event_name) ⇒ Boolean

Returns:

  • (Boolean)

134
135
136
# File 'lib/gitlab/usage_data_counters/hll_redis_counter.rb', line 134

def known_event?(event_name)
  event_for(event_name).present?
end

.known_eventsObject


138
139
140
# File 'lib/gitlab/usage_data_counters/hll_redis_counter.rb', line 138

def known_events
  @known_events ||= load_events(KNOWN_EVENTS_PATH)
end

.track_event(event_name, values:, time: Time.current) ⇒ Object

Track unique events

event_name - The event name. values - One or multiple values counted. time - Time of the action, set to Time.current.


67
68
69
# File 'lib/gitlab/usage_data_counters/hll_redis_counter.rb', line 67

def track_event(event_name, values:, time: Time.current)
  track(values, event_name, time: time)
end

.track_event_in_context(event_name, values:, context:, time: Time.zone.now) ⇒ Object

Track unique events

event_name - The event name. values - One or multiple values counted. context - Event context, plan level tracking. time - Time of the action, set to Time.current.


77
78
79
80
81
82
# File 'lib/gitlab/usage_data_counters/hll_redis_counter.rb', line 77

def track_event_in_context(event_name, values:, context:, time: Time.zone.now)
  return if context.blank?
  return unless context.in?(valid_context_list)

  track(values, event_name, context: context, time: time)
end

.unique_events(event_names:, start_date:, end_date:, context: '') ⇒ Object

Count unique events for a given time range.

event_names - The list of the events to count. start_date - The start date of the time range. end_date - The end date of the time range. context - Event context, plan level tracking. Available if set when tracking.


90
91
92
93
94
95
96
97
# File 'lib/gitlab/usage_data_counters/hll_redis_counter.rb', line 90

def unique_events(event_names:, start_date:, end_date:, context: '')
  count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events|
    raise SlotMismatch, events unless events_in_same_slot?(events)
    raise CategoryMismatch, events unless events_in_same_category?(events)
    raise AggregationMismatch, events unless events_same_aggregation?(events)
    raise InvalidContext if context.present? && !context.in?(valid_context_list)
  end
end

.unique_events_dataObject

Recent 7 or 28 days unique events data for events defined in /lib/gitlab/usage_data_counters/known_events/

- For metrics for which we store a key per day, we have the last 7 days or last 28 days of data.
- For metrics for which we store a key per week, we have the last complete week or last 4 complete weeks
  daily or weekly information is in the file we have for events definition /lib/gitlab/usage_data_counters/known_events/
- Most of the metrics have weekly aggregation. We recommend this as it generates fewer keys in Redis to store.
- The aggregation used doesn't affect data granulation.

116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
# File 'lib/gitlab/usage_data_counters/hll_redis_counter.rb', line 116

def unique_events_data
  categories.each_with_object({}) do |category, category_results|
    events_names = events_for_category(category)

    event_results = events_names.each_with_object({}) do |event, hash|
      hash["#{event}_weekly"] = unique_events(**weekly_time_range.merge(event_names: [event])) unless event == "i_package_composer_deploy_token"
      hash["#{event}_monthly"] = unique_events(**monthly_time_range.merge(event_names: [event]))
    end

    if eligible_for_totals?(events_names) && CATEGORIES_FOR_TOTALS.include?(category)
      event_results["#{category}_total_unique_counts_weekly"] = unique_events(**weekly_time_range.merge(event_names: events_names))
      event_results["#{category}_total_unique_counts_monthly"] = unique_events(**monthly_time_range.merge(event_names: events_names))
    end

    category_results["#{category}"] = event_results
  end
end