Class: Gitlab::Database::BatchCounter
- Inherits:
-
Object
- Object
- Gitlab::Database::BatchCounter
- Defined in:
- lib/gitlab/database/batch_counter.rb
Constant Summary collapse
- FALLBACK =
-1
- MIN_REQUIRED_BATCH_SIZE =
1_250
- DEFAULT_SUM_BATCH_SIZE =
1_000
- MAX_ALLOWED_LOOPS =
10_000
- SLEEP_TIME_IN_SECONDS =
10 msec sleep
0.01
- ALLOWED_MODES =
[:itself, :distinct].freeze
- FALLBACK_FINISH =
0
- OFFSET_BY_ONE =
1
- DEFAULT_DISTINCT_BATCH_SIZE =
Each query should take < 500ms gitlab.com/gitlab-org/gitlab/-/merge_requests/22705
10_000
- DEFAULT_BATCH_SIZE =
100_000
Instance Method Summary collapse
- #count(batch_size: nil, mode: :itself, start: nil, finish: nil) ⇒ Object
-
#initialize(relation, column: nil, operation: :count, operation_args: nil) ⇒ BatchCounter
constructor
A new instance of BatchCounter.
- #merge_results(results, object) ⇒ Object
- #transaction_open? ⇒ Boolean
- #unwanted_configuration?(finish, batch_size, start) ⇒ Boolean
Constructor Details
#initialize(relation, column: nil, operation: :count, operation_args: nil) ⇒ BatchCounter
Returns a new instance of BatchCounter.
19 20 21 22 23 24 |
# File 'lib/gitlab/database/batch_counter.rb', line 19 def initialize(relation, column: nil, operation: :count, operation_args: nil) @relation = relation @column = column || relation.primary_key @operation = operation @operation_args = operation_args end |
Instance Method Details
#count(batch_size: nil, mode: :itself, start: nil, finish: nil) ⇒ Object
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 |
# File 'lib/gitlab/database/batch_counter.rb', line 33 def count(batch_size: nil, mode: :itself, start: nil, finish: nil) raise 'BatchCount can not be run inside a transaction' if transaction_open? check_mode!(mode) # non-distinct have better performance batch_size ||= batch_size_for_mode_and_operation(mode, @operation) start = actual_start(start) finish = actual_finish(finish) raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 return FALLBACK if unwanted_configuration?(finish, batch_size, start) results = nil batch_start = start while batch_start < finish begin batch_end = [batch_start + batch_size, finish].min batch_relation = build_relation_batch(batch_start, batch_end, mode) results = merge_results(results, batch_relation.send(@operation, *@operation_args)) # rubocop:disable GitlabSecurity/PublicSend batch_start = batch_end rescue ActiveRecord::QueryCanceled => error # retry with a safe batch size & warmer cache if batch_size >= 2 * MIN_REQUIRED_BATCH_SIZE batch_size /= 2 else log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) return FALLBACK end end sleep(SLEEP_TIME_IN_SECONDS) end results end |
#merge_results(results, object) ⇒ Object
77 78 79 80 81 82 83 84 85 |
# File 'lib/gitlab/database/batch_counter.rb', line 77 def merge_results(results, object) return object unless results if object.is_a?(Hash) results.merge!(object) { |_, a, b| a + b } else results + object end end |
#transaction_open? ⇒ Boolean
73 74 75 |
# File 'lib/gitlab/database/batch_counter.rb', line 73 def transaction_open? @relation.connection.transaction_open? end |
#unwanted_configuration?(finish, batch_size, start) ⇒ Boolean
26 27 28 29 30 31 |
# File 'lib/gitlab/database/batch_counter.rb', line 26 def unwanted_configuration?(finish, batch_size, start) (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || start >= finish end |