Class: ClickHouse::SyncStrategies::BaseSyncStrategy

Inherits:
Object
  • Object
show all
Includes:
Gitlab::ExclusiveLeaseHelpers, Gitlab::Utils::StrongMemoize
Defined in:
app/services/click_house/sync_strategies/base_sync_strategy.rb

Direct Known Subclasses

AuditEventSyncStrategy

Constant Summary collapse

MAX_TTL =

the job is scheduled every 3 minutes and we will allow maximum 2.5 minutes runtime

2.5.minutes.to_i
MAX_RUNTIME = 
120.seconds
BATCH_SIZE = 
500
INSERT_BATCH_SIZE = 
5000

Constants included from Gitlab::ExclusiveLeaseHelpers

Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError

Instance Method Summary collapse

Methods included from Gitlab::ExclusiveLeaseHelpers

#in_lock

Instance Method Details

#execute ⇒ Object 



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'app/services/click_house/sync_strategies/base_sync_strategy.rb', line 15

def execute
  return { status: :disabled } unless enabled?

  metadata = { status: :processed }

  begin
    # Prevent parallel jobs
    in_lock(self.class.to_s, ttl: MAX_TTL, retries: 0) do
      loop { break unless next_batch }

      metadata.merge!(records_inserted: context.total_record_count,
        reached_end_of_table: context.no_more_records?)

      if context.last_processed_id
        ClickHouse::SyncCursor.update_cursor_for(model_class.table_name,
          context.last_processed_id)
      end
    end
  rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError
    # Skip retrying, just let the next worker to start after a few minutes
    metadata = { status: :skipped }
  end

  metadata
end