Class: GitlabQuality::TestTooling::ClickHouse::Client

Inherits:
Object
  • Object
show all
Defined in:
lib/gitlab_quality/test_tooling/click_house/client.rb

Constant Summary collapse

DEFAULT_BATCH_SIZE = 
100_000
LOG_PREFIX = 
"[ClickHouse]"

Instance Method Summary collapse

Constructor Details

#initialize(url:, database:, username: nil, password: nil, logger: ::Logger.new($stdout, level: 1)) ⇒ Client

Returns a new instance of Client.



15
16
17
18
19
20
21
 
# File 'lib/gitlab_quality/test_tooling/click_house/client.rb', line 15

def initialize(url:, database:, username: nil, password: nil, logger: ::Logger.new($stdout, level: 1))
  @url = url
  @database = database
  @username = username
  @password = password
  @logger = logger
end

Instance Method Details

#insert_json_data(table_name, data, batch_size: DEFAULT_BATCH_SIZE) ⇒ void

This method returns an undefined value.

Push data to ClickHouse

Parameters:

  • table_name (String)
  • data (Array<Hash>)
  • batch_size (Integer) (defaults to: DEFAULT_BATCH_SIZE)

Raises:

  • (ArgumentError) 


52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/gitlab_quality/test_tooling/click_house/client.rb', line 52

def insert_json_data(table_name, data, batch_size: DEFAULT_BATCH_SIZE) # rubocop:disable Metrics/AbcSize
  raise ArgumentError, "Expected data to be an Array, got #{data.class}" unless data.is_a?(Array)
  raise ArgumentError, "Expected all elements of array to be hashes" unless data.is_a?(Array) && data.all?(Hash)
  raise ArgumentError, "Expected data to not be empty" if data.empty?

  total_batches = (data.size.to_f / batch_size).ceil
  results = data.each_slice(batch_size).with_index.map do |batch, index|
    logger.debug("#{LOG_PREFIX} Pushing batch #{index + 1} of #{total_batches}")
    send_batch(table_name, batch)
  end
  logger.debug("#{LOG_PREFIX} Processed #{results.size} result batches")
  return if results.all? { |res| res[:success] }

  err = results
    .reject { |res| res[:success] }
    .map { |res| "batch_size: #{res[:count]}, err: #{res[:error]}" }
    .join("\n")
  raise "Failures detected when pushing data to ClickHouse, errors:\n#{err}"
end

#query(sql, format: "JSONEachRow") ⇒ Array, ...

Perform sql query

Parameters:

  • sql (String)
  • format (String) (defaults to: "JSONEachRow")

Returns:

  • (Array, Hash, String) 


28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/gitlab_quality/test_tooling/click_house/client.rb', line 28

def query(sql, format: "JSONEachRow")
  logger.debug("Running #{sql}")
  response = post(
    body: sql,
    content_type: "text/plain",
    query_opts: { default_format: format }
  )
  raise "ClickHouse query failed: code: #{response.code}, error: #{response.body}" if response.code != 200

  if format == "JSONEachRow"
    response.body.split("\n").map { |row| JSON.parse(row) }
  elsif %w[JSON JSONCompact].include?(format)
    JSON.parse(response.body.presence || "{}")
  else
    response.body
  end
end