Class: GitLab::Exporter::Database::ZoektCollector

Inherits:
Base
  • Object
show all
Defined in:
lib/gitlab_exporter/database/zoekt.rb

Overview

A helper class to collect zoekt metrics. rubocop:disable Metrics/ClassLength

Constant Summary collapse

ZOEKT_TASKS_PROCESSING_QUERY =

Query to get processing zoekt_tasks distribution by zoekt_node_id

"WITH task_counts AS (\n  SELECT\n    zoekt_node_id,\n    COUNT(*) AS count\n  FROM\n    zoekt_tasks\n  WHERE\n    perform_at <= $1\n    AND state IN (0, 1)\n  GROUP BY\n    zoekt_node_id\n)\nSELECT\n  n.id AS node_id,\n  n.metadata ->> 'name' AS node_name,\n  COALESCE(tc.count, 0) AS task_count\nFROM\n  zoekt_nodes n\nLEFT JOIN\n  task_counts tc ON n.id = tc.zoekt_node_id\n".freeze
ZOEKT_NODES_QUERY = 
"SELECT schema_version, id, metadata ->> 'name' AS node_name FROM zoekt_nodes\n".freeze
ZOEKT_NODES_STATUS_QUERY = 
"SELECT id, metadata ->> 'name' AS node_name,\n  CASE\n    WHEN last_seen_at < NOW() - INTERVAL '2 minute' THEN 0\n    ELSE 1\n  END AS status\nFROM zoekt_nodes;\n".freeze
ZOEKT_REPOSITORY_SCHEMA_VERSION_QUERY = 
"SELECT\n  COUNT(*)\nFROM\n  zoekt_repositories\nINNER JOIN\n  zoekt_indices\n  ON zoekt_repositories.zoekt_index_id = zoekt_indices.id\nWHERE\n  zoekt_indices.zoekt_node_id = $1\n  AND zoekt_repositories.schema_version < $2\n  AND zoekt_repositories.state = 10;\n".freeze
ZOEKT_ENABLED_QUERY = 
"SELECT\n  zoekt_settings ->> 'zoekt_indexing_enabled' AS zoekt_indexing_enabled\nFROM application_settings\nORDER BY ID DESC\nLIMIT 1\n".freeze
ZOEKT_NODE_STORAGE_QUERY =

unclaimed_storage_bytes and storage_percent_used are not real columns on zoekt_nodes; they are derived in Search::Zoekt::Node. We compute them in SQL here to avoid loading every row through ActiveRecord.

"SELECT\n  zn.id,\n  zn.metadata ->> 'name' AS node_name,\n  (zn.usable_storage_bytes - COALESCE(zi.reserved_total, 0))::bigint\n    AS unclaimed_storage_bytes,\n  CASE\n    WHEN zn.total_bytes > 0\n      THEN zn.used_bytes::float8 / zn.total_bytes::float8\n    ELSE 0\n  END AS storage_percent_used\nFROM zoekt_nodes zn\nLEFT JOIN (\n  SELECT zoekt_node_id, SUM(reserved_storage_bytes) AS reserved_total\n  FROM zoekt_indices\n  GROUP BY zoekt_node_id\n) zi ON zi.zoekt_node_id = zn.id\n".freeze
ZOEKT_REPOSITORIES_STATE_QUERY = 
"SELECT state, COUNT(*) FROM zoekt_repositories GROUP BY state\n".freeze
ZOEKT_INDICES_STATE_QUERY = 
"SELECT state, COUNT(*) FROM zoekt_indices GROUP BY state\n".freeze
ZOEKT_INDICES_WATERMARK_QUERY = 
"SELECT watermark_level, COUNT(*) FROM zoekt_indices GROUP BY watermark_level\n".freeze
ZOEKT_INDICES_STORAGE_QUERY = 
"SELECT\n  zoekt_indices.id,\n  zoekt_indices.reserved_storage_bytes,\n  zoekt_indices.used_storage_bytes,\n  zoekt_nodes.metadata ->> 'name' AS node_name\nFROM\n  zoekt_indices\nINNER JOIN\n  zoekt_nodes ON zoekt_indices.zoekt_node_id = zoekt_nodes.id\n".freeze
ZOEKT_NODE_ENABLED_NAMESPACES_QUERY = 
"SELECT\n  zn.id AS node_id,\n  zn.metadata ->> 'name' AS node_name,\n  COUNT(DISTINCT zi.zoekt_enabled_namespace_id) AS count\nFROM zoekt_nodes zn\nLEFT JOIN zoekt_indices zi ON zi.zoekt_node_id = zn.id\nGROUP BY zn.id, zn.metadata ->> 'name'\n".freeze
ZOEKT_NODE_TASKS_QUERY = 
"SELECT\n  zn.id AS node_id,\n  zn.metadata ->> 'name' AS node_name,\n  zt.state,\n  COUNT(*) AS count\nFROM zoekt_nodes zn\nLEFT JOIN zoekt_tasks zt ON zt.zoekt_node_id = zn.id\nGROUP BY zn.id, zn.metadata ->> 'name', zt.state\n".freeze
ZOEKT_INDICES_STALE_USED_STORAGE_QUERY = 
"SELECT COUNT(*) AS count FROM zoekt_indices WHERE last_indexed_at >= used_storage_bytes_updated_at\n".freeze
ZOEKT_TASK_STATES = 
{
  0 => "pending",
  1 => "processing",
  10 => "done",
  250 => "skipped",
  255 => "failed",
  256 => "orphaned"
}.freeze

Constants inherited from Base

Base::POOL_SIZE, Base::POOL_TIMEOUT

Instance Method Summary collapse

Methods inherited from Base

configure_type_map_for_results, connection_pool, #connection_pool, #initialize, #with_connection_pool

Constructor Details

This class inherits a constructor from GitLab::Exporter::Database::Base

Instance Method Details

#run ⇒ Object 



146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/gitlab_exporter/database/zoekt.rb', line 146

def run
  return {} unless zoekt_indexing_enabled?

  {
    task_processing_query_result: execute(ZOEKT_TASKS_PROCESSING_QUERY, [Time.now.utc]),
    repositories_schema_version_query_result: repositories_schema_version_query_result,
    zoekt_nodes_status_query_result: execute(ZOEKT_NODES_STATUS_QUERY),
    node_storage_query_result: execute(ZOEKT_NODE_STORAGE_QUERY),
    repositories_state_query_result: execute(ZOEKT_REPOSITORIES_STATE_QUERY),
    indices_state_query_result: execute(ZOEKT_INDICES_STATE_QUERY),
    indices_watermark_query_result: execute(ZOEKT_INDICES_WATERMARK_QUERY),
    indices_storage_query_result: execute(ZOEKT_INDICES_STORAGE_QUERY),
    node_enabled_namespaces_query_result: execute(ZOEKT_NODE_ENABLED_NAMESPACES_QUERY),
    node_tasks_query_result: execute(ZOEKT_NODE_TASKS_QUERY),
    indices_stale_used_storage_query_result: execute(ZOEKT_INDICES_STALE_USED_STORAGE_QUERY)
  }.compact
end