Class: GitLab::Exporter::Database::RowCountCollector

Inherits:
Base
  • Object
show all
Defined in:
lib/gitlab_exporter/database/row_count.rb

Overview

A helper class that executes the query its given and returns an int of the row count This class works under the assumption you do COUNT(*) queries, define queries in the QUERIES constant. If in doubt how these work, read #construct_query rubocop:disable Metrics/ClassLength

Constant Summary collapse

WHERE_MIRROR_ENABLED =

We ignore mirrors with a next_execution_timestamp before 2020-03-28 because this is when we stopped processing mirrors for private projects on the free plan. Skipping those can significantly improve query performance: gitlab.com/gitlab-org/gitlab/-/issues/216252#note_334514544

"projects.mirror = true\nAND projects.archived = false\nAND project_mirror_data.retry_count <= 14\nAND (\n  (projects.visibility_level = 20 AND root_namespaces.visibility_level = 20)\n  OR\n  plans.name IN ('early_adopter', 'bronze', 'silver', 'gold')\n)\nAND project_mirror_data.next_execution_timestamp > '2020-03-28'\n".freeze
MIRROR_QUERY = 
{
  select: :projects,
  joins: "    INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id\n    INNER JOIN namespaces AS root_namespaces ON root_namespaces.id = (\n      WITH RECURSIVE \"base_and_ancestors\" AS (\n        (SELECT \"namespaces\".* FROM \"namespaces\" WHERE \"namespaces\".\"id\" = projects.namespace_id)\n        UNION\n        (SELECT \"namespaces\".* FROM \"namespaces\", \"base_and_ancestors\" WHERE \"namespaces\".\"id\" = \"base_and_ancestors\".\"parent_id\")\n      ) SELECT \"namespaces\".id FROM \"base_and_ancestors\" AS \"namespaces\" WHERE \"namespaces\".\"parent_id\" IS NULL\n    )\n    LEFT JOIN gitlab_subscriptions ON gitlab_subscriptions.namespace_id = root_namespaces.id\n    LEFT JOIN plans ON plans.id = gitlab_subscriptions.hosted_plan_id\n  SQL\n  check: \"SELECT 1 FROM information_schema.tables WHERE table_name='plans'\"\n}.freeze\n",
QUERIES = 
{
  mirrors_ready_to_sync: MIRROR_QUERY.merge( # EE only
    where: "      \#{WHERE_MIRROR_ENABLED}\n      AND project_mirror_data.status NOT IN ('scheduled', 'started')\n      AND project_mirror_data.next_execution_timestamp <= NOW()\n    SQL\n  ),\n  mirrors_not_updated_recently: MIRROR_QUERY.merge( # EE only\n    where: <<~SQL\n      \#{WHERE_MIRROR_ENABLED}\n      AND project_mirror_data.status NOT IN ('scheduled', 'started')\n      AND (project_mirror_data.next_execution_timestamp - project_mirror_data.last_update_at) <= '30 minutes'::interval\n      AND project_mirror_data.last_update_at < NOW() - '30 minutes'::interval\n    SQL\n  ),\n  mirrors_updated_very_recently: MIRROR_QUERY.merge( # EE only\n    where: <<~SQL\n      \#{WHERE_MIRROR_ENABLED}\n      AND project_mirror_data.status NOT IN ('scheduled', 'started')\n      AND project_mirror_data.last_update_at >= NOW() - '30 seconds'::interval\n    SQL\n  ),\n  mirrors_behind_schedule: MIRROR_QUERY.merge( # EE only\n    where: <<~SQL\n      \#{WHERE_MIRROR_ENABLED}\n      AND project_mirror_data.status NOT IN ('scheduled', 'started')\n      AND project_mirror_data.next_execution_timestamp <= NOW() - '10 seconds'::interval\n    SQL\n  ),\n  mirrors_scheduled_or_started: MIRROR_QUERY.merge( # EE only\n    where: <<~SQL\n      \#{WHERE_MIRROR_ENABLED}\n      AND project_mirror_data.status IN ('scheduled', 'started')\n    SQL\n  ),\n  mirrors_scheduled: MIRROR_QUERY.merge( # EE only\n    where: <<~SQL\n      \#{WHERE_MIRROR_ENABLED}\n      AND project_mirror_data.status = 'scheduled'\n    SQL\n  ),\n  mirrors_started: MIRROR_QUERY.merge( # EE only\n    where: <<~SQL\n      \#{WHERE_MIRROR_ENABLED}\n      AND project_mirror_data.status = 'started'\n    SQL\n  ),\n  soft_deleted_projects: { select: :projects, where: \"pending_delete=true\" },\n  orphaned_projects: {\n    select: :projects,\n    joins: \"LEFT JOIN namespaces ON projects.namespace_id = namespaces.id\",\n    where: \"namespaces.id IS NULL\"\n  },\n  uploads: { select: :uploads },\n  users: {\n    select: :users,\n    joins: \"LEFT JOIN\n      (\n        SELECT\n          members.user_id,\n          MAX(access_level) as access_level\n        FROM members\n        GROUP BY members.user_id\n      ) AS u\n      ON users.id = u.user_id\",\n    where: \"user_type IS NULL\",\n    fields: {\n      admin: {},\n      external: {},\n      state: {},\n      access_level: { definition: \"COALESCE(u.access_level, 0)\" }\n    }\n  },\n  projects: {\n    select: :projects,\n    fields: {\n      visibility_level: {},\n      archived: {}\n    }\n  },\n  groups: {\n    select: :namespaces,\n    fields: {\n      visibility_level: {},\n      root: { definition: \"(parent_id IS NULL)\" }\n    }\n  }\n}.freeze\n"

Constants inherited from Base

Base::POOL_SIZE, Base::POOL_TIMEOUT

Instance Method Summary collapse

Methods inherited from Base

configure_type_map_for_results, connection_pool, #connection_pool, #with_connection_pool

Constructor Details

#initialize(selected_queries: nil, **args) ⇒ RowCountCollector 



137
138
139
140
141
 
# File 'lib/gitlab_exporter/database/row_count.rb', line 137

def initialize(selected_queries: nil, **args)
  super(**args)

  @selected_queries = Set.new(selected_queries.map(&:to_sym)) unless selected_queries.nil?
end

Instance Method Details

#run ⇒ Object 



143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/gitlab_exporter/database/row_count.rb', line 143

def run
  results = Hash.new(0)

  QUERIES.each do |key, query_hash|
    next if query_hash[:check] && !successful_check?(query_hash[:check])
    next if !@selected_queries.nil? && !@selected_queries.include?(key)

    results[key] = count_from_query_hash(query_hash)
  end

  results
end