Module: Gitlab::Database::PartitioningMigrationHelpers::IndexHelpers

Includes:
MigrationHelpers, SchemaHelpers
Included in:
Gitlab::Database::PartitioningMigrationHelpers
Defined in:
lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb

Constant Summary collapse

DuplicatedIndexesError = 
Class.new(StandardError)
ERROR_SCOPE = 
'index'

Constants included from MigrationHelpers

MigrationHelpers::DEFAULT_TIMESTAMP_COLUMNS

Constants included from DynamicModelHelpers

DynamicModelHelpers::BATCH_SIZE

Constants included from Migrations::RedisHelpers

Migrations::RedisHelpers::SCAN_START_CURSOR

Constants included from Migrations::SidekiqHelpers

Migrations::SidekiqHelpers::DEFAULT_MAX_ATTEMPTS, Migrations::SidekiqHelpers::DEFAULT_TIMES_IN_A_ROW

Constants included from Migrations::ConstraintsHelpers

Migrations::ConstraintsHelpers::MAX_IDENTIFIER_NAME_LENGTH

Constants included from Migrations::BatchedBackgroundMigrationHelpers

Migrations::BatchedBackgroundMigrationHelpers::BATCH_CLASS_NAME, Migrations::BatchedBackgroundMigrationHelpers::BATCH_MIN_DELAY, Migrations::BatchedBackgroundMigrationHelpers::BATCH_MIN_VALUE, Migrations::BatchedBackgroundMigrationHelpers::BATCH_SIZE, Migrations::BatchedBackgroundMigrationHelpers::NonExistentMigrationError, Migrations::BatchedBackgroundMigrationHelpers::SUB_BATCH_SIZE

Constants included from Migrations::BackgroundMigrationHelpers

Migrations::BackgroundMigrationHelpers::BATCH_SIZE, Migrations::BackgroundMigrationHelpers::JOB_BUFFER_SIZE

Instance Method Summary collapse

Methods included from SchemaHelpers

#assert_not_in_transaction_block, #create_comment, #create_trigger, #create_trigger_function, #drop_function, #drop_trigger, #function_exists?, #object_name, #tmp_table_name, #trigger_exists?

Methods included from MigrationHelpers

#add_concurrent_foreign_key, #add_concurrent_index, #add_primary_key_using_index, #add_sequence, #add_timestamps_with_timezone, #backfill_conversion_of_integer_to_bigint, #backfill_iids, #change_column_type_concurrently, #check_trigger_permissions!, #cleanup_concurrent_column_rename, #cleanup_concurrent_column_type_change, #column_for, #concurrent_foreign_key_name, #convert_to_bigint_column, #convert_to_type_column, #copy_foreign_keys, #copy_indexes, #create_or_update_plan_limit, #create_temporary_columns_and_triggers, #define_batchable_model, #drop_sequence, #each_batch, #each_batch_range, #false_value, #foreign_key_exists?, #foreign_keys_for, #index_exists_by_name?, #index_invalid?, #indexes_for, #initialize_conversion_of_integer_to_bigint, #install_rename_triggers, #partition?, #postgres_exists_by_name?, #remove_column_default, #remove_concurrent_index, #remove_concurrent_index_by_name, #remove_foreign_key_if_exists, #remove_foreign_key_without_error, #remove_rename_triggers, #remove_timestamps, #rename_column_concurrently, #rename_trigger_name, #replace_sql, #restore_conversion_of_integer_to_bigint, #revert_backfill_conversion_of_integer_to_bigint, #revert_initialize_conversion_of_integer_to_bigint, #swap_primary_key, #table_partitioned?, #true_value, #undo_change_column_type_concurrently, #undo_cleanup_concurrent_column_rename, #undo_cleanup_concurrent_column_type_change, #undo_rename_column_concurrently, #update_column_in_batches, #validate_foreign_key

Methods included from MigrationHelpers::WraparoundVacuumHelpers

#check_if_wraparound_in_progress

Methods included from AsyncConstraints::MigrationHelpers

#prepare_async_check_constraint_validation, #prepare_async_foreign_key_validation, #prepare_partitioned_async_foreign_key_validation, #unprepare_async_check_constraint_validation, #unprepare_async_foreign_key_validation, #unprepare_partitioned_async_foreign_key_validation

Methods included from AsyncIndexes::MigrationHelpers

#async_index_creation_available?, #prepare_async_index, #prepare_async_index_from_sql, #prepare_async_index_removal, #unprepare_async_index, #unprepare_async_index_by_name

Methods included from RenameTableHelpers

#finalize_table_rename, #rename_table_safely, #undo_finalize_table_rename, #undo_rename_table_safely

Methods included from DynamicModelHelpers

#define_batchable_model, #each_batch, #each_batch_range

Methods included from Migrations::RedisHelpers

#queue_redis_migration_job

Methods included from Migrations::SidekiqHelpers

#sidekiq_queue_length, #sidekiq_queue_migrate, #sidekiq_remove_jobs

Methods included from Migrations::ExtensionHelpers

#create_extension, #drop_extension

Methods included from Migrations::ConstraintsHelpers

#add_check_constraint, #add_not_null_constraint, #add_text_limit, #check_constraint_exists?, check_constraint_exists?, #check_constraint_name, #check_not_null_constraint_exists?, #check_text_limit_exists?, #copy_check_constraints, #drop_constraint, #remove_check_constraint, #remove_not_null_constraint, #remove_text_limit, #rename_constraint, #switch_constraint_names, #text_limit_name, #validate_check_constraint, #validate_check_constraint_name!, #validate_not_null_constraint, #validate_text_limit

Methods included from Migrations::TimeoutHelpers

#disable_statement_timeout

Methods included from Migrations::LockRetriesHelpers

#with_lock_retries

Methods included from Migrations::BatchedBackgroundMigrationHelpers

#delete_batched_background_migration, #ensure_batched_background_migration_is_finished, #finalize_batched_background_migration, #gitlab_schema_from_context, #queue_batched_background_migration

Methods included from Migrations::BackgroundMigrationHelpers

#delete_job_tracking, #delete_queued_jobs, #finalize_background_migration, #migrate_in, #queue_background_migration_jobs_by_range_at_intervals, #requeue_background_migration_jobs_by_range_at_intervals

Methods included from Migrations::ReestablishedConnectionStack

#with_restored_connection_stack

Instance Method Details

#add_concurrent_partitioned_index(table_name, column_names, options = {}) ⇒ Object

Concurrently creates a new index on a partitioned table. In concept this works similarly to ‘add_concurrent_index`, and won’t block reads or writes on the table while the index is being built.

A special helper is required for partitioning because Postgres does not support concurrently building indexes on partitioned tables. This helper concurrently adds the same index to each partition, and creates the final index on the parent table once all of the partitions are indexed. This is the recommended safe way to add indexes to partitioned tables.

Example:

add_concurrent_partitioned_index :users, :some_column

See Rails’ ‘add_index` for more info on the available arguments.

Raises:

  • (ArgumentError) 


27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 27

def add_concurrent_partitioned_index(table_name, column_names, options = {})
  assert_not_in_transaction_block(scope: ERROR_SCOPE)

  raise ArgumentError, 'A name is required for indexes added to partitioned tables' unless options[:name]

  partitioned_table = find_partitioned_table(table_name)

  if index_name_exists?(table_name, options[:name])
    Gitlab::AppLogger.warn "Index not created because it already exists (this may be due to an aborted" \
      " migration or similar): table_name: #{table_name}, index_name: #{options[:name]}"

    return
  end

  partitioned_table.postgres_partitions.order(:name).each do |partition|
    partition_index_name = generated_index_name(partition.identifier, options[:name])
    partition_options = options.merge(name: partition_index_name, allow_partition: true)

    add_concurrent_index(partition.identifier, column_names, partition_options)
  end

  with_lock_retries do
    add_index(table_name, column_names, **options)
  end
end

#find_duplicate_indexes(table_name, schema_name: connection.current_schema) ⇒ Object

Finds duplicate indexes for a given schema and table. This finds indexes where the index definition is identical but the names are different. Returns an array of arrays containing duplicate index name pairs.

Example:

find_duplicate_indexes('table_name_goes_here')



90
91
92
93
94
95
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 90

def find_duplicate_indexes(table_name, schema_name: connection.current_schema)
  find_indexes(table_name, schema_name: schema_name)
    .group_by { |r| r['index_id'] }
    .select { |_, v| v.size > 1 }
    .map { |_, indexes| indexes.map { |index| index['index_name'] } }
end

#indexes_by_definition_for_table(table_name, schema_name: connection.current_schema) ⇒ Object

Retrieves a hash of index names for a given table and schema, by index definition.

Example:

indexes_by_definition_for_table('table_name_goes_here')

Returns:

{
  "CREATE _ btree (created_at)" => "index_on_created_at"
}



109
110
111
112
113
114
115
116
117
118
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 109

def indexes_by_definition_for_table(table_name, schema_name: connection.current_schema)
  duplicate_indexes = find_duplicate_indexes(table_name, schema_name: schema_name)

  unless duplicate_indexes.empty?
    raise DuplicatedIndexesError, "#{table_name} has duplicate indexes: #{duplicate_indexes}"
  end

  find_indexes(table_name, schema_name: schema_name)
    .each_with_object({}) { |row, hash| hash[row['index_id']] = row['index_name'] }
end

#remove_concurrent_partitioned_index_by_name(table_name, index_name) ⇒ Object

Safely removes an existing index from a partitioned table. The method name is a bit inaccurate as it does not drop the index concurrently, but it’s named as such to maintain consistency with other similar helpers, and indicate that this should be safe to use in a production environment.

In current versions of Postgres it’s impossible to drop an index concurrently, or drop an index from an individual partition that exists across the entire partitioned table. As a result this helper drops the index from the parent table, which automatically cascades to all partitions. While this does require an exclusive lock, dropping an index is a fast operation that won’t block the table for a significant period of time.

Example:

remove_concurrent_partitioned_index_by_name :users, 'index_name_goes_here'



65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 65

def remove_concurrent_partitioned_index_by_name(table_name, index_name)
  assert_not_in_transaction_block(scope: ERROR_SCOPE)

  find_partitioned_table(table_name)

  unless index_name_exists?(table_name, index_name)
    Gitlab::AppLogger.warn "Index not removed because it does not exist (this may be due to an aborted " \
      "migration or similar): table_name: #{table_name}, index_name: #{index_name}"

    return
  end

  with_lock_retries do
    remove_index(table_name, name: index_name)
  end
end

#rename_indexes_for_table(table_name, new_index_names, schema_name: connection.current_schema) ⇒ Object

Renames indexes for a given table and schema, mapping by index definition, to a hash of new index names.

Example:

index_names = indexes_by_definition_for_table('source_table_name_goes_here')
drop_table('source_table_name_goes_here')
rename_indexes_for_table('destination_table_name_goes_here', index_names)



128
129
130
131
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 128

def rename_indexes_for_table(table_name, new_index_names, schema_name: connection.current_schema)
  current_index_names = indexes_by_definition_for_table(table_name, schema_name: schema_name)
  rename_indexes(current_index_names, new_index_names, schema_name: schema_name)
end