Module: Gitlab::Database::PartitioningMigrationHelpers::IndexHelpers

Includes:
MigrationHelpers, MigrationHelpers::Swapping, SchemaHelpers
Included in:
Gitlab::Database::PartitioningMigrationHelpers
Defined in:
lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb

Constant Summary collapse

DuplicatedIndexesError = 
Class.new(StandardError)
ERROR_SCOPE = 
'index'

Constants included from MigrationHelpers

MigrationHelpers::DEFAULT_TIMESTAMP_COLUMNS, MigrationHelpers::ENFORCE_INITIALIZE_ALL_INT_IDS_FROM_MILESTONE, MigrationHelpers::INTEGER_IDS_YET_TO_INITIALIZED_TO_BIGINT_FILE_PATH, MigrationHelpers::PENDING_INT_IDS_ERROR_MSG, MigrationHelpers::TABLE_INT_IDS_YAML_FILE_COMMENT

Constants included from DynamicModelHelpers

DynamicModelHelpers::BATCH_SIZE

Constants included from Migrations::RedisHelpers

Migrations::RedisHelpers::SCAN_START_CURSOR

Constants included from Migrations::SidekiqHelpers

Migrations::SidekiqHelpers::DEFAULT_MAX_ATTEMPTS, Migrations::SidekiqHelpers::DEFAULT_TIMES_IN_A_ROW

Constants included from Migrations::ConstraintsHelpers

Migrations::ConstraintsHelpers::MAX_IDENTIFIER_NAME_LENGTH

Constants included from Migrations::BatchedBackgroundMigrationHelpers

Migrations::BatchedBackgroundMigrationHelpers::BATCH_CLASS_NAME, Migrations::BatchedBackgroundMigrationHelpers::BATCH_MIN_DELAY, Migrations::BatchedBackgroundMigrationHelpers::BATCH_MIN_VALUE, Migrations::BatchedBackgroundMigrationHelpers::BATCH_SIZE, Migrations::BatchedBackgroundMigrationHelpers::EARLY_FINALIZATION_ERROR, Migrations::BatchedBackgroundMigrationHelpers::ENFORCE_EARLY_FINALIZATION_FROM_VERSION, Migrations::BatchedBackgroundMigrationHelpers::MIGRATION_NOT_FOUND_MESSAGE, Migrations::BatchedBackgroundMigrationHelpers::MINIMUM_PAUSE_MS, Migrations::BatchedBackgroundMigrationHelpers::NonExistentMigrationError, Migrations::BatchedBackgroundMigrationHelpers::SUB_BATCH_SIZE

Instance Method Summary collapse

Methods included from MigrationHelpers::Swapping

#swap_columns, #swap_columns_default, #swap_foreign_keys, #swap_indexes

Methods included from MigrationHelpers::V2

#cleanup_concurrent_column_rename, #create_table, #rename_column_concurrently, #rename_index_with_schema, #truncate_tables!, #undo_cleanup_concurrent_column_rename, #undo_rename_column_concurrently, #with_lock_retries

Methods included from MigrationHelpers

#add_concurrent_index, #add_primary_key_using_index, #add_sequence, #add_timestamps_with_timezone, #backfill_conversion_of_integer_to_bigint, #backfill_iids, #change_column_type_concurrently, #check_trigger_permissions!, #cleanup_concurrent_column_rename, #cleanup_concurrent_column_type_change, #cleanup_conversion_of_integer_to_bigint, #column_for, #convert_to_bigint_column, #copy_foreign_keys, #copy_indexes, #create_or_update_plan_limit, #define_batchable_model, #drop_sequence, #each_batch, #each_batch_range, #ensure_backfill_conversion_of_integer_to_bigint_is_finished, #false_value, #foreign_keys_for, #index_exists_by_name?, #index_invalid?, #indexes_for, #initialize_conversion_of_integer_to_bigint, #install_rename_triggers, #install_sharding_key_assignment_trigger, #lock_tables, #remove_column_default, #remove_concurrent_index, #remove_concurrent_index_by_name, #remove_rename_triggers, #remove_sharding_key_assignment_trigger, #remove_timestamps, #rename_column_concurrently, #rename_trigger_name, #replace_sql, #restore_conversion_of_integer_to_bigint, #revert_backfill_conversion_of_integer_to_bigint, #revert_initialize_conversion_of_integer_to_bigint, #swap_primary_key, #table_integer_ids, #true_value, #undo_change_column_type_concurrently, #undo_cleanup_concurrent_column_rename, #undo_cleanup_concurrent_column_type_change, #undo_rename_column_concurrently, #update_column_in_batches

Methods included from Gitlab::Database::PartitionHelpers

#partition?, #table_partitioned?

Methods included from MigrationHelpers::WraparoundVacuumHelpers

#check_if_wraparound_in_progress

Methods included from AsyncConstraints::MigrationHelpers

#prepare_async_check_constraint_validation, #prepare_async_foreign_key_validation, #prepare_partitioned_async_check_constraint_validation, #prepare_partitioned_async_foreign_key_validation, #unprepare_async_check_constraint_validation, #unprepare_async_foreign_key_validation, #unprepare_partitioned_async_check_constraint_validation, #unprepare_partitioned_async_foreign_key_validation

Methods included from AsyncIndexes::MigrationHelpers

#async_index_creation_available?, #prepare_async_index, #prepare_async_index_from_sql, #prepare_async_index_removal, #unprepare_async_index, #unprepare_async_index_by_name

Methods included from RenameTableHelpers

#finalize_table_rename, #rename_table_safely, #undo_finalize_table_rename, #undo_rename_table_safely

Methods included from MigrationHelpers::FeatureFlagMigratorHelpers

#down_migrate_to_jsonb_setting, #down_migrate_to_setting, #up_migrate_to_jsonb_setting, #up_migrate_to_setting

Methods included from DynamicModelHelpers

define_batchable_model, #each_batch, #each_batch_range

Methods included from Migrations::ForeignKeyHelpers

#add_concurrent_foreign_key, #concurrent_foreign_key_name, #foreign_key_exists?, #remove_foreign_key_if_exists, #remove_foreign_key_without_error, #validate_foreign_key

Methods included from Migrations::LockRetriesHelpers

#with_lock_retries

Methods included from Migrations::TimeoutHelpers

#disable_statement_timeout

Methods included from Migrations::RedisHelpers

#queue_redis_migration_job

Methods included from Migrations::SidekiqHelpers

#migrate_across_instance, #migrate_within_instance, #sidekiq_queue_migrate, #sidekiq_remove_jobs

Methods included from Migrations::ExtensionHelpers

#create_extension, #drop_extension

Methods included from Migrations::ConstraintsHelpers

#add_check_constraint, #add_multi_column_not_null_constraint, #add_not_null_constraint, #add_text_limit, #check_constraint_exists?, check_constraint_exists?, #check_constraint_name, #check_not_null_constraint_exists?, #check_text_limit_exists?, #copy_check_constraints, #drop_constraint, #remove_check_constraint, #remove_multi_column_not_null_constraint, #remove_not_null_constraint, #remove_text_limit, #rename_constraint, #switch_constraint_names, #text_limit_name, #validate_check_constraint, #validate_check_constraint_name!, #validate_multi_column_not_null_constraint, #validate_not_null_constraint, #validate_text_limit

Methods included from Migrations::BatchedBackgroundMigrationHelpers

#delete_batched_background_migration, #ensure_batched_background_migration_is_finished, #finalize_batched_background_migration, #gitlab_schema_from_context, #queue_batched_background_migration

Methods included from Migrations::ReestablishedConnectionStack

#with_restored_connection_stack

Methods included from SchemaHelpers

#assert_not_in_transaction_block, #create_comment, #create_trigger, #create_trigger_function, #drop_function, #drop_trigger, #find_all_id_columns_sql, #function_exists?, #object_name, #reset_all_trigger_functions, #reset_trigger_function, #tmp_table_name, #trigger_exists?

Instance Method Details

#add_concurrent_partitioned_index(table_name, column_names, options = {}) ⇒ Object

Concurrently creates a new index on a partitioned table. In concept this works similarly to add_concurrent_index, and won't block reads or writes on the table while the index is being built.

A special helper is required for partitioning because Postgres does not support concurrently building indexes on partitioned tables. This helper concurrently adds the same index to each partition, and creates the final index on the parent table once all of the partitions are indexed. This is the recommended safe way to add indexes to partitioned tables.

Example:

add_concurrent_partitioned_index :users, :some_column

See Rails' add_index for more info on the available arguments.

Raises:



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 28

def add_concurrent_partitioned_index(table_name, column_names, options = {})
  assert_not_in_transaction_block(scope: ERROR_SCOPE)

  raise ArgumentError, 'A name is required for indexes added to partitioned tables' unless options[:name]

  partitioned_table = find_partitioned_table(table_name)

  if index_name_exists?(table_name, options[:name])
    Gitlab::AppLogger.warn "Index not created because it already exists (this may be due to an aborted " \
      "migration or similar): table_name: #{table_name}, index_name: #{options[:name]}"

    return
  end

  partitioned_table.postgres_partitions.order(:name).each do |partition|
    partition_index_name = generated_index_name(partition.identifier, options[:name])
    partition_options = options.merge(name: partition_index_name, allow_partition: true)

    add_concurrent_index(partition.identifier, column_names, partition_options)
  end

  with_lock_retries do
    add_index(table_name, column_names, **options)
  end
end

#find_duplicate_indexes(table_name, schema_name: connection.current_schema) ⇒ Object

Finds duplicate indexes for a given schema and table. This finds indexes where the index definition is identical but the names are different. Returns an array of arrays containing duplicate index name pairs.

Example:

find_duplicate_indexes('table_name_goes_here')



137
138
139
140
141
142
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 137

def find_duplicate_indexes(table_name, schema_name: connection.current_schema)
  find_indexes(table_name, schema_name: schema_name)
    .group_by { |r| r['index_id'] }
    .select { |_, v| v.size > 1 }
    .map { |_, indexes| indexes.map { |index| index['index_name'] } }
end

#indexes_by_definition_for_table(table_name, schema_name: connection.current_schema) ⇒ Object

Retrieves a hash of index names for a given table and schema, by index definition.

Example:

indexes_by_definition_for_table('table_name_goes_here')

Returns:

{
  "CREATE _ btree (created_at)" => "index_on_created_at"
}



156
157
158
159
160
161
162
163
164
165
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 156

def indexes_by_definition_for_table(table_name, schema_name: connection.current_schema)
  duplicate_indexes = find_duplicate_indexes(table_name, schema_name: schema_name)

  unless duplicate_indexes.empty?
    raise DuplicatedIndexesError, "#{table_name} has duplicate indexes: #{duplicate_indexes}"
  end

  find_indexes(table_name, schema_name: schema_name)
    .each_with_object({}) { |row, hash| hash[row['index_id']] = row['index_name'] }
end

#prepare_partitioned_async_index(table_name, column_name, **options) ⇒ Object

Prepare async index creation for partitions Once the partition indexes are created, we can then use #add_concurrent_partitioned_index below to synchronously create the partitioned index



183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 183

def prepare_partitioned_async_index(table_name, column_name, **options)
  Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
  return unless async_index_creation_available?

  partitioned_table = find_partitioned_table(table_name)

  if index_exists?(table_name, column_name, **options)
    return Gitlab::AppLogger.warn(
      message: 'Partitioned index not prepared because it already exists',
      table_name: table_name,
      column_name: column_name,
      index_name: options[:name]
    )
  end

  options[:name] ||= index_name(table_name, column_name)

  partitioned_table
    .postgres_partitions
    .order(:name)
    .each do |partition|
    partition_index_name = generated_index_name(partition.identifier, options[:name])
    prepare_async_index(partition.identifier, column_name, **options.merge(name: partition_index_name))
  end
end

#remove_concurrent_partitioned_index_by_name(table_name, index_name) ⇒ Object

Safely removes an existing index from a partitioned table. The method name is a bit inaccurate as it does not drop the index concurrently, but it's named as such to maintain consistency with other similar helpers, and indicate that this should be safe to use in a production environment.

In current versions of Postgres it's impossible to drop an index concurrently, or drop an index from an individual partition that exists across the entire partitioned table. As a result this helper drops the index from the parent table, which automatically cascades to all partitions. While this does require an exclusive lock, dropping an index is a fast operation that won't block the table for a significant period of time.

Example:

remove_concurrent_partitioned_index_by_name :users, 'index_name_goes_here'



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 66

def remove_concurrent_partitioned_index_by_name(table_name, index_name)
  assert_not_in_transaction_block(scope: ERROR_SCOPE)

  find_partitioned_table(table_name)

  unless index_name_exists?(table_name, index_name)
    Gitlab::AppLogger.warn "Index not removed because it does not exist (this may be due to an aborted " \
      "migration or similar): table_name: #{table_name}, index_name: #{index_name}"

    return
  end

  with_lock_retries do
    remove_index(table_name, name: index_name)
  end
end

#rename_indexes_for_table(table_name, new_index_names, schema_name: connection.current_schema) ⇒ Object

Renames indexes for a given table and schema, mapping by index definition, to a hash of new index names.

Example:

index_names = indexes_by_definition_for_table('source_table_name_goes_here')
drop_table('source_table_name_goes_here')
rename_indexes_for_table('destination_table_name_goes_here', index_names)



175
176
177
178
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 175

def rename_indexes_for_table(table_name, new_index_names, schema_name: connection.current_schema)
  current_index_names = indexes_by_definition_for_table(table_name, schema_name: schema_name)
  rename_indexes(current_index_names, new_index_names, schema_name: schema_name)
end

#rename_partitioned_index(table_name, old_index_name, new_index_name) ⇒ Object

Rename the index for partitioned table and its partitions. The new_index_name will be the new name of the partitioned index. The new name of the partition indexes will be generated by the partition table name and new_index_name and look like e.g. index_000925dbd7.

Example:

rename_partitioned_index :users, 'existing_partitioned_index_name', 'new_index_name'



91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 91

def rename_partitioned_index(table_name, old_index_name, new_index_name)
  partitioned_table = find_partitioned_table(table_name)

  old_index_json = index_json(find_index(partitioned_table.name) { |i| i.name == old_index_name.to_s })

  partitioned_table.postgres_partitions.order(:name).each do |partition|
    old_partition_index_name = find_index(partition.identifier) { |i| index_json(i) == old_index_json }.name
    new_partition_index_name = generated_index_name(partition.identifier, new_index_name)
    rename_index_with_schema(
      partition.identifier, old_partition_index_name, new_partition_index_name, schema: partition.schema
    )
  end

  rename_index_with_schema(partitioned_table.name, old_index_name, new_index_name)
end

#swap_partitioned_indexes(table_name, old_index_name, new_index_name) ⇒ Object

Swap the index names for partitioned table and its partitions.

Example:

swap_partitioned_indexes :users, 'existing_partitioned_index_name_1', 'existing_partitioned_index_name_2'



112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 112

def swap_partitioned_indexes(table_name, old_index_name, new_index_name)
  partitioned_table = find_partitioned_table(table_name)

  old_index_json = index_json(find_index(partitioned_table.name) { |i| i.name == old_index_name.to_s })
  new_index_json = index_json(find_index(partitioned_table.name) { |i| i.name == new_index_name.to_s })

  partitioned_table.postgres_partitions.order(:name).each do |partition|
    old_partition_index_name = find_index(partition.identifier) { |i| index_json(i) == old_index_json }.name
    new_partition_index_name = find_index(partition.identifier) { |i| index_json(i) == new_index_json }.name
    swap_indexes(
      partition.identifier, old_partition_index_name, new_partition_index_name, schema: partition.schema
    )
  end

  swap_indexes(partitioned_table.name, old_index_name, new_index_name)
end

#unprepare_partitioned_async_index(table_name, column_name, **options) ⇒ Object 



209
210
211
212
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 209

def unprepare_partitioned_async_index(table_name, column_name, **options)
  partitioned_index_name = options[:name] || index_name(table_name, column_name)
  unprepare_partitioned_async_index_by_name(table_name, partitioned_index_name)
end

#unprepare_partitioned_async_index_by_name(table_name, index_name) ⇒ Object

Raises:



214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb', line 214

def unprepare_partitioned_async_index_by_name(table_name, index_name)
  raise ArgumentError, 'Partitioned index name is required' if index_name.blank?

  Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
  return unless async_index_creation_available?

  find_partitioned_table(table_name)
    .postgres_partitions
    .order(:name)
    .each do |partition|
    partition_index_name = generated_index_name(partition.identifier, index_name)
    unprepare_async_index_by_name(partition.identifier, partition_index_name)
  end
end