Module: Gitlab::Database::MigrationHelpers::V2

Includes:
Gitlab::Database::MigrationHelpers
Included in:
Gitlab::Database::Migration::V1_0
Defined in:
lib/gitlab/database/migration_helpers/v2.rb

Constant Summary

Constants included from Gitlab::Database::MigrationHelpers

DEFAULT_TIMESTAMP_COLUMNS

Constants included from DynamicModelHelpers

DynamicModelHelpers::BATCH_SIZE

Constants included from Gitlab::Database::Migrations::RedisHelpers

Gitlab::Database::Migrations::RedisHelpers::SCAN_START_CURSOR

Constants included from Gitlab::Database::Migrations::SidekiqHelpers

Gitlab::Database::Migrations::SidekiqHelpers::DEFAULT_MAX_ATTEMPTS, Gitlab::Database::Migrations::SidekiqHelpers::DEFAULT_TIMES_IN_A_ROW

Constants included from Gitlab::Database::Migrations::ConstraintsHelpers

Gitlab::Database::Migrations::ConstraintsHelpers::MAX_IDENTIFIER_NAME_LENGTH

Constants included from Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers

Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::BATCH_CLASS_NAME, Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::BATCH_MIN_DELAY, Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::BATCH_MIN_VALUE, Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::BATCH_SIZE, Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::NonExistentMigrationError, Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::SUB_BATCH_SIZE

Constants included from Gitlab::Database::Migrations::BackgroundMigrationHelpers

Gitlab::Database::Migrations::BackgroundMigrationHelpers::BATCH_SIZE, Gitlab::Database::Migrations::BackgroundMigrationHelpers::JOB_BUFFER_SIZE

Instance Method Summary collapse

Methods included from Gitlab::Database::MigrationHelpers

#add_concurrent_foreign_key, #add_concurrent_index, #add_primary_key_using_index, #add_sequence, #add_timestamps_with_timezone, #backfill_conversion_of_integer_to_bigint, #backfill_iids, #change_column_type_concurrently, #check_trigger_permissions!, #cleanup_concurrent_column_type_change, #column_for, #concurrent_foreign_key_name, #convert_to_bigint_column, #convert_to_type_column, #copy_foreign_keys, #copy_indexes, #create_or_update_plan_limit, #create_temporary_columns_and_triggers, #define_batchable_model, #drop_sequence, #each_batch, #each_batch_range, #false_value, #foreign_key_exists?, #foreign_keys_for, #index_exists_by_name?, #index_invalid?, #indexes_for, #initialize_conversion_of_integer_to_bigint, #install_rename_triggers, #partition?, #postgres_exists_by_name?, #remove_column_default, #remove_concurrent_index, #remove_concurrent_index_by_name, #remove_foreign_key_if_exists, #remove_foreign_key_without_error, #remove_rename_triggers, #remove_timestamps, #rename_trigger_name, #replace_sql, #restore_conversion_of_integer_to_bigint, #revert_backfill_conversion_of_integer_to_bigint, #revert_initialize_conversion_of_integer_to_bigint, #swap_primary_key, #table_partitioned?, #true_value, #undo_change_column_type_concurrently, #undo_cleanup_concurrent_column_type_change, #update_column_in_batches, #validate_foreign_key

Methods included from WraparoundVacuumHelpers

#check_if_wraparound_in_progress

Methods included from AsyncConstraints::MigrationHelpers

#prepare_async_check_constraint_validation, #prepare_async_foreign_key_validation, #prepare_partitioned_async_foreign_key_validation, #unprepare_async_check_constraint_validation, #unprepare_async_foreign_key_validation, #unprepare_partitioned_async_foreign_key_validation

Methods included from AsyncIndexes::MigrationHelpers

#async_index_creation_available?, #prepare_async_index, #prepare_async_index_from_sql, #prepare_async_index_removal, #unprepare_async_index, #unprepare_async_index_by_name

Methods included from RenameTableHelpers

#finalize_table_rename, #rename_table_safely, #undo_finalize_table_rename, #undo_rename_table_safely

Methods included from DynamicModelHelpers

#define_batchable_model, #each_batch, #each_batch_range

Methods included from Gitlab::Database::Migrations::RedisHelpers

#queue_redis_migration_job

Methods included from Gitlab::Database::Migrations::SidekiqHelpers

#sidekiq_queue_length, #sidekiq_queue_migrate, #sidekiq_remove_jobs

Methods included from Gitlab::Database::Migrations::ExtensionHelpers

#create_extension, #drop_extension

Methods included from Gitlab::Database::Migrations::ConstraintsHelpers

#add_check_constraint, #add_not_null_constraint, #add_text_limit, #check_constraint_exists?, check_constraint_exists?, #check_constraint_name, #check_not_null_constraint_exists?, #check_text_limit_exists?, #copy_check_constraints, #drop_constraint, #remove_check_constraint, #remove_not_null_constraint, #remove_text_limit, #rename_constraint, #switch_constraint_names, #text_limit_name, #validate_check_constraint, #validate_check_constraint_name!, #validate_not_null_constraint, #validate_text_limit

Methods included from Gitlab::Database::Migrations::TimeoutHelpers

#disable_statement_timeout

Methods included from Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers

#delete_batched_background_migration, #ensure_batched_background_migration_is_finished, #finalize_batched_background_migration, #gitlab_schema_from_context, #queue_batched_background_migration

Methods included from Gitlab::Database::Migrations::BackgroundMigrationHelpers

#delete_job_tracking, #delete_queued_jobs, #finalize_background_migration, #migrate_in, #queue_background_migration_jobs_by_range_at_intervals, #requeue_background_migration_jobs_by_range_at_intervals

Methods included from Gitlab::Database::Migrations::ReestablishedConnectionStack

#with_restored_connection_stack

Instance Method Details

#cleanup_concurrent_column_rename(table, old_column, new_column) ⇒ Object

Cleans up a concurrent column name.

This method takes care of removing previously installed triggers as well as removing the old column.

table - The name of the database table. old_column - The name of the old column. new_column - The name of the new column.



148
149
150
 
# File 'lib/gitlab/database/migration_helpers/v2.rb', line 148

def cleanup_concurrent_column_rename(table, old_column, new_column)
  teardown_rename_mechanism(table, old_column, new_column, column_to_remove: old_column)
end

#create_table(table_name, *args, **kwargs, &block) ⇒ Object

Creates a new table, optionally allowing the caller to add text limit constraints to the table. This method only extends Rails’ ‘create_table` method

Example:

create_table :db_guides do |t|
  t.bigint :stars, default: 0, null: false
  t.text :title, limit: 128
  t.text :notes, limit: 1024

  t.check_constraint 'stars > 1000', name: 'so_many_stars'
end

See Rails’ ‘create_table` for more info on the available arguments.

When adding foreign keys to other tables, consider wrapping the call into a with_lock_retries block to avoid traffic stalls.



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/gitlab/database/migration_helpers/v2.rb', line 25

def create_table(table_name, *args, **kwargs, &block)
  helper_context = self

  super do |t|
    t.define_singleton_method(:text) do |column_name, **kwargs|
      limit = kwargs.delete(:limit)

      super(column_name, **kwargs)

      if limit
        # rubocop:disable GitlabSecurity/PublicSend
        name = helper_context.send(:text_limit_name, table_name, column_name)
        # rubocop:enable GitlabSecurity/PublicSend

        column_name = helper_context.quote_column_name(column_name)
        definition = "char_length(#{column_name}) <= #{limit}"

        t.check_constraint(definition, name: name)
      end
    end

    yield t unless block.nil?
  end
end

#rename_column_concurrently(table, old_column, new_column, type: nil, batch_column_name: :id) ⇒ Object

Renames a column without requiring downtime.

Concurrent renames work by using database triggers to ensure both the old and new column are in sync. However, this method will not remove the triggers or the old column automatically; this needs to be done manually in a post-deployment migration. This can be done using the method ‘cleanup_concurrent_column_rename`.

table - The name of the database table containing the column. old_column - The old column name. new_column - The new column name. type - The type of the new column. If no type is given the old column’s

type is used.

batch_column_name - option is for tables without primary key, in this

case another unique integer column can be used. Example: :user_id



118
119
120
121
122
123
124
125
126
 
# File 'lib/gitlab/database/migration_helpers/v2.rb', line 118

def rename_column_concurrently(table, old_column, new_column, type: nil, batch_column_name: :id)
  Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!

  setup_renamed_column(__callee__, table, old_column, new_column, type, batch_column_name)

  with_lock_retries do
    install_bidirectional_triggers(table, old_column, new_column)
  end
end

#truncate_tables!(*table_names, connection: self.connection) ⇒ Object

TRUNCATE is a DDL statement (it drops the table and re-creates it), so we want to run the migration in DDL mode, but we also don’t want to execute it against all schemas because it will be prevented by the lock_writes trigger.

For example, a ‘gitlab_main` table on `:gitlab_main` database will be truncated, and a `gitlab_main` table on `:gitlab_ci` database will be skipped.

Note Rails already has a truncate_tables, see github.com/rails/rails/blob/6-1-stable/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb#L193

Raises:

  • (ArgumentError) 


187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/gitlab/database/migration_helpers/v2.rb', line 187

def truncate_tables!(*table_names, connection: self.connection)
  table_schemas = Gitlab::Database::GitlabSchema.table_schemas!(table_names)

  raise ArgumentError, "`table_names` must resolve to only one `gitlab_schema`" if table_schemas.size != 1

  return unless Gitlab::Database.gitlab_schemas_for_connection(connection).include?(table_schemas.first)

  quoted_tables = table_names.map { |table_name| quote_table_name(table_name) }.join(', ')

  execute("TRUNCATE TABLE #{quoted_tables}")
end

#undo_cleanup_concurrent_column_rename(table, old_column, new_column, type: nil, batch_column_name: :id) ⇒ Object

Reverses the operations performed by cleanup_concurrent_column_rename.

This method adds back the old_column removed by cleanup_concurrent_column_rename. It also adds back the triggers that are removed by cleanup_concurrent_column_rename.

table - The name of the database table containing the column. old_column - The old column name. new_column - The new column name. type - The type of the old column. If no type is given the new column’s

type is used.

batch_column_name - option is for tables without primary key, in this

case another unique integer column can be used. Example: :user_id



167
168
169
170
171
172
173
174
175
 
# File 'lib/gitlab/database/migration_helpers/v2.rb', line 167

def undo_cleanup_concurrent_column_rename(table, old_column, new_column, type: nil, batch_column_name: :id)
  Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!

  setup_renamed_column(__callee__, table, new_column, old_column, type, batch_column_name)

  with_lock_retries do
    install_bidirectional_triggers(table, old_column, new_column)
  end
end

#undo_rename_column_concurrently(table, old_column, new_column) ⇒ Object

Reverses operations performed by rename_column_concurrently.

This method takes care of removing previously installed triggers as well as removing the new column.

table - The name of the database table. old_column - The name of the old column. new_column - The name of the new column.



136
137
138
 
# File 'lib/gitlab/database/migration_helpers/v2.rb', line 136

def undo_rename_column_concurrently(table, old_column, new_column)
  teardown_rename_mechanism(table, old_column, new_column, column_to_remove: new_column)
end

#with_lock_retries(*args, **kwargs, &block) ⇒ Object

Executes the block with a retry mechanism that alters the lock_timeout and sleep_time between attempts. The timings can be controlled via the timing_configuration parameter. If the lock was not acquired within the retry period, a last attempt is made without using lock_timeout.

In order to retry the block, the method wraps the block into a transaction.

When called inside an open transaction it will execute the block directly if lock retries are enabled with ‘enable_lock_retries!` at migration level, otherwise it will raise an error.

Examples

# Invoking without parameters
with_lock_retries do
  drop_table :my_table
end

# Invoking with custom +timing_configuration+
t = [
  [1.second, 1.second],
  [2.seconds, 2.seconds]
]

with_lock_retries(timing_configuration: t) do
  drop_table :my_table # this will be retried twice
end

# Disabling the retries using an environment variable
> export DISABLE_LOCK_RETRIES=true

with_lock_retries do
  drop_table :my_table # one invocation, it will not retry at all
end

Parameters

  • timing_configuration - [[ActiveSupport::Duration, ActiveSupport::Duration], …] lock timeout for the block, sleep time before the next iteration, defaults to ‘Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION`

  • logger - [Gitlab::JsonLogger]

  • env - [Hash] custom environment hash, see the example with ‘DISABLE_LOCK_RETRIES`



86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/gitlab/database/migration_helpers/v2.rb', line 86

def with_lock_retries(*args, **kwargs, &block)
  if transaction_open?
    if enable_lock_retries?
      Gitlab::AppLogger.warn 'Lock retries already enabled, executing the block directly'
      yield
    else
      raise <<~EOF
      #{__callee__} can not be run inside an already open transaction

      Use migration-level lock retries instead, see https://docs.gitlab.com/ee/development/migration_style_guide.html#retry-mechanism-when-acquiring-database-locks
      EOF
    end
  else
    super(*args, **kwargs.merge(allow_savepoints: false), &block)
  end
end