Module: Gitlab::Database

Defined in:
lib/gitlab/database.rb,
lib/gitlab/database/count.rb,
lib/gitlab/database/grant.rb,
lib/gitlab/database/median.rb,
lib/gitlab/database/date_time.rb,
lib/gitlab/database/batch_count.rb,
lib/gitlab/database/sha_attribute.rb,
lib/gitlab/database/schema_cleaner.rb,
lib/gitlab/database/schema_helpers.rb,
lib/gitlab/database/connection_timer.rb,
lib/gitlab/database/custom_structure.rb,
lib/gitlab/database/sha256_attribute.rb,
lib/gitlab/database/similarity_score.rb,
lib/gitlab/database/migration_helpers.rb,
lib/gitlab/database/with_lock_retries.rb,
lib/gitlab/database/read_only_relation.rb,
lib/gitlab/database/schema_version_files.rb,
lib/gitlab/database/dynamic_model_helpers.rb,
lib/gitlab/database/background_migration_job.rb,
lib/gitlab/database/multi_threaded_migration.rb,
lib/gitlab/database/obsolete_ignored_columns.rb,
lib/gitlab/database/count/exact_count_strategy.rb,
lib/gitlab/database/partitioning/time_partition.rb,
lib/gitlab/database/x509_serial_number_attribute.rb,
lib/gitlab/database/partitioning/monthly_strategy.rb,
lib/gitlab/database/count/reltuples_count_strategy.rb,
lib/gitlab/database/partitioning/partition_creator.rb,
lib/gitlab/database/partitioning_migration_helpers.rb,
lib/gitlab/database/count/tablesample_count_strategy.rb,
lib/gitlab/database/partitioning/partition_monitoring.rb,
lib/gitlab/database/rename_reserved_paths_migration/v1.rb,
lib/gitlab/database/migrations/background_migration_helpers.rb,
lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb,
lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb,
lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb,
lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb,
lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb,
lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb,
lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb,
lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb,
lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb,
lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb,
lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb,
lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb

Defined Under Namespace

Modules: ActiveRecordBaseTransactionMetrics, BatchCount, Count, DateTime, DynamicModelHelpers, Median, MigrationHelpers, Migrations, MultiThreadedMigration, Partitioning, PartitioningMigrationHelpers, PostgresqlAdapter, PostgresqlDatabaseTasks, ReadOnlyRelation, RenameReservedPathsMigration, SchemaHelpers Classes: BackgroundMigrationJob, BatchCounter, ConnectionTimer, CustomStructure, Grant, ObsoleteIgnoredColumns, SchemaCleaner, SchemaVersionFiles, Sha256Attribute, ShaAttribute, SimilarityScore, WithLockRetries, X509SerialNumberAttribute

Constant Summary collapse

MINIMUM_POSTGRES_VERSION =

Minimum PostgreSQL version requirement per documentation: docs.gitlab.com/ee/install/requirements.html#postgresql-requirements

11
MAX_INT_VALUE =

www.postgresql.org/docs/9.2/static/datatype-numeric.html

2147483647
MIN_INT_VALUE = 
-2147483648
MAX_TIMESTAMP_VALUE =

The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz: www.postgresql.org/docs/9.1/static/datatype-datetime.html dev.mysql.com/doc/refman/5.7/en/datetime.html FIXME: this should just be the max value of timestampz

Time.at((1 << 31) - 1).freeze
MAX_TEXT_SIZE_LIMIT =

The maximum number of characters for text fields, to avoid DoS attacks via parsing huge text fields gitlab.com/gitlab-org/gitlab-foss/issues/61974

1_000_000
MIN_SCHEMA_VERSION =

Minimum schema version from which migrations are supported Migrations before this version may have been removed

20190506135400
MIN_SCHEMA_GITLAB_VERSION = 
'11.11.0'
DYNAMIC_PARTITIONS_SCHEMA =

Schema we store dynamically managed partitions in (e.g. for time partitioning)

:gitlab_partitions_dynamic
STATIC_PARTITIONS_SCHEMA =

Schema we store static partitions in (e.g. for hash partitioning)

:gitlab_partitions_static
EXTRA_SCHEMAS =

This is an extensive list of postgres schemas owned by GitLab It does not include the default public schema

[DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze
BINARY_TYPE =

PostgreSQL defines its own class with slightly different behaviour from the default Binary type.

ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea

Class Method Summary collapse

Class Method Details

.adapter_name ⇒ Object 


50
51
52
 
# File 'lib/gitlab/database.rb', line 50

def self.adapter_name
  config['adapter']
end

.add_post_migrate_path_to_rails(force: false) ⇒ Object 


281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
 
# File 'lib/gitlab/database.rb', line 281

def self.add_post_migrate_path_to_rails(force: false)
  return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force

  Rails.application.config.paths['db'].each do |db_path|
    path = Rails.root.join(db_path, 'post_migrate').to_s

    unless Rails.application.config.paths['db/migrate'].include? path
      Rails.application.config.paths['db/migrate'] << path

      # Rails memoizes migrations at certain points where it won't read the above
      # path just yet. As such we must also update the following list of paths.
      ActiveRecord::Migrator.migrations_paths << path
    end
  end
end

.bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil) ⇒ Object

Bulk inserts a number of rows into a table, optionally returning their IDs.

table - The name of the table to insert the rows into. rows - An Array of Hash instances, each mapping the columns to their

values.

return_ids - When set to true the return value will be an Array of IDs of

the inserted rows

disable_quote - A key or an Array of keys to exclude from quoting (You

become responsible for protection from SQL injection for
these keys!)

on_conflict - Defines an upsert. Values can be: :disabled (default) or

:do_nothing


195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/gitlab/database.rb', line 195

def self.bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil)
  return if rows.empty?

  keys = rows.first.keys
  columns = keys.map { |key| connection.quote_column_name(key) }

  disable_quote = Array(disable_quote).to_set
  tuples = rows.map do |row|
    keys.map do |k|
      disable_quote.include?(k) ? row[k] : connection.quote(row[k])
    end
  end

  sql = <<-EOF
    INSERT INTO #{table} (#{columns.join(', ')})
    VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')}
  EOF

  sql = "#{sql} ON CONFLICT DO NOTHING" if on_conflict == :do_nothing

  sql = "#{sql} RETURNING id" if return_ids

  result = connection.execute(sql)

  if return_ids
    result.values.map { |tuple| tuple[0].to_i }
  else
    []
  end
end

.cached_column_exists?(table_name, column_name) ⇒ Boolean

Returns:

  • (Boolean) 

257
258
259
 
# File 'lib/gitlab/database.rb', line 257

def self.cached_column_exists?(table_name, column_name)
  connection.schema_cache.columns_hash(table_name).has_key?(column_name.to_s)
end

.cached_table_exists?(table_name) ⇒ Boolean

Returns:

  • (Boolean) 

261
262
263
 
# File 'lib/gitlab/database.rb', line 261

def self.cached_table_exists?(table_name)
  exists? && connection.schema_cache.data_source_exists?(table_name)
end

.check_postgres_version_and_print_warning ⇒ Object 


103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/gitlab/database.rb', line 103

def self.check_postgres_version_and_print_warning
  return if Gitlab::Database.postgresql_minimum_supported_version?
  return if Gitlab::Runtime.rails_runner?

  Kernel.warn ERB.new(Rainbow.new.wrap(<<~EOS).red).result

              ██     ██  █████  ██████  ███    ██ ██ ███    ██  ██████ 
              ██     ██ ██   ██ ██   ██ ████   ██ ██ ████   ██ ██      
              ██  █  ██ ███████ ██████  ██ ██  ██ ██ ██ ██  ██ ██   ███ 
              ██ ███ ██ ██   ██ ██   ██ ██  ██ ██ ██ ██  ██ ██ ██    ██ 
               ███ ███  ██   ██ ██   ██ ██   ████ ██ ██   ████  ██████  

    ******************************************************************************
      You are using PostgreSQL <%= Gitlab::Database.version %>, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %>
      is required for this version of GitLab.
      <% if Rails.env.development? || Rails.env.test? %>
      If using gitlab-development-kit, please find the relevant steps here:
        https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/postgresql.md#upgrade-postgresql
      <% end %>
      Please upgrade your environment to a supported PostgreSQL version, see
      https://docs.gitlab.com/ee/install/requirements.html#database for details.
    ******************************************************************************
  EOS
rescue ActiveRecord::ActiveRecordError, PG::Error
  # ignore - happens when Rake tasks yet have to create a database, e.g. for testing
end

.config ⇒ Object 


38
39
40
 
# File 'lib/gitlab/database.rb', line 38

def self.config
  ActiveRecord::Base.configurations[Rails.env]
end

.create_connection_pool(pool_size, host = nil, port = nil) ⇒ Object

pool_size - The size of the DB pool. host - An optional host name to use instead of the default one.


232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
 
# File 'lib/gitlab/database.rb', line 232

def self.create_connection_pool(pool_size, host = nil, port = nil)
  env = Rails.env
  original_config = ActiveRecord::Base.configurations.to_h

  env_config = original_config[env].merge('pool' => pool_size)
  env_config['host'] = host if host
  env_config['port'] = port if port

  config = ActiveRecord::DatabaseConfigurations.new(
    original_config.merge(env => env_config)
  )

  spec =
    ActiveRecord::
      ConnectionAdapters::
      ConnectionSpecification::Resolver.new(config).spec(env.to_sym)

  ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec)
end

.database_name ⇒ Object 


46
47
48
 
# File 'lib/gitlab/database.rb', line 46

def self.database_name
  config['database']
end

.db_read_only? ⇒ Boolean

Check whether the underlying database is in read-only mode

Returns:

  • (Boolean) 

76
77
78
79
80
81
82
83
84
85
 
# File 'lib/gitlab/database.rb', line 76

def self.db_read_only?
  pg_is_in_recovery =
    ActiveRecord::Base
      .connection
      .execute('SELECT pg_is_in_recovery()')
      .first
      .fetch('pg_is_in_recovery')

  Gitlab::Utils.to_boolean(pg_is_in_recovery)
end

.db_read_write? ⇒ Boolean

Returns:

  • (Boolean) 

87
88
89
 
# File 'lib/gitlab/database.rb', line 87

def self.db_read_write?
  !self.db_read_only?
end

.exists? ⇒ Boolean

Returns:

  • (Boolean) 

271
272
273
274
275
276
277
 
# File 'lib/gitlab/database.rb', line 271

def self.exists?
  connection

  true
rescue
  false
end

.false_value ⇒ Object 


168
169
170
 
# File 'lib/gitlab/database.rb', line 168

def self.false_value
  "'f'"
end

.human_adapter_name ⇒ Object 


54
55
56
57
58
59
60
 
# File 'lib/gitlab/database.rb', line 54

def self.human_adapter_name
  if postgresql?
    'PostgreSQL'
  else
    'Unknown'
  end
end

.inside_transaction? ⇒ Boolean

inside_transaction? will return true if the caller is running within a transaction. Handles special cases when running inside a test environment, where tests may be wrapped in transactions

Returns:

  • (Boolean) 

299
300
301
302
303
304
305
 
# File 'lib/gitlab/database.rb', line 299

def self.inside_transaction?
  if Rails.env.test?
    ActiveRecord::Base.connection.open_transactions > open_transactions_baseline
  else
    ActiveRecord::Base.connection.open_transactions > 0
  end
end

.install_monkey_patches ⇒ Object

Monkeypatch rails with upgraded database observability


324
325
326
 
# File 'lib/gitlab/database.rb', line 324

def self.install_monkey_patches
  ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics)
end

.nulls_first_order(field, direction = 'ASC') ⇒ Object 


156
157
158
 
# File 'lib/gitlab/database.rb', line 156

def self.nulls_first_order(field, direction = 'ASC')
  Arel.sql("#{field} #{direction} NULLS FIRST")
end

.nulls_last_order(field, direction = 'ASC') ⇒ Object 


152
153
154
 
# File 'lib/gitlab/database.rb', line 152

def self.nulls_last_order(field, direction = 'ASC')
  Arel.sql("#{field} #{direction} NULLS LAST")
end

.observe_transaction_duration(duration_seconds) ⇒ Object

observe_transaction_duration is called from ActiveRecordBaseTransactionMetrics.transaction and used to record transaction durations.


330
331
332
333
334
335
336
337
338
339
 
# File 'lib/gitlab/database.rb', line 330

def self.observe_transaction_duration(duration_seconds)
  if current_transaction = ::Gitlab::Metrics::Transaction.current
    current_transaction.observe(:gitlab_database_transaction_seconds, duration_seconds) do
      docstring "Time spent in database transactions, in seconds"
    end
  end
rescue Prometheus::Client::LabelSetValidator::LabelSetError => err
  # Ensure that errors in recording these metrics don't affect the operation of the application
  Gitlab::AppLogger.error("Unable to observe database transaction duration: #{err}")
end

.pg_current_wal_insert_lsn ⇒ Object 


136
137
138
 
# File 'lib/gitlab/database.rb', line 136

def self.pg_current_wal_insert_lsn
  Gitlab::Database.postgresql_9_or_less? ? 'pg_current_xlog_insert_location' : 'pg_current_wal_insert_lsn'
end

.pg_last_wal_receive_lsn ⇒ Object 


140
141
142
 
# File 'lib/gitlab/database.rb', line 140

def self.pg_last_wal_receive_lsn
  Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_receive_location' : 'pg_last_wal_receive_lsn'
end

.pg_last_wal_replay_lsn ⇒ Object 


144
145
146
 
# File 'lib/gitlab/database.rb', line 144

def self.pg_last_wal_replay_lsn
  Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_replay_location' : 'pg_last_wal_replay_lsn'
end

.pg_last_xact_replay_timestamp ⇒ Object 


148
149
150
 
# File 'lib/gitlab/database.rb', line 148

def self.pg_last_xact_replay_timestamp
  'pg_last_xact_replay_timestamp'
end

.pg_wal_lsn_diff ⇒ Object

map some of the function names that changed between PostgreSQL 9 and 10 wiki.postgresql.org/wiki/New_in_postgres_10


132
133
134
 
# File 'lib/gitlab/database.rb', line 132

def self.pg_wal_lsn_diff
  Gitlab::Database.postgresql_9_or_less? ? 'pg_xlog_location_diff' : 'pg_wal_lsn_diff'
end

.postgresql? ⇒ Boolean

Deprecated.

Returns:

  • (Boolean) 

63
64
65
 
# File 'lib/gitlab/database.rb', line 63

def self.postgresql?
  adapter_name.casecmp('postgresql') == 0
end

.postgresql_9_or_less? ⇒ Boolean

Returns:

  • (Boolean) 

95
96
97
 
# File 'lib/gitlab/database.rb', line 95

def self.postgresql_9_or_less?
  version.to_f < 10
end

.postgresql_minimum_supported_version? ⇒ Boolean

Returns:

  • (Boolean) 

99
100
101
 
# File 'lib/gitlab/database.rb', line 99

def self.postgresql_minimum_supported_version?
  version.to_f >= MINIMUM_POSTGRES_VERSION
end

.random ⇒ Object 


160
161
162
 
# File 'lib/gitlab/database.rb', line 160

def self.random
  "RANDOM()"
end

.read_only? ⇒ Boolean

Returns:

  • (Boolean) 

67
68
69
 
# File 'lib/gitlab/database.rb', line 67

def self.read_only?
  false
end

.read_write? ⇒ Boolean

Returns:

  • (Boolean) 

71
72
73
 
# File 'lib/gitlab/database.rb', line 71

def self.read_write?
  !self.read_only?
end

.reset_open_transactions_baseline ⇒ Object 


314
315
316
 
# File 'lib/gitlab/database.rb', line 314

def self.reset_open_transactions_baseline
  @open_transactions_baseline = 0
end

.sanitize_timestamp(timestamp) ⇒ Object 


226
227
228
 
# File 'lib/gitlab/database.rb', line 226

def self.sanitize_timestamp(timestamp)
  MAX_TIMESTAMP_VALUE > timestamp ? timestamp : MAX_TIMESTAMP_VALUE.dup
end

.set_open_transactions_baseline ⇒ Object

These methods that access @open_transactions_baseline are not thread-safe. These are fine though because we only call these in RSpec's main thread. If we decide to run specs multi-threaded, we would need to use something like ThreadGroup to keep track of this value


310
311
312
 
# File 'lib/gitlab/database.rb', line 310

def self.set_open_transactions_baseline
  @open_transactions_baseline = ActiveRecord::Base.connection.open_transactions
end

.true_value ⇒ Object 


164
165
166
 
# File 'lib/gitlab/database.rb', line 164

def self.true_value
  "'t'"
end

.username ⇒ Object 


42
43
44
 
# File 'lib/gitlab/database.rb', line 42

def self.username
  config['username'] || ENV['USER']
end

.version ⇒ Object 


91
92
93
 
# File 'lib/gitlab/database.rb', line 91

def self.version
  @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
end

.with_connection_pool(pool_size) ⇒ Object 


172
173
174
175
176
177
178
179
180
 
# File 'lib/gitlab/database.rb', line 172

def self.with_connection_pool(pool_size)
  pool = create_connection_pool(pool_size)

  begin
    yield(pool)
  ensure
    pool.disconnect!
  end
end