Class: Gitlab::ClickHouse::SiphonGenerator

Inherits:
Rails::Generators::Base
  • Object
show all
Defined in:
lib/generators/gitlab/click_house/siphon_generator.rb

Constant Summary collapse

PG_TYPE_MAP =

Data types table Postgresql OID reference - jdbc.postgresql.org/documentation/publicapi/org/postgresql/core/Oid.html

{
  16 => 'Bool',
  17 => 'String',
  20 => 'Int64',
  21 => 'Int8',
  23 => 'Int64',
  25 => 'String',
  869 => 'String', # ip address type
  1016 => 'Array(Int64)',
  1043 => 'String',
  1082 => 'Date32',
  1184 => "DateTime64(6, 'UTC')",
  1114 => "DateTime64(6, 'UTC')",
  3802 => "String" # JSONB
}.freeze
DICTIONARIES =

The generator needs to look up traversal_path values from parent entities (organizations, namespaces, or projects). Instead of expensive JOINs, ClickHouse uses dictionaries for O(1) lookups. Organization, Namespace and Project are the only entities that have traversal_ids/traversal_path in PostgreSQL.

{
  'projects' => 'project_traversal_paths_dict',
  'namespaces' => 'namespace_traversal_paths_dict',
  'organizations' => 'organization_traversal_paths_dict'
}.freeze
PG_TO_CH_DEFAULT_MAP = 
{
  /^nextval/ => ->(default) {
    warn "Sequences like #{default} are not supported in ClickHouse"
    nil
  },
  /^ARRAY\[.*\]::.*$/ => ->(default) {
    warn "Array defaults like (#{default}) are not supported in ClickHouse."
    nil
  },
  /'\{\}'::\w+\[\]/ => ->(_) {
    '[]' # For arrays with empty as default
  },
  'now()' => ->(_) {
    'now()'
  },
  /^\d+(\.\d+)?$/ => ->(default) {
    default # numeric default
  },
  /::.*$/ => ->(default) {
    default.split('::').first # extract string default
  },
  'true' => ->(_) {
    'true'
  },
  'false' => ->(_) {
    'false'
  }
}.freeze

Instance Method Summary collapse

Instance Method Details

#generate_ch_table ⇒ Object 



91
92
93
94
95
96
97
 
# File 'lib/generators/gitlab/click_house/siphon_generator.rb', line 91

def generate_ch_table
  timestamp = Time.current.strftime('%Y%m%d%H%M%S')

  migration_path = "db/click_house/migrate/main/#{timestamp}_create_siphon_#{table_name}.rb"

  template 'siphon_table.rb.template', migration_path
end

#validate! ⇒ Object

Raises:

  • (ArgumentError) 


85
86
87
88
89
 
# File 'lib/generators/gitlab/click_house/siphon_generator.rb', line 85

def validate!
  return unless pg_fields_metadata.count == 0

  raise ArgumentError, "PG #{table_name} table does not exist"
end