Class: Flydata::SourcePostgresql::TableMeta

Inherits:
TableMeta
  • Object
show all
Defined in:
lib/flydata/source_postgresql/table_meta.rb

Overview

Fetch and keep table meta information

<table-name(Symbol)>:

primary_keys: <Array of String>    # Set primary key names. ex: ['group_id', 'category_id']
pk_positions: <Array of Integer>   # Set the ordinal position of primary keys. ex: [1, 3]
max_row_size: <Integer>            # byte, calculated based on column size
max_num_rows_per_query>: <Integer> # max number of rows per query
raw_columns: <Hash>                # raw information schema data
columns: table_def.columns
table_def:
 columns:
   table:
   column:
   type:
   not_null:
   primary_key:
   default:
   column_size: (new)  # Set in `PostgresqlTableDef.parse_one_column_def`

Constant Summary collapse

GET_TABLE_META_SQL = 
"SELECT\n  c.table_catalog,\n  c.table_schema,\n  c.table_name,\n  c.column_name,\n  c.ordinal_position,\n  c.column_default,\n  c.is_nullable,\n  c.data_type,\n  c.character_maximum_length,\n  c.character_octet_length,\n  CASE WHEN c.data_type='money' THEN 19\n   ELSE c.numeric_precision END AS numeric_precision,\n  c.numeric_precision_radix,\n  CASE WHEN c.data_type='money' THEN (\n    SELECT CASE WHEN scale IS NULL THEN 0 ELSE scale END\n      FROM (SELECT length(substring(999999::money::char varying, '\\.([0-9]+)$')) AS scale) s)\n   ELSE c.numeric_scale END AS numeric_scale,\n  c.datetime_precision,\n  i.indisprimary AS is_primary\nFROM\n  pg_index i\n  JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)\n  RIGHT JOIN\n    (SELECT\n      (table_catalog ||'.'|| table_schema ||'.'|| table_name)::regclass AS regid,\n      *\n     FROM information_schema.columns\n    ) c\n  ON i.indrelid = c.regid AND a.attname = c.column_name\nWHERE\n  c.table_catalog = '%{database}'\n  AND c.table_schema IN (%{schema})\n  AND c.table_name IN (%{tables})\nORDER BY\n  c.table_name, c.ordinal_position;\n"
GET_CURRENT_SNAPSHOT_SQL = 
"SELECT txid_current_snapshot();"
DEFAULT_MAX_FETCH_RECORD_SIZE = 
50000

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods inherited from TableMeta

#[]

Constructor Details

#initialize(dbconf, tables, schema = nil) ⇒ TableMeta

DEFAULT_MAX_FETCH_RECORD_SIZE = 8



75
76
77
78
79
80
 
# File 'lib/flydata/source_postgresql/table_meta.rb', line 75

def initialize(dbconf, tables, schema = nil) # 64mb
  @dbconf = dbconf
  @database = dbconf[:dbname] || dbconf[:database] || dbconf['database']
  @tables = tables
  @schema = schema || dbconf[:schema] || dbconf['schema']
end

Instance Attribute Details

#current_snapshot ⇒ Object (readonly)

Returns the value of attribute current_snapshot.



82
83
84
 
# File 'lib/flydata/source_postgresql/table_meta.rb', line 82

def current_snapshot
  @current_snapshot
end

Instance Method Details

#build_table_meta(columns) ⇒ Object 



111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
 
# File 'lib/flydata/source_postgresql/table_meta.rb', line 111

def build_table_meta(columns)
  ret = Hash.new{|h,k| h[k]={} }

  # Put ret[<table-name-sym>][:raw_columns]
  columns.each do |col|
    column_name = col['column_name'].to_sym
    table_name = col['table_name'].to_sym
    t_meta = ret[table_name]
    t_meta[:raw_columns] ||= {}
    t_meta[:raw_columns][column_name] = col
  end

  ret.each do |table_name, t_meta|
    begin
      table_def = FlydataCore::TableDef::PostgresqlTableDef.create(
                  t_meta[:raw_columns].values, :skip_primary_key_check)
    rescue FlydataCore::TableDefError => e
      t_meta.merge!(
        table_name: table_name,
        table_def_err: e,
      )
      # Skip when getting an error when parsing the columns
      next
    end

    primary_keys = []
    pk_positions = []
    table_def.columns.each.with_index(1) do |col, index|
      col_name = col[:column]
      if col[:primary_key]
        primary_keys << col_name
        pk_positions << index.to_s
      end
    end

    t_meta.merge!(
      table_name: table_name.to_s,
      primary_keys: primary_keys,
      pk_positions: pk_positions,
      #max_row_size: max_row_size,                  #TODO: calculation
      #max_num_rows_per_query: max_row_size + 128,  #TODO: calculation
      max_num_rows_per_query: DEFAULT_MAX_FETCH_RECORD_SIZE,
      columns: table_def.columns,
      table_def: table_def,
    )
  end

  ret
end

#calc_column_size(column) ⇒ Object 



161
162
163
164
 
# File 'lib/flydata/source_postgresql/table_meta.rb', line 161

def calc_column_size(column)
  #TODO: Implement the check logic based on column type
  124
end

#reload(pg_client = nil) ⇒ Object 



84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/flydata/source_postgresql/table_meta.rb', line 84

def reload(pg_client = nil)
  schema = @schema ? "'#{@schema}'" : "select current_schema"
  sql = GET_TABLE_META_SQL % {
    database: @database,
    schema: schema,
    tables: @tables.collect{|t| "'#{t}'"}.join(','),
  }

  conn = pg_client
  if conn.nil?
    local_conn = conn = FlydataCore::Postgresql::PGClient.new(@dbconf)
  end

  # Set table_meta
  columns = conn.query(sql)
  @table_meta = build_table_meta(columns)

  # Set current snapshot
  current_snapshot_str = conn.
    query(GET_CURRENT_SNAPSHOT_SQL).first['txid_current_snapshot']
  @current_snapshot = FlydataCore::Postgresql::Snapshot.new(current_snapshot_str)

  self
ensure
  local_conn.close rescue nil if local_conn
end