Class: Rdt::Model

Inherits:
Object
  • Object
show all
Includes:
SqlTemplateHelpers
Defined in:
lib/rdt/model.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from SqlTemplateHelpers

#j, #j_except, #j_numeric, #j_numeric_comma, #star, #x, #x_date, #x_except, #x_numeric, #x_numeric_comma

Constructor Details

#initialize(filepath, schema = SCHEMA) ⇒ Model

Returns a new instance of Model.



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/rdt/model.rb', line 16

def initialize(filepath, schema = SCHEMA)
  @filepath = filepath
  @name = File.basename(filepath, ".sql")
  @original_code = File.read(filepath)
  @sources = []
  @refs = []
  @built = false
  @materialize_as = "VIEW"
  @schema = schema

  def source(table)
    @sources << table.to_s
    table.to_s
  end

  def ref(model)
    @refs << model.to_s
    "#{@schema}.#{model}"
  end

  def this
    "#{@schema}.#{@name}"
  end

  def build_as kind, unique_by: "unique_by"
    case kind.to_s.downcase
      when "view"
        @materialize_as = "VIEW"
      when "table"
        @materialize_as = "TABLE"
      when "incremental"
        if get_relation_type(this) != "TABLE"
          @materialize_as = "TABLE"
          @is_incremental = false
        else
          @is_incremental = true
          @unique_by_column = unique_by
        end
    else
      raise "Invalid build_as materialization: #{kind}"
    end
    ""
  end

  def materialize
    # legacy, use build_as :table
    # will add warning in the future
    build_as :table
    ""
  end

  def skip
    @skip = true
    ""
  end

  @code = ERB.new(@original_code).result(binding)
end

Instance Attribute Details

#built ⇒ Object (readonly)

Returns the value of attribute built.



5
6
7
 
# File 'lib/rdt/model.rb', line 5

def built
  @built
end

#code ⇒ Object (readonly)

Returns the value of attribute code.



5
6
7
 
# File 'lib/rdt/model.rb', line 5

def code
  @code
end

#filepath ⇒ Object (readonly)

Returns the value of attribute filepath.



5
6
7
 
# File 'lib/rdt/model.rb', line 5

def filepath
  @filepath
end

#is_incremental ⇒ Object (readonly)

Returns the value of attribute is_incremental.



5
6
7
 
# File 'lib/rdt/model.rb', line 5

def is_incremental
  @is_incremental
end

#materialize_as ⇒ Object (readonly)

Returns the value of attribute materialize_as.



5
6
7
 
# File 'lib/rdt/model.rb', line 5

def materialize_as
  @materialize_as
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



5
6
7
 
# File 'lib/rdt/model.rb', line 5

def name
  @name
end

#refs ⇒ Object (readonly)

Returns the value of attribute refs.



5
6
7
 
# File 'lib/rdt/model.rb', line 5

def refs
  @refs
end

#skip ⇒ Object (readonly)

Returns the value of attribute skip.



5
6
7
 
# File 'lib/rdt/model.rb', line 5

def skip
  @skip
end

#sources ⇒ Object (readonly)

Returns the value of attribute sources.



5
6
7
 
# File 'lib/rdt/model.rb', line 5

def sources
  @sources
end

#unique_by_column ⇒ Object (readonly)

Returns the value of attribute unique_by_column.



5
6
7
 
# File 'lib/rdt/model.rb', line 5

def unique_by_column
  @unique_by_column
end

Instance Method Details

#assert_column_uniqueness(column, relation) ⇒ Object 



178
179
180
181
182
183
184
185
186
 
# File 'lib/rdt/model.rb', line 178

def assert_column_uniqueness(column, relation)
  result = ActiveRecord::Base.connection.execute <<~SQL
    SELECT COUNT(*) = COUNT(DISTINCT #{column}) FROM #{relation};
  SQL
  if result.values.first.first == false
    raise "Column #{column} is not unique in #{relation}"
  end
  true
end

#build ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
 
# File 'lib/rdt/model.rb', line 75

def build
  if @skip
    puts "SKIPPING #{@name}"
  elsif @is_incremental
    puts "INCREMENTAL #{@name}"
    assert_column_uniqueness(unique_by_column, this)
    temp_table = "#{@schema}.#{@name}_incremental_build_temp_table"

    # drop the temp table if it exists
    ActiveRecord::Base.connection.execute <<~SQL
    DROP TABLE IF EXISTS #{temp_table};
    SQL

    # create a temp table with the same schema as the source
    ActiveRecord::Base.connection.execute <<~SQL
      CREATE TABLE #{temp_table} AS (
        #{code}
      );
    SQL
    assert_column_uniqueness(unique_by_column, temp_table)
    # delete rows from the table that are in the source
    ActiveRecord::Base.connection.execute <<~SQL
      DELETE FROM #{this}
      USING #{temp_table}
      WHERE #{this}.#{unique_by_column} = #{temp_table}.#{unique_by_column};
    SQL

    # insert rows from the source into the table
    ActiveRecord::Base.connection.execute <<~SQL
      INSERT INTO #{this}
      SELECT * FROM #{temp_table};
    SQL

    # drop the temp table
    ActiveRecord::Base.connection.execute <<~SQL
      DROP TABLE #{temp_table};
    SQL

  else
    puts "BUILDING #{@name}"
    curent_relation_type = get_relation_type(this)
    case @materialize_as
      when "VIEW"
        ActiveRecord::Base.connection.execute <<~SQL
          BEGIN;
          #{drop_relation(this)}
          CREATE VIEW #{this} AS (
            #{@code}
          );
          COMMIT;
        SQL
      when "TABLE"
        temp_table = "#{@schema}.#{@name}_build_step_temp_table"
        ActiveRecord::Base.connection.execute <<~SQL
          DROP TABLE IF EXISTS #{temp_table};
          CREATE TABLE #{temp_table} AS (
            #{@code}
          );
          BEGIN;
          #{drop_relation(this)}
          ALTER TABLE #{temp_table} RENAME TO #{@name};
          DROP TABLE IF EXISTS #{temp_table};
          COMMIT;
        SQL
      else
        raise "Invalid materialize_as: #{@materialize_as}"
    end

    @built = true
  end
end

#build_as(kind, unique_by: "unique_by") ⇒ Object 



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/rdt/model.rb', line 40

def build_as kind, unique_by: "unique_by"
  case kind.to_s.downcase
    when "view"
      @materialize_as = "VIEW"
    when "table"
      @materialize_as = "TABLE"
    when "incremental"
      if get_relation_type(this) != "TABLE"
        @materialize_as = "TABLE"
        @is_incremental = false
      else
        @is_incremental = true
        @unique_by_column = unique_by
      end
  else
    raise "Invalid build_as materialization: #{kind}"
  end
  ""
end

#drop_relation(relation) ⇒ Object 



147
148
149
150
151
152
153
154
 
# File 'lib/rdt/model.rb', line 147

def drop_relation(relation)
  type = get_relation_type(relation)
  if type.present?
    "DROP #{type} #{relation} CASCADE;"
  else
    ""
  end
end

#get_relation_type(relation) ⇒ Object 



156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
 
# File 'lib/rdt/model.rb', line 156

def get_relation_type(relation)
  relnamespace, relname = relation.split(".")
  type =
    ActiveRecord::Base
      .connection
      .execute(
        "
  SELECT
    CASE c.relkind
      WHEN 'r' THEN 'TABLE'
      WHEN 'v' THEN 'VIEW'
      WHEN 'm' THEN 'MATERIALIZED VIEW'
    END AS relation_type
  FROM pg_class c
  JOIN pg_namespace n ON n.oid = c.relnamespace
  WHERE c.relname = '#{relname}' AND n.nspname = '#{relnamespace}';"
      )
      .values
      .first
      &.first
end

#materialize ⇒ Object 



60
61
62
63
64
65
 
# File 'lib/rdt/model.rb', line 60

def materialize
  # legacy, use build_as :table
  # will add warning in the future
  build_as :table
  ""
end

#ref(model) ⇒ Object 



31
32
33
34
 
# File 'lib/rdt/model.rb', line 31

def ref(model)
  @refs << model.to_s
  "#{@schema}.#{model}"
end

#source(table) ⇒ Object 



26
27
28
29
 
# File 'lib/rdt/model.rb', line 26

def source(table)
  @sources << table.to_s
  table.to_s
end

#this ⇒ Object 



36
37
38
 
# File 'lib/rdt/model.rb', line 36

def this
  "#{@schema}.#{@name}"
end