Class: DataDuck::ETL

Inherits:
Object
  • Object
show all
Defined in:
lib/dataduck/etl.rb

Class Attribute Summary collapse

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ ETL

Returns a new instance of ETL.



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/dataduck/etl.rb', line 17

def initialize(options = {})
  self.class.destinations ||= []
  @tables = options[:tables] || []
  @destinations = options[:destinations] || []

  @autoload_tables = options[:autoload_tables].nil? ? true : options[:autoload_tables]
  if @autoload_tables
    Dir[DataDuck.project_root + "/src/tables/*.rb"].each do |file|
      table_name_underscores = file.split("/").last.gsub(".rb", "")
      table_name_camelized = DataDuck::Util.underscore_to_camelcase(table_name_underscores)
      require file
      table = Object.const_get(table_name_camelized)
      if table <= DataDuck::Table
        @tables << table
      end
    end
  end
end

Class Attribute Details

.destinations ⇒ Object

Returns the value of attribute destinations.



6
7
8
 
# File 'lib/dataduck/etl.rb', line 6

def destinations
  @destinations
end

Instance Attribute Details

#destinations ⇒ Object

Returns the value of attribute destinations.



14
15
16
 
# File 'lib/dataduck/etl.rb', line 14

def destinations
  @destinations
end

#tables ⇒ Object

Returns the value of attribute tables.



15
16
17
 
# File 'lib/dataduck/etl.rb', line 15

def tables
  @tables
end

Class Method Details

.destination(destination_name) ⇒ Object 



9
10
11
12
 
# File 'lib/dataduck/etl.rb', line 9

def self.destination(destination_name)
  self.destinations ||= []
  self.destinations << DataDuck::Destination.destination(destination_name)
end

Instance Method Details

#process! ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/dataduck/etl.rb', line 36

def process!
  Logs.info("Processing ETL on pid #{ Process.pid }...")

  destinations_to_use = []
  destinations_to_use = destinations_to_use.concat(self.class.destinations)
  destinations_to_use = destinations_to_use.concat(self.destinations)
  destinations_to_use.uniq!

  @tables.each do |table_class|
    table_to_etl = table_class.new
    table_to_etl.etl!(destinations_to_use)
  end
end

#process_table!(table) ⇒ Object 



50
51
52
53
54
55
56
57
58
59
 
# File 'lib/dataduck/etl.rb', line 50

def process_table!(table)
  Logs.info("Processing ETL for table #{ table.name } on pid #{ Process.pid }...")

  destinations_to_use = []
  destinations_to_use = destinations_to_use.concat(self.class.destinations)
  destinations_to_use = destinations_to_use.concat(self.destinations)
  destinations_to_use.uniq!

  table.etl!(destinations_to_use)
end