Class: DataDuck::ETL

Inherits:
Object
  • Object
show all
Defined in:
lib/dataduck/etl.rb

Class Attribute Summary collapse

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ ETL

Returns a new instance of ETL.



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/dataduck/etl.rb', line 18

def initialize(options = {})
  self.class.destinations ||= []
  @tables = options[:tables] || []
  @destinations = options[:destinations] || []
  @errored_tables = []

  @autoload_tables = options[:autoload_tables].nil? ? true : options[:autoload_tables]
  if @autoload_tables
    Dir[DataDuck.project_root + "/src/tables/*.rb"].each do |file|
      table_name_underscores = file.split("/").last.gsub(".rb", "")
      table_name_camelized = DataDuck::Util.underscore_to_camelcase(table_name_underscores)
      require file
      table_class = Object.const_get(table_name_camelized)
      if table_class <= DataDuck::Table && table_class.new.include_with_all?
        @tables << table_class
      end
    end
  end
end

Class Attribute Details

.destinations ⇒ Object

Returns the value of attribute destinations.



6
7
8
 
# File 'lib/dataduck/etl.rb', line 6

def destinations
  @destinations
end

Instance Attribute Details

#destinations ⇒ Object

Returns the value of attribute destinations.



14
15
16
 
# File 'lib/dataduck/etl.rb', line 14

def destinations
  @destinations
end

#errored_tables ⇒ Object

Returns the value of attribute errored_tables.



16
17
18
 
# File 'lib/dataduck/etl.rb', line 16

def errored_tables
  @errored_tables
end

#tables ⇒ Object

Returns the value of attribute tables.



15
16
17
 
# File 'lib/dataduck/etl.rb', line 15

def tables
  @tables
end

Class Method Details

.destination(destination_name) ⇒ Object 



9
10
11
12
 
# File 'lib/dataduck/etl.rb', line 9

def self.destination(destination_name)
  self.destinations ||= []
  self.destinations << DataDuck::Destination.destination(destination_name)
end

Instance Method Details

#errored? ⇒ Boolean

Returns:

  • (Boolean) 


38
39
40
 
# File 'lib/dataduck/etl.rb', line 38

def errored?
  @errored_tables.length > 0
end

#process! ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/dataduck/etl.rb', line 42

def process!
  Logs.info("Processing ETL on pid #{ Process.pid }...")

  destinations_to_use = []
  destinations_to_use = destinations_to_use.concat(self.class.destinations)
  destinations_to_use = destinations_to_use.concat(self.destinations)
  destinations_to_use.uniq!
  if destinations_to_use.length == 0
    destinations_to_use << DataDuck::Destination.only_destination
  end

  @errored_tables = []

  @tables.each do |table_or_class|
    table = table_or_class.kind_of?(DataDuck::Table) ? table_or_class : table_or_class.new
    Logs.info("Processing table '#{ table.name }'...")
    begin
      table.etl!(destinations_to_use)
    rescue => err
      Logs.error("Error while processing table '#{ table.name }': #{ err.to_s }\n#{ err.backtrace.join("\n") }")
      @errored_tables << table
    end
  end

  Logs.info("Finished ETL processing for pid #{ Process.pid }, #{ @tables.length - @errored_tables.length } succeeded, #{ @errored_tables.length } failed")
  if @errored_tables.length > 0
    Logs.info("The following tables encountered errors: '#{ @errored_tables.map(&:name).join("', '") }'")
  end
end