Class: GoodData::Bam::Compiler::EtlVisitor

Inherits:
Object
  • Object
show all
Defined in:
lib/compiler/etl_visitor.rb

Instance Method Summary collapse

Instance Method Details

#visit(node, state) ⇒ Object



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
# File 'lib/compiler/etl_visitor.rb', line 6

def visit(node, state)
  case node[:type]
  when :project
    FileUtils::cd(Pathname(state[:home]) + GoodData::Bam::Commands::ETL_HOME) do
      flows = if state[:only]
        node[:flows].find_all {|f| f[:name] == state[:only]}
      else
        node[:flows]
      end
      result = flows.reduce({
        :flows => [],
        :params => state
      }) {|state, node| visit(node, state)}
      GoodData::CloudConnect::Helpers::create_run_graph("graphs/main.grf", {
        :subgraphs => result[:flows],
        :flow => "main"
      })
    end
  when :flow
    visit_flow(node, state)
  when :tap
    case node[:direct]
    when false
      visit_incremental_tap(node, state)
    else
      case node[:source_type]
      when :salesforce
        visit_direct_sf_tap(node, state)
      else
        visit_direct_file_tap(node, state)
      end
    end
  when :sink
    case node[:target_type]
    when :file, :web
      visit_file_sink(node, state)
    else
      visit_gd_sink(node, state)
    end
  when :graph
    visit_graph(node, state)
  else
    state
  end
end

#visit_direct_file_tap(node, state) ⇒ Object



141
142
143
144
145
146
147
148
# File 'lib/compiler/etl_visitor.rb', line 141

def visit_direct_file_tap(node, state)
  graph_path = Step.get_graph_path(node)
  GoodData::Bam::Generators::Etl::create_file_downloading_graph(graph_path, node, state)
  state.merge({
    :graphs => state[:graphs].concat([graph_path]),
    :metadata => state[:metadata].merge({ node[:id] => Metadata.(node) })
  })
end

#visit_direct_sf_tap(node, state) ⇒ Object



130
131
132
133
134
135
136
137
138
139
# File 'lib/compiler/etl_visitor.rb', line 130

def visit_direct_sf_tap(node, state)
  puts "Visiting direct tap"
  graph_path = Step.get_graph_path(node)
  tap = Tap.prepare_for_sf_downloader(node)
  GoodData::Bam::Generators::Etl::create_sf_downloading_graph(graph_path, tap, state)
  state.merge({
    :graphs => state[:graphs].concat([graph_path]),
    :metadata => state[:metadata].merge({node[:id] => Metadata.(node)})
  })
end

#visit_file_sink(node, state) ⇒ Object



103
104
105
106
107
108
109
110
111
112
113
114
# File 'lib/compiler/etl_visitor.rb', line 103

def visit_file_sink(node, state)
  graph_path = Step.get_graph_path(node)

  sink_fields = Metadata.(node)[:fields].map {|f| f[:name]}
   = state[:metadata][node[:in][:id]][:fields].map {|f| f[:name]}
  fail "You have fields \"#{(sink_fields - ).join(', ')}\" in sink \"#{node[:id]}\" that are not filled in by data" unless (sink_fields - ).empty?

  GoodData::Bam::Generators::Etl::create_file_uploading_graph(graph_path, node, node[:in][:id], state)
  state.merge({
    :graphs => state[:graphs].concat([graph_path])
  })
end

#visit_flow(flow, state) ⇒ Object



52
53
54
55
56
57
58
59
60
61
62
63
# File 'lib/compiler/etl_visitor.rb', line 52

def visit_flow(flow, state)
  puts "Visiting flow"
  result = flow[:steps].reduce(state.merge({
    :graphs => [],
    :metadata => {}
  })) {|state, node| visit(node, state)}
    GoodData::CloudConnect::Helpers::create_run_graph("graphs/#{flow[:name]}_main.grf", {
      :subgraphs => result[:graphs],
      :flow => flow[:name]
    })
  state.merge(:flows => state[:flows].concat(["graphs/#{flow[:name]}_main.grf"]))
end

#visit_gd_sink(node, state) ⇒ Object



117
118
119
120
121
122
123
124
125
126
127
128
# File 'lib/compiler/etl_visitor.rb', line 117

def visit_gd_sink(node, state)
  graph_path = Step.get_graph_path(node)

  sink_fields = Metadata.(node)[:fields].map {|f| f[:name]}
   = state[:metadata][node[:in][:id]][:fields].map {|f| f[:name]}
  fail "You have fields \"#{(sink_fields - ).join(', ')}\" in sink \"#{node[:id]}\" that are not filled in by data" unless (sink_fields - ).empty?

  GoodData::Bam::Generators::Etl::create_gd_uploading_graph(graph_path, node, node[:in][:id], state)
  state.merge({
    :graphs => state[:graphs].concat([graph_path])
  })
end

#visit_graph(node, state) ⇒ Object



65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
# File 'lib/compiler/etl_visitor.rb', line 65

def visit_graph(node, state)
  puts "Visiting graph"
  move_graphs = []
  node[:input_ports].each_with_index do |input, i|
    j = i+1
    name = input[:id]
    graph_path = "graphs/#{node[:flow_id]}_#{input[:id]}_move_in_#{j}.grf"
    graph_name = "#{node[:flow_id]}_#{input[:id]}_move_in_#{j}.grf"
    state = state.merge({:graphs => state[:graphs].concat([graph_path])})
    GoodData::CloudConnect::Helpers::create_moving_graph(graph_path, {
      :source => "${DATA}/#{name}.csv",
      :target => "${DATA}/#{j}_in.csv",
      :operation => "MOVE"
    })

    FileUtils.cp(Graph.get_path(node), (Pathname("graphs") + node[:path].basename).to_s)
     = Compiler.(state[:metadata], input)

     = "./metadata/#{node[:flow_id]}/#{Step.step_name(node)}/#{j}_in.xml"
    GoodData::CloudConnect::Helpers::(, state[:metadata][name].merge(:name => "in_#{j}"))
     = "./metadata/#{node[:flow_id]}/#{Step.step_name(node)}/#{j}_out.xml"
    GoodData::CloudConnect::Helpers::(, .merge(:name => "out_#{j}"))
    state = state.merge({:metadata => state[:metadata].merge([:id] => )})
  end

  state = state.merge({:graphs => state[:graphs].concat([(Pathname("graphs") + node[:path].basename).to_s])})

   = node[:output_port]
  graph_path = "graphs/#{node[:flow_id]}_#{Step.step_name(node)}_move_out.grf"
  state = state.merge({:graphs => state[:graphs].concat([graph_path])})
  GoodData::CloudConnect::Helpers::create_moving_graph(graph_path, {
    :source => "${DATA}/out.csv",
    :target => "${DATA}/#{[:id]}.csv",
    :operation => "MOVE"
  })
  state
end

#visit_incremental_tap(node, state) ⇒ Object



150
151
152
153
154
155
156
157
158
159
160
# File 'lib/compiler/etl_visitor.rb', line 150

def visit_incremental_tap(node, state)
  puts "Visiting Incremental tap"
  # Get rid of all the shit. Timestamps, pull ids check for snapshots. And shove them downstream.
  tap = Tap.prepare_for_es_downloader(node)
  graph_path = Step.get_graph_path(tap)
  GoodData::Bam::Generators::Etl::create_es_downloading_graph(graph_path, tap, state)
  state.merge({
    :graphs => state[:graphs].concat([graph_path]),
    :metadata => state[:metadata].merge({tap[:id] => Metadata.(tap)})
  })
end