Module: GoodData::Bam::Tap
- Defined in:
- lib/base/tap.rb
Class Method Summary collapse
- .add_field(tap, field) ⇒ Object
- .add_source_type(tap) ⇒ Object
- .add_timestamp_field(tap) ⇒ Object
-
.assert_presence_of_id_field(tap) ⇒ Object
Every incremental tap has to have.
- .collect_validations(tap) ⇒ Object
- .create(tap) ⇒ Object
- .find_field(tap, name) ⇒ Object
- .find_output_field(tap, name) ⇒ Object
- .get_mandatory_fields(tap) ⇒ Object
- .has_field?(tap, name) ⇒ Boolean
- .has_output_field?(tap, name) ⇒ Boolean
- .prepare_for_es_downloader(tap) ⇒ Object
- .prepare_for_s3_backup(tap) ⇒ Object
-
.prepare_for_sf_downloader(tap) ⇒ Object
This method prepares the tap for data after downloading from SF but BEFORE it goes to ES We want to be able to load to ES as unchanged data as posible.
- .pull_id_from_output_to_input(tap) ⇒ Object
- .pull_timestamp_from_output_to_input(tap) ⇒ Object
- .remove_acts_as(tap) ⇒ Object
- .remove_field(tap, name) ⇒ Object
- .remove_timestamp_field(tap) ⇒ Object
- .source_target_mappging(tap) ⇒ Object
- .validate(tap) ⇒ Object
Class Method Details
.add_field(tap, field) ⇒ Object
14 15 16 17 18 19 20 21 |
# File 'lib/base/tap.rb', line 14 def self.add_field(tap, field) fail "You have to specify name. You specified \"#{field}\"" unless field.respond_to?(:has_key?) && field.has_key?(:name) fail "There already is a field named #{field[:name]}" if has_field?(tap, field[:name]) position = field[:position] || 0 fields = tap[:fields].clone fields.insert(position - 1, field) Tap.create(tap.merge({:fields => fields})) end |
.add_source_type(tap) ⇒ Object
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 |
# File 'lib/base/tap.rb', line 28 def self.add_source_type(tap) source = tap[:source].to_s path = Pathname(source) uri = URI(source) if uri.absolute? && (uri.scheme == "http" || uri.scheme == "https") tap.merge({ :source => uri, :source_type => :web }) elsif source =~ /^\./ || source =~ /^\// tap.merge({ :source => path, :source_type => :file }) elsif source == :dummy tap.merge({ :source_type => :dummy }) else tap.merge({ :source_type => :salesforce }) end end |
.add_timestamp_field(tap) ⇒ Object
80 81 82 83 84 85 86 |
# File 'lib/base/tap.rb', line 80 def self.(tap) if has_field?(tap, "Timestamp") tap else add_field(tap, {:name => "Timestamp"}) end end |
.assert_presence_of_id_field(tap) ⇒ Object
Every incremental tap has to have
182 183 184 185 186 187 |
# File 'lib/base/tap.rb', line 182 def self.assert_presence_of_id_field(tap) id_present = Tap.has_output_field?(tap, "Id") direct = tap[:direct] fail IdInTapNotPresentError.new(tap) if (!id_present && !direct) end |
.collect_validations(tap) ⇒ Object
132 133 134 |
# File 'lib/base/tap.rb', line 132 def self.collect_validations(tap) tap[:fields].find_all {|f| f.has_key?(:validates_as)} end |
.create(tap) ⇒ Object
6 7 8 9 10 11 12 |
# File 'lib/base/tap.rb', line 6 def self.create(tap) tap = ({:fields => [], :direct => false}.merge(tap).merge(:type => :tap)) tap = Tap.add_source_type(tap) fail "Tap has to have id defined" if tap[:id].nil? tap end |
.find_field(tap, name) ⇒ Object
58 59 60 |
# File 'lib/base/tap.rb', line 58 def self.find_field(tap, name) tap[:fields].detect {|f| f[:name] == name} end |
.find_output_field(tap, name) ⇒ Object
62 63 64 65 66 67 68 69 70 |
# File 'lib/base/tap.rb', line 62 def self.find_output_field(tap, name) acts_as_field = tap[:fields].detect {|f| f[:acts_as] && f[:acts_as].include?(name)} source_field = Tap.find_field(tap, name) if acts_as_field.blank? source_field if source_field && (!source_field.has_key?(:acts_as) || !source_field[:acts_as].include(name)) else acts_as_field end end |
.get_mandatory_fields(tap) ⇒ Object
173 174 175 |
# File 'lib/base/tap.rb', line 173 def self.get_mandatory_fields(tap) tap[:fields].reject {|f| f[:is_mandatory] == false } end |
.has_field?(tap, name) ⇒ Boolean
54 55 56 |
# File 'lib/base/tap.rb', line 54 def self.has_field?(tap, name) !Tap.find_field(tap, name).nil? end |
.has_output_field?(tap, name) ⇒ Boolean
72 73 74 |
# File 'lib/base/tap.rb', line 72 def self.has_output_field?(tap, name) !find_output_field(tap, name).nil? end |
.prepare_for_es_downloader(tap) ⇒ Object
161 162 163 164 165 166 167 |
# File 'lib/base/tap.rb', line 161 def self.prepare_for_es_downloader(tap) tap = Tap.pull_id_from_output_to_input(tap) tap = Tap.(Tap.(tap)) tap = tap[:snapshots] == true ? Tap.add_field(tap, {:name => "Snapshot"}) : tap tap = tap[:has_deleted_records] == true ? Tap.add_field(tap, {:name => "IsDeleted"}) : tap tap = tap[:has_deleted_records] == true ? Tap.add_field(tap, {:name => "DeletedAt"}) : tap end |
.prepare_for_s3_backup(tap) ⇒ Object
169 170 171 |
# File 'lib/base/tap.rb', line 169 def self.prepare_for_s3_backup(tap) Tap.remove_acts_as(tap) end |
.prepare_for_sf_downloader(tap) ⇒ Object
This method prepares the tap for data after downloading from SF but BEFORE it goes to ES We want to be able to load to ES as unchanged data as posible. Only Id and Timestamp (if provided is needed) other acts as should be used on the output of ES. The reason is that even if we change acts_as we should still pick up the same data
147 148 149 150 151 152 153 154 155 156 157 158 159 |
# File 'lib/base/tap.rb', line 147 def self.prepare_for_sf_downloader(tap) id_field = Tap.find_output_field(tap, "Id") Tap.validate(tap) time_field = Tap.find_output_field(tap, "Timestamp") tap = Tap.remove_acts_as(tap) fields = tap[:fields].map {|f| f[:name] == id_field[:name] ? f.merge(:acts_as => ["Id"]) : f} if time_field fields = fields.map {|f| f[:name] == time_field[:name] ? f.merge(:acts_as => ["Timestamp"]) : f} end Tap.create(tap.merge({ :fields => fields })) end |
.pull_id_from_output_to_input(tap) ⇒ Object
106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 |
# File 'lib/base/tap.rb', line 106 def self.pull_id_from_output_to_input(tap) if has_field?(tap, "Id") tap elsif has_output_field?(tap, "Id") field = find_output_field(tap, "Id") fields = tap[:fields].clone index = fields.index(field) field = field.clone field[:name] = "Id" fields.delete_at(index) fields.insert(index, field) Tap.create(tap.merge(:fields => fields)) else tap end end |
.pull_timestamp_from_output_to_input(tap) ⇒ Object
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 |
# File 'lib/base/tap.rb', line 88 def self.(tap) if has_field?(tap, "Timestamp") tap elsif has_output_field?(tap, "Timestamp") field = find_output_field(tap, "Timestamp") fields = tap[:fields].clone index = fields.index(field) field = field.clone field[:name] = "Timestamp" fields.delete_at(index) fields.insert(index, field) Tap.create(tap.merge(:fields => fields)) else tap end end |
.remove_acts_as(tap) ⇒ Object
136 137 138 139 140 |
# File 'lib/base/tap.rb', line 136 def self.remove_acts_as(tap) Tap.create(tap.merge({ :fields => tap[:fields].map {|f| f = f.clone; f.delete_if {|key, value| key == :acts_as }} })) end |
.remove_field(tap, name) ⇒ Object
23 24 25 26 |
# File 'lib/base/tap.rb', line 23 def self.remove_field(tap, name) fields = tap[:fields].find_all {|f| f[:name] != name} Tap.create(tap.merge({:fields => fields})) end |
.remove_timestamp_field(tap) ⇒ Object
76 77 78 |
# File 'lib/base/tap.rb', line 76 def self.(tap) remove_field(tap, "Timestamp") end |
.source_target_mappging(tap) ⇒ Object
125 126 127 128 129 130 |
# File 'lib/base/tap.rb', line 125 def self.source_target_mappging(tap) tap[:fields].reduce([]) do |acc, f| transformation = f[:acts_as] ? f[:acts_as].map {|i| [f[:name], i]} : [[f[:name], f[:name]]] acc.concat(transformation) end end |
.validate(tap) ⇒ Object
177 178 179 |
# File 'lib/base/tap.rb', line 177 def self.validate(tap) assert_presence_of_id_field(tap) end |