Class: ExternalMigration::Migration

Inherits:
Object
  • Object
show all
Defined in:
lib/external_migration/migration.rb

Overview

Migrate data between data source and transforme to destination

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(schema_url = nil) ⇒ Migration

constructor



78
79
80
 
# File 'lib/external_migration/migration.rb', line 78

def initialize(schema_url=nil)
  self.load_schema(schema_url) unless schema_url.nil?
end

Instance Attribute Details

#name ⇒ Object

Returns the value of attribute name.



75
76
77
 
# File 'lib/external_migration/migration.rb', line 75

def name
  @name
end

#processor ⇒ Object

Returns the value of attribute processor.



75
76
77
 
# File 'lib/external_migration/migration.rb', line 75

def processor
  @processor
end

#schema_from ⇒ Object

Returns the value of attribute schema_from.



75
76
77
 
# File 'lib/external_migration/migration.rb', line 75

def schema_from
  @schema_from
end

#schema_to ⇒ Object

Returns the value of attribute schema_to.



75
76
77
 
# File 'lib/external_migration/migration.rb', line 75

def schema_to
  @schema_to
end

#transformer ⇒ Object

Returns the value of attribute transformer.



75
76
77
 
# File 'lib/external_migration/migration.rb', line 75

def transformer
  @transformer
end

Instance Method Details

#begin_migration ⇒ Object 



192
193
194
195
 
# File 'lib/external_migration/migration.rb', line 192

def begin_migration
  # TODO: make transactional
  res = @transformer.begin(@schema_from, @schema_to) unless @transformer.nil?
end

#end_migration ⇒ Object 



198
199
200
 
# File 'lib/external_migration/migration.rb', line 198

def end_migration
  res = @transformer.end(@schema_from, @schema_to) unless @transformer.nil?
end

#load_schema(url) ⇒ Object

load yml schemas from and to



83
84
85
86
87
 
# File 'lib/external_migration/migration.rb', line 83

def load_schema(url)
  schema = YAML::load(File.open(url)).keys_to_sym
  self.schema_from = schema[:from]
  self.schema_to = schema[:to]
end

#load_schema_from(url) ⇒ Object

Deprecated.

loads yml file and convert to hash on schema_from



232
233
234
 
# File 'lib/external_migration/migration.rb', line 232

def load_schema_from(url)
  self.schema_from = YAML::load(File.open(url))
end

#load_schema_to(url) ⇒ Object

Deprecated.

load yml file and convert to hash on schema_to



239
240
241
 
# File 'lib/external_migration/migration.rb', line 239

def load_schema_to(url)
  self.schema_to = YAML::load(File.open(url))      
end

#migrate! ⇒ Object

Running migration from configured files

ps> Default Behaviour Ignore First line - assumes head line



95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/external_migration/migration.rb', line 95

def migrate!

  raise "schema_from needs" if @schema_from.nil?
  raise "schema_to needs" if @schema_to.nil?
  
  res = @transformer.begin_transaction(@schema_from, @schema_to) unless @transformer.nil?
  
  @line = 0
  
  ActiveRecord::Base.transaction do      
    begin_migration()

    # TODO: Make flexible configurable and more input formats
    case @schema_from[:format].to_s.to_sym 
    when :XLS
      xls_migrate()
    when :TXT_FIXED
      decoder = ExternalMigration::TextFixed.new(@schema_from)
      decoder.migration = self
      decoder.migrate!
    end
  
    end_migration()
  end
  
  res = @transformer.end_transaction(@schema_from, @schema_to) unless @transformer.nil?
  
  return true
end

#migrate_row!(row_to) ⇒ Object 



125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/external_migration/migration.rb', line 125

def migrate_row!(row_to)
  @line += 1
  begin
    if @processor.nil?
      #transform row to @schema_to
      res = true
      res = @transformer.transform(row_to) unless @transformer.nil?
    
      if (res!=:ignore)
        res = res==true && send_row_to_schema(row_to)
        raise_migration if (res==false)
      
        @transformer.after_row_saved(row_to, @last_object) unless @transformer.nil?
      end
    else
      @processor.migrate_row row_to
    end
  rescue Exception => e
    line = @line.nil? ? 0 : @line
    column = @column.nil? ? 0 : @column
    
    obj = @last_object || (e.respond_to?(:record) && (e.record)) || nil
    
    if !obj.nil?
      raise ActiveMigartionDataSourceError.new obj.errors.to_yaml.to_s.concat("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)         
    else
      raise ActiveMigartionDataSourceError.new ("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
    end
  end

end

#raise_migration ⇒ Object 



202
203
204
 
# File 'lib/external_migration/migration.rb', line 202

def raise_migration
  raise "failing migration %s.  Line: %d, Column: %d" % [@name, @line, @column]
end

#send_row_to_schema(row) ⇒ Object 



206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/external_migration/migration.rb', line 206

def send_row_to_schema(row)
  
  if @schema_to[:format].to_sym == :ACTIVE_RECORD
    
    # TODO: optimize on initialize migration
    class_schema_to = eval @schema_to[:url]
    
    @last_object = class_schema_to.new(row)
    res = @last_object.save
  
    if (!res)
      msg = "[Schema:%s] Error on send to ACTIVE_RECORD %s. \n%s \nrow: \n%s" % [@name, @schema_to[:url], @last_object.errors.to_yaml, row.to_yaml]
      Rails.logger.error msg
      raise AciteMigrationInvalidRecordError.new msg
    end
  
    return res
  
  else
    raise "Not valid schema::TO format! %s" % @name  
  end
end

#xls_migrate ⇒ Object 



158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
 
# File 'lib/external_migration/migration.rb', line 158

def xls_migrate
  begin
    @xls = Spreadsheet.open @schema_from[:url]
    # TODO: make others workbook accessible by configuration
    sheet = @xls.worksheet 0

    # ignore head line
    sheet.each 1 do |row|
      @column = 0
      row_to = { }
      
      #read schema columns and types
      @schema_from[:columns].each do |schema_column, schema_type|
        row_to.merge!(schema_column.to_sym => row[@column])
        @column+=1
      end
      
      self.migrate_row! row_to
    end
  rescue Exception => e
    line = @line.nil? ? 0 : @line
    column = @column.nil? ? 0 : @column
    
    obj = @last_object || (e.respond_to?(:record) && (e.record)) || nil
    
    if !obj.nil?
      raise ActiveMigartionDataSourceError.new obj.errors.to_yaml.to_s.concat("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)         
    else
      raise ActiveMigartionDataSourceError.new ("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
    end
  end
end