Class: Myreplicator::Export

Inherits:
ActiveRecord::Base
  • Object
show all
Defined in:
app/models/myreplicator/export.rb

Defined Under Namespace

Classes: SourceDb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#filename ⇒ Object (readonly)

Returns the value of attribute filename.



30
31
32
 
# File 'app/models/myreplicator/export.rb', line 30

def filename
  @filename
end

Class Method Details

.available_dbs ⇒ Object

List of all avaiable databases from database.yml file All Export/Load jobs can use these databases



227
228
229
230
231
232
233
234
235
236
237
238
 
# File 'app/models/myreplicator/export.rb', line 227

def self.available_dbs
  dbs = ActiveRecord::Base.configurations.keys
  available = [] 

  dbs.each do |db|
    db_config = ActiveRecord::Base.configurations[db]
    unless db_config["myreplicator"].nil?
      available << db if db_config["myreplicator"]
    end
  end
  return available
end

.available_tables ⇒ Object

Returns a hash of => [TableName1,…], DB => …



214
215
216
217
218
219
220
221
 
# File 'app/models/myreplicator/export.rb', line 214

def self.available_tables
  metadata = {}
  available_dbs.each do |db|
    tables = SourceDb.get_tables(db)
    metadata[db] = tables
  end
  return metadata
end

.perform(export_id, *args) ⇒ Object

Perfoms the export job, Provided for Resque



37
38
39
40
41
42
43
44
45
46
47
 
# File 'app/models/myreplicator/export.rb', line 37

def self.perform(export_id, *args)
  options = args.extract_options!
  ActiveRecord::Base.verify_active_connections!
  ActiveRecord::Base.connection.reconnect!
  export_obj = Export.find(export_id)

  if export_obj.active
    export_obj.export
  end

end

.schedule_in_resque ⇒ Object

NOTE: Provided for Resque use Schedules all the exports in resque Requires Resque Scheduler



245
246
247
248
249
250
251
252
253
254
255
 
# File 'app/models/myreplicator/export.rb', line 245

def self.schedule_in_resque
  exports = Export.find(:all)
  exports.each do |export|
    if export.active
      export.schedule
    else
      Resque.remove_schedule(export.schedule_name)
    end
  end
  Resque.reload_schedule! # Reload all schedules in Resque
end

Instance Method Details

#connection_factory(type) ⇒ Object

Connects to the server via ssh/sftp



189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
 
# File 'app/models/myreplicator/export.rb', line 189

def connection_factory type
  config = Myreplicator.configs[self.source_schema]
  puts self.source_schema
  puts config
  case type
  when :ssh
    if config.has_key? "ssh_password"
      return Net::SSH.start(config["ssh_host"], config["ssh_user"], :password => config["ssh_password"])

    elsif(config.has_key? "ssh_private_key")
      return Net::SSH.start(config["ssh_host"], config["ssh_user"], :keys => [config["ssh_private_key"]])
    end
  when :sftp
    if config.has_key? "ssh_password"
      return Net::SFTP.start(config["ssh_host"], config["ssh_user"], :password => config["ssh_password"])

    elsif(config.has_key? "ssh_private_key")
      return Net::SFTP.start(config["ssh_host"], config["ssh_user"], :keys => [config["ssh_private_key"]])
    end          
  end
end

#destination_max_incremental_value ⇒ Object 



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
 
# File 'app/models/myreplicator/export.rb', line 106

def destination_max_incremental_value      
  if self.export_to == 'vertica'
    sql = SqlCommands.max_value_vsql(:incremental_col => self.incremental_column,
                                          :max_incremental_value => self.max_incremental_value,
                                          :db => self.destination_schema,
                                          :incremental_col_type => self.incremental_column_type,
                                          :table => self.table_name)
    puts sql
    begin
      result = Myreplicator::DB.exec_sql('vertica',sql)
      if result.rows.first[:max].blank?
        return "0"
      else
        case result.rows.first[:max].class.to_s
        when "DateTime"
          return result.rows.first[:max].strftime('%Y-%m-%d %H:%M:%S')
        else
          return result.rows.first[:max].to_s
        end
      end
    rescue Exception => e
      puts "Vertica Table Not Existed"
    end
  else
    begin
      sql = SqlCommands.max_value_sql(:incremental_col => self.incremental_column,
                                            :max_incremental_value => self.max_incremental_value,
                                            :db => self.source_schema,
                                            :incremental_col_type => self.incremental_column_type,
                                            :table => self.table_name)
      puts sql
      result = Myreplicator::DB.exec_sql(self.destination_schema,sql)
      if result.first.nil?
        return "0"
      else
        return result.first.first
      end
    end
  end
  return "0"
end

#exec_on_source(sql) ⇒ Object 



171
172
173
174
 
# File 'app/models/myreplicator/export.rb', line 171

def exec_on_source sql
  result = SourceDb.exec_sql(self.source_schema, sql)
  return result
end

#export ⇒ Object

Runs the export process using the required Exporter library



69
70
71
72
73
74
75
 
# File 'app/models/myreplicator/export.rb', line 69

def export
  Log.run(:job_type => "export", :name => schedule_name, 
          :file => filename, :export_id => id) do |log|
    exporter = MysqlExporter.new
    exporter.export_table self # pass current object to exporter
  end
end

#export_type? ⇒ Boolean

end

Returns:

  • (Boolean) 


87
88
89
90
91
92
93
 
# File 'app/models/myreplicator/export.rb', line 87

def export_type?
  if state == "new"
    return :new
  elsif incremental_export?
    return :incremental
  end
end

#incremental_export? ⇒ Boolean

Returns:

  • (Boolean) 


95
96
97
98
99
100
 
# File 'app/models/myreplicator/export.rb', line 95

def incremental_export?
  if export_type == "incremental"
    return true
  end
  return false
end

#is_running? ⇒ Boolean

Throws ExportIgnored if the job is still running Checks the state of the job using PID and state

Returns:

  • (Boolean) 


280
281
282
283
284
285
286
287
288
 
# File 'app/models/myreplicator/export.rb', line 280

def is_running?
  return false if state != "exporting"
  begin
    Process.getpgid(exporter_pid)
    raise Exceptions::ExportIgnored.new("Ignored")
  rescue Errno::ESRCH
    return false
  end
end

#max_value ⇒ Object 



148
149
150
151
152
153
154
155
156
157
158
159
160
 
# File 'app/models/myreplicator/export.rb', line 148

def max_value
  sql = SqlCommands.max_value_sql(:incremental_col => self.incremental_column,
                                  :max_incremental_value => self.max_incremental_value,
                                  :db => self.source_schema,
                                  :incremental_col_type => self.incremental_column_type,
                                  :table => self.table_name)
  result = exec_on_source(sql)
  if result.first.nil?
    return "0"
  else
    return result.first.first
  end
end

#reload ⇒ Object 



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'app/models/myreplicator/export.rb', line 49

def reload
  
  # TRUNCATE TABLE & Reset incremental value if there is any
  sql = "TRUNCATE TABLE #{self.destination_schema}.#{self.table_name};"
  if self.export_to == "vertica"
    Myreplicator::DB.exec_sql("vertica",sql)
  else
    Myreplicator::DB.exec_sql("#{self.destination_schema}",sql)
  end
    
  if self.export_type != "all"
    self.max_incremental_value = nil
    self.save!
  end
    
  Resque.enqueue(Myreplicator::Export, id)
end

#schedule ⇒ Object

Schedules the export job in Resque



267
268
269
270
271
272
273
274
 
# File 'app/models/myreplicator/export.rb', line 267

def schedule
  Resque.set_schedule(schedule_name, {
                        :cron => cron,
                        :class => "Myreplicator::Export",
                        :queue => "myreplicator_export",
                        :args => id
                      })
end

#schedule_name ⇒ Object

Name used for the job in Resque



260
261
262
 
# File 'app/models/myreplicator/export.rb', line 260

def schedule_name
  name = "#{source_schema}_#{destination_schema}_#{table_name}"
end

#sftp_to_source ⇒ Object 



181
182
183
184
 
# File 'app/models/myreplicator/export.rb', line 181

def sftp_to_source
  puts "Connecting SFTP..."
  return connection_factory(:sftp)
end

#ssh_to_source ⇒ Object 



176
177
178
179
 
# File 'app/models/myreplicator/export.rb', line 176

def ssh_to_source
  puts "Connecting SSH..."
  return connection_factory(:ssh) 
end

#update_max_val(max_val = nil) ⇒ Object 



162
163
164
165
166
167
168
169
 
# File 'app/models/myreplicator/export.rb', line 162

def update_max_val(max_val = nil)
  if max_val.nil?
    self.max_incremental_value = max_value
  else
    self.max_incremental_value = max_val
    self.save!
  end
end