Class: Flydata::SyncFileManager
- Inherits:
-
Object
- Object
- Flydata::SyncFileManager
- Defined in:
- lib/flydata/sync_file_manager.rb
Constant Summary collapse
- DUMP_DIR =
ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
- BACKUP_DIR =
ENV['FLYDATA_BACKUP'] || File.join(FLYDATA_HOME, 'backup')
- TABLE_POSITIONS_DIR =
ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
- SYNC_TABLE_POSITIONS =
0
Instance Method Summary collapse
- #backup_dir ⇒ Object
- #backup_dump_dir ⇒ Object
- #binlog_path ⇒ Object
- #close ⇒ Object
- #delete_dump_file ⇒ Object
- #delete_table_binlog_pos(table_name) ⇒ Object
- #dump_file_path ⇒ Object
-
#dump_pos_path ⇒ Object
dump pos file for resume.
- #get_new_table_list(tables, file_type) ⇒ Object
- #get_table_binlog_pos(table_name) ⇒ Object
- #get_table_binlog_pos_init(table_name) ⇒ Object
- #get_table_position(table_name) ⇒ Object
-
#increment_and_save_table_position(table_name) {|seq| ... } ⇒ Object
Read a sequence number from the table’s position file, increment the number and pass the number to a block.
- #increment_table_rev(table_name, base_rev) ⇒ Object
-
#initialize(data_entry) ⇒ SyncFileManager
constructor
A new instance of SyncFileManager.
- #install_table_binlog_files(tables) ⇒ Object
- #load_binlog(file_path = binlog_path) ⇒ Object
- #load_dump_pos ⇒ Object
- #load_generated_ddl(tables) ⇒ Object
- #load_sent_binlog(file_path = sent_binlog_path) ⇒ Object
- #load_stats ⇒ Object
- #load_sync_info ⇒ Object
-
#mysql_table_marshal_dump_path ⇒ Object
MysqlTable marshal file.
-
#reset_table_position_files(tables) ⇒ Object
table files.
-
#save_binlog(binlog_pos) ⇒ Object
master binlog.pos file.
- #save_dump_pos(status, table_name, last_pos, binlog_pos, state = nil, substate = nil) ⇒ Object
- #save_generated_ddl(tables, contents = "1") ⇒ Object
- #save_mysql_table_marshal_dump(mysql_table) ⇒ Object
- #save_record_count_stat(table, record_count) ⇒ Object
-
#save_sent_binlog(binlog_pos) ⇒ Object
sent binlog.pos file.
- #save_ssl_ca(ssl_ca_content, path = ssl_ca_path) ⇒ Object
- #save_ssl_cipher(ssl_cipher_content, path = ssl_cipher_path) ⇒ Object
- #save_sync_info(initial_sync, tables) ⇒ Object
- #save_table_binlog_pos(tables, binlog_pos, options = {}) ⇒ Object
- #save_table_position(table_name, seq) ⇒ Object
- #sent_binlog_path(master_binlog_path = binlog_path) ⇒ Object
-
#ssl_ca_path(master_binlog_path = binlog_path) ⇒ Object
ssl_ca file path.
-
#ssl_cipher_path(master_binlog_path = binlog_path) ⇒ Object
ssl_cipher file path.
- #stats_path ⇒ Object
- #sync_info_file ⇒ Object
- #table_binlog_pos_init_paths(*tables) ⇒ Object
- #table_binlog_pos_paths(*tables) ⇒ Object
- #table_ddl_file_paths(*tables) ⇒ Object
- #table_position_file_paths(*tables) ⇒ Object
- #table_positions_dir_path ⇒ Object
- #table_rev(table_name) ⇒ Object
- #table_rev_file_path(table_name) ⇒ Object
- #table_rev_file_paths(*tables) ⇒ Object
Constructor Details
#initialize(data_entry) ⇒ SyncFileManager
Returns a new instance of SyncFileManager.
11 12 13 14 15 |
# File 'lib/flydata/sync_file_manager.rb', line 11 def initialize(data_entry) @data_entry = data_entry @table_position_files = {} # File objects keyed by table name @sync_table_positions_count = SYNC_TABLE_POSITIONS end |
Instance Method Details
#backup_dir ⇒ Object
357 358 359 |
# File 'lib/flydata/sync_file_manager.rb', line 357 def backup_dir BACKUP_DIR end |
#backup_dump_dir ⇒ Object
347 348 349 350 351 352 353 354 355 |
# File 'lib/flydata/sync_file_manager.rb', line 347 def backup_dump_dir backup_dir = BACKUP_DIR.dup FileUtils.mkdir_p(backup_dir) unless Dir.exists?(backup_dir) dest_dir = File.join(backup_dir, Time.now.strftime("%Y%m%d%H%M%S")) FileUtils.mkdir(dest_dir) %w(info pos stats mysql_table).each do |ext| FileUtils.mv(Dir.glob("#{dump_dir}/*.#{ext}"), dest_dir) end end |
#binlog_path ⇒ Object
107 108 109 |
# File 'lib/flydata/sync_file_manager.rb', line 107 def binlog_path File.join(FLYDATA_HOME, @data_entry['name'] + ".binlog.pos") end |
#close ⇒ Object
17 18 19 20 |
# File 'lib/flydata/sync_file_manager.rb', line 17 def close @table_position_files.values.each {|f| f.close } @table_position_files = {} end |
#delete_dump_file ⇒ Object
343 344 345 |
# File 'lib/flydata/sync_file_manager.rb', line 343 def delete_dump_file FileUtils.rm(dump_file_path) if File.exists?(dump_file_path) end |
#delete_table_binlog_pos(table_name) ⇒ Object
297 298 299 300 301 302 303 304 |
# File 'lib/flydata/sync_file_manager.rb', line 297 def delete_table_binlog_pos(table_name) file = File.join(table_positions_dir_path, table_name + ".binlog.pos") if File.exists?(file) FileUtils.rm(file, :force => true) else puts "#{file} does not exist. Something is wrong. Did you delete the file manually when flydata was running?" end end |
#dump_file_path ⇒ Object
22 23 24 |
# File 'lib/flydata/sync_file_manager.rb', line 22 def dump_file_path File.join(dump_dir, @data_entry['name']) + ".dump" end |
#dump_pos_path ⇒ Object
dump pos file for resume
27 28 29 |
# File 'lib/flydata/sync_file_manager.rb', line 27 def dump_pos_path dump_file_path + ".pos" end |
#get_new_table_list(tables, file_type) ⇒ Object
72 73 74 75 76 77 78 79 |
# File 'lib/flydata/sync_file_manager.rb', line 72 def get_new_table_list(tables, file_type) table_positions_dir_path = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions') new_tables = [] tables.each do |table| new_tables << table unless File.exists?(File.join(table_positions_dir_path, "#{table}.#{file_type}")) end new_tables end |
#get_table_binlog_pos(table_name) ⇒ Object
322 323 324 325 326 |
# File 'lib/flydata/sync_file_manager.rb', line 322 def get_table_binlog_pos(table_name) file = File.join(table_positions_dir_path, table_name + ".binlog.pos") return nil unless File.exists?(file) File.open(file, 'r').readline end |
#get_table_binlog_pos_init(table_name) ⇒ Object
260 261 262 263 264 |
# File 'lib/flydata/sync_file_manager.rb', line 260 def get_table_binlog_pos_init(table_name) file = File.join(table_positions_dir_path, table_name + ".binlog.pos.init") return nil unless File.exists?(file) File.open(file, 'r').readline end |
#get_table_position(table_name) ⇒ Object
236 237 238 239 240 241 |
# File 'lib/flydata/sync_file_manager.rb', line 236 def get_table_position(table_name) file = File.join(table_positions_dir_path, table_name + ".pos") return nil unless File.exists?(file) File.open(file) {|f| f.read} end |
#increment_and_save_table_position(table_name) {|seq| ... } ⇒ Object
Read a sequence number from the table’s position file, increment the number and pass the number to a block. After executing the block, saves the value to the position file.
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 |
# File 'lib/flydata/sync_file_manager.rb', line 192 def increment_and_save_table_position(table_name) file = File.join(table_positions_dir_path, table_name + ".pos") retry_count = 0 begin @table_position_files[table_name] ||= (f = File.open(file, File::RDWR); f.sync = true; f) rescue Errno::ENOENT raise if retry_count > 0 # Already retried. Must be a differentfile causing the error # File not exist. Create one with initial value of '0' File.open(file, "w") {|f| f.write('0') } retry_count += 1 retry end f = @table_position_files[table_name] seq = f.read prev_seq_len = seq.size f.rewind seq = seq.to_i + 1 seq = FlydataCore::QueryJob::SYNC_FIRST_SEQ if seq == 1 new_seq_len = seq.to_s.size seq_to_write = seq.to_s if new_seq_len < prev_seq_len seq_to_write += " " * (prev_seq_len - new_seq_len) end # logical transaction starts yield(seq) f.write(seq_to_write) if @sync_table_positions_count > 1 @sync_table_positions_count -= 1 elsif @sync_table_positions_count == 1 f.fsync @sync_table_positions_count = SYNC_TABLE_POSITIONS end # logical transaction ends f.truncate(new_seq_len) if new_seq_len < prev_seq_len f.rewind end |
#increment_table_rev(table_name, base_rev) ⇒ Object
288 289 290 291 292 293 294 295 |
# File 'lib/flydata/sync_file_manager.rb', line 288 def increment_table_rev(table_name, base_rev) file = table_rev_file_path(table_name) new_rev = base_rev + 1 File.open(file, "w") do |f| f.write(new_rev) end new_rev end |
#install_table_binlog_files(tables) ⇒ Object
328 329 330 331 332 333 334 335 336 337 338 339 340 341 |
# File 'lib/flydata/sync_file_manager.rb', line 328 def install_table_binlog_files(tables) FileUtils.mkdir_p(table_positions_dir_path) unless Dir.exists?(table_positions_dir_path) tables.each do |table_name| file_name = table_name + ".binlog.pos" src_file = File.join(dump_dir, file_name) if ! File.exists?(src_file) raise "#{src_file} does not exist. Error!!" end FileUtils.mv(src_file, table_positions_dir_path) # save the position at initial sync. this is used for repair if # necessary. FileUtils.cp(File.join(table_positions_dir_path, file_name), File.join(table_positions_dir_path, file_name + ".init")) end end |
#load_binlog(file_path = binlog_path) ⇒ Object
100 101 102 103 104 105 |
# File 'lib/flydata/sync_file_manager.rb', line 100 def load_binlog(file_path = binlog_path) return nil unless File.exists?(file_path) f, pos = IO.read(file_path).strip.split("\t") return nil if f.nil? || f.empty? || pos.nil? { binfile: f, pos: pos.to_i } end |
#load_dump_pos ⇒ Object
37 38 39 40 41 42 43 44 45 46 |
# File 'lib/flydata/sync_file_manager.rb', line 37 def load_dump_pos path = dump_pos_path return {} unless File.exists?(path) items = File.open(path, 'r').readline.split("\t") raise "Invalid dump.pos file: #{path}" unless items.length >= 5 && items.length <= 7 mysql_table = load_mysql_table_marshal_dump { status: items[0], table_name: items[1], last_pos: items[2].to_i, binlog_pos: {binfile: items[3], pos: items[4].to_i}, state: items[5], substate: items[6], mysql_table: mysql_table} end |
#load_generated_ddl(tables) ⇒ Object
48 49 50 51 52 53 54 55 56 57 58 |
# File 'lib/flydata/sync_file_manager.rb', line 48 def load_generated_ddl(tables) tables = [ tables ] unless tables.kind_of?(Array) paths = table_ddl_file_paths(*tables) paths.collect{|path| begin File.open(path) {|f| f.read } rescue Errno::ENOENT nil end } end |
#load_sent_binlog(file_path = sent_binlog_path) ⇒ Object
118 119 120 121 122 123 |
# File 'lib/flydata/sync_file_manager.rb', line 118 def load_sent_binlog(file_path = sent_binlog_path) return nil unless File.exists?(file_path) f, pos = IO.read(file_path).strip.split("\t") return nil if f.nil? || f.empty? || pos.nil? { binfile: f, pos: pos.to_i } end |
#load_stats ⇒ Object
371 372 373 374 |
# File 'lib/flydata/sync_file_manager.rb', line 371 def load_stats return nil unless File.exists?(stats_path) Hash[*File.read(stats_path).split(/\t/)] end |
#load_sync_info ⇒ Object
253 254 255 256 257 258 |
# File 'lib/flydata/sync_file_manager.rb', line 253 def load_sync_info return nil unless File.exists?(sync_info_file) items = File.open(sync_info_file, 'r').readline.split("\t") { initial_sync: (items[0] == 'true'), tables: items[1].split(" ") } end |
#mysql_table_marshal_dump_path ⇒ Object
MysqlTable marshal file
82 83 84 |
# File 'lib/flydata/sync_file_manager.rb', line 82 def mysql_table_marshal_dump_path dump_file_path + ".mysql_table" end |
#reset_table_position_files(tables) ⇒ Object
table files
157 158 159 160 161 162 |
# File 'lib/flydata/sync_file_manager.rb', line 157 def reset_table_position_files(tables) tables.each do |table_name| file = File.join(table_positions_dir_path, table_name + ".pos") File.open(file, "w") {|f| f.write('0') } end end |
#save_binlog(binlog_pos) ⇒ Object
master binlog.pos file
93 94 95 96 97 98 |
# File 'lib/flydata/sync_file_manager.rb', line 93 def save_binlog(binlog_pos) path = binlog_path File.open(path, 'w') do |f| f.write(binlog_content(binlog_pos)) end end |
#save_dump_pos(status, table_name, last_pos, binlog_pos, state = nil, substate = nil) ⇒ Object
31 32 33 34 35 |
# File 'lib/flydata/sync_file_manager.rb', line 31 def save_dump_pos(status, table_name, last_pos, binlog_pos, state = nil, substate = nil) File.open(dump_pos_path, 'w') do |f| f.write(dump_pos_content(status, table_name, last_pos, binlog_pos, state, substate)) end end |
#save_generated_ddl(tables, contents = "1") ⇒ Object
60 61 62 63 64 65 66 67 68 69 70 |
# File 'lib/flydata/sync_file_manager.rb', line 60 def save_generated_ddl(tables, contents = "1") tables = [ tables ] unless tables.kind_of?(Array) table_positions_dir_path = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions') #Create positions if dir does not exist unless File.directory?(table_positions_dir_path) FileUtils.mkdir_p(table_positions_dir_path) end tables.each do |tab| File.open(File.join(table_positions_dir_path, "#{tab}.generated_ddl"), 'w') {|f| f.write(contents) } end end |
#save_mysql_table_marshal_dump(mysql_table) ⇒ Object
86 87 88 89 90 |
# File 'lib/flydata/sync_file_manager.rb', line 86 def save_mysql_table_marshal_dump(mysql_table) File.open(mysql_table_marshal_dump_path, 'w') do |f| f.write Marshal.dump(mysql_table) end end |
#save_record_count_stat(table, record_count) ⇒ Object
365 366 367 368 369 |
# File 'lib/flydata/sync_file_manager.rb', line 365 def save_record_count_stat(table, record_count) stats = load_stats || Hash.new stats[table] = stats[table] ? stats[table].to_i + record_count : record_count save_stats(stats) end |
#save_sent_binlog(binlog_pos) ⇒ Object
sent binlog.pos file
112 113 114 115 116 |
# File 'lib/flydata/sync_file_manager.rb', line 112 def save_sent_binlog(binlog_pos) File.open(sent_binlog_path, 'w') do |f| f.write(binlog_content(binlog_pos)) end end |
#save_ssl_ca(ssl_ca_content, path = ssl_ca_path) ⇒ Object
137 138 139 140 141 |
# File 'lib/flydata/sync_file_manager.rb', line 137 def save_ssl_ca(ssl_ca_content, path = ssl_ca_path) File.open(path, 'w') do |f| f.write(ssl_ca_content) end end |
#save_ssl_cipher(ssl_cipher_content, path = ssl_cipher_path) ⇒ Object
150 151 152 153 154 |
# File 'lib/flydata/sync_file_manager.rb', line 150 def save_ssl_cipher(ssl_cipher_content, path = ssl_cipher_path) File.open(path, 'w') do |f| f.write(ssl_cipher_content) end end |
#save_sync_info(initial_sync, tables) ⇒ Object
247 248 249 250 251 |
# File 'lib/flydata/sync_file_manager.rb', line 247 def save_sync_info(initial_sync, tables) File.open(sync_info_file, "w") do |f| f.write([initial_sync, tables.join(" ")].join("\t")) end end |
#save_table_binlog_pos(tables, binlog_pos, options = {}) ⇒ Object
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 |
# File 'lib/flydata/sync_file_manager.rb', line 306 def save_table_binlog_pos(tables, binlog_pos, = {}) dest_dir = case [:destination] when :positions; table_positions_dir_path when :dump; dump_dir else dump_dir end tables = [ tables ] unless tables.kind_of?(Array) tables.each do |table_name| file = File.join(dest_dir, table_name + ".binlog.pos") File.open(file, "w") do |f| f.write(binlog_content(binlog_pos)) end end end |
#save_table_position(table_name, seq) ⇒ Object
230 231 232 233 234 |
# File 'lib/flydata/sync_file_manager.rb', line 230 def save_table_position(table_name, seq) file = File.join(table_positions_dir_path, table_name + ".pos") File.open(file, "w") {|f| f.write(seq)} end |
#sent_binlog_path(master_binlog_path = binlog_path) ⇒ Object
125 126 127 128 |
# File 'lib/flydata/sync_file_manager.rb', line 125 def sent_binlog_path(master_binlog_path = binlog_path) validate_master_binlog_path(master_binlog_path) "#{master_binlog_path[0..-5]}.sent.pos" end |
#ssl_ca_path(master_binlog_path = binlog_path) ⇒ Object
ssl_ca file path
131 132 133 134 135 |
# File 'lib/flydata/sync_file_manager.rb', line 131 def ssl_ca_path(master_binlog_path = binlog_path) validate_master_binlog_path(master_binlog_path) # <data-entry-name>.ssl_ca.pem "#{master_binlog_path[0..-12]}.ssl_ca.pem" end |
#ssl_cipher_path(master_binlog_path = binlog_path) ⇒ Object
ssl_cipher file path
144 145 146 147 148 |
# File 'lib/flydata/sync_file_manager.rb', line 144 def ssl_cipher_path(master_binlog_path = binlog_path) validate_master_binlog_path(master_binlog_path) # <data-entry-name>.ssl_cipher "#{master_binlog_path[0..-12]}.ssl_cipher" end |
#stats_path ⇒ Object
361 362 363 |
# File 'lib/flydata/sync_file_manager.rb', line 361 def stats_path File.join(dump_dir, @data_entry['name']) + ".stats" end |
#sync_info_file ⇒ Object
243 244 245 |
# File 'lib/flydata/sync_file_manager.rb', line 243 def sync_info_file File.join(dump_dir, "sync.info") end |
#table_binlog_pos_init_paths(*tables) ⇒ Object
183 184 185 186 |
# File 'lib/flydata/sync_file_manager.rb', line 183 def table_binlog_pos_init_paths(*tables) tables.empty? ? Dir.glob(File.join(table_positions_dir_path, '*.binlog.pos.init')) : tables.map{|table| File.join(table_positions_dir_path, table + '.binlog.pos.init')} end |
#table_binlog_pos_paths(*tables) ⇒ Object
178 179 180 181 |
# File 'lib/flydata/sync_file_manager.rb', line 178 def table_binlog_pos_paths(*tables) tables.empty? ? Dir.glob(File.join(table_positions_dir_path, '*.binlog.pos')) : tables.map{|table| File.join(table_positions_dir_path, table + '.binlog.pos')} end |
#table_ddl_file_paths(*tables) ⇒ Object
173 174 175 176 |
# File 'lib/flydata/sync_file_manager.rb', line 173 def table_ddl_file_paths(*tables) tables.empty? ? Dir.glob(File.join(table_positions_dir_path, '*.generated_ddl')) : tables.map{|table| File.join(table_positions_dir_path, table + '.generated_ddl')} end |
#table_position_file_paths(*tables) ⇒ Object
168 169 170 171 |
# File 'lib/flydata/sync_file_manager.rb', line 168 def table_position_file_paths(*tables) tables.empty? ? Dir.glob(File.join(table_positions_dir_path, '*.pos')) : tables.map{|table| File.join(table_positions_dir_path, table + '.pos')} end |
#table_positions_dir_path ⇒ Object
164 165 166 |
# File 'lib/flydata/sync_file_manager.rb', line 164 def table_positions_dir_path TABLE_POSITIONS_DIR end |
#table_rev(table_name) ⇒ Object
275 276 277 278 279 280 281 282 283 284 285 286 |
# File 'lib/flydata/sync_file_manager.rb', line 275 def table_rev(table_name) file = table_rev_file_path(table_name) return 1 unless File.exists?(file) #default revision is 1 File.open(file, "r+") do |f| seq = f.read if seq.empty? return 1 else return seq.to_i end end end |
#table_rev_file_path(table_name) ⇒ Object
266 267 268 |
# File 'lib/flydata/sync_file_manager.rb', line 266 def table_rev_file_path(table_name) File.join(table_positions_dir_path, table_name + ".rev") end |
#table_rev_file_paths(*tables) ⇒ Object
270 271 272 273 |
# File 'lib/flydata/sync_file_manager.rb', line 270 def table_rev_file_paths(*tables) tables.empty? ? Dir.glob(File.join(table_positions_dir_path, "*.rev")) : tables.map{|table| table_rev_file_path(table)} end |