Class: MongoOplogBackup::Backup

Inherits:
Object
  • Object
show all
Includes:
Lockable
Defined in:
lib/mongo_oplog_backup/backup.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Lockable

included, #lock

Constructor Details

#initialize(config, backup_name = nil) ⇒ Backup

Returns a new instance of Backup.



19
20
21
22
23
24
25
26
27
28
 
# File 'lib/mongo_oplog_backup/backup.rb', line 19

def initialize(config, backup_name=nil)
  @config = config
  @backup_name = backup_name
  if backup_name.nil?
    state_file = config.global_state_file
    state = JSON.parse(File.read(state_file)) rescue nil
    state ||= {}
    @backup_name = state['backup']
  end
end

Instance Attribute Details

#backup_name ⇒ Object (readonly)

Returns the value of attribute backup_name.



8
9
10
 
# File 'lib/mongo_oplog_backup/backup.rb', line 8

def backup_name
  @backup_name
end

#config ⇒ Object (readonly)

Returns the value of attribute config.



8
9
10
 
# File 'lib/mongo_oplog_backup/backup.rb', line 8

def config
  @config
end

Instance Method Details

#backup_folder ⇒ Object 



10
11
12
13
 
# File 'lib/mongo_oplog_backup/backup.rb', line 10

def backup_folder
  return nil unless backup_name
  File.join(config.backup_dir, backup_name)
end

#backup_full ⇒ Object 



114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
 
# File 'lib/mongo_oplog_backup/backup.rb', line 114

def backup_full
  position = latest_oplog_timestamp
  raise "Cannot backup with empty oplog" if position.nil?
  @backup_name = "backup-#{position}"
  if File.exists? backup_folder
    raise "Backup folder '#{backup_folder}' already exists; not performing backup."
  end
  dump_folder = File.join(backup_folder, 'dump')
  dump_args = ['--out', dump_folder]
  dump_args << '--gzip' if config.use_compression?
  result = config.mongodump(dump_args)
  unless File.directory? dump_folder
    MongoOplogBackup.log.error 'Backup folder does not exist'
    raise 'Full backup failed'
  end

  File.write(File.join(dump_folder, 'debug.log'), result.standard_output)

  unless result.standard_error.length == 0
    File.write(File.join(dump_folder, 'error.log'), result.standard_error)
  end

  write_state({
    'position' => position
  })

  return {
    position: position,
    backup: backup_name
  }
end

#backup_oplog(options = {}) ⇒ Object

Raises:

  • (ArgumentError) 


34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/mongo_oplog_backup/backup.rb', line 34

def backup_oplog(options={})
  raise ArgumentError, "No state in #{backup_name}" unless File.exists? state_file

  backup_state = JSON.parse(File.read(state_file))
  start_at = options[:start] || BSON::Timestamp.from_json(backup_state['position'])
  raise ArgumentError, ":start is required" unless start_at

  query = ['--query', "{ts : { $gte : { $timestamp : { t : #{start_at.seconds}, i : #{start_at.increment} } } }}"]

  dump_args = ['--out', config.oplog_dump_folder, '--db', 'local', '--collection', 'oplog.rs']
  dump_args += query
  dump_args << '--gzip' if config.use_compression?
  config.mongodump(dump_args)

  unless File.exists? config.oplog_dump
    raise "mongodump failed"
  end
  MongoOplogBackup.log.debug "Checking timestamps..."
  timestamps = Oplog.oplog_timestamps(config.oplog_dump)

  unless timestamps.increasing?
    raise "Something went wrong - oplog is not ordered."
  end

  first = timestamps[0]
  last = timestamps[-1]

  if first > start_at
    raise "Expected first oplog entry to be #{start_at.inspect} but was #{first.inspect}\n" +
      "The oplog is probably too small.\n" +
      "Increase the oplog size, the start with another full backup."
  elsif first < start_at
    raise "Expected first oplog entry to be #{start_at.inspect} but was #{first.inspect}\n" +
      "Something went wrong in our query."
  end

  result = {
    entries: timestamps.count,
    first: first,
    position: last
  }

  if timestamps.count == 1
    result[:empty] = true
  else
    outfile = "oplog-#{first}-#{last}.bson"
    outfile += '.gz' if config.use_compression?
    full_path = File.join(backup_folder, outfile)
    FileUtils.mkdir_p backup_folder
    FileUtils.mv config.oplog_dump, full_path

    write_state({
      'position' => result[:position]
    })
    result[:file] = full_path
    result[:empty] = false
  end

  FileUtils.rm_r config.oplog_dump_folder rescue nil
  result
end

#latest_oplog_timestamp ⇒ Object 



102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/mongo_oplog_backup/backup.rb', line 102

def latest_oplog_timestamp
  script = File.expand_path('../../oplog-last-timestamp.js', File.dirname(__FILE__))
  result_text = config.mongo('admin', script).standard_output
  begin
    response = JSON.parse(strip_warnings_which_should_be_in_stderr_anyway(result_text))
    return nil unless response['position']
    BSON::Timestamp.from_json(response['position'])
  rescue JSON::ParserError => e
    raise StandardError, "Failed to connect to MongoDB: #{result_text}"
  end
end

#latest_oplog_timestamp_mongo ⇒ Object 



183
184
185
186
187
188
189
190
191
192
193
194
 
# File 'lib/mongo_oplog_backup/backup.rb', line 183

def latest_oplog_timestamp_mongo
  # Alternative implementation for `latest_oplog_timestamp`
  require 'mongo'
  client = Mongo::Client.new([ "127.0.0.1:27017" ], database: 'local')
  oplog = client['oplog.rs']
  entry = oplog.find.limit(1).sort('$natural' => -1).first
  if entry
    entry['ts']
  else
    nil
  end
end

#perform(mode = :auto, options = {}) ⇒ Object 



146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
 
# File 'lib/mongo_oplog_backup/backup.rb', line 146

def perform(mode=:auto, options={})
  FileUtils.mkdir_p config.backup_dir
  have_backup = backup_folder != nil

  if mode == :auto
    if have_backup
      mode = :oplog
    else
      mode = :full
    end
  end

  if mode == :oplog
    raise "Unknown backup position - cannot perform oplog backup. Have you completed a full backup?" unless have_backup
    MongoOplogBackup.log.info "Performing incremental oplog backup"
    lock(File.join(backup_folder, 'backup.lock')) do
      result = backup_oplog
      unless result[:empty]
        new_entries = result[:entries] - 1
        MongoOplogBackup.log.info "Backed up #{new_entries} new entries to #{result[:file]}"
      else
        MongoOplogBackup.log.info "Nothing new to backup"
      end
    end
  elsif mode == :full
    lock(config.global_lock_file) do
      MongoOplogBackup.log.info "Performing full backup"
      result = backup_full
      File.write(config.global_state_file, {
        'backup' => result[:backup]
      }.to_json)
      MongoOplogBackup.log.info "Performed full backup"
    end
    perform(:oplog, options)
  end
end

#state_file ⇒ Object 



15
16
17
 
# File 'lib/mongo_oplog_backup/backup.rb', line 15

def state_file
  File.join(backup_folder, 'state.json')
end

#strip_warnings_which_should_be_in_stderr_anyway(data) ⇒ Object

Because jira.mongodb.org/browse/SERVER-18643 Mongo shell warns (in stdout) about self-signed certs, regardless of ‘allowInvalidCertificates’ option.



98
99
100
 
# File 'lib/mongo_oplog_backup/backup.rb', line 98

def strip_warnings_which_should_be_in_stderr_anyway data
  data.gsub(/^.*[thread\d.*].* certificate.*$/,'')
end

#write_state(state) ⇒ Object 



30
31
32
 
# File 'lib/mongo_oplog_backup/backup.rb', line 30

def write_state(state)
  File.write(state_file, state.to_json)
end