Class: S3TarBackup::Main

Inherits:
Object
  • Object
show all
Defined in:
lib/s3_tar_backup.rb

Constant Summary collapse

UPLOAD_TRIES = 
5

Instance Method Summary collapse

Instance Method Details

#absolute_path_from_config_file(config, path) ⇒ Object 



81
82
83
 
# File 'lib/s3_tar_backup.rb', line 81

def absolute_path_from_config_file(config, path)
  File.expand_path(File.join(File.expand_path(File.dirname(config.file_path)), path))
end

#backup(config, backup, verbose = false) ⇒ Object 



215
216
217
218
219
220
221
222
223
224
 
# File 'lib/s3_tar_backup.rb', line 215

def backup(config, backup, verbose=false)
  FileUtils.rm(backup.tmp_snar_path) if File.exists?(backup.tmp_snar_path)
  FileUtils.cp(backup.snar_path, backup.tmp_snar_path) if backup.snar_exists?
  exec(backup.backup_cmd(verbose))
  puts "Uploading #{config[:backend].prefix}/#{File.basename(backup.archive)} (#{bytes_to_human(File.size(backup.archive))})"
  upload(config[:backend], backup.archive, File.basename(backup.archive), true)
  FileUtils.mv(backup.tmp_snar_path, backup.snar_path, :force => true)
  puts "Uploading snar (#{bytes_to_human(File.size(backup.snar_path))})"
  upload(config[:backend], backup.snar_path, File.basename(backup.snar), false)
end

#backup_full(config, verbose = false) ⇒ Object 



207
208
209
210
211
212
213
 
# File 'lib/s3_tar_backup.rb', line 207

def backup_full(config, verbose=false)
  puts "Starting new full backup"
  backup = Backup.new(config[:backup_dir], config[:name], config[:sources], config[:exclude], config[:compression], config[:encryption])
  # Nuke the snar file -- forces a full backup
  File.delete(backup.snar_path) if File.exists?(backup.snar_path)
  backup(config, backup, verbose)
end

#backup_incr(config, verbose = false) ⇒ Object

Config should have the keys backup_dir, name, soruces, exclude



189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/s3_tar_backup.rb', line 189

def backup_incr(config, verbose=false)
  puts "Starting new incremental backup"
  backup = Backup.new(config[:backup_dir], config[:name], config[:sources], config[:exclude], config[:compression], config[:encryption])

  # Try and get hold of the snar file
  unless backup.snar_exists?
    puts "Failed to find snar file. Attempting to download..."
    if config[:backend].item_exists?(backup.snar)
      puts "Found file on S3. Downloading"
      config[:backend].download_item(backup.snar, backup.snar_path)
    else
      puts "Failed to download snar file. Defaulting to full backup"
    end
  end

  backup(config, backup, verbose)
end

#bytes_to_human(n) ⇒ Object 



335
336
337
338
339
340
341
342
343
 
# File 'lib/s3_tar_backup.rb', line 335

def bytes_to_human(n)
  count = 0
  while n >= 1014 && count < 4
    n /= 1024.0
    count += 1
  end
  fmt = (count == 0) ? '%i' : '%.2f'
  format(fmt, n) << %w(B KB MB GB TB)[count]
end

#create_backend(config, dest_prefix) ⇒ Object 



85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/s3_tar_backup.rb', line 85

def create_backend(config, dest_prefix)
    if dest_prefix.start_with?('file://')
      Backend::FileBackend.new(dest_prefix['file://'.length..-1])
    elsif dest_prefix.start_with?('/')
      Backend::FileBackend.new(dest_prefix)
    else
      Backend::S3Backend.new(
        ENV['AWS_ACCESS_KEY_ID'] || config['settings.aws_access_key_id'],
        ENV['AWS_SECRET_ACCESS_KEY'] || config['settings.aws_secret_access_key'],
        config.get('settings.aws_region', false),
        (config.get('settings.dest', false) || config["profile.#{profiles[0]}.dest"]).sub(%r{^s3://}, '')
      )
    end
end

#exec(cmd) ⇒ Object 



345
346
347
348
349
350
351
 
# File 'lib/s3_tar_backup.rb', line 345

def exec(cmd)
  puts "Executing: #{cmd}"
  result = system(cmd)
  unless result
    raise "Unable to run command. See above output for clues."
  end
end

#full_required?(interval_str, objects) ⇒ Boolean

Returns:

  • (Boolean) 


330
331
332
333
 
# File 'lib/s3_tar_backup.rb', line 330

def full_required?(interval_str, objects)
  time = parse_interval(interval_str)
  objects.select{ |o| o[:type] == :full && o[:date] > time }.empty?
end

#gen_backup_config(profile, config) ⇒ Object 



100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
# File 'lib/s3_tar_backup.rb', line 100

def gen_backup_config(profile, config)
  top_gpg_key = config.get('settings.gpg_key', false)
  profile_gpg_key = config.get("profile.#{profile}.gpg_key", false)
  top_password_file = config.get('settings.password_file', false)
  profile_password_file = config.get("profile.#{profile}.password_file", false)
  raise "Cannot specify gpg_key and password_file together at the top level" if top_gpg_key && top_password_file 
  raise "Cannot specify both gpg_key and password_file for profile #{profile}" if profile_gpg_key && profile_password_file

  encryption = nil
  if profile_password_file
    encryption = profile_password_file.empty? ? nil : { :type => :password_file, :password_file => absolute_path_from_config_file(config, profile_password_file) }
  elsif profile_gpg_key
    encryption = profile_gpg_key.empty? ? nil : { :type => :gpg_key, :gpg_key => profile_gpg_key }
  elsif top_password_file
    encryption = top_password_file.empty? ? nil : { :type => :password_file, :password_file => absolute_path_from_config_file(config, top_password_file) }
  elsif top_gpg_key
    encryption = top_gpg_key.empty? ? nil : { :type => :gpg_key, :gpg_key => top_gpg_key }
  end

  backup_config = {
    :backup_dir => config.get("profile.#{profile}.backup_dir", false) || config.get('settings.backup_dir', '~/.s3-tar-backup/tmp'),
    :name => profile,
    :encryption => encryption,
    :password_file => profile_password_file || top_password_file || '',
    :sources => [*config.get("profile.#{profile}.source", [])] + [*config.get("settings.source", [])],
    :exclude => [*config.get("profile.#{profile}.exclude", [])] + [*config.get("settings.exclude", [])],
    :pre_backup => [*config.get("profile.#{profile}.pre-backup", [])] + [*config.get('settings.pre-backup', [])],
    :post_backup => [*config.get("profile.#{profile}.post-backup", [])] + [*config.get('settings.post-backup', [])],
    :full_if_older_than => config.get("profile.#{profile}.full_if_older_than", false) || config['settings.full_if_older_than'],
    :remove_older_than => config.get("profile.#{profile}.remove_older_than", false) || config.get('settings.remove_older_than', false),
    :remove_all_but_n_full => config.get("profile.#{profile}.remove_all_but_n_full", false) || config.get('settings.remove_all_but_n_full', false),
    :compression => (config.get("profile.#{profile}.compression", false) || config.get('settings.compression', 'none')).to_sym,
    :always_full => config.get('settings.always_full', false) || config.get("profile.#{profile}.always_full", false),
    :backend => create_backend(config,config.get("profile.#{profile}.dest", false) || config['settings.dest']),
  }
  backup_config
end

#get_objects(config, profile) ⇒ Object 



311
312
313
314
315
316
 
# File 'lib/s3_tar_backup.rb', line 311

def get_objects(config, profile)
  objects = config[:backend].list_items.map do |object|
    Backup.parse_object(object, profile)
  end
  objects.compact.sort_by{ |o| o[:date] }
end

#parse_interval(interval_str) ⇒ Object 



318
319
320
321
322
323
324
325
326
327
328
 
# File 'lib/s3_tar_backup.rb', line 318

def parse_interval(interval_str)
  time = Time.now
  time -= $1.to_i if interval_str =~ /(\d+)s/
  time -= $1.to_i*60 if interval_str =~ /(\d+)m/
  time -= $1.to_i*3600 if interval_str =~ /(\d+)h/
  time -= $1.to_i*86400 if interval_str =~ /(\d+)D/
  time -= $1.to_i*604800 if interval_str =~ /(\d+)W/
  time -= $1.to_i*2592000 if interval_str =~ /(\d+)M/
  time -= $1.to_i*31536000 if interval_str =~ /(\d+)Y/
  time
end

#perform_backup(opts, prev_backups, backup_config) ⇒ Object 



138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/s3_tar_backup.rb', line 138

def perform_backup(opts, prev_backups, backup_config)
  puts "===== Backing up profile #{backup_config[:name]} ====="
  backup_config[:pre_backup].each_with_index do |cmd, i|
    puts "Executing pre-backup hook #{i+1}"
    exec(cmd)
  end
  full_required = full_required?(backup_config[:full_if_older_than], prev_backups)
  puts "Last full backup is too old. Forcing a full backup" if full_required && !opts[:full] && backup_config[:always_full]
  if full_required || opts[:full] || backup_config[:always_full]
    backup_full(backup_config, opts[:verbose])
  else
    backup_incr(backup_config, opts[:verbose])
  end
  backup_config[:post_backup].each_with_index do |cmd, i|
    puts "Executing post-backup hook #{i+1}"
    exec(cmd)
  end
end

#perform_cleanup(prev_backups, backup_config) ⇒ Object 



157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
 
# File 'lib/s3_tar_backup.rb', line 157

def perform_cleanup(prev_backups, backup_config)
  puts "===== Cleaning up profile #{backup_config[:name]} ====="
  remove = []
  if age_str = backup_config[:remove_older_than]
    age = parse_interval(age_str)
    remove = prev_backups.select{ |o| o[:date] < age }
    # Don't want to delete anything before the last full backup
    unless remove.empty?
      kept = remove.slice!(remove.rindex{ |o| o[:type] == :full }..-1).count
      puts "Keeping #{kept} old backups as part of a chain" if kept > 1
    end
  elsif keep_n = backup_config[:remove_all_but_n_full]
    keep_n = keep_n.to_i
    # Get the date of the last full backup to keep
    if last_full_to_keep = prev_backups.select{ |o| o[:type] == :full }[-keep_n]
      # If there is a last full one...
      remove = prev_backups.select{ |o| o[:date] < last_full_to_keep[:date] }
    end
  end

  if remove.empty?
    puts "Nothing to do"
  else
    puts "Removing #{remove.count} old backup files"
  end
  remove.each do |object|
    backup_config[:backend].remove_item(object[:name])
  end
end

#perform_list_backups(prev_backups, backup_config) ⇒ Object 



276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
 
# File 'lib/s3_tar_backup.rb', line 276

def perform_list_backups(prev_backups, backup_config)
  # prev_backups alreays contains just the files for the current profile
  puts "===== Backups list for #{backup_config[:name]} ====="
  puts "Type: N:  Date:#{' '*18}Size:       Chain Size:   Compression:   Encryption:\n\n"
  prev_type = ''
  total_size = 0
  chain_length = 0
  chain_cum_size = 0
  prev_backups.each do |object|
    type = object[:type] == prev_type && object[:type] == :incr ? " -- " : object[:type].to_s.capitalize
    prev_type = object[:type]
    chain_length += 1
    chain_length = 0 if object[:type] == :full
    chain_cum_size = 0 if object[:type] == :full
    chain_cum_size += object[:size]

    chain_length_str = (chain_length == 0 ? '' : chain_length.to_s).ljust(3)
    chain_cum_size_str = (object[:type] == :full ? '' : bytes_to_human(chain_cum_size)).ljust(8)
    encryption = case object[:encryption]
    when :gpg_key
      'Key'
    when :password_file
      'Password'
    else
      'None'
    end
    puts "#{type}  #{chain_length_str} #{object[:date].strftime('%F %T')}    #{bytes_to_human(object[:size]).ljust(8)}    " \
      "#{chain_cum_size_str}      #{object[:compression].to_s.ljust(12)}   #{encryption}"
    total_size += object[:size]
  end
  puts "\n"
  puts "Total size: #{bytes_to_human(total_size)}"
  puts "\n"
end

#perform_restore(opts, prev_backups, backup_config) ⇒ Object 



243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
 
# File 'lib/s3_tar_backup.rb', line 243

def perform_restore(opts, prev_backups, backup_config)
  puts "===== Restoring profile #{backup_config[:name]} ====="
  # If restore date given, parse
  if opts[:restore_date_given]
    m = opts[:restore_date].match(/(\d\d\d\d)(\d\d)(\d\d)?(\d\d)?(\d\d)?(\d\d)?/)
    raise "Unknown date format in --restore-to" if m.nil?
    restore_to = Time.new(*m[1..-1].map{ |s| s.to_i if s })
  else
    restore_to = Time.now
  end

  # Find the index of the first backup, incremental or full, before that date
  restore_end_index = prev_backups.rindex{ |o| o[:date] < restore_to }
  raise "Failed to find a backup for that date" unless restore_end_index

  # Find the first full backup before that one
  restore_start_index = prev_backups[0..restore_end_index].rindex{ |o| o[:type] == :full }

  restore_dir = opts[:restore].chomp('/') << '/'

  Dir.mkdir(restore_dir) unless Dir.exists?(restore_dir)
  raise "Destination dir is not a directory" unless File.directory?(restore_dir)

  prev_backups[restore_start_index..restore_end_index].each do |object|
    puts "Fetching #{backup_config[:backend].prefix}/#{object[:name]} (#{bytes_to_human(object[:size])})"
    dl_file = "#{backup_config[:backup_dir]}/#{object[:name]}"
    backup_config[:backend].download_item(object[:name], dl_file)
    puts "Extracting..."
    exec(Backup.restore_cmd(restore_dir, dl_file, opts[:verbose], opts[:password_file] || backup_config[:password_file]))
    File.delete(dl_file)
  end
end

#run ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/s3_tar_backup.rb', line 12

def run
  opts = Trollop::options do
    version VERSION
    banner "Backs up files to, and restores files from, Amazon's S3 storage, using tar incremental backups\n\n" \
      "Usage:\ns3-tar-backup -c config.ini [-p profile] --backup [--full] [-v]\n" \
      "s3-tar-backup -c config.ini [-p profile] --cleanup [-v]\n" \
      "s3-tar-backup -c config.ini [-p profile] --restore restore_dir\n\t[--restore_date date] [-v]\n" \
      "s3-tar-backup -c config.ini [-p profile] --backup-config [--verbose]\n" \
      "s3-tar-backup -c config.ini [-p profile] --list-backups\n\n" \
      "Option details:\n"
    opt :config, "Configuration file", :short => 'c', :type => :string
    opt :backup, "Make an incremental backup"
    opt :full, "Make the backup a full backup"
    opt :profile, "The backup profile(s) to use (default all)", :short => 'p', :type => :strings
    opt :cleanup, "Clean up old backups"
    opt :restore, "Restore a backup to the specified dir", :type => :string
    opt :restore_date, "Restore a backup from the specified date. Format YYYYMM[DD[hh[mm[ss]]]]", :type => :string
    opt :backup_config, "Backs up the specified configuration file"
    opt :list_backups, "List the stored backup info for one or more profiles"
    opt :password_file, "Override the password file used to decrypt backups", :type => :string
    opt :verbose, "Show verbose output", :short => 'v'
    conflicts :backup, :cleanup, :restore, :backup_config, :list_backups
  end


  Trollop::die "--full requires --backup" if opts[:full] && !opts[:backup]
  Trollop::die "--restore-date requires --restore" if opts[:restore_date_given] && !opts[:restore_given]
  Trollop::die "--password-file requires --restore" if opts[:password_file_given] && !opts[:restore_given]
  unless opts[:backup] || opts[:cleanup] || opts[:restore_given] || opts[:backup_config] || opts[:list_backups]
    Trollop::die "Need one of --backup, --cleanup, --restore, --backup-config, --list-backups"
  end

  config_file = opts[:config] || '~/.s3-tar-backup/config.ini'

  begin
    raise "Config file #{config_file} not found.#{opts[:config] ? '' : ' You can specify a config file to use with --config'}" unless File.exists?(config_file)
    config = IniParser.new(config_file).load
    profiles = opts[:profile] || config.find_sections(/^profile\./).keys.map{ |k| k.to_s.split('.', 2)[1] }

    # This is a bit of a special case
    if opts[:backup_config]
      dest = config.get('settings.dest', false)
      raise "You must specify a single profile (used to determine the location to back up to) " \
        "if backing up config and dest key is not in [settings]" if !dest && profiles.count != 1
      dest ||= config["profile.#{profiles[0]}.dest"]
      puts "===== Backing up config file #{config_file} ====="
      prefix = config.get('settings.dest', false) || config["profile.#{profiles[0]}.dest"]
      puts "Uploading #{config_file} to #{prefix}/#{File.basename(config_file)}"
      backend = create_backend(config, prefix)
      upload(backend, config_file, File.basename(config_file), false)
      return
    end

    profiles.dup.each do |profile|
      raise "No such profile: #{profile}" unless config.has_section?("profile.#{profile}")
      opts[:profile] = profile
      backup_config = gen_backup_config(opts[:profile], config)
      prev_backups = get_objects(backup_config, opts[:profile])
      perform_backup(opts, prev_backups, backup_config) if opts[:backup]
      perform_cleanup(prev_backups, backup_config) if opts[:backup] || opts[:cleanup]
      perform_restore(opts, prev_backups, backup_config) if opts[:restore_given]
      perform_list_backups(prev_backups, backup_config) if opts[:list_backups]
    end
  rescue Exception => e
    raise e
    Trollop::die e.to_s
  end
end

#upload(backend, source, dest_name, remove_original) ⇒ Object 



226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/s3_tar_backup.rb', line 226

def upload(backend, source, dest_name, remove_original)
  tries = 0
  begin
    backend.upload_item(dest_name, source, remove_original)
  rescue Backend::UploadItemFailedError => e
    tries += 1
    if tries <= UPLOAD_TRIES
      puts "Upload Exception: #{e}"
      puts "Retrying #{tries}/#{UPLOAD_TRIES}..."
      retry
    else
      raise e
    end
  end
  puts "Succeeded" if tries > 0
end