Class: BackupUtility::Base

Inherits:
Object
  • Object
show all
Defined in:
lib/backup_utility/base.rb

Direct Known Subclasses

File, Mongo, Postgres

Instance Method Summary collapse

Constructor Details

#initialize(backup_info) ⇒ Base

Returns a new instance of Base.



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/backup_utility/base.rb', line 7

def initialize(backup_info)
  @backup_dir = backup_info[:backup_dir] || BACKUP_DIR
  @shared_dir = backup_info[:shared_dir] || SHARED_DIR
  @code_dir = backup_info[:code_dir] || CODE_DIR
  @backup_machine = backup_info[:backup_machine] || 'a1-stats'

  @send_to_shared = backup_info.fetch(:send_to_shared, true)
  @send_to_s3 = backup_info.fetch(:send_to_s3, true)
  @move_backups = backup_info.fetch(:move_backups, true)
  @use_code = backup_info.fetch(:use_code, false)
  @use_scp = backup_info.fetch(:use_scp, false)
  @save = backup_info.fetch(:save, true)
  @clean_shared_age = backup_info[:clean_shared_age]
  @clean_shared_age = ::Rails.env == 'production' ? 30 : 14 if @clean_shared_age.blank?

  @clean_backups_age = backup_info[:clean_backups_age]
  if @clean_backups_age.blank?
    if ::Rails.env == 'production'
      @cleanup_backups_age = @move_backups ? 7 : 14
    else
      @cleanup_backups_age = @move_backups ? 3 : 7
    end
  end
  @last_dst = nil
end

Instance Method Details

#backup(settings = {}, append = nil) ⇒ Object 



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/backup_utility/base.rb', line 46

def backup(settings = {}, append = nil)
  now = Time.now
  storage_name, working_dir, backup_file = get_paths(settings, append)
  if !perform_dump(settings, working_dir)
    log("could not perform dump properly")
    return false
  end
  if custom?
    # no need to tar up the directory, all we need to do is rename
    # the file in the backup_dir to the backup name
    log("moving file #{@last_dst} to #{backup_file}")
    File.mv(@last_dst, backup_file)
    ret = 0
  else
    base_name = File.basename(working_dir)
    out = log_and_time("taring up backup file #{backup_file}") do
      # tar up the combined directory
      `tar czf #{backup_file} -C #{@backup_dir} #{base_name}`
    end
    ret=$?
  end
  if ret.to_i != 0
    log("error taring file : #{out}")
    exit 1
  end
  if File.exists?(backup_file)
    log("cleaning up working dir #{@working_dir}")
    FileUtils.rm_rf(working_dir)
  else
    log("backup file #{backup_file} was not created properly")
    return false
  end
  store_backup(storage_name, backup_file)
  taken = Time.now - now
  log("time taken to dump backup #{taken}")
end

#cleanup_backups(age = nil) ⇒ Object 



144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'lib/backup_utility/base.rb', line 144

def cleanup_backups(age = nil)
  return false if !File.exists?(@backup_dir)
  age = @cleanup_backups_age if age.blank?

  clean_dir = @backup_dir
  cleanup_dir(clean_dir, age) do |file|
    if @move_backups
      dst_file = shared_dst(true)
      next if dst_file.blank? || !File.exists?(dst_file)
      FileUtils.mv(file, dst_file)
    else
      File.delete(file)
    end
  end
end

#cleanup_dir(clean_dir, age, display = true) ⇒ Object 



169
170
171
172
173
174
175
176
177
178
179
180
181
182
 
# File 'lib/backup_utility/base.rb', line 169

def cleanup_dir(clean_dir, age, display = true)
  cleaned = 0
  total = 0
  Dir["#{clean_dir}/*.tar.gz"].each do |file|
    stat = File.stat(file)
    if stat.mtime < age.to_i.days.ago
      yield file
      cleaned += stat.size
      total += 1
    end
  end
  log("cleaned up #{total} for #{cleaned} bytes in #{clean_dir}") if display
  [total, cleaned]
end

#cleanup_shared(age = nil) ⇒ Object 



160
161
162
163
164
165
166
167
 
# File 'lib/backup_utility/base.rb', line 160

def cleanup_shared(age = nil)
  clean_dir = shared_dst
  return false if clean_dir.nil? || !File.exists?(clean_dir)
  age = @clean_shared_age if age.blank?
  cleanup_dir(clean_dir, age) do |file|
    File.delete(file)
  end
end

#custom? ⇒ Boolean

Returns:

  • (Boolean) 


106
107
108
 
# File 'lib/backup_utility/base.rb', line 106

def custom?
  false
end

#get_paths(settings = {}, append = nil) ⇒ Object 



83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/backup_utility/base.rb', line 83

def get_paths(settings = {}, append = nil)
  append = settings[:append] if append.nil?
  prepend = settings[:prepend]
  now = Time.now
  storage_name = "#{::Rails.env}_latest"
  format = settings.fetch(:date_format, '%Y-%m-%d-%H')
  date = now.strftime(format)
  storage_name += "_#{append}" if append
  base_name = "#{prepend}#{date}-#{::Rails.env}"
  base_name += "-#{append}" if append
  if custom?
    ext = '.dump.gz'
  else
    ext = '.tar.gz'
  end
  storage_name += ext
  back_name = "#{base_name}#{ext}"
  working_dir = File.join(@backup_dir, base_name)
  Dir.mkdir(working_dir) if !File.exists?(working_dir)
  backup_file = File.join(@backup_dir, back_name)
  [storage_name, working_dir, backup_file]
end

#log(out) ⇒ Object 



42
43
44
 
# File 'lib/backup_utility/base.rb', line 42

def log(out)
  puts "#{Time.now} - #{out}"
end

#log_and_time(out) ⇒ Object 



33
34
35
36
37
38
39
40
 
# File 'lib/backup_utility/base.rb', line 33

def log_and_time(out)
  log(out)
  start = Time.now
  ret = yield
  took = Time.now - start
  log("took #{took}")
  ret
end

#send_dir_to_s3 ⇒ Object 



184
185
186
187
188
189
190
 
# File 'lib/backup_utility/base.rb', line 184

def send_dir_to_s3
  Dir.foreach(@backup_dir) do |f|
    next if f[0] == ?.
    file = File.join(backup_dir, f)
    send_file_to_s3(file)
  end
end

#send_file_to_s3(backup_file) ⇒ Object 



192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/backup_utility/base.rb', line 192

def send_file_to_s3(backup_file)
  if @use_code
    `/usr/local/bin/ruby #{@code_dir}/s3_storage.rb upload #{backup_file}`
    return true
  end
  bucket = 'twistage-backup'
  # first check if backup file exists on s3, and then upload it
  name = File.basename(backup_file)
  s3 = S3Storage.default_connection
  files = s3.list_bucket({:prefix => name}, bucket)
  size = File.size(backup_file)
  amt = 5
  if files.size == 0
    amt.times do |x|
      log("uploading #{backup_file} to s3")
      begin
        start = Time.now
        s3.put_file(backup_file, bucket)
        taken = Time.now - start
        mb = (size / 1.megabyte) / taken
        log("file uploaded in #{taken} seconds (#{mb} MB/s)")
        return true
      rescue StandardError => err
        if x+1 == amt
          log("could not upload file #{backup_file} : #{err.to_s}")
          return false
        end
        log("could not send file #{err.to_s}, retrying (#{x+1} of #{amt})")
        sleep(15)
      end
    end
  else
    log("#{backup_file} already exists in s3")
  end
  true
end

#shared_dst(create_if_missing = false) ⇒ Object 



137
138
139
140
141
142
 
# File 'lib/backup_utility/base.rb', line 137

def shared_dst(create_if_missing = false)
  return nil if !File.exists?(@shared_dir)
  dst = File.join(@shared_dir, "#{::Rails.env}_daily")
  Dir.mkdir(dst) if !File.exists?(dst) && create_if_missing
  dst
end

#store_backup(storage_name, backup_file) ⇒ Object 



110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/backup_utility/base.rb', line 110

def store_backup(storage_name, backup_file)
  if @save
    # send file to s3
    if @send_to_s3
      log("sending backup file to s3")
      send_file_to_s3(backup_file)
    end
    if @send_to_shared
      if @use_scp
        # maintain the name so we can send it to the storage properly
        back_name = File.basename(backup_file)
        shared_file = File.join(@shared_dir, back_name)
        log("scping backup file to #{shared_file}")
        `/usr/bin/scp #{backup_file} #{@backup_user}@#{@backup_machine}:#{shared_file}`
      else
        shared_file = File.join(@shared_dir, storage_name)
        log("copying backup file to #{shared_file}")
        FileUtils.copy_file(backup_file, shared_file)
      end
    end
    cleanup_backups
    cleanup_shared
  else
    log("skip save")
  end
end