Class: LL::InnoBackup

Inherits:
Object
  • Object
show all
Defined in:
lib/ll-innobackup.rb

Defined Under Namespace

Classes: NoStateError

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ InnoBackup

Returns a new instance of InnoBackup.



67
68
69
70
71
72
73
74
 
# File 'lib/ll-innobackup.rb', line 67

def initialize(options = {})
  @now = Time.now
  @date = @now.to_date
  @options = options
  @lock_files = {}
  @state_files = {}
  @type = backup_type
end

Instance Attribute Details

#date ⇒ Object (readonly)

Returns the value of attribute date.



60
61
62
 
# File 'lib/ll-innobackup.rb', line 60

def date
  @date
end

#lock_files ⇒ Object (readonly)

Returns the value of attribute lock_files.



60
61
62
 
# File 'lib/ll-innobackup.rb', line 60

def lock_files
  @lock_files
end

#now ⇒ Object (readonly)

Returns the value of attribute now.



60
61
62
 
# File 'lib/ll-innobackup.rb', line 60

def now
  @now
end

#options ⇒ Object (readonly)

Returns the value of attribute options.



60
61
62
 
# File 'lib/ll-innobackup.rb', line 60

def options
  @options
end

#state_files ⇒ Object (readonly)

Returns the value of attribute state_files.



60
61
62
 
# File 'lib/ll-innobackup.rb', line 60

def state_files
  @state_files
end

#type ⇒ Object (readonly)

Returns the value of attribute type.



60
61
62
 
# File 'lib/ll-innobackup.rb', line 60

def type
  @type
end

Class Method Details

.innobackup_log(t) ⇒ Object 



55
56
57
 
# File 'lib/ll-innobackup.rb', line 55

def innobackup_log(t)
  "/tmp/backup_#{t}_innobackup_log"
end

.lock_file(type) ⇒ Object 



51
52
53
 
# File 'lib/ll-innobackup.rb', line 51

def lock_file(type)
  "/tmp/backup_#{type}.lock"
end

.options ⇒ Object

Use this in case the log file is massive



9
10
11
12
13
 
# File 'lib/ll-innobackup.rb', line 9

def options
  JSON.parse(File.read('/etc/mysql/innobackupex.json'))
rescue Errno::ENOENT
  {}
end

.state_file(t) ⇒ Object 



47
48
49
 
# File 'lib/ll-innobackup.rb', line 47

def state_file(t)
  "/tmp/backup_#{t}_state"
end

.tail_file(path, n) ⇒ Object 



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/ll-innobackup.rb', line 15

def tail_file(path, n)
  file = File.open(path, 'r')
  buffer_s = 512
  line_count = 0
  file.seek(0, IO::SEEK_END)

  offset = file.pos # we start at the end

  while line_count <= n && offset > 0
    to_read = if (offset - buffer_s) < 0
                offset
              else
                buffer_s
              end

    file.seek(offset - to_read)
    data = file.read(to_read)

    data.reverse.each_char do |c|
      if line_count > n
        offset += 1
        break
      end
      offset -= 1
      line_count += 1 if c == "\n|"
    end
  end

  file.seek(offset)
  file.read
end

Instance Method Details

#aws_backup_file ⇒ Object 



294
295
296
297
298
299
 
# File 'lib/ll-innobackup.rb', line 294

def aws_backup_file
  return "#{hostname}/#{now.iso8601}/percona_full_backup" if type == 'full'
  "#{hostname}/#{Time.parse(state('full')['date']).iso8601}/percona_incremental_#{now.iso8601}"
rescue NoMethodError
  raise NoStateError, 'incremental state missing or corrupt'
end

#aws_bin ⇒ Object 



147
148
149
 
# File 'lib/ll-innobackup.rb', line 147

def aws_bin
  @aws_bin = options['aws_bin'] ||= '/usr/local/bin/aws'
end

#aws_bucket ⇒ Object

Raises:



151
152
153
154
 
# File 'lib/ll-innobackup.rb', line 151

def aws_bucket
  raise NoStateError, 'aws_bucket not provided' unless options['aws_bucket']
  @aws_bucket = options['aws_bucket']
end

#aws_command ⇒ Object 



202
203
204
205
 
# File 'lib/ll-innobackup.rb', line 202

def aws_command
  "#{aws_bin} s3 cp - s3://#{aws_bucket}/#{aws_backup_file}"\
  " #{expected_size} #{expires}"
end

#aws_debug ⇒ Object 



207
208
209
 
# File 'lib/ll-innobackup.rb', line 207

def aws_debug
  "--debug" if debug_aws?
end

#aws_log ⇒ Object 



76
77
78
 
# File 'lib/ll-innobackup.rb', line 76

def aws_log
  "/tmp/backup_#{type}_aws_log"
end

#backup ⇒ Object 



219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
 
# File 'lib/ll-innobackup.rb', line 219

def backup
  require 'English'
  exc = "#{innobackup_command} 2> #{innobackup_log} | "\
  "#{aws_command} #{aws_debug} > #{aws_log} 2>&1"
  `#{exc}`if valid_commands?
  resp = $CHILD_STATUS
  @completed = resp = 0
  puts "Function Returned: #{resp}"
  return record if success? && completed?
  revert_aws if valid_commands?
rescue InnoBackup::NoStateError => e
  STDERR.puts e.message
ensure
  report
end

#backup_bin ⇒ Object 



127
128
129
 
# File 'lib/ll-innobackup.rb', line 127

def backup_bin
  @backup_bin = options['backup_bin'] ||= '/usr/bin/innobackupex'
end

#backup_compress_threads ⇒ Object 



135
136
137
 
# File 'lib/ll-innobackup.rb', line 135

def backup_compress_threads
  @backup_compress_threads = options['backup_compress_threads'] ||= 4
end

#backup_parallel ⇒ Object 



131
132
133
 
# File 'lib/ll-innobackup.rb', line 131

def backup_parallel
  @backup_parallel = options['backup_parallel'] ||= 4
end

#backup_type ⇒ Object 



121
122
123
124
125
 
# File 'lib/ll-innobackup.rb', line 121

def backup_type
  return 'full' unless fully_backed_up_today? || full_backup_running?
  return 'incremental' unless incremental_backup_running?
  raise 'Unable to backup as backups are running'
end

#can_full_backup? ⇒ Boolean

Returns:

  • (Boolean) 


109
110
111
 
# File 'lib/ll-innobackup.rb', line 109

def can_full_backup?
  !fully_backed_up_today? && lock?('full')
end

#completed? ⇒ Boolean

Returns:

  • (Boolean) 


301
302
303
 
# File 'lib/ll-innobackup.rb', line 301

def completed?
  @completed == true
end

#debug_aws? ⇒ Boolean

Returns:

  • (Boolean) 


211
212
213
 
# File 'lib/ll-innobackup.rb', line 211

def debug_aws?
  @debug_aws = (options['debug_aws'] == true)
end

#expected_full_size ⇒ Object 



156
157
158
 
# File 'lib/ll-innobackup.rb', line 156

def expected_full_size
  @expected_full_size = options['expected_full_size'] ||= 15_000_000_000
end

#expected_incremental_size ⇒ Object 



160
161
162
 
# File 'lib/ll-innobackup.rb', line 160

def expected_incremental_size
  @expected_incremental_size = options['expected_incremental_size'] ||= 5_000_000_000
end

#expected_size ⇒ Object 



196
197
198
199
200
 
# File 'lib/ll-innobackup.rb', line 196

def expected_size
  es = backup_type == 'full' ? expected_full_size : expected_incremental_size

  "--expected-size=#{es}" if es.to_i > 0
end

#expires ⇒ Object 



191
192
193
194
 
# File 'lib/ll-innobackup.rb', line 191

def expires
  ed = expires_date
  "--expires=#{ed}" if ed
end

#expires_date ⇒ Object 



179
180
181
182
183
184
185
186
187
188
189
 
# File 'lib/ll-innobackup.rb', line 179

def expires_date
  require 'active_support/all'
  # Keep incrementals for 2 days
  return (@now + 2.days).iso8601 if type == 'incremental'
  # Keep first backup of month for 180 days
  return (@now + 6.months).iso8601 if @date.yesterday.month != @date.month
  # Keep first backup of week for 31 days (monday)
  return (@now + 1.month).iso8601 if @date.cwday == 1
  # Keep daily backups for 14 days
  (@now + 2.weeks).iso8601
end

#full_backup_running? ⇒ Boolean

Returns:

  • (Boolean) 


113
114
115
 
# File 'lib/ll-innobackup.rb', line 113

def full_backup_running?
  !lock?('full')
end

#fully_backed_up_today? ⇒ Boolean

Returns:

  • (Boolean) 


99
100
101
102
103
104
105
106
107
 
# File 'lib/ll-innobackup.rb', line 99

def fully_backed_up_today?
  require 'active_support/all'
  date = state('full')['date']
  Time.parse(date).today? if date
rescue Errno::ENOENT
  false
rescue NoMethodError
  false
end

#hostname ⇒ Object 



288
289
290
291
292
 
# File 'lib/ll-innobackup.rb', line 288

def hostname
  return options['hostname'] if options['hostname']
  require 'socket'
  Socket.gethostbyname(Socket.gethostname).first
end

#incremental ⇒ Object 



260
261
262
263
264
265
 
# File 'lib/ll-innobackup.rb', line 260

def incremental
  return unless backup_type == 'incremental'
  "--incremental --incremental-lsn=#{lsn_from_state}"
rescue Errno::ENOENT
  ''
end

#incremental_backup_running? ⇒ Boolean

Returns:

  • (Boolean) 


117
118
119
 
# File 'lib/ll-innobackup.rb', line 117

def incremental_backup_running?
  !lock?('incremental')
end

#innobackup_command ⇒ Object 



174
175
176
177
 
# File 'lib/ll-innobackup.rb', line 174

def innobackup_command
  "#{backup_bin} #{sql_authentication} "\
  "#{incremental} #{innobackup_options} /tmp/sql"
end

#innobackup_log ⇒ Object 



80
81
82
 
# File 'lib/ll-innobackup.rb', line 80

def innobackup_log
  "/tmp/backup_#{type}_innobackup_log"
end

#innobackup_options ⇒ Object 



168
169
170
171
172
 
# File 'lib/ll-innobackup.rb', line 168

def innobackup_options
  "--parallel=#{backup_parallel} "\
  "--compress-threads=#{backup_compress_threads} "\
  '--stream=xbstream --compress'
end

#lock?(t = type) ⇒ Boolean

Returns:

  • (Boolean) 


84
85
86
87
88
89
 
# File 'lib/ll-innobackup.rb', line 84

def lock?(t = type)
  lock_files[t] ||= File.new(InnoBackup.lock_file(t), File::CREAT)
  lock_files[t].flock(File::LOCK_NB | File::LOCK_EX).zero?
rescue NoMethodError
  false
end

#lsn_from_backup_log ⇒ Object 



280
281
282
283
284
285
286
 
# File 'lib/ll-innobackup.rb', line 280

def lsn_from_backup_log
  matches = InnoBackup.tail_file(
    InnoBackup.innobackup_log(type),
    30
  ).match(/The latest check point \(for incremental\): '(\d+)'/)
  matches[1] if matches
end

#lsn_from_full_backup_state? ⇒ Boolean

Returns:

  • (Boolean) 


267
268
269
270
271
 
# File 'lib/ll-innobackup.rb', line 267

def lsn_from_full_backup_state?
  Time.parse(state('full')['date']) > Time.parse(state('incremental')['date'])
rescue Errno::ENOENT
  true
end

#lsn_from_state ⇒ Object 



273
274
275
276
277
278
 
# File 'lib/ll-innobackup.rb', line 273

def lsn_from_state
  return state('full')['lsn'] if lsn_from_full_backup_state?
  state('incremental')['lsn']
rescue NoMethodError
  raise NoStateError, 'no state file for incremental backup'
end

#record ⇒ Object 



249
250
251
252
253
254
255
256
257
258
 
# File 'lib/ll-innobackup.rb', line 249

def record
  File.write(
    InnoBackup.state_file(type),
    {
      date: now,
      lsn: lsn_from_backup_log,
      file: aws_backup_file
    }.to_json
  )
end

#report ⇒ Object 



305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
 
# File 'lib/ll-innobackup.rb', line 305

def report
  # Eventually Tell Zabbix
  if success? && completed?
    STDERR.puts "#{$PROGRAM_NAME}: success: completed #{type} backup"
    return
  end
  STDERR.puts "#{$PROGRAM_NAME}: failed" 
  STDERR.puts 'missing binaries' unless valid_commands?
  inno_tail = InnoBackup.tail_file(innobackup_log, 10)
  STDERR.puts 'invalid sql user' if inno_tail =~ /Option user requires an argument/
  STDERR.puts 'unable to connect to DB' if inno_tail =~ /Access denied for user/
  STDERR.puts 'insufficient file access' if inno_tail =~ /Can't change dir to/
  aws_tail = InnoBackup.tail_file(aws_log, 10)
  STDERR.puts 'bucket incorrect' if aws_tail =~ /The specified bucket does not exist/
  STDERR.puts 'invalid AWS key' if aws_tail =~ /The AWS Access Key Id you/
  STDERR.puts 'invalid Secret key' if aws_tail =~ /The request signature we calculated/
end

#revert_aws ⇒ Object 



235
236
237
238
 
# File 'lib/ll-innobackup.rb', line 235

def revert_aws
  exc = "#{aws_bin} s3 rm s3://#{aws_bucket}/#{aws_backup_file} > /dev/null 2>/dev/null"
  `#{exc}`
end

#sql_authentication ⇒ Object 



164
165
166
 
# File 'lib/ll-innobackup.rb', line 164

def sql_authentication
  "--user=#{sql_backup_user} --password=#{sql_backup_password}"
end

#sql_backup_password ⇒ Object 



143
144
145
 
# File 'lib/ll-innobackup.rb', line 143

def sql_backup_password
  @sql_backup_password ||= options['sql_backup_password']
end

#sql_backup_user ⇒ Object 



139
140
141
 
# File 'lib/ll-innobackup.rb', line 139

def sql_backup_user
  @sql_backup_user ||= options['sql_backup_user']
end

#state(t) ⇒ Object 



91
92
93
94
95
96
97
 
# File 'lib/ll-innobackup.rb', line 91

def state(t)
  state_files[t] ||= JSON.parse(File.read(InnoBackup.state_file(t)))
rescue Errno::ENOENT
  {}
rescue JSON::ParserError
  {}
end

#success? ⇒ Boolean

Returns:

  • (Boolean) 


240
241
242
243
244
245
246
247
 
# File 'lib/ll-innobackup.rb', line 240

def success?
  InnoBackup.tail_file(
    innobackup_log,
    1
  ) =~ /: completed OK/
rescue Errno::ENOENT
  false
end

#valid_commands? ⇒ Boolean

Returns:

  • (Boolean) 


215
216
217
 
# File 'lib/ll-innobackup.rb', line 215

def valid_commands?
  File.exist?(backup_bin) && File.exist?(aws_bin)
end