Class: Backup::Files

Inherits:
Task
  • Object
show all
Extended by:
Gitlab::Utils::Override
Includes:
Helper
Defined in:
lib/backup/files.rb

Constant Summary collapse

DEFAULT_EXCLUDE = 
'lost+found'

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Gitlab::Utils::Override

extended, extensions, included, method_added, override, prepended, queue_verification, verify!

Methods included from Helper

#access_denied_error, #gzip_cmd, #resource_busy_error

Methods inherited from Task

#post_restore_warning, #pre_restore_warning

Constructor Details

#initialize(progress, app_files_dir, excludes: []) ⇒ Files

Returns a new instance of Files.



14
15
16
17
18
19
 
# File 'lib/backup/files.rb', line 14

def initialize(progress, app_files_dir, excludes: [])
  super(progress)

  @app_files_dir = app_files_dir
  @excludes = [DEFAULT_EXCLUDE].concat(excludes)
end

Instance Attribute Details

#excludes ⇒ Object (readonly)

Returns the value of attribute excludes.



12
13
14
 
# File 'lib/backup/files.rb', line 12

def excludes
  @excludes
end

Instance Method Details

#backup_existing_files_dir(backup_tarball) ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/backup/files.rb', line 75

def backup_existing_files_dir(backup_tarball)
  name = File.basename(backup_tarball, '.tar.gz')

  timestamped_files_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}.#{Time.now.to_i}")
  if File.exist?(app_files_realpath)
    # Move all files in the existing repos directory except . and .. to
    # repositories.<timestamp> directory
    FileUtils.mkdir_p(timestamped_files_path, mode: 0700)
    files = Dir.glob(File.join(app_files_realpath, "*"), File::FNM_DOTMATCH) - [File.join(app_files_realpath, "."), File.join(app_files_realpath, "..")]
    begin
      FileUtils.mv(files, timestamped_files_path)
    rescue Errno::EACCES
      access_denied_error(app_files_realpath)
    rescue Errno::EBUSY
      resource_busy_error(app_files_realpath)
    end
  end
end

#dump(backup_tarball, backup_id) ⇒ Object 



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/backup/files.rb', line 23

def dump(backup_tarball, backup_id)
  FileUtils.mkdir_p(Gitlab.config.backup.path)
  FileUtils.rm_f(backup_tarball)

  if ENV['STRATEGY'] == 'copy'
    cmd = [%w[rsync -a --delete], exclude_dirs(:rsync), %W[#{app_files_realpath} #{Gitlab.config.backup.path}]].flatten
    output, status = Gitlab::Popen.popen(cmd)

    # Retry if rsync source files vanish
    if status == 24
      $stdout.puts "Warning: files vanished during rsync, retrying..."
      output, status = Gitlab::Popen.popen(cmd)
    end

    unless status == 0
      puts output
      raise_custom_error(backup_tarball)
    end

    tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{backup_files_realpath} -cf - .]].flatten
    status_list, output = run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600])
    FileUtils.rm_rf(backup_files_realpath)
  else
    tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{app_files_realpath} -cf - .]].flatten
    status_list, output = run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600])
  end

  unless pipeline_succeeded?(tar_status: status_list[0], gzip_status: status_list[1], output: output)
    raise_custom_error(backup_tarball)
  end
end

#exclude_dirs(fmt) ⇒ Object 



137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/backup/files.rb', line 137

def exclude_dirs(fmt)
  excludes.map do |s|
    if s == DEFAULT_EXCLUDE
      '--exclude=' + s
    elsif fmt == :rsync
      '--exclude=/' + File.join(File.basename(app_files_realpath), s)
    elsif fmt == :tar
      '--exclude=./' + s
    end
  end
end

#noncritical_warning?(warning) ⇒ Boolean

Returns:

  • (Boolean) 


103
104
105
106
107
108
109
 
# File 'lib/backup/files.rb', line 103

def noncritical_warning?(warning)
  noncritical_warnings = [
    /^g?tar: \.: Cannot mkdir: No such file or directory$/
  ]

  noncritical_warnings.map { |w| warning =~ w }.any?
end

#pipeline_succeeded?(tar_status:, gzip_status:, output:) ⇒ Boolean

Returns:

  • (Boolean) 


111
112
113
114
115
 
# File 'lib/backup/files.rb', line 111

def pipeline_succeeded?(tar_status:, gzip_status:, output:)
  return false unless gzip_status&.success?

  tar_status&.success? || tar_ignore_non_success?(tar_status.exitstatus, output)
end

#raise_custom_error(backup_tarball) ⇒ Object

Raises:



149
150
151
 
# File 'lib/backup/files.rb', line 149

def raise_custom_error(backup_tarball)
  raise FileBackupError.new(app_files_realpath, backup_tarball)
end

#restore(backup_tarball) ⇒ Object 



56
57
58
59
60
61
62
63
64
 
# File 'lib/backup/files.rb', line 56

def restore(backup_tarball)
  backup_existing_files_dir(backup_tarball)

  cmd_list = [%w[gzip -cd], %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_realpath} -xf -]]
  status_list, output = run_pipeline!(cmd_list, in: backup_tarball)
  unless pipeline_succeeded?(gzip_status: status_list[0], tar_status: status_list[1], output: output)
    raise Backup::Error, "Restore operation failed: #{output}"
  end
end

#run_pipeline!(cmd_list, options = {}) ⇒ Object 



94
95
96
97
98
99
100
101
 
# File 'lib/backup/files.rb', line 94

def run_pipeline!(cmd_list, options = {})
  err_r, err_w = IO.pipe
  options[:err] = err_w
  status_list = Open3.pipeline(*cmd_list, options)
  err_w.close

  [status_list, err_r.read]
end

#tar ⇒ Object 



66
67
68
69
70
71
72
73
 
# File 'lib/backup/files.rb', line 66

def tar
  if system(*%w[gtar --version], out: '/dev/null')
    # It looks like we can get GNU tar by running 'gtar'
    'gtar'
  else
    'tar'
  end
end

#tar_ignore_non_success?(exitstatus, output) ⇒ Boolean

Returns:

  • (Boolean) 


117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/backup/files.rb', line 117

def tar_ignore_non_success?(exitstatus, output)
  # tar can exit with nonzero code:
  #  1 - if some files changed (i.e. a CI job is currently writes to log)
  #  2 - if it cannot create `.` directory (see issue https://gitlab.com/gitlab-org/gitlab/-/issues/22442)
  #  http://www.gnu.org/software/tar/manual/html_section/tar_19.html#Synopsis
  #  so check tar status 1 or stderr output against some non-critical warnings
  if exitstatus == 1
    $stdout.puts "Ignoring tar exit status 1 'Some files differ': #{output}"
    return true
  end

  # allow tar to fail with other non-success status if output contain non-critical warning
  if noncritical_warning?(output)
    $stdout.puts "Ignoring non-success exit status #{exitstatus} due to output of non-critical warning(s): #{output}"
    return true
  end

  false
end