Class: SnapshotReload::SnapshotReload

Inherits:
Object
  • Object
show all
Includes:
Methadone::CLILogging, Methadone::SH
Defined in:
lib/snapshot_reload.rb,
lib/snapshot_reload/fetch.rb,
lib/snapshot_reload/reload.rb,
lib/snapshot_reload/validate.rb

Instance Method Summary collapse

Constructor Details

#initialize(config, options = nil) ⇒ SnapshotReload

Returns a new instance of SnapshotReload.



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/snapshot_reload.rb', line 25

def initialize(config, options=nil)

  options = Hash.new if options.nil?

  @config = validate_configuration(config)
  @env = validate_environment(options[:env], @config)

  @host = check_field(@config,@env,'host')
  @database = check_field(@config,@env,'database')
  @username = check_field(@config,@env,'username')
  @password = check_field(@config,@env,'password',false)

  @source = validate_source(options[:source])

  if @source.match('^s3://')
    aws = validate_aws(options['aws-conf'],
      options['aws-key'], options['aws-secret'])
    @aws_key = aws[0]
    @aws_secret = aws[1]
  end

  @save_before = options.has_key?(:save)
  @save_file = options[:save].gsub(/[^-_.\/[:alnum:]]*/, '') + ".sql.gz" if @save_before

  @dry_run = options['dry-run'] ||= false

  @verbose = options[:verbose] ||= false
  @quiet = options[:quiet] ||= false

  @verbose = false if @quiet

  @sql_file = ''

  if @verbose
    info("config: #{@config.to_s}")
    info("env: #{@env.to_s}")
    info("host: #{@host.to_s}")
    info("database: #{@database.to_s}")
    info("username: #{@username.to_s}")
    info("password: #{@password.to_s}")
    info("source: #{@source.to_s}")
    info("aws key: #{@aws_key}")
    info("aws secret: #{@aws_secret}")
    info("save before? #{@save_before.to_s}")
    info("save file: #{@save_file}")
    info("dry run: #{@dry_run}")
    info("verbose: #{@verbose}")
    info("quiet: #{@quiet}")
  end

  reload # and here all the magic happens!!  
  
end

Instance Method Details

#aws_key ⇒ Object 



107
108
109
 
# File 'lib/snapshot_reload.rb', line 107

def aws_key
  @aws_key
end

#aws_secret ⇒ Object 



111
112
113
 
# File 'lib/snapshot_reload.rb', line 111

def aws_secret
  @aws_secret
end

#check_field(conf, env, field, required = true) ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/snapshot_reload/validate.rb', line 95

def check_field(conf,env,field,required=true)
  if conf[env].has_key?(field)
    conf[env][field]
  else
    if required
      fatal("Must provide #{field} in configuration")
      exit(ENOFIELD)
    else
      nil
    end
  end
end

#check_sql_file ⇒ Object

want to see if the sql file is good



63
64
65
66
67
68
69
70
71
72
 
# File 'lib/snapshot_reload/reload.rb', line 63

def check_sql_file
  unzipped_file = '/tmp/' + File.basename(self.sql_file.gsub(/\.gz$/,''))
  cmd = "gunzip < #{self.sql_file} > #{unzipped_file}"
  run_it(cmd)
  unless File.size?(unzipped_file) or @dry_run
    fatal("#{self.sql_file} is empty!!")
    exit(EEMPTYSQL)
  end
  # File.unlink(unzipped_file)
end

#config ⇒ Object 



79
80
81
 
# File 'lib/snapshot_reload.rb', line 79

def config
  @config
end

#create_db ⇒ Object

method drop



43
44
45
 
# File 'lib/snapshot_reload/reload.rb', line 43

def create_db
  mysqladmin_go("create")
end

#database ⇒ Object 



91
92
93
 
# File 'lib/snapshot_reload.rb', line 91

def database
  @database
end

#drop_db ⇒ Object 



39
40
41
 
# File 'lib/snapshot_reload/reload.rb', line 39

def drop_db
  mysqladmin_go("drop")
end

#dry_run? ⇒ Boolean

Returns:

  • (Boolean) 


127
128
129
 
# File 'lib/snapshot_reload.rb', line 127

def dry_run?
  @dry_run      
end

#env ⇒ Object 



83
84
85
 
# File 'lib/snapshot_reload.rb', line 83

def env
  @env
end

#fetch_snapshot ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/snapshot_reload/fetch.rb', line 17

def fetch_snapshot
  
  if @source.match('^s3://')
    @sql_file = s3_fetch
  else
    @sql_file = @source
  end

  unless File.exists?(@sql_file)
    fatal("#{@sql_file} does not exist!")
    exit(ENOSQLFILE)
  end

  info("SQL file: #{@sql_file}") if @verbose

  @sql_file

end

#get_the_object(cnxn, bucket, name, range_start, range_end) ⇒ Object

method s3_fetch



148
149
150
151
152
153
154
155
156
157
 
# File 'lib/snapshot_reload/fetch.rb', line 148

def get_the_object(cnxn, bucket, name, range_start, range_end)

  get_object_options = { 'Range' => "bytes=%d-%d" % [ range_start, range_end ] }
  info("Getting #{name} from #{bucket}, range #{get_object_options['Range']}") if @verbose
  return '' if @dry_run
  response = cnxn.get_object(bucket,name,get_object_options)
  debug("Response.class: #{response.class}")
  debug("Response.status: #{response.status}")
  response.body
end

#host ⇒ Object 



87
88
89
 
# File 'lib/snapshot_reload.rb', line 87

def host
  @host
end

#load_db ⇒ Object 



47
48
49
50
51
52
53
 
# File 'lib/snapshot_reload/reload.rb', line 47

def load_db
  passopt = @password.present? ? "--password=#{@password}" : ''
  fetch_snapshot
  check_sql_file
  cmd = "gunzip < #{self.sql_file} | mysql --host=#{@host} --user=#{@username} #{passopt} #{@database}"
  run_it(cmd)
end

#mysqladmin_go(command) ⇒ Object 



55
56
57
58
59
60
 
# File 'lib/snapshot_reload/reload.rb', line 55

def mysqladmin_go(command)
  return false unless "drop create".include?(command)
  passopt = @password.present? ? "--password=#{@password}" : ''
  cmd = "mysqladmin --host=#{@host} --user=#{@username} #{passopt} --force #{command} #{@database}"
  run_it(cmd)
end

#password ⇒ Object 



99
100
101
 
# File 'lib/snapshot_reload.rb', line 99

def password
  @password
end

#quiet? ⇒ Boolean

Returns:

  • (Boolean) 


135
136
137
 
# File 'lib/snapshot_reload.rb', line 135

def quiet?
  @quiet
end

#reload ⇒ Object 



19
20
21
22
23
24
 
# File 'lib/snapshot_reload/reload.rb', line 19

def reload
  save_db if @save_before
  drop_db
  create_db
  load_db
end

#run_it(cmd) ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/snapshot_reload/reload.rb', line 74

def run_it(cmd)
  info("Dry-run, no commands will be executed.") if @dry_run and @verbose
  info("Command issued: #{cmd}") if @verbose
  unless @dry_run
    sh cmd do |stdout, stderr, retval|
      if retval != 0
        unless cmd.match("mysqladmin.*drop") and stderr.match("database doesn't exist")
          fatal("mysqladmin drop command failed:")
          fatal(stdout)
          fatal(stderr)
          exit(EDROPFAILED)
        else
          
        end
      end
    end
  else
    true
  end
end

#s3_fetch ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/snapshot_reload/fetch.rb', line 44

def s3_fetch

  matches = @source.match('^s3://([^/]+)/(.*)$')
  if matches
    s3_bucket_name=matches[1]
    s3_object_name=matches[2]
  else
    fatal("#{@source} is not a valid s3 uri (must be s3://bucket/object)")
    exit(EBADS3URI)
  end


  warn("Dry run, nothing will be fetched from S3") if @dry_run

  info("Connecting to AWS S3") if @verbose
  
  connection = Fog::Storage.new(:provider => 'AWS',
    :aws_access_key_id => @aws_key,
    :aws_secret_access_key => @aws_secret) unless @dry_run


  unless @dry_run
    
    buckets = connection.directories.select do |dir|
      debug("Dir: #{dir.key}")
      dir if dir.key == s3_bucket_name
    end

    if buckets.nil? or buckets.empty?
      fatal("no bucket #{s3_bucket_name} found")
      exit(ENOBUCKET)
    end

    s3_bucket = buckets.first
    
    info("S3 Bucket #{s3_bucket.key} found") if @verbose
    
    if s3_bucket.files.nil?
      error("No files in S3 bucket #{s3_bucket_name}")
      exit(ENOS3FILES)
    end

    files = s3_bucket.files.select do |file| 
      debug("File: #{file.key}")
      file if file.key == s3_object_name 
    end

    if files.nil? or files.empty?
      fatal("no object #{s3_object_name} found")
      exit(ENOOBJECT)
    end

    s3_object = files.first

    info("S3 Object #{s3_object_name} found in #{s3_bucket_name}") if @verbose

    s3_file = File.basename(s3_object.key)
    s3_object_content_length = s3_object.content_length
  
  else # this is just a dry run, make up stuff
  
    s3_file = File.basename(s3_object_name)
    s3_object_content_length = 0
  
  end

  if File.exists?(s3_file) and
    not (File.stat(s3_file).file? and File.stat(s3_file).writable?)
    fatal("#{s3_file} is not writable!")
    exit(ENOWRITE)
  end

  info("Writing to file #{s3_file}") if @verbose

  File.open(s3_file,'w') do |file|

    # Calculate number of batches for extremely large files

    batches = s3_object_content_length / CHUNK_SIZE

    (0..batches).each do |batch|
      start_byte = batch * CHUNK_SIZE
      end_byte = start_byte + CHUNK_SIZE - 1
      contents = get_the_object(connection, s3_bucket_name, s3_object_name, start_byte, end_byte)
      info("Writing #{contents.length} bytes to #{s3_file}") if @verbose
      file.write(contents)
    end

    # Now get the remainder
    start_byte = batches * CHUNK_SIZE
    end_byte = s3_object_content_length
    contents = get_the_object(connection, s3_bucket_name, s3_object_name, start_byte, end_byte)
    info("Writing #{contents.length} bytes to #{s3_file}") if @verbose
    file.write(contents)

  end # File.open
  
  s3_file_stat = File.stat(s3_file)
  info("Wrote #{s3_file_stat.size} bytes to #{s3_file}") if @verbose

  s3_file # return the name of file written

end

#save_before? ⇒ Boolean

Returns:

  • (Boolean) 


119
120
121
 
# File 'lib/snapshot_reload.rb', line 119

def save_before?
  @save_before
end

#save_db ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/snapshot_reload/reload.rb', line 26

def save_db
  info("saving to file #{@save_file}") if @verbose
  passopt = @password.present? ? "--password=#{password}" : ''
  cmd = "mysqldump --host=#{@host} --user=#{username} #{passopt} #{@database} | gzip > #{@save_file}"
  run_it(cmd)
  unless File.exists?(@save_file) or @dry_run
    fatal("Could not save data to #{@save_file}")
    exit(ENOSAVEFILE)
  else
    info("data written to #{@save_file}") if @verbose
  end
end

#save_file ⇒ Object 



123
124
125
 
# File 'lib/snapshot_reload.rb', line 123

def save_file
  @save_file
end

#source ⇒ Object 



103
104
105
 
# File 'lib/snapshot_reload.rb', line 103

def source
  @source
end

#sql_file ⇒ Object 



115
116
117
 
# File 'lib/snapshot_reload.rb', line 115

def sql_file
  @sql_file
end

#username ⇒ Object 



95
96
97
 
# File 'lib/snapshot_reload.rb', line 95

def username
  @username
end

#validate_aws(aws_conf, aws_key = nil, aws_secret = nil) ⇒ Object 



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/snapshot_reload/validate.rb', line 51

def validate_aws(aws_conf, aws_key = nil, aws_secret = nil)

  # DONE: fix scenarios so they run this code (pass in an S3 source!!)

  # aws_key and/or aws_secret were not nil -- therefore specified on the command line

  # **Both** aws_key and aws_secret must be given.
  # if only one but not the other is given, it's an error

  debug("Is aws_key present? #{aws_key.present? ? "yes" : "no"} key: #{aws_key}")
  debug("Is aws_secret present? #{aws_secret.present? ? "yes" : "no"} secret: #{aws_secret}")

  if (aws_key.present? ^ aws_secret.present?) # exclusive or, one must be true, but not the other
    fatal("Must provide *both* aws-key and aws-secret")
    exit(EMISSINGKEYORSECRET)  
  end

  if aws_key.nil? and aws_secret.nil?

    aws_conf ||= DEFAULT_AWS_CREDS

    if File.exists?(aws_conf)
      File.open(aws_conf, 'r') do |awsfile|
        awsfile.each_line do |line|
          m = line.match('^access_key\s*=\s*(.*)$')
          aws_key = m[1] if m

          m = line.match('^secret_key\s*=\s*(.*)$')
          debug("secret_key match -> #{m[0]} -> #{m[1]}") if m
          aws_secret = m[1] if m
        end
      end
    else
      fatal("#{aws_conf} does not exist!")
      exit(ENOCRED)
    end

  end

  debug("key: #{aws_key}\nsecret: #{aws_secret}")
  [ aws_key, aws_secret ]

end

#validate_configuration(config_file) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/snapshot_reload/validate.rb', line 18

def validate_configuration(config_file)

  unless File.exists?(config_file)
    fatal("#{config_file} file does not exist") 
    exit(ENOCONFIG)
  end

  config = YAML.load(File.read(config_file))
  unless config.class == Hash
    fatal("#{config_file} is not YAML")
    exit(ECONFIGNOTYAML)      
  end

  debug("Config: #{config.to_s}")

  config
end

#validate_environment(env, config) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
 
# File 'lib/snapshot_reload/validate.rb', line 36

def validate_environment(env,config)
  env ||= ENV['RAILS_ENV'] ||= DEFAULT_ENVIRONMENT
  unless config.has_key?(env)
    fatal("#{env} not found in configuration")
    exit(ENOENVINCONFIG)
  end

  debug("Env: #{env}")
  env
end

#validate_source(source) ⇒ Object 



47
48
49
 
# File 'lib/snapshot_reload/validate.rb', line 47

def validate_source(source)
  source ||= DEFAULT_S3_SOURCE
end

#verbose? ⇒ Boolean

Returns:

  • (Boolean) 


131
132
133
 
# File 'lib/snapshot_reload.rb', line 131

def verbose?
  @verbose
end