Class: S3Rsync::Sync
- Inherits:
-
Base
- Object
- Base
- S3Rsync::Sync
show all
- Defined in:
- lib/s3rsync/sync.rb
Instance Method Summary
collapse
Methods inherited from Base
#error_exit, #init_logger, #init_statsd, #pass_cli
Constructor Details
#initialize(opt) ⇒ Sync
Returns a new instance of Sync.
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
|
# File 'lib/s3rsync/sync.rb', line 9
def initialize(opt)
@log = init_logger(opt['log-path'])
@log.debug "opt: #{opt.inspect}"
@params = {}
@params[:path] = File.expand_path(opt['path'])
@params[:s3_prefix] = opt['s3-prefix']
@params[:config_path] = opt['config-path']
@params[:s3cmd_conf] = @params[:config_path].sub('.json', '_s3rsync.conf')
@params[:dry_run] = opt['dry-run']
@params[:make_public] = opt['make-public']
@params[:enable_lock] = opt['enable-lock']
@log.debug "@params[:config_path]: #{@params[:config_path]}"
begin
conf = JSON.parse File.read(@params[:config_path])
rescue Exception => e
error_exit "ERROR: opening or parsing config file failed: #{e.inspect}"
end
@log.debug "conf: #{conf}"
@params[:s3_bucket] = conf['env.static.s3.bucket']
@params[:s3_access] = conf['env.static.s3.key.user']
@params[:s3_secret] = conf['env.static.s3.key.secret']
@params[:statsd_host] = conf['env.metrics.host'] || 'localhost'
@params[:statsd_port] = conf['env.metrics.port'] || 8125
@log.debug "@params: #{@params.inspect}"
@statsd = init_statsd @params[:statsd_host], @params[:statsd_port]
@log.debug "@statsd: #{@statsd.inspect}"
init_s3cmd_conf
@s3cmd = "s3cmd"
end
|
Instance Method Details
#get_lock(name = 'common') ⇒ Object
41
42
43
44
45
46
47
48
|
# File 'lib/s3rsync/sync.rb', line 41
def get_lock(name = 'common')
lockfile = Lockfile.new(
"/tmp/s3rsync_#{name}.lock",
:timeout => 3,
:max_age => 3600,
:debug => !ENV['DEBUG'].nil?
)
end
|
#init_s3cmd_conf ⇒ Object
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
|
# File 'lib/s3rsync/sync.rb', line 50
def init_s3cmd_conf
@log.debug "@params[:s3cmd_conf]: #{@params[:s3cmd_conf]}"
if !File.exists? @params[:s3cmd_conf]
@log.info "Generating dynamic s3cfg: #{@params[:s3cmd_conf]}"
begin
erb_path = File.join(File.dirname(__FILE__), 'templates', 's3cmd_conf.erb')
erb_template = IO.read erb_path
@log.debug "erb_template: #{erb_template.inspect}"
erb = ERB.new erb_template
rendered_template = erb.result(binding)
@log.debug "rendered_template: #{rendered_template}"
File.write @params[:s3cmd_conf], rendered_template
rescue Exception => e
error_exit "generating s3cmd config failed: #{e.inspect}"
end
@log.info "Dynamic s3cfg generated: #{@params[:s3cmd_conf]}"
end
end
|
#run(mode = :upload) ⇒ Object
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
# File 'lib/s3rsync/sync.rb', line 69
def run(mode = :upload)
init_s3cmd_conf
valid_file_upload = (mode == :upload && File.exists?(@params[:path]))
if !Dir.exists?(@params[:path]) && !valid_file_upload
@log.error "Local dir or file not exist on disk: #{@params[:path]}"
return 4
end
@params[:path] += '/' if Dir.exists?(@params[:path])
sync_locations = "#{@params[:path]} s3://#{@params[:s3_bucket]}/#{@params[:s3_prefix]}#{@params[:path]}"
@log.debug "original sync_locations: #{sync_locations}"
if @params[:enable_lock]
lockfile = get_lock(mode)
@log.debug "lockfile: #{lockfile.inspect}"
if lockfile.locked?
@log.warn "lock file found, exiting: #{lockfile.inspect}"
return 5
end
end
if mode == :download
@log.debug "reversing sync_locations"
sync_locations = sync_locations.split(' ').reverse.join(' ')
@log.debug "download sync_locations: #{sync_locations}"
end
sync_output = 1
begin
lockfile.lock if @params[:enable_lock]
s3cmd_args = {}
s3cmd_args['-c'] = @params[:s3cmd_conf]
s3cmd_args['--dry-run'] = '' if @params[:dry_run]
s3cmd_args['--acl-public'] = '' if @params[:make_public]
@log.debug "s3cmd_args: #{s3cmd_args.inspect}"
compiled_args = s3cmd_args.to_a.flatten.reject { |v| v.empty? }.join(' ')
@log.debug "compiled_args: #{compiled_args.inspect}"
cmd = "#{@s3cmd} #{compiled_args} sync #{sync_locations}"
@log.debug "cmd: #{cmd}"
sync_output = pass_cli cmd
@log.debug "sync_output: #{sync_output.inspect}"
rescue Exception => e
sync_output = 1
@log.error "something wrong while trying to run the sync: #{e.inspect}"
ensure
lockfile.unlock if @params[:enable_lock]
end
if (sync_output == 1) || sync_output.downcase.include?('error')
@log.error "errors found in output: #{sync_output}"
return 1
else
return 0
end
end
|