3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
|
# File 'lib/s3_uploader/s3_uploader.rb', line 3
def self.upload_directory(source, bucket, options = {})
options = {
:destination_dir => '',
:threads => 5,
:s3_key => ENV['S3_KEY'],
:s3_secret => ENV['S3_SECRET'],
:public => false,
:region => 'us-east-1',
:metadata => {},
:path_style => false,
:regexp => /.*/,
:gzip => false,
:gzip_working_dir => source,
:time_range => Time.at(0)..(Time.now + (60 * 60 * 24))
}.merge(options)
log = options[:logger] || Logger.new(STDOUT)
raise 'Source must be a directory' unless File.directory?(source)
if options[:gzip_working_dir] != source && options[:gzip_working_dir][source]
raise 'gzip_working_dir may not be located within source-folder'
end
if options[:connection]
connection = options[:connection]
else
raise "Missing access keys" if options[:s3_key].nil? or options[:s3_secret].nil?
connection = Fog::Storage.new({
:provider => 'AWS',
:aws_access_key_id => options[:s3_key],
:aws_secret_access_key => options[:s3_secret],
:region => options[:region],
:path_style => options[:path_style]
})
end
source = source.chop if source.end_with?('/')
options[:gzip_working_dir] = options[:gzip_working_dir].chop if options[:gzip_working_dir].end_with?('/')
if options[:destination_dir] != '' and !options[:destination_dir].end_with?('/')
options[:destination_dir] = "#{options[:destination_dir]}/"
end
total_size = 0
files = Queue.new
Dir.glob("#{source}/**/*").select { |f| !File.directory?(f) }.each do |f|
if File.basename(f).match(options[:regexp]) and options[:time_range].cover?(File.mtime(f))
if options[:gzip] && File.extname(f) != '.gz'
dir, base = File.split(f)
dir = dir.sub(source, options[:gzip_working_dir])
gz_file = "#{dir}/#{base}.gz"
FileUtils.mkdir_p(dir) unless File.directory?(dir)
Zlib::GzipWriter.open(gz_file) do |gz|
gz.mtime = File.mtime(f)
gz.orig_name = f
gz.write IO.binread(f)
end
files << gz_file
total_size += File.size(gz_file)
else
files << f
total_size += File.size(f)
end
end
end
directory = connection.directories.new(:key => bucket)
start = Time.now
total_files = files.size
file_number = 0
@mutex = Mutex.new
threads = []
options[:threads].times do |i|
threads[i] = Thread.new {
until files.empty?
@mutex.synchronize do
file_number += 1
Thread.current["file_number"] = file_number
end
file = files.pop rescue nil
if file
key = file.gsub(source, '').gsub(options[:gzip_working_dir], '')[1..-1]
dest = "#{options[:destination_dir]}#{key}"
log.info("[#{Thread.current["file_number"]}/#{total_files}] Uploading #{key} to s3://#{bucket}/#{dest}")
directory.files.create(
:key => dest,
:body => File.open(file),
:public => options[:public],
:metadata => options[:metadata]
)
end
end
}
end
threads.each { |t| t.join }
finish = Time.now
elapsed = finish.to_f - start.to_f
mins, secs = elapsed.divmod 60.0
log.info("Uploaded %d (%.#{0}f KB) in %d:%04.2f" % [total_files, total_size / KILO_SIZE, mins.to_i, secs])
end
|