Class: S3File

Inherits:
Object
  • Object
show all
Defined in:
lib/s3file.rb,
lib/s3file/cp.rb,
lib/s3file/ls.rb,
lib/s3file/mv.rb,
lib/s3file/rm.rb,
lib/s3file/bucket.rb,
lib/s3file/errors.rb

Defined Under Namespace

Classes: AuthorizationError, ConnectionError, InitializationError, PathError, S3CommandError, S3PathError, UnknownError

Constant Summary collapse

@@config_file = 
File.join(File.expand_path("~/"), ".s3file.cfg")

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(keys = {}) ⇒ S3File

Intialize the object to the bucket given the access_key and secret_access_key You need to have the following set *:access_key and *:secret_access_key s3cmd requires a config file to be created. Based upon the values provided - a .s3file.cfg is created in the users home directory. If the keys are not provided - it will raise an InitializationError



21
22
23
24
25
26
 
# File 'lib/s3file.rb', line 21

def initialize(keys = {})
  if keys.nil? || keys[:access_key].nil? || keys[:secret_access_key].nil?
    raise(InitializationError, "Keys not set when initializing S3 connection")
  end
  S3File.create_config_file(keys)
end

Class Method Details

.config_file_exists? ⇒ Boolean

Returns true or false if s3cmd config file exists

Returns:

  • (Boolean) 


62
63
64
 
# File 'lib/s3file.rb', line 62

def config_file_exists?
  File.exists?(@@config_file)
end

.cp(source, destination) ⇒ Object

Copy either from local to S3 or the other way around. Used for copying files and not folders. Raises a S3CommandError in case attempt to copy a folder.



5
6
7
8
9
10
11
12
 
# File 'lib/s3file/cp.rb', line 5

def cp(source, destination)
  if s3?(source)
    raise(S3CommandError, "Attempting to copy folder using cp. Use cp_r instead.") if source.match(/\/\z/)
    run_command("s3cmd get #{source} #{destination} -c #{@@config_file}")
  elsif local?(source)
    run_command("s3cmd put #{source} #{destination} -c #{@@config_file}")
  end
end

.cp_f(source, destination) ⇒ Object

Force copy from local to S3 or the other way around. This will over write local files. Raises a S3CommandError in case attempt to copy a folder.



16
17
18
19
20
21
22
23
 
# File 'lib/s3file/cp.rb', line 16

def cp_f(source, destination)
  if s3?(source)
    raise(S3CommandError, "Attempting to copy folder using cp. Use cp_rf instead.") if source.match(/\/\z/)
    run_command("s3cmd get #{source} #{destination} -c #{@@config_file} --force")
  elsif local?(source)
    run_command("s3cmd put #{source} #{destination} -c #{@@config_file} --force")
  end
end

.cp_r(source, destination) ⇒ Object

Recursively copy from local to S3 or the other way around. This is used for copying of folders.



26
27
28
29
30
31
32
 
# File 'lib/s3file/cp.rb', line 26

def cp_r(source, destination)
  if s3?(source)
    run_command("s3cmd get #{source} #{destination} -c #{@@config_file} --recursive")
  elsif local?(source)
    run_command("s3cmd put #{source} #{destination} -c #{@@config_file} --recursive")
  end
end

.cp_rf(source, destination) ⇒ Object

Recursively and forcefully copy a folder between S3 and local or the other way around.



35
36
37
38
39
40
41
 
# File 'lib/s3file/cp.rb', line 35

def cp_rf(source, destination)
  if s3?(source)
    run_command("s3cmd get #{source} #{destination} -c #{@@config_file} --recursive --force")
  elsif local?(source)
    run_command("s3cmd put #{source} #{destination} -c #{@@config_file} --recursive --force")
  end
end

.create_config_file(keys = {}) ⇒ Object

Creates the config_file



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/s3file.rb', line 67

def create_config_file(keys={})
  # Data to be put into the config file
  config_data = %Q{[default]
access_key = #{keys[:access_key].to_s}
acl_public = False
bucket_location = US
cloudfront_host = cloudfront.amazonaws.com
cloudfront_resource = /2008-06-30/distribution
default_mime_type = binary/octet-stream
delete_removed = False
dry_run = False
encoding = UTF-8
encrypt = False
force = False
get_continue = False
gpg_command = /usr/bin/gpg
gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_passphrase = 
guess_mime_type = True
host_base = s3.amazonaws.com
host_bucket = %(bucket)s.s3.amazonaws.com
human_readable_sizes = False
list_md5 = False
preserve_attrs = True
progress_meter = True
proxy_host = 
proxy_port = 0
recursive = False
recv_chunk = 4096
secret_key = #{keys[:secret_access_key]}
send_chunk = 4096
simpledb_host = sdb.amazonaws.com
skip_existing = False
urlencoding_mode = normal
use_https = False
verbosity = WARNING}

  File.delete(@@config_file) if File.exists?(@@config_file)
  file = File.new(@@config_file, "w")
  file.puts config_data
  file.close
end

.delete_config_file! ⇒ Object

Deletes the config file that was created.



30
31
32
 
# File 'lib/s3file.rb', line 30

def delete_config_file!
  File.delete(@@config_file) if S3File.config_file_exists?
end

.disk_usage(bucket) ⇒ Object

Gets the disk usage of the bucket. Raises a S3PathError in case bucket specified does not start with s3://

Raises:



19
20
21
22
 
# File 'lib/s3file/bucket.rb', line 19

def disk_usage(bucket)
  raise(S3PathError, "Invalid S3 path") unless s3?(bucket)
  run_command("s3cmd du #{bucket} -c #{@@config_file}")
end

.equal(location1, location2) ⇒ Object

Checks if the file is the same on the source and destination The check is done via md5_hash to determine if both the files are the same.

Raises:



43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/s3file.rb', line 43

def equal(location1, location2)
  if s3?(location1)
    s3_location = location1
    local_file  = location2
  elsif local?(location1)
    s3_location = location2
    local_file  = location1
  else
    raise(S3PathError, "S3 location not specified")     
  end
  raise(PathError, "Local file not found") unless File.exists?(local_file)
  md5_hash(s3_location) == Digest::MD5.hexdigest(File.read(location2))
end

.fix_bucket(bucket) ⇒ Object

Fixes any invalid file names in the bucket. Raises S3PathError in case bucket specified does not start with s3://

Raises:



26
27
28
29
 
# File 'lib/s3file/bucket.rb', line 26

def fix_bucket(bucket)
  raise(S3PathError, "Invalid S3 path") unless s3?(bucket)
  run_command("s3cmd fixbucket #{bucket} -c #{@@config_file}")
end

.list(location = nil, display = nil) ⇒ Object

Gets a list of all the files in the given location in the bucket

Raises:



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/s3file/ls.rb', line 21

def list(location = nil, display = nil)
  return [] if location.nil?
  raise(PathError, "Not S3 Path") unless s3?(location)
  # display can take 3 possible values "all", "dir" or "file"
  # if display is nil or any other value other than dir or file then
  # display = "all"
  # if display is 'dir' or 'file' then
  # display takes that value
  if display.nil? || !display.match(/dir|file/)
    display = "all"
  end

  directories = []
  files = []
  location += "/" if location == s3_bucket(location)
  entries = run_command("s3cmd ls #{location} -c #{@@config_file}")

  directory = s3_directory(location)
  entries.split("\n").each do |entry|
    if display.match(/all|dir/) && entry.strip.match(/DIR/)
      dir = entry.strip.sub(/DIR/, '').strip.sub("#{directory}", "")
      if dir == "/"
        directories << (location + "/")
      else
        directories << dir
      end
    elsif display.match(/all|file/) && !entry.strip.match(/DIR/)
      file = entry.strip.sub(/\d{4}-\d{2}-\d{2}\s*\d{2}:\d{2}\s*/, '').split(" ")[1].sub("#{directory}","")
      if file == ""
        files << location
      else
        files << file
      end
    end
  end
  directories + files
end

.local?(location) ⇒ Boolean

Checks if the location is on local or not Checking that the location does not start with s3:// Short cut - use the opposite of s3? to get the solution

Returns:

  • (Boolean) 


122
123
124
125
126
127
128
 
# File 'lib/s3file.rb', line 122

def local?(location)
  if s3?(location)
    return false
  else
    return File.exists?(File.expand_path(location))
  end
end

.ls(location = nil) ⇒ Object

Gets a list of all the entries in a given directory



4
5
6
 
# File 'lib/s3file/ls.rb', line 4

def ls(location=nil)
  list(location, 'all')
end

.ls_directories(location = nil) ⇒ Object

Gets a list of all the directories in the given location in the bucket



9
10
11
 
# File 'lib/s3file/ls.rb', line 9

def ls_directories(location=nil)
  list(location, 'dir')
end

.ls_files(location = nil) ⇒ Object

Gets a list of all the files in the given location in the bucket



14
15
16
 
# File 'lib/s3file/ls.rb', line 14

def ls_files(location=nil)
  list(location, 'file')
end

.make_bucket(bucket) ⇒ Object

Makes a bucket in S3. Raises a S3PathError in case bucket specified does not start with s3://

Raises:



5
6
7
8
 
# File 'lib/s3file/bucket.rb', line 5

def make_bucket(bucket)
  raise(S3PathError, "Invalid S3 path") unless s3?(bucket)
  run_command("s3cmd mb #{bucket} -c #{@@config_file}")
end

.md5_hash(location) ⇒ Object

Gets the md5 hash of the s3 file. This will be used to confirm if the copy occurred successfully or not.

Raises:



36
37
38
39
 
# File 'lib/s3file.rb', line 36

def md5_hash(location)
  raise(S3PathError, "Not an S3 location") unless s3?(location)
  run_command("s3cmd --list-md5 ls #{location} -c #{@@config_file}").split(" ")[3]
end

.mv(source, destination) ⇒ Object

Moves a file from a bucket to another. Requires that both source and destination be s3 location - raises S3PathError otherwise.

Raises:



5
6
7
8
 
# File 'lib/s3file/mv.rb', line 5

def mv(source, destination)
  raise(S3PathError, "Either source or destination (or both) not S3") unless (s3?(source) && s3?(destination))
  run_command("s3cmd mv #{source} #{destination} -c #{@@config_file}")
end

.remove_bucket(bucket) ⇒ Object

Removes a bucket from S3. Raises a S3PathError in case bucket specified does not start with s3://

Raises:



12
13
14
15
 
# File 'lib/s3file/bucket.rb', line 12

def remove_bucket(bucket)
  raise(S3PathError, "Invalid S3 path") unless s3?(bucket)
  run_command("s3cmd rb #{bucket} -c #{@@config_file}")
end

.rm(location) ⇒ Object

Deletes the file from S3. Raises an S3CommandError in case of directory deletion. Use s3_r instead. Raises a S3PathError in case the location provided is not S3



6
7
8
9
10
11
12
13
 
# File 'lib/s3file/rm.rb', line 6

def rm(location)
  if s3?(location)
    raise(S3CommandError, "Attempting to delete folder using rm. Use rm_r instead.") if location.match(/\/\z/)
    run_command("s3cmd del #{location} -c #{@@config_file}")
  else
    raise(S3PathError, "Incorrect S3 path")
  end
end

.rm_r(location) ⇒ Object

Recursively deletes a folder from S3 Raises a S3PathError in case the location provided is not S3



17
18
19
20
21
22
23
 
# File 'lib/s3file/rm.rb', line 17

def rm_r(location)
  if s3?(location)
    run_command("s3cmd del #{location} -c #{@@config_file} --recursive")
  else
    raise(S3PathError, "Incorrect S3 path")
  end
end

.rm_rf(location) ⇒ Object

Recursively and forcefully deletes a folder from S3 Raises S3PathError in case the location provided is not an S3 location



27
28
29
30
31
32
33
 
# File 'lib/s3file/rm.rb', line 27

def rm_rf(location)
  if s3?(location)
    run_command("s3cmd del #{location} -c #{@@config_file} --recursive --force")
  else
    raise(S3PathError, "Incorrect S3 path")
  end
end

.run_command(command) ⇒ Object 



141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/s3file.rb', line 141

def run_command(command)
  output = status = err = nil
  status =
          Open4::popen4(command) do |pid, stdin, stdout, stderr|
            output = stdout.read
            err = stderr.read
          end

  if err.empty?
    return output
  else
    puts "Error #{err}"
    raise(AuthorizationError, "Authorization keys failed") if err.match("S3 error: 403")
    raise(S3PathError, "S3 file does not exist") if err.match("S3 error: 404")
    raise(S3PathError, "S3 bucket does not exist") if err.match("does not exist")
    raise(S3PathError, "S3 bucket does not exist") if err.match("was denied")
    raise(PathError, "Local file already exists") if err.match("already exists")
    raise(ConnectionError, "Check your connection and try again") if err.match("Errno -2")
    raise(UnknownError, "Unknown Error caught - #{err}")
  end
end

.s3?(location) ⇒ Boolean

Checks if the location is on s3 or not Checking if the location starts with s3://

Returns:

  • (Boolean) 


113
114
115
116
 
# File 'lib/s3file.rb', line 113

def s3?(location)
  return true if location.match(/s3:\/\//)
  return false
end

.s3_bucket(location) ⇒ Object

Gets the s3 bucket given a location



136
137
138
 
# File 'lib/s3file.rb', line 136

def s3_bucket(location)
  location.match("s3://[^\/]*").to_s
end

.s3_directory(location) ⇒ Object

Gets the directory of the file given the location



131
132
133
 
# File 'lib/s3file.rb', line 131

def s3_directory(location)
  location.sub(%r{#{location.match(/[^\/]*\z/)}\z}, "")
end