Class: Mir::Disk::Amazon

Inherits:
Object
  • Object
show all
Defined in:
lib/mir/disk/amazon.rb

Constant Summary collapse

DEFAULT_CHUNK_SIZE =

This is the default size in bytes at which files will be split and stored on S3. From trial and error, 5MB seems to be a good default size for chunking large files.

5*(2**20)

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(settings = {}) ⇒ Amazon

Returns a new instance of Amazon.



29
30
31
32
33
34
35
 
# File 'lib/mir/disk/amazon.rb', line 29

def initialize(settings = {})
  @bucket_name = settings[:bucket_name]
  @access_key_id = settings[:access_key_id]
  @secret_access_key = settings[:secret_access_key]
  @chunk_size = settings[:chunk_size] || DEFAULT_CHUNK_SIZE
  @connection = try_connect
end

Instance Attribute Details

#bucket_name ⇒ Object (readonly)

Returns the value of attribute bucket_name.



14
15
16
 
# File 'lib/mir/disk/amazon.rb', line 14

def bucket_name
  @bucket_name
end

#connection ⇒ Object (readonly)

Returns the value of attribute connection.



14
15
16
 
# File 'lib/mir/disk/amazon.rb', line 14

def connection
  @connection
end

Class Method Details

.s3_key(path) ⇒ String

Converts a path name to a key that can be stored on s3

Parameters:

  • the (String) —

    path to the file

Returns:

  • (String) —

    an S3-safe key with leading slashes removed



21
22
23
24
25
26
27
 
# File 'lib/mir/disk/amazon.rb', line 21

def self.s3_key(path)
  if path[0] == File::SEPARATOR
    path[1..-1] 
  else
    path
  end
end

Instance Method Details

#chunk_size ⇒ Object 



47
48
49
 
# File 'lib/mir/disk/amazon.rb', line 47

def chunk_size
  @chunk_size
end

#chunk_size=(n) ⇒ Object

Raises:

  • (ArgumentError) 


42
43
44
45
 
# File 'lib/mir/disk/amazon.rb', line 42

def chunk_size=(n)
  raise ArgumentError unless n > 0
  @chunk_size = n
end

#collections ⇒ Object

Returns the buckets available from S3



38
39
40
 
# File 'lib/mir/disk/amazon.rb', line 38

def collections
  @connection.list_bucket.select(:key)
end

#connected? ⇒ Boolean

Returns:

  • (Boolean) 


81
82
83
 
# File 'lib/mir/disk/amazon.rb', line 81

def connected?
  @connection_success
end

#copy(from, dest) ⇒ Object

Copies the remote resource to the local filesystem

Parameters:

  • the (String) —

    remote name of the resource to copy

  • the (String) —

    local name of the destination



68
69
70
71
72
73
74
 
# File 'lib/mir/disk/amazon.rb', line 68

def copy(from, dest)
  open(dest, 'w') do |file|
    key = self.class.s3_key(from)
    remote_file = MultiPartFile.new(self, key)
    remote_file.get(dest)
  end
end

#delete(file_path) ⇒ Boolean

Deletes the remote version of the file

Returns:

  • (Boolean) —

    true if operation succeeded



91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/mir/disk/amazon.rb', line 91

def delete(file_path)
  key = self.class.s3_key(file_path)
  Mir.logger.info "Deleting remote object #{file_path}"

  begin
    remote_file = MultiPartFile.new(self, key)
  rescue Disk::RemoteFileNotFound => e
    Mir.logger.warn "Could not find remote resource '#{key}'"
    return false
  end
  
  if remote_file.multipart?
    delete_parts(key)
  else
    connection.delete(bucket_name, key)
  end
end

#key_exists?(key) ⇒ Boolean

Whether the key exists in S3

Parameters:

  • the (String) —

    S3 key name

Returns:

  • (Boolean) 


55
56
57
58
59
60
61
62
63
 
# File 'lib/mir/disk/amazon.rb', line 55

def key_exists?(key)
  begin
    connection.head(bucket_name, key)
  rescue RightAws::AwsError => e
    return false
  end

  true
end

#read(key) ⇒ Object

Retrieves the complete object from S3 without streaming



77
78
79
 
# File 'lib/mir/disk/amazon.rb', line 77

def read(key)
  connection.get_object(bucket_name, key)
end

#volume ⇒ Object 



85
86
87
 
# File 'lib/mir/disk/amazon.rb', line 85

def volume
  connection.bucket(bucket_name, true)
end

#write(file_path) ⇒ Object

Writes a file to Amazon S3. If the file size exceeds the chunk size, the file will be written in chunks

Parameters:

  • the (String) —

    absolute path of the file to be written

Raises:



114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
 
# File 'lib/mir/disk/amazon.rb', line 114

def write(file_path)
  key = self.class.s3_key(file_path)
  
  if File.size(file_path) <= chunk_size
    connection.put(bucket_name, key, open(file_path))
    raise Disk::IncompleteTransmission unless equals?(file_path, key)
  else          
    delete_parts(file_path) # clean up remaining part files if any exist
              
    open(file_path, "rb") do |source|
      part_id = 1
      while part = source.read(chunk_size) do
        part_name = Mir::Utils.filename_with_sequence(key, part_id)
        Mir.logger.debug "Writing part #{part_name}"
        
        temp_file(part_name) do |tmp|
          tmp.binmode                
          tmp.write(part)
          tmp.rewind
          connection.put(bucket_name, part_name, open(tmp.path))
          raise Disk::IncompleteTransmission unless equals?(tmp.path, part_name)
        end

        part_id += 1
      end
    end
  end
  Mir.logger.info "Completed upload #{file_path}"
end