Class: Rubber::Cloud::FogStorage

Inherits:
Object
  • Object
show all
Defined in:
lib/rubber/cloud/fog_storage.rb

Constant Summary collapse

RETRYABLE_EXCEPTIONS = 
[Excon::Errors::Error]

Instance Method Summary collapse

Constructor Details

#initialize(storage_provider, bucket) ⇒ FogStorage

Returns a new instance of FogStorage.



12
13
14
15
16
17
18
 
# File 'lib/rubber/cloud/fog_storage.rb', line 12

def initialize(storage_provider, bucket)
  raise "bucket required" unless bucket && bucket.size > 0
  
  @storage_provider = storage_provider
  @bucket = bucket
  @directory = @storage_provider.directories.get(@bucket)
end

Instance Method Details

#delete(key, opts = {}) ⇒ Object 



124
125
126
127
128
129
130
131
132
133
 
# File 'lib/rubber/cloud/fog_storage.rb', line 124

def delete(key, opts={})
  raise "a key is required" unless key && key.size > 0
  
  # store the object
  logger.debug "Deleting object: #{key}"
  Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
    file = @directory.files.get(key, opts);
    file.destroy if file
  end
end

#ensure_bucket ⇒ Object

create the bucket if needed



21
22
23
24
25
26
 
# File 'lib/rubber/cloud/fog_storage.rb', line 21

def ensure_bucket
  Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
    @directory = @storage_provider.directories.create(:key => @bucket) unless @directory
  end
  return self
end

#fetch(key, opts = {}, &block) ⇒ Object 



102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/rubber/cloud/fog_storage.rb', line 102

def fetch(key, opts={}, &block)
  raise "a key is required" unless key && key.size > 0
  
  if block_given?
    # TODO (nirvdrum 05/24/13) Remove when https://github.com/fog/fog/issues/1832 is fixed.
    begin
      @directory.files.get(key, opts, &block)
    rescue Excon::Errors::NotFound
      nil
    end
  else
    Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
      begin
        file = @directory.files.get(key, opts)
        file.body if file
      rescue Excon::Errors::NotFound
        nil
      end
    end
  end
end

#logger ⇒ Object 



8
9
10
 
# File 'lib/rubber/cloud/fog_storage.rb', line 8

def logger
  Rubber.logger
end

#multipart_store(key, data, opts = {}) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/rubber/cloud/fog_storage.rb', line 56

def multipart_store(key, data, opts={})
  raise "a key is required" unless key && key.size > 0
  
  opts = {:chunk_size => (5 * 2**20)}.merge(opts)
  
  chunk = data.read(opts[:chunk_size])
  
  if chunk.size < opts[:chunk_size]
    singlepart_store(key, chunk, opts)
  else
    logger.info "Multipart uploading #{key}"
  
    part_ids = []
  
    ensure_bucket()
    
    response = @storage_provider.initiate_multipart_upload(@bucket, key)
    begin
      upload_id = response.body['UploadId']
  
      while chunk
        next_chunk = data.read(opts[:chunk_size])
        if data.eof?
          chunk << next_chunk
          next_chunk = nil
        end
        part_number = part_ids.size + 1
        logger.info("Uploading part #{part_number}")
        Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
          response = @storage_provider.upload_part(@bucket, key, upload_id, part_number, chunk)
          part_ids << response.headers['ETag']
        end
        chunk = next_chunk
      end
  
      @storage_provider.complete_multipart_upload(@bucket, key, upload_id, part_ids)
      logger.info("Completed multipart upload: #{upload_id}")
  
    rescue Exception => e
      logger.error("Aborting multipart upload: #{upload_id}")
      @storage_provider.abort_multipart_upload(@bucket, key, upload_id)
      raise
    end
  end
end

#singlepart_store(key, data, opts = {}) ⇒ Object 



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/rubber/cloud/fog_storage.rb', line 37

def singlepart_store(key, data, opts={})
  raise "a key is required" unless key && key.size > 0
  
  file = nil
  
  # store the object
  logger.debug "Storing object: #{key}"
  
  ensure_bucket()
  
  Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
    file = @directory.files.new(opts.merge(:key => key))
    file.body = data
    file.save
  end
  
  file
end

#store(key, data, opts = {}) ⇒ Object

data can be a string or io handle



29
30
31
32
33
34
35
 
# File 'lib/rubber/cloud/fog_storage.rb', line 29

def store(key, data, opts={})
  if data.respond_to?(:read)
    multipart_store(key, data, opts)
  else
    singlepart_store(key, data, opts)
  end
end

#walk_tree(prefix = nil) ⇒ Object 



135
136
137
138
139
140
141
142
143
 
# File 'lib/rubber/cloud/fog_storage.rb', line 135

def walk_tree(prefix=nil)
  # Grab a new bucket connection since this method destroys
  # the cached copy of files once the prefix is applied.
  Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
    @storage_provider.directories.get(@bucket, :prefix => prefix).files.each do |f|
      yield f
    end
  end
end