Class: RightAws::S3Interface

Inherits:
Object
  • Object
show all
Defined in:
lib/s3/right_s3_interface.rb

Defined Under Namespace

Classes: S3AclParser, S3HttpResponseBodyParser, S3HttpResponseHeadParser, S3HttpResponseParser, S3ImprovedListBucketParser, S3ListAllMyBucketsParser, S3ListBucketParser, S3TrueParser

Constant Summary collapse

DEFAULT_HOST = 
's3.amazonaws.com'
DEFAULT_PORT = 
443
DEFAULT_PROTOCOL = 
'https'
REQUEST_TTL = 
30
DEFAULT_EXPIRES_AFTER =

One day’s worth of seconds

1 * 24 * 60 * 60
AMAZON_HEADER_PREFIX = 
'x-amz-'
AMAZON_METADATA_PREFIX = 
'x-amz-meta-'
@@amazon_problems =

A list if Amazons problems we can handle by AWSErrorHandler.

RightAws::AMAZON_PROBLEMS
@@bench_s3 = 
Benchmark::Tms.new()
@@bench_xml = 
Benchmark::Tms.new()

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(aws_access_key_id, aws_secret_access_key, params = {}) ⇒ S3Interface

Creates new RightS3 instance.

s3 = RightAws::S3Interface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX', {:multi_thread => true, :logger => Logger.new('/tmp/x.log')}) #=> #<RightS3:0xb7b3c27c>

Params is a hash:

{:server       => 's3.amazonaws.com'   # Amazon service host: 's3.amazonaws.com'(default)
 :port         => 443                  # Amazon service port: 80 or 443(default)
 :protocol     => 'https'              # Amazon service protocol: 'http' or 'https'(default)
 :multi_thread => true|false           # Multi-threaded (connection per each thread): true or false(default)
 :logger       => Logger Object}       # Logger instance: logs to STDOUT if omitted }

Raises:



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/s3/right_s3_interface.rb', line 96

def initialize(aws_access_key_id, aws_secret_access_key, params={})
  @params = params
  raise AwsError.new("AWS access keys are required to operate on S3") \
    if aws_access_key_id.blank? || aws_secret_access_key.blank?

  # TODO TRB 6/19/07 - keys, basic params, and logger are all 
  # candidates to break out into a helper class common to all 
  # service gems.  Stick the helper in right_awsbase
  @aws_access_key_id     = aws_access_key_id
  @aws_secret_access_key = aws_secret_access_key
    # params
  @params[:server]       ||= DEFAULT_HOST
  @params[:port]         ||= DEFAULT_PORT
  @params[:protocol]     ||= DEFAULT_PROTOCOL
  @params[:multi_thread] ||= defined?(AWS_DAEMON)
    # set logger
  @logger = @params[:logger]
  @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
  @logger = Logger.new(STDOUT)   if !@logger
  @logger.info "New #{self.class.name} using #{@params[:multi_thread] ? 'multi' : 'single'}-threaded mode"
  @error_handler = nil
end

Instance Attribute Details

#aws_access_key_id ⇒ Object (readonly)

Current aws_access_key_id



48
49
50
 
# File 'lib/s3/right_s3_interface.rb', line 48

def aws_access_key_id
  @aws_access_key_id
end

#last_errors ⇒ Object

Last AWS errors list (used by AWSErrorHandler)



54
55
56
 
# File 'lib/s3/right_s3_interface.rb', line 54

def last_errors
  @last_errors
end

#last_request ⇒ Object (readonly)

Last HTTP request object



50
51
52
 
# File 'lib/s3/right_s3_interface.rb', line 50

def last_request
  @last_request
end

#last_request_id ⇒ Object

Last AWS request id (used by AWSErrorHandler)



56
57
58
 
# File 'lib/s3/right_s3_interface.rb', line 56

def last_request_id
  @last_request_id
end

#last_response ⇒ Object (readonly)

Last HTTP response object



52
53
54
 
# File 'lib/s3/right_s3_interface.rb', line 52

def last_response
  @last_response
end

#logger ⇒ Object

Logger object



58
59
60
 
# File 'lib/s3/right_s3_interface.rb', line 58

def logger
  @logger
end

#params ⇒ Object

Initial params hash



60
61
62
 
# File 'lib/s3/right_s3_interface.rb', line 60

def params
  @params
end

Class Method Details

.amazon_problems ⇒ Object

Returns a list of Amazon service responses which are known to be transient problems. We have to re-request if we get any of them, because the problem will probably disappear. By default this method returns the same value as the AMAZON_PROBLEMS const.



74
75
76
 
# File 'lib/s3/right_s3_interface.rb', line 74

def self.amazon_problems
  @@amazon_problems
end

.amazon_problems=(problems_list) ⇒ Object

Sets the list of Amazon side problems. Use in conjunction with the getter to append problems.



80
81
82
 
# File 'lib/s3/right_s3_interface.rb', line 80

def self.amazon_problems=(problems_list)
  @@amazon_problems = problems_list
end

.bench_s3 ⇒ Object

Benchmark::Tms instance for S3 access benchmark.



66
 
# File 'lib/s3/right_s3_interface.rb', line 66

def self.bench_s3;  @@bench_s3;  end

.bench_xml ⇒ Object

Benchmark::Tms instance for XML parsing benchmark.



69
 
# File 'lib/s3/right_s3_interface.rb', line 69

def self.bench_xml; @@bench_xml; end

Instance Method Details

#canonical_string(method, path, headers = {}, expires = nil) ⇒ Object

Produces canonical string for signing.



135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/s3/right_s3_interface.rb', line 135

def canonical_string(method, path, headers={}, expires=nil) # :nodoc:
  s3_headers = {}
  headers.each do |key, value|
    key = key.downcase
    s3_headers[key] = value.to_s.strip if key[/^#{AMAZON_HEADER_PREFIX}|^content-md5$|^content-type$|^date$/o]
  end
  s3_headers['content-type'] ||= ''
  s3_headers['content-md5']  ||= ''
  s3_headers['date']           = ''      if s3_headers.has_key? 'x-amz-date'
  s3_headers['date']           = expires if expires
    # prepare output string
  out_string = "#{method}\n"
  s3_headers.sort { |a, b| a[0] <=> b[0] }.each do |key, value|
    out_string << (key[/^#{AMAZON_HEADER_PREFIX}/o] ? "#{key}:#{value}\n" : "#{value}\n")
  end
    # ignore everything after the question mark...
  out_string << path.gsub(/\?.*$/, '')
   # ...unless there is an acl or torrent parameter
  out_string << '?acl'    if path[/[&?]acl($|&|=)/]
  out_string << '?torrent'if path[/[&?]torrent($|&|=)/]
  out_string
end

#clear_bucket(bucket) ⇒ Object

Removes all keys from bucket. Returns true or an exception.

s3.clear_bucket('my_awesome_bucket') #=> true



530
531
532
533
534
535
536
537
 
# File 'lib/s3/right_s3_interface.rb', line 530

def clear_bucket(bucket)
  incrementally_list_bucket(bucket) do |results|
    results[:contents].each { |key| delete(bucket, key[:key]) }
  end
  true
rescue
  on_exception
end

#create_bucket(bucket, headers = {}) ⇒ Object

Creates new bucket. Returns true or an exception.

s3.create_bucket('my_awesome_bucket') #=> true



262
263
264
265
266
267
 
# File 'lib/s3/right_s3_interface.rb', line 262

def create_bucket(bucket, headers={})
  req_hash = generate_rest_request('PUT', headers.merge(:url=>bucket))
  request_info(req_hash, S3TrueParser.new)
rescue
  on_exception
end

#create_bucket_link(bucket, expires = nil, headers = {}) ⇒ Object

Generates link for ‘CreateBucket’.

s3.create_bucket_link('my_awesome_bucket') #=> url string



617
618
619
620
621
 
# File 'lib/s3/right_s3_interface.rb', line 617

def create_bucket_link(bucket, expires=nil, headers={})
  generate_link('PUT', headers.merge(:url=>bucket), expires)
rescue
  on_exception
end

#delete(bucket, key = '', headers = {}) ⇒ Object

Deletes key. Returns true or an exception.

s3.delete('my_awesome_bucket', 'log/curent/1.log') #=> true



426
427
428
429
430
431
 
# File 'lib/s3/right_s3_interface.rb', line 426

def delete(bucket, key='', headers={})
  req_hash = generate_rest_request('DELETE', headers.merge(:url=>"#{bucket}/#{CGI::escape key}"))
  request_info(req_hash, S3TrueParser.new)
rescue
  on_exception
end

#delete_bucket(bucket, headers = {}) ⇒ Object

Deletes new bucket. Bucket must be empty! Returns true or an exception.

s3.delete_bucket('my_awesome_bucket')  #=> true

See also: force_delete_bucket method



275
276
277
278
279
280
 
# File 'lib/s3/right_s3_interface.rb', line 275

def delete_bucket(bucket, headers={})
  req_hash = generate_rest_request('DELETE', headers.merge(:url=>bucket))
  request_info(req_hash, S3TrueParser.new)
rescue
  on_exception
end

#delete_bucket_link(bucket, expires = nil, headers = {}) ⇒ Object

Generates link for ‘DeleteBucket’.

s3.delete_bucket_link('my_awesome_bucket') #=> url string



627
628
629
630
631
 
# File 'lib/s3/right_s3_interface.rb', line 627

def delete_bucket_link(bucket, expires=nil,  headers={})
  generate_link('DELETE', headers.merge(:url=>bucket), expires)
rescue
  on_exception
end

#delete_folder(bucket, folder_key, separator = '/') ⇒ Object

Deletes all keys where the ‘folder_key’ may be assumed as ‘folder’ name. Returns an array of string keys that have been deleted.

s3.list_bucket('my_awesome_bucket').map{|key_data| key_data[:key]} #=> ['test','test/2/34','test/3','test1','test1/logs']
s3.delete_folder('my_awesome_bucket','test')                       #=> ['test','test/2/34','test/3']



555
556
557
558
559
560
561
562
563
564
565
566
 
# File 'lib/s3/right_s3_interface.rb', line 555

def delete_folder(bucket, folder_key, separator='/')
  folder_key.chomp!(separator)
  allkeys = []
  incrementally_list_bucket(bucket, { 'prefix' => folder_key }) do |results|
    keys = results[:contents].map{ |s3_key| s3_key[:key][/^#{folder_key}($|#{separator}.*)/] ? s3_key[:key] : nil}.compact
    keys.each{ |key| delete(bucket, key) }
    allkeys << keys
  end
  allkeys
rescue
  on_exception
end

#delete_link(bucket, key, expires = nil, headers = {}) ⇒ Object

Generates link for ‘DeleteObject’.

s3.delete_link('my_awesome_bucket',key) #=> url string



678
679
680
681
682
 
# File 'lib/s3/right_s3_interface.rb', line 678

def delete_link(bucket, key, expires=nil, headers={})
  generate_link('DELETE', headers.merge(:url=>"#{bucket}/#{CGI::escape key}"), expires)
rescue
  on_exception
end

#force_delete_bucket(bucket) ⇒ Object

Deletes all keys in bucket then deletes bucket. Returns true or an exception.

s3.force_delete_bucket('my_awesome_bucket')



543
544
545
546
547
548
 
# File 'lib/s3/right_s3_interface.rb', line 543

def force_delete_bucket(bucket)
  clear_bucket(bucket)
  delete_bucket(bucket)
rescue
  on_exception
end

#generate_link(method, headers = {}, expires = nil) ⇒ Object

Generates link for QUERY API



583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
 
# File 'lib/s3/right_s3_interface.rb', line 583

def generate_link(method, headers={}, expires=nil) #:nodoc:
  path = headers[:url]
  path = "/#{path}" unless path[/^\//]
   # expiration time
  expires ||= DEFAULT_EXPIRES_AFTER
  expires   = Time.now.utc.since(expires) if expires.is_a?(Fixnum) && (expires<1.year)
  expires   = expires.to_i
    # remove unset(==optional) and symbolyc keys
  headers.each{ |key, value| headers.delete(key) if (value.nil? || key.is_a?(Symbol)) }
    #generate auth strings
  auth_string = canonical_string(method, path, headers, expires)
  signature   = CGI::escape(Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new("sha1"), @aws_secret_access_key, auth_string)).strip)
    # path building
  addon = "Signature=#{signature}&Expires=#{expires}&AWSAccessKeyId=#{@aws_access_key_id}"
  path += path[/\?/] ? "&#{addon}" : "?#{addon}"
  "#{@params[:protocol]}://#{@params[:server]}:#{@params[:port]}#{path}"
rescue
  on_exception
end

#generate_rest_request(method, headers) ⇒ Object

Generates request hash for REST API.



159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/s3/right_s3_interface.rb', line 159

def generate_rest_request(method, headers)  # :nodoc:
  path = headers[:url]
  path = "/#{path}" unless path[/^\//]
  data = headers[:data]
    # remove unset(==optional) and symbolyc keys
  headers.each{ |key, value| headers.delete(key) if (value.nil? || key.is_a?(Symbol)) }
    #
  headers['content-type'] ||= ''
  headers['date']           = Time.now.httpdate
    # create request
  request      = "Net::HTTP::#{method.capitalize}".constantize.new(URI::escape(CGI::unescape(path)))
  request.body = data if data
    # set request headers and meta headers
  headers.each      { |key, value| request[key.to_s] = value }
    #generate auth strings
  auth_string = canonical_string(request.method, request.path, request.to_hash)
  signature   = Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new("sha1"), @aws_secret_access_key, auth_string)).strip
    # set other headers
  request['Authorization'] = "AWS #{@aws_access_key_id}:#{signature}"
    # prepare output hash
  { :request  => request, 
    :server   => @params[:server],
    :port     => @params[:port],
    :protocol => @params[:protocol] }
end

#get(bucket, key, headers = {}, &block) ⇒ Object

Retrieves object data from Amazon. Returns a hash or an exception.

s3.get('my_awesome_bucket', 'log/curent/1.log') #=>

    {:object  => "Ola-la!", 
     :headers => {"last-modified"     => "Wed, 23 May 2007 09:08:04 GMT", 
                  "content-type"      => "", 
                  "etag"              => "\"000000000096f4ee74bc4596443ef2a4\"", 
                   "date"              => "Wed, 23 May 2007 09:08:03 GMT", 
                   "x-amz-id-2"        => "ZZZZZZZZZZZZZZZZZZZZ1HJXZoehfrS4QxcxTdNGldR7w/FVqblP50fU8cuIMLiu", 
                   "x-amz-meta-family" => "Woho556!",
                   "x-amz-request-id"  => "0000000C246D770C", 
                   "server"            => "AmazonS3", 
                   "content-length"    => "7"}}

If a block is provided, yields incrementally to the block as the response is read. For large responses, this function is ideal as the response can be ‘streamed’. The hash containing header fields is still returned. Example: foo = File.new(‘./chunder.txt’, File::CREAT|File::RDWR) rhdr = s3.get(‘aws-test’, ‘Cent5V1_7_1.img.part.00’) do |chunk|

foo.write(chunk)

end foo.close



395
396
397
398
399
400
 
# File 'lib/s3/right_s3_interface.rb', line 395

def get(bucket, key, headers={}, &block)
  req_hash = generate_rest_request('GET', headers.merge(:url=>"#{bucket}/#{CGI::escape key}"))
  request_info(req_hash, S3HttpResponseBodyParser.new, &block)
rescue
  on_exception
end

#get_acl(bucket, key = '', headers = {}) ⇒ Object

Retieves the ACL (access control policy) for a bucket or object. Returns a hash of headers and xml doc with ACL data. See: docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAccessPolicy.html.

s3.get_acl('my_awesome_bucket', 'log/curent/1.log') #=>
  {:headers => {"x-amz-id-2"=>"B3BdDMDUz+phFF2mGBH04E46ZD4Qb9HF5PoPHqDRWBv+NVGeA3TOQ3BkVvPBjgxX",
                "content-type"=>"application/xml;charset=ISO-8859-1",
                "date"=>"Wed, 23 May 2007 09:40:16 GMT",
                "x-amz-request-id"=>"B183FA7AB5FBB4DD",
                "server"=>"AmazonS3",
                "transfer-encoding"=>"chunked"},
   :object  => "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<AccessControlPolicy xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Owner>
                <ID>16144ab2929314cc309ffe736daa2b264357476c7fea6efb2c3347ac3ab2792a</ID><DisplayName>root</DisplayName></Owner>
                <AccessControlList><Grant><Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"CanonicalUser\"><ID>
                16144ab2929314cc309ffe736daa2b264357476c7fea6efb2c3347ac3ab2792a</ID><DisplayName>root</DisplayName></Grantee>
                <Permission>FULL_CONTROL</Permission></Grant></AccessControlList></AccessControlPolicy>" }



449
450
451
452
453
454
455
 
# File 'lib/s3/right_s3_interface.rb', line 449

def get_acl(bucket, key='', headers={})
  key = key.blank? ? '' : "/#{CGI::escape key}"
  req_hash = generate_rest_request('GET', headers.merge(:url=>"#{bucket}#{key}?acl"))
  request_info(req_hash, S3HttpResponseBodyParser.new) 
rescue
  on_exception
end

#get_acl_link(bucket, key = '', headers = {}) ⇒ Object

Generates link for ‘GetACL’.

s3.get_acl_link('my_awesome_bucket',key) #=> url string



689
690
691
692
693
 
# File 'lib/s3/right_s3_interface.rb', line 689

def get_acl_link(bucket, key='', headers={})
  return generate_link('GET', headers.merge(:url=>"#{bucket}/#{CGI::escape key}?acl"))
rescue
  on_exception
end

#get_acl_parse(bucket, key = '', headers = {}) ⇒ Object

Retieves the ACL (access control policy) for a bucket or object. Returns a hash of :grantees

s3.get_acl_parse('my_awesome_bucket', 'log/curent/1.log') #=>

{ :grantees=>
  { "16...2a"=>
    { :display_name=>"root",
      :permissions=>["FULL_CONTROL"],
      :attributes=>
       { "xsi:type"=>"CanonicalUser",
         "xmlns:xsi"=>"http://www.w3.org/2001/XMLSchema-instance"}},
   "http://acs.amazonaws.com/groups/global/AllUsers"=>
     { :display_name=>"AllUsers",
       :permissions=>["READ"],
       :attributes=>
        { "xsi:type"=>"Group",
          "xmlns:xsi"=>"http://www.w3.org/2001/XMLSchema-instance"}}},
 :owner=>
   { :id=>"16..2a",
     :display_name=>"root"}}



479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
 
# File 'lib/s3/right_s3_interface.rb', line 479

def get_acl_parse(bucket, key='', headers={})
  key = key.blank? ? '' : "/#{CGI::escape key}"
  req_hash = generate_rest_request('GET', headers.merge(:url=>"#{bucket}#{key}?acl"))
  acl = request_info(req_hash, S3AclParser.new(:logger => @logger))
  result = {}
  result[:owner]    = acl[:owner]
  result[:grantees] = {}
  acl[:grantees].each do |grantee|
    key = grantee[:id] || grantee[:uri]
    if result[:grantees].key?(key)
      result[:grantees][key][:permissions] << grantee[:permissions]
    else
      result[:grantees][key] = 
        { :display_name => grantee[:display_name] || grantee[:uri].to_s[/[^\/]*$/],
          :permissions  => grantee[:permissions].to_a,
          :attributes   => grantee[:attributes] }
    end
  end
  result
rescue
  on_exception
end

#get_bucket_acl(bucket, headers = {}) ⇒ Object

Retieves the ACL (access control policy) for a bucket. Returns a hash of headers and xml doc with ACL data.



512
513
514
515
516
 
# File 'lib/s3/right_s3_interface.rb', line 512

def get_bucket_acl(bucket, headers={})
  return get_acl(bucket, '', headers)
rescue
  on_exception
end

#get_bucket_acl_link(bucket, headers = {}) ⇒ Object

Generates link for ‘GetBucketACL’.

s3.get_acl_link('my_awesome_bucket',key) #=> url string



709
710
711
712
713
 
# File 'lib/s3/right_s3_interface.rb', line 709

def get_bucket_acl_link(bucket, headers={})
  return get_acl_link(bucket, '', headers)
rescue
  on_exception
end

#get_link(bucket, key, expires = nil, headers = {}) ⇒ Object

Generates link for ‘GetObject’.

s3.get_link('my_awesome_bucket',key) #=> url string



658
659
660
661
662
 
# File 'lib/s3/right_s3_interface.rb', line 658

def get_link(bucket, key, expires=nil, headers={})
  generate_link('GET', headers.merge(:url=>"#{bucket}/#{CGI::escape key}"), expires)
rescue
  on_exception
end

#get_object(bucket, key, headers = {}) ⇒ Object

Retrieves object data only (headers are omitted). Returns string or an exception.

s3.get('my_awesome_bucket', 'log/curent/1.log') #=> 'Ola-la!'



572
573
574
575
576
 
# File 'lib/s3/right_s3_interface.rb', line 572

def get_object(bucket, key, headers={})
  get(bucket, key, headers)[:object]
rescue
  on_exception
end

#head(bucket, key, headers = {}) ⇒ Object

Retrieves object metadata. Returns a hash of http_response_headers.

s3.head('my_awesome_bucket', 'log/curent/1.log') #=>
  {"last-modified"     => "Wed, 23 May 2007 09:08:04 GMT", 
   "content-type"      => "", 
   "etag"              => "\"000000000096f4ee74bc4596443ef2a4\"", 
   "date"              => "Wed, 23 May 2007 09:08:03 GMT", 
   "x-amz-id-2"        => "ZZZZZZZZZZZZZZZZZZZZ1HJXZoehfrS4QxcxTdNGldR7w/FVqblP50fU8cuIMLiu", 
   "x-amz-meta-family" => "Woho556!",
   "x-amz-request-id"  => "0000000C246D770C", 
   "server"            => "AmazonS3", 
   "content-length"    => "7"}



415
416
417
418
419
420
 
# File 'lib/s3/right_s3_interface.rb', line 415

def head(bucket, key, headers={})
  req_hash = generate_rest_request('HEAD', headers.merge(:url=>"#{bucket}/#{CGI::escape key}"))
  request_info(req_hash, S3HttpResponseHeadParser.new)
rescue
  on_exception
end

#head_link(bucket, key, expires = nil, headers = {}) ⇒ Object

Generates link for ‘HeadObject’.

s3.head_link('my_awesome_bucket',key) #=> url string



668
669
670
671
672
 
# File 'lib/s3/right_s3_interface.rb', line 668

def head_link(bucket, key, expires=nil,  headers={})
  generate_link('HEAD', headers.merge(:url=>"#{bucket}/#{CGI::escape key}"), expires)
rescue
  on_exception
end

#incrementally_list_bucket(bucket, options = {}, headers = {}, &block) ⇒ Object

Incrementally list the contents of a bucket. Yields the following hash to a block:

s3.incrementally_list_bucket('my_awesome_bucket', { 'prefix'=>'t', 'marker'=>'', 'max-keys'=>5, delimiter=>'' }) yields  
 {
   :name => 'bucketname',
   :prefix => 'subfolder/',
   :marker => 'fileN.jpg',
   :max_keys => 234,
   :delimiter => '/',
   :is_truncated => true,
   :next_marker => 'fileX.jpg',
   :contents => [
     { :key => "file1",
       :last_modified => "2007-05-18T07:00:59.000Z",
       :e_tag => "000000000059075b964b07152d234b70",
       :size => 3,
       :storage_class => "STANDARD",
       :owner_id => "00000000009314cc309ffe736daa2b264357476c7fea6efb2c3347ac3ab2792a",
       :owner_display_name => "root"
     }, { :key, ...}, ... {:key, ...}
   ]
   :common_prefixes => [
     "prefix1",
     "prefix2",
     ...,
     "prefixN"
   ]
 }



333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
 
# File 'lib/s3/right_s3_interface.rb', line 333

def incrementally_list_bucket(bucket, options={}, headers={}, &block)
  internal_options = options.dup
  begin 
    internal_bucket = bucket.dup
    internal_bucket  += '?'+internal_options.map{|k, v| "#{k.to_s}=#{CGI::escape v.to_s}"}.join('&') unless internal_options.blank?
    req_hash = generate_rest_request('GET', headers.merge(:url=>internal_bucket))
    response = request_info(req_hash, S3ImprovedListBucketParser.new(:logger => @logger))
    there_are_more_keys = response[:is_truncated]
    if(there_are_more_keys)
      if(response[:next_marker])
        internal_options['marker'] = response[:next_marker]
      else 
        internal_options['marker'] = response[:contents].last[:key]
      end
      internal_options['max-keys'] ? (internal_options['max-keys'] -= response[:contents].length) : nil 
    end
    yield response
  end while there_are_more_keys 
  true
rescue
  on_exception
end

#list_all_my_buckets(headers = {}) ⇒ Object

Returns an array of customer’s buckets. Each item is a hash.

s3.list_all_my_buckets #=> 
  [{:owner_id           => "00000000009314cc309ffe736daa2b264357476c7fea6efb2c3347ac3ab2792a",
    :owner_display_name => "root",
    :name               => "bucket_name",
    :creation_date      => "2007-04-19T18:47:43.000Z"}, ..., {...}]



251
252
253
254
255
256
 
# File 'lib/s3/right_s3_interface.rb', line 251

def list_all_my_buckets(headers={})
  req_hash = generate_rest_request('GET', headers.merge(:url=>''))
  request_info(req_hash, S3ListAllMyBucketsParser.new(:logger => @logger))
rescue
  on_exception
end

#list_all_my_buckets_link(expires = nil, headers = {}) ⇒ Object

Generates link for ‘ListAllMyBuckets’.

s3.list_all_my_buckets_link #=> url string



607
608
609
610
611
 
# File 'lib/s3/right_s3_interface.rb', line 607

def list_all_my_buckets_link(expires=nil, headers={})
  generate_link('GET', headers.merge(:url=>''), expires)
rescue
  on_exception
end

#list_bucket(bucket, options = {}, headers = {}) ⇒ Object

Returns an array of bucket’s keys. Each array item (key data) is a hash.

s3.list_bucket('my_awesome_bucket', { 'prefix'=>'t', 'marker'=>'', 'max-keys'=>5, delimiter=>'' }) #=>
  [{:key                => "test1",
    :last_modified      => "2007-05-18T07:00:59.000Z",
    :owner_id           => "00000000009314cc309ffe736daa2b264357476c7fea6efb2c3347ac3ab2792a",
    :owner_display_name => "root",
    :e_tag              => "000000000059075b964b07152d234b70",
    :storage_class      => "STANDARD",
    :size               => 3,
    :service=> {'is_truncated' => false,
                'prefix'       => "t",
                'marker'       => "",
                'name'         => "my_awesome_bucket",
                'max-keys'     => "5"}, ..., {...}]



298
299
300
301
302
303
304
 
# File 'lib/s3/right_s3_interface.rb', line 298

def list_bucket(bucket, options={}, headers={})
  bucket  += '?'+options.map{|k, v| "#{k.to_s}=#{CGI::escape v.to_s}"}.join('&') unless options.blank?
  req_hash = generate_rest_request('GET', headers.merge(:url=>bucket))
  request_info(req_hash, S3ListBucketParser.new(:logger => @logger))
rescue
  on_exception
end

#list_bucket_link(bucket, options = nil, expires = nil, headers = {}) ⇒ Object

Generates link for ‘ListBucket’.

s3.list_bucket_link('my_awesome_bucket') #=> url string



637
638
639
640
641
642
 
# File 'lib/s3/right_s3_interface.rb', line 637

def list_bucket_link(bucket, options=nil, expires=nil, headers={})
  bucket += '?' + options.map{|k, v| "#{k.to_s}=#{CGI::escape v.to_s}"}.join('&') unless options.blank?
  generate_link('GET', headers.merge(:url=>bucket), expires)
rescue
  on_exception
end

#multi_thread ⇒ Object

Return the true if this RightS3 instance works in multi_thread state and false otherwise.



127
128
129
 
# File 'lib/s3/right_s3_interface.rb', line 127

def multi_thread
  @params[:multi_thread]
end

#on_exception(options = {:raise=>true, :log=>true}) ⇒ Object

TODO TRB 6/19/07 - Service gem common method



120
121
122
 
# File 'lib/s3/right_s3_interface.rb', line 120

def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
  RightAws::AwsError::on_aws_exception(self, options)
end

#put(bucket, key, data = nil, headers = {}) ⇒ Object

Saves object to Amazon. Returns true or an exception. Any header starting with AMAZON_METADATA_PREFIX is considered user metadata. It will be stored with the object and returned when you retrieve the object. The total size of the HTTP request, not including the body, must be less than 4 KB.

s3.put('my_awesome_bucket', 'log/curent/1.log', 'Ola-la!', 'x-amz-meta-family'=>'Woho556!') #=> true



361
362
363
364
365
366
 
# File 'lib/s3/right_s3_interface.rb', line 361

def put(bucket, key, data=nil, headers={})
  req_hash = generate_rest_request('PUT', headers.merge(:url=>"#{bucket}/#{CGI::escape key}", :data=>data))
  request_info(req_hash, S3TrueParser.new)
rescue
  on_exception
end

#put_acl(bucket, key, acl_xml_doc, headers = {}) ⇒ Object

Sets the ACL on a bucket or object.



503
504
505
506
507
508
509
 
# File 'lib/s3/right_s3_interface.rb', line 503

def put_acl(bucket, key, acl_xml_doc, headers={})
  key = key.blank? ? '' : "/#{CGI::escape key}"
  req_hash = generate_rest_request('PUT', headers.merge(:url=>"#{bucket}#{key}?acl", :data=>acl_xml_doc))
  request_info(req_hash, S3HttpResponseBodyParser.new)
rescue
  on_exception
end

#put_acl_link(bucket, key = '', headers = {}) ⇒ Object

Generates link for ‘PutACL’.

s3.put_acl_link('my_awesome_bucket',key) #=> url string



699
700
701
702
703
 
# File 'lib/s3/right_s3_interface.rb', line 699

def put_acl_link(bucket, key='', headers={})
  return generate_link('PUT', headers.merge(:url=>"#{bucket}/#{CGI::escape key}?acl"))
rescue
  on_exception
end

#put_bucket_acl(bucket, acl_xml_doc, headers = {}) ⇒ Object

Sets the ACL on a bucket only.



519
520
521
522
523
 
# File 'lib/s3/right_s3_interface.rb', line 519

def put_bucket_acl(bucket, acl_xml_doc, headers={})
  return put_acl(bucket, '', acl_xml_doc, headers)
rescue
  on_exception
end

#put_bucket_acl_link(bucket, acl_xml_doc, headers = {}) ⇒ Object

Generates link for ‘PutBucketACL’.

s3.put_acl_link('my_awesome_bucket',key) #=> url string



719
720
721
722
723
 
# File 'lib/s3/right_s3_interface.rb', line 719

def put_bucket_acl_link(bucket, acl_xml_doc, headers={})
  return put_acl_link(bucket, '', acl_xml_doc, headers)
rescue
  on_exception
end

#put_link(bucket, key, data = nil, expires = nil, headers = {}) ⇒ Object

Generates link for ‘PutObject’.

s3.put_link('my_awesome_bucket',key, object) #=> url string



648
649
650
651
652
 
# File 'lib/s3/right_s3_interface.rb', line 648

def put_link(bucket, key, data=nil, expires=nil, headers={})
  generate_link('PUT', headers.merge(:url=>"#{bucket}/#{CGI::escape key}", :data=>data), expires)
rescue
  on_exception
end

#request_info(request, parser, &block) ⇒ Object

Sends request to Amazon and parses the response. Raises AwsError if any banana happened. TODO TRB 6/19/07: request_info is a candidate to move to right_awsbase

because it currently appears (in identical form) in right_s3, right_ec2, and right_sqs



191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
 
# File 'lib/s3/right_s3_interface.rb', line 191

def request_info(request, parser, &block) # :nodoc:
  thread = @params[:multi_thread] ? Thread.current : Thread.main
  thread[:s3_connection] ||= Rightscale::HttpConnection.new(:exception => RightAws::AwsError)
  @last_request  = request[:request]
  @last_response = nil
  response=nil

  if(block != nil)
    @@bench_s3.add! do
      responsehdr = thread[:s3_connection].request(request) do |response|
        if response.is_a?(Net::HTTPSuccess)
          @error_handler = nil
          response.read_body(&block)
        else
          @error_handler = AWSErrorHandler.new(self, parser, @@amazon_problems) unless @error_handler
          check_result   = @error_handler.check(request)
          if check_result
            @error_handler = nil
            return check_result 
          end
          raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
        end
      end
      @@bench_xml.add! do
        parser.parse(responsehdr)
      end
      return parser.result
    end
  else
    @@bench_s3.add!{ response = thread[:s3_connection].request(request) }
      # check response for errors...
    @last_response = response
    if response.is_a?(Net::HTTPSuccess)
      @error_handler = nil
      @@bench_xml.add! { parser.parse(response) }
      return parser.result
    else
      @error_handler = AWSErrorHandler.new(self, parser, @@amazon_problems) unless @error_handler
      check_result   = @error_handler.check(request)
      if check_result
        @error_handler = nil
        return check_result 
      end
      raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
    end
  end
rescue
  @error_handler = nil
  raise
end