Module: RightScale::CloudApi::Utils::AWS

Defined in:
lib/cloud/aws/base/helpers/utils.rb

Overview

AWS helpers namespace

Constant Summary collapse

@@digest1 = 
OpenSSL::Digest.new("sha1")
@@digest256 =

Some installations may not support sha256

OpenSSL::Digest.new("sha256") rescue nil

Class Method Summary collapse

Class Method Details

.amz_escape(string) ⇒ String

Escapes a string accordingly to Amazon rules

Examples:

RightScale::CloudApi::Utils::AWS.amz_escape('something >= 13') #=>
  'something%20%3E%3D%2013'

Parameters:

  • string (String)

Returns:

  • (String)

See Also:



85
86
87
88
89
90
91
92
93
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 85

def self.amz_escape(string)
  string = string.to_s
  # Use UTF-8 if current ruby supports it (1.9+)
  string = string.encode("UTF-8") if string.respond_to?(:encode)
  # CGI::escape is too clever:
  #  - it escapes '~' when Amazon wants it to be un-escaped
  #  - it escapes ' ' as '+' but Amazon loves it as '%20'
  CGI.escape(string).gsub('%7E','~').gsub('+','%20')
end

.hex_encode(bindata) ⇒ Object 



396
397
398
399
400
401
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 396

def self.hex_encode(bindata)
  result=""
  data=bindata.unpack("C*")
  data.each {|b| result+= "%02x" % b}
  result
end

.is_dns_bucket?(bucket_name) ⇒ Boolean

Returns true if the provided bucket name is a DNS compliant bucket name

Examples:

RightScale::CloudApi::Utils::AWS.is_dns_bucket?('my') #=> false

RightScale::CloudApi::Utils::AWS.is_dns_bucket?('my_bycket') #=> false

RightScale::CloudApi::Utils::AWS.is_dns_bucket?('my-bucket') #=> true

Parameters:

  • bucket_name (String)

Returns:

  • (Boolean)

See Also:



153
154
155
156
157
158
159
160
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 153

def self.is_dns_bucket?(bucket_name)
  bucket_name = bucket_name.to_s
  return false unless (3..63) === bucket_name.size
  bucket_name.split('.').each do |component|
    return false unless component[/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/]
  end
  true
end

.parametrize(data) ⇒ Hash

Parametrizes data to the format that Amazon EC2 (and compatible APIs) loves

Examples:

# Where hash is:
{ Name.?.Mask  => Value | [ Values ],
  NamePrefix.? => [{ SubNameA.1 => ValueA.1,  SubNameB.1 => ValueB.1 },   # any simple parameter
                  ...,
                   { SubNameN.X => ValueN.X,  SubNameM.X => ValueN.X }]   # see BlockDeviceMapping case

parametrize( 'ParamA'             => 'a',
             'ParamB'             => ['b', 'c'],
             'ParamC.?.Something' => ['d', 'e'],
             'Filter'             => [ { 'Key' => 'A', 'Value' => ['aa','ab']},
                                       { 'Key' => 'B', 'Value' => ['ba','bb']}] ) #=>
  {
    "Filter.1.Key"       => "A",
    "Filter.1.Value.1"   => "aa",
    "Filter.1.Value.2"   => "ab",
    "Filter.2.Key"       => "B",
    "Filter.2.Value.1"   => "ba",
    "Filter.2.Value.2"   => "bb",
    "ParamA"             => "a",
    "ParamB.1"           => "b",
    "ParamB.2"           => "c",
    "ParamC.1.Something" => "d",
    "ParamC.2.Something" => "e"
  }

# BlockDeviceMapping example
parametrize( 'ImageId'  => 'i-01234567',
             'MinCount' => 1,
             'MaxCount' => 2,
             'KeyName'  => 'my-key',
             'SecurityGroupId' => ['sg-01234567', 'sg-12345670', 'sg-23456701'],
             'BlockDeviceMapping' => [
                { 'DeviceName'     => '/dev/sda1',
                  'Ebs.SnapshotId' => 'snap-01234567',
                  'Ebs.VolumeSize' => 20,
                  'Ebs.DeleteOnTermination' => true },
                { 'DeviceName'     => '/dev/sdb1',
                  'Ebs.SnapshotId' => 'snap-12345670',
                  'Ebs.VolumeSize' => 10,
                  'Ebs.DeleteOnTermination' => false } ] ) #=>
  {
    "BlockDeviceMapping.1.DeviceName"              => "/dev/sda1",
    "BlockDeviceMapping.1.Ebs.DeleteOnTermination" => true,
    "BlockDeviceMapping.1.Ebs.SnapshotId"          => "snap-01234567",
    "BlockDeviceMapping.1.Ebs.VolumeSize"          => 20,
    "BlockDeviceMapping.2.DeviceName"              => "/dev/sdb1",
    "BlockDeviceMapping.2.Ebs.DeleteOnTermination" => false,
    "BlockDeviceMapping.2.Ebs.SnapshotId"          => "snap-12345670",
    "BlockDeviceMapping.2.Ebs.VolumeSize"          => 10,
    "ImageId"                                      => "i-01234567",
    "KeyName"                                      => "my-key",
    "MaxCount"                                     => 2,
    "MinCount"                                     => 1,
    "SecurityGroupId.1"                            => "sg-01234567",
    "SecurityGroupId.2"                            => "sg-12345670",
    "SecurityGroupId.3"                            => "sg-23456701"
  }

parametrize( 'DomainName' => 'kdclient',
             'Item' => [ { 'ItemName'  => 'konstantin',
                           'Attribute' => [ { 'Name' => 'sex',    'Value' => 'male' },
                                            { 'Name' => 'age',    'Value' => '38'} ] },
                         { 'ItemName'  => 'alex',
                           'Attribute' => [ { 'Name' => 'sex',    'Value' => 'male' },
                                            { 'Name' => 'weight', 'Value' => '188'},
                                            { 'Name' => 'age',    'Value' => '42'} ] },
                         { 'ItemName'  => 'diana',
                           'Attribute' => [ { 'Name' => 'sex',    'Value' => 'female' },
                                            { 'Name' => 'weight', 'Value' => '120'},
                                            { 'Name' => 'age',    'Value' => '25'} ] } ] ) #=>
  { "DomainName"               => "kdclient",
    "Item.1.ItemName"          => "konstantin",
    "Item.1.Attribute.1.Name"  => "sex",
    "Item.1.Attribute.1.Value" => "male",
    "Item.1.Attribute.2.Name"  => "weight",
    "Item.1.Attribute.2.Value" => "170",
    "Item.1.Attribute.3.Name"  => "age",
    "Item.1.Attribute.3.Value" => "38",
    "Item.2.ItemName"          => "alex",
    "Item.2.Attribute.1.Name"  => "sex",
    "Item.2.Attribute.1.Value" => "male",
    "Item.2.Attribute.2.Name"  => "weight",
    "Item.2.Attribute.2.Value" => "188",
    "Item.2.Attribute.3.Name"  => "age",
    "Item.2.Attribute.3.Value" => "42",
    "Item.3.ItemName"          => "diana",
    "Item.3.Attribute.1.Name"  => "sex",
    "Item.3.Attribute.1.Value" => "female",
    "Item.3.Attribute.2.Name"  => "weight",
    "Item.3.Attribute.2.Value" => "120",
    "Item.3.Attribute.3.Name"  => "age",
    "Item.3.Attribute.3.Value" => "25"}

Parameters:

  • data (Hash)

Returns:

  • (Hash) 


507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 507

def self.parametrize(data)
  return data unless data.is_a?(Hash)
  result = {}
  #
  data.each do |mask, values|
    current_values = Utils::arrayify(values)
    current_mask   = mask.dup.to_s
    current_mask  << ".?" unless current_mask[/\?/] if current_values.size > 1
    #
    current_values.dup.each_with_index do |value, idx|
      key  = current_mask.sub('?', (idx+1).to_s)
      item = parametrize(value)
      if item.is_a?(Hash)
        item.each{ |k, v| result["#{key}.#{k}"] = v }
      else
        result[key] = item
      end
    end
  end
  result
end

.sign(aws_secret_access_key, auth_string, digest = nil) ⇒ String

Generates a signature for the given string, secret access key and digest

Examples:

RightScale::CloudApi::Utils::AWS.sign('my-secret-key', 'something-that-needs-to-be-signed') #=>
  'kdHo0Ks4KkypU1CkYZzAxFIIX+0='

Parameters:

  • aws_secret_access_key (String)
  • auth_string (String)
  • digest (String) (defaults to: nil)

Returns:

  • (String) —

    The signature.



50
51
52
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 50

def self.sign(aws_secret_access_key, auth_string, digest=nil)
  Utils::base64en(OpenSSL::HMAC.digest(digest || @@digest1, aws_secret_access_key, auth_string))
end

.sign_s3_signature(aws_secret_access_key, verb, canonicalized_resource, _headers = {}) ⇒ String

Signs and Authenticates REST Requests

Examples:

sign_s3_signature('secret', :get, 'xxx/yyy/zzz/object', {'header'=>'value'}) #=>
  "i85igH0sftHD/cGZcLiBKcYEuks="

Parameters:

  • aws_secret_access_key (String)
  • verb (String, Symbol) —

    ‘get’ | ‘post’

  • canonicalized_resource (String)
  • _headers (Hash) (defaults to: {})

Returns:

  • (String)

See Also:



178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 178

def self.sign_s3_signature(aws_secret_access_key, verb, canonicalized_resource, _headers={})
  headers = {}
  # Make sure all our headers ara downcased
  _headers.each do |key, value|
    headers[key.to_s.downcase] = value.is_a?(Array) ? value.join(',') : value
  end
  content_md5                 = headers['content-md5']
  content_type                = headers['content-type']
  date                        = headers['x-amz-date'] || headers['date'] || headers['expires']
  canonicalized_x_amz_headers = headers.select{|key, value|  key[/^x-amz-/]}.sort.map{|key, value| "#{key}:#{value}"}.join("\n")
  canonicalized_x_amz_headers << "\n" unless canonicalized_x_amz_headers._blank?
  # StringToSign
  string_to_sign = "#{verb.to_s.upcase}\n"         +
                   "#{content_md5}\n"              +
                   "#{content_type}\n"             +
                   "#{date}\n"                     +
                   "#{canonicalized_x_amz_headers}"+
                   "#{canonicalized_resource}"
  sign(aws_secret_access_key, string_to_sign)
end

.sign_v2_signature(aws_secret_access_key, params, verb, host, urn) ⇒ String

Signature Version 2

EC2, SQS and SDB requests must be signed by this guy

Examples:

params = {'InstanceId' => 'i-00000000'}
sign_v2_signature('secret', params, :get, 'ec2.amazonaws.com', '/') #=>
  "InstanceId=i-00000000&SignatureMethod=HmacSHA256&SignatureVersion=2&
   Timestamp=2014-03-12T21%3A52%3A21.000Z&Signature=gR2x3oWmNbh4bdZksPS
   sg3t7U0zbTcnFOfizWF3Zujw%3D"

Parameters:

  • aws_secret_access_key (String)
  • params (Hash)
  • verb (String, Symbol) —

    ‘get’ | ‘post’

  • host (String)
  • urn (String)

Returns:

  • (String)

See Also:



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 118

def self.sign_v2_signature(aws_secret_access_key, params, verb, host, urn)
  params["Timestamp"]      ||= utc_iso8601(Time.now) unless params["Expires"]
  params["SignatureVersion"] = '2'
  # select a signing method (make an old openssl working with sha1)
  # make 'HmacSHA256' to be a default one
  params['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(params['SignatureMethod'])
  params['SignatureMethod'] = 'HmacSHA1'   unless @@digest256
  # select a digest
  digest = (params['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
  # form string to sign
  canonical_string = Utils::params_to_urn(params){ |value| amz_escape(value) }
  string_to_sign = "#{verb.to_s.upcase}\n" +
                   "#{host.downcase}\n"    +
                   "#{urn}\n"              +
                   "#{canonical_string}"
  "#{canonical_string}&Signature=#{amz_escape(sign(aws_secret_access_key, string_to_sign, digest))}"
end

.sign_v4_get_canonical_headers(headers) ⇒ Object 



326
327
328
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 326

def self.sign_v4_get_canonical_headers(headers)
  headers.sort.map{ |key, value| "#{key}:#{value}" }.join("\n")
end

.sign_v4_get_canonical_path(path) ⇒ Object 



307
308
309
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 307

def self.sign_v4_get_canonical_path(path)
  path
end

.sign_v4_get_canonical_string(verb, path, query_string, headers, signed_headers, payload) ⇒ String

Signature V4: Returns Canonical String

Returns:

  • (String) 


364
365
366
367
368
369
370
371
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 364

def self.sign_v4_get_canonical_string(verb, path, query_string, headers, signed_headers, payload)
  verb           + "\n"   +
  path           + "\n"   +
  query_string   + "\n"   +
  headers        + "\n\n" +
  signed_headers + "\n"   +
  payload
end

.sign_v4_get_canonical_verb(verb) ⇒ Object 



302
303
304
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 302

def self.sign_v4_get_canonical_verb(verb)
  verb.to_s.upcase
end

.sign_v4_get_service_and_region(host) ⇒ Object 



200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 200

def self.sign_v4_get_service_and_region(host)
  result =
    case
    when host[                  /^iam\.amazonaws\.com$/i ] then ['iam', 'us-east-1']
    when host[              /^route53\.amazonaws\.com$/i ] then ['route53', 'us-east-1']
    when host[            /^(.*\.)?s3\.amazonaws\.com$/i ] then ['s3',  'us-east-1']
    when host[ /^(.*\.)?s3-external-1\.amazonaws\.com$/i ] then ['s3',  'us-east-1']
    when host[    /s3-website(-|\.)([^.]+)\.amazonaws\.com(\.cn)?$/i ] then ['s3', $2]
    when host[     /^(.*\.)?s3-([^.]+).amazonaws\.com(\.cn)?$/i ] then ['s3', $2]
    when host[   /^(.*\.)?s3\.([^.]+)\.amazonaws\.com(\.cn)?$/i ] then ['s3', $2]
    else host[     /^([^.]+)\.([^.]+)\.amazonaws\.com(\.cn)?$/i ]   && [$1,   $2]
    end
  fail(ArgumentError, "Cannot extract service name from %s host" % host.inspect) if !result || result[0].to_s.empty?
  fail(ArgumentError, "Cannot extract region name from %s host"  % host.inspect) if result[1].to_s.empty?
  result
end

.sign_v4_get_signature_key(key, string_to_sign, date, region, service, digest = @@digest256) ⇒ Object

Helpers from AWS documentation docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html



387
388
389
390
391
392
393
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 387

def self.sign_v4_get_signature_key(key, string_to_sign, date, region, service, digest = @@digest256)
  k_date    = OpenSSL::HMAC.digest(digest, "AWS4" + key, date)
  k_region  = OpenSSL::HMAC.digest(digest, k_date,       region)
  k_service = OpenSSL::HMAC.digest(digest, k_region,     service)
  k_signing = OpenSSL::HMAC.digest(digest, k_service,    "aws4_request")
  hex_encode  OpenSSL::HMAC.digest(digest, k_signing,    string_to_sign)
end

.sign_v4_get_signed_headers(headers) ⇒ Object 



331
332
333
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 331

def self.sign_v4_get_signed_headers(headers)
  headers.keys.sort.join(';')
end

.sign_v4_get_string_to_sign(algorithm, current_time, creds_scope, canonical_string) ⇒ String

Signature V4: A string to sign value

Returns:

  • (String) 


378
379
380
381
382
383
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 378

def self.sign_v4_get_string_to_sign(algorithm, current_time, creds_scope, canonical_string)
  algorithm    + "\n" +
  current_time + "\n" +
  creds_scope  + "\n" +
  hex_encode(Digest::SHA256.digest(canonical_string)).downcase
end

.sign_v4_headers(request, host, current_time) ⇒ Object 



336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 336

def self.sign_v4_headers(request, host, current_time)
  expires_at = request[:headers]['X-Amz-Expires'].first || 3600
  expires_at = expires_at.to_i if expires_at.is_a?(Time)

  if request[:body].is_a?(IO)
    canonical_payload = ''
    request[:headers].set_if_blank('X-Amz-Content-Sha256', 'UNSIGNED-PAYLOAD')
  else
    request[:body]    = request[:body].to_s
    canonical_payload = hex_encode(Digest::SHA256.digest(request[:body]))
    content_type      = 'application/x-www-form-urlencoded; charset=utf-8'
    content_md5       = Base64::encode64(Digest::MD5::digest(request[:body])).strip
    request[:headers].set_if_blank('Content-Length',       request[:body].bytesize)
    request[:headers].set_if_blank('Content-Type',         content_type)
    request[:headers].set_if_blank('Content-Md5',          content_md5)
    request[:headers].set_if_blank('X-Amz-Content-Sha256', canonical_payload)
  end
  request[:headers]['X-Amz-Date']    = current_time
  request[:headers]['X-Amz-Expires'] = expires_at

  canonical_payload
end

.sign_v4_query_params(request, algorithm, current_time, signed_headers, aws_access_key, creds_scope) ⇒ Object 



312
313
314
315
316
317
318
319
320
321
322
323
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 312

def self.sign_v4_query_params(request, algorithm, current_time, signed_headers, aws_access_key, creds_scope)
  expires_at = request[:params]['X-Amz-Expires'] || 3600
  expires_at = expires_at.to_i if expires_at.is_a?(Time)

  request[:params]['X-Amz-Date']          = current_time
  request[:params]['X-Amz-Expires']       = expires_at
  request[:params]['X-Amz-Algorithm']     = algorithm
  request[:params]['X-Amz-SignedHeaders'] = signed_headers
  request[:params]['X-Amz-Credential']    = "%s/%s" % [aws_access_key, creds_scope]

  'UNSIGNED-PAYLOAD'
end

.sign_v4_signature(aws_access_key, aws_secret_access_key, host, request, method = :headers) ⇒ String

Signs and Authenticates REST Requests

Parameters:

  • aws_secret_access_key (String)
  • aws_access_key (String)
  • host (String)
  • request (Hash)

Returns:

  • (String)

See Also:



229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 229

def self.sign_v4_signature(aws_access_key, aws_secret_access_key, host, request, method=:headers)
  now             = Time.now.utc
  current_date    = now.strftime("%Y%m%d")
  current_time    = now.strftime("%Y%m%dT%H%M%SZ")
  host            = host.downcase
  service, region = sign_v4_get_service_and_region(host)
  creds_scope     = "%s/%s/%s/aws4_request" % [current_date, region, service]
  algorithm       = "AWS4-HMAC-SHA256"

  # Verb
  canonical_verb = sign_v4_get_canonical_verb(request[:verb])

  # Path
  request[:path] ||= '/'
  canonical_path   = sign_v4_get_canonical_path(request[:path])

  # Headers (Auth)
  request[:headers].delete('Authorization')
  if method == :headers
    canonical_payload = sign_v4_headers(request, host, current_time)
  end
  # Headers (Standard)
  request[:headers]['Host'] = host
  _headers = {}
  request[:headers].each do |key, value|
    _headers[key.to_s.downcase] = value.is_a?(Array) ? value.join(',') : value
  end
  canonical_headers = sign_v4_get_canonical_headers(_headers)
  signed_headers    = sign_v4_get_signed_headers(_headers)

  # Params (Auth)
  if method != :headers
    canonical_payload = sign_v4_query_params(
      request,
      algorithm,
      current_time,
      signed_headers,
      aws_access_key,
      creds_scope
    )
  end
  # Params (Standard)
  canonical_query_string = Utils::params_to_urn(request[:params]){ |value| amz_escape(value) }

  # Canonical String
  canonical_string = sign_v4_get_canonical_string(
    canonical_verb,
    canonical_path,
    canonical_query_string,
    canonical_headers,
    signed_headers,
    canonical_payload
  )

  # StringToSign
  string_to_sign = sign_v4_get_string_to_sign(algorithm, current_time, creds_scope, canonical_string)

  # Signature
  signature = sign_v4_get_signature_key(aws_secret_access_key, string_to_sign, current_date, region, service)

  request[:path] += "?%s" % canonical_query_string unless canonical_query_string.empty?

  if method == :headers
    # Authorization Header
    authorization_header = "%s Credential=%s/%s, SignedHeaders=%s, Signature=%s" %
                           [algorithm, aws_access_key, creds_scope, signed_headers, signature]
    request[:headers]['Authorization'] = authorization_header
  else
    request[:path] += "&X-Amz-Signature=%s" % signature
  end
end

.utc_iso8601(time) ⇒ String

Returns ISO-8601 representation for the given time

Examples:

RightScale::CloudApi::Utils::AWS.utc_iso8601(Time.now) #=> '2013-03-22T21:00:21.000Z'

RightScale::CloudApi::Utils::AWS.utc_iso8601(0) #=> '1970-01-01T00:00:00.000Z'

Parameters:

  • time (Time, Fixnum)

Returns:

  • (String) 


66
67
68
69
70
71
72
 
# File 'lib/cloud/aws/base/helpers/utils.rb', line 66

def self.utc_iso8601(time)
  case
  when time.is_a?(Fixnum) then Time::at(time)
  when time.is_a?(String) then Time::parse(time)
  else                         time
  end.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
end