Class: CloudProviders::Ec2

Inherits:
CloudProvider show all
Defined in:
lib/cloud_providers/ec2/ec2.rb

Instance Attribute Summary

Attributes inherited from CloudProvider

#init_opts, #name

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from CloudProvider

#after_initialized, #default_keypair_path, default_keypair_path, #initialize, #method_missing

Constructor Details

This class inherits a constructor from CloudProviders::CloudProvider

Dynamic Method Handling

This class handles dynamic methods through the method_missing method in the class CloudProviders::CloudProvider

Class Method Details

.default_access_key ⇒ Object

Set the aws keys from the environment, or load from /etc/poolparty/env.yml if the environment variable is not set



18
19
20
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 18

def self.default_access_key
  ENV['EC2_ACCESS_KEY'] || load_keys_from_file[:access_key] || load_keys_from_credential_file[:access_key]
end

.default_cert ⇒ Object 



30
31
32
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 30

def self.default_cert
  ENV['EC2_CERT'] || load_keys_from_file[:cert]
end

.default_cloud_cert ⇒ Object 



46
47
48
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 46

def self.default_cloud_cert
  ENV['CLOUD_CERT'] || ENV['EUCALYPTUS_CERT'] || load_keys_from_file[:cloud_cert]
end

.default_credential_file ⇒ Object 



50
51
52
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 50

def self.default_credential_file
  ENV['AWS_CREDENTIAL_FILE'] || load_keys_from_file[:credential_file]
end

.default_ec2_url ⇒ Object 



38
39
40
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 38

def self.default_ec2_url
  ENV['EC2_URL'] || load_keys_from_file[:ec2_url]
end

.default_private_key ⇒ Object 



26
27
28
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 26

def self.default_private_key
  ENV['EC2_PRIVATE_KEY'] || load_keys_from_file[:private_key]
end

.default_s3_url ⇒ Object 



42
43
44
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 42

def self.default_s3_url
  ENV['S3_URL'] || load_keys_from_file[:s3_url]
end

.default_secret_access_key ⇒ Object 



22
23
24
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 22

def self.default_secret_access_key
  ENV['EC2_SECRET_KEY'] || load_keys_from_file[:secret_access_key] || load_keys_from_credential_file[:secret_access_key]
end

.default_user_id ⇒ Object 



34
35
36
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 34

def self.default_user_id
  ENV['EC2_USER_ID'] || load_keys_from_file[:user_id]
end

.load_keys_from_credential_file(filename = default_credential_file, caching = true) ⇒ Object

Load credentials from file



63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 63

def self.load_keys_from_credential_file(filename=default_credential_file, caching=true)
  return {:access_key => @access_key, :secret_access_key => @secret_access_key} if @access_key and @secret_access_key and caching
  return {} if filename.nil? or not File.exists?(filename)
  puts("Reading keys from file: #{filename}")
  File.open(filename).each_line { |line|
    if line =~ /AWSAccessKeyId=([a-zA-Z0-9]+)$/
      @access_key=$1.chomp
    elsif line =~ /AWSSecretKey=([^   ]+)$/
      @secret_access_key=$1.chomp
    end
  }
  return {:access_key => @access_key, :secret_access_key => @secret_access_key}
end

.load_keys_from_file(filename = "#{ENV["HOME"]}/.poolparty/aws", caching = true) ⇒ Object

Load the yaml file containing keys. If the file does not exist, return an empty hash



55
56
57
58
59
60
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 55

def self.load_keys_from_file(filename="#{ENV["HOME"]}/.poolparty/aws", caching=true)
  return @aws_yml if @aws_yml && caching==true
  return {} unless File.exists?(filename)
  puts("Reading keys from file: #{filename}")
  @aws_yml = YAML::load( open(filename).read ) || {}
end

Instance Method Details

#all_nodes ⇒ Object 



318
319
320
321
322
323
324
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 318

def all_nodes
  @nodes ||= describe_instances.select { |i| 
    !(security_group_names & tags(i)).empty? 
  }.sort {|a,b| 
    DateTime.parse(a.launchTime) <=> DateTime.parse(b.launchTime)
  }
end

#as ⇒ Object

Proxy to the raw Grempe amazon-aws autoscaling instance



387
388
389
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 387

def as
  @as = AWS::Autoscaling::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
end

#assign_ebs_volumes ⇒ Object 



431
432
433
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 431

def assign_ebs_volumes
  ebs_volume_groups.each{|ebs_volume_group| ebs_volume_group.attach(nodes)}
end

#assign_elastic_ips ⇒ Object 



268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 268

def assign_elastic_ips
  unless elastic_ips.empty?
    unused_elastic_ip_addresses = ElasticIp.unused_elastic_ips(self).map {|i| i.public_ip }
    used_elastic_ip_addresses = ElasticIp.elastic_ips(self).map {|i| i.public_ip }

    elastic_ip_objects = ElasticIp.unused_elastic_ips(self).select {|ip_obj| elastic_ips.include?(ip_obj.public_ip) }

    assignee_nodes = nodes.select {|n| !ElasticIp.elastic_ips(self).include?(n.public_ip) }

    elastic_ip_objects.each_with_index do |eip, idx|
      # Only get the nodes that do not have elastic ips associated with them
      begin
        if assignee_nodes[idx]
          puts "Assigning elastic ip: #{eip.public_ip} to node: #{assignee_nodes[idx].instance_id}"
          ec2.associate_address(:instance_id => assignee_nodes[idx].instance_id, :public_ip => eip.public_ip)
        end
      rescue Exception => e
        p [:error, e.inspect]
      end
      reset!
    end
  end
end

#autoscale(given_name = cloud.proper_name, o = {}, &block) ⇒ Object 



356
357
358
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 356

def autoscale(given_name=cloud.proper_name, o={}, &block)
  autoscalers << ElasticAutoScaler.new(given_name, sub_opts.merge(o || {}), &block)
end

#autoscalers ⇒ Object 



409
410
411
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 409

def autoscalers
  @autoscalers ||= []
end

#available_rds_instances ⇒ Object 



370
371
372
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 370

def available_rds_instances
  rds_instances.select{|r| r.available? }
end

#awsrds ⇒ Object 



396
397
398
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 396

def awsrds
  @awsrds ||= AWS::RDS::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
end

#block_device_mapping(o = [], given_name = cloud.proper_name) ⇒ Object

Extras!



349
350
351
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 349

def block_device_mapping(o=[], given_name=cloud.proper_name )
  @mappings ||= o
end

#bootstrap_nodes!(tmp_path = nil) ⇒ Object 



242
243
244
245
246
247
248
249
250
251
252
253
254
255
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 242

def bootstrap_nodes!(tmp_path=nil)
  unless security_groups.map {|a| a.authorizes.map {|t| t.from_port.to_i }.flatten }.flatten.include?(22)
    warn "Cloud security_groups are not authorized for ssh. Cannot bootstrap."
    return
  end
  tmp_path ||= cloud.tmp_path
  nodes.each do |node|
    next unless node.in_service?
    node.cloud_provider = self
    node.rsync_dir(tmp_path)
    node.bootstrap_chef!
    node.run_chef!
  end
end

#cleanup_ssh_known_hosts!(nodes_to_cleanup = nodes, even_unavailable = false) ⇒ Object 



292
293
294
295
296
297
298
299
300
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 292

def cleanup_ssh_known_hosts!(nodes_to_cleanup=nodes,
                             even_unavailable=false)
  puts "cleaning up .ssh/known_hosts"
  nodes_to_cleanup.find_all do |node|
    even_unavailable || node.ssh_available?
  end.each do |node|
    node.ssh_cleanup_known_hosts!
  end
end

#configure_nodes!(tmp_path = nil) ⇒ Object 



257
258
259
260
261
262
263
264
265
266
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 257

def configure_nodes!(tmp_path=nil)
  # removed duplicated code (now configure_nodes! invokes
  # node.bootstrap_chef!, while old version did not, but I believe
  # this is harmless)
  bootstrap_nodes!(tmp_path)

  ebs_volume_groups.each do |vol_grp|
    vol_grp.verify_attachments nodes
  end
end

#contract_by(num = 1) ⇒ Object

Raises:

  • (RuntimeError) 


231
232
233
234
235
236
237
238
239
240
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 231

def contract_by(num=1)
  raise RuntimeError, "Contracting instances by #{num} will lower the number of instances below specified minimum" unless nodes.size - num > minimum_instances
  num.times do |i|
    node = nodes[-num]
    id = node.instance_id
    node.ssh_cleanup_known_hosts!
    Ec2Instance.terminate!(:instance_id => id, :cloud => cloud)
  end
  reset!
end

#create! ⇒ Object

Called when the create command is called on the cloud



102
103
104
105
106
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 102

def create!
  [:security_groups, :load_balancers, :rds_instances].each do |type|
    self.send(type).each {|ele| ele.create! }
  end
end

#credential_file(file = nil) ⇒ Object

Read credentials from credential_file if one exists



467
468
469
470
471
472
473
474
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 467

def credential_file(file=nil)
  unless file.nil?
    dsl_options[:credential_file]=file
    dsl_options.merge!(Ec2.load_keys_from_credential_file(file))
  else
    fetch(:credential_file)
  end
end

#decoded_user_data ⇒ Object 



215
216
217
218
219
220
221
222
223
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 215

def decoded_user_data
  if user_data
    if File.file?(user_data)
      open(user_data).read
    else
      user_data
    end
  end
end

#describe_instances(id = nil) ⇒ Object

Describe instances Describe the instances that are available on this cloud with the id given will be returned if not given, details for all instances will be returned



331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 331

def describe_instances(id=nil)
  begin
    @describe_instances = ec2.describe_instances.reservationSet.item.map do |r|
      r.instancesSet.item.map do |i|
        inst_options = i.merge(r.merge(:cloud => cloud)).merge(cloud.cloud_provider.dsl_options)
        Ec2Instance.new(inst_options)
      end
    end.flatten
  rescue AWS::InvalidClientTokenId => e # AWS credentials invalid
    puts "Error contacting AWS: #{e}"
    raise e
  rescue Exception => e
    []
  end
end

#ebs_volume_groups ⇒ Object 



416
417
418
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 416

def ebs_volume_groups
  @ebs_volume_groups ||= []
end

#ebs_volumes(name = nil, &block) ⇒ Object

dsl method for EBS volumes. E.G.:

ebs_volumes do
  volumes "vol-001248ff", "vol-01ff4b85" # use existing volumes, not mandatory
  device "/dev/sdf"
  snapshot_id "snap-602030dd"
  size 200
end



427
428
429
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 427

def ebs_volumes(name=nil, &block)
  ebs_volume_groups << ElasticBlockStoreGroup.new(sub_opts,&block) if block
end

#ec2 ⇒ Object

Proxy to the raw Grempe amazon-aws @ec2 instance



375
376
377
378
379
380
381
382
383
384
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 375

def ec2
  @ec2 ||= begin
   AWS::EC2::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
  rescue AWS::ArgumentError => e # AWS credentials missing?
    puts "Error contacting AWS: #{e}"
    raise e
  rescue Exception => e
    puts "Generic error #{e.class}: #{e}"
  end
end

#elastic_ip(*ips) ⇒ Object 



362
363
364
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 362

def elastic_ip(*ips)
  ips.each {|ip| elastic_ips << ip}
end

#elastic_ips ⇒ Object 



412
413
414
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 412

def elastic_ips
  @elastic_ips ||= []
end

#elb ⇒ Object

Proxy to the raw Grempe amazon-aws elastic_load_balancing instance



392
393
394
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 392

def elb
  @elb ||= AWS::ELB::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
end

#expand_by(num = 1) ⇒ Object 



192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 192

def expand_by(num=1)
  e = Ec2Instance.run!({
    :image_id => image_id,
    :min_count => num,
    :max_count => num,
    :key_name => keypair.basename,
    :security_groups => security_groups,
    :user_data => decoded_user_data,
    :instance_type => instance_type,
    :availability_zone => availability_zones.first,
    :base64_encoded => true,
    :cloud => cloud,
    :block_device_mapping => block_device_mapping,
    :disable_api_termination => disable_api_termination,
    :instance_initiated_shutdown_behavior => instance_initiated_shutdown_behavior,
    :subnet_id => subnet_id,
  })
  progress_bar_until("Waiting for node to launch...") do
    wait_for_node(e)
  end
  all_nodes.detect {|n| n.instance_id == e.instance_id }
end

#list_ec2_volumes(filters = nil) ⇒ Object

Get existing volumes on EC2. filters is a hash of filters, either single valued or multivalued (value is an array of possible values). The function will return volumes matching all filters. A volume is a filter match if any one of the filter values equals the volume parameter value.



446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 446

def list_ec2_volumes(filters=nil)
  @volumes_on_ec2=ec2.describe_volumes.volumeSet.item unless @volumes_on_ec2
  (if filters.nil? # no filter to check, so return at once
    @volumes_on_ec2
  else
    @volumes_on_ec2.select{|vol| # select volumes for which no filter failed
      not filters.map {|filter_key, filter_val|
        filter_key=filter_key.to_s if filter_key.is_a?(Symbol) # filter_key may be given as a symbol
        raise ArgumentError, "Filter key #{filter_key} is invalid" unless vol.has_key?(filter_key)
        if filter_val.is_a?(Array) # Deal with multiple filter values
          filter_val.map{|val| val.is_a?(String) ? val : val.to_s}.member?(vol[filter_key]) # make sure fiter_val array values are Strings before checking for match
        else
          (filter_val.is_a?(String) ? filter_val : filter_val.to_s)==vol[filter_key] # make sure fiter_val is a String before comparing
        end
        }.member?(false) # Check if a filter failed, the 'not' statement at the beginning of the map block negates this so 'select' will choose only when no filter failed
      }.compact # remove nil results from volume set.
    end
    ).map{|vol| ElasticBlockStore.new(vol,:cloud => cloud)}
end

#load_balancer(given_name = cloud.proper_name, o = {}, &block) ⇒ Object 



353
354
355
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 353

def load_balancer(given_name=cloud.proper_name, o={}, &block)
  load_balancers << ElasticLoadBalancer.new(given_name, sub_opts.merge(o || {}), &block)
end

#load_balancers ⇒ Object 



406
407
408
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 406

def load_balancers
  @load_balancers ||= []
end

#nodes ⇒ Object 



302
303
304
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 302

def nodes
  all_nodes.select {|i| i.in_service? }#describe_instances.select {|i| i.in_service? && security_groups.include?(i.security_groups) }
end

#rds(given_name = cloud.proper_name, o = {}, &block) ⇒ Object 



366
367
368
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 366

def rds(given_name=cloud.proper_name, o={}, &block)
  rds_instances << RdsInstance.new(given_name, sub_opts.merge(o || {}), &block)
end

#rds_instances ⇒ Object 



435
436
437
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 435

def rds_instances
  @rds_instances ||= []
end

#reset! ⇒ Object

Clear the cache



440
441
442
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 440

def reset!
  @nodes = @describe_instances = nil
end

#run ⇒ Object 



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 108

def run
  puts "  for cloud: #{cloud.name}"
  puts "  minimum_instances: #{minimum_instances}"
  puts "  maximum_instances: #{maximum_instances}"
  puts "  security_groups: #{security_group_names.join(", ")}"
  puts "  using keypair: #{keypair}"
  puts "  with user_data #{user_data.to_s[0..100]}"
  puts "  user: #{user}\n"

  security_groups.each do |sg|
    sg.run
  end

  unless load_balancers.empty?
    load_balancers.each do |lb|
      puts "    load balancer: #{lb.name}"
      lb.run
    end
  end

  unless rds_instances.empty?
    rds_instances.each do |rdsi|
      puts "    rds instance: #{rdsi.name}"
      rdsi.run
    end
  end

  if autoscalers.empty? # not using autoscaling
    puts "---- live, running instances (#{nodes.size}) ----"
    if nodes.size < minimum_instances
      expansion_count = minimum_instances - nodes.size
      puts "-----> expanding the cloud because the #{expansion_count} minimum_instances is not satisified: "
      expand_by(expansion_count)
    elsif nodes.size > maximum_instances
      contraction_count = nodes.size - maximum_instances
      puts "-----> contracting the cloud because the instances count exceeds the #{maximum_instances} maximum_instances by #{contraction_count}"
      contract_by(contraction_count)
    end
    progress_bar_until("Waiting for the instances to be launched") do
      reset!
      running_nodes = nodes.select {|n| n.running? }
      running_nodes.size >= minimum_instances
    end
    reset!
    # ELASTIC IPS
  else
    autoscalers.each do |a|
      puts "    autoscaler: #{a.name}"
      puts "-----> The autoscaling groups will launch the instances"
      a.run

      progress_bar_until("Waiting for autoscaler to launch instances") do
        reset!
        running_nodes = nodes.select {|n| n.running? }
        running_nodes.size >= minimum_instances
      end
      reset!
    end
  end

  from_ports = security_groups.map {|a| a.authorizes.map {|t| t.from_port.to_i }.flatten }.flatten
  if from_ports.include?(22)
    progress_bar_until("Waiting for the instances to be accessible by ssh") do
      running_nodes = nodes.select {|n| n.running? }
      accessible_count = running_nodes.map do |node|
        node.accessible?
      end.size
      accessible_count == running_nodes.size
    end
  end

  assign_elastic_ips
  cleanup_ssh_known_hosts!
  puts "Attaching EBS volumes"
  assign_ebs_volumes # Assign EBS volumes
end

#security_group(given_name = cloud.proper_name, o = {}, &block) ⇒ Object 



359
360
361
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 359

def security_group(given_name=cloud.proper_name, o={}, &block)
  security_groups << SecurityGroup.new(given_name, sub_opts.merge(o || {}), &block)
end

#security_group_names ⇒ Object 



400
401
402
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 400

def security_group_names
  security_groups.map {|a| a.to_s }
end

#security_groups ⇒ Object 



403
404
405
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 403

def security_groups
  @security_groups ||= []
end

#tags(instance) ⇒ Object

Description

Return all the security groups of the instance that are prefixed with #poolparty.

These are special security groups used only for tagging

Parameters

instance    -   An ec2 instance as returned from describe_instances



314
315
316
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 314

def tags instance
  instance.groupSet.item.collect{|g| g.groupId }.select {|s| s.start_with? "#poolparty"}
end

#teardown ⇒ Object 



185
186
187
188
189
190
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 185

def teardown
  puts "------ Tearing down and cleaning up #{cloud.name} cloud"
  unless autoscalers.empty?
    puts "Tearing down autoscalers"
  end
end

#wait_for_node(instance) ⇒ Object 



225
226
227
228
229
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 225

def wait_for_node(instance)
  reset!
  inst = all_nodes.detect {|n| n.instance_id == instance.instance_id }
  inst.running? if inst
end