Class: Cloudmaster::InstancePool

Inherits:
Object
  • Object
show all
Includes:
Enumerable
Defined in:
app/instance_pool.rb

Overview

Stores and operates on a collection of instances

Internally, instances are stored as an array of Instance objects.

Instance Method Summary collapse

Constructor Details

#initialize(reporter, config) ⇒ InstancePool

Create an instance pool.

This class knows how to start and stop instances, and how to detect that new instance have come about, or existing ones have gone away. The constructor takes:

[config] describes configurable instance properties



19
20
21
22
23
24
25
 
# File 'app/instance_pool.rb', line 19

def initialize(reporter, config)
  @ec2 = AwsContext.instance.ec2
  @reporter = reporter
  @config = config
  @state_change_time = Clock.now
  @instances = []         # holds Instance objects
end

Instance Method Details

#above_maximum_count ⇒ Object

Return count of instance above maximum



114
115
116
 
# File 'app/instance_pool.rb', line 114

def above_maximum_count
  greater_than_maximum? ? size - maximum : 0
end

#active_idle_instances ⇒ Object

Return instances that are active and have load <= target_load



152
153
154
155
 
# File 'app/instance_pool.rb', line 152

def active_idle_instances
  target_load = 0
  active_instances.find_all {|i| i.load_estimate <= target_load}
end

#active_instances ⇒ Object

Return all instances in active state



142
143
144
 
# File 'app/instance_pool.rb', line 142

def active_instances
  find_all {|i| i.state == :active}
end

#active_set ⇒ Object

Return a YAML encoded representation of the active set. The active set describes the id, public DNS, and load average of each active instance in the pool.



217
218
219
220
221
222
223
224
 
# File 'app/instance_pool.rb', line 217

def active_set
  message = active_instances.collect do |instance|
    { :id => instance.id, 
      :public_dns => instance.public_dns, 
      :load_estimate => instance.load_estimate }
  end
  YAML.dump(message)
end

#add(id, public_dns) ⇒ Object

Create an instance and add to the list. Return the newly created instance.



67
68
69
70
71
 
# File 'app/instance_pool.rb', line 67

def add(id, public_dns)
  new_instance =  Instance.new(id, public_dns, @config)
  @instances << new_instance
  new_instance
end

#audit_existing_instances ⇒ Object

Audit the list of instances based on what is currently known to EC2. In other words, bring our list of instances into agreement with the instances EC2 knows about by (1) adding instances that EC2 knows but that we do not, and (2) deleting instance that EC2 no longer knows about. This is used initially to build the instance list, and periodically thereafter to catch instances started or stopped outside cloudmaster.



272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
 
# File 'app/instance_pool.rb', line 272

def audit_existing_instances
  running_instances = our_running_instances
  
  # add running instances that we don't have
  running_instances.each do |running|
    if ! find_by_id(running[:id])
      add(running[:id], running[:public_dns])
      @reporter.info("Instance discovered #{running[:public_dns]}", running[:id])
    end
  end
  # delete instances that are no longer running
  each do |instance|
    if ! running_instances.find {|running| running[:id] == instance.id}
      delete(instance)
      @reporter.info("Instance disappeared #{instance.public_dns}", instance.id)
    end
  end
end

#below_minimum_count ⇒ Object

Return count of instance below minimum



109
110
111
 
# File 'app/instance_pool.rb', line 109

def below_minimum_count
  less_than_minimum? ? minimum - size : 0
end

#delete(instance) ⇒ Object

Delete the instance from the list



74
75
76
 
# File 'app/instance_pool.rb', line 74

def delete(instance)
  @instances.delete(instance)
end

#each ⇒ Object

Allows iteration through instances. So enumeration on InstancePool is implicitly enumeration

on @instances.



51
52
53
 
# File 'app/instance_pool.rb', line 51

def each
  @instances.each {|i| yield i}
end

#excess_capacity ⇒ Object

Return the sum of all the extra capacity of active instances that have excess capacity (load less than target load).



182
183
184
185
186
187
 
# File 'app/instance_pool.rb', line 182

def excess_capacity
  target_load = @config[:target_upper_load].to_f
  active_instances.inject(0) do |sum, instance|
    sum + max2(target_load - instance.load_estimate, 0)
  end
end

#find_by_id(id) ⇒ Object

Find an instance given its instance id.



79
80
81
 
# File 'app/instance_pool.rb', line 79

def find_by_id(id)
  find {|i| i.id == id}
end

#first ⇒ Object

return first instance



56
57
58
 
# File 'app/instance_pool.rb', line 56

def first
  @instances.first
end

#greater_than_maximum? ⇒ Boolean

return true if number of instances is more than maximum

Returns:

  • (Boolean) 


104
105
106
 
# File 'app/instance_pool.rb', line 104

def greater_than_maximum?
  size > maximum
end

#hung_instances ⇒ Object

Return instances that have not seen status in watchdog_interval



137
138
139
 
# File 'app/instance_pool.rb', line 137

def hung_instances
  find_all {|i| i.watchdog_time_elapsed?}
end

#id_list ⇒ Object

Return a list of all instance ids.



84
85
86
 
# File 'app/instance_pool.rb', line 84

def id_list
  map {|i| i.id}
end

#less_than_minimum? ⇒ Boolean

Return true if the number of instances is less than the minimum.

Returns:

  • (Boolean) 


99
100
101
 
# File 'app/instance_pool.rb', line 99

def less_than_minimum?
  size < minimum
end

#max2(a, b) ⇒ Object 



44
45
46
 
# File 'app/instance_pool.rb', line 44

def max2(a, b)
  a > b ? a : b
end

#maximum ⇒ Object

Return the maximum number of instances allowed.



89
90
91
 
# File 'app/instance_pool.rb', line 89

def maximum
  @config[:maximum_number_of_instances].to_i
end

#minimum ⇒ Object

Return the minimum number of instances allowed.



94
95
96
 
# File 'app/instance_pool.rb', line 94

def minimum
  @config[:minimum_number_of_instances].to_i
end

#missing_public_dns_ids ⇒ Object

Return ids of all instances missing a public_dns.



124
125
126
 
# File 'app/instance_pool.rb', line 124

def missing_public_dns_ids
  missing_public_dns_instances.collect {|i| i.id}
end

#missing_public_dns_instances ⇒ Object

Return a list of instances missing public_dns.



119
120
121
 
# File 'app/instance_pool.rb', line 119

def missing_public_dns_instances
  find_all {|i| i.public_dns.nil? || i.public_dns.empty? }
end

#our_running_instances ⇒ Object

Return instances that match our ami_id that are either pending or running. These instances are as returned by ec2: containing fields such as id and :public_dns.



257
258
259
260
261
262
 
# File 'app/instance_pool.rb', line 257

def our_running_instances
  EC2InstanceEnumerator.new.find_all do |instance|
    instance[:image_id] == @config[:ami_id] && 
      %w[pending running].include?(instance[:state])
  end
end

#over_capacity ⇒ Object

Return the sum of capacity in excess of the target upper load



190
191
192
193
194
195
 
# File 'app/instance_pool.rb', line 190

def over_capacity
  target_load = @config[:target_upper_load].to_f
  active_instances.inject(0) do |sum, instance|
    sum + max2(instance.load_estimate - target_load, 0)
  end
end

#shut_down(instances_to_shut_down) ⇒ Object

Shut down the given set of instances. Set the state to shut_down Return an array of shut down instances.



319
320
321
322
323
324
 
# File 'app/instance_pool.rb', line 319

def shut_down(instances_to_shut_down)
  instances_to_shut_down.collect do |instance|
    instance.shutdown
    instance
  end
end

#shut_down_idle_instances ⇒ Object

Shut down all instances who have a load below the target. Shut down is not the same as stop – the instances continue to provide service, but are no longer allocated new clients.



160
161
162
163
 
# File 'app/instance_pool.rb', line 160

def shut_down_idle_instances
  target_load = @config[:shut_down_threshold].to_i
  shut_down_instances.find_all {|i| i.load_estimate <= target_load}
end

#shut_down_instances ⇒ Object

Return all instances in shut_down state



147
148
149
 
# File 'app/instance_pool.rb', line 147

def shut_down_instances
  find_all {|i| i.state == :shut_down}
end

#shut_down_timeout_instances ⇒ Object

Return instances that are shut_down and have time_since_state_change > shut_down_interval.



167
168
169
170
171
 
# File 'app/instance_pool.rb', line 167

def shut_down_timeout_instances
  shut_down_interval = @config[:shut_down_interval].to_i * 60

  shut_down_instances.find_all {|i| i.time_since_state_change > shut_down_interval}
end

#size ⇒ Object

Return the number of instances in the pool.



61
62
63
 
# File 'app/instance_pool.rb', line 61

def size
  @instances.size
end

#sorted_by_lowest_load ⇒ Object

Return all the instance, sortd by lowest load estimate.



227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'app/instance_pool.rb', line 227

def sorted_by_lowest_load
  @instances.sort do |a,b|
    # Compare the elapsed lifetime status. If the status differs, instances
    # that have lived beyond the minimum lifetime will be sorted earlier.
    if a.minimum_lifetime_elapsed? != b.minimum_lifetime_elapsed?
      if a.minimum_lifetime_elapsed?
        -1   # This instance has lived long enough, the other hasn't
      else
        1    # The other instance has lived long enough, this one hasn't
      end
    else
      a.load_estimate - b.load_estimate
    end
  end
end

#start_n_instances(number_to_start) ⇒ Object

Start the given number of instances. Remember started instances by creating an Instance object and storing it in the pool. Return an array of the ones we just started.



295
296
297
298
299
300
301
302
303
 
# File 'app/instance_pool.rb', line 295

def start_n_instances(number_to_start)
  return [] if number_to_start <= 0 
  started_instances = @ec2.run_instances(@config[:ami_id], 1, 
                           number_to_start, start_opts)[:instances]
  started_instances.collect do |started_instance|
    # the public dns is not available yet
    add(started_instance[:id], nil)
  end
end

#start_opts ⇒ Object

Create and return options, in a way that is acceptable to EC2.



30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'app/instance_pool.rb', line 30

def start_opts
  groups = @config[:security_groups]
  # this can throw an exception if groups is not formetteed properly
  begin
    groups = eval(groups) if groups.kind_of?(String)
  rescue
    groups = [:default]
  end
  {:key_name => @config[:key_pair_name], 
      :user_data => YAML.dump(@config[:user_data]), 
      :security_groups => groups, 
      :instance_type => @config[:instance_type]}
end

#state_change_time ⇒ Object

Return the latest time since any state change of any instance.



174
175
176
177
178
 
# File 'app/instance_pool.rb', line 174

def state_change_time
  @state_change_time = inject(@state_change_time) do |latest, instance|
    max2(latest, instance.state_change_time)
  end
end

#stop_instances(instances_to_stop) ⇒ Object

Stop the given set of instances. Remove stopped instance from the pool. Return an array of stopped instances.



308
309
310
311
312
313
314
 
# File 'app/instance_pool.rb', line 308

def stop_instances(instances_to_stop)
  instances_to_stop.collect do |instance|
    @ec2.terminate_instances(instance.id.to_s)
    delete(instance)
    instance
  end
end

#total_load ⇒ Object

Return the total load of all active instances



198
199
200
201
202
 
# File 'app/instance_pool.rb', line 198

def total_load
  active_instances.inject(0) do |sum, instance|
    sum + instance.load_estimate
  end
end

#update_public_dns(id, public_dns) ⇒ Object

Find the instance identified by id and update its public_dns If there is no dns information, then skip it.



130
131
132
133
134
 
# File 'app/instance_pool.rb', line 130

def update_public_dns(id, public_dns)
  return if public_dns.nil? || public_dns.empty?
  i = find_by_id(id)
  i.public_dns = public_dns if i
end

#update_public_dns_all ⇒ Object

Find all instances for which we don’t have a public_dns, For each one,see of EC2 now has the public DNS. If so, store it.



245
246
247
248
249
250
251
 
# File 'app/instance_pool.rb', line 245

def update_public_dns_all
  missing_ids = missing_public_dns_ids
  return if missing_ids.size == 0
  EC2InstanceEnumerator.new(missing_ids).each do |instance|
    update_public_dns(instance[:id], instance[:public_dns])
  end
end

#update_status(msg) ⇒ Object

Update the status of an instance using the contents of the status message.



205
206
207
208
209
210
211
212
 
# File 'app/instance_pool.rb', line 205

def update_status(msg)
  id = msg[:instance_id]
  if instance = find_by_id(id)
    instance.update_status(msg)
  else
    @reporter.error("Received status message from unknown instance: #{id}") unless id == 'unknown'
  end
end