Class: OpenStudioAwsWrapper

Inherits:
Object
  • Object
show all
Includes:
Logging
Defined in:
lib/openstudio/lib/openstudio_aws_wrapper.rb

Constant Summary collapse

VALID_OPTIONS = 
[:proxy, :credentials]

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Logging

configure_logger_for, #logger, logger_for

Constructor Details

#initialize(options = {}, group_uuid = nil) ⇒ OpenStudioAwsWrapper

Returns a new instance of OpenStudioAwsWrapper.



55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 55

def initialize(options = {}, group_uuid = nil)
  @group_uuid = group_uuid || (SecureRandom.uuid).gsub('-', '')

  @security_group_name = nil
  @key_pair_name = nil
  @private_key_file_name = nil

  @private_key = nil # Private key data

  # List of instances
  @server = nil
  @workers = []

  # store an instance variable with the proxy for passing to instances for use in scp/ssh
  @proxy = options[:proxy] ? options[:proxy] : nil

  # need to remove the prxoy information here
  @aws = Aws::EC2::Client.new(options[:credentials])
end

Instance Attribute Details

#group_uuid ⇒ Object (readonly)

Returns the value of attribute group_uuid.



44
45
46
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 44

def group_uuid
  @group_uuid
end

#key_pair_name ⇒ Object (readonly)

Returns the value of attribute key_pair_name.



46
47
48
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 46

def key_pair_name
  @key_pair_name
end

#private_key_file_name ⇒ Object

Returns the value of attribute private_key_file_name.



51
52
53
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 51

def private_key_file_name
  @private_key_file_name
end

#proxy ⇒ Object (readonly)

Returns the value of attribute proxy.



49
50
51
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 49

def proxy
  @proxy
end

#security_group_name ⇒ Object (readonly)

Returns the value of attribute security_group_name.



45
46
47
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 45

def security_group_name
  @security_group_name
end

#server ⇒ Object (readonly)

Returns the value of attribute server.



47
48
49
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 47

def server
  @server
end

#workers ⇒ Object (readonly)

Returns the value of attribute workers.



48
49
50
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 48

def workers
  @workers
end

Instance Method Details

#configure_server_and_workers ⇒ Object

blocking method that waits for servers and workers to be fully configured (i.e. execution of user_data has occured on all nodes)



323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 323

def configure_server_and_workers
  logger.info('waiting for server user_data to complete')
  @server.wait_command('[ -e /home/ubuntu/user_data_done ] && echo "true"')
  logger.info('waiting for worker user_data to complete')
  @workers.each { |worker| worker.wait_command('[ -e /home/ubuntu/user_data_done ] && echo "true"') }

  ips = "master|#{@server.data.ip}|#{@server.data.dns}|#{@server.data.procs}|ubuntu|ubuntu|true\n"
  @workers.each { |worker| ips << "worker|#{worker.data.ip}|#{worker.data.dns}|#{worker.data.procs}|ubuntu|ubuntu|true\n" }
  file = Tempfile.new('ip_addresses')
  file.write(ips)
  file.close
  @server.upload_file(file.path, 'ip_addresses')
  file.unlink
  logger.info("ips #{ips}")
  @server.shell_command('chmod 664 /home/ubuntu/ip_addresses')
  @server.shell_command('~/setup-ssh-keys.sh')
  @server.shell_command('~/setup-ssh-worker-nodes.sh ip_addresses')

  mongoid = File.read(File.expand_path(File.dirname(__FILE__)) + '/mongoid.yml.template')
  mongoid.gsub!(/SERVER_IP/, @server.data.ip)
  file = Tempfile.new('mongoid.yml')
  file.write(mongoid)
  file.close
  @server.upload_file(file.path, '/mnt/openstudio/rails-models/mongoid.yml')
  @workers.each { |worker| worker.upload_file(file.path, '/mnt/openstudio/rails-models/mongoid.yml') }
  file.unlink

  @server.shell_command('chmod 664 /mnt/openstudio/rails-models/mongoid.yml')
  @workers.each { |worker| worker.shell_command('chmod 664 /mnt/openstudio/rails-models/mongoid.yml') }

  sleep 60 # wait 60 more seconds for everything -- this is cheesy
  true
end

#create_new_ami_json(version = 1) ⇒ Object

method to hit the existing list of available amis and compare to the list of AMIs on Amazon and then generate the new ami list



388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 388

def create_new_ami_json(version = 1)
  # get the list of existing amis from developer.nrel.gov
  existing_amis = OpenStudioAmis.new(1).list

  # list of available AMIs from AWS
  available_amis = describe_amis

  amis = transform_ami_lists(existing_amis, available_amis)

  if version == 1
    puts 'Creating version 1 of the AMI file'
    version1 = {}

    # now grab the good keys - they should be sorted newest to older... so go backwards
    amis[:openstudio_server].keys.reverse.each do |key|
      next if amis[:openstudio_server][key][:deprecate]

      a = amis[:openstudio_server][key]
      # this will override any of the old ami/os version
      version1[a[:openstudio_version].to_sym] = a[:amis]
    end

    # create the default version. First sort, then grab the first hash's values

    version1.sort_by
    default_v = nil
    version1 = Hash[version1.sort_by { |k, _| k.to_s.to_version }.reverse]
    default_v = version1.keys[0]

    version1[:default] = version1[default_v]
    puts "Pretty version 1: #{JSON.pretty_generate(version1)}"

    amis = version1
  elsif version == 2
    # don't need to transform anything right now
    puts "Pretty version 2: #{JSON.pretty_generate(amis)}"
  end

  amis
end

#create_or_retrieve_key_pair(key_pair_name = nil) ⇒ Object 



231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 231

def create_or_retrieve_key_pair(key_pair_name = nil)
  tmp_name = key_pair_name || "os-key-pair-#{@group_uuid}"

  # the describe_key_pairs method will raise an expectation if it can't find the key pair, so catch it
  resp = nil
  begin
    resp = @aws.describe_key_pairs(key_names: [tmp_name]).data
    fail 'looks like there are 2 key pairs with the same name' if resp.key_pairs.size >= 2
  rescue
    logger.info "could not find key pair '#{tmp_name}'"
  end

  if resp.nil? || resp.key_pairs.size == 0
    # create the new key_pair
    # check if the key pair name exists
    # create a new key pair everytime
    keypair = @aws.create_key_pair(key_name: tmp_name)

    # save the private key to memory (which can later be persisted via the save_private_key method)
    @private_key = keypair.data.key_material
    @key_pair_name = keypair.data.key_name
  else
    logger.info "found existing keypair #{resp.key_pairs.first}"
    @key_pair_name = resp.key_pairs.first[:key_name]

    # This will not set the private key because it doesn't live on the remote system
  end

  logger.info("create key pair: #{@key_pair_name}")
end

#create_or_retrieve_security_group(sg_name = nil) ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 75

def create_or_retrieve_security_group(sg_name = nil)
  tmp_name = sg_name || 'openstudio-server-sg-v1'
  group = @aws.describe_security_groups(filters: [{ name: 'group-name', values: [tmp_name] }])
  logger.info "Length of the security group is: #{group.data.security_groups.length}"
  if group.data.security_groups.length == 0
    logger.info 'server group not found --- will create a new one'
    @aws.create_security_group(group_name: tmp_name, description: "group dynamically created by #{__FILE__}")
    @aws.authorize_security_group_ingress(
        group_name: tmp_name,
        ip_permissions: [
          { ip_protocol: 'tcp', from_port: 1, to_port: 65_535, ip_ranges: [cidr_ip: '0.0.0.0/0'] }
        ]
    )
    @aws.authorize_security_group_ingress(
        group_name: tmp_name,
        ip_permissions: [
          { ip_protocol: 'icmp', from_port: -1, to_port: -1, ip_ranges: [cidr_ip: '0.0.0.0/0']
          }
        ]
    )

    # reload group information
    group = @aws.describe_security_groups(filters: [{ name: 'group-name', values: [tmp_name] }])
  end
  @security_group_name = group.data.security_groups.first.group_name
  logger.info("server_group #{group.data.security_groups.first.group_name}")
end

#describe_amis(filter = nil, image_ids = [], owned_by_me = true) ⇒ Object 



173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 173

def describe_amis(filter = nil, image_ids = [], owned_by_me = true)
  resp = nil

  # todo: test the filter.  i don't think that it is exposed in the AWS gem?
  if owned_by_me
    if filter
      resp = @aws.describe_images(owners: [:self], filter: filter).data
    else
      resp = @aws.describe_images(owners: [:self]).data
    end
  else
    if filter
      resp = @aws.describe_images(filter: filter).data
    else
      resp = @aws.describe_images(image_ids: image_ids).data
    end
  end

  resp = resp.to_hash

  # map the tags to hashes
  resp[:images].each do |image|
    image[:tags_hash] = {}
    image[:tags_hash][:tags] = []
    image[:tags].each do |tag|
      if tag[:value]
        image[:tags_hash][tag[:key].to_sym] = tag[:value]
      else
        image[:tags_hash][:tags] << tag[:key]
      end
    end
  end

  resp
end

#describe_availability_zones ⇒ Object 



103
104
105
106
107
108
109
110
111
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 103

def describe_availability_zones
  resp = @aws.describe_availability_zones
  map = []
  resp.data.availability_zones.each do |zn|
    map << zn.to_hash
  end

  { availability_zone_info: map }
end

#describe_availability_zones_json ⇒ Object 



113
114
115
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 113

def describe_availability_zones_json
  describe_availability_zones.to_json
end

#describe_running_instances(group_uuid = nil, openstudio_instance_type = nil) ⇒ Object

return all of the running instances, or filter by the group_uuid & instance type



129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 129

def describe_running_instances(group_uuid = nil, openstudio_instance_type = nil)
  resp = nil
  if group_uuid
    resp = @aws.describe_instances(
        filters: [
          { name: 'instance-state-code', values: [0.to_s, 16.to_s] }, # running or pending
          { name: 'tag-key', values: ['GroupUUID'] },
          { name: 'tag-value', values: [group_uuid.to_s] }
        ]
    )
  else
    resp = @aws.describe_instances
  end

  instance_data = nil
  if resp
    instance_data = []
    resp.reservations.each do |r|
      r.instances.each do |i|
        i_h = i.to_hash
        if group_uuid && openstudio_instance_type
          # {:key=>"Purpose", :value=>"OpenStudioWorker"}
          if i_h[:tags].any? { |h| (h[:key] == 'Purpose') && (h[:value] == "OpenStudio#{openstudio_instance_type.capitalize}") } &&
                    i_h[:tags].any? { |h|(h[:key] == 'GroupUUID') && (h[:value] == group_uuid.to_s) }
            instance_data << i_h
          end
        elsif group_uuid
          if i_h[:tags].any? { |h|(h[:key] == 'GroupUUID') && (h[:value] == group_uuid.to_s) }
            instance_data << i_h
          end
        elsif openstudio_instance_type
          if i_h[:tags].any? { |h| (h[:key] == 'Purpose') && (h[:value] == "OpenStudio#{openstudio_instance_type.capitalize}") }
            instance_data << i_h
          end
        else
          instance_data << i_h
        end
      end
    end
  end

  instance_data
end

#describe_total_instances ⇒ Object 



117
118
119
120
121
122
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 117

def describe_total_instances
  resp = @aws.describe_instance_status

  region = resp.instance_statuses.length > 0 ? resp.instance_statuses.first.availability_zone : 'no_instances'
  { total_instances: resp.instance_statuses.length, region: region }
end

#describe_total_instances_json ⇒ Object 



124
125
126
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 124

def describe_total_instances_json
  describe_total_instances.to_json
end

#find_server(server_data_hash) ⇒ Object

method to query the amazon api to find the server (if it exists), based on the group id if it is found, then it will set the @server member variable. Note that the information around keys and security groups is pulled from the instance information.



360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 360

def find_server(server_data_hash)
  group_uuid = server_data_hash[:group_id] || @group_uuid
  load_private_key(server_data_hash[:server][:private_key_file_name])

  logger.info "finding the server for groupid of #{group_uuid}"
  fail 'no group uuid defined either in member variable or method argument' if group_uuid.nil?

  resp = describe_running_instances(group_uuid, :server)
  if resp
    fail "more than one server running with group uuid of #{group_uuid} found, expecting only one" if resp.size > 1
    resp = resp.first
    if !@server
      logger.info "Server found and loading data into object [instance id is #{resp[:instance_id]}]"
      @server = OpenStudioAwsInstance.new(@aws, :server, resp[:key_name], resp[:security_groups].first[:group_name], group_uuid, @private_key, @private_key_file_name)
      @server.load_instance_data(resp)
    else
      logger.info "Server instance is already defined with instance #{resp[:instance_id]}"
    end
  else
    fail 'could not find a running server instance'
  end

  # Really don't need to return anything because this sets the class instance variable
  @server
end

#get_next_version(base, list_of_svs) ⇒ Object

take the base version and increment the patch until



448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 448

def get_next_version(base, list_of_svs)
  b = base.to_version

  # first go through the array and test that there isn't any other versions > that in the array
  list_of_svs.each do |v|
    b = v.to_version if v.to_version.satisfies("#{b.major}.#{b.minor}.*") && v.to_version.patch > b.patch
  end

  # get the max version in the list_of_svs
  while list_of_svs.include?(b.to_s)
    b.patch += 1
  end

  # return the value back as a string
  b.to_s
end

#launch_server(image_id, instance_type, options = {}) ⇒ Object 



278
279
280
281
282
283
284
285
286
287
288
289
290
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 278

def launch_server(image_id, instance_type, options = {})
  defaults = { user_id: 'unknown_user' }
  options = defaults.merge(options)

  user_data = File.read(File.expand_path(File.dirname(__FILE__)) + '/server_script.sh')
  @server = OpenStudioAwsInstance.new(@aws, :server, @key_pair_name, @security_group_name, @group_uuid, @private_key, @private_key_file_name, @proxy)

  # create the EBS volumes instead of the ephemeral storage - needed especially for the m3 instances (SSD)

  fail 'image_id is nil' unless image_id
  fail 'instance type is nil' unless instance_type
  @server.launch_instance(image_id, instance_type, user_data, options[:user_id], options[:ebs_volume_size])
end

#launch_workers(image_id, instance_type, num, options = {}) ⇒ Object 



292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 292

def launch_workers(image_id, instance_type, num, options = {})
  defaults = { user_id: 'unknown_user' }
  options = defaults.merge(options)

  user_data = File.read(File.expand_path(File.dirname(__FILE__)) + '/worker_script.sh.template')
  user_data.gsub!(/SERVER_IP/, @server.data.ip)
  user_data.gsub!(/SERVER_HOSTNAME/, 'master')
  user_data.gsub!(/SERVER_ALIAS/, '')
  logger.info("worker user_data #{user_data.inspect}")

  # thread the launching of the workers

  num.times do
    @workers << OpenStudioAwsInstance.new(@aws, :worker, @key_pair_name, @security_group_name, @group_uuid, @private_key, @private_key_file_name, @proxy)
  end

  threads = []
  @workers.each do |worker|
    threads << Thread.new do
      # create the EBS volumes instead of the ephemeral storage - needed especially for the m3 instances (SSD)
      worker.launch_instance(image_id, instance_type, user_data, options[:user_id], options[:ebs_volume_size])
    end
  end
  threads.each { |t| t.join }

  # todo: do we need to have a flag if the worker node is successful?
  # todo: do we need to check the current list of running workers?
end

#load_private_key(filename) ⇒ Object 



262
263
264
265
266
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 262

def load_private_key(filename)
  fail "Could not find private key #{filename}" unless File.exist? filename
  @private_key_file_name = File.expand_path filename
  @private_key = File.read(filename)
end

#save_private_key(filename) ⇒ Object 



268
269
270
271
272
273
274
275
276
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 268

def save_private_key(filename)
  if @private_key
    @private_key_file_name = File.expand_path filename
    File.open(filename, 'w') { |f| f << @private_key }
    File.chmod(0600, filename)
  else
    fail "No private key found in which to persist with filename #{filename}"
  end
end

#stop_instances(ids) ⇒ Object 



209
210
211
212
213
214
215
216
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 209

def stop_instances(ids)
  resp = @aws.stop_instances(
      instance_ids: ids,
      force: true
  )

  resp
end

#terminate_instances(ids) ⇒ Object 



218
219
220
221
222
223
224
225
226
227
228
229
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 218

def terminate_instances(ids)
  begin
    resp = @aws.terminate_instances(
        instance_ids: ids,
    )
  rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
    # Log that the instances couldn't be found?
    return resp = {error: 'instances could not be found'}
  end
    
  resp
end

#to_os_worker_hash ⇒ Object 



429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
 
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 429

def to_os_worker_hash
  worker_hash = []
  @workers.each do |worker|
    worker_hash.push(
        id: worker.data.id,
        ip: "http://#{worker.data.ip}",
        dns: worker.data.dns,
        procs: worker.data.procs,
        private_key_file_name: worker.private_key_file_name
    )
  end

  out = { workers: worker_hash }
  logger.info out

  out
end