Class: OpenStudioAwsWrapper
- Inherits:
-
Object
- Object
- OpenStudioAwsWrapper
- Includes:
- Logging
- Defined in:
- lib/openstudio/lib/openstudio_aws_wrapper.rb
Constant Summary collapse
- VALID_OPTIONS =
[:proxy, :credentials]
Instance Attribute Summary collapse
-
#group_uuid ⇒ Object
readonly
Returns the value of attribute group_uuid.
-
#key_pair_name ⇒ Object
readonly
Returns the value of attribute key_pair_name.
-
#private_key_file_name ⇒ Object
Returns the value of attribute private_key_file_name.
-
#proxy ⇒ Object
readonly
Returns the value of attribute proxy.
-
#security_groups ⇒ Object
Returns the value of attribute security_groups.
-
#server ⇒ Object
readonly
Returns the value of attribute server.
-
#worker_keys ⇒ Object
readonly
Returns the value of attribute worker_keys.
-
#workers ⇒ Object
readonly
Returns the value of attribute workers.
Instance Method Summary collapse
-
#configure_server_and_workers ⇒ Boolean
blocking method that waits for servers and workers to be fully configured (i.e. execution of user_data has occured on all nodes).
-
#configure_swarm_cluster(save_directory) ⇒ Object
blocking method that executes required commands for creating and provisioning a docker swarm cluster.
-
#create_new_ami_json(version = 1) ⇒ Object
method to hit the existing list of available amis and compare to the list of AMIs on Amazon and then generate the new ami list.
- #create_or_retrieve_default_security_group(tmp_name = 'openstudio-server-sg-v2.2', vpc_id = nil) ⇒ Object
- #create_or_retrieve_key_pair(key_pair_name = nil) ⇒ Object
-
#delete_key_pair(key_pair_name = nil) ⇒ Object
Delete the key pair from aws.
- #describe_all_instances ⇒ Object
-
#describe_amis(image_ids = [], owned_by_me = true) ⇒ Hash
Describe the list of AMIs adn return the hash.
- #describe_availability_zones ⇒ Object
- #describe_availability_zones_json ⇒ Object
-
#describe_instances ⇒ Object
describe the instances by group id (this is the default method).
-
#describe_running_instances(group_uuid = nil, openstudio_instance_type = nil) ⇒ Object
return all of the running instances, or filter by the group_uuid & instance type.
-
#find_server(server_data_hash) ⇒ Object
method to query the amazon api to find the server (if it exists), based on the group id if it is found, then it will set the @server instance variable.
-
#get_next_version(base, list_of_svs) ⇒ Object
take the base version and increment the patch until.
-
#initialize(options = {}, group_uuid = nil) ⇒ OpenStudioAwsWrapper
constructor
A new instance of OpenStudioAwsWrapper.
- #launch_server(image_id, instance_type, launch_options = {}) ⇒ Object
- #launch_workers(image_id, instance_type, num, launch_options = {}) ⇒ Object
- #load_private_key(filename) ⇒ Object
-
#load_worker_key(private_key_filename) ⇒ Object
Load the worker key for communicating between the server and worker instances on AWS.
-
#save_private_key(directory = '.', filename = 'ec2_server_key.pem') ⇒ Object
Save the private key to disk.
-
#save_worker_keys(directory = '.') ⇒ Object
save off the worker public/private keys that were created.
-
#stop_instances(ids) ⇒ Object
Stop specific instances based on the instance_ids.
- #terminate_instances(ids) ⇒ Object
-
#to_os_hash ⇒ Object
save off the instance configuration and instance information into a JSON file for later use.
- #total_instances_count ⇒ Object
Methods included from Logging
configure_logger_for, #logger, logger_for
Constructor Details
#initialize(options = {}, group_uuid = nil) ⇒ OpenStudioAwsWrapper
Returns a new instance of OpenStudioAwsWrapper.
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 53 def initialize( = {}, group_uuid = nil) @group_uuid = group_uuid || (SecureRandom.uuid).delete('-') @security_groups = [] @key_pair_name = nil @private_key_file_name = nil @region = [:region] || 'unknown-region' # If the keys exist in the directory then load those, otherwise create new ones. @work_dir = File. [:save_directory] if File.exist?(File.join(@work_dir, 'ec2_worker_key.pem')) && File.exist?(File.join(@work_dir, 'ec2_worker_key.pub')) logger.info "Worker keys already exist, loading from #{@work_dir}" load_worker_key(File.join(@work_dir, 'ec2_worker_key.pem')) else logger.info 'Generating new worker keys' @worker_keys = SSHKey.generate end @private_key = nil # Private key data # List of instances @server = nil @workers = [] # store an instance variable with the proxy for passing to instances for use in scp/ssh @proxy = [:proxy] ? [:proxy] : nil # need to remove the prxoy information here @aws = Aws::EC2::Client.new([:credentials]) end |
Instance Attribute Details
#group_uuid ⇒ Object (readonly)
Returns the value of attribute group_uuid.
41 42 43 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 41 def group_uuid @group_uuid end |
#key_pair_name ⇒ Object (readonly)
Returns the value of attribute key_pair_name.
42 43 44 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 42 def key_pair_name @key_pair_name end |
#private_key_file_name ⇒ Object
Returns the value of attribute private_key_file_name.
48 49 50 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 48 def private_key_file_name @private_key_file_name end |
#proxy ⇒ Object (readonly)
Returns the value of attribute proxy.
45 46 47 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 45 def proxy @proxy end |
#security_groups ⇒ Object
Returns the value of attribute security_groups.
49 50 51 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 49 def security_groups @security_groups end |
#server ⇒ Object (readonly)
Returns the value of attribute server.
43 44 45 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 43 def server @server end |
#worker_keys ⇒ Object (readonly)
Returns the value of attribute worker_keys.
46 47 48 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 46 def worker_keys @worker_keys end |
#workers ⇒ Object (readonly)
Returns the value of attribute workers.
44 45 46 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 44 def workers @workers end |
Instance Method Details
#configure_server_and_workers ⇒ Boolean
blocking method that waits for servers and workers to be fully configured (i.e. execution of user_data has occured on all nodes). Ideally none of these methods would ever need to exist.
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 447 def configure_server_and_workers logger.info('waiting for server user_data to complete') @server.wait_command('[ -e /home/ubuntu/user_data_done ] && echo "true"') logger.info('waiting for worker user_data to complete') @workers.each { |worker| worker.wait_command('[ -e /home/ubuntu/user_data_done ] && echo "true"') } ips = "master|#{@server.data.private_ip_address}|#{@server.data.dns}|#{@server.data.procs}|ubuntu|ubuntu|true\n" @workers.each { |worker| ips << "worker|#{worker.data.private_ip_address}|#{worker.data.dns}|#{worker.data.procs}|ubuntu|ubuntu|true\n" } file = Tempfile.new('ip_addresses') file.write(ips) file.close @server.upload_file(file.path, 'ip_addresses') file.unlink logger.info("ips #{ips}") @server.shell_command('chmod 664 /home/ubuntu/ip_addresses') mongoid = File.read(File.(File.dirname(__FILE__)) + '/mongoid.yml.template') mongoid.gsub!(/SERVER_IP/, @server.data.private_ip_address) file = Tempfile.new('mongoid.yml') file.write(mongoid) file.close @server.upload_file(file.path, '/mnt/openstudio/rails-models/mongoid.yml') @workers.each { |worker| worker.upload_file(file.path, '/mnt/openstudio/rails-models/mongoid.yml') } file.unlink @server.shell_command('chmod 664 /mnt/openstudio/rails-models/mongoid.yml') @workers.each { |worker| worker.shell_command('chmod 664 /mnt/openstudio/rails-models/mongoid.yml') } true end |
#configure_swarm_cluster(save_directory) ⇒ Object
blocking method that executes required commands for creating and provisioning a docker swarm cluster
479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 479 def configure_swarm_cluster(save_directory) logger.info('waiting for server user_data to complete') @server.wait_command('while ! [ -e /home/ubuntu/user_data_done ]; do sleep 5; done && echo "true"') logger.info('Running the configuration script for the server.') @server.wait_command('echo $(env) &> /home/ubuntu/env.log && echo "true"') @server.wait_command('cp /home/ubuntu/server_provision.sh /home/ubuntu/server_provision.sh.bak && echo "true"') @server.wait_command('sudo /home/ubuntu/server_provision.sh &> /home/ubuntu/server_provision.log && echo "true"') logger.info('Downloading the swarm join command.') swarm_file = File.join(save_directory, 'worker_swarm_join.sh') @server.download_file('/home/ubuntu/swarmjoin.sh', swarm_file) logger.info('waiting for worker user_data to complete') @workers.each { |worker| worker.wait_command('while ! [ -e /home/ubuntu/user_data_done ]; do sleep 5; done && echo "true"') } logger.info('Running the configuration script for the worker(s).') @workers.each { |worker| worker.wait_command('sudo /home/ubuntu/worker_provision.sh &> /home/ubuntu/worker_provision.log && echo "true"') } logger.info('Successfully re-sized storage devices for all nodes. Joining server nodes to the swarm.') worker_join_cmd = "#{File.read(swarm_file).strip} && echo \"true\"" @workers.each { |worker| worker.wait_command(worker_join_cmd) } logger.info('All worker nodes have been added to the swarm. Setting environment variables and starting the cluster') total_procs = @server.procs @workers.each { |worker| total_procs += worker.procs } max_requests = ((total_procs + 10) * 1.2).round mongo_cores = (total_procs / 64.0).ceil web_cores = (total_procs / 32.0).ceil max_pool = 16 * web_cores rez_mem = 512 * max_pool total_procs = total_procs - mongo_cores - web_cores + 2 @server.shell_command("sed -i -e 's/AWS_MAX_REQUESTS/#{max_requests}/g' /home/ubuntu/docker-compose.yml && echo \"true\"") @server.shell_command("sed -i -e 's/AWS_MONGO_CORES/#{mongo_cores}/g' /home/ubuntu/docker-compose.yml && echo \"true\"") @server.shell_command("sed -i -e 's/AWS_WEB_CORES/#{web_cores}/g' /home/ubuntu/docker-compose.yml && echo \"true\"") @server.shell_command("sed -i -e 's/AWS_MAX_POOL/#{max_pool}/g' /home/ubuntu/docker-compose.yml && echo \"true\"") @server.shell_command("sed -i -e 's/AWS_REZ_MEM/#{rez_mem}/g' /home/ubuntu/docker-compose.yml && echo \"true\"") @server.shell_command("sed -i -e 's/AWS_OS_SERVER_NUMBER_OF_WORKERS/#{total_procs}/g' /home/ubuntu/docker-compose.yml && echo \"true\"") @server.shell_command("echo '' >> /home/ubuntu/.env && echo \"true\"") @server.shell_command("docker stack deploy --compose-file docker-compose.yml osserver && echo \"true\"") sleep 10 logger.info('The OpenStudio Server stack has been started. Waiting for the server to become available.') @server.wait_command("while ( nc -zv #{@server.ip} 80 3>&1 1>&2- 2>&3- ) | awk -F \":\" '$3 != \" Connection refused\" {exit 1}'; do sleep 5; done && echo \"true\"") logger.info('The OpenStudio Server stack has become available. Scaling the worker nodes.') @server.wait_command("docker service scale osserver_worker=#{total_procs} && echo \"true\"") logger.info('Waiting up to two minutes for the osserver_worker service to scale.') @server.wait_command("timeout 120 bash -c -- 'while [ $( docker service ls -f name=osserver_worker --format=\"{{.Replicas}}\" ) != \"#{total_procs}/#{total_procs}\" ]; do sleep 5; done'; echo \"true\"") logger.info('The OpenStudio Server stack is booted and ready for analysis submissions.') end |
#create_new_ami_json(version = 1) ⇒ Object
method to hit the existing list of available amis and compare to the list of AMIs on Amazon and then generate the new ami list
591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 591 def create_new_ami_json(version = 1) # get the list of existing amis from developer.nrel.gov existing_amis = OpenStudioAmis.new(1).list # list of available AMIs from AWS available_amis = describe_amis amis = transform_ami_lists(existing_amis, available_amis) if version == 1 version1 = {} # now grab the good keys - they should be sorted newest to older... so go backwards amis[:openstudio_server].keys.reverse_each do |key| a = amis[:openstudio_server][key] # this will override any of the old ami/os version version1[a[:openstudio_version].to_sym] = a[:amis] end # create the default version. First sort, then grab the first hash's values version1.sort_by default_v = nil version1 = Hash[version1.sort_by { |k, _| k.to_s.to_version }.reverse] default_v = version1.keys[0] version1[:default] = version1[default_v] amis = version1 elsif version == 2 # don't need to transform anything right now, only flag which ones are stable version so that the uploaded ami JSON has the # stable server for OpenStudio PAT to use. stable = JSON.parse File.read(File.join(File.dirname(__FILE__), 'ami_stable_version.json')), symbolize_names: true # go through and tag the versions of the openstudio instances that are stable, stable[:openstudio].each do |k, v| if amis[:openstudio][k.to_s] && amis[:openstudio][k.to_s][v.to_sym] amis[:openstudio][k.to_s][:stable] = v end end # I'm not sure what the below code is trying to accomplish. Are we even using the default? k, v = stable[:openstudio].first if k && v if amis[:openstudio][k.to_s] amis[:openstudio][:default] = v end end # now go through and if the OpenStudio version does not have a stable key, then assign it the most recent # stable AMI. This allows for easy testing so a new version of OpenStudio can use an existing AMI. stable[:openstudio].each do |stable_openstudio, stable_server| amis[:openstudio].each do |k, v| next if k == :default if k.to_s.to_version > stable_openstudio.to_s.to_version && v[:stable].nil? amis[:openstudio][k.to_s][:stable] = stable_server.to_s end end end end amis end |
#create_or_retrieve_default_security_group(tmp_name = 'openstudio-server-sg-v2.2', vpc_id = nil) ⇒ Object
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 84 def create_or_retrieve_default_security_group(tmp_name = 'openstudio-server-sg-v2.2', vpc_id = nil) group = @aws.describe_security_groups(filters: [{ name: 'group-name', values: [tmp_name] }]) logger.info "Length of the security group is: #{group.data.security_groups.length}" if group.data.security_groups.length == 0 logger.info 'security group not found --- will create a new one' if vpc_id r = @aws.create_security_group(group_name: tmp_name, description: "group dynamically created by #{__FILE__}", vpc_id: vpc_id) else r = @aws.create_security_group(group_name: tmp_name, description: "group dynamically created by #{__FILE__}") end group_id = r[:group_id] @aws.( group_id: group_id, ip_permissions: [ { ip_protocol: 'tcp', from_port: 22, to_port: 22, ip_ranges: [cidr_ip: '0.0.0.0/0'] }, # Eventually make this only the user's IP address seen by the internet { ip_protocol: 'tcp', from_port: 80, to_port: 80, ip_ranges: [cidr_ip: '0.0.0.0/0'] }, { ip_protocol: 'tcp', from_port: 443, to_port: 443, ip_ranges: [cidr_ip: '0.0.0.0/0'] }, { ip_protocol: 'tcp', from_port: 0, to_port: 65535, user_id_group_pairs: [{ group_name: tmp_name }] }, # allow all machines in the security group talk to each other openly { ip_protocol: 'udp', from_port: 0, to_port: 65535, user_id_group_pairs: [{ group_name: tmp_name }] }, # allow all machines in the security group talk to each other openly { ip_protocol: 'icmp', from_port: -1, to_port: -1, ip_ranges: [cidr_ip: '0.0.0.0/0'] } ] ) # reload group information group = @aws.describe_security_groups(filters: [{ name: 'group-name', values: [tmp_name] }]) else logger.info 'Found existing security group' end @security_groups = [group.data.security_groups.first.group_id] logger.info("server_group #{group.data.security_groups.first.group_name}:#{group.data.security_groups.first.group_id}") group.data.security_groups.first end |
#create_or_retrieve_key_pair(key_pair_name = nil) ⇒ Object
285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 285 def create_or_retrieve_key_pair(key_pair_name = nil) tmp_name = key_pair_name || "os-key-pair-#{@group_uuid}" # the describe_key_pairs method will raise an expectation if it can't find the key pair, so catch it resp = nil begin resp = @aws.describe_key_pairs(key_names: [tmp_name]).data fail 'looks like there are 2 key pairs with the same name' if resp.key_pairs.size >= 2 rescue logger.info "could not find key pair '#{tmp_name}'" end if resp.nil? || resp.key_pairs.size == 0 # create the new key_pair # check if the key pair name exists # create a new key pair everytime keypair = @aws.create_key_pair(key_name: tmp_name) # save the private key to memory (which can later be persisted via the save_private_key method) @private_key = keypair.data.key_material @key_pair_name = keypair.data.key_name else logger.info "found existing keypair #{resp.key_pairs.first}" @key_pair_name = resp.key_pairs.first[:key_name] # This will not set the private key because it doesn't live on the remote system end logger.info("create key pair: #{@key_pair_name}") end |
#delete_key_pair(key_pair_name = nil) ⇒ Object
Delete the key pair from aws
317 318 319 320 321 322 323 324 325 326 327 328 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 317 def delete_key_pair(key_pair_name = nil) tmp_name = key_pair_name || "os-key-pair-#{@group_uuid}" resp = nil begin logger.info "Trying to delete key pair #{tmp_name}" resp = @aws.delete_key_pair(key_name: tmp_name) rescue logger.info "could not delete the key pair '#{tmp_name}'" end resp end |
#describe_all_instances ⇒ Object
142 143 144 145 146 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 142 def describe_all_instances resp = @aws.describe_instance_status resp end |
#describe_amis(image_ids = [], owned_by_me = true) ⇒ Hash
Describe the list of AMIs adn return the hash.
229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 229 def describe_amis(image_ids = [], owned_by_me = true) resp = nil if owned_by_me resp = @aws.describe_images(owners: [:self]).data else resp = @aws.describe_images(image_ids: image_ids).data end resp = resp.to_hash # map the tags to hashes resp[:images].each do |image| image[:tags_hash] = {} image[:tags_hash][:tags] = [] # If the image is being created then its tags may be empty if image[:tags] image[:tags].each do |tag| if tag[:value] image[:tags_hash][tag[:key].to_sym] = tag[:value] else image[:tags_hash][:tags] << tag[:key] end end end end resp end |
#describe_availability_zones ⇒ Object
120 121 122 123 124 125 126 127 128 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 120 def describe_availability_zones resp = @aws.describe_availability_zones map = [] resp.data.availability_zones.each do |zn| map << zn.to_hash end { availability_zone_info: map } end |
#describe_availability_zones_json ⇒ Object
130 131 132 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 130 def describe_availability_zones_json describe_availability_zones.to_json end |
#describe_instances ⇒ Object
describe the instances by group id (this is the default method)
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 149 def describe_instances resp = nil if group_uuid resp = @aws.describe_instances( filters: [ # {name: 'instance-state-code', values: [0.to_s, 16.to_s]}, # running or pending -- any state { name: 'tag-key', values: ['GroupUUID'] }, { name: 'tag-value', values: [group_uuid.to_s] } ] ) else resp = @aws.describe_instances end # Any additional filters instance_data = nil if resp instance_data = [] resp.reservations.each do |r| r.instances.each do |i| i_h = i.to_hash if i_h[:tags].any? { |h| (h[:key] == 'GroupUUID') && (h[:value] == group_uuid.to_s) } instance_data << i_h end end end end instance_data end |
#describe_running_instances(group_uuid = nil, openstudio_instance_type = nil) ⇒ Object
return all of the running instances, or filter by the group_uuid & instance type
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 181 def describe_running_instances(group_uuid = nil, openstudio_instance_type = nil) resp = nil if group_uuid resp = @aws.describe_instances( filters: [ { name: 'instance-state-code', values: [0.to_s, 16.to_s] }, # running or pending { name: 'tag-key', values: ['GroupUUID'] }, { name: 'tag-value', values: [group_uuid.to_s] } ] ) else resp = @aws.describe_instances end instance_data = nil if resp instance_data = [] resp.reservations.each do |r| r.instances.each do |i| i_h = i.to_hash if group_uuid && openstudio_instance_type # {:key=>"Purpose", :value=>"OpenStudioWorker"} if i_h[:tags].any? { |h| (h[:key] == 'Purpose') && (h[:value] == "OpenStudio#{openstudio_instance_type.capitalize}") } && i_h[:tags].any? { |h| (h[:key] == 'GroupUUID') && (h[:value] == group_uuid.to_s) } instance_data << i_h end elsif group_uuid if i_h[:tags].any? { |h| (h[:key] == 'GroupUUID') && (h[:value] == group_uuid.to_s) } instance_data << i_h end elsif openstudio_instance_type if i_h[:tags].any? { |h| (h[:key] == 'Purpose') && (h[:value] == "OpenStudio#{openstudio_instance_type.capitalize}") } instance_data << i_h end else instance_data << i_h end end end end instance_data end |
#find_server(server_data_hash) ⇒ Object
method to query the amazon api to find the server (if it exists), based on the group id if it is found, then it will set the @server instance variable. The security groups are assigned from the server node information on AWS if the security groups have not been initialized yet.
Note that the information around keys and security groups is pulled from the instance information.
531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 531 def find_server(server_data_hash) @group_uuid = server_data_hash[:group_id] || @group_uuid load_private_key(server_data_hash[:server][:private_key_file_name]) logger.info "Finding the server for GroupUUID of #{group_uuid}" fail 'no GroupUUID defined either in member variable or method argument' if @group_uuid.nil? # This should really just be a single call to describe running instances @server = nil resp = describe_running_instances(group_uuid, :server) if resp fail "more than one server running with group uuid of #{group_uuid} found, expecting only one" if resp.size > 1 resp = resp.first if !@server if resp logger.info "Server found and loading data into object [instance id is #{resp[:instance_id]}]" sg = resp[:security_groups].map { |s| s[:group_id] } # Set the security groups of the object if these groups haven't been assigned yet. @security_groups = sg if @security_groups.empty? logger.info "The security groups in aws wrapper are #{@security_groups}" # set the key name from AWS if it isn't yet assigned logger.info 'Setting the keyname in the aws wrapper' @key_pair_name = resp[:key_name] unless @key_pair_name @server = OpenStudioAwsInstance.new(@aws, :server, @key_pair_name, sg, @group_uuid, @private_key, @private_key_file_name, @proxy) @server.load_instance_data(resp) end else logger.info "Server instance is already defined with instance #{resp[:instance_id]}" end else logger.info 'could not find a running server instance' end # Find the worker instances. if @workers.size == 0 resp = describe_running_instances(group_uuid, :worker) if resp resp.each do |r| @workers << OpenStudioAwsInstance.new(@aws, :worker, r[:key_name], r[:security_groups].map { |s| s[:group_id] }, @group_uuid, @private_key, @private_key_file_name, @proxy) @workers.last.load_instance_data(r) end end else logger.info 'Worker nodes are already defined' end # set the private key from the hash load_private_key server_data_hash[:server][:private_key_file_name] load_worker_key server_data_hash[:server][:worker_private_key_file_name] # Really don't need to return anything because this sets the class instance variable @server end |
#get_next_version(base, list_of_svs) ⇒ Object
take the base version and increment the patch until. TODO: DEPRECATE
675 676 677 678 679 680 681 682 683 684 685 686 687 688 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 675 def get_next_version(base, list_of_svs) b = base.to_version # first go through the array and test that there isn't any other versions > that in the array list_of_svs.each do |v| b = v.to_version if v.to_version.satisfies("#{b.major}.#{b.minor}.*") && v.to_version.patch > b.patch end # get the max version in the list_of_svs b.patch += 1 while list_of_svs.include?(b.to_s) # return the value back as a string b.to_s end |
#launch_server(image_id, instance_type, launch_options = {}) ⇒ Object
382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 382 def launch_server(image_id, instance_type, = {}) defaults = { user_id: 'unknown_user', tags: [], ebs_volume_size: nil, user_data_file: 'server_script.sh.template' } = defaults.merge() # replace the server_script.sh.template with the keys to add user_data = File.read(File.join(File.(File.dirname(__FILE__)), [:user_data_file])) user_data.gsub!(/SERVER_HOSTNAME/, 'openstudio.server') user_data.gsub!(/WORKER_PRIVATE_KEY_TEMPLATE/, worker_keys.private_key.gsub("\n", '\\n')) user_data.gsub!(/WORKER_PUBLIC_KEY_TEMPLATE/, worker_keys.ssh_public_key) @server = OpenStudioAwsInstance.new(@aws, :server, @key_pair_name, @security_groups, @group_uuid, @private_key, @private_key_file_name, @proxy) # TODO: create the EBS volumes instead of the ephemeral storage - needed especially for the m3 instances (SSD) fail 'image_id is nil' unless image_id fail 'instance type is nil' unless instance_type @server.launch_instance(image_id, instance_type, user_data, [:user_id], ) end |
#launch_workers(image_id, instance_type, num, launch_options = {}) ⇒ Object
408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 408 def launch_workers(image_id, instance_type, num, = {}) defaults = { user_id: 'unknown_user', tags: [], ebs_volume_size: nil, availability_zone: @server.data.availability_zone, user_data_file: 'worker_script.sh.template' } = defaults.merge() user_data = File.read(File.join(File.(File.dirname(__FILE__)), [:user_data_file])) user_data.gsub!(/SERVER_IP/, @server.data.private_ip_address) user_data.gsub!(/SERVER_HOSTNAME/, 'openstudio.server') user_data.gsub!(/WORKER_PUBLIC_KEY_TEMPLATE/, worker_keys.ssh_public_key) logger.info("worker user_data #{user_data.inspect}") # thread the launching of the workers num.times do @workers << OpenStudioAwsInstance.new(@aws, :worker, @key_pair_name, @security_groups, @group_uuid, @private_key, @private_key_file_name, @proxy) end threads = [] @workers.each do |worker| threads << Thread.new do # create the EBS volumes instead of the ephemeral storage - needed especially for the m3 instances (SSD) worker.launch_instance(image_id, instance_type, user_data, [:user_id], ) end end threads.each(&:join) # TODO: do we need to have a flag if the worker node is successful? # TODO: do we need to check the current list of running workers? end |
#load_private_key(filename) ⇒ Object
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 330 def load_private_key(filename) unless File.exist? filename # check if the file basename exists in your user directory filename = File.("~/.ssh/#{File.basename(filename)}") if File.exist? filename logger.info "Found key of same name in user's home ssh folder #{filename}" # using the key in your home directory else fail "Could not find private key #{filename}" unless File.exist? filename end end @private_key_file_name = File. filename @private_key = File.read(filename) end |
#load_worker_key(private_key_filename) ⇒ Object
Load the worker key for communicating between the server and worker instances on AWS. The public key will be automatically created when loading the private key
350 351 352 353 354 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 350 def load_worker_key(private_key_filename) logger.info "Loading worker keys from #{private_key_filename}" @worker_keys_filename = private_key_filename @worker_keys = SSHKey.new(File.read(@worker_keys_filename)) end |
#save_private_key(directory = '.', filename = 'ec2_server_key.pem') ⇒ Object
Save the private key to disk
357 358 359 360 361 362 363 364 365 366 367 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 357 def save_private_key(directory = '.', filename = 'ec2_server_key.pem') if @private_key @private_key_file_name = File. "#{directory}/#{filename}" logger.info "Saving server private key in #{@private_key_file_name}" File.open(@private_key_file_name, 'w') { |f| f << @private_key } logger.info 'Setting permissions of server private key to 0600' File.chmod(0600, @private_key_file_name) else fail "No private key found in which to persist with filename #{filename}" end end |
#save_worker_keys(directory = '.') ⇒ Object
save off the worker public/private keys that were created
370 371 372 373 374 375 376 377 378 379 380 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 370 def save_worker_keys(directory = '.') @worker_keys_filename = "#{directory}/ec2_worker_key.pem" logger.info "Saving worker private key in #{@worker_keys_filename}" File.open(@worker_keys_filename, 'w') { |f| f << @worker_keys.private_key } logger.info 'Setting permissions of worker private key to 0600' File.chmod(0600, @worker_keys_filename) wk = "#{directory}/ec2_worker_key.pub" logger.info "Saving worker public key in #{wk}" File.open(wk, 'w') { |f| f << @worker_keys.public_key } end |
#stop_instances(ids) ⇒ Object
Stop specific instances based on the instance_ids
262 263 264 265 266 267 268 269 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 262 def stop_instances(ids) resp = @aws.stop_instances( instance_ids: ids, force: true ) resp end |
#terminate_instances(ids) ⇒ Object
271 272 273 274 275 276 277 278 279 280 281 282 283 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 271 def terminate_instances(ids) resp = nil begin resp = @aws.terminate_instances( instance_ids: ids ) rescue Aws::EC2::Errors::InvalidInstanceIDNotFound # Log that the instances couldn't be found? resp = { error: 'instances could not be found' } end resp end |
#to_os_hash ⇒ Object
save off the instance configuration and instance information into a JSON file for later use
655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 655 def to_os_hash h = @server.to_os_hash h[:server][:worker_private_key_file_name] = @worker_keys_filename h[:workers] = @workers.map do |worker| { id: worker.data.id, ip: "http://#{worker.data.ip}", dns: worker.data.dns, procs: worker.data.procs, private_key_file_name: worker.private_key_file_name, private_ip_address: worker.private_ip_address } end h end |
#total_instances_count ⇒ Object
134 135 136 137 138 139 140 |
# File 'lib/openstudio/lib/openstudio_aws_wrapper.rb', line 134 def total_instances_count resp = @aws.describe_instance_status availability_zone = resp.instance_statuses.length > 0 ? resp.instance_statuses.first.availability_zone : 'no_instances' { total_instances: resp.instance_statuses.length, region: @region, availability_zone: availability_zone } end |