Class: OpenStudio::Aws::Aws

Inherits:
Object
  • Object
show all
Includes:
Logging
Defined in:
lib/openstudio/aws/aws.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Logging

configure_logger_for, #logger, logger_for

Constructor Details

#initialize(options = {}) ⇒ Aws

default constructor to create the AWS class that can spin up server and worker instances. options are optional with the following support:

credentials => {:access_key_id, :secret_access_key, :region, :ssl_verify_peer}
proxy => {:host => "192.168.0.1", :port => "8808", :username => "user", :password => "password"}}

Parameters:

  • options (Hash) (defaults to: {})

Options Hash (options):

  • :openstudio_version (Boolean) —

    Version of OpenStudio in which to do the lookup for the server AMIs. This cannot be used in conjunction with the :openstudio_server_version.

  • :openstudio_server_version (Boolean) —

    Version of OpenStudio Server in which to do the lookup for the server AMIs. This cannot be used in conjunction with the :openstudio_server_version

  • :stable (Boolean) — default: false —

    Find a stable version less than or equal to the version that is passed in the version field



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/openstudio/aws/aws.rb', line 64

def initialize(options = {})
  invalid_options = options.keys - VALID_OPTIONS
  if invalid_options.any?
    fail ArgumentError, "invalid option(s): #{invalid_options.join(', ')}"
  end

  # merge in some defaults
  defaults = {
    ami_lookup_version: 1,
    region: 'us-east-1',
    ssl_verify_peer: false,
    save_directory: '.'
  }
  options = defaults.merge(options)
  logger.info "AWS initialized with the options: #{options.except(:credentials)}"

  # set the save path
  @save_directory = File.expand_path options[:save_directory]
  FileUtils.mkdir_p @save_directory unless Dir.exist? @save_directory

  # read in the config.yml file to get the secret/private key
  if !options[:credentials]
    config_file = OpenStudio::Aws::Config.new

    # populate the credentials
    options[:credentials] =
        {
          access_key_id: config_file.access_key,
          secret_access_key: config_file.secret_key,
          region: options[:region],
          ssl_verify_peer: options[:ssl_verify_peer]
        }
  else
    options[:credentials][:region] = options[:region]
    options[:credentials][:ssl_verify_peer] = options[:ssl_verify_peer]
  end

  if options[:proxy]
    proxy_uri = nil
    if options[:proxy][:username]
      proxy_uri = "https://#{options[:proxy][:username]}:#{options[:proxy][:password]}@#{options[:proxy][:host]}:#{options[:proxy][:port]}"
    else
      proxy_uri = "https://#{options[:proxy][:host]}:#{options[:proxy][:port]}"
    end
    options[:proxy_uri] = proxy_uri
  end

  @os_aws = OpenStudioAwsWrapper.new(options)
  @os_cloudwatch = OpenStudioCloudWatch.new(options)

  @dockerized = options[:ami_lookup_version] == 3 ? true : false

  # this will grab the default version of openstudio ami versions
  # get the arugments for the AMI lookup
  ami_options = {}
  ami_options[:openstudio_server_version] = options[:openstudio_server_version] if options[:openstudio_server_version]
  ami_options[:openstudio_version] = options[:openstudio_version] if options[:openstudio_version]
  ami_options[:host] = options[:host] if options[:host]
  ami_options[:url] = options[:url] if options[:url]
  ami_options[:stable] = options[:stable] if options[:stable]

  @default_amis = OpenStudioAmis.new(options[:ami_lookup_version], ami_options).get_amis
end

Instance Attribute Details

#default_amis ⇒ Object (readonly)

Returns the value of attribute default_amis.



50
51
52
 
# File 'lib/openstudio/aws/aws.rb', line 50

def default_amis
  @default_amis
end

#os_aws ⇒ Object (readonly)

Deprecate OS_AWS object



49
50
51
 
# File 'lib/openstudio/aws/aws.rb', line 49

def os_aws
  @os_aws
end

#save_directory ⇒ Object (readonly)

Returns the value of attribute save_directory.



51
52
53
 
# File 'lib/openstudio/aws/aws.rb', line 51

def save_directory
  @save_directory
end

Instance Method Details

#cluster_info ⇒ Hash

Return information on the cluster instances as a hash. This includes IP addresses, host names, number of processors, etc.

Returns:

  • (Hash) —

    Data about the configured cluster



286
287
288
 
# File 'lib/openstudio/aws/aws.rb', line 286

def cluster_info
  @os_aws.to_os_hash
end

#create_server(options = {}) ⇒ Object

command line call to create a new instance. This should be more tightly integrated with the os-aws.rb gem



135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
 
# File 'lib/openstudio/aws/aws.rb', line 135

def create_server(options = {})
  defaults = {
    instance_type: 'm3.xlarge',
    security_groups: [],
    image_id: @default_amis[:server],
    user_id: 'unknown_user',

    # optional -- will default later
    associate_public_ip_address: nil,
    subnet_id: nil,
    ebs_volume_id: nil,
    aws_key_pair_name: nil,
    private_key_file_name: nil, # required if using an existing "aws_key_pair_name"
    tags: [],
    vpc_id: nil
  }
  options = defaults.merge(options)

  # for backwards compatibilty, still allow security_group
  if options[:security_group]
    warn 'Pass security_groups as an array instead of security_group. security_group will be deprecated in 0.4.0'
    options[:security_groups] = [options[:security_group]]
  end

  if options[:aws_key_pair_name]
    fail 'Must pass in the private_key_file_name' unless options[:private_key_file_name]
    fail "Private key was not found: #{options[:private_key_file_name]}" unless File.exist? options[:private_key_file_name]
  end

  if options[:security_groups].empty?
    # if the user has not specified any security groups, then create one called: 'openstudio-server-sg-v2'
    @os_aws.create_or_retrieve_default_security_group(tmp_name = 'openstudio-server-sg-v2.2',
                                                      vpc_id=options[:vpc_id])
  else
    @os_aws.security_groups = options[:security_groups]
  end

  @os_aws.create_or_retrieve_key_pair options[:aws_key_pair_name]

  # If using an already_existing key_pair, then you must pass in the private key file name
  if options[:aws_key_pair_name]
    @os_aws.load_private_key options[:private_key_file_name]
    @os_aws.private_key_file_name = options[:private_key_file_name]
  else
    # Save the private key if you did not pass in an already existing key_pair_name
    @os_aws.save_private_key @save_directory
  end

  user_data_file = @dockerized ? 'server_script.sh.docker.template' : 'server_script.sh.template'

  server_options = {
    user_id: options[:user_id],
    tags: options[:tags],
    subnet_id: options[:subnet_id],
    associate_public_ip_address: options[:associate_public_ip_address],
    user_data_file: user_data_file
  }

  server_options[:availability_zone] = options[:availability_zone] if options[:availability_zone]

  # save the worker pem and public to the directory
  # presently, this will always overwrite the worker key, is that okay? Is this really needed later?
  @os_aws.save_worker_keys @save_directory

  @os_aws.launch_server(options[:image_id], options[:instance_type], server_options)
end

#create_workers(number_of_instances, options = {}, user_id = 'unknown_user') ⇒ Object

create workers after the server has been created.

Parameters:

  • number_of_instances (Integer) —

    Number of worker instances to create

  • options (Hash) (defaults to: {})

Options Hash (options):

  • :instance_type (String) —

    Type of server to start (e.g. m3.medium, m3.xlarge, etc.)



207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
 
# File 'lib/openstudio/aws/aws.rb', line 207

def create_workers(number_of_instances, options = {}, user_id = 'unknown_user')
  defaults = {
    instance_type: 'm2.4xlarge',
    security_groups: [],
    image_id: nil, # don't prescribe the image id so that it can determine later
    user_id: user_id,

    # optional -- will default later
    associate_public_ip_address: nil,
    subnet_id: nil,
    ebs_volume_id: nil,
    aws_key_pair_name: nil,
    private_key_file_name: nil, # required if using an existing "aws_key_pair_name",
    tags: []
  }
  options = defaults.merge(options)

  # for backwards compatibility, still allow security_group
  if options[:security_group]
    warn 'Pass security_groups as an array instead of security_group. security_group will be deprecated in 0.4.0'
    options[:security_groups] = [options[:security_group]]
  end

  # Get the right worker AMI ids based on the type of instance
  if options[:image_id].nil?
    options[:image_id] = determine_image_type(options[:instance_type])
  end

  fail "Can't create workers without a server instance running" if @os_aws.server.nil?
  user_data_file = @dockerized ? 'worker_script.sh.docker.template' : 'worker_script.sh.template'

  unless number_of_instances == 0
    worker_options = {
      user_id: options[:user_id],
      tags: options[:tags],
      subnet_id: options[:subnet_id],
      associate_public_ip_address: options[:associate_public_ip_address],
      user_data_file: user_data_file
    }

    # if options[:ebs_volume_size]
    #   worker_options[:ebs_volume_size] = options[:ebs_volume_size]
    # end

    @os_aws.launch_workers(options[:image_id], options[:instance_type], number_of_instances, worker_options)
  end

  logger.info 'Waiting for server/worker configurations'

  begin
    if @dockerized
      @os_aws.configure_swarm_cluster(@save_directory)
    else
      @os_aws.configure_server_and_workers
    end
  rescue => e
    fail "Configuring the cluster failed with error `#{e.message}` in:\n#{e.backtrace.join('\n')}"
  end
end

#delete_key_pair ⇒ Object

Delete the key pair. Make sure that this happens at the end of whatever you are running, because you will not be able to connect to the instance after you do this.



304
305
306
 
# File 'lib/openstudio/aws/aws.rb', line 304

def delete_key_pair
  @os_aws.delete_key_pair
end

#describe_all_instances ⇒ Object

List the description of all instances on AWS in the predefined region



350
351
352
 
# File 'lib/openstudio/aws/aws.rb', line 350

def describe_all_instances
  @os_aws.describe_all_instances
end

#describe_availability_zones ⇒ Object

Return the availability zones for the AWS region specified in the options hash



298
299
300
 
# File 'lib/openstudio/aws/aws.rb', line 298

def describe_availability_zones
  @os_aws.describe_availability_zones
end

#describe_instances ⇒ Object

Return the description of the instances in the GroupUUID

Examples:

Return Example

[{:instance_id=>"i-45f924ac",
  :image_id=>"ami-845a54ec",
  :state=>{:code=>48, :name=>"terminated"},
  :private_dns_name=>"",
  :public_dns_name=>"",
  :state_transition_reason=>"User initiated (2015-06-01 21:50:40 GMT)",
  :key_name=>"os-key-pair-275a3bf436004c04a1a347ff36337f16",
  :ami_launch_index=>0,
  :product_codes=>[],
  :instance_type=>"m3.medium",
  :launch_time=>2015-06-01 21:13:18 UTC,
  :placement=>
      {:availability_zone=>"us-east-1e", :group_name=>"", :tenancy=>"default"},
      :monitoring=>{:state=>"disabled"},
      :state_reason=>
      {:code=>"Client.UserInitiatedShutdown",
       :message=>"Client.UserInitiatedShutdown: User initiated shutdown"},
      :architecture=>"x86_64",
      :root_device_type=>"ebs",
      :root_device_name=>"/dev/sda1",
      :block_device_mappings=>[],
      :virtualization_type=>"hvm",
      :client_token=>"",
      :tags=>
      [{:key=>"Purpose", :value=>"OpenStudioServer"},
       {:key=>"NumberOfProcessors", :value=>"1"},
       {:key=>"GroupUUID", :value=>"275a3bf436004c04a1a347ff36337f16"},
       {:key=>"Name", :value=>"OpenStudio-Server"},
       {:key=>"UserID", :value=>"unknown_user"}],
      :security_groups=>
      [{:group_name=>"openstudio-server-sg-v2.1", :group_id=>"sg-8740f3ea"}],
      :hypervisor=>"xen",
      :network_interfaces=>[],
      :ebs_optimized=>false}]


345
346
347
 
# File 'lib/openstudio/aws/aws.rb', line 345

def describe_instances
  @os_aws.describe_instances
end

#determine_image_type(instance_type) ⇒ Object 



478
479
480
481
482
483
484
485
486
487
 
# File 'lib/openstudio/aws/aws.rb', line 478

def determine_image_type(instance_type)
  image = nil
  if instance_type =~ /cc2/
    image = @default_amis[:cc2worker]
  else
    image = @default_amis[:worker]
  end

  image
end

#download_remote_file(server_or_workers, remote_file, local_file) ⇒ Object

Download remote files that are on the server or worker. note that the worker at the moment will not work because it would simply overwrite the downloaded filas at this time.



468
469
470
471
472
473
474
475
476
 
# File 'lib/openstudio/aws/aws.rb', line 468

def download_remote_file(server_or_workers, remote_file, local_file)
  case server_or_workers
    when :server
      fail 'Server node is nil' unless @os_aws.server
      return @os_aws.server.download_file(remote_file, local_file)
    when :worker
      fail 'Worker file download is not available'
  end
end

#estimated_charges ⇒ Object

Return the estimated cost for EC2 instances



360
361
362
 
# File 'lib/openstudio/aws/aws.rb', line 360

def estimated_charges
  @os_cloudwatch.estimated_charges
end

#group_uuid ⇒ String

Return the Group UUID as defined in the AWS wrapper

Returns:

  • (String) —

    UUID



492
493
494
 
# File 'lib/openstudio/aws/aws.rb', line 492

def group_uuid
  @os_aws.group_uuid
end

#load_instance_info_from_file(filename) ⇒ Object 



425
426
427
428
429
430
431
432
433
434
435
436
437
 
# File 'lib/openstudio/aws/aws.rb', line 425

def load_instance_info_from_file(filename)
  fail 'Could not find instance description JSON file' unless File.exist? filename

  h = JSON.parse(File.read(filename), symbolize_names: true)
  if h[:location] == 'AWS'
    @os_aws.find_server(h)
  else
    logger.info "Instance file '#{filename}' does not have the location of 'AWS'"
    return false
  end

  true
end

#print_connection_info ⇒ nil

Write out to the terminal the connection information for the servers and workers

Returns:

  • (nil) —

    Only prints to the screen. No return is expected



270
271
272
273
274
275
276
277
278
279
280
281
282
 
# File 'lib/openstudio/aws/aws.rb', line 270

def print_connection_info
  # Print out some debugging commands (probably work on mac/linux only)
  puts ''
  puts 'Server SSH Command:'
  puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{@os_aws.server.data[:dns]}"
  if @os_aws.workers.size > 0
    puts ''
    puts 'Worker SSH Command:'
    @os_aws.workers.each do |worker|
      puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{worker.data[:dns]}"
    end
  end
end

#save_cluster_info(filename) ⇒ Object

Save a JSON with information about the cluster that was configured.

Parameters:

  • filename (String) —

    Path and filename to save the JSON file



293
294
295
 
# File 'lib/openstudio/aws/aws.rb', line 293

def save_cluster_info(filename)
  File.open(filename, 'w') { |f| f << JSON.pretty_generate(cluster_info) }
end

#server ⇒ Object 



439
440
441
 
# File 'lib/openstudio/aws/aws.rb', line 439

def server
  @os_aws.server
end

#shell_command(server_or_workers, command, load_env = true) ⇒ Object 



455
456
457
458
459
460
461
462
463
464
 
# File 'lib/openstudio/aws/aws.rb', line 455

def shell_command(server_or_workers, command, load_env = true)
  case server_or_workers
    when :server
      fail 'Server node is nil' unless @os_aws.server
      return @os_aws.server.shell_command(command, load_env)
    when :worker
      fail 'Worker list is empty' if @os_aws.workers.empty?
      return @os_aws.workers.each { |w| w.shell_command(command, load_env) }
  end
end

#stop ⇒ Object

Stop the entire cluster



365
366
367
368
369
 
# File 'lib/openstudio/aws/aws.rb', line 365

def stop
  puts "Stoping any instance with group ID: #{@os_aws.group_uuid}"

  stop_instances_by_group_id(@os_aws.group_uuid)
end

#stop_instances(group_id, openstudio_instance_type) ⇒ Object

Stop running instances

Parameters:

  • group_id (String) —

    The unique group identifier for the OpenStudio cluster.

  • openstudio_instance_type (Symbol) —

    The type of instance (:server or :worker)



375
376
377
378
379
380
381
382
 
# File 'lib/openstudio/aws/aws.rb', line 375

def stop_instances(group_id, openstudio_instance_type)
  instances = @os_aws.describe_running_instances(group_id, openstudio_instance_type.to_sym)
  ids = instances.map { |k, _| k[:instance_id] }

  resp = []
  resp = @os_aws.stop_instances(ids).to_hash unless ids.empty?
  resp
end

#stop_instances_by_group_id(group_id) ⇒ Object

Warning, it appears that this stops all the instances



385
386
387
388
389
390
391
392
393
 
# File 'lib/openstudio/aws/aws.rb', line 385

def stop_instances_by_group_id(group_id)
  instances = @os_aws.describe_running_instances(group_id)
  ids = instances.map { |k, _| k[:instance_id] }

  puts "Stoping the following instances #{ids}"
  resp = []
  resp = @os_aws.stop_instances(ids).to_hash unless ids.empty?
  resp
end

#terminate ⇒ Object

Terminate the entire cluster based on the member variable’s group_uuid.



419
420
421
422
423
 
# File 'lib/openstudio/aws/aws.rb', line 419

def terminate
  logger.info "Terminating any instance with GroupUUID: #{@os_aws.group_uuid}"

  terminate_instances_by_group_id(@os_aws.group_uuid)
end

#terminate_instances(ids) ⇒ Object

@params(ids): array of instance ids



396
397
398
399
400
401
 
# File 'lib/openstudio/aws/aws.rb', line 396

def terminate_instances(ids)
  logger.info "Terminating the following instances #{ids}"
  resp = []
  resp = @os_aws.terminate_instances(ids).to_hash unless ids.empty?
  resp
end

#terminate_instances_by_group_id(group_id) ⇒ Object

Warning, it appears that this terminates all the instances



404
405
406
407
408
409
410
411
412
413
414
415
416
 
# File 'lib/openstudio/aws/aws.rb', line 404

def terminate_instances_by_group_id(group_id)
  fail 'Group ID not defined' unless group_id

  instances = @os_aws.describe_running_instances(group_id)
  logger.info instances
  ids = instances.map { |k, _| k[:instance_id] }

  logger.info "Terminating the following instances #{ids}"
  resp = []
  resp = @os_aws.terminate_instances(ids).to_hash unless ids.empty?

  resp[:terminating_instances].first[:current_state][:name] == 'shutting-down'
end

#total_instances_count ⇒ Object

Return the list of all the instances that are running on the account in the availablity zone



355
356
357
 
# File 'lib/openstudio/aws/aws.rb', line 355

def total_instances_count
  @os_aws.total_instances_count
end

#upload_file(server_or_workers, local_file, remote_file) ⇒ Object

Send a file to the server or worker nodes



444
445
446
447
448
449
450
451
452
453
 
# File 'lib/openstudio/aws/aws.rb', line 444

def upload_file(server_or_workers, local_file, remote_file)
  case server_or_workers
    when :server
      fail 'Server node is nil' unless @os_aws.server
      return @os_aws.server.upload_file(local_file, remote_file)
    when :worker
      fail 'Worker list is empty' if @os_aws.workers.empty?
      return @os_aws.workers.each { |w| w.upload_file(local_file, remote_file) }
  end
end