Class: OpenStudio::Aws::Aws

Inherits:
Object
  • Object
show all
Includes:
Logging
Defined in:
lib/openstudio/aws/aws.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Logging

configure_logger_for, #logger, logger_for

Constructor Details

#initialize(options = {}) ⇒ Aws

default constructor to create the AWS class that can spin up server and worker instances. options are optional with the following support:

credentials => {:access_key_id, :secret_access_key, :region, :ssl_verify_peer}
proxy => {:host => "192.168.0.1", :port => "8808", :username => "user", :password => "password"}}

Parameters:

  • options (Hash) (defaults to: {})

Options Hash (options):

  • :openstudio_version (Boolean) —

    Version of OpenStudio in which to do the lookup for the server AMIs. This cannot be used in conjunction with the :openstudio_server_version.

  • :openstudio_server_version (Boolean) —

    Version of OpenStudio Server in which to do the lookup for the server AMIs. This cannot be used in conjunction with the :openstudio_server_version

  • :stable (Boolean) — default: false —

    Find a stable version less than or equal to the version that is passed in the version field



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/openstudio/aws/aws.rb', line 64

def initialize(options = {})
  invalid_options = options.keys - VALID_OPTIONS
  if invalid_options.any?
    raise ArgumentError, "invalid option(s): #{invalid_options.join(', ')}"
  end

  # merge in some defaults
  defaults = {
    ami_lookup_version: 1,
    region: 'us-east-1',
    ssl_verify_peer: false,
    save_directory: '.'
  }
  options = defaults.merge(options)
  logger.info "AWS initialized with the options: #{options.except(:credentials)}"

  # set the save path
  @save_directory = File.expand_path options[:save_directory]
  FileUtils.mkdir_p @save_directory unless Dir.exist? @save_directory

  # read in the config.yml file to get the secret/private key
  if !options[:credentials]
    config_file = OpenStudio::Aws::Config.new

    # populate the credentials
    options[:credentials] =
      {
        access_key_id: config_file.access_key,
        secret_access_key: config_file.secret_key,
        region: options[:region],
        ssl_verify_peer: options[:ssl_verify_peer]
      }
  else
    options[:credentials][:region] = options[:region]
    options[:credentials][:ssl_verify_peer] = options[:ssl_verify_peer]
  end

  if options[:proxy]
    proxy_uri = nil
    if options[:proxy][:username]
      proxy_uri = "https://#{options[:proxy][:username]}:#{options[:proxy][:password]}@#{options[:proxy][:host]}:#{options[:proxy][:port]}"
    else
      proxy_uri = "https://#{options[:proxy][:host]}:#{options[:proxy][:port]}"
    end
    options[:proxy_uri] = proxy_uri
  end

  @os_aws = OpenStudioAwsWrapper.new(options)
  @os_cloudwatch = OpenStudioCloudWatch.new(options)

  @dockerized = options[:ami_lookup_version] == 3

  # this will grab the default version of openstudio ami versions
  # get the arugments for the AMI lookup
  ami_options = {}
  ami_options[:openstudio_server_version] = options[:openstudio_server_version] if options[:openstudio_server_version]
  ami_options[:openstudio_version] = options[:openstudio_version] if options[:openstudio_version]
  ami_options[:host] = options[:host] if options[:host]
  ami_options[:url] = options[:url] if options[:url]
  ami_options[:stable] = options[:stable] if options[:stable]

  @default_amis = OpenStudioAmis.new(options[:ami_lookup_version], ami_options).get_amis
end

Instance Attribute Details

#default_amis ⇒ Object (readonly)

Returns the value of attribute default_amis.



50
51
52
 
# File 'lib/openstudio/aws/aws.rb', line 50

def default_amis
  @default_amis
end

#os_aws ⇒ Object (readonly)

Deprecate OS_AWS object



49
50
51
 
# File 'lib/openstudio/aws/aws.rb', line 49

def os_aws
  @os_aws
end

#save_directory ⇒ Object (readonly)

Returns the value of attribute save_directory.



51
52
53
 
# File 'lib/openstudio/aws/aws.rb', line 51

def save_directory
  @save_directory
end

Instance Method Details

#cluster_info ⇒ Hash

Return information on the cluster instances as a hash. This includes IP addresses, host names, number of processors, etc.

Returns:

  • (Hash) —

    Data about the configured cluster



287
288
289
 
# File 'lib/openstudio/aws/aws.rb', line 287

def cluster_info
  @os_aws.to_os_hash
end

#create_server(options = {}) ⇒ Object

command line call to create a new instance. This should be more tightly integrated with the os-aws.rb gem



135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
 
# File 'lib/openstudio/aws/aws.rb', line 135

def create_server(options = {})
  defaults = {
    instance_type: 'm3.xlarge',
    security_groups: [],
    image_id: @default_amis[:server],
    user_id: 'unknown_user',

    # optional -- will default later
    associate_public_ip_address: nil,
    subnet_id: nil,
    ebs_volume_id: nil,
    aws_key_pair_name: nil,
    private_key_file_name: nil, # required if using an existing "aws_key_pair_name"
    tags: [],
    vpc_id: nil
  }
  options = defaults.merge(options)

  # for backwards compatibilty, still allow security_group
  if options[:security_group]
    warn 'Pass security_groups as an array instead of security_group. security_group will be deprecated in 0.4.0'
    options[:security_groups] = [options[:security_group]]
  end

  if options[:aws_key_pair_name]
    raise 'Must pass in the private_key_file_name' unless options[:private_key_file_name]
    raise "Private key was not found: #{options[:private_key_file_name]}" unless File.exist? options[:private_key_file_name]
  end

  if options[:security_groups].empty?
    # if the user has not specified any security groups, then create one called: 'openstudio-server-sg-v2'
    @os_aws.create_or_retrieve_default_security_group(tmp_name = 'openstudio-server-sg-v2.2',
                                                      vpc_id = options[:vpc_id])
  else
    @os_aws.security_groups = options[:security_groups]
  end

  @os_aws.create_or_retrieve_key_pair options[:aws_key_pair_name]

  # If using an already_existing key_pair, then you must pass in the private key file name
  if options[:aws_key_pair_name]
    @os_aws.load_private_key options[:private_key_file_name]
    @os_aws.private_key_file_name = options[:private_key_file_name]
  else
    # Save the private key if you did not pass in an already existing key_pair_name
    @os_aws.save_private_key @save_directory
  end

  user_data_file = @dockerized ? 'server_script.sh.docker.template' : 'server_script.sh.template'

  server_options = {
    user_id: options[:user_id],
    tags: options[:tags],
    subnet_id: options[:subnet_id],
    associate_public_ip_address: options[:associate_public_ip_address],
    user_data_file: user_data_file
  }

  server_options[:availability_zone] = options[:availability_zone] if options[:availability_zone]

  # save the worker pem and public to the directory
  # presently, this will always overwrite the worker key, is that okay? Is this really needed later?
  @os_aws.save_worker_keys @save_directory

  @os_aws.launch_server(options[:image_id], options[:instance_type], server_options)
end

#create_workers(number_of_instances, options = {}, user_id = 'unknown_user') ⇒ Object

create workers after the server has been created.

Parameters:

  • number_of_instances (Integer) —

    Number of worker instances to create

  • options (Hash) (defaults to: {})

Options Hash (options):

  • :instance_type (String) —

    Type of server to start (e.g. m3.medium, m3.xlarge, etc.)



207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
 
# File 'lib/openstudio/aws/aws.rb', line 207

def create_workers(number_of_instances, options = {}, user_id = 'unknown_user')
  defaults = {
    instance_type: 'm2.4xlarge',
    security_groups: [],
    image_id: nil, # don't prescribe the image id so that it can determine later
    user_id: user_id,

    # optional -- will default later
    associate_public_ip_address: nil,
    subnet_id: nil,
    ebs_volume_id: nil,
    aws_key_pair_name: nil,
    private_key_file_name: nil, # required if using an existing "aws_key_pair_name",
    tags: []
  }
  options = defaults.merge(options)

  # for backwards compatibility, still allow security_group
  if options[:security_group]
    warn 'Pass security_groups as an array instead of security_group. security_group will be deprecated in 0.4.0'
    options[:security_groups] = [options[:security_group]]
  end

  # Get the right worker AMI ids based on the type of instance
  if options[:image_id].nil?
    options[:image_id] = determine_image_type(options[:instance_type])
  end

  raise "Can't create workers without a server instance running" if @os_aws.server.nil?

  user_data_file = @dockerized ? 'worker_script.sh.docker.template' : 'worker_script.sh.template'

  unless number_of_instances == 0
    worker_options = {
      user_id: options[:user_id],
      tags: options[:tags],
      subnet_id: options[:subnet_id],
      associate_public_ip_address: options[:associate_public_ip_address],
      user_data_file: user_data_file
    }

    # if options[:ebs_volume_size]
    #   worker_options[:ebs_volume_size] = options[:ebs_volume_size]
    # end

    @os_aws.launch_workers(options[:image_id], options[:instance_type], number_of_instances, worker_options)
  end

  logger.info 'Waiting for server/worker configurations'

  begin
    if @dockerized
      @os_aws.configure_swarm_cluster(@save_directory)
    else
      @os_aws.configure_server_and_workers
    end
  rescue StandardError => e
    raise "Configuring the cluster failed with error `#{e.message}` in:\n#{e.backtrace.join('\n')}"
  end
end

#delete_key_pair ⇒ Object

Delete the key pair. Make sure that this happens at the end of whatever you are running, because you will not be able to connect to the instance after you do this.



305
306
307
 
# File 'lib/openstudio/aws/aws.rb', line 305

def delete_key_pair
  @os_aws.delete_key_pair
end

#describe_all_instances ⇒ Object

List the description of all instances on AWS in the predefined region



351
352
353
 
# File 'lib/openstudio/aws/aws.rb', line 351

def describe_all_instances
  @os_aws.describe_all_instances
end

#describe_availability_zones ⇒ Object

Return the availability zones for the AWS region specified in the options hash



299
300
301
 
# File 'lib/openstudio/aws/aws.rb', line 299

def describe_availability_zones
  @os_aws.describe_availability_zones
end

#describe_instances ⇒ Object

Return the description of the instances in the GroupUUID

Examples:

Return Example

[{:instance_id=>"i-45f924ac",
  :image_id=>"ami-845a54ec",
  :state=>{:code=>48, :name=>"terminated"},
  :private_dns_name=>"",
  :public_dns_name=>"",
  :state_transition_reason=>"User initiated (2015-06-01 21:50:40 GMT)",
  :key_name=>"os-key-pair-275a3bf436004c04a1a347ff36337f16",
  :ami_launch_index=>0,
  :product_codes=>[],
  :instance_type=>"m3.medium",
  :launch_time=>2015-06-01 21:13:18 UTC,
  :placement=>
      {:availability_zone=>"us-east-1e", :group_name=>"", :tenancy=>"default"},
      :monitoring=>{:state=>"disabled"},
      :state_reason=>
      {:code=>"Client.UserInitiatedShutdown",
       :message=>"Client.UserInitiatedShutdown: User initiated shutdown"},
      :architecture=>"x86_64",
      :root_device_type=>"ebs",
      :root_device_name=>"/dev/sda1",
      :block_device_mappings=>[],
      :virtualization_type=>"hvm",
      :client_token=>"",
      :tags=>
      [{:key=>"Purpose", :value=>"OpenStudioServer"},
       {:key=>"NumberOfProcessors", :value=>"1"},
       {:key=>"GroupUUID", :value=>"275a3bf436004c04a1a347ff36337f16"},
       {:key=>"Name", :value=>"OpenStudio-Server"},
       {:key=>"UserID", :value=>"unknown_user"}],
      :security_groups=>
      [{:group_name=>"openstudio-server-sg-v2.1", :group_id=>"sg-8740f3ea"}],
      :hypervisor=>"xen",
      :network_interfaces=>[],
      :ebs_optimized=>false}]


346
347
348
 
# File 'lib/openstudio/aws/aws.rb', line 346

def describe_instances
  @os_aws.describe_instances
end

#determine_image_type(instance_type) ⇒ Object 



484
485
486
487
488
489
490
491
492
493
 
# File 'lib/openstudio/aws/aws.rb', line 484

def determine_image_type(instance_type)
  image = nil
  if instance_type =~ /cc2/
    image = @default_amis[:cc2worker]
  else
    image = @default_amis[:worker]
  end

  image
end

#download_remote_file(server_or_workers, remote_file, local_file) ⇒ Object

Download remote files that are on the server or worker. note that the worker at the moment will not work because it would simply overwrite the downloaded filas at this time.



473
474
475
476
477
478
479
480
481
482
 
# File 'lib/openstudio/aws/aws.rb', line 473

def download_remote_file(server_or_workers, remote_file, local_file)
  case server_or_workers
    when :server
      raise 'Server node is nil' unless @os_aws.server

      return @os_aws.server.download_file(remote_file, local_file)
    when :worker
      raise 'Worker file download is not available'
  end
end

#estimated_charges ⇒ Object

Return the estimated cost for EC2 instances



361
362
363
 
# File 'lib/openstudio/aws/aws.rb', line 361

def estimated_charges
  @os_cloudwatch.estimated_charges
end

#group_uuid ⇒ String

Return the Group UUID as defined in the AWS wrapper

Returns:

  • (String) —

    UUID



498
499
500
 
# File 'lib/openstudio/aws/aws.rb', line 498

def group_uuid
  @os_aws.group_uuid
end

#load_instance_info_from_file(filename) ⇒ Object 



426
427
428
429
430
431
432
433
434
435
436
437
438
 
# File 'lib/openstudio/aws/aws.rb', line 426

def load_instance_info_from_file(filename)
  raise 'Could not find instance description JSON file' unless File.exist? filename

  h = JSON.parse(File.read(filename), symbolize_names: true)
  if h[:location] == 'AWS'
    @os_aws.find_server(h)
  else
    logger.info "Instance file '#{filename}' does not have the location of 'AWS'"
    return false
  end

  true
end

#print_connection_info ⇒ nil

Write out to the terminal the connection information for the servers and workers

Returns:

  • (nil) —

    Only prints to the screen. No return is expected



271
272
273
274
275
276
277
278
279
280
281
282
283
 
# File 'lib/openstudio/aws/aws.rb', line 271

def print_connection_info
  # Print out some debugging commands (probably work on mac/linux only)
  puts ''
  puts 'Server SSH Command:'
  puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{@os_aws.server.data[:dns]}"
  if !@os_aws.workers.empty?
    puts ''
    puts 'Worker SSH Command:'
    @os_aws.workers.each do |worker|
      puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{worker.data[:dns]}"
    end
  end
end

#save_cluster_info(filename) ⇒ Object

Save a JSON with information about the cluster that was configured.

Parameters:

  • filename (String) —

    Path and filename to save the JSON file



294
295
296
 
# File 'lib/openstudio/aws/aws.rb', line 294

def save_cluster_info(filename)
  File.open(filename, 'w') { |f| f << JSON.pretty_generate(cluster_info) }
end

#server ⇒ Object 



440
441
442
 
# File 'lib/openstudio/aws/aws.rb', line 440

def server
  @os_aws.server
end

#shell_command(server_or_workers, command, load_env = true) ⇒ Object 



458
459
460
461
462
463
464
465
466
467
468
469
 
# File 'lib/openstudio/aws/aws.rb', line 458

def shell_command(server_or_workers, command, load_env = true)
  case server_or_workers
    when :server
      raise 'Server node is nil' unless @os_aws.server

      return @os_aws.server.shell_command(command, load_env)
    when :worker
      raise 'Worker list is empty' if @os_aws.workers.empty?

      return @os_aws.workers.each { |w| w.shell_command(command, load_env) }
  end
end

#stop ⇒ Object

Stop the entire cluster



366
367
368
369
370
 
# File 'lib/openstudio/aws/aws.rb', line 366

def stop
  puts "Stoping any instance with group ID: #{@os_aws.group_uuid}"

  stop_instances_by_group_id(@os_aws.group_uuid)
end

#stop_instances(group_id, openstudio_instance_type) ⇒ Object

Stop running instances

Parameters:

  • group_id (String) —

    The unique group identifier for the OpenStudio cluster.

  • openstudio_instance_type (Symbol) —

    The type of instance (:server or :worker)



376
377
378
379
380
381
382
383
 
# File 'lib/openstudio/aws/aws.rb', line 376

def stop_instances(group_id, openstudio_instance_type)
  instances = @os_aws.describe_running_instances(group_id, openstudio_instance_type.to_sym)
  ids = instances.map { |k, _| k[:instance_id] }

  resp = []
  resp = @os_aws.stop_instances(ids).to_hash unless ids.empty?
  resp
end

#stop_instances_by_group_id(group_id) ⇒ Object

Warning, it appears that this stops all the instances



386
387
388
389
390
391
392
393
394
 
# File 'lib/openstudio/aws/aws.rb', line 386

def stop_instances_by_group_id(group_id)
  instances = @os_aws.describe_running_instances(group_id)
  ids = instances.map { |k, _| k[:instance_id] }

  puts "Stoping the following instances #{ids}"
  resp = []
  resp = @os_aws.stop_instances(ids).to_hash unless ids.empty?
  resp
end

#terminate ⇒ Object

Terminate the entire cluster based on the member variable’s group_uuid.



420
421
422
423
424
 
# File 'lib/openstudio/aws/aws.rb', line 420

def terminate
  logger.info "Terminating any instance with GroupUUID: #{@os_aws.group_uuid}"

  terminate_instances_by_group_id(@os_aws.group_uuid)
end

#terminate_instances(ids) ⇒ Object

@params(ids): array of instance ids



397
398
399
400
401
402
 
# File 'lib/openstudio/aws/aws.rb', line 397

def terminate_instances(ids)
  logger.info "Terminating the following instances #{ids}"
  resp = []
  resp = @os_aws.terminate_instances(ids).to_hash unless ids.empty?
  resp
end

#terminate_instances_by_group_id(group_id) ⇒ Object

Warning, it appears that this terminates all the instances



405
406
407
408
409
410
411
412
413
414
415
416
417
 
# File 'lib/openstudio/aws/aws.rb', line 405

def terminate_instances_by_group_id(group_id)
  raise 'Group ID not defined' unless group_id

  instances = @os_aws.describe_running_instances(group_id)
  logger.info instances
  ids = instances.map { |k, _| k[:instance_id] }

  logger.info "Terminating the following instances #{ids}"
  resp = []
  resp = @os_aws.terminate_instances(ids).to_hash unless ids.empty?

  resp[:terminating_instances].first[:current_state][:name] == 'shutting-down'
end

#total_instances_count ⇒ Object

Return the list of all the instances that are running on the account in the availablity zone



356
357
358
 
# File 'lib/openstudio/aws/aws.rb', line 356

def total_instances_count
  @os_aws.total_instances_count
end

#upload_file(server_or_workers, local_file, remote_file) ⇒ Object

Send a file to the server or worker nodes



445
446
447
448
449
450
451
452
453
454
455
456
 
# File 'lib/openstudio/aws/aws.rb', line 445

def upload_file(server_or_workers, local_file, remote_file)
  case server_or_workers
    when :server
      raise 'Server node is nil' unless @os_aws.server

      return @os_aws.server.upload_file(local_file, remote_file)
    when :worker
      raise 'Worker list is empty' if @os_aws.workers.empty?

      return @os_aws.workers.each { |w| w.upload_file(local_file, remote_file) }
  end
end