Class: Reyes::AwsManager

Inherits:
Object
  • Object
show all
Includes:
Chalk::Log
Defined in:
lib/reyes/aws_manager.rb

Defined Under Namespace

Classes: Error

Constant Summary collapse

RegionShortNames =

Short names for AWS regions to save space in ipset names

{
  'us-east-1' => 'VA',
  'us-west-2' => 'OR',
  'us-west-1' => 'CA',
  'eu-west-1' => 'IE',
  'eu-central-1' => 'DE',
  'ap-southeast-1' => 'SG',
  'ap-southeast-2' => 'AU',
  'ap-northeast-1' => 'JP',
  'sa-east-1' => 'BR',
  'us-gov-west-1' => 'GV',
  'cn-north-1' => 'CN',
}

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config_path = nil) ⇒ AwsManager

Returns a new instance of AwsManager.



42
43
44
 
# File 'lib/reyes/aws_manager.rb', line 42

def initialize(config_path=nil)
  @config ||= Reyes::Config.new(config_path)
end

Class Method Details

.with_retries(retries = 5, delay = 2) ⇒ Object

Raises:

  • (ArgumentError) 


26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/reyes/aws_manager.rb', line 26

def self.with_retries(retries=5, delay=2)
  raise ArgumentError.new('Block is required') unless block_given?
  begin
    yield
  rescue AWS::EC2::Errors::RequestLimitExceeded => err
    log.warn(err.inspect)
    if retries > 0
      retries -= 1
      sleep delay
      retry
    else
      raise
    end
  end
end

Instance Method Details

#connections ⇒ Object 



58
59
60
 
# File 'lib/reyes/aws_manager.rb', line 58

def connections
  @connections ||= connect!
end

#dump_fake_data(filename) ⇒ Object 



261
262
263
264
265
266
267
268
269
270
 
# File 'lib/reyes/aws_manager.rb', line 261

def dump_fake_data(filename)
  log.info("Dumping AWS data to #{filename.inspect}")
  data = generate_fake_data

  File.open(filename, File::WRONLY | File::CREAT | File::EXCL) do |fh|
    fh.write(JSON.pretty_generate(data))
  end

  log.info('Wrote data to file')
end

#ec2(region) ⇒ Object 



46
47
48
 
# File 'lib/reyes/aws_manager.rb', line 46

def ec2(region)
  connections.fetch(:ec2).fetch(region)
end

#generate_fake_data ⇒ Hash

Generate AWS data suitable for offline rule processing.

The data will include EC2 instance and security group information for each region. (see ‘#regions`)

Generated DATA will be a hash including these keys:

DATA:

'metadata' => hash including information about the generation process
'vpcs' => A mapping of VPC ID => VPC_DATA
'classic_cidr_blocks' => from config: list of EC2 classic CIDR blocks
'excluded_group_names' => from config: list of security group names
  to ignore
'security_groups_by_name' => An index of global security groups by
  name. See SG_GROUPS_BY_NAME below.
'regions' => A hash of {region_name => REGION_DATA}

VPC_DATA: information about a VPC, including:

- region
- cidr_block

SG_GROUPS_BY_NAME: a mapping of => SG_REFS

some_group_name:
  - region: us-east-1
    group_id: sg-1234eeee
    vpc: vpc-eeeeeeee
  - region: us-west-1
    group_id: sg-1234cccc
    vpc: null

REGION_DATA

'instances' => mapping of {instance_id => INSTANCE_DATA}
'security_groups' => mapping of {group_id => SG_DATA}

INSTANCE_DATA: information about an instance, including:

- tags
- region
- vpc (ID)
- availability_zone
- private_ip_address
- public_ip_address
- security_groups (IDs)
- security_group_names

SG_DATA: information about a security group, including:

- name
- description
- vpc (ID)
- region
- ipset_suffix (a name appropriate for an IPset)
- ingress_ip_permissions => list of IP_PERMISSION_DATA
- instances (IDs)

IP_PERMISSION_DATA: information about an IP Permission, including:

- protocol
- port_start
- port_end
- ip_ranges => list of CIDR block strings
- group_names => list of security group names

Returns:

  • (Hash) 


209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
 
# File 'lib/reyes/aws_manager.rb', line 209

def generate_fake_data
  log.info('Generating AWS data for serialization')
  start = Time.now.utc
  data = {
    'metadata' => {
      'format_version' => Reyes::JSON_FORMAT_VERSION,
      'generated' => start,
      'generated_stamp' => start.to_i,
      'hostname' => Socket.gethostname,
      'pid' => Process.pid,
    },
    'vpcs' => {},
    'classic_cidr_blocks' => aws_config.fetch('classic_cidr_blocks'),
    'excluded_group_names' => aws_config.fetch('excluded_group_names'),
    'security_groups_by_name' => {},
    'regions' => {},
  }

  vpcs.each do |vpc|
    data['vpcs'][vpc.vpc_id] = {
      'region' => vpc.client.config.ec2_region,
      'cidr_block' => vpc.cidr_block,
    }
  end

  regions.each do |region|
    data['regions'][region] = fake_data_for_region(region)
  end

  # populate security_groups_by_name
  data['regions'].each_pair do |region, rdata|
    rdata.fetch('security_groups').each_pair do |sg_id, sg_data|
      data['security_groups_by_name'][sg_data.fetch('name')] ||= []
      data['security_groups_by_name'][sg_data.fetch('name')] << {
        'region' => region,
        'group_id' => sg_id,
        'vpc' => sg_data.fetch('vpc_id'),
      }
    end
  end

  data['metadata']['generated_in'] = Time.now - start

  log.info("Generated AWS data in #{(Time.now - start).round(1)}s")

  data
end

#generate_fake_data_json ⇒ Object 



257
258
259
 
# File 'lib/reyes/aws_manager.rb', line 257

def generate_fake_data_json
  JSON.pretty_generate(generate_fake_data)
end

#instances_in_security_group(sg) ⇒ Array<AWS::EC2::Instance>

List instances in a given VPC security group. Use this method to get better cache behavior for repeated calls to list security group members in a VPC.

Parameters:

  • sg (AWS::EC2::SecurityGroup)

Returns:

  • (Array<AWS::EC2::Instance>) 


113
114
115
116
117
118
119
 
# File 'lib/reyes/aws_manager.rb', line 113

def instances_in_security_group(sg)
  unless sg.vpc
    raise ArgumentError.new("SG #{sg.security_group_id} is not in VPC")
  end

  sg.vpc.instances.find_all {|i| i.security_groups.include?(sg)}
end

#regions ⇒ Object 



62
63
64
 
# File 'lib/reyes/aws_manager.rb', line 62

def regions
  aws_config.fetch('regions')
end

#s3 ⇒ Object 



50
51
52
53
54
55
56
 
# File 'lib/reyes/aws_manager.rb', line 50

def s3
  @s3 ||= AWS::S3.new({
    access_key_id: @config.aws_credentials.fetch(:access_key_id),
    secret_access_key: @config.aws_credentials.fetch(:secret_access_key),
    logger: Chalk::Log::Logger.new("s3"),
  })
end

#security_groups(region) ⇒ Object 



84
85
86
 
# File 'lib/reyes/aws_manager.rb', line 84

def security_groups(region)
  ec2(region).security_groups.to_a
end

#vpc_security_groups_by_name(name) ⇒ Array<AWS::EC2::SecurityGroup>

Look up foreign VPC security groups by name.

Does not use native AWS API filtering to promote better cache behavior.

Parameters:

  • name (String)

Returns:

  • (Array<AWS::EC2::SecurityGroup>) 


95
96
97
98
99
100
101
102
103
 
# File 'lib/reyes/aws_manager.rb', line 95

def vpc_security_groups_by_name(name)
  unless name.is_a?(String)
    raise ArgumentError.new("#{name.inspect} must be a String")
  end

  vpcs.map { |vpc|
    vpc.security_groups.find_all {|sg| sg.name == name }
  }.flatten
end

#vpcs ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/reyes/aws_manager.rb', line 66

def vpcs
  return @vpcs if @vpcs

  @vpcs = aws_config.fetch('vpcs').map do |row|
    unless row.length == 2
      raise Error.new("Invalid VPC row: #{row.inspect}")
    end

    region, vpc_id = row
    vpc = ec2(region).vpcs[vpc_id]

    # warm cidr_block cache & ensure VPC exists
    vpc.cidr_block

    vpc
  end
end

#warm_sg_cache ⇒ Object 



121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/reyes/aws_manager.rb', line 121

def warm_sg_cache
  unless AWS.memoizing?
    log.warn("Calling warm_sg_cache doesn't make sense unless memoizing")
  end
  log.info("Warming security group caches")
  vpcs.each do |vpc|
    log.debug("Warming security group cache for #{vpc.id}")
    sg = vpc.security_groups.to_a.first
    instances_in_security_group(sg)
  end
end