Class: Reyes::AwsManager

Inherits:
Object
  • Object
show all
Includes:
Chalk::Log
Defined in:
lib/reyes/aws_manager.rb

Defined Under Namespace

Classes: Error

Constant Summary collapse

JSON_NOT_AFTER =

Validity period for signed JSON documents

3600
RegionShortNames =

Short names for AWS regions to save space in ipset names

{
  'us-east-1' => 'VA',
  'us-west-2' => 'OR',
  'us-west-1' => 'CA',
  'eu-west-1' => 'IE',
  'eu-central-1' => 'DE',
  'ap-southeast-1' => 'SG',
  'ap-southeast-2' => 'AU',
  'ap-northeast-1' => 'JP',
  'sa-east-1' => 'BR',
  'us-gov-west-1' => 'GV',
  'cn-north-1' => 'CN',
}

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config) ⇒ AwsManager

Returns a new instance of AwsManager.



45
46
47
 
# File 'lib/reyes/aws_manager.rb', line 45

def initialize(config)
  @config = config
end

Class Method Details

.with_retries(retries = 5, delay = 2) ⇒ Object

Raises:

  • (ArgumentError) 


29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/reyes/aws_manager.rb', line 29

def self.with_retries(retries=5, delay=2)
  raise ArgumentError.new('Block is required') unless block_given?
  begin
    yield
  rescue AWS::EC2::Errors::RequestLimitExceeded => err
    log.warn(err.inspect)
    if retries > 0
      retries -= 1
      sleep delay
      retry
    else
      raise
    end
  end
end

Instance Method Details

#connections ⇒ Object 



61
62
63
 
# File 'lib/reyes/aws_manager.rb', line 61

def connections
  @connections ||= connect!
end

#dump_fake_data(filename) ⇒ Object 



265
266
267
268
269
270
271
272
273
274
 
# File 'lib/reyes/aws_manager.rb', line 265

def dump_fake_data(filename)
  log.info("Dumping AWS data to #{filename.inspect}")
  data = generate_fake_data

  File.open(filename, File::WRONLY | File::CREAT | File::EXCL) do |fh|
    fh.write(JSON.pretty_generate(data))
  end

  log.info('Wrote data to file')
end

#ec2(region) ⇒ Object 



49
50
51
 
# File 'lib/reyes/aws_manager.rb', line 49

def ec2(region)
  connections.fetch(:ec2).fetch(region)
end

#generate_fake_data ⇒ Hash

Generate AWS data suitable for offline rule processing.

The data will include EC2 instance and security group information for each region. (see ‘#regions`)

Generated DATA will be a hash including these keys:

DATA:

'metadata' => hash including information about the generation process
'vpcs' => A mapping of VPC ID => VPC_DATA
'classic_cidr_blocks' => from config: list of EC2 classic CIDR blocks
'excluded_group_names' => from config: list of security group names
  to ignore
'security_groups_by_name' => An index of global security groups by
  name. See SG_GROUPS_BY_NAME below.
'regions' => A hash of {region_name => REGION_DATA}

VPC_DATA: information about a VPC, including:

- region
- cidr_block

SG_GROUPS_BY_NAME: a mapping of => SG_REFS

some_group_name:
  - region: us-east-1
    group_id: sg-1234eeee
    vpc: vpc-eeeeeeee
  - region: us-west-1
    group_id: sg-1234cccc
    vpc: null

REGION_DATA

'instances' => mapping of {instance_id => INSTANCE_DATA}
'security_groups' => mapping of {group_id => SG_DATA}

INSTANCE_DATA: information about an instance, including:

- tags
- region
- vpc (ID)
- availability_zone
- private_ip_address
- public_ip_address
- security_groups (IDs)
- security_group_names

SG_DATA: information about a security group, including:

- name
- description
- vpc (ID)
- region
- ipset_suffix (a name appropriate for an IPset)
- ingress_ip_permissions => list of IP_PERMISSION_DATA
- instances (IDs)

IP_PERMISSION_DATA: information about an IP Permission, including:

- protocol
- port_start
- port_end
- ip_ranges => list of CIDR block strings
- group_names => list of security group names

Returns:

  • (Hash) 


212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
 
# File 'lib/reyes/aws_manager.rb', line 212

def generate_fake_data
  log.info('Generating AWS data for serialization')
  start = Time.now.utc
  data = {
    'metadata' => {
      'format_version' => Reyes::JSON_FORMAT_VERSION,
      'generated' => start,
      'generated_stamp' => start.to_i,
      'hostname' => Socket.gethostname,
      'pid' => Process.pid,
      'not_after_stamp' => Time.now.to_i + JSON_NOT_AFTER,
    },
    'vpcs' => {},
    'classic_cidr_blocks' => aws_config.fetch('classic_cidr_blocks'),
    'excluded_group_names' => aws_config.fetch('excluded_group_names'),
    'security_groups_by_name' => {},
    'regions' => {},
  }

  vpcs.each do |vpc|
    data['vpcs'][vpc.vpc_id] = {
      'region' => vpc.client.config.ec2_region,
      'cidr_block' => vpc.cidr_block,
    }
  end

  regions.each do |region|
    data['regions'][region] = fake_data_for_region(region)
  end

  # populate security_groups_by_name
  data['regions'].each_pair do |region, rdata|
    rdata.fetch('security_groups').each_pair do |sg_id, sg_data|
      data['security_groups_by_name'][sg_data.fetch('name')] ||= []
      data['security_groups_by_name'][sg_data.fetch('name')] << {
        'region' => region,
        'group_id' => sg_id,
        'vpc' => sg_data.fetch('vpc_id'),
      }
    end
  end

  data['metadata']['generated_in'] = Time.now - start

  log.info("Generated AWS data in #{(Time.now - start).round(1)}s")

  data
end

#generate_fake_data_json ⇒ Object 



261
262
263
 
# File 'lib/reyes/aws_manager.rb', line 261

def generate_fake_data_json
  JSON.pretty_generate(generate_fake_data)
end

#instances_in_security_group(sg) ⇒ Array<AWS::EC2::Instance>

List instances in a given VPC security group. Use this method to get better cache behavior for repeated calls to list security group members in a VPC.

Parameters:

  • sg (AWS::EC2::SecurityGroup)

Returns:

  • (Array<AWS::EC2::Instance>) 


116
117
118
119
120
121
122
 
# File 'lib/reyes/aws_manager.rb', line 116

def instances_in_security_group(sg)
  unless sg.vpc
    raise ArgumentError.new("SG #{sg.security_group_id} is not in VPC")
  end

  sg.vpc.instances.find_all {|i| i.security_groups.include?(sg)}
end

#regions ⇒ Object 



65
66
67
 
# File 'lib/reyes/aws_manager.rb', line 65

def regions
  aws_config.fetch('regions')
end

#s3 ⇒ Object 



53
54
55
56
57
58
59
 
# File 'lib/reyes/aws_manager.rb', line 53

def s3
  @s3 ||= AWS::S3.new({
    access_key_id: @config.aws_credentials.fetch(:access_key_id),
    secret_access_key: @config.aws_credentials.fetch(:secret_access_key),
    logger: Chalk::Log::Logger.new("s3"),
  })
end

#security_groups(region) ⇒ Object 



87
88
89
 
# File 'lib/reyes/aws_manager.rb', line 87

def security_groups(region)
  ec2(region).security_groups.to_a
end

#vpc_security_groups_by_name(name) ⇒ Array<AWS::EC2::SecurityGroup>

Look up foreign VPC security groups by name.

Does not use native AWS API filtering to promote better cache behavior.

Parameters:

  • name (String)

Returns:

  • (Array<AWS::EC2::SecurityGroup>) 


98
99
100
101
102
103
104
105
106
 
# File 'lib/reyes/aws_manager.rb', line 98

def vpc_security_groups_by_name(name)
  unless name.is_a?(String)
    raise ArgumentError.new("#{name.inspect} must be a String")
  end

  vpcs.map { |vpc|
    vpc.security_groups.find_all {|sg| sg.name == name }
  }.flatten
end

#vpcs ⇒ Object 



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/reyes/aws_manager.rb', line 69

def vpcs
  return @vpcs if @vpcs

  @vpcs = aws_config.fetch('vpcs').map do |row|
    unless row.length == 2
      raise Error.new("Invalid VPC row: #{row.inspect}")
    end

    region, vpc_id = row
    vpc = ec2(region).vpcs[vpc_id]

    # warm cidr_block cache & ensure VPC exists
    vpc.cidr_block

    vpc
  end
end

#warm_sg_cache ⇒ Object 



124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/reyes/aws_manager.rb', line 124

def warm_sg_cache
  unless AWS.memoizing?
    log.warn("Calling warm_sg_cache doesn't make sense unless memoizing")
  end
  log.info("Warming security group caches")
  vpcs.each do |vpc|
    log.debug("Warming security group cache for #{vpc.id}")
    sg = vpc.security_groups.to_a.first
    instances_in_security_group(sg)
  end
end