Module: Sumomo::Stack

Defined in:
lib/sumomo/api.rb,
lib/sumomo/cdn.rb,
lib/sumomo/dns.rb,
lib/sumomo/ec2.rb,
lib/sumomo/ecs.rb,
lib/sumomo/stack.rb,
lib/sumomo/network.rb

Defined Under Namespace

Classes: APIGenerator, EC2Tasks

Instance Method Summary collapse

Instance Method Details

#allow_port(thing) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/sumomo/ec2.rb', line 13

def allow_port(thing)
  if (thing == :all)
    {
      "IpProtocol" => "-1",
      "ToPort" => 65535,
      "FromPort" => 0,
      "CidrIp" => "0.0.0.0/0"
    }
  elsif thing.is_a? Integer and thing > 0 and thing < 65536
    # its a port!
    {
      "IpProtocol" => "tcp",
      "ToPort" => thing,
      "FromPort" => thing,
      "CidrIp" => "0.0.0.0/0"
    }
  elsif thing.is_a? String and /[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\/[0-9]+/.match(thing)
    # its a cidr!
    {
      "IpProtocol" => "tcp",
      "ToPort" => 65535,
      "FromPort" => 0,
      "CidrIp" => thing
    }
  elsif thing.is_a? Hash
    # more shit
    {
      "IpProtocol" => thing[:protocol] || "tcp",
      "ToPort" => thing[:port] || thing[:end_port] || 0,
      "FromPort" => thing[:port] || thing[:start_port] || 65535,
      "CidrIp" => thing[:cidr] || "0.0.0.0/0"
    }
  else
    raise "utils.rb allow: please allow something"
  end
end

#cloudflare_dns(key:, email:) ⇒ Object 



188
189
190
 
# File 'lib/sumomo/api.rb', line 188

def cloudflare_dns(key:, email:)
    {type: :cloudflare, key: key, email: email}
end

#cloudflare_hosted_zone(domain_name:, key:, email:) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/sumomo/dns.rb', line 5

def cloudflare_hosted_zone(domain_name:,key:,email:)
  root_name = /(?<root_name>[^.]+\.[^.]+)$/.match(domain_name)[:root_name]

  hz = make "AWS::Route53::HostedZone" do
    Name domain_name
  end

  for i in 0..3
    make "Custom::CloudflareDNSEntry" do
      Key key
      Email email
      Domain root_name
      Entry domain_name.sub(/#{root_name}$/, "").chomp(".")
      NS hz.NameServers[i]
    end
  end

  return hz
end

#define_custom_resource(name: nil, code:) ⇒ Object 



71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/sumomo/stack.rb', line 71

def define_custom_resource(name: nil,code:)

  name ||= make_default_resource_name("CustomResource")

  func = make_lambda(
    name: name, 
    files:[
      {
        name: "index.js", 
        code: File.read( File.join(Gem.loaded_specs['sumomo'].full_gem_path, "data", "sumomo", "custom_resource_utils.js") ).sub("{{ CODE }}", code)
      }
    ],
    description: "CF Resource Custom::#{name}",
    function_key: "cloudformation/custom_resources/function_#{name}")

  @custom_resources["Custom::#{name}"] = func
end

#elb_health_check(port: 80, healthy_threshold: 2, interval: 10, timeout: 5, unhealthy_threshold: 10, path: "/", check_type: "HTTP") ⇒ Object 



77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/sumomo/ec2.rb', line 77

def elb_health_check(port: 80, 
  healthy_threshold: 2, 
  interval: 10, 
  timeout: 5, 
  unhealthy_threshold: 10, 
  path: "/", 
  check_type: "HTTP")

  options[:path] = "/#{options[:path]}"
  options[:path].gsub!(/^[\/]+/, "/")
  {
    "HealthyThreshold" => options[:healthy_threshold] || 2,
    "Interval" => options[:interval] || 10,
    "Target" => "#{check_type}:#{port}#{options[:path]}",
    "Timeout" => options[:timeout] || 5,
    "UnhealthyThreshold" => options[:unhealthy_threshold] || 10
  }
end

#elb_tcp_health_check(port: 80, healthy_threshold: 2, interval: 10, timeout: 5, unhealthy_threshold: 10, path: "/") ⇒ Object 



67
68
69
70
71
72
73
74
75
 
# File 'lib/sumomo/ec2.rb', line 67

def elb_tcp_health_check(port: 80, healthy_threshold: 2, interval: 10, timeout: 5, unhealthy_threshold: 10, path: "/")
  elb_health_check(port: port, 
    healthy_threshold: healthy_threshold, 
    interval: interval, 
    timeout: timeout, 
    unhealthy_threshold: unhealthy_threshold, 
    path: path, 
    check_type: "TCP")
end

#exec_role(with_statements: []) ⇒ Object 



119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
 
# File 'lib/sumomo/stack.rb', line 119

def exec_role(with_statements: [])

  if @exec_roles == nil
    @exec_roles = {}
  end

  statement_key = JSON.parse(with_statements.to_json)

  if !@exec_roles.has_key?(statement_key)
    name = make_default_resource_name("LambdaExecRole")

    role_policy_doc = {
      "Version" => "2012-10-17",
      "Statement" => [{
        "Effect" => "Allow",
        "Principal" => {"Service" => ["lambda.amazonaws.com"]},
        "Action" => ["sts:AssumeRole"]
      }]
    }

    bucket_name = @bucket_name

    statement_list = [
      {
        "Effect" => "Allow",
        "Action" => ["logs:CreateLogStream","logs:PutLogEvents"],
        "Resource" => "arn:aws:logs:*:*:*"
      },
      {
        "Effect" => "Allow",
        "Action" => ["cloudformation:DescribeStacks", "ec2:Describe*", ],
        "Resource" => "*"
      },
      {
        "Effect" => "Allow",
        "Action" => ["s3:DeleteObject", "s3:GetObject", "s3:PutObject"],
        "Resource" => "arn:aws:s3:::#{bucket_name}/*"
      },
      {
        "Effect" => "Allow",
        "Action" => ["cloudfront:CreateCloudFrontOriginAccessIdentity", "cloudfront:DeleteCloudFrontOriginAccessIdentity"],
        "Resource" => "*"
      },
      {
        "Effect" => "Allow",
        "Action" => ["apigateway:*", "cloudfront:UpdateDistribution"],
        "Resource" => "*"
      },
      {
        "Effect" => "Allow",
        "Action" => ["acm:RequestCertificate", "acm:DeleteCertificate", "acm:DescribeCertificate"],
        "Resource" => "*"
      }] + with_statements

    @exec_roles[statement_key] = make "AWS::IAM::Role", name: name do
      AssumeRolePolicyDocument role_policy_doc
      Path "/"
      Policies [
        {
          "PolicyName" => name,
          "PolicyDocument" => {
            "Version" => "2012-10-17",
            "Statement" => statement_list
          }
        }
      ]
    end
  end

  @exec_roles[statement_key]

end

#get_azs ⇒ Object 



5
6
7
8
9
10
11
 
# File 'lib/sumomo/ec2.rb', line 5

def get_azs
  resp = @ec2.describe_availability_zones

  Array(resp.availability_zones.map do |x|
    x.zone_name
  end)
end

#hidden_value(value) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
 
# File 'lib/sumomo/stack.rb', line 5

def hidden_value(value)
  name = make_default_resource_name("HiddenValue")
  if !@hidden_values
    @hidden_values = []
  end

  @hidden_values << {
    parameter_key: name,
    parameter_value: value
  }

  param name, type: :string
end

#http_listener(port: 80, instance_port: port) ⇒ Object 



50
51
52
53
54
55
56
 
# File 'lib/sumomo/ec2.rb', line 50

def http_listener(port: 80, instance_port: port)
  {
    "LoadBalancerPort" => port,
    "InstancePort" => instance_port,
    "Protocol" => "HTTP"
  }
end

#https_listener(cert_arn:, instance_port: 80, port: 443) ⇒ Object 



58
59
60
61
62
63
64
65
 
# File 'lib/sumomo/ec2.rb', line 58

def https_listener(cert_arn:, instance_port: 80, port: 443)
  res = http_listener(instance_port)
  res["LoadBalancerPort"] = lb_port
  res["Protocol"] = "HTTPS"
  res["SSLCertificateId"] = cert_arn

  return res
end

#initscript(wait_handle, asgname, script) ⇒ Object 



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/sumomo/ec2.rb', line 96

def initscript(wait_handle, asgname, script)

  call("Fn::Base64", 
    call("Fn::Join", "", [

      "#!/bin/bash -v\n",
      "yum update -y aws-cfn-bootstrap\n",

      "# Helper function\n",
      "function error_exit\n",
      "{\n",
      "  /opt/aws/bin/cfn-signal -e 1 -r \"$1\" \"", wait_handle, "\"\n",
      "  exit 1\n",
      "}\n",

      "# Run init meta\n",
      "/opt/aws/bin/cfn-init -s ", ref("AWS::StackId"), " -r ", asgname, " ",
      "    --region ", ref("AWS::Region"), " || error_exit 'Failed to run cfn-init'\n",

      "# Run script\n",
      script,

      "\n",

      "# All is well so signal success\n",
      "/opt/aws/bin/cfn-signal -e 0 -r \"Setup complete\" \"", wait_handle, "\"\n"
    ]))
end

#make(type, options = {}, &block) ⇒ Object 



99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/sumomo/stack.rb', line 99

def make(type, options = {}, &block)
  match = /^Custom\:\:(?<name>[a-z0-9]+)/i.match(type)
  if match
    if !@custom_resources[type]

      resource_function_source = File.join(Gem.loaded_specs['sumomo'].full_gem_path, "data", "sumomo", "custom_resources", "#{match[:name]}.js")
      
      if File.exists? resource_function_source
        define_custom_resource(name: match[:name], code: File.read(resource_function_source))
      else
        throw "#{resource_function_source} does not exist"

      end
    end
    make_custom(@custom_resources[type], options, &block)
  else
    stack_make(type, options, &block)
  end
end

#make_api(domain_name, name:, script: nil, dns: nil, cert: nil, with_statements: [], &block) ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
 
# File 'lib/sumomo/api.rb', line 75

def make_api(domain_name, name:, script:nil, dns:nil, cert:nil, with_statements:[], &block)

    api = make "AWS::ApiGateway::RestApi", name: name do
        Name name
    end

    script ||= File.read(File.join(Gem.loaded_specs['sumomo'].full_gem_path, "data", "sumomo", "api_modules", "real_script.js"))

    apigen = APIGenerator.new(&block);
    script.sub!("// {{ ROUTES }}", apigen.generate);

    script.gsub!("{{ REGION }}", @region);
    script.gsub!("{{ BUCKET }}", @bucket_name);
    script.gsub!("{{ STORE_PREFIX }}", "functions/" + name);

    module_dir = File.join(Gem.loaded_specs['sumomo'].full_gem_path, "data", "sumomo", "api_modules")

    files = Dir[File.join(module_dir, "**/*")].select{|x| File.file?(x)}.map do |x| 
        { name: x.sub(/^#{module_dir}\//, ""), code: File.read(x) } 
    end

    files += [ {name:"index.js", code:script} ]

    fun = make_lambda(name: "#{name}Lambda#{@version_number}", files:files, with_statements:with_statements)

    resource = make "AWS::ApiGateway::Resource", name: "#{name}Resource" do
        ParentId api.RootResourceId
        PathPart "{proxy+}"
        RestApiId api
    end

    meth = make "AWS::ApiGateway::Method", name: "#{name}MethodOther" do
        RestApiId api
        ResourceId resource
        HttpMethod "ANY"
        AuthorizationType "NONE"
        Integration ({
            Type: "AWS_PROXY",
            IntegrationHttpMethod: "POST",
            Uri: call("Fn::Join", "", ["arn:aws:apigateway:" ,ref("AWS::Region") ,":lambda:path", "/2015-03-31/functions/", fun.Arn, "/invocations"])
        })
    end

    meth2 = make "AWS::ApiGateway::Method", name: "#{name}MethodRoot" do
        RestApiId api
        ResourceId api.RootResourceId
        HttpMethod "ANY"
        AuthorizationType "NONE"
        Integration ({
            Type: "AWS_PROXY",
            IntegrationHttpMethod: "POST",
            Uri: call("Fn::Join", "", ["arn:aws:apigateway:" ,ref("AWS::Region") ,":lambda:path", "/2015-03-31/functions/", fun.Arn, "/invocations"])
        })
    end

    make "AWS::Lambda::Permission", name: "#{name}LambdaPermission" do
        FunctionName fun.Arn
        Action "lambda:InvokeFunction"
        Principal "apigateway.amazonaws.com"
    end

    deployment = make "AWS::ApiGateway::Deployment", name: "#{name}Deployment#{@version_number}" do
        depends_on meth
        depends_on meth2
        RestApiId api
    end

    stage = make "AWS::ApiGateway::Stage", name: "#{name}Stage" do
        RestApiId api
        DeploymentId deployment
        StageName "test"
    end

    root_name = /(?<root_name>[^.]+\.[^.]+)$/.match(domain_name)[:root_name]

    cert ||= make "Custom::USEastCertificate", name: "#{name}Certificate" do
        DomainName domain_name
    end

    domain = make "Custom::APIDomainName", name: "#{name}DomainName" do
        DomainName domain_name
        CertificateArn cert
    end

    make "AWS::ApiGateway::BasePathMapping", name: "#{name}BasePathMapping" do
        BasePath "(none)"
        DomainName domain
        RestApiId api
        Stage stage
    end

    if dns[:type] == :cloudflare
        make "Custom::CloudflareDNSEntry", name: "#{name}CloudFlareEntry" do
            Key dns[:key]
            Email dns[:email]
            Domain root_name
            Entry domain_name.sub(/#{root_name}$/, "").chomp(".")
            CNAME call("Fn::Join", "", [api, ".execute-api.", ref("AWS::Region"), ".amazonaws.com"])
        end
        domain_name
    elsif dns[:type] == :route53
        make "AWS::Route53::RecordSet", name: "#{name}Route53Entry" do
            HostedZoneId dns[:hosted_zone]
            Name domain_name
            Type "CNAME"
        end
        domain_name
    else
        call("Fn::Join", "", [api, ".execute-api.", ref("AWS::Region"), ".amazonaws.com"])
    end
    
end

#make_autoscaling_group(network:, layer:, zone: nil, type: "m3.medium", name: nil, elb: nil, min_size: 1, max_size: min_size, vol_size: 10, vol_type: "gp2", keypair: @master_key_name, has_public_ips: true, ingress: nil, egress: nil, machine_tag: nil, ec2_sns_arn: nil, ami_name: nil, ebs_root_device: nil, spot_price: nil, script: nil, ecs_cluster: nil, docker_username: "", docker_email: "", docker_password: "", eip: nil, policies: [], &block) ⇒ Object 



247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
 
# File 'lib/sumomo/ec2.rb', line 247

def make_autoscaling_group(
  network:, 
  layer:,
  zone:nil,
  type:"m3.medium", 
  name:nil,
  elb:nil,
  min_size:1,
  max_size:min_size,
  vol_size:10,
  vol_type:"gp2",
  keypair:@master_key_name,
  has_public_ips:true,
  ingress:nil,
  egress:nil,
  machine_tag:nil,
  ec2_sns_arn:nil,
  ami_name:nil,
  ebs_root_device:nil,
  spot_price:nil,
  script: nil,
  ecs_cluster: nil,
  docker_username:"",
  docker_email:"",
  docker_password: "",
  eip:nil,
  policies:[],
  &block)

  if ami_name == nil

    @ami_lookup_resources ||= {}

    if !@ami_lookup_resources[type]
      @ami_lookup_resources[type] = make "Custom::AMILookup" do
        InstanceType type
      end
    end

    ami_name = @ami_lookup_resources[type]
    ebs_root_device = @ami_lookup_resources[type].RootDeviceName if ebs_root_device == nil
  end

  tasks = EC2Tasks.new(@bucket_name, &block)

  task_script = tasks.script

  ingress ||= [ allow_port(:all) ]
  egress ||= [ allow_port(:all) ]
  machine_tag ||= ref("AWS::StackName")
  name ||= make_default_resource_name("AutoScalingGroup")
  script ||= ""

  bucket_name = @bucket_name

  script += "\n#{task_script}\n"

  if ecs_cluster
    script += "\nyum update\nyum groupinstall \"Development Tools\"\nyum install -y python screen git gcc-c++ ecs-init\ncurl -sSL https://get.docker.com/ | sh\n\ncp /ecs.config /etc/ecs/ecs.config\n\nservice docker start\nstart ecs\n\ncurl http://localhost:51678/v1/metadata > /home/ec2-user/ecs_info\n\n    ECS_START\n  end\n\n  if eip\n    script += <<-EIP_ALLOCATE\naws ec2 associate-address --region `cat /etc/aws_region` --instance-id `curl http://169.254.169.254/latest/meta-data/instance-id` --allocation-id `cat /etc/eip_allocation_id`\n    EIP_ALLOCATE\n  end\n\n  script += \"\\nservice spot-watcher start\" if spot_price and ec2_sns_arn\n\n  raise \"ec2: ingress option needs to be an array\" if !ingress.is_a? Array\n  raise \"ec2: egress option needs to be an array\" if !egress.is_a? Array\n\n  web_sec_group = make \"AWS::EC2::SecurityGroup\" do\n    GroupDescription \"Security group for layer: \#{layer}\"\n    SecurityGroupIngress ingress\n    SecurityGroupEgress egress\n    VpcId network.vpc\n  end\n\n  wait_handle = make \"AWS::CloudFormation::WaitConditionHandle\"\n\n  user_data = initscript(wait_handle, name, script)\n\n  role_policy_doc = {\n    \"Version\" => \"2012-10-17\",\n    \"Statement\" => [{\n      \"Effect\" => \"Allow\",\n      \"Principal\" => {\"Service\" => [\"ec2.amazonaws.com\"]},\n      \"Action\" => [\"sts:AssumeRole\"]\n    }]\n  }\n\n  asg_role = make \"AWS::IAM::Role\" do\n    AssumeRolePolicyDocument role_policy_doc\n    Path \"/\"\n    Policies [{\n      \"PolicyName\" => \"root\",\n      \"PolicyDocument\" => {\n        \"Version\" => \"2012-10-17\",\n        \"Statement\" => [{\n          \"Effect\" => \"Allow\",\n          \"Action\" => [\"sns:Publish\"],\n          \"Resource\" => \"*\"\n        },\n        {\n          \"Effect\" => \"Allow\",\n          \"Action\" => [\"s3:DeleteObject\", \"s3:GetObject\", \"s3:PutObject\"],\n          \"Resource\" => \"arn:aws:s3:::\#{bucket_name}/uploads/*\"\n        },\n        {\n          \"Effect\" => \"Allow\",\n          \"Action\" => [\n            \"ec2:AllocateAddress\", \n            \"ec2:AssociateAddress\", \n            \"ec2:DescribeAddresses\", \n            \"ec2:DisassociateAddress\"\n          ],\n          \"Resource\" => \"*\"\n        },\n        {\n          \"Effect\" => \"Allow\",\n          \"Action\" => [\n            \"ecs:DeregisterContainerInstance\",\n            \"ecs:DiscoverPollEndpoint\",\n            \"ecs:Poll\",\n            \"ecs:RegisterContainerInstance\",\n            \"ecs:StartTelemetrySession\",\n            \"ecs:Submit*\",\n            \"ecr:GetAuthorizationToken\",\n            \"ecr:BatchCheckLayerAvailability\",\n            \"ecr:GetDownloadUrlForLayer\",\n            \"ecr:BatchGetImage\",\n            \"logs:CreateLogStream\",\n            \"logs:PutLogEvents\"\n          ],\n          \"Resource\": \"*\"\n        }] + policies\n      }\n    }]\n  end\n\n  asg_profile = make \"AWS::IAM::InstanceProfile\" do\n    Path \"/\"\n    Roles [ asg_role ]\n  end\n\n  launch_config = make \"AWS::AutoScaling::LaunchConfiguration\" do\n    AssociatePublicIpAddress has_public_ips\n    KeyName keypair \n    SecurityGroups [ web_sec_group ]\n    ImageId ami_name\n    UserData user_data\n    InstanceType type\n    IamInstanceProfile asg_profile\n    SpotPrice spot_price if spot_price\n    BlockDeviceMappings [{\n        \"DeviceName\" => ebs_root_device,\n        \"Ebs\" => {\n            \"VolumeType\" => vol_type,\n            \"VolumeSize\" => vol_size,\n          }\n      }]\n  end\n\n  zones_used = network.azs\n  subnet_ids = network.subnets[layer].map { |x| x[:name] }\n\n  if zone\n    # if we only specified a single zone, then we have to do some processing\n    res = define_custom_resource(name: \"SubnetIdentifierCodeFor\#{name}\", code: <<-CODE\n      var ids = {};\n      var zones = request.ResourceProperties.SubnetZones;\n      for (var i=0;i<zones.length;i++)\n      {\n        ids[zones[i]] = request.ResourceProperties.SubnetIds[i];\n      }\n\n      Cloudformation.send(request, context, Cloudformation.SUCCESS, {}, \"Success\", ids[request.ResourceProperties.Zone]);\n    CODE\n    )\n\n    identifier = make_custom res, name: \"SubnetIdentifierFor\#{name}\" do\n      SubnetIds network.subnets[layer].map { |x| x[:name] }\n      SubnetZones network.subnets[layer].map { |x| x[:zone] }\n      Zone zone\n    end\n\n    zones_used = [ zone ]\n    subnet_ids = [ identifier ]\n  end\n\n\n  asg = make \"AWS::AutoScaling::AutoScalingGroup\", name: name do\n    depends_on network.attachment\n\n    AvailabilityZones zones_used\n\n    Cooldown 30\n    MinSize min_size\n    MaxSize max_size\n\n    VPCZoneIdentifier subnet_ids\n\n    LaunchConfigurationName launch_config\n    LoadBalancerNames [ elb ] if elb\n\n    NotificationConfigurations [\n      {\n        \"NotificationTypes\" => [\n          \"autoscaling:EC2_INSTANCE_LAUNCH\", \n          \"autoscaling:EC2_INSTANCE_LAUNCH_ERROR\", \n          \"autoscaling:EC2_INSTANCE_TERMINATE\", \n          \"autoscaling:EC2_INSTANCE_TERMINATE_ERROR\",\n          \"autoscaling:TEST_NOTIFICATION\"\n        ],\n        \"TopicARN\" => ec2_sns_arn\n      }\n    ] if ec2_sns_arn\n\n    file \"/etc/aws_region\", content: \"{{ region }}\", context: {\n      region: ref(\"AWS::Region\")\n    }\n\n    if ec2_sns_arn\n      file \"/etc/sns_arn\", content: \"{{ sns_arn }}\", context: {\n        sns_arn: ec2_sns_arn\n      }\n    end\n\n    if eip\n      file \"/etc/eip_allocation_id\", content: \"{{ id }}\", context: {\n        id: eip.AllocationId\n      }\n    end\n\n    if spot_price and ec2_sns_arn\n      watcher = File.read( File.join( Gem.loaded_specs['sumomo'].full_gem_path, \"data\", \"sumomo\", \"sources\", \"spot-watcher.sh\" ) )\n      poller = File.read( File.join( Gem.loaded_specs['sumomo'].full_gem_path, \"data\", \"sumomo\", \"sources\", \"spot-watcher-poller.sh\" ) )\n\n      file \"/etc/init.d/spot-watcher\", content: watcher, mode: \"000700\"\n      file \"/bin/spot-watcher\", content: poller, mode: \"000700\", context: {\n        sns_arn: ec2_sns_arn,\n        region: ref(\"AWS::Region\")\n      }\n    end\n\n    if ecs_cluster\n      ecs_config = <<-CONFIG\nECS_CLUSTER={{cluster_name}}\nECS_ENGINE_AUTH_TYPE=docker\nECS_ENGINE_AUTH_DATA={\"https://index.docker.io/v1/\":{\"username\":\"{{docker_username}}\",\"password\":\"{{docker_password}}\",\"email\":\"{{docker_email}}\"}}\n      CONFIG\n\n      file \"/ecs.config\", content: ecs_config, context: {\n        cluster_name: ecs_cluster,\n        docker_username: docker_username,\n        docker_password: docker_password,\n        docker_email: docker_email\n      }\n    end\n\n    tag \"Name\", machine_tag, propagate_at_launch: true\n\n    tasks.tags.each do |t|\n      tag t[0], t[1], propagate_at_launch: true\n    end\n\n  end\n\n  asg\n\nend\n"

#make_cdn_from_dir(domain:, dir:, low_ttl: []) ⇒ Object 



4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/sumomo/cdn.rb', line 4

def make_cdn_from_dir(domain:, dir:, low_ttl: [])

    bucket_name = @bucket_name

    puts "Uploading files..."
    `aws s3 sync #{dir} "s3://#{bucket_name}/uploads/#{domain}" --size-only --delete`
    puts "Done."

    oai = make "Custom::OriginAccessIdentity"

    make "AWS::S3::BucketPolicy" do
        Bucket "#{bucket_name}"
        PolicyDocument({
            Version: "2008-10-17",
            Id: "PolicyForCloudFrontPrivateContent",
            Statement: [
                {
                    Effect: "Allow",
                    Principal: {
                        CanonicalUser: oai.S3CanonicalUserId
                    },
                    Action: "s3:GetObject",
                    Resource: "arn:aws:s3:::#{bucket_name}/uploads/#{domain}/*"
                }
            ]
        })
    end

    make "AWS::CloudFront::Distribution" do
        DistributionConfig do
            Origins [{
                Id: "originBucket",
                DomainName: "#{bucket_name}.s3.amazonaws.com",
                OriginPath: "/uploads/#{domain}",
                S3OriginConfig: {
                    OriginAccessIdentity: oai
                }
            }]

            CacheBehaviors low_ttl.map{ |pattern|
                {
                    PathPattern: pattern,
                    ForwardedValues: {
                        QueryString: "false",
                        Cookies: { Forward: "none" }
                    },
                    TargetOriginId: "originBucket",
                    ViewerProtocolPolicy: "allow-all",
                    DefaultTTL: 60,
                    MaxTTL: 60,
                    MinTTL: 60
                }
            }.to_a

            Enabled "true"
            DefaultRootObject "index.html"
            Aliases [ domain ]
            ViewerCertificate { CloudFrontDefaultCertificate "true" } 

            DefaultCacheBehavior do
                AllowedMethods ["GET", "HEAD", "OPTIONS"]
                TargetOriginId "originBucket"
                ViewerProtocolPolicy "allow-all"
                ForwardedValues {
                    QueryString "false"
                    Cookies { Forward "none" }
                }
            end

            Logging do
                IncludeCookies "false"
                Bucket "#{bucket_name}.s3.amazonaws.com"
                Prefix "logs/#{domain}/"
            end
        end
    end
end

#make_custom(custom_resource, options = {}, &block) ⇒ Object 



89
90
91
92
93
94
95
96
97
 
# File 'lib/sumomo/stack.rb', line 89

def make_custom(custom_resource, options = {}, &block)
  bucket_name = @bucket_name
  stack_make "Custom::#{custom_resource.name}", options do
    ServiceToken custom_resource.Arn
    Region ref("AWS::Region")
    Bucket bucket_name
    instance_eval(&block) if block
  end
end

#make_ecs_cluster(name: make_default_resource_name("ECSCluster"), services: [], machine_config: {}, network:, log_retention: 30, dependencies: []) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
 
# File 'lib/sumomo/ecs.rb', line 47

def make_ecs_cluster(name:make_default_resource_name("ECSCluster"),services:[],machine_config:{},network:,log_retention:30,dependencies:[])

  ecs = make "AWS::ECS::Cluster", name: "#{name}" do
    dependencies.each do |x|
      depends_on x
    end
  end

  volumes = []
  machine_volume_locations = {}

  service_number = 0

  services.each do |service|

    alb = service[:alb]
    certificate = service[:certificate]
    alb_ports = {}

    service_number += 1

    containers = service[:containers]
    service_name = service[:name] || "Service#{service_number}"
    service_count = service[:count] || 1

    container_defs = containers.map do |container|
      definition = {}

      definition["Name"] = "#{sluggify(container[:image]).camelize}"
      definition["Name"] = container[:name] if container[:name]

      definition["Memory"] = container[:memory] || 1024

      loggroup = make "AWS::Logs::LogGroup", name: "#{name}#{definition["Name"]}Logs" do
        LogGroupName "#{definition["Name"].underscore}_logs"
        RetentionInDays log_retention
      end

      definition["LogConfiguration"] = {
        "LogDriver" => "awslogs",
        "Options" => {
          "awslogs-group" => loggroup,
          "awslogs-region" => ref("AWS::Region")
        }
      }

      if container[:files]
        definition["MountPoints"] = container[:files].map do |file, destination|

          s3_location = "container_files/#{sluggify(service_name)}/#{definition["Name"]}/#{file}"
          volume_name = sluggify("#{definition["Name"].underscore}_#{destination}").camelize

          upload_file s3_location, File.read(file)

          machine_volume_locations[s3_location] = "/opt/s3/#{s3_location}"

          volumes << {
            "Name" => volume_name,
            "Host" => { "SourcePath" => machine_volume_locations[s3_location] }
          }

          {
            "ContainerPath" => destination,
            "SourceVolume" => volume_name
          }
        end
        container.delete(:files)
      end

      if container[:ports]
        if !alb
          definition["PortMappings"] = container[:ports].map do |from_port, to_port|
            {
              "ContainerPort" => from_port,
              "HostPort" => to_port
            }
          end

        else
          definition["PortMappings"] = container[:ports].map do |from_port, to_port|
            {
              "ContainerPort" => from_port
            }
          end

          container[:ports].each do |container_port, host_port|

            if alb_ports[host_port.to_i]
              raise "Container #{alb_ports[host_port][:name]} is already using #{host_port}"
            end

            alb_target = make "AWS::ElasticLoadBalancingV2::TargetGroup", name: "#{name}#{definition["Name"]}Target" do
              HealthCheckIntervalSeconds 60
              UnhealthyThresholdCount 10
              HealthCheckPath "/"
              Name "#{name}Port#{host_port}Target"
              Port container_port
              Protocol "HTTP"
              VpcId network[:vpc]

              if container[:alb_sticky]
                TargetGroupAttributes({
                  "stickiness.enabled" => true,
                  "stickiness.type" => "lb_cookie"
                }.map{|k,v| {Key: k, Value: v} })
                container.delete(:alb_sticky)
              end
            end

            alb_action = {
              "Type" => "forward",
              "TargetGroupArn" => alb_target
            }

            if certificate
              alb_listener = make "AWS::ElasticLoadBalancingV2::Listener", name: "#{name}#{definition["Name"]}Listener" do
                Certificates [{ CertificateArn: certificate }]
                DefaultActions [ alb_action ]
                LoadBalancerArn alb
                Port host_port
                Protocol "HTTPS"
              end
            else
              alb_listener = make "AWS::ElasticLoadBalancingV2::Listener", name: "#{name}#{definition["Name"]}Listener" do
                DefaultActions [ alb_action ]
                LoadBalancerArn alb
                Port host_port
                Protocol "HTTP"
              end
            end

            alb_ports[host_port.to_i] = {
              listener: alb_listener,
              target: alb_target,
              port: container_port,
              name: definition["Name"]
            }

          end

        end
        container.delete(:ports)
      end

      if container[:envvars]
        definition["Environment"] = container[:envvars].map do |var_name, var_value|
          {
            "Name" => var_name,
            "Value" => var_value
          }
        end
        container.delete(:envvars)
      end

      container.each do |key, value|
        definition["#{key}".camelize] = value
      end

      definition
    end 


    deployment_config = {
      "MaximumPercent" => 200,
      "MinimumHealthyPercent" => 50
    }

    ecs_task = make "AWS::ECS::TaskDefinition", name: "#{name}#{service_name}Task" do
      ContainerDefinitions container_defs
      Volumes volumes
    end

    stack = self

    ecs_service = make "AWS::ECS::Service", name: "#{name}#{service_name}" do
      alb_ports.each do |host_port, info|
        depends_on info[:listener]
      end

      Cluster ecs
      DesiredCount service_count
      TaskDefinition ecs_task
      DeploymentConfiguration deployment_config

      if alb_ports.keys.count != 0
        Role stack.make_ecs_role
        LoadBalancers alb_ports.values.map { |info|
          {
            "TargetGroupArn" => info[:target],
            "ContainerPort" => info[:port],
            "ContainerName" => info[:name]
          }
        }
      end

    end
  end #services

  machine_config[:methods].each do |method_name|
    parameters = {ecs_cluster: ecs}

    method(method_name).parameters.each do |param|
      parameters[param[1]] = machine_config[param[1]] if (param[0] == :keyreq or param[0] == :key) and machine_config[param[1]]
    end

    parameters[:network] = network if !parameters[:network]

    method(method_name).call(parameters) do
      machine_volume_locations.each do |s3_loc, machine_loc|
        mkdir File.dirname(machine_loc)
        download_file s3_loc, machine_loc
      end
    end
  end

  ecs
end

#make_ecs_role ⇒ Object 



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/sumomo/ecs.rb', line 9

def make_ecs_role
  make "AWS::IAM::Role", name: "ECSServiceRole" do

    role_policy_doc = {
      "Version" => "2012-10-17",
      "Statement" => [{
        "Effect" => "Allow",
        "Principal" => {"Service" => ["ecs.amazonaws.com"]},
        "Action" => ["sts:AssumeRole"]
      }]
    }

    AssumeRolePolicyDocument role_policy_doc
    Path "/"
    Policies [
      {
        "PolicyName" => "ecs-service",
        "PolicyDocument" => {
          "Version" => "2012-10-17",
          "Statement" => [{
            "Effect" => "Allow",
            "Action" => [
              "ec2:AuthorizeSecurityGroupIngress",
              "ec2:Describe*",
              "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
              "elasticloadbalancing:DeregisterTargets",
              "elasticloadbalancing:Describe*",
              "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
              "elasticloadbalancing:RegisterTargets"
                  ],
            "Resource" => "*"
          }]
        }
      }
    ]
  end
end

#make_lambda(name: nil, files: [{name:"index.js", code:""}], description: "Lambda Function in #{@bucket_name}", function_key: "cloudformation/lambda/function_#{name}", handler: "index.handler", runtime: "nodejs4.3", memory_size: 128, timeout: 30, with_statements: []) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/sumomo/stack.rb', line 24

def make_lambda(name: nil, files:[{name:"index.js", code:""}],
  description: "Lambda Function in #{@bucket_name}", 
  function_key: "cloudformation/lambda/function_#{name}",
  handler: "index.handler",
  runtime: "nodejs4.3",
  memory_size: 128,
  timeout: 30,
  with_statements: [])

  name ||= make_default_resource_name("Lambda")

  stringio = Zip::OutputStream.write_buffer do |zio|
    files.each do |file|
      zio.put_next_entry(file[:name])
      if file[:code]
        zio.write file[:code]
      elsif file[:path]
        zio.write File.read(file[:path])
      else
        raise "Files needs to be an array of objects with :name and :code or :path members"
      end
    end
  end

  @store.set_raw(function_key, stringio.string)

  stack = self

  code_location = {"S3Bucket": @bucket_name, "S3Key": function_key}
  fun = make "AWS::Lambda::Function", name: name do
    Code code_location
    Description description
    MemorySize memory_size
    Handler handler
    Runtime runtime
    Timeout timeout
    Role stack.exec_role(with_statements: with_statements).Arn
  end

  log_group = make "AWS::Logs::LogGroup", name: "#{name}LogGroup" do
    LogGroupName call("Fn::Join", "", ["/aws/lambda/", fun])
    RetentionInDays 30
  end

  fun
end

#make_network(layers: []) ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/sumomo/network.rb', line 7

def make_network(layers: [])

  zones = get_azs()

  region = @region

  vpc = make "AWS::EC2::VPC" do
    CidrBlock "10.0.0.0/16"
    EnableDnsSupport true
    EnableDnsHostnames true
    tag "Name", call("Fn::Join", "-", [ref("AWS::StackName")])
  end

  gateway = make "AWS::EC2::InternetGateway" do
    tag "Name", call("Fn::Join", "-", [ref("AWS::StackName")])
  end

  attachment = make "AWS::EC2::VPCGatewayAttachment" do
    VpcId vpc
    InternetGatewayId gateway
  end

  inet_route_table = make "AWS::EC2::RouteTable" do
    depends_on attachment
    VpcId vpc
    tag "Name", call("Fn::Join", "-", ["public", ref("AWS::StackName")])
  end

  make "AWS::EC2::Route" do
    RouteTableId inet_route_table
    DestinationCidrBlock "0.0.0.0/0"
    GatewayId gateway
  end

  last_unused_number = 0

  subnet_numbers = []

  #load current config
  number_hash = {}
  subnet_hash = {}

  ec2 = Aws::EC2::Client.new(region: @region)
  ec2_subnets = ec2.describe_subnets().subnets
  ec2_subnets.each do |subnet|
    if subnet.tags.select {|x| x.key == "aws:cloudformation:stack-name" && x.value == @bucket_name}.length == 1
      layer = /^#{@bucket_name}-(?<layer_name>.+)-[a-z]+$/.match(subnet.tags.select{|x| x.key == "Name"}.first.value)[:layer_name]
      zone = subnet.availability_zone
      number = /^10.0.(?<num>[0-9]+).0/.match(subnet.cidr_block)[:num].to_i

      key = "#{layer}/#{zone}"
      number_hash[number] = key
      subnet_hash[key] = number
    end
  end

  # assign numbers to unassigned subnets
  layers.product(zones).each do |e|
    key = "#{e[0]}/#{e[1]}"
    if !subnet_hash.has_key?(key)
      loop do
        break if !number_hash.has_key?(last_unused_number)
        last_unused_number += 1
      end
      number_hash[last_unused_number] = key
      subnet_hash[key] = last_unused_number
      subnet_numbers << [e, last_unused_number]
    else
      subnet_numbers << [e, subnet_hash[key]]
    end
  end

  subnets = {}

  subnet_numbers.each do |e, subnet_number|
    layer = e[0]
    zone = e[1]

    zone_letter = zone.sub("#{region}", "")
    cidr = "10.0.#{subnet_number}.0/24"

    subnet = make "AWS::EC2::Subnet", name: "SubnetFor#{layer.camelize}Layer#{zone_letter.upcase}" do
      AvailabilityZone zone
      VpcId vpc
      CidrBlock cidr

      tag("Name", call("Fn::Join", "-", [ ref("AWS::StackName"), "#{layer}", zone_letter] ) )
    end

    make "AWS::EC2::SubnetRouteTableAssociation", name: "SubnetRTAFor#{layer.camelize}Layer#{zone_letter.upcase}" do
      SubnetId subnet
      RouteTableId inet_route_table
    end

    subnets[layer] ||= []
    subnets[layer] << {name: subnet, cidr: cidr, zone: zone}
  end

  Hashie::Mash.new vpc: vpc, subnets: subnets, azs: zones, attachment: attachment
end

#make_spotter(price:, network:, layer:, ec2_sns_arn: nil, ecs_cluster: nil, eip: nil, &block) ⇒ Object 



158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
 
# File 'lib/sumomo/ec2.rb', line 158

def make_spotter(
  price:,
  network:,
  layer:,
  ec2_sns_arn:nil,
  ecs_cluster:nil,
  eip:nil,
  &block)
  update_time = Time.now.to_i

  spot = make "Custom::SelectSpot" do
    DateTime update_time
    ExcludeString "1.,2.,small,micro"
    LookBack 3
    TargetPrice price
  end 

  switcher1_src = define_custom_resource(name: "ASGSelector1", code: "\n    store.get(\"num1\", function(num) {\n      num = parseInt(num);\n      if (request.RequestType != \"Delete\")\n      {\n        store.put(\"num1\", String(num+1));\n      }\n      else\n      {\n        store.put(\"num1\", String(0));\n      }\n\n      Cloudformation.send(request, context, Cloudformation.SUCCESS, {Num: String(num)}, \"Success\", String(num % 2));\n    }, function() {\n      store.put(\"num1\", String(1));\n      Cloudformation.send(request, context, Cloudformation.SUCCESS, {Num: 1}, \"Success\", String(1));\n    });\n  CODE\n  )\n\n  switcher2_src = define_custom_resource(name: \"ASGSelector2\", code: <<-CODE\n    store.get(\"num2\", function(num) {\n      num = parseInt(num);\n      if (request.RequestType != \"Delete\")\n      {\n        store.put(\"num2\", String(num+1));\n      }\n      else\n      {\n        store.put(\"num1\", String(0));\n      }\n\n      Cloudformation.send(request, context, Cloudformation.SUCCESS, {Num: String(num)}, \"Success\", String((num + 1) % 2));\n    }, function() {\n      store.put(\"num2\", String(1));\n      Cloudformation.send(request, context, Cloudformation.SUCCESS, {Num: 1}, \"Success\", String(0));\n    });\n  CODE\n  )\n\n  size_1 = make_custom switcher1_src, name: \"ASGSelector1Value\" do\n    DateTime update_time\n  end\n\n  size_2 = make_custom switcher2_src, name: \"ASGSelector2Value\" do\n    DateTime update_time\n  end\n\n  make_autoscaling_group(\n    type: spot,\n    network: network, \n    layer: \"ecs\", \n    zone: spot.Zone, \n    spot_price: price, \n    min_size: size_1,\n    ec2_sns_arn: ec2_sns_arn,\n    ecs_cluster: ecs_cluster,\n    eip: eip, &block)\n\n  make_autoscaling_group(\n    type: spot,\n    network: network,\n    layer: \"ecs\", \n    zone: spot.Zone, \n    spot_price: price, \n    min_size: size_2,\n    ec2_sns_arn: ec2_sns_arn,\n    ecs_cluster: ecs_cluster,\n    eip: eip)\nend\n"

#route53_dns(hosted_zone:) ⇒ Object 



192
193
194
 
# File 'lib/sumomo/api.rb', line 192

def route53_dns(hosted_zone:)
    {type: :route53, hosted_zone: hosted_zone}
end

#sluggify(str) ⇒ Object 



5
6
7
 
# File 'lib/sumomo/ecs.rb', line 5

def sluggify(str)
  str.gsub(/[^0-9a-zA-Z]/, "_")
end

#upload_file(name, content) ⇒ Object 



19
20
21
22
 
# File 'lib/sumomo/stack.rb', line 19

def upload_file(name,content)
  @store.set_raw("uploads/#{name}", content)
  puts "Uploaded #{name}"
end