Module: StackBuilder

Defined in:
lib/modulator/stack/builder.rb,
lib/modulator/stack/policies.rb,
lib/modulator/stack/uploader.rb

Defined Under Namespace

Modules: LambdaPolicy

Constant Summary collapse

RUBY_VERSION = 
'ruby2.5'
GEM_PATH_RUBY_VERSION = 
'2.5.0'
GEM_PATH = 
"/opt/ruby/#{GEM_PATH_RUBY_VERSION}"
LAMBDA_HANDLER_FILE_NAME = 
'modulator-lambda-handler'
S3Client = 
Aws::S3::Client.new

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.api_gateway_deployment ⇒ Object

Returns the value of attribute api_gateway_deployment.



17
18
19
 
# File 'lib/modulator/stack/builder.rb', line 17

def api_gateway_deployment
  @api_gateway_deployment
end

.api_gateway_id ⇒ Object

Returns the value of attribute api_gateway_id.



17
18
19
 
# File 'lib/modulator/stack/builder.rb', line 17

def api_gateway_id
  @api_gateway_id
end

.app_dir ⇒ Object

Returns the value of attribute app_dir.



15
16
17
 
# File 'lib/modulator/stack/builder.rb', line 15

def app_dir
  @app_dir
end

.app_name ⇒ Object

Returns the value of attribute app_name.



15
16
17
 
# File 'lib/modulator/stack/builder.rb', line 15

def app_name
  @app_name
end

.app_path ⇒ Object

Returns the value of attribute app_path.



15
16
17
 
# File 'lib/modulator/stack/builder.rb', line 15

def app_path
  @app_path
end

.hidden_dir ⇒ Object

Returns the value of attribute hidden_dir.



16
17
18
 
# File 'lib/modulator/stack/builder.rb', line 16

def hidden_dir
  @hidden_dir
end

.lambda_handler_s3_key ⇒ Object

Returns the value of attribute lambda_handler_s3_key.



18
19
20
 
# File 'lib/modulator/stack/builder.rb', line 18

def lambda_handler_s3_key
  @lambda_handler_s3_key
end

.lambda_handler_s3_object_version ⇒ Object

Returns the value of attribute lambda_handler_s3_object_version.



16
17
18
 
# File 'lib/modulator/stack/builder.rb', line 16

def lambda_handler_s3_object_version
  @lambda_handler_s3_object_version
end

.lambda_handlers ⇒ Object

Returns the value of attribute lambda_handlers.



18
19
20
 
# File 'lib/modulator/stack/builder.rb', line 18

def lambda_handlers
  @lambda_handlers
end

.lambda_policies ⇒ Object

Returns the value of attribute lambda_policies.



17
18
19
 
# File 'lib/modulator/stack/builder.rb', line 17

def lambda_policies
  @lambda_policies
end

.s3_bucket ⇒ Object

Returns the value of attribute s3_bucket.



16
17
18
 
# File 'lib/modulator/stack/builder.rb', line 16

def s3_bucket
  @s3_bucket
end

.stack ⇒ Object

Returns the value of attribute stack.



15
16
17
 
# File 'lib/modulator/stack/builder.rb', line 15

def stack
  @stack
end

.stack_opts ⇒ Object

Returns the value of attribute stack_opts.



15
16
17
 
# File 'lib/modulator/stack/builder.rb', line 15

def stack_opts
  @stack_opts
end

Class Method Details

.add_api_gateway ⇒ Object

gateway



92
93
94
95
 
# File 'lib/modulator/stack/builder.rb', line 92

def add_api_gateway
  self.api_gateway_id = 'ApiGateway'
  stack.add(api_gateway_id, Humidifier::ApiGateway::RestApi.new(name: app_name, description: app_name + ' API'))
end

.add_api_gateway_deployment ⇒ Object

gateway deployment



98
99
100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/modulator/stack/builder.rb', line 98

def add_api_gateway_deployment
  self.api_gateway_deployment = Humidifier::ApiGateway::Deployment.new(
    rest_api_id: Humidifier.ref(api_gateway_id),
    stage_name: Humidifier.ref("ApiGatewayStageName")
  )
  stack.add('ApiGatewayDeployment', api_gateway_deployment)
  stack.add_output('ApiGatewayInvokeURL',
      value: Humidifier.fn.sub("https://${#{api_gateway_id}}.execute-api.${AWS::Region}.amazonaws.com/${ApiGatewayStageName}"),
      description: 'API root url',
      export_name: app_name + 'RootUrl'
  )
  api_gateway_deployment.depends_on = []
end

.add_api_gateway_resources(gateway:, lambda:) ⇒ Object

gateway method



195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
 
# File 'lib/modulator/stack/builder.rb', line 195

def add_api_gateway_resources(gateway:, lambda:)

  # example: calculator/algebra/:x/:y/sum -> module name, args, method name
  path = gateway[:path].split('/')

  # root resource
  root_resource = path.shift
  stack.add(root_resource.camelize, Humidifier::ApiGateway::Resource.new(
      rest_api_id: Humidifier.ref(api_gateway_id),
      parent_id: Humidifier.fn.get_att(["ApiGateway", "RootResourceId"]),
      path_part: root_resource
    )
  )

  # args and method name are nested resources
  parent_resource = root_resource.camelize
  path.each do |fragment|
    if fragment.start_with?(':')
      fragment = fragment[1..-1]
      dynamic_fragment = "{#{fragment}}"
    end
    stack.add(parent_resource + fragment.camelize, Humidifier::ApiGateway::Resource.new(
        rest_api_id: Humidifier.ref(api_gateway_id),
        parent_id: Humidifier.ref(parent_resource),
        path_part: dynamic_fragment || fragment
      )
    )
    parent_resource = parent_resource + fragment.camelize
  end

  # attach lambda to last resource
  id = 'EndpointFor' + (gateway[:path].gsub(':', '').gsub('/', '_')).camelize
  stack.add(id, Humidifier::ApiGateway::Method.new(
      authorization_type: 'NONE',
      http_method: gateway[:verb].to_s.upcase,
      integration: {
        integration_http_method: 'POST',
        type: "AWS_PROXY",
        uri: Humidifier.fn.sub([
          "arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${lambdaArn}/invocations",
          'lambdaArn' => Humidifier.fn.get_att([lambda, 'Arn'])
        ])
      },
      rest_api_id: Humidifier.ref(api_gateway_id),
      resource_id: Humidifier.ref(parent_resource) # last evaluated resource
    )
  )

  # deployment depends on each endpoint
  api_gateway_deployment.depends_on << id
end

.add_generic_lambda(gateway: {}, mod: {}, wrapper: {}, env: {}, settings: {}) ⇒ Object

generic lambda function for gateway



127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'lib/modulator/stack/builder.rb', line 127

def add_generic_lambda(gateway: {}, mod: {}, wrapper: {}, env: {}, settings: {})
  lambda_config = {}
  name_parts = mod[:name].split('::')
  {gateway: gateway, module: mod, wrapper: wrapper}.each do |env_group_prefix, env_group|
    env_group.each{|env_key, env_value| lambda_config["#{env_group_prefix}_#{env_key}"] = env_value}
  end
  env_vars = env
      .reduce({}){|env_as_string, (k, v)| env_as_string.update(k.to_s => v.to_s)}
      .merge(lambda_config)
      .merge(
        'GEM_PATH' => GEM_PATH,
        'app_dir'  => app_dir,
        'app_env'  => Humidifier.ref('AppEnvironment')
      )

  lambda_resource = generate_lambda_resource(
    description: "Lambda for #{mod[:name]}.#{mod[:method]}",
    function_name: [app_name, name_parts, mod[:method]].flatten.join('-').dasherize,
    handler: "#{LAMBDA_HANDLER_FILE_NAME}.AwsLambdaHandler.call",
    s3_key: LAMBDA_HANDLER_FILE_NAME + '.rb.zip',
    env_vars: env_vars,
    role: Humidifier.fn.get_att(['LambdaRole', 'Arn']),
    settings: settings,
    layers: [Humidifier.ref(app_name + 'Layer'), Humidifier.ref(app_name + 'GemsLayer')]
  )

  # add to stack
  ['Lambda', name_parts, mod[:method].capitalize].join.tap do |id|
    stack.add(id, lambda_resource)
    stack.add_lambda_invoke_permission(id: id, gateway: gateway)
  end
end

.add_lambda(handler:, env: {}, settings: {}) ⇒ Object

custom lambda function



113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/modulator/stack/builder.rb', line 113

def add_lambda(handler:, env: {}, settings: {})
  lambda_resource = generate_lambda_resource(
    description: "Lambda for #{handler}",
    function_name: ([app_name] << handler.split('.')).flatten.join('-').dasherize,
    handler: handler,
    s3_key: lambda_handler_s3_key,
    env_vars: env.merge('app_env' => Humidifier.ref('AppEnvironment')),
    role: Humidifier.fn.get_att(['LambdaRole', 'Arn']),
    settings: settings
  )
  stack.add(handler.gsub('.', '_').camelize, lambda_resource)
end

.add_lambda_endpoint(**opts) ⇒ Object

gateway:, mod:, wrapper: {}, env: {}, settings: {}



86
87
88
89
 
# File 'lib/modulator/stack/builder.rb', line 86

def add_lambda_endpoint(**opts) # gateway:, mod:, wrapper: {}, env: {}, settings: {}
  # add api resources and its lambda
  stack.add_api_gateway_resources(gateway: opts[:gateway], lambda: stack.add_generic_lambda(opts))
end

.add_lambda_invoke_permission(id:, gateway:) ⇒ Object

invoke permission



180
181
182
183
184
185
186
187
188
189
190
191
192
 
# File 'lib/modulator/stack/builder.rb', line 180

def add_lambda_invoke_permission(id:, gateway:)
  arn_path_matcher = gateway[:path].split('/').each_with_object([]) do |fragment, matcher|
    fragment = '*' if fragment.start_with?(':')
    matcher << fragment
  end.join('/')
  stack.add(id + 'InvokePermission' , Humidifier::Lambda::Permission.new(
      action: "lambda:InvokeFunction",
      function_name: Humidifier.fn.get_att([id, 'Arn']),
      principal: "apigateway.amazonaws.com",
      source_arn: Humidifier.fn.sub("arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${#{api_gateway_id}}/*/#{gateway[:verb]}/#{arn_path_matcher}")
    )
  )
end

.add_layer(name:, description:, s3_key:, s3_object_version:) ⇒ Object

add layer



163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/modulator/stack/uploader.rb', line 163

def add_layer(name:, description:, s3_key:, s3_object_version:)
  stack.add(name + 'Layer', Humidifier::Lambda::LayerVersion.new(
      compatible_runtimes: [RUBY_VERSION],
      layer_name: name,
      description: description,
      content: {
        s3_bucket: s3_bucket,
        s3_key: s3_key,
        s3_object_version: s3_object_version
      }
    )
  )
end

.generate_lambda_resource(description:, function_name:, handler:, s3_key:, env_vars:, role:, settings:, layers: []) ⇒ Object 



160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/modulator/stack/builder.rb', line 160

def generate_lambda_resource(description:, function_name:, handler:, s3_key:, env_vars:, role:, settings:, layers: [])
  lambda_function = Humidifier::Lambda::Function.new(
    description: description,
    function_name: function_name,
    handler: handler,
    environment: {variables: env_vars},
    role: role,
    timeout: settings[:timeout] || stack_opts[:timeout] || 15,
    memory_size: settings[:memory_size] || stack_opts[:memory_size] || 128,
    runtime: RUBY_VERSION,
    code: {
      s3_bucket: s3_bucket,
      s3_key: s3_key,
      s3_object_version: lambda_handler_s3_object_version
    },
    layers: layers
  )
end

.init(app_name:, s3_bucket:, **stack_opts) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/modulator/stack/builder.rb', line 21

def init(app_name:, s3_bucket:, **stack_opts)
  puts 'Initializing stack'
  @app_name   = app_name.camelize
  @s3_bucket  = s3_bucket
  @app_path   = Pathname.getwd
  @app_dir    = app_path.basename.to_s
  @hidden_dir = '.modulator'
  @stack_opts = stack_opts
  @lambda_handlers = stack_opts[:lambda_handlers] || []
  @lambda_policies = Array(stack_opts[:lambda_policies]) << :cloudwatch

  # create hidden dir for build artifacts
  app_path.join(hidden_dir).mkpath

  # init stack instance
  self.stack = Humidifier::Stack.new(name: app_name, aws_template_format_version: '2010-09-09')

  # app environment -  test, development, production ...
  app_envs = stack_opts[:app_envs] || ['development']
  stack.add_parameter('AppEnvironment', description: 'Application environment', type: 'String', allowed_values: app_envs, constraint_description: "Must be one of #{app_envs.join(', ')}")

  if lambda_handlers.empty?
    # api stage
    stack.add_parameter('ApiGatewayStageName', description: 'Gateway deployment stage', type: 'String', default: 'v1')

    # add gateway
    stack.add_api_gateway
    stack.add_api_gateway_deployment
  end

  # add role
  stack.add_lambda_iam_role

  # add policies to role
  stack.lambda_policies.each do |policy|
    stack.add_policy(policy) if policy.is_a?(Symbol)
    stack.add_policy(policy[:name], **policy) if policy.is_a?(Hash)
  end

  # simple lambda app
  if lambda_handlers.any?
    stack.upload_lambda_files
    lambda_handlers.each do |handler|
      stack.add_lambda(handler: handler, env: stack_opts[:env] || {}, settings: stack_opts[:settings] || {})
    end
  else
    # upload handlers and layers
    stack.upload_files
  end

  # return humidifier instance
  stack
end

.upload_app_layer(sub_dirs: 'ruby/lib', add_layer_to_stack: true) ⇒ Object 



113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
 
# File 'lib/modulator/stack/uploader.rb', line 113

def upload_app_layer(sub_dirs: 'ruby/lib', add_layer_to_stack: true)
  zip_file_name = app_dir + '.zip'
  app_zip_path  = app_path.join(hidden_dir, zip_file_name)

  # copy app code to ruby/lib in outside temp dir
  temp_dir_name = '.modulator_temp'
  temp_sub_dirs = sub_dirs
  temp_path     = app_path.parent.join(temp_dir_name)
  temp_path.join(temp_sub_dirs).mkpath
  FileUtils.copy_entry app_path, temp_path.join(temp_sub_dirs)

  # calculate checksum for app folder
  checksum_path = app_path.join(hidden_dir, 'app_checksum')
  old_checksum  = (checksum_path.read rescue nil)
  new_checksum  = Utils.checksum(app_path)

  if old_checksum != new_checksum
    puts '- uploading app files'
    checksum_path.write(new_checksum)
    ZipFileGenerator.new(temp_path, app_zip_path).write
    # upload zipped file
    app_layer = S3Client.put_object(
      bucket: s3_bucket,
      key: zip_file_name,
      body: app_zip_path.read
    )
    # delete zipped file
    app_zip_path.delete
  else
    puts '- using existing app files'
    app_layer = S3Client.get_object(bucket: s3_bucket, key: zip_file_name)
  end

  # delete temp dir
  FileUtils.remove_dir(temp_path)

  if add_layer_to_stack
    add_layer(
      name: app_name,
      description: "App source. MD5: #{new_checksum}",
      s3_key: zip_file_name,
      s3_object_version: app_layer.version_id
    )
  else # for simple lambda apps
    self.lambda_handler_s3_key = zip_file_name
    self.lambda_handler_s3_object_version = app_layer.version_id
  end
end

.upload_files ⇒ Object 



75
76
77
78
79
80
81
82
83
84
 
# File 'lib/modulator/stack/builder.rb', line 75

def upload_files
  if stack_opts[:skip_upload]
    puts 'Skipping upload'
    return
  end
  stack.upload_generic_lambda_handler
  puts 'Generating layers'
  stack.upload_gems_layer
  stack.upload_app_layer
end

.upload_gems_layer ⇒ Object 



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/modulator/stack/uploader.rb', line 64

def upload_gems_layer
  if !app_path.join('Gemfile').exist?
    puts '- no Gemfile detected'
    return
  end

  # calculate Gemfile checksum
  checksum_path = app_path.join(hidden_dir, 'gemfile_checksum')
  old_checksum  = (checksum_path.read rescue nil)
  new_checksum  = Digest::MD5.hexdigest(File.read(app_path.join('Gemfile.lock')))

  zip_file_name = app_dir + '_gems.zip'
  gems_path     = app_path.join(hidden_dir, 'gems')
  gems_zip_path = app_path.join(hidden_dir, zip_file_name)

  if old_checksum != new_checksum
    puts '- uploading gems layer'
    checksum_path.write(new_checksum)

    # bundle gems
    Bundler.with_clean_env do
      Dir.chdir(app_path) do
        `bundle install --path=./#{hidden_dir}/gems --clean --without development`
      end
    end
    ZipFileGenerator.new(gems_path, gems_zip_path).write

    # upload zipped file
    gem_layer = S3Client.put_object(
      bucket: s3_bucket,
      key: zip_file_name,
      body: gems_zip_path.read
    )
    # delete zipped file
    FileUtils.remove_dir(gems_path)
    gems_zip_path.delete
  else
    puts '- using existing gems layer'
    gem_layer = S3Client.get_object(bucket: s3_bucket, key: zip_file_name)
  end

  add_layer(
    name: app_name + 'Gems',
    description: "App gems",
    s3_key: zip_file_name,
    s3_object_version: gem_layer.version_id
  )
end

.upload_generic_lambda_handler ⇒ Object

generic handler for all lambda



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/modulator/stack/uploader.rb', line 26

def upload_generic_lambda_handler
  lambda_handler_key = LAMBDA_HANDLER_FILE_NAME + '.rb.zip'
  modulator_handler_source = Pathname.new(__FILE__).dirname.parent.join('lambda_handler.rb').read
  source = "    # DO NOT EDIT THIS FILE\n\n    \#{modulator_handler_source}\n    Dir.chdir('/opt/ruby/lib')\n  SOURCE\n\n  existing_handler = S3Client.get_object(\n    bucket: s3_bucket,\n    key: lambda_handler_key\n  ) rescue false # not found\n\n  if existing_handler\n    existing_source = Zip::InputStream.open(existing_handler.body) do |zip_file|\n      zip_file.get_next_entry\n      zip_file.read\n    end\n    self.lambda_handler_s3_object_version = existing_handler.version_id\n  end\n\n  if existing_source != source\n    puts '- uploading generic lambda handler'\n    source_zip_file = Zip::OutputStream.write_buffer do |zip|\n      zip.put_next_entry LAMBDA_HANDLER_FILE_NAME + '.rb'\n      zip.print source\n    end\n    new_handler = S3Client.put_object(\n      bucket: s3_bucket,\n      key: lambda_handler_key,\n      body: source_zip_file.tap(&:rewind).read\n    )\n    self.lambda_handler_s3_object_version = new_handler.version_id\n  end\nend\n"

.upload_lambda_files ⇒ Object

bundle gems and upload all for simple lambda apps



11
12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/modulator/stack/uploader.rb', line 11

def upload_lambda_files
  puts '- bundling dependencies'
  Bundler.with_clean_env do
    Dir.chdir(app_path) do
      `bundle install`
      `bundle install --deployment --without development`
    end
  end
  FileUtils.remove_dir(app_path.join("vendor/bundle/ruby/#{GEM_PATH_RUBY_VERSION}/cache")) # remove cache dir
  upload_app_layer(sub_dirs: '', add_layer_to_stack: false) # reuse layer upload
  FileUtils.remove_dir(app_path.join('.bundle'))
  FileUtils.remove_dir(app_path.join('vendor'))
end