Class: Jets::Builders::CodeBuilder

Inherits:
Object
  • Object
show all
Extended by:
Memoist
Includes:
AwsServices, Util, Timing
Defined in:
lib/jets/builders/code_builder.rb

Constant Summary collapse

AWS_CODE_SIZE_LIMIT =

docs.aws.amazon.com/lambda/latest/dg/limits.html

250 * 1024 * 1024

Constants included from Timing

Timing::RECORD_LOG_PATH

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Util

#full, #headline, #poly_only?, #sh

Methods included from AwsServices

#cfn, #lambda, #logs, #s3, #s3_resource, #sns, #sts

Methods included from AwsServices::StackStatus

#stack_exists?, #stack_in_progress?

Methods included from Timing

clear, #record_data, #record_log, report

Constructor Details

#initialize ⇒ CodeBuilder

Returns a new instance of CodeBuilder.



62
63
64
65
66
 
# File 'lib/jets/builders/code_builder.rb', line 62

def initialize
  # Expanding to the full path and capture now.
  # Dir.chdir gets called later and we'll lose this info.
  @full_project_path = File.expand_path(Jets.root) + "/"
end

Instance Attribute Details

#full_project_path ⇒ Object (readonly)

Returns the value of attribute full_project_path.



61
62
63
 
# File 'lib/jets/builders/code_builder.rb', line 61

def full_project_path
  @full_project_path
end

Class Method Details

.tmp_code ⇒ Object

Group all the path settings together here



395
396
397
 
# File 'lib/jets/builders/code_builder.rb', line 395

def self.tmp_code
  Jets::Commands::Build.tmp_code
end

Instance Method Details

#build ⇒ Object 



68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/jets/builders/code_builder.rb', line 68

def build
  cache_check_message
  check_ruby_version

  clean_start
  compile_assets # easier to do before we copy the project because node and yarn has been likely setup in the that dir
  compile_rails_assets
  copy_project
  Dir.chdir(full(tmp_code)) do
    # These commands run from project root
    code_setup
    package_ruby
    code_finish
  end
end

#cache_check_message ⇒ Object 



370
371
372
373
374
 
# File 'lib/jets/builders/code_builder.rb', line 370

def cache_check_message
  if File.exist?("#{Jets.build_root}/cache")
    puts "The #{Jets.build_root}/cache folder exists. Incrementally re-building the jets using the cache.  To clear the cache: rm -rf #{Jets.build_root}/cache"
  end
end

#calculate_md5s ⇒ Object

Resolves the chicken-and-egg problem with md5 checksums. The handlers need to reference files with the md5 checksum. The files are the:

jets/code/rack-checksum.zip
jets/code/bundled-checksum.zip

We compute the checksums before we generate the node shim handlers.



92
93
94
 
# File 'lib/jets/builders/code_builder.rb', line 92

def calculate_md5s
  Md5.compute! # populates Md5.checksums hash
end

#check_ruby_version ⇒ Object 



376
377
378
379
380
381
382
 
# File 'lib/jets/builders/code_builder.rb', line 376

def check_ruby_version
  unless ruby_version_supported?
    puts "You are using ruby version #{RUBY_VERSION} which is not supported by Jets."
    ruby_variant = Jets::RUBY_VERSION.split('.')[0..1].join('.') + '.x'
    abort("Jets uses ruby #{Jets::RUBY_VERSION}.  You should use a variant of ruby #{ruby_variant}".colorize(:red))
  end
end

#clean_start ⇒ Object

Cleans out non-cached files like code-*.zip in Jets.build_root for a clean start. Also ensure that the /tmp/jets/project build root exists.

Most files are kept around after the build process for inspection and debugging. So we have to clean out the files. But we only want to clean out some of the files.



293
294
295
296
 
# File 'lib/jets/builders/code_builder.rb', line 293

def clean_start
  Dir.glob("#{Jets.build_root}/code/code-*.zip").each { |f| FileUtils.rm_f(f) }
  FileUtils.mkdir_p(Jets.build_root) # /tmp/jets/demo
end

#code_finish ⇒ Object 



168
169
170
171
172
173
174
175
 
# File 'lib/jets/builders/code_builder.rb', line 168

def code_finish
  update_lazy_load_config # at the top, must be called before Jets.lazy_load? is used
  store_s3_base_url
  setup_tmp
  calculate_md5s # must be called before generate_node_shims and create_zip_files
  generate_node_shims
  create_zip_files
end

#code_setup ⇒ Object 



163
164
165
 
# File 'lib/jets/builders/code_builder.rb', line 163

def code_setup
  reconfigure_development_webpacker
end

#compile_assets ⇒ Object

This happens in the current app directory not the tmp code for simplicity. This is because the node and yarn has likely been set up correctly there.



236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/jets/builders/code_builder.rb', line 236

def compile_assets
  if ENV['JETS_SKIP_ASSETS']
    puts "Skip compiling assets".colorize(:yellow) # useful for debugging
    return
  end

  headline "Compling assets in current project directory"
  # Thanks: https://stackoverflow.com/questions/4195735/get-list-of-gems-being-used-by-a-bundler-project
  webpacker_loaded = Gem.loaded_specs.keys.include?("webpacker")
  return unless webpacker_loaded

  sh("yarn install")
  webpack_command = File.exist?("#{Jets.root}bin/webpack") ?
      "bin/webpack" :
      `which webpack`.strip
  sh("JETS_ENV=#{Jets.env} #{webpack_command}")
end

#compile_rails_assets ⇒ Object

This happens in the current app directory not the tmp code for simplicity This is because the node likely been set up correctly there.



257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
 
# File 'lib/jets/builders/code_builder.rb', line 257

def compile_rails_assets
  return unless rails?

  if ENV['JETS_SKIP_ASSETS']
    puts "Skip compiling rack assets".colorize(:yellow) # useful for debugging
    return
  end

  return unless Jets.rack?

  Bundler.with_clean_env do
    rails_assets(:clobber)
    rails_assets(:precompile)
  end
end

#copy_project ⇒ Object

Copy project into temporary directory. Do this so we can keep the project directory untouched and we can also remove a bunch of unnecessary files like logs before zipping it up.



301
302
303
304
305
306
307
308
309
310
311
312
313
 
# File 'lib/jets/builders/code_builder.rb', line 301

def copy_project
  headline "Copying current project directory to temporary build area: #{full(tmp_code)}"
  FileUtils.rm_rf(stage_area) # clear out from previous build
  FileUtils.mkdir_p(stage_area)
  FileUtils.rm_rf(full(tmp_code)) # remove current code folder
  move_node_modules(Jets.root, Jets.build_root)
  begin
    # puts "cp -r #{@full_project_path} #{full(tmp_code)}".colorize(:yellow) # uncomment to debug
    FileUtils.cp_r(@full_project_path, full(tmp_code))
  ensure
    move_node_modules(Jets.build_root, Jets.root) # move node_modules directory back
  end
end

#create_zip_files ⇒ Object 



104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/jets/builders/code_builder.rb', line 104

def create_zip_files
  folders = Md5.stage_folders
  folders.each do |folder|
    zip = Md5Zip.new(folder)
    if exist_on_s3?(zip.md5_name)
      puts "Already exists: s3://#{s3_bucket}/jets/code/#{zip.md5_name}"
    else
      zip = Md5Zip.new(folder)
      zip.create
    end
  end
end

#dir_size(folder) ⇒ Object

Thanks stackoverflow.com/questions/9354595/recursively-getting-the-size-of-a-directory Seems to overestimate a little bit but close enough.



191
192
193
194
195
196
 
# File 'lib/jets/builders/code_builder.rb', line 191

def dir_size(folder)
  Dir.glob(File.join(folder, '**', '*'))
    .select { |f| File.file?(f) }
    .map{ |f| File.size(f) }
    .inject(:+)
end

#exist_on_s3?(filename) ⇒ Boolean

Returns:

  • (Boolean) 


118
119
120
121
122
123
124
125
126
 
# File 'lib/jets/builders/code_builder.rb', line 118

def exist_on_s3?(filename)
  s3_key = "jets/code/#{filename}"
  begin
    s3.head_object(bucket: s3_bucket, key: s3_key)
    true
  rescue Aws::S3::Errors::NotFound
    false
  end
end

#generate_node_shims ⇒ Object 



96
97
98
99
100
101
102
 
# File 'lib/jets/builders/code_builder.rb', line 96

def generate_node_shims
  headline "Generating shims in the handlers folder."
  # Crucial that the Dir.pwd is in the tmp_code because for
  # Jets::Builders::app_files because Jets.boot set ups
  # autoload_paths and this is how project classes are loaded.
  Jets::Builders::HandlerGenerator.build!
end

#move_node_modules(source_folder, dest_folder) ⇒ Object

Move the node modules to the tmp build folder to speed up project copying. A little bit risky because a ctrl-c in the middle of the project copying results in a missing node_modules but user can easily rebuild that.

Tesing shows 6.623413 vs 0.027754 speed improvement.



321
322
323
324
325
326
327
 
# File 'lib/jets/builders/code_builder.rb', line 321

def move_node_modules(source_folder, dest_folder)
  source = "#{source_folder}/node_modules"
  dest = "#{dest_folder}/node_modules"
  if File.exist?(source)
    FileUtils.mv(source, dest)
  end
end

#package_ruby ⇒ Object 



354
355
356
357
358
359
360
361
362
 
# File 'lib/jets/builders/code_builder.rb', line 354

def package_ruby
  return if Jets.poly_only?

  ruby_packager.install
  reconfigure_rails
  rack_packager.install
  ruby_packager.finish
  rack_packager.finish
end

#rack_packager ⇒ Object 



349
350
351
 
# File 'lib/jets/builders/code_builder.rb', line 349

def rack_packager
  RackPackager.new("#{tmp_code}/rack")
end

#rails? ⇒ Boolean

Rudimentary rails detection

Returns:

  • (Boolean) 


281
282
283
284
285
 
# File 'lib/jets/builders/code_builder.rb', line 281

def rails?
  config_ru = "#{Jets.root}rack/config.ru"
  return false unless File.exist?(config_ru)
  !IO.readlines(config_ru).grep(/Rails.application/).empty?
end

#rails_assets(cmd) ⇒ Object 



273
274
275
276
277
278
 
# File 'lib/jets/builders/code_builder.rb', line 273

def rails_assets(cmd)
  # rake is available in both rails 4 and 5. rails command only in 5
  command = "rake assets:#{cmd} --trace"
  command = "RAILS_ENV=#{Jets.env} #{fulL_cmd}" unless Jets.env.development?
  sh("cd rack && #{command}")
end

#reconfigure_development_webpacker ⇒ Object

Bit hacky but this saves the user from accidentally forgetting to change this when they deploy a jets project in development mode



331
332
333
334
335
336
337
338
339
340
341
342
 
# File 'lib/jets/builders/code_builder.rb', line 331

def reconfigure_development_webpacker
  return unless Jets.env.development?
  headline "Reconfiguring webpacker development settings for AWS Lambda."

  webpacker_yml = "#{full(tmp_code)}/config/webpacker.yml"
  return unless File.exist?(webpacker_yml)

  config = YAML.load_file(webpacker_yml)
  config["development"]["compile"] = false # force this to be false for deployment
  new_yaml = YAML.dump(config)
  IO.write(webpacker_yml, new_yaml)
end

#reconfigure_rails ⇒ Object

TODO: Move logic into plugin instead



366
367
368
 
# File 'lib/jets/builders/code_builder.rb', line 366

def reconfigure_rails
  ReconfigureRails.new("#{full(tmp_code)}/rack").run
end

#ruby_packager ⇒ Object 



344
345
346
 
# File 'lib/jets/builders/code_builder.rb', line 344

def ruby_packager
  RubyPackager.new(tmp_code)
end

#ruby_version_supported? ⇒ Boolean

Returns:

  • (Boolean) 


384
385
386
387
388
389
390
391
392
 
# File 'lib/jets/builders/code_builder.rb', line 384

def ruby_version_supported?
  pattern = /(\d+)\.(\d+)\.(\d+)/
  md = RUBY_VERSION.match(pattern)
  ruby = {major: md[1], minor: md[2]}
  md = Jets::RUBY_VERSION.match(pattern)
  jets = {major: md[1], minor: md[2]}

  ruby[:major] == jets[:major] && ruby[:minor] == jets[:minor]
end

#s3_base_url ⇒ Object 



215
216
217
218
219
220
221
222
223
224
225
226
227
228
 
# File 'lib/jets/builders/code_builder.rb', line 215

def s3_base_url
  # Allow user to set assets.base_url
  #
  #   Jets.application.configure do
  #     config.assets.base_url = "https://cloudfront.com/my/base/path"
  #   end
  #
  return Jets.config.assets.base_url if Jets.config.assets.base_url

  region = Jets.aws.region

  asset_base_url = "https://s3-#{region}.amazonaws.com"
  "#{asset_base_url}/#{s3_bucket}/jets" # s3_base_url
end

#s3_bucket ⇒ Object 



230
231
232
 
# File 'lib/jets/builders/code_builder.rb', line 230

def s3_bucket
  Jets.aws.s3_bucket
end

#setup_tmp ⇒ Object

Moves code/bundled and code/rack to build_root. These files will be packaged separated and lazy loaded as part of the node shim. This keeps the code zipfile smaller in size and helps with the 250MB extract limited. /tmp permits up to 512MB. AWS Lambda Limits: amzn.to/2A7y6v6

> Each Lambda function receives an additional 512MB of non-persistent disk space in its own /tmp directory. The /tmp directory can be used for loading additional resources like dependency libraries or data sets during function initialization.



136
137
138
139
 
# File 'lib/jets/builders/code_builder.rb', line 136

def setup_tmp
  tmp_symlink("bundled") if Jets.lazy_load?
  tmp_symlink("rack")
end

#stage_area ⇒ Object 



141
142
143
 
# File 'lib/jets/builders/code_builder.rb', line 141

def stage_area
  "#{Jets.build_root}/stage"
end

#store_s3_base_url ⇒ Object

Store s3 base url is needed for asset serving from s3 later. Need to package this as part of the code so we have a reference to it. At this point the minimal stack exists, so we can grab it with the AWS API. We do not want to grab this as part of the live request because it is slow.



202
203
204
205
206
207
 
# File 'lib/jets/builders/code_builder.rb', line 202

def store_s3_base_url
  return if poly_only?

  write_s3_base_url("config/s3_base_url.txt")
  write_s3_base_url("rack/config/s3_base_url.txt") if Jets.rack?
end

#tmp_code ⇒ Object 



399
400
401
 
# File 'lib/jets/builders/code_builder.rb', line 399

def tmp_code
  self.class.tmp_code
end

#tmp_symlink(folder) ⇒ Object

Moves folder to a stage folder and create a symlink its place that links from /var/task to /tmp. Example:

/var/task/bundled => /tmp/bundled



150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/jets/builders/code_builder.rb', line 150

def tmp_symlink(folder)
  src = "#{full(tmp_code)}/#{folder}"
  return unless File.exist?(src)

  dest = "#{stage_area}/#{folder}"
  dir = File.dirname(dest)
  FileUtils.mkdir_p(dir) unless File.exist?(dir)
  FileUtils.mv(src, dest)

  # Create symlink
  FileUtils.ln_sf("/tmp/#{folder}", "/#{full(tmp_code)}/#{folder}")
end

#update_lazy_load_config ⇒ Object 



178
179
180
181
182
183
184
185
186
187
 
# File 'lib/jets/builders/code_builder.rb', line 178

def update_lazy_load_config
  size_limit = AWS_CODE_SIZE_LIMIT
  code_size = dir_size(full(tmp_code))
  if code_size > size_limit && !Jets.config.ruby.lazy_load
    # override the setting because we dont have to a choice but to lazy load
    mb_limit = AWS_CODE_SIZE_LIMIT / 1024 / 1024
    puts "Code size close to AWS code size limit of #{mb_limit}MB. Lazy loading automatically enabled."
    Jets.config.ruby.lazy_load = true
  end
end

#write_s3_base_url(relative_path) ⇒ Object 



209
210
211
212
213
 
# File 'lib/jets/builders/code_builder.rb', line 209

def write_s3_base_url(relative_path)
  full_path = "#{full(tmp_code)}/#{relative_path}"
  FileUtils.mkdir_p(File.dirname(full_path))
  IO.write(full_path, s3_base_url)
end