Class: Jets::Builders::CodeBuilder

Inherits:
Object
  • Object
show all
Extended by:
Memoist
Includes:
AwsServices, Util
Defined in:
lib/jets/builders/code_builder.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from AwsServices

#apigateway, #cfn, #lambda, #logs, #s3, #s3_resource, #sns, #sqs, #sts

Methods included from AwsServices::StackStatus

#lookup, #stack_exists?, #stack_in_progress?

Constructor Details

#initialize ⇒ CodeBuilder

Returns a new instance of CodeBuilder.



22
23
24
25
26
27
 
# File 'lib/jets/builders/code_builder.rb', line 22

def initialize
  # Expanding to the full path and capture now.
  # Dir.chdir gets called later and we'll lose this info.
  @full_project_path = File.expand_path(Jets.root) + "/"
  @version_purger = Purger.new
end

Instance Attribute Details

#full_project_path ⇒ Object (readonly)

Returns the value of attribute full_project_path.



21
22
23
 
# File 'lib/jets/builders/code_builder.rb', line 21

def full_project_path
  @full_project_path
end

Class Method Details

.tmp_code ⇒ Object

Group all the path settings together here



361
362
363
 
# File 'lib/jets/builders/code_builder.rb', line 361

def self.tmp_code
  Jets::Commands::Build.tmp_code
end

Instance Method Details

#build ⇒ Object 



29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/jets/builders/code_builder.rb', line 29

def build
  check_ruby_version
  @version_purger.purge
  cache_check_message

  clean_start
  compile_assets # easier to do before we copy the project because node and yarn has been likely setup in the that dir
  compile_rails_assets
  copy_project
  Dir.chdir("#{stage_area}/code") do
    # These commands run from project root
    code_setup
    package_ruby
    code_finish
  end
end

#build_lambda_layer ⇒ Object 



325
326
327
328
329
 
# File 'lib/jets/builders/code_builder.rb', line 325

def build_lambda_layer
  return if Jets.poly_only?
  lambda_layer = LambdaLayer.new
  lambda_layer.build
end

#cache_check_message ⇒ Object 



336
337
338
339
340
 
# File 'lib/jets/builders/code_builder.rb', line 336

def cache_check_message
  if File.exist?("#{Jets.build_root}/cache")
    puts "The #{Jets.build_root}/cache folder exists. Incrementally re-building the jets using the cache.  To clear the cache: rm -rf #{Jets.build_root}/cache"
  end
end

#calculate_md5s ⇒ Object

Resolves the chicken-and-egg problem with md5 checksums. The handlers need to reference files with the md5 checksum. The files are the:

jets/code/rack-checksum.zip
jets/code/opt-checksum.zip

We compute the checksums before we generate the node shim handlers.



53
54
55
 
# File 'lib/jets/builders/code_builder.rb', line 53

def calculate_md5s
  Md5.compute! # populates Md5.checksums hash
end

#check_agree ⇒ Object 



320
321
322
323
 
# File 'lib/jets/builders/code_builder.rb', line 320

def check_agree
  agree = Jets::Gems::Agree.new
  agree.prompt
end

#check_code_size! ⇒ Object 



106
107
108
 
# File 'lib/jets/builders/code_builder.rb', line 106

def check_code_size!
  CodeSize.check!
end

#check_ruby_version ⇒ Object 



342
343
344
345
346
347
348
 
# File 'lib/jets/builders/code_builder.rb', line 342

def check_ruby_version
  unless ruby_version_supported?
    puts "You are using Ruby version #{RUBY_VERSION} which is not supported by Jets."
    ruby_variant = Jets::RUBY_VERSION.split('.')[0..1].join('.') + '.x'
    abort("Jets uses Ruby #{Jets::RUBY_VERSION}.  You should use a variant of Ruby #{ruby_variant}".colorize(:red))
  end
end

#clean_start ⇒ Object

Cleans out non-cached files like code-*.zip in Jets.build_root for a clean start. Also ensure that the /tmp/jets/project build root exists.

Most files are kept around after the build process for inspection and debugging. So we have to clean out the files. But we only want to clean out some of the files.



247
248
249
250
 
# File 'lib/jets/builders/code_builder.rb', line 247

def clean_start
  Dir.glob("#{Jets.build_root}/code/code-*.zip").each { |f| FileUtils.rm_f(f) }
  FileUtils.mkdir_p(Jets.build_root) # /tmp/jets/demo
end

#code_finish ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/jets/builders/code_builder.rb', line 93

def code_finish
  # Reconfigure code
  store_s3_base_url
  disable_webpacker_middleware
  copy_internal_jets_code

  # Code prep and zipping
  check_code_size!
  calculate_md5s # must be called before generate_node_shims and create_zip_files
  generate_node_shims
  create_zip_files
end

#code_setup ⇒ Object 



89
90
91
 
# File 'lib/jets/builders/code_builder.rb', line 89

def code_setup
  reconfigure_development_webpacker
end

#compile_assets ⇒ Object

This happens in the current app directory not the tmp code for simplicity. This is because the node and yarn has likely been set up correctly there.



173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
 
# File 'lib/jets/builders/code_builder.rb', line 173

def compile_assets
  if ENV['JETS_SKIP_ASSETS']
    puts "Skip compiling assets".colorize(:yellow) # useful for debugging
    return
  end

  headline "Compling assets in current project directory"
  return unless webpacker_included?

  sh("yarn install")
  webpack_command = File.exist?("#{Jets.root}bin/webpack") ?
      "bin/webpack" :
      `which webpack`.strip
  sh "JETS_ENV=#{Jets.env} #{webpack_command}"
end

#compile_rails_assets ⇒ Object

This happens in the current app directory not the tmp code for simplicity This is because the node likely been set up correctly there.



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
 
# File 'lib/jets/builders/code_builder.rb', line 203

def compile_rails_assets
  return unless rails?

  if ENV['JETS_SKIP_ASSETS']
    puts "Skip compiling rack assets".colorize(:yellow) # useful for debugging
    return
  end

  return unless Jets.rack?

  # Need to capture JETS_ROOT since can be changed by Turbo mode
  jets_root = Jets.root
  Bundler.with_clean_env do
    # Switch gemfile for Afterburner mode
    gemfile = ENV['BUNDLE_GEMFILE']
    ENV['BUNDLE_GEMFILE'] = "#{jets_root}/rack/Gemfile"
    sh "cd #{jets_root} && bundle install"
    ENV['BUNDLE_GEMFILE'] = gemfile

    rails_assets(:clobber, jets_root: jets_root)
    rails_assets(:precompile, jets_root: jets_root)
  end
end

#copy_internal_jets_code ⇒ Object

We copy the files into the project because we cannot require simple functions directly since they are wrapped by an anonymous class. TODO: Do this with the other files we required the same way.



113
114
115
116
117
118
119
120
121
 
# File 'lib/jets/builders/code_builder.rb', line 113

def copy_internal_jets_code
  files = []
  files.each do |relative_path|
    src = File.expand_path("../internal/#{relative_path}", File.dirname(__FILE__))
    dest = "#{"#{stage_area}/code"}/#{relative_path}"
    FileUtils.mkdir_p(File.dirname(dest))
    FileUtils.cp(src, dest)
  end
end

#copy_project ⇒ Object

Copy project into temporary directory. Do this so we can keep the project directory untouched and we can also remove a bunch of unnecessary files like logs before zipping it up.



255
256
257
258
259
260
261
262
263
264
265
266
267
 
# File 'lib/jets/builders/code_builder.rb', line 255

def copy_project
  headline "Copying current project directory to temporary build area: #{"#{stage_area}/code"}"
  FileUtils.rm_rf("#{build_area}/stage") # clear out from previous build's stage area
  FileUtils.mkdir_p("#{build_area}/stage")
  FileUtils.rm_rf("#{stage_area}/code") # remove current code folder
  move_node_modules(Jets.root, Jets.build_root)
  begin
    # puts "cp -r #{@full_project_path} #{"#{stage_area}/code"}".colorize(:yellow) # uncomment to debug
    Jets::Util.cp_r(@full_project_path, "#{stage_area}/code")
  ensure
    move_node_modules(Jets.build_root, Jets.root) # move node_modules directory back
  end
end

#create_zip_files ⇒ Object 



65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/jets/builders/code_builder.rb', line 65

def create_zip_files
  folders = Md5.stage_folders
  # Md5.stage_folders ["stage/bundled", "stage/code"]
  folders.each do |folder|
    zip = Md5Zip.new(folder)
    if exist_on_s3?(zip.md5_name)
      puts "Already exists: s3://#{s3_bucket}/jets/code/#{zip.md5_name}"
    else
      zip = Md5Zip.new(folder)
      zip.create
    end
  end
end

#dir_size(folder) ⇒ Object

Thanks stackoverflow.com/questions/9354595/recursively-getting-the-size-of-a-directory Seems to overestimate a little bit but close enough.



125
126
127
128
129
130
 
# File 'lib/jets/builders/code_builder.rb', line 125

def dir_size(folder)
  Dir.glob(File.join(folder, '**', '*'))
    .select { |f| File.file?(f) }
    .map{ |f| File.size(f) }
    .inject(:+)
end

#disable_webpacker_middleware ⇒ Object 



165
166
167
168
169
 
# File 'lib/jets/builders/code_builder.rb', line 165

def disable_webpacker_middleware
  full_path = "#{"#{stage_area}/code"}/config/disable-webpacker-middleware.txt"
  FileUtils.mkdir_p(File.dirname(full_path))
  FileUtils.touch(full_path)
end

#exist_on_s3?(filename) ⇒ Boolean

Returns:

  • (Boolean) 


79
80
81
82
83
84
85
86
87
 
# File 'lib/jets/builders/code_builder.rb', line 79

def exist_on_s3?(filename)
  s3_key = "jets/code/#{filename}"
  begin
    s3.head_object(bucket: s3_bucket, key: s3_key)
    true
  rescue Aws::S3::Errors::NotFound
    false
  end
end

#generate_node_shims ⇒ Object 



57
58
59
60
61
62
63
 
# File 'lib/jets/builders/code_builder.rb', line 57

def generate_node_shims
  headline "Generating shims in the handlers folder."
  # Crucial that the Dir.pwd is in the tmp_code because for
  # Jets::Builders::app_files because Jets.boot set ups
  # autoload_paths and this is how project classes are loaded.
  Jets::Builders::HandlerGenerator.build!
end

#move_node_modules(source_folder, dest_folder) ⇒ Object

Move the node modules to the tmp build folder to speed up project copying. A little bit risky because a ctrl-c in the middle of the project copying results in a missing node_modules but user can easily rebuild that.

Tesing shows 6.623413 vs 0.027754 speed improvement.



274
275
276
277
278
279
280
 
# File 'lib/jets/builders/code_builder.rb', line 274

def move_node_modules(source_folder, dest_folder)
  source = "#{source_folder}/node_modules"
  dest = "#{dest_folder}/node_modules"
  if File.exist?(source)
    FileUtils.mv(source, dest)
  end
end

#package_ruby ⇒ Object 



307
308
309
310
311
312
313
314
315
316
317
318
 
# File 'lib/jets/builders/code_builder.rb', line 307

def package_ruby
  return if Jets.poly_only?

  check_agree
  ruby_packager.install
  reconfigure_rails # call here after "#{stage_area}/code" is available
  rack_packager.install
  ruby_packager.finish # by this time we have a /tmp/jets/demo/stage/code/vendor/gems
  rack_packager.finish

  build_lambda_layer
end

#rack_packager ⇒ Object 



302
303
304
 
# File 'lib/jets/builders/code_builder.rb', line 302

def rack_packager
  RackPackager.new("#{tmp_code}/rack")
end

#rails? ⇒ Boolean

Rudimentary rails detection

Returns:

  • (Boolean) 


235
236
237
238
239
 
# File 'lib/jets/builders/code_builder.rb', line 235

def rails?
  config_ru = "#{Jets.root}rack/config.ru"
  return false unless File.exist?(config_ru)
  !IO.readlines(config_ru).grep(/Rails.application/).empty?
end

#rails_assets(cmd, jets_root:) ⇒ Object 



227
228
229
230
231
232
 
# File 'lib/jets/builders/code_builder.rb', line 227

def rails_assets(cmd, jets_root:)
  # rake is available in both rails 4 and 5. rails command only in 5
  command = "bundle exec rake assets:#{cmd} --trace"
  command = "RAILS_ENV=#{Jets.env} #{command}" unless Jets.env.development?
  sh("cd #{jets_root}rack && #{command}")
end

#reconfigure_development_webpacker ⇒ Object

Bit hacky but this saves the user from accidentally forgetting to change this when they deploy a jets project in development mode



284
285
286
287
288
289
290
291
292
293
294
295
 
# File 'lib/jets/builders/code_builder.rb', line 284

def reconfigure_development_webpacker
  return unless Jets.env.development?
  headline "Reconfiguring webpacker development settings for AWS Lambda."

  webpacker_yml = "#{"#{stage_area}/code"}/config/webpacker.yml"
  return unless File.exist?(webpacker_yml)

  config = YAML.load_file(webpacker_yml)
  config["development"]["compile"] = false # force this to be false for deployment
  new_yaml = YAML.dump(config)
  IO.write(webpacker_yml, new_yaml)
end

#reconfigure_rails ⇒ Object

TODO: Move logic into plugin instead



332
333
334
 
# File 'lib/jets/builders/code_builder.rb', line 332

def reconfigure_rails
  ReconfigureRails.new("#{"#{stage_area}/code"}/rack").run
end

#ruby_packager ⇒ Object 



297
298
299
 
# File 'lib/jets/builders/code_builder.rb', line 297

def ruby_packager
  RubyPackager.new(tmp_code)
end

#ruby_version_supported? ⇒ Boolean

Returns:

  • (Boolean) 


350
351
352
353
354
355
356
357
358
 
# File 'lib/jets/builders/code_builder.rb', line 350

def ruby_version_supported?
  pattern = /(\d+)\.(\d+)\.(\d+)/
  md = RUBY_VERSION.match(pattern)
  ruby = {major: md[1], minor: md[2]}
  md = Jets::RUBY_VERSION.match(pattern)
  jets = {major: md[1], minor: md[2]}

  ruby[:major] == jets[:major] && ruby[:minor] == jets[:minor]
end

#s3_base_url ⇒ Object 



146
147
148
149
150
151
152
153
154
155
156
157
158
159
 
# File 'lib/jets/builders/code_builder.rb', line 146

def s3_base_url
  # Allow user to set assets.base_url
  #
  #   Jets.application.configure do
  #     config.assets.base_url = "https://cloudfront.com/my/base/path"
  #   end
  #
  return Jets.config.assets.base_url if Jets.config.assets.base_url

  region = Jets.aws.region

  asset_base_url = "https://s3-#{region}.amazonaws.com"
  "#{asset_base_url}/#{s3_bucket}/jets" # s3_base_url
end

#s3_bucket ⇒ Object 



161
162
163
 
# File 'lib/jets/builders/code_builder.rb', line 161

def s3_bucket
  Jets.aws.s3_bucket
end

#store_s3_base_url ⇒ Object

Store s3 base url is needed for asset serving from s3 later. Need to package this as part of the code so we have a reference to it. At this point the minimal stack exists, so we can grab it with the AWS API. We do not want to grab this as part of the live request because it is slow.



136
137
138
139
 
# File 'lib/jets/builders/code_builder.rb', line 136

def store_s3_base_url
  write_s3_base_url("#{stage_area}/code/config/s3_base_url.txt")
  write_s3_base_url("#{stage_area}/rack/config/s3_base_url.txt") if Jets.rack?
end

#tmp_code ⇒ Object 



365
366
367
 
# File 'lib/jets/builders/code_builder.rb', line 365

def tmp_code
  self.class.tmp_code
end

#webpacker_included? ⇒ Boolean

Returns:

  • (Boolean) 


189
190
191
192
193
194
195
196
197
198
199
 
# File 'lib/jets/builders/code_builder.rb', line 189

def webpacker_included?
  # Old code, leaving around for now:
  # Thanks: https://stackoverflow.com/questions/4195735/get-list-of-gems-being-used-by-a-bundler-project
  # webpacker_loaded = Gem.loaded_specs.keys.include?("webpacker")
  # return unless webpacker_loaded

  # Checking this way because when using jets standalone for Afterburner mode we don't want to run into
  # bundler gem collisions.  TODO: figure out the a better way to handle the collisions.
  lines = IO.readlines("#{Jets.root}Gemfile")
  lines.detect { |l| l =~ /webpacker/ }
end

#write_s3_base_url(full_path) ⇒ Object 



141
142
143
144
 
# File 'lib/jets/builders/code_builder.rb', line 141

def write_s3_base_url(full_path)
  FileUtils.mkdir_p(File.dirname(full_path))
  IO.write(full_path, s3_base_url)
end