Class: Lono::FileUploader

Inherits:
Object
  • Object
show all
Extended by:
Memoist
Includes:
Template::AwsService
Defined in:
lib/lono/file_uploader.rb

Instance Method Summary collapse

Constructor Details

#initialize(blueprint, options = {}) ⇒ FileUploader

Returns a new instance of FileUploader.



6
7
8
9
10
 
# File 'lib/lono/file_uploader.rb', line 6

def initialize(blueprint, options={})
  @blueprint, @options = blueprint, options
  @checksums = {}
  @prefix = "#{folder_key}/#{Lono.env}/#{blueprint}/files" # s3://s3-bucket/folder/development/files
end

Instance Method Details

#folder_key ⇒ Object

The folder_key is the s3_folder setting with the s3 bucket.

Example:

s3_bucket('s3://mybucket/files/storage/path')
=> files/storage/path



106
107
108
109
110
 
# File 'lib/lono/file_uploader.rb', line 106

def folder_key
  return nil if @options[:noop] # to get spec passing
  return nil unless s3_folder
  s3_folder.sub('s3://','').split('/')[1..-1].join('/')
end

#load_checksums! ⇒ Object 



23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/lono/file_uploader.rb', line 23

def load_checksums!
  resp = s3.list_objects(bucket: s3_bucket, prefix: @prefix)
  resp.contents.each do |object|
    # key does not include the bucket name
    #    full path = s3://my-bucket/s3_folder/files/production/my-template.yml
    #    key = s3_folder/files/production/my-template.yml
    # etag is the checksum as long as the file is not a multi-part file upload
    # it has extra double quotes wrapped around it.
    #    etag = "\"9cb437490cee2cc96101baf326e5ca81\""
    @checksums[object.key] = strip_surrounding_quotes(object.etag)
  end
  @checksums
end

#md5(path) ⇒ Object

used for file_s3_key helper



38
39
40
 
# File 'lib/lono/file_uploader.rb', line 38

def md5(path)
  Digest::MD5.file(path).to_s[0..7]
end

#md5_key(path) ⇒ Object 



43
44
45
46
47
48
49
50
51
 
# File 'lib/lono/file_uploader.rb', line 43

def md5_key(path)
  pretty_path = path.sub(/^\.\//, '')
  key = "#{@prefix}/#{pretty_path.sub(%r{app/files/},'')}"
  # add the short md5sum to the file
  key = key.sub(/\.(\w+)$/,'') # strip extension
  ext = $1
  md5 = md5(path)
  "#{key}-#{md5}.#{ext}"
end

#remote_checksum(key) ⇒ Object

key example: cloudformation/production/files/lifecycle-0719ab81.zip s3 path: s3://boltops-dev/cloudformation/production/files/lifecycle-0719ab81.zip s3_folder: s3://boltops-dev/cloudformation



88
89
90
 
# File 'lib/lono/file_uploader.rb', line 88

def remote_checksum(key)
  @checksums[key]
end

#s3 ⇒ Object 



120
121
122
 
# File 'lib/lono/file_uploader.rb', line 120

def s3
  @s3 ||= Aws::S3::Client.new
end

#s3_bucket ⇒ Object 



92
93
94
 
# File 'lib/lono/file_uploader.rb', line 92

def s3_bucket
  s3_folder.sub('s3://','').split('/').first
end

#s3_folder ⇒ Object 



96
97
98
99
 
# File 'lib/lono/file_uploader.rb', line 96

def s3_folder
  setting = Lono::Setting.new
  setting.s3_folder
end

#s3_resource ⇒ Object 



116
117
118
 
# File 'lib/lono/file_uploader.rb', line 116

def s3_resource
  @s3_resource ||= Aws::S3::Resource.new
end

#s3_upload(path) ⇒ Object

Inputs:

path: can be full path or relative path



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/lono/file_uploader.rb', line 57

def s3_upload(path)
  path = path.gsub("#{Lono.root}/",'') # remove Lono.root
  pretty_path = path.sub(/^\.\//, '')
  key = md5_key(path)
  s3_full_path = "s3://#{s3_bucket}/#{key}"

  local_checksum = Digest::MD5.hexdigest(IO.read(path))
  remote_checksum = remote_checksum(key)
  if local_checksum == remote_checksum
    puts("Not modified: #{pretty_path} to #{s3_full_path}".color(:yellow)) unless @options[:noop]
    return # do not upload unless the checksum has changed
  else
    # Example output:
    # Uploaded: app/files/docker.yml to s3://boltops-dev/s3_folder/templates/development/docker.yml
    # Uploaded: app/files/ecs/private.yml to s3://boltops-dev/s3_folder/templates/development/ecs/private.yml
    message = "Uploading: #{pretty_path} to #{s3_full_path}".color(:green)
    message = "NOOP: #{message}" if @options[:noop]
    puts message
  end

  resp = s3.put_object(
    body: IO.read(path),
    bucket: s3_bucket,
    key: key,
    storage_class: "REDUCED_REDUNDANCY"
  ) unless @options[:noop]
end

#strip_surrounding_quotes(string) ⇒ Object 



112
113
114
 
# File 'lib/lono/file_uploader.rb', line 112

def strip_surrounding_quotes(string)
  string.sub(/^"/,'').sub(/"$/,'')
end

#upload_all ⇒ Object 



12
13
14
15
16
17
18
19
20
21
 
# File 'lib/lono/file_uploader.rb', line 12

def upload_all
  puts "Uploading app/files..."
  load_checksums!

  pattern = "#{Lono.blueprint_root}/app/files/**/*"
  Dir.glob(pattern).each do |path|
    next if ::File.directory?(path)
    s3_upload(path)
  end
end