Class: BoxGrinder::S3Plugin

Inherits:
BasePlugin show all
Defined in:
lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb

Instance Attribute Summary

Attributes inherited from BasePlugin

#deliverables, #plugin_info

Instance Method Summary collapse

Methods inherited from BasePlugin

#current_platform, #deliverables_exists?, #init, #initialize, #is_supported_os?, #is_supported_platform?, #merge_plugin_config, #read_plugin_config, #register_deliverable, #register_supported_os, #register_supported_platform, #run, #set_default_config_value, #subtype, #supported_oses, #validate_plugin_config

Methods included from Plugins

#plugin

Constructor Details

This class inherits a constructor from BoxGrinder::BasePlugin

Instance Method Details

#after_init ⇒ Object 



32
33
34
35
 
# File 'lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb', line 32

def after_init
  @ami_build_dir = "#{@dir.base}/ami"
  @ami_manifest = "#{@ami_build_dir}/#{@appliance_config.name}.ec2.manifest.xml"
end

#ami_by_manifest_key(ami_manifest_key) ⇒ Object 



222
223
224
225
226
227
 
# File 'lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb', line 222

def ami_by_manifest_key(ami_manifest_key)
  ami = @ec2.images.with_owner(@plugin_config['account_number']).
      filter("manifest-location","#{@plugin_config['bucket']}/#{ami_manifest_key.key}")
  return nil unless ami.any?
  ami.first
end

#ami_key(appliance_name, path) ⇒ Object 



229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
 
# File 'lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb', line 229

def ami_key(appliance_name, path)
  base_path = "#{@s3helper.parse_path(path)}#{appliance_name}/#{@appliance_config.os.name}/#{@appliance_config.os.version}/#{@appliance_config.version}.#{@appliance_config.release}"

  return "#{base_path}/#{@appliance_config.hardware.arch}" unless @plugin_config['snapshot']

  @log.info "Determining snapshot name"
  snapshot = 1
  while @s3helper.stub_s3obj(@bucket, "#{base_path}-snapshot-#{snapshot}/#{@appliance_config.hardware.arch}/").exists?
    snapshot += 1
  end
  # Reuse the last key (if there was one)
  snapshot -=1 if snapshot > 1 and @plugin_config['overwrite']

  "#{base_path}-snapshot-#{snapshot}/#{@appliance_config.hardware.arch}"
end

#asset_bucket(create_if_missing = true, permissions = :private) ⇒ Object 



143
144
145
146
147
148
149
 
# File 'lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb', line 143

def asset_bucket(create_if_missing = true, permissions = :private)
  @s3helper.bucket(:bucket => @plugin_config['bucket'],
    :create_if_missing => create_if_missing,
    :acl => permissions,
    :location_constraint => @s3_endpoints[@plugin_config['region']][:location]
  )
end

#bundle_image(deliverables) ⇒ Object 



151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb', line 151

def bundle_image(deliverables)
  if @plugin_config['snapshot']
    @log.debug "Removing bundled image from local disk..."
    FileUtils.rm_rf(@ami_build_dir)
  end

  return if File.exists?(@ami_build_dir)

  @log.info "Bundling AMI..."

  FileUtils.mkdir_p(@ami_build_dir)

  cmd_str = "euca-bundle-image --ec2cert #{File.dirname(__FILE__)}/src/cert-ec2.pem " <<
    "-i #{deliverables[:disk]} " <<
    "--kernel #{@plugin_config['kernel'] || @s3_endpoints[@plugin_config['region']][:kernel][@appliance_config.hardware.base_arch.intern][:aki]} " <<
    "-c #{@plugin_config['cert_file']} "<<
    "-k #{@plugin_config['key_file']} " <<
    "-u #{@plugin_config['account_number']} " <<
    "-r #{@appliance_config.hardware.base_arch} " <<
    "-d #{@ami_build_dir} "

  cmd_str << "--ramdisk #{@plugin_config['ramdisk']}" if @plugin_config['ramdisk']

  @exec_helper.execute(cmd_str, :redacted => [@plugin_config['account_number'], @plugin_config['key_file'], @plugin_config['cert_file']])

  @log.info "Bundling AMI finished."
end

#constraint_equal?(region, constraint) ⇒ Boolean

US default constraint is often represented as '' or nil



246
247
248
 
# File 'lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb', line 246

def constraint_equal?(region, constraint) 
  [region, constraint].collect{ |v| v.nil? || v == '' ? 'us-east-1' : v }.reduce(:==)
end

#deregister_image(ami_manifest_key) ⇒ Object 



212
213
214
215
216
217
218
219
220
 
# File 'lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb', line 212

def deregister_image(ami_manifest_key)
  if ami = ami_by_manifest_key(ami_manifest_key)
    @log.info "Preexisting image '#{ami.location}' for #{@appliance_config.name} will be de-registered, it had id: #{ami.id} (region: #{@plugin_config['region']})."
    ami.deregister
    @ec2helper.wait_for_image_death(ami)
  else # This occurs when the AMI is de-registered externally but the file structure is left intact in S3. In this instance, we simply overwrite and register the image as if it were "new".
    @log.debug "Possible dangling/unregistered AMI skeleton structure in S3, there is nothing to deregister"
  end
end

#execute ⇒ Object 



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb', line 87

def execute
  case @type
    when :s3
      upload_to_bucket(@previous_deliverables)
    when :cloudfront
      upload_to_bucket(@previous_deliverables, :public_read)
    when :ami
      @log.info Banner.message(S3::Messages::EPHEMERAL_MESSAGE)

      ami_dir = ami_key(@appliance_config.name, @plugin_config['path'])
      ami_manifest_key = @s3helper.stub_s3obj(@bucket, "#{ami_dir}/#{@appliance_config.name}.ec2.manifest.xml")

      @log.debug "Going to check whether s3 object exists"

      if ami_manifest_key.exists? and @plugin_config['overwrite']
        @log.info "Object exists, attempting to deregister an existing image"
        deregister_image(ami_manifest_key) # Remove existing image
        @s3helper.delete_folder(@bucket, ami_dir) # Avoid triggering dupe detection
      end

      if !ami_manifest_key.exists? or @plugin_config['snapshot']
        @log.info "Doing bundle/snapshot"
        bundle_image(@previous_deliverables)
        upload_image(ami_dir)
      end
      register_image(ami_manifest_key)
  end
end

#register_image(ami_manifest_key) ⇒ Object 



194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb', line 194

def register_image(ami_manifest_key)
  if ami = ami_by_manifest_key(ami_manifest_key)
    @log.info "Image for #{@appliance_config.name} is already registered under id: #{ami.id} (region: #{@plugin_config['region']})."
  else
    optmap = { :image_location =>  "#{@plugin_config['bucket']}/#{ami_manifest_key.key}" }
    
    unless @plugin_config['block_device_mappings'].empty?
      optmap.merge!(:block_device_mappings => @plugin_config['block_device_mappings']) 
    end

    @log.debug("Options map: #{optmap.inspect}")

    ami = @ec2.images.create(optmap)
    @ec2helper.wait_for_image_state(:available, ami)
    @log.info "Image for #{@appliance_config.name} successfully registered under id: #{ami.id} (region: #{@plugin_config['region']})."
  end
end

#upload_image(ami_dir) ⇒ Object 



179
180
181
182
183
184
185
186
187
188
189
190
191
192
 
# File 'lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb', line 179

def upload_image(ami_dir)
  @log.info "Uploading #{@appliance_config.name} AMI to bucket '#{@plugin_config['bucket']}'..."

  endpoint = @plugin_config['url'] || "http://#{@s3_endpoints[@plugin_config['region']][:endpoint]}"

  cmd_str = "euca-upload-bundle " << 
    "-U #{endpoint} " <<
    "-b #{@plugin_config['bucket']}/#{ami_dir} " <<
    "-m #{@ami_manifest} " <<
    "-a #{@plugin_config['access_key']} " << 
    "-s #{@plugin_config['secret_access_key']}"

  @exec_helper.execute(cmd_str, :redacted => [@plugin_config['access_key'], @plugin_config['secret_access_key']])
end

#upload_to_bucket(previous_deliverables, permissions = :private) ⇒ Object 



116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb', line 116

def upload_to_bucket(previous_deliverables, permissions = :private)
  register_deliverable(
      :package => "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz"
  )

  # quick and dirty workaround to use @deliverables[:package] later in code
  FileUtils.mv(@target_deliverables[:package], @deliverables[:package]) if File.exists?(@target_deliverables[:package])

  PackageHelper.new(@config, @appliance_config, :log => @log, :exec_helper => @exec_helper).package(File.dirname(previous_deliverables[:disk]), @deliverables[:package])

  remote_path = "#{@s3helper.parse_path(@plugin_config['path'])}#{File.basename(@deliverables[:package])}"
  size_m = File.size(@deliverables[:package])/1024**2
  s3_obj = @s3helper.stub_s3obj(@bucket,remote_path.gsub(/^\//, '').gsub(/\/\//, ''))
  # Does it really exist?
  obj_exists = s3_obj.exists?

  if !obj_exists or @plugin_config['overwrite']
    @log.info "Will overwrite existing file #{remote_path}" if obj_exists and @plugin_config['overwrite']
    @log.info "Uploading #{File.basename(@deliverables[:package])} (#{size_m}MB) to '#{@plugin_config['bucket']}#{remote_path}' path..."
    s3_obj.write(:file => @deliverables[:package],
                    :acl => permissions)
    @log.info "Appliance #{@appliance_config.name} uploaded to S3."
  else
    @log.info "File '#{@plugin_config['bucket']}#{remote_path}' already uploaded, skipping."
  end
end

#validate ⇒ Object

Raises:

  • (PluginValidationError) 


37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb', line 37

def validate
  set_default_config_value('overwrite', false)
  set_default_config_value('kernel', false)
  set_default_config_value('ramdisk', false)
  set_default_config_value('path', '/')
  set_default_config_value('region', 'us-east-1')

  set_default_config_value('block_device_mappings', {}) do |k, m, v|
    EC2Helper::block_device_mappings_validator(k, m, v)
  end
  
  validate_plugin_config(['bucket', 'access_key', 'secret_access_key'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin')

  subtype(:ami) do
    set_default_config_value('snapshot', false)
    validate_plugin_config(['cert_file', 'key_file', 'account_number'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin')

    raise PluginValidationError, "AWS certificate file doesn't exists, please check the path: '#{@plugin_config['cert_file']}'." unless File.exists?(File.expand_path(@plugin_config['cert_file']))
    raise PluginValidationError, "AWS key file doesn't exists, please check the path: '#{@plugin_config['key_file']}'." unless File.exists?(File.expand_path(@plugin_config['key_file']))
  end

  @s3_endpoints = S3Helper::endpoints
  raise PluginValidationError, "Invalid region specified: #{@plugin_config['region']}. This plugin is only aware of the following regions: #{@s3_endpoints.keys.join(", ")}." unless @s3_endpoints.has_key?(@plugin_config['region'])

  @plugin_config['account_number'] = @plugin_config['account_number'].to_s.gsub(/-/, '')

  # Set global AWS configuration
  AWS.config(:access_key_id => @plugin_config['access_key'],
    :secret_access_key => @plugin_config['secret_access_key'],
    :ec2_endpoint => EC2Helper::endpoints[@plugin_config['region']][:endpoint],
    :s3_endpoint => @s3_endpoints[@plugin_config['region']][:endpoint],
    :max_retries => 5,
    :use_ssl => @plugin_config['use_ssl'])

  @ec2 = AWS::EC2.new
  @s3 = AWS::S3.new
  @s3helper = S3Helper.new(@ec2, @s3, :log => @log)
  @ec2helper = EC2Helper.new(@ec2, :log => @log)

  subtype(:ami) do
    # If there is an existing bucket, determine whether its location_constraint matches the region selected
    if existing_bucket = asset_bucket(false)
      raise PluginValidationError, "Existing bucket #{@plugin_config['bucket']} has a location constraint that does not match the region selected. " <<
        "AMI region and bucket location constraint must match." unless constraint_equal?(@plugin_config['region'], existing_bucket.location_constraint)
    end
  end

  @bucket = asset_bucket(true)
end