Class: VCAP::Services::Base::AsyncJob::Serialization::BaseImportFromURLJob

Inherits:
SerializationJob show all
Defined in:
lib/base/job/serialization.rb

Overview

Create a new snapshot of service using given URL

Constant Summary

Constants included from VCAP::Services::Base::AsyncJob::Snapshot

VCAP::Services::Base::AsyncJob::Snapshot::FILTER_KEYS, VCAP::Services::Base::AsyncJob::Snapshot::MAX_NAME_LENGTH, VCAP::Services::Base::AsyncJob::Snapshot::SNAPSHOT_ID, VCAP::Services::Base::AsyncJob::Snapshot::SNAPSHOT_KEY_PREFIX

Constants included from VCAP::Services::Base::AsyncJob::Serialization

SERIALIZATION_KEY_PREFIX

Instance Attribute Summary collapse

Attributes inherited from SerializationJob

#name

Instance Method Summary collapse

Methods inherited from SerializationJob

#create_lock, #delete_download_token, #get_dump_path, #handle_error, #init_worker_logger, #initialize, #parse_config, queue_lookup_key, #required_options, select_queue, #snapshot_filename, #update_download_token, #validate_input

Methods included from Error

#failure, #internal_fail, #parse_msg, #success, #timeout_fail

Methods included from VCAP::Services::Base::AsyncJob::Snapshot

#client, #delete_snapshot, #filter_keys, #fmt_time, #new_snapshot_id, redis_connect, redis_init, #save_snapshot, #service_snapshots, #service_snapshots_count, #snapshot_details, #snapshot_filepath, #update_name

Methods included from VCAP::Services::Base::AsyncJob::Serialization

#fmt_error, redis_connect, #redis_key

Constructor Details

This class inherits a constructor from VCAP::Services::Base::AsyncJob::Serialization::SerializationJob

Instance Attribute Details

#snapshot_id ⇒ Object (readonly)

Returns the value of attribute snapshot_id.



203
204
205
 
# File 'lib/base/job/serialization.rb', line 203

def snapshot_id
  @snapshot_id
end

#url ⇒ Object (readonly)

Returns the value of attribute url.



203
204
205
 
# File 'lib/base/job/serialization.rb', line 203

def url
  @url
end

Instance Method Details

#execute ⇒ Object

empty by default



297
298
299
 
# File 'lib/base/job/serialization.rb', line 297

def execute
  true
end

#fetch_url(url, file_path) ⇒ Object

Fetch remote uri and stream content to file.



258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
 
# File 'lib/base/job/serialization.rb', line 258

def fetch_url(url, file_path)
  max_download_size = (@config["serialization"] && @config["serialization"]["max_download_size_mb"] || 10).to_i * 1024 * 1024 # 10M by default
  max_redirects = @config["serialization"] && @config["serialization"]["max_download_redirects"] || 5

  File.open(file_path, "wb+") do |f|
    c = Curl::Easy.new(url)
    # force use ipv4 dns
    c.resolve_mode = :ipv4
    # auto redirect
    c.follow_location = true
    c.max_redirects = max_redirects

    c.on_header do |header|
      if c.downloaded_content_length > max_download_size
        raise ServiceError.new(ServiceError::FILESIZE_TOO_LARGE, url, c.downloaded_content_length, max_download_size)
      end

      header.size
    end

    bytes_downloaded = 0
    c.on_body do |data|
      # calculate bytes downloaded for chucked response
      bytes_downloaded += data.size
      if bytes_downloaded > max_download_size
        raise ServiceError.new(ServiceError::FILESIZE_TOO_LARGE, url, bytes_downloaded, max_download_size)
      end
      f.write(data)
    end

    begin
      c.perform
    rescue Curl::Err::TooManyRedirectsError
      raise ServiceError.new(ServiceError::TOO_MANY_REDIRECTS, url, max_redirects)
    end
  end
end

#perform ⇒ Object

Sub class should return true for a successful import job.



206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
 
# File 'lib/base/job/serialization.rb', line 206

def perform
  begin
    required_options :service_id, :url
    @name = options["service_id"]
    @url = options["url"]
    @logger.info("Launch job: #{self.class} for #{name} with options:#{options.inspect}")

    lock = create_lock
    lock.lock do
      quota = @config["snapshot_quota"]
      if quota
        current = service_snapshots_count(name)
        @logger.debug("Current snapshots count for #{name}: #{current}, max: #{quota}")
        raise ServiceError.new(ServiceError::OVER_QUOTA, name, current, quota) if current >= quota
      end

      @snapshot_id = new_snapshot_id
      @snapshot_path = get_dump_path(name, snapshot_id)
      @snapshot_file = File.join(@snapshot_path, "#{snapshot_id}.zip")

      # clean any data in snapshot folder
      FileUtils.rm_rf(@snapshot_path)
      FileUtils.mkdir_p(@snapshot_path)

      fetch_url(url, @snapshot_file)
      raise ServiceError.new(ServiceError::BAD_SERIALIZED_DATAFILE, url) unless validate_package(@snapshot_file)

      result = execute
      @logger.info("Results of import from url: #{result}")

      snapshot = {
        :snapshot_id => snapshot_id,
        :size => File.open(@snapshot_file) {|f| f.size },
        :date => fmt_time,
        :file => "#{snapshot_id}.zip"
      }
      save_snapshot(name, snapshot)
      @logger.info("Create new snapshot for #{name}:#{snapshot}")

      completed(Yajl::Encoder.encode(filter_keys(snapshot)))
      @logger.info("Complete job: #{self.class} for #{name}")
    end
  rescue => e
    handle_error(e)
    delete_snapshot(name, snapshot_id) if snapshot_id
    FileUtils.rm_rf(@snapshot_path) if @snapshot_path
  ensure
    set_status({:complete_time => Time.now.to_s})
  end
end