Class: Backarch::ParallelDownloader

Inherits:
Object
  • Object
show all
Defined in:
lib/backarch/parallel_downloader.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(destination, s3_inputs) ⇒ ParallelDownloader

Returns a new instance of ParallelDownloader.



13
14
15
16
17
18
19
 
# File 'lib/backarch/parallel_downloader.rb', line 13

def initialize(destination, s3_inputs)
  validate_parallel_installed

  @s3_inputs = s3_inputs
  @destination = destination
  @outputs = generate_output_files(destination, s3_inputs)
end

Instance Attribute Details

#destination ⇒ Object (readonly)

Returns the value of attribute destination.



5
6
7
 
# File 'lib/backarch/parallel_downloader.rb', line 5

def destination
  @destination
end

#outputs ⇒ Object (readonly)

Returns the value of attribute outputs.



5
6
7
 
# File 'lib/backarch/parallel_downloader.rb', line 5

def outputs
  @outputs
end

#s3_inputs ⇒ Object (readonly)

Returns the value of attribute s3_inputs.



5
6
7
 
# File 'lib/backarch/parallel_downloader.rb', line 5

def s3_inputs
  @s3_inputs
end

Class Method Details

.download_folder(s3_path, destination) ⇒ Object 



7
8
9
10
11
 
# File 'lib/backarch/parallel_downloader.rb', line 7

def self.download_folder(s3_path, destination)
  s3_inputs = retrieve_volumes_from_s3(s3_path)
  instance = new destination, s3_inputs
  instance.download
end

Instance Method Details

#download ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/backarch/parallel_downloader.rb', line 21

def download
  LOG.info "Downloading: #{s3_inputs}"
  tuples = s3_inputs.zip(outputs)
  file = Tempfile.new('parallel_input')
  tuples.each do |pair|
    file.puts "s3cmd get -v #{pair[0]} #{pair[1]}"
  end
  file.close

  command = "parallel --no-notice -j 70% --joblog #{joblog_path} -a #{file.path}"
  LOG.info "Running in parallel: #{command}"
  if !system(command)
    LOG.info "Parallel downloads failed. Retrying."
    raise StandardError, "Failed to complete parallel download" unless system(command + " --resume-failed")
  end
ensure
  file.unlink
end