Method: Cnvrg::Files#download_multpile_files_s3

Defined in:

lib/cnvrg/files.rb

#download_multpile_files_s3(files, project_home) ⇒ Object

# File 'lib/cnvrg/files.rb', line 458

def download_multpile_files_s3(files, project_home)
  begin
      props = Cnvrg::Helpers.get_s3_props(files)
      client = props[:client]
      iv = props[:iv]
      key = props[:key]
      bucket = props[:bucket]
      download_succ_count = 0
      parallel_options = {
          :progress => {
              :title => "Download Progress",
              :progress_mark => '=',
              :format => "%b>>%i| %p%% %t",
              :starting_at => 0,
              :total => files["keys"].size,
              :autofinish => true
          },
          in_threads: Cnvrg::Helpers.parallel_threads,
          isolation: true
      }
      Parallel.map(files["keys"], parallel_options) do |f|

        file_path = f["name"]
        if file_path.end_with? "/"
          # dir
          if download_dir(file_path, file_path, project_home)
            download_succ_count += 1
          else
            return Cnvrg::Result.new(false,"Could not create directory: #{file_path}")
            raise Parallel::Kill
          end
        else
          # blob
          begin
          file_key =  Cnvrg::Helpers.decrypt(key,iv, f["path"])
          resp = false
          File.open(project_home+"/"+file_path, 'w+') do |file|
            resp = client.get_object({bucket:bucket,
                                      key:file_key}, target: file)
            #TODO update idx here!
          end
          if resp
            download_succ_count +=1
          else
            return Cnvrg::Result.new(false,"Could not create file: #{file_path}")
          end



          rescue => e
            return Cnvrg::Result.new(false,"Could not create file: #{file_path}", e.message, e.backtrace)
            raise Parallel::Kill
          end



          end
      end
    if download_succ_count == files["keys"].size
      return Cnvrg::Result.new(true,"Done.\nDownloaded #{download_succ_count} files")
    end
  rescue => e

    return Cnvrg::Result.new(false,"Could not download some files", e.message, e.backtrace)
  end




end