Class: Cnvrg::Downloader::Client

Inherits:
Object
  • Object
show all
Defined in:
lib/cnvrg/downloader/client.rb

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(params) ⇒ Client

Returns a new instance of Client.



9
10
11
12
13
14
 
# File 'lib/cnvrg/downloader/client.rb', line 9

def initialize(params)
  @key = ''
  @iv = ''
  @client = ''
  @bucket = ''
end

Class Method Details

.factory(params) ⇒ Object 



82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/cnvrg/downloader/client.rb', line 82

def self.factory(params)
  params = params.as_json
  case params["storage"]
  when 's3', 'minio'
    return Cnvrg::Downloader::Clients::S3Client.new(sts_path: params["path_sts"], access_key: params["sts_a"], secret: params["sts_s"], session_token: params["sts_st"], region: params["region"], bucket: params["bucket"], encryption: params["encryption"], endpoint: params["endpoint"], storage: params["storage"])
  when 'azure'
    azure_params = params.symbolize_keys.slice(*[:storage_account_name, :storage_access_key, :container, :sts])
    return Cnvrg::Downloader::Clients::AzureClient.new(**azure_params)
  when 'gcp'
    return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"])
  end
end

Instance Method Details

#cut_prefix(prefix, file) ⇒ Object 



29
30
31
 
# File 'lib/cnvrg/downloader/client.rb', line 29

def cut_prefix(prefix, file)
  file.gsub(prefix, '').gsub(/^\/*/, '')
end

#decrypt(str) ⇒ Object 



49
50
51
 
# File 'lib/cnvrg/downloader/client.rb', line 49

def decrypt(str)
  Cnvrg::Helpers.decrypt(@key, @iv, str)
end

#download(storage_path, local_path) ⇒ Object 



33
34
35
 
# File 'lib/cnvrg/downloader/client.rb', line 33

def download(storage_path, local_path)
  ### need to be implemented..
end

#extract_key_iv(sts_path) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/cnvrg/downloader/client.rb', line 16

def extract_key_iv(sts_path)
  count = 20
  begin
    count += 1
    sts = open(sts_path, {ssl_verify_mode: 0}).read rescue nil
  rescue => e
    Cnvrg::Logger.log_error(e)
    retry if count <= 20
    raise StandardError.new("Cant access storage: #{e.message}")
  end
  sts.split("\n")
end

#mkdir(path, recursive: false) ⇒ Object 



41
42
43
 
# File 'lib/cnvrg/downloader/client.rb', line 41

def mkdir(path, recursive: false)
  recursive ? FileUtils.mkdir_p(path) : FileUtils.mkdir(path)
end

#prepare_download(local_path) ⇒ Object 



45
46
47
 
# File 'lib/cnvrg/downloader/client.rb', line 45

def prepare_download(local_path)
  mkdir(File.dirname(local_path), recursive: true)
end

#safe_upload(storage_path, local_path) ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/cnvrg/downloader/client.rb', line 53

def safe_upload(storage_path, local_path)
  n = 1
  error = nil
  while n <= RETRIES
    begin
      self.upload(storage_path, local_path)
      error = nil
      break
    rescue => e
      backoff_time_seconds = backoff_time(n)

      message = "Got error: #{e.class.name} with message: #{e.message} while uploading a single file: #{local_path}, retry: #{n} of: #{RETRIES}"
      if n < RETRIES
        message += ", next retry in: #{backoff_time_seconds} seconds"
      else
        message += ", done retry, continuing to the next file"
      end
      Cnvrg::Logger.log_error_message(message)

      sleep backoff_time_seconds

      n += 1
      error = e
    end
  end
  raise error if error.present?
  true
end

#upload(storage_path, local_path) ⇒ Object 



37
38
39
 
# File 'lib/cnvrg/downloader/client.rb', line 37

def upload(storage_path, local_path)
  ### need to be implemented..
end