Class: Cnvrg::Downloader::Clients::S3Client

Inherits:
Cnvrg::Downloader::Client show all
Defined in:
lib/cnvrg/downloader/clients/s3_client.rb

Instance Method Summary collapse

Methods inherited from Cnvrg::Downloader::Client

#decrypt, #extract_key_iv, factory, #mkdir, #prepare_download

Constructor Details

#initialize(sts_path: nil, access_key: nil, secret: nil, session_token: nil, region: nil, bucket: nil, encryption: nil, endpoint: nil, storage: nil) ⇒ S3Client

Returns a new instance of S3Client.



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/cnvrg/downloader/clients/s3_client.rb', line 5

def initialize(sts_path: nil, access_key: nil, secret: nil, session_token: nil, region: nil, bucket: nil, encryption: nil, endpoint: nil, storage: nil)
  @key, @iv = extract_key_iv(sts_path)
  @access_key = Cnvrg::Helpers.decrypt(@key, @iv, access_key)
  @secret = Cnvrg::Helpers.decrypt(@key, @iv, secret)
  @session_token = Cnvrg::Helpers.decrypt(@key, @iv, session_token)
  @region = Cnvrg::Helpers.decrypt(@key, @iv, region)
  @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket)
  @endpoint = Cnvrg::Helpers.decrypt(@key, @iv, endpoint)
  options = {
      :access_key_id => @access_key,
      :secret_access_key => @secret,
      :session_token => @session_token,
      :region => @region,
      :http_open_timeout => 60, :retry_limit => 20
  }
  if storage == 'minio'
    options.delete(:session_token)
    options = options.merge({
                                :force_path_style => true,
                                :ssl_verify_peer => false,
                                :endpoint => @endpoint,
                            })
  end

  @client = Aws::S3::Client.new(options)
  @bucket = Aws::S3::Resource.new(client: @client).bucket(@bucket_name)
  @upload_options = {:use_accelerate_endpoint => storage == 's3'}
  if encryption.present?
    @upload_options[:server_side_encryption] = encryption
  end
end

Instance Method Details

#download(storage_path, local_path) ⇒ Object 



37
38
39
40
41
42
43
44
45
 
# File 'lib/cnvrg/downloader/clients/s3_client.rb', line 37

def download(storage_path, local_path)
  prepare_download(local_path)
  storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path)
  File.open(local_path, 'w+') do |file|
    resp = @client.get_object({bucket: @bucket_name,
                               key: storage_path}, target: file)
  end
  resp
end

#upload(storage_path, local_path) ⇒ Object 



47
48
49
50
51
52
 
# File 'lib/cnvrg/downloader/clients/s3_client.rb', line 47

def upload(storage_path, local_path)
  ### storage path is the path inside s3 (after the bucket)
  # local path is fullpath for the file /home/ubuntu/user.../hazilim.py
  o = @bucket.object(storage_path)
  o.upload_file(local_path, @upload_options)
end