Class: Cnvrg::Downloader::Clients::S3Client

Inherits:
Cnvrg::Downloader::Client show all
Defined in:
lib/cnvrg/downloader/clients/s3_client.rb

Instance Method Summary collapse

Methods inherited from Cnvrg::Downloader::Client

#cut_prefix, #decrypt, #extract_key_iv, factory, #link_file, #mkdir, #prepare_download, #safe_download, #safe_operation, #safe_upload

Constructor Details

#initialize(sts_path: nil, access_key: nil, secret: nil, session_token: nil, region: nil, bucket: nil, encryption: nil, endpoint: nil, storage: nil) ⇒ S3Client

Returns a new instance of S3Client.



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/cnvrg/downloader/clients/s3_client.rb', line 5

def initialize(sts_path: nil, access_key: nil, secret: nil, session_token: nil, region: nil, bucket: nil, encryption: nil, endpoint: nil, storage: nil)
  @key, @iv = extract_key_iv(sts_path)
  @access_key = Cnvrg::Helpers.decrypt(@key, @iv, access_key)
  @secret = Cnvrg::Helpers.decrypt(@key, @iv, secret)
  @session_token = Cnvrg::Helpers.decrypt(@key, @iv, session_token)
  @region = Cnvrg::Helpers.decrypt(@key, @iv, region)
  @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket)
  @endpoint = Cnvrg::Helpers.decrypt(@key, @iv, endpoint)
  options = {
      :access_key_id => @access_key,
      :secret_access_key => @secret,
      :session_token => @session_token,
      :region => @region,
      :http_open_timeout => 60, :retry_limit => 20
  }
  if storage == 'minio'
    options.delete(:session_token)
    options = options.merge({
                                :force_path_style => true,
                                :ssl_verify_peer => false,
                                :endpoint => @endpoint,
                            })
  end

  @options = options

  #@client = Aws::S3::Client.new(options)
  #@bucket = Aws::S3::Resource.new(client: @client).bucket(@bucket_name)
  @upload_options = {:use_accelerate_endpoint => storage == 's3'}
  if encryption.present?
    @upload_options[:server_side_encryption] = encryption
  end
end

Instance Method Details

#download(storage_path, local_path, decrypt: true) ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/cnvrg/downloader/clients/s3_client.rb', line 39

def download(storage_path, local_path, decrypt: true)
  prepare_download(local_path)
  storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt
  resp = nil
  File.open(local_path, 'w+') do |file|
    resp = aws_client.get_object({bucket: @bucket_name, key: storage_path}, target: file)
  end
  resp
rescue => e
  Cnvrg::Logger.log_error(e)
  raise e
end

#fetch_files(prefix: nil, marker: nil, limit: 1000) ⇒ Object 



62
63
64
65
 
# File 'lib/cnvrg/downloader/clients/s3_client.rb', line 62

def fetch_files(prefix: nil, marker: nil, limit: 1000)
  batch_files = aws_bucket.objects(prefix: prefix, marker: marker).first(limit)
  batch_files.to_a.map(&:key)
end

#upload(storage_path, local_path) ⇒ Object 



52
53
54
55
56
57
58
59
60
 
# File 'lib/cnvrg/downloader/clients/s3_client.rb', line 52

def upload(storage_path, local_path)
  ### storage path is the path inside s3 (after the bucket)
  # local path is fullpath for the file /home/ubuntu/user.../hazilim.py
  o = aws_bucket.object(storage_path)
  success = o.upload_file(local_path, @upload_options)
  return success
rescue => e
  raise e
end