Class: Cnvrg::Downloader::Clients::AzureClient

Inherits:
Cnvrg::Downloader::Client show all
Defined in:
lib/cnvrg/downloader/clients/azure_client.rb

Instance Method Summary collapse

Methods inherited from Cnvrg::Downloader::Client

#cut_prefix, #decrypt, #extract_key_iv, factory, #link_file, #mkdir, #prepare_download, #safe_download, #safe_operation, #safe_upload

Constructor Details

#initialize(storage_account_name: nil, storage_access_key: nil, container: nil, sts: nil) ⇒ AzureClient

Returns a new instance of AzureClient.



9
10
11
12
13
14
 
# File 'lib/cnvrg/downloader/clients/azure_client.rb', line 9

def initialize(storage_account_name: nil, storage_access_key: nil, container: nil, sts: nil)
  @key, @iv = extract_key_iv(sts)
  @account_name = Cnvrg::Helpers.decrypt(@key, @iv, )
  @access_key = Cnvrg::Helpers.decrypt(@key, @iv, storage_access_key)
  @container = Cnvrg::Helpers.decrypt(@key, @iv, container)
end

Instance Method Details

#download(storage_path, local_path, decrypt: true) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/cnvrg/downloader/clients/azure_client.rb', line 16

def download(storage_path, local_path, decrypt: true)
  prepare_download(local_path)

  storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt

  # We generate a temp uri in order to stream the file instead of using "get_blob" that overflows memory
  uri = client.send(:blob_uri, @container, storage_path)

  generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@account_name, @access_key)

  expiring_url = generator.signed_uri(
    uri,
    false,
    service: 'b',
    resource: 'b',
    permissions: 'r',
    start: (Time.now - (5 * 60)).utc.iso8601, # start 5 minutes ago
    expiry: (Time.now + 60 * 60 * 2).utc.iso8601 # expire in 2 hours
  )

  # Stream the file without loading it all into memory
  download = open(expiring_url)
  IO.copy_stream(download, local_path)
end

#fetch_files(prefix: nil, marker: nil, limit: 10000) ⇒ Object 



49
50
51
52
53
54
 
# File 'lib/cnvrg/downloader/clients/azure_client.rb', line 49

def fetch_files(prefix: nil, marker: nil, limit: 10000)
  blobs = client.list_blobs(@container, prefix: prefix, max_results: limit, marker: marker)
  next_marker = blobs.continuation_token
  files = blobs.map{|x| x.name}
  [files, next_marker]
end

#upload(storage_path, local_path) ⇒ Object 



41
42
43
44
45
46
47
 
# File 'lib/cnvrg/downloader/clients/azure_client.rb', line 41

def upload(storage_path, local_path)
  begin
    client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
  rescue => e
    raise e
  end
end