Class: Cnvrg::Downloader::Clients::AzureClient

Inherits:
Cnvrg::Downloader::Client show all
Defined in:
lib/cnvrg/downloader/clients/azure_client.rb

Instance Method Summary collapse

Methods inherited from Cnvrg::Downloader::Client

#cut_prefix, #decrypt, #extract_key_iv, factory, #link_file, #mkdir, #prepare_download, #safe_download, #safe_operation, #safe_upload

Constructor Details

#initialize(storage_account_name: nil, storage_access_key: nil, container: nil, sts: nil) ⇒ AzureClient

Returns a new instance of AzureClient.



9
10
11
12
13
14
 
# File 'lib/cnvrg/downloader/clients/azure_client.rb', line 9

def initialize(storage_account_name: nil, storage_access_key: nil, container: nil, sts: nil)
  @key, @iv = extract_key_iv(sts)
  @account_name = Cnvrg::Helpers.decrypt(@key, @iv, )
  @access_key = Cnvrg::Helpers.decrypt(@key, @iv, storage_access_key)
  @container = Cnvrg::Helpers.decrypt(@key, @iv, container)
end

Instance Method Details

#download(storage_path, local_path, decrypt: true) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/cnvrg/downloader/clients/azure_client.rb', line 16

def download(storage_path, local_path, decrypt: true)
  prepare_download(local_path)
  storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt

  # We generate a temp uri in order to stream the file instead of using "get_blob" that overflows memory
  uri = client.send(:blob_uri, @container, storage_path)


  expiring_url = self.signed_uri_custom(
    uri,
    false,
    service: 'b',
    resource: 'b',
    permissions: 'r',
    start: (Time.now - (5 * 60)).utc.iso8601, # start 5 minutes ago
    expiry: (Time.now + 60 * 60 * 2).utc.iso8601 # expire in 2 hours
  )
  # Stream the file without loading it all into memory
  download = open(expiring_url)
  IO.copy_stream(download, local_path)
end

#fetch_files(prefix: nil, marker: nil, limit: 10000) ⇒ Object 



67
68
69
70
71
72
 
# File 'lib/cnvrg/downloader/clients/azure_client.rb', line 67

def fetch_files(prefix: nil, marker: nil, limit: 10000)
  blobs = client.list_blobs(@container, prefix: prefix, max_results: limit, marker: marker)
  next_marker = blobs.continuation_token
  files = blobs.map{|x| x.name}
  [files, next_marker]
end

#signed_uri_custom(uri, use_account_sas, options) ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/cnvrg/downloader/clients/azure_client.rb', line 38

def signed_uri_custom(uri, , options)
  # url sent to generate_service_sas_token should be DECODED (file names with spaces should not be encoded with %20)
  url = URI.decode(uri.path)
  generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@account_name, @access_key)

  CGI::parse(uri.query || "").inject({}) { |memo, (k, v)| memo[k.to_sym] = v; memo }

  if options[:service] == (nil) && uri.host != (nil)
    host_splits = uri.host.split(".")
    options[:service] = host_splits[1].chr if host_splits.length > 1 && host_splits[0] == @account_name
  end

  sas_params = if 
                 generator.(options)
               else
                 generator.generate_service_sas_token(url, options)
               end

  URI.parse(uri.to_s + (uri.query.nil? ? "?" : "&") + sas_params)
end

#upload(storage_path, local_path) ⇒ Object 



59
60
61
62
63
64
65
 
# File 'lib/cnvrg/downloader/clients/azure_client.rb', line 59

def upload(storage_path, local_path)
  begin
    client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
  rescue => e
    raise e
  end
end