Class: Fog::AzureRM::Storage::Real
- Inherits:
-
Object
- Object
- Fog::AzureRM::Storage::Real
show all
- Includes:
- Utilities::General
- Defined in:
- lib/fog/azurerm/storage.rb,
lib/fog/azurerm/requests/storage/get_blob.rb,
lib/fog/azurerm/requests/storage/copy_blob.rb,
lib/fog/azurerm/requests/storage/get_object.rb,
lib/fog/azurerm/requests/storage/list_blobs.rb,
lib/fog/azurerm/requests/storage/copy_object.rb,
lib/fog/azurerm/requests/storage/delete_blob.rb,
lib/fog/azurerm/requests/storage/get_blob_url.rb,
lib/fog/azurerm/requests/storage/delete_object.rb,
lib/fog/azurerm/requests/storage/get_object_url.rb,
lib/fog/azurerm/requests/storage/put_blob_block.rb,
lib/fog/azurerm/requests/storage/put_blob_pages.rb,
lib/fog/azurerm/requests/storage/put_object_url.rb,
lib/fog/azurerm/requests/storage/save_page_blob.rb,
lib/fog/azurerm/requests/storage/list_containers.rb,
lib/fog/azurerm/requests/storage/create_container.rb,
lib/fog/azurerm/requests/storage/create_page_blob.rb,
lib/fog/azurerm/requests/storage/delete_container.rb,
lib/fog/azurerm/requests/storage/create_block_blob.rb,
lib/fog/azurerm/requests/storage/delete_object_url.rb,
lib/fog/azurerm/requests/storage/get_blob_http_url.rb,
lib/fog/azurerm/requests/storage/get_container_acl.rb,
lib/fog/azurerm/requests/storage/get_container_url.rb,
lib/fog/azurerm/requests/storage/put_blob_http_url.rb,
lib/fog/azurerm/requests/storage/put_blob_metadata.rb,
lib/fog/azurerm/requests/storage/put_container_acl.rb,
lib/fog/azurerm/requests/storage/acquire_blob_lease.rb,
lib/fog/azurerm/requests/storage/commit_blob_blocks.rb,
lib/fog/azurerm/requests/storage/copy_blob_from_uri.rb,
lib/fog/azurerm/requests/storage/get_blob_https_url.rb,
lib/fog/azurerm/requests/storage/put_blob_https_url.rb,
lib/fog/azurerm/requests/storage/release_blob_lease.rb,
lib/fog/azurerm/requests/storage/get_blob_properties.rb,
lib/fog/azurerm/requests/storage/put_blob_properties.rb,
lib/fog/azurerm/requests/storage/delete_blob_https_url.rb,
lib/fog/azurerm/requests/storage/check_container_exists.rb,
lib/fog/azurerm/requests/storage/put_container_metadata.rb,
lib/fog/azurerm/requests/storage/acquire_container_lease.rb,
lib/fog/azurerm/requests/storage/compare_container_blobs.rb,
lib/fog/azurerm/requests/storage/release_container_lease.rb,
lib/fog/azurerm/requests/storage/get_container_properties.rb,
lib/fog/azurerm/requests/storage/wait_blob_copy_operation_to_finish.rb
Overview
This class provides the actual implementation for service calls.
Defined Under Namespace
Classes: BlobChunk, BlobFileStream
Constant Summary
collapse
- BLOCK_SIZE =
32 * 1024 * 1024
Instance Method Summary
collapse
-
#acquire_blob_lease(container_name, name, options = {}) ⇒ Object
-
#acquire_container_lease(name, options = {}) ⇒ Object
-
#check_container_exists(name) ⇒ Object
-
#commit_blob_blocks(container_name, blob_name, blocks, options = {}) ⇒ Object
-
#compare_container_blobs(container1, container2) ⇒ Object
-
#copy_blob(destination_container, destination_blob, source_container, source_blob, options = {}) ⇒ Object
-
#copy_blob_from_uri(destination_container, destination_blob, source_blob_uri, options = {}) ⇒ Object
-
#copy_object(source_container, source_blob, target_container, target_blob, options = {}) ⇒ Object
-
#create_block_blob(container_name, blob_name, body, options = {}) ⇒ Object
-
#create_container(name, options = {}) ⇒ Object
-
#create_page_blob(container_name, blob_name, blob_size, options = {}) ⇒ Object
-
#delete_blob(container_name, blob_name, options = {}) ⇒ Object
(also: #delete_object)
-
#delete_blob_https_url(container_name, blob_name, expires) ⇒ String
Get a pre-signed URL to delete an object in Azure blob storage.
-
#delete_container(name) ⇒ Object
-
#delete_object_url(container_name, blob_name, expires) ⇒ String
Get a pre-signed URL to delete an object in Azure blob storage This is to make this library compatible with CarrierWave.
-
#get_blob(container_name, blob_name, options = {}, &block) ⇒ Object
-
#get_blob_http_url(container_name, blob_name, expires, options = {}) ⇒ String
Get an expiring http blob url from Azure blob storage.
-
#get_blob_https_url(container_name, blob_name, expires, options = {}) ⇒ String
Get an expiring https blob url from Azure blob storage.
-
#get_blob_properties(container_name, name, options = {}) ⇒ Object
-
#get_blob_url(container_name, blob_name, options = {}) ⇒ Object
Get a public blob url from Azure blob storage.
-
#get_blob_with_block_given(container_name, blob_name, options, &block) ⇒ Object
-
#get_container_acl(container_name, options = {}) ⇒ Object
-
#get_container_properties(name, options = {}) ⇒ Object
-
#get_container_url(container_name, options = {}) ⇒ String
Get a public container url from Azure storage container.
-
#get_object ⇒ Object
-
#get_object_url(container_name, blob_name, expires, options = {}) ⇒ Object
Get a public blob url from Azure blob storage This is to make this library compatible with CarrierWave.
-
#initialize(options) ⇒ Real
constructor
-
#list_blobs(container_name, options = {}) ⇒ Object
-
#list_containers ⇒ Object
-
#put_blob_block(container_name, blob_name, block_id, data, options = {}) ⇒ Object
-
#put_blob_http_url(container_name, blob_name, expires) ⇒ String
Generate a pre-signed URL for create an object in an Azure blob storage.
-
#put_blob_https_url(container_name, blob_name, expires) ⇒ String
Generate a pre-signed URL for create an object in an Azure blob storage.
-
#put_blob_metadata(container_name, name, metadata, options = {}) ⇒ Object
-
#put_blob_pages(container_name, blob_name, start_range, end_range, data, options = {}) ⇒ Object
-
#put_blob_properties(container_name, name, options = {}) ⇒ Object
-
#put_container_acl(name, acl, options = {}) ⇒ Object
-
#put_container_metadata(name, metadata, options = {}) ⇒ Object
-
#put_object_url(bucket_name, object_name, expire_at, _upload_options) ⇒ String
Generate a pre-signed URL for create an object in an Azure blob storage This is to make this library compatible with CarrierWave.
-
#release_blob_lease(container_name, name, lease_id, options = {}) ⇒ Object
-
#release_container_lease(name, lease_id, options = {}) ⇒ Object
-
#save_page_blob(container_name, blob_name, body, options) ⇒ Object
-
#wait_blob_copy_operation_to_finish(container_name, blob_name, copy_id, copy_status, timeout = nil) ⇒ Object
#get_blob_endpoint, #get_blob_endpoint_with_domain, #get_circuit_name_from_id, #get_end_point_type, #get_hash_from_object, #get_image_name, #get_record_set_from_id, #get_record_type, #get_resource_from_resource_id, #get_resource_group_from_id, #get_subscription_id, #get_traffic_manager_profile_name_from_endpoint_id, #get_type_from_recordset_type, #get_virtual_machine_from_id, #get_virtual_network_from_id, #parse_storage_object, #raise_azure_exception, #random_string, #remove_trailing_periods_from_path_segments, #storage_endpoint_suffix, #validate_params
Constructor Details
#initialize(options) ⇒ Real
Returns a new instance of Real.
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
# File 'lib/fog/azurerm/storage.rb', line 83
def initialize(options)
begin
require 'azure/storage/common'
require 'azure/storage/blob'
require 'securerandom'
@debug = ENV['DEBUG'] || options[:debug]
require 'azure/core/http/debug_filter' if @debug
require 'fog/azurerm/identity_encoding_filter'
rescue LoadError => e
retry if require('rubygems')
raise e.message
end
return unless @azure_storage_account_name != options[:azure_storage_account_name] ||
@azure_storage_access_key != options[:azure_storage_access_key] ||
@azure_storage_token_signer != options[:azure_storage_token_signer]
@azure_storage_account_name = options[:azure_storage_account_name]
@azure_storage_access_key = options[:azure_storage_access_key]
@azure_storage_token_signer = options[:azure_storage_token_signer]
@azure_storage_endpoint = options[:azure_storage_endpoint]
@azure_storage_domain = options[:azure_storage_domain]
options[:environment] = 'AzureCloud' if options[:environment].nil?
@environment = options[:environment]
storage_blob_host =
@azure_storage_endpoint ||
if @azure_storage_domain.nil? || @azure_storage_domain.empty?
get_blob_endpoint(@azure_storage_account_name, true, @environment)
else
get_blob_endpoint_with_domain(@azure_storage_account_name, true, @azure_storage_domain)
end
azure_client = Azure::Storage::Common::Client.create({
storage_account_name: @azure_storage_account_name,
storage_access_key: @azure_storage_access_key,
signer: @azure_storage_token_signer
}.compact)
azure_client.storage_blob_host = storage_blob_host
@blob_client = Azure::Storage::Blob::BlobService.new(client: azure_client)
@blob_client.with_filter(Fog::AzureRM::IdentityEncodingFilter.new)
@blob_client.with_filter(Azure::Storage::Common::Core::Filter::ExponentialRetryPolicyFilter.new)
@blob_client.with_filter(Azure::Core::Http::DebugFilter.new) if @debug
end
|
Instance Method Details
#acquire_blob_lease(container_name, name, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/acquire_blob_lease.rb', line 6
def acquire_blob_lease(container_name, name, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Leasing blob: #{name} of container #{container_name} options: #{options}"
Fog::Logger.debug msg
begin
lease_id = @blob_client.acquire_blob_lease(container_name, name, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Blob #{name} leased successfully."
lease_id
end
|
#acquire_container_lease(name, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/acquire_container_lease.rb', line 6
def acquire_container_lease(name, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Leasing container: #{name} options: #{options}"
Fog::Logger.debug msg
begin
lease_id = @blob_client.acquire_container_lease(name, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Container #{name} leased successfully."
lease_id
end
|
#check_container_exists(name) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/check_container_exists.rb', line 6
def check_container_exists(name)
msg = "Checking container #{name}."
Fog::Logger.debug msg
begin
get_container_properties(name)
Fog::Logger.debug "Container #{name} exists."
true
rescue Exception => e
if e.message.include? 'NotFound'
Fog::Logger.debug "The specified container #{name} does not exist."
false
end
end
end
|
#commit_blob_blocks(container_name, blob_name, blocks, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/commit_blob_blocks.rb', line 6
def commit_blob_blocks(container_name, blob_name, blocks, options = {})
options[:request_id] = SecureRandom.uuid
msg = "commit_blob_blocks: Complete uploading #{blob_name} to the container #{container_name}. options: #{options}"
Fog::Logger.debug msg
begin
@blob_client.commit_blob_blocks(container_name, blob_name, blocks, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Block blob #{blob_name} is uploaded successfully."
true
end
|
#compare_container_blobs(container1, container2) ⇒ Object
6
7
8
9
10
11
12
13
14
15
|
# File 'lib/fog/azurerm/requests/storage/compare_container_blobs.rb', line 6
def compare_container_blobs(container1, container2)
msg = "Comparing blobs from container #{container1} to container #{container2}"
Fog::Logger.debug msg
begin
identical_blobs = get_identical_blobs_from_containers(container1, container2)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
identical_blobs
end
|
#copy_blob(destination_container, destination_blob, source_container, source_blob, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/copy_blob.rb', line 6
def copy_blob(destination_container, destination_blob, source_container, source_blob, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Copying blob: #{source_blob} from container #{source_container} to container #{destination_container} options: #{options}"
Fog::Logger.debug msg
begin
copy_id, copy_status = @blob_client.copy_blob(destination_container, destination_blob, source_container, source_blob, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Copying blob: x-ms-copy-id: #{copy_id}, x-ms-copy-status: #{copy_status}"
[copy_id, copy_status]
end
|
#copy_blob_from_uri(destination_container, destination_blob, source_blob_uri, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/copy_blob_from_uri.rb', line 6
def copy_blob_from_uri(destination_container, destination_blob, source_blob_uri, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Copying blob: #{source_blob_uri} to container #{destination_container} options: #{options}"
Fog::Logger.debug msg
begin
copy_id, copy_status = @blob_client.copy_blob_from_uri(destination_container, destination_blob, source_blob_uri, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Copying blob: x-ms-copy-id: #{copy_id}, x-ms-copy-status: #{copy_status}"
[copy_id, copy_status]
end
|
#copy_object(source_container, source_blob, target_container, target_blob, options = {}) ⇒ Object
7
8
9
|
# File 'lib/fog/azurerm/requests/storage/copy_object.rb', line 7
def copy_object(source_container, source_blob, target_container, target_blob, options = {})
copy_blob(target_container, target_blob, source_container, source_blob, options)
end
|
#create_block_blob(container_name, blob_name, body, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
|
# File 'lib/fog/azurerm/requests/storage/create_block_blob.rb', line 6
def create_block_blob(container_name, blob_name, body, options = {})
options[:request_id] = SecureRandom.uuid
msg = "create_block_blob #{blob_name} to the container #{container_name}. options: #{options}"
Fog::Logger.debug msg
@blob_client.create_block_blob(container_name, blob_name, body, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
|
#create_container(name, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/create_container.rb', line 6
def create_container(name, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Creating container: #{name}. options: #{options}"
Fog::Logger.debug msg
begin
container = @blob_client.create_container(name, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Container #{name} created successfully."
container
end
|
#create_page_blob(container_name, blob_name, blob_size, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/create_page_blob.rb', line 6
def create_page_blob(container_name, blob_name, blob_size, options = {})
options[:request_id] = SecureRandom.uuid
msg = "create_page_blob #{blob_name} to the container #{container_name}. blob_size: #{blob_size}, options: #{options}"
Fog::Logger.debug msg
begin
@blob_client.create_page_blob(container_name, blob_name, blob_size, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Page blob #{blob_name} created successfully."
true
end
|
#delete_blob(container_name, blob_name, options = {}) ⇒ Object
Also known as:
delete_object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
|
# File 'lib/fog/azurerm/requests/storage/delete_blob.rb', line 6
def delete_blob(container_name, blob_name, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Deleting blob: #{blob_name} in container #{container_name}. options: #{options}"
Fog::Logger.debug msg
begin
@blob_client.delete_blob(container_name, blob_name, options)
rescue Azure::Core::Http::HTTPError => ex
return true if ex.message.include?('(404)')
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Blob #{blob_name} deleted successfully."
true
end
|
#delete_blob_https_url(container_name, blob_name, expires) ⇒ String
Get a pre-signed URL to delete an object in Azure blob storage
16
17
18
19
20
21
22
23
24
25
26
27
28
|
# File 'lib/fog/azurerm/requests/storage/delete_blob_https_url.rb', line 16
def delete_blob_https_url(container_name, blob_name, expires)
relative_path = "#{container_name}/#{blob_name}"
params = {
service: 'b',
resource: 'b',
permissions: 'd',
expiry: expires.utc.iso8601,
protocol: 'https'
}
token = signature_client(expires).generate_service_sas_token(relative_path, params)
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
"#{uri}?#{token}"
end
|
#delete_container(name) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
|
# File 'lib/fog/azurerm/requests/storage/delete_container.rb', line 6
def delete_container(name)
options = { request_id: SecureRandom.uuid }
msg = "Deleting container: #{name}. options: #{options}"
Fog::Logger.debug msg
begin
@blob_client.delete_container(name, options)
rescue Azure::Core::Http::HTTPError => ex
return true if ex.message.include?('(404)')
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Container #{name} deleted successfully."
true
end
|
#delete_object_url(container_name, blob_name, expires) ⇒ String
Get a pre-signed URL to delete an object in Azure blob storage This is to make this library compatible with CarrierWave.
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/delete_object_url.rb', line 17
def delete_object_url(container_name, blob_name, expires)
delete_blob_https_url(container_name, blob_name, expires)
end
|
#get_blob(container_name, blob_name, options = {}, &block) ⇒ Object
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
# File 'lib/fog/azurerm/requests/storage/get_blob.rb', line 65
def get_blob(container_name, blob_name, options = {}, &block)
if block_given?
get_blob_with_block_given(container_name, blob_name, options, &block)
else
options[:request_id] = SecureRandom.uuid
msg = "get_blob blob #{blob_name} in the container #{container_name}. options: #{options}"
Fog::Logger.debug msg
begin
blob, content = @blob_client.get_blob(container_name, blob_name, options)
Fog::Logger.debug "Get blob #{blob_name} successfully."
[blob, content]
rescue Azure::Core::Http::HTTPError => ex
raise 'NotFound' if ex.message.include?('(404)')
raise_azure_exception(ex, msg)
end
end
end
|
#get_blob_http_url(container_name, blob_name, expires, options = {}) ⇒ String
Get an expiring http blob url from Azure blob storage
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
# File 'lib/fog/azurerm/requests/storage/get_blob_http_url.rb', line 16
def get_blob_http_url(container_name, blob_name, expires, options = {})
relative_path = "#{container_name}/#{blob_name}"
relative_path = remove_trailing_periods_from_path_segments(relative_path)
params = {
service: 'b',
resource: 'b',
permissions: 'r',
expiry: expires.utc.iso8601,
content_disposition: options[:content_disposition]
}
token = signature_client(expires).generate_service_sas_token(relative_path, params)
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
url = "#{uri}?#{token}"
url.sub('https:', 'http:')
end
|
#get_blob_https_url(container_name, blob_name, expires, options = {}) ⇒ String
Get an expiring https blob url from Azure blob storage
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
|
# File 'lib/fog/azurerm/requests/storage/get_blob_https_url.rb', line 16
def get_blob_https_url(container_name, blob_name, expires, options = {})
relative_path = "#{container_name}/#{blob_name}"
relative_path = remove_trailing_periods_from_path_segments(relative_path)
params = {
service: 'b',
resource: 'b',
permissions: 'r',
expiry: expires.utc.iso8601,
protocol: 'https',
content_disposition: options[:content_disposition],
content_type: options[:content_type]
}
token = signature_client(expires).generate_service_sas_token(relative_path, params)
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
"#{uri}?#{token}"
end
|
#get_blob_properties(container_name, name, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
|
# File 'lib/fog/azurerm/requests/storage/get_blob_properties.rb', line 6
def get_blob_properties(container_name, name, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Get Blob #{name} properties in container #{container_name}, options: #{options}."
Fog::Logger.debug msg
begin
blob = @blob_client.get_blob_properties(container_name, name, options)
rescue Azure::Core::Http::HTTPError => ex
raise 'NotFound' if ex.message.include?('(404)')
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Getting properties of blob #{name} successfully."
blob
end
|
#get_blob_url(container_name, blob_name, options = {}) ⇒ Object
Get a public blob url from Azure blob storage
7
8
9
10
11
12
13
14
15
|
# File 'lib/fog/azurerm/requests/storage/get_blob_url.rb', line 7
def get_blob_url(container_name, blob_name, options = {})
uri = @blob_client.generate_uri("#{container_name}/#{blob_name}", {}, { encode: true })
if options[:scheme] == 'http'
uri.to_s.gsub('https:', 'http:')
else
uri.to_s
end
end
|
#get_blob_with_block_given(container_name, blob_name, options, &block) ⇒ Object
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
|
# File 'lib/fog/azurerm/requests/storage/get_blob.rb', line 8
def get_blob_with_block_given(container_name, blob_name, options, &block)
options[:request_id] = SecureRandom.uuid
msg = "get_blob_with_block_given: blob #{blob_name} in the container #{container_name}. options: #{options}"
Fog::Logger.debug msg
begin
blob = @blob_client.get_blob_properties(container_name, blob_name, options)
rescue Azure::Core::Http::HTTPError => ex
raise 'NotFound' if ex.message.include?('(404)')
raise_azure_exception(ex, msg)
end
content_length = blob.properties[:content_length]
if content_length.zero?
block.call('', 0, 0)
return [blob, '']
end
start_range = 0
end_range = content_length - 1
start_range = options[:start_range] if options[:start_range]
end_range = options[:end_range] if options[:end_range]
raise ArgumentError.new(':end_range MUST be greater than :start_range') if start_range > end_range
if start_range == end_range
block.call('', 0, 0)
return [blob, '']
end
buffer_size = BLOCK_SIZE
buffer_size = options[:block_size] if options[:block_size]
buffer_start_range = start_range
total_bytes = end_range - start_range + 1
params = options.dup
while buffer_start_range < end_range
buffer_end_range = [end_range, buffer_start_range + buffer_size - 1].min
params[:start_range] = buffer_start_range
params[:end_range] = buffer_end_range
params[:request_id] = SecureRandom.uuid
begin
msg = "get_blob_with_block_given: blob #{blob_name} in the container #{container_name}. options: #{params}"
Fog::Logger.debug msg
_, content = @blob_client.get_blob(container_name, blob_name, params)
rescue Azure::Core::Http::HTTPError => ex
raise 'NotFound' if ex.message.include?('(404)')
raise_azure_exception(ex, msg)
end
block.call(content, end_range - buffer_end_range, total_bytes)
buffer_start_range += buffer_size
end
[blob, '']
end
|
#get_container_acl(container_name, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/get_container_acl.rb', line 6
def get_container_acl(container_name, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Get container ACL: #{container_name}. options: #{options}"
Fog::Logger.debug msg
begin
container, signed_identifiers = @blob_client.get_container_acl(container_name, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Getting ACL of container #{container_name} successfully."
[container.public_access_level, signed_identifiers]
end
|
#get_container_properties(name, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
|
# File 'lib/fog/azurerm/requests/storage/get_container_properties.rb', line 6
def get_container_properties(name, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Getting container properties: #{name}, options: #{options}."
Fog::Logger.debug msg
begin
container = @blob_client.get_container_properties(name, options)
rescue Azure::Core::Http::HTTPError => ex
raise 'NotFound' if ex.message.include?('(404)')
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Getting properties of container #{name} successfully."
container
end
|
#get_container_url(container_name, options = {}) ⇒ String
Get a public container url from Azure storage container
12
13
14
15
16
17
18
19
20
21
|
# File 'lib/fog/azurerm/requests/storage/get_container_url.rb', line 12
def get_container_url(container_name, options = {})
query = { 'comp' => 'list', 'restype' => 'container' }
uri = @blob_client.generate_uri(container_name, query, { encode: true })
if options[:scheme] == 'http'
uri.to_s.gsub('https:', 'http:')
else
uri.to_s
end
end
|
#get_object ⇒ Object
6
7
8
9
10
11
|
# File 'lib/fog/azurerm/requests/storage/get_object.rb', line 6
def get_object(...)
blob, body = get_blob(...)
blob[:body] = body
blob
end
|
#get_object_url(container_name, blob_name, expires, options = {}) ⇒ Object
Get a public blob url from Azure blob storage This is to make this library compatible with CarrierWave.
8
9
10
|
# File 'lib/fog/azurerm/requests/storage/get_object_url.rb', line 8
def get_object_url(container_name, blob_name, expires, options = {})
get_blob_https_url(container_name, blob_name, expires, options)
end
|
#list_blobs(container_name, options = {}) ⇒ Object
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
|
# File 'lib/fog/azurerm/requests/storage/list_blobs.rb', line 7
def list_blobs(container_name, options = {})
options = options.dup
options[:metadata] = true
next_marker = nil
blobs = []
msg = nil
max_results = -1
max_results = options[:max_results].to_i if options[:max_results]
begin
loop do
options[:request_id] = SecureRandom.uuid
msg = "Listing blobs in container: #{container_name}, options: #{options}"
Fog::Logger.debug msg
temp = @blob_client.list_blobs(container_name, options)
raise temp unless temp.instance_of?(Azure::Storage::Common::Service::EnumerationResults)
blobs += temp unless temp.empty?
break if temp.continuation_token.nil? || temp.continuation_token.empty?
options[:marker] = temp.continuation_token
next if max_results == -1
options[:max_results] = max_results - blobs.size
if options[:max_results].zero?
next_marker = temp.continuation_token
break
end
end
rescue Azure::Core::Http::HTTPError => ex
raise 'NotFound' if ex.message.include?('(404)')
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Listing blobs in container: #{container_name} successfully."
{
next_marker: next_marker,
blobs: blobs
}
end
|
#list_containers ⇒ Object
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
|
# File 'lib/fog/azurerm/requests/storage/list_containers.rb', line 7
def list_containers
options = { metadata: true }
containers = []
msg = nil
begin
loop do
options[:request_id] = SecureRandom.uuid
msg = "Listing containers. options: #{options}"
Fog::Logger.debug msg
temp = @blob_client.list_containers(options)
raise temp unless temp.instance_of?(Azure::Storage::Common::Service::EnumerationResults)
containers += temp unless temp.empty?
break if temp.continuation_token.nil? || temp.continuation_token.empty?
options[:marker] = temp.continuation_token
end
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug 'Listing containers successfully.'
containers
end
|
#put_blob_block(container_name, blob_name, block_id, data, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/put_blob_block.rb', line 6
def put_blob_block(container_name, blob_name, block_id, data, options = {})
options[:request_id] = SecureRandom.uuid
msg = "put_blob_block block_id: #{block_id} / #{blob_name} to the container #{container_name}. options: #{options}"
Fog::Logger.debug msg
begin
@blob_client.put_blob_block(container_name, blob_name, block_id, data, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "block_id #{block_id} is uploaded successfully."
true
end
|
#put_blob_http_url(container_name, blob_name, expires) ⇒ String
Generate a pre-signed URL for create an object in an Azure blob storage
16
17
18
19
20
21
22
23
24
25
26
27
28
|
# File 'lib/fog/azurerm/requests/storage/put_blob_http_url.rb', line 16
def put_blob_http_url(container_name, blob_name, expires)
relative_path = "#{container_name}/#{blob_name}"
params = {
service: 'b',
resource: 'b',
permissions: 'c',
expiry: expires.utc.iso8601
}
token = signature_client(expires).generate_service_sas_token(relative_path, params)
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
url = "#{uri}?#{token}"
url.sub('https:', 'http:')
end
|
#put_blob_https_url(container_name, blob_name, expires) ⇒ String
Generate a pre-signed URL for create an object in an Azure blob storage
16
17
18
19
20
21
22
23
24
25
26
27
28
|
# File 'lib/fog/azurerm/requests/storage/put_blob_https_url.rb', line 16
def put_blob_https_url(container_name, blob_name, expires)
relative_path = "#{container_name}/#{blob_name}"
params = {
service: 'b',
resource: 'b',
permissions: 'c',
expiry: expires.utc.iso8601,
protocol: 'https'
}
token = signature_client(expires).generate_service_sas_token(relative_path, params)
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
"#{uri}?#{token}"
end
|
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/put_blob_metadata.rb', line 6
def put_blob_metadata(container_name, name, metadata, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Setting Blob #{name} metadata in a container #{container_name}. options: #{options}"
Fog::Logger.debug msg
begin
@blob_client.set_blob_metadata(container_name, name, metadata, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Setting metadata of blob #{name} successfully."
true
end
|
#put_blob_pages(container_name, blob_name, start_range, end_range, data, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/put_blob_pages.rb', line 6
def put_blob_pages(container_name, blob_name, start_range, end_range, data, options = {})
options[:request_id] = SecureRandom.uuid
msg = "put_blob_pages [#{start_range}-#{end_range}] / #{blob_name} to the container #{container_name}. options: #{options}"
Fog::Logger.debug msg
begin
@blob_client.put_blob_pages(container_name, blob_name, start_range, end_range, data, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "[#{start_range}-#{end_range}] / #{blob_name} is uploaded successfully."
true
end
|
#put_blob_properties(container_name, name, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/put_blob_properties.rb', line 6
def put_blob_properties(container_name, name, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Set Blob #{name} properties #{options} in container #{container_name}."
Fog::Logger.debug msg
begin
@blob_client.set_blob_properties(container_name, name, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Setting properties of blob #{name} successfully."
true
end
|
#put_container_acl(name, acl, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/put_container_acl.rb', line 6
def put_container_acl(name, acl, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Setting Container #{name} acl #{acl.nil? ? 'nil' : acl}. options: #{options}"
Fog::Logger.debug msg
begin
@blob_client.set_container_acl(name, acl, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Setting acl of container #{name} successfully."
true
end
|
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/put_container_metadata.rb', line 6
def put_container_metadata(name, metadata, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Setting Container #{name} metadata. options: #{options}"
Fog::Logger.debug msg
begin
@blob_client.set_container_metadata(name, metadata, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Setting metadata of container #{name} successfully."
true
end
|
#put_object_url(bucket_name, object_name, expire_at, _upload_options) ⇒ String
Generate a pre-signed URL for create an object in an Azure blob storage This is to make this library compatible with CarrierWave.
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/put_object_url.rb', line 17
def put_object_url(bucket_name, object_name, expire_at, _upload_options)
put_blob_https_url(bucket_name, object_name, expire_at)
end
|
#release_blob_lease(container_name, name, lease_id, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/release_blob_lease.rb', line 6
def release_blob_lease(container_name, name, lease_id, options = {})
options[:request_id] = SecureRandom.uuid
msg = "Releasing blob: #{name} of container #{container_name} having lease_id #{lease_id} options: #{options}"
Fog::Logger.debug msg
begin
@blob_client.release_blob_lease(container_name, name, lease_id, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Blob #{name} released successfully."
true
end
|
#release_container_lease(name, lease_id, options = {}) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# File 'lib/fog/azurerm/requests/storage/release_container_lease.rb', line 6
def release_container_lease(name, lease_id, options={})
options[:request_id] = SecureRandom.uuid
msg = "Releasing container: #{name} having lease_id #{lease_id} options: #{options}"
Fog::Logger.debug msg
begin
@blob_client.release_container_lease(name, lease_id, options)
rescue Azure::Core::Http::HTTPError => ex
raise_azure_exception(ex, msg)
end
Fog::Logger.debug "Container #{name} released successfully."
true
end
|
#save_page_blob(container_name, blob_name, body, options) ⇒ Object
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
|
# File 'lib/fog/azurerm/requests/storage/save_page_blob.rb', line 57
def save_page_blob(container_name, blob_name, body, options)
threads_num = options.delete(:worker_thread_num)
threads_num = Fog::AzureRM::UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1
begin
blob_size = Fog::Storage.get_body_size(body)
raise "The page blob size must be aligned to a 512-byte boundary. But the file size is #{blob_size}." if (blob_size % 512).nonzero?
Fog::Logger.debug "Creating the page blob #{container_name}/#{blob_name}. options: #{options}"
create_page_blob(container_name, blob_name, blob_size, options)
options.delete(:content_md5)
iostream = BlobFileStream.new(body)
threads = []
threads_num.times do |id|
thread = Thread.new do
Fog::Logger.debug "Created upload thread #{id}."
while (chunk = iostream.read(Fog::AzureRM::MAXIMUM_CHUNK_SIZE))
Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, start_range: #{chunk.start_range}, size: #{chunk.data.size}."
put_blob_pages(container_name, blob_name, chunk.start_range, chunk.end_range, chunk.data, options) if Digest::MD5.hexdigest(chunk.data) != Fog::AzureRM::HASH_OF_4MB_EMPTY_CONTENT
end
Fog::Logger.debug "Upload thread #{id} finished."
end
thread.abort_on_exception = true
threads << thread
end
threads.each(&:join)
rescue
begin
delete_blob(container_name, blob_name)
rescue => ex
Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after save_page_blob failed. #{ex.inspect}"
end
raise
end
Fog::Logger.debug "Successfully save the page blob: #{container_name}/#{blob_name}."
true
end
|
#wait_blob_copy_operation_to_finish(container_name, blob_name, copy_id, copy_status, timeout = nil) ⇒ Object
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
|
# File 'lib/fog/azurerm/requests/storage/wait_blob_copy_operation_to_finish.rb', line 6
def wait_blob_copy_operation_to_finish(container_name, blob_name, copy_id, copy_status, timeout = nil)
begin
start_time = Time.new
while copy_status == Fog::AzureRM::COPY_STATUS[:PENDING]
blob = get_blob_properties(container_name, blob_name)
blob_props = blob.properties
if !copy_id.nil? && blob_props[:copy_id] != copy_id
raise "The progress of copying to #{container_name}/#{blob_name} was interrupted by other copy operations."
end
copy_status_description = blob_props[:copy_status_description]
copy_status = blob_props[:copy_status]
break if copy_status != Fog::AzureRM::COPY_STATUS[:PENDING]
elapse_time = Time.new - start_time
raise Timeout::Error.new("The copy operation cannot be finished in #{timeout} seconds") if !timeout.nil? && elapse_time >= timeout
copied_bytes, total_bytes = blob_props[:copy_progress].split('/').map(&:to_i)
interval = copied_bytes.zero? ? 5 : (total_bytes - copied_bytes).to_f / copied_bytes * elapse_time
interval = 30 if interval > 30
interval = 1 if interval < 1
sleep(interval)
end
if copy_status != Fog::AzureRM::COPY_STATUS[:SUCCESS]
raise "Failed to copy to #{container_name}/#{blob_name}: \n\tcopy status: #{copy_status}\n\tcopy description: #{copy_status_description}"
end
rescue
begin
delete_blob(container_name, blob_name)
rescue => ex
Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after the copy operation failed. #{ex.inspect}"
end
raise
end
Fog::Logger.debug "Successfully copied the blob: #{container_name}/#{blob_name}."
true
end
|