Class: Fluent::Plugin::AzureStorageGen2Output
- Inherits:
-
Output
- Object
- Output
- Fluent::Plugin::AzureStorageGen2Output
show all
- Defined in:
- lib/fluent/plugin/out_azurestorage_gen2.rb
Defined Under Namespace
Classes: Compressor, GzipCompressor, JsonCompressor, TextCompressor
Constant Summary
collapse
- DEFAULT_FORMAT_TYPE =
"out_file"
- ACCESS_TOKEN_API_VERSION =
"2018-02-01"
- ABFS_API_VERSION =
"2018-11-09"
- AZURE_BLOCK_SIZE_LIMIT =
4 * 1024 * 1024 - 1
Instance Method Summary
collapse
Constructor Details
Returns a new instance of AzureStorageGen2Output.
18
19
20
21
|
# File 'lib/fluent/plugin/out_azurestorage_gen2.rb', line 18
def initialize
super
@compressor = nil
end
|
Instance Method Details
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
# File 'lib/fluent/plugin/out_azurestorage_gen2.rb', line 71
def configure(conf)
compat_parameters_convert(conf, :buffer, :formatter, :inject)
super
if @store_as.nil? || @store_as == "none"
log.info "azurestorage_gen2: Compression is disabled (store_as: #{@store_as})"
else
begin
@compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
rescue => e
log.warn "#{@store_as} not found. Use 'text' instead"
@compressor = TextCompressor.new
end
@compressor.configure(conf)
end
@formatter = formatter_create
if @azure_container.nil?
raise Fluent::ConfigError, "azure_container is needed"
end
@azure_storage_path = ''
@last_azure_storage_path = ''
@current_index = 0
if @store_as.nil? || @store_as == "none"
@final_file_extension = @file_extension
else
@final_file_extension = @compressor.ext
end
@values_for_object_chunk = {}
end
|
133
134
135
136
|
# File 'lib/fluent/plugin/out_azurestorage_gen2.rb', line 133
def format(tag, time, record)
r = inject_values_to_record(tag, time, record)
@formatter.format(tag, time, r)
end
|
#multi_workers_ready? ⇒ Boolean
105
106
107
|
# File 'lib/fluent/plugin/out_azurestorage_gen2.rb', line 105
def multi_workers_ready?
true
end
|
#start ⇒ Object
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
# File 'lib/fluent/plugin/out_azurestorage_gen2.rb', line 109
def start
setup_access_token
if !@skip_container_check
if @failsafe_container_check
begin
if @write_only && @auto_create_container
create_container
else
ensure_container
end
rescue Exception => e
log.warn("#{e.message}, container list/create failsafe is enabled. Continue without those operations.")
end
else
if @write_only && @auto_create_container
create_container
else
ensure_container
end
end
end
super
end
|
#write(chunk) ⇒ Object
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
# File 'lib/fluent/plugin/out_azurestorage_gen2.rb', line 138
def write(chunk)
if @store_as.nil? || @store_as == "none"
generate_log_name(chunk, @current_index)
if @last_azure_storage_path != @azure_storage_path
@current_index = 0
generate_log_name(chunk, @current_index)
end
raw_data = chunk.read
unless raw_data.empty?
log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
upload_blob(raw_data, chunk)
end
chunk.close rescue nil
@last_azure_storage_path = @azure_storage_path
else
tmp = Tempfile.new("azure-")
tmp.binmode
begin
@compressor.compress(chunk, tmp)
tmp.rewind
generate_log_name(chunk, @current_index)
if @last_azure_storage_path != @azure_storage_path
@current_index = 0
generate_log_name(chunk, @current_index)
end
log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
content = File.open(tmp.path, 'rb') { |file| file.read }
upload_blob(content, chunk)
@last_azure_storage_path = @azure_storage_path
ensure
tmp.close(true) rescue nil
end
@values_for_object_chunk.delete(chunk.unique_id)
end
end
|