Class: LogStash::Outputs::Kusto
- Inherits:
-
Base
- Object
- Base
- LogStash::Outputs::Kusto
- Defined in:
- lib/logstash/outputs/kusto.rb,
lib/logstash/outputs/kusto/interval.rb,
lib/logstash/outputs/kusto/ingestor.rb
Overview
This plugin sends messages to Azure Kusto in batches.
Defined Under Namespace
Constant Summary collapse
- FIELD_REF =
/%\{[^}]+\}/
Instance Attribute Summary collapse
-
#failure_path ⇒ Object
readonly
Returns the value of attribute failure_path.
Instance Method Summary collapse
Instance Attribute Details
#failure_path ⇒ Object (readonly)
Returns the value of attribute failure_path.
19 20 21 |
# File 'lib/logstash/outputs/kusto.rb', line 19 def failure_path @failure_path end |
Instance Method Details
#close ⇒ Object
197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 |
# File 'lib/logstash/outputs/kusto.rb', line 197 def close @flusher.stop unless @flusher.nil? @cleaner.stop unless @cleaner.nil? @io_mutex.synchronize do @logger.debug('Close: closing files') @files.each do |path, fd| begin fd.close @logger.debug("Closed file #{path}", fd: fd) kusto_send_file(path) rescue Exception => e @logger.error('Exception while flushing and closing files.', exception: e) end end end @ingestor.stop unless @ingestor.nil? end |
#multi_receive_encoded(events_and_encoded) ⇒ Object
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 |
# File 'lib/logstash/outputs/kusto.rb', line 177 def multi_receive_encoded(events_and_encoded) encoded_by_path = Hash.new { |h, k| h[k] = [] } events_and_encoded.each do |event, encoded| file_output_path = event_path(event) encoded_by_path[file_output_path] << encoded end @io_mutex.synchronize do encoded_by_path.each do |path, chunks| fd = open(path) # append to the file chunks.each { |chunk| fd.write(chunk) } fd.flush unless @flusher && @flusher.alive? end close_stale_files if @stale_cleanup_type == 'events' end end |
#register ⇒ Object
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 |
# File 'lib/logstash/outputs/kusto.rb', line 107 def register require 'fileutils' # For mkdir_p @files = {} @io_mutex = Mutex.new # TODO: add id to the tmp path to support multiple outputs of the same type # add fields from the meta that will note the destination of the events in the file @path = if dynamic_event_routing File.("#{path}.%{[@metadata][database]}.%{[@metadata][table]}.%{[@metadata][mapping]}") else File.("#{path}.#{database}.#{table}") end validate_path @file_root = if path_with_field_ref? extract_file_root else File.dirname(path) end @failure_path = File.join(@file_root, @filename_failure) executor = Concurrent::ThreadPoolExecutor.new(min_threads: 1, max_threads: upload_concurrent_count, max_queue: upload_queue_size, fallback_policy: :caller_runs) @ingestor = Ingestor.new(ingest_url, app_id, app_key, app_tenant, database, table, mapping, delete_temp_files, @logger, executor) # send existing files recover_past_files if recovery @last_stale_cleanup_cycle = Time.now @flush_interval = @flush_interval.to_i if @flush_interval > 0 @flusher = Interval.start(@flush_interval, -> { flush_pending_files }) end if (@stale_cleanup_type == 'interval') && (@stale_cleanup_interval > 0) @cleaner = Interval.start(stale_cleanup_interval, -> { close_stale_files }) end end |