Class: LogStash::Outputs::Application_insights

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/outputs/application_insights/storage_cleanup.rb,
lib/logstash/outputs/application_insights/multi_io_logger.rb,
lib/logstash/outputs/application_insights/flow_control.rb,
lib/logstash/outputs/application_insights/sub_channel.rb,
lib/logstash/outputs/application_insights/exceptions.rb,
lib/logstash/outputs/application_insights/telemetry.rb,
lib/logstash/outputs/application_insights/constants.rb,
lib/logstash/outputs/application_insights/shutdown.rb,
lib/logstash/outputs/application_insights/channels.rb,
lib/logstash/outputs/application_insights/clients.rb,
lib/logstash/outputs/application_insights/channel.rb,
lib/logstash/outputs/application_insights/config.rb,
lib/logstash/outputs/application_insights/client.rb,
lib/logstash/outputs/application_insights/utils.rb,
lib/logstash/outputs/application_insights/timer.rb,
lib/logstash/outputs/application_insights/state.rb,
lib/logstash/outputs/application_insights/block.rb,
lib/logstash/outputs/application_insights/blob.rb,
lib/logstash/outputs/application_insights.rb

Overview

Logstash Output Application Insights

Copyright © Microsoft Corporation

All rights reserved.

Licensed under the Apache License, Version 2.0 (the License); you may not use this file except in compliance with the License. You may obtain a copy of the License at www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

See the Apache Version 2.0 License for specific language governing permissions and limitations under the License.

Defined Under Namespace

Classes: Blob, Block, BlockOverflowError, BlockTooSmallError, Channel, ChannelExistError, Channels, Client, Clients, Config, ConfigurationError, Flow_control, Multi_io_logger, NoChannelError, NotRecoverableError, Shutdown, State, StorageAccountsOffError, Storage_cleanup, Sub_channel, Telemetry, Timer, UnexpectedBranchError, UploadRetryError, Utils

Constant Summary collapse

BOOLEAN_PROPERTIES = 
[  :disable_notification, :disable_blob_upload, 
:stop_on_unknown_io_errors, :disable_telemetry, 
:disable_cleanup, :delete_not_notified_blobs, 
:save_notified_blobs_records, :case_insensitive_columns,
:table_columns, :serialized_event_field ]
GUID_NULL = 
"00000000-0000-0000-0000-000000000000"
INSTRUMENTATION_KEY_TEMPLATE = 
"KKKKKKKK-KKKK-KKKK-KKKK-KKKKKKKKKKKK"
TABLE_ID_TEMPLATE = 
"SSSSSSSS-SSSS-SSSS-SSSS-SSSSSSSSSSSS"
REQUEST_NAME =

notification payload constants

"Microsoft.ApplicationInsights.OpenSchema"
BASE_DATA_REQUIRED_VERSION = 
"2"
DATA_BASE_TYPE = 
"OpenSchemaData"
LOGGER_LEVEL_MAP =

logger constants

{:DEBUG => Logger::DEBUG, :INFO => Logger::INFO, :WARN => Logger::WARN, :ERROR => Logger::ERROR, :FATAL => Logger::FATAL, :UNKNOWN => Logger::UNKNOWN}
BLOB_BLOCK_MAX_BYTESIZE =

in bytes - 4 Mega bytes - blob limitation

4 * 1024 * 1024
BLOB_MAX_BLOCKS =

in blocks - 50,0000 blocks - blob limitation

50000
MIN_BLOB_MAX_BYTESIZE =

BLOB_BLOCK_MAX_BYTESIZE

BLOB_BLOCK_MAX_BYTESIZE
MAX_BLOB_MAX_BYTESIZE =

192 Giga bytes

BLOB_MAX_BLOCKS * BLOB_BLOCK_MAX_BYTESIZE
MIN_BLOB_MAX_EVENTS =

256 Kilo events

1
MAX_BLOB_MAX_EVENTS =

No Limit

0
MIN_BLOB_MAX_DELAY =

in seconds - one second

1
MAX_BLOB_MAX_DELAY =

in seconds - one day

24 * 3600
MIN_BLOB_RETENTION_TIME =

in seconds - one minute

60
MAX_BLOB_RETENTION_TIME =

in seconds - one year

60 * 60 * 24 * 365
MIN_BLOB_ACCESS_EXPIRY_TIME =

in seconds - one minute

60
MAX_BLOB_ACCESS_EXPIRY_TIME =

in seconds - one year

60 * 60 * 24 * 365
MIN_STORAGE_RESURRECT_DELAY =

in seconds - one second

1
MAX_STORAGE_RESURRECT_DELAY =

in seconds - one hour

3600
MIN_LOGGER_SHIFT_SIZE =

in bytes - 1 Kilo bytes

1024
MAX_LOGGER_SHIFT_SIZE =

in bytes - 1 Giga bytes

1 * 1024 * 1024 * 1024
MIN_LOGGER_SHIFT_AGE =

in retension version - no

0
MAX_LOGGER_SHIFT_AGE =

in retension version - almost limitless

100000
MIN_IO_RETRY_DELAY =

in seconds

0
MAX_IO_RETRY_DELAY =

in seconds - 5 minutes

300
MIN_IO_MAX_RETRIES = 
0
MAX_IO_MAX_RETRIES = 
3
MIN_FLOW_CONTROL_SUSPEND_BYTES =

in bytes,

0
MAX_FLOW_CONTROL_SUSPEND_BYTES =

in bytes, 0 means no upper limit

0
MIN_FLOW_CONTROL_RESUME_BYTES =

in bytes

0
MAX_FLOW_CONTROL_RESUME_BYTES =

in bytes, 0 means no upper limit

0
MIN_FLOW_CONTROL_DELAY =

in seconds, 1 seconds, can be less than 1 seconds, like 0.5, 0.1

0.1
MAX_FLOW_CONTROL_DELAY =

in seconds, 1 seconds, can be less than 1 seconds, like 0.5, 0.1

0
METADATA_FIELD_INSTRUMENTATION_KEY = 
"[@metadata]intrumentation_key"
METADATA_FIELD_TABLE_ID = 
"[@metadata]table_id"
FIELD_INSTRUMENTATION_KEY = 
"intrumentation_key"
FIELD_TABLE_ID = 
"table_id"
STATE_TABLE_NAME = 
"BlobsState"
AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX =

lower case only, dash allowed

"logstash"
AZURE_STORAGE_BLOB_LOGSTASH_PREFIX = 
"logstash"
AZURE_STORAGE_TABLE_LOGSTASH_PREFIX =

case sensitive, no dash

"Logstash"
AZURE_STORAGE_ORPHAN_BLOBS_CONTAINER_NAME = 
"orphan-blobs"
AZURE_STORAGE_NOTIFIED_BLOBS_TABLE_NAME = 
"orphan-blobs"
COLUMN_PROPERTY_NAME = 
"name"
COLUMN_PROPERTY_TYPE = 
"type"
COLUMN_PROPERTY_DEFAULT = 
"default"
VALID_FIELDS_MAP_TYPES = 
[ "string", "hash", "array", "number", "json", "boolean", "float", "integer", "dynamic", "datetime", "object" ]
VALID_LOGGER_SHIFT_AGES = 
[ "daily", "weekly", "monthly" ]
EXT_EVENT_FORMAT_JSON = 
"json"
EXT_EVENT_FORMAT_CSV = 
"csv"
VALID_EXT_EVENT_FORMAT = 
[EXT_EVENT_FORMAT_JSON, EXT_EVENT_FORMAT_CSV ]
DEFAULT_INSTRUMENTATION_KEY = 
GUID_NULL
DEFAULT_TABLE_ID = 
GUID_NULL
DEFAULT_EVENT_SEPARATOR = 
"\r\n"
DEFAULT_CSV_SEPARATOR = 
","
DEFAULT_CSV_DEFAULT_VALUE = 
""
DEFAULT_AZURE_STORAGE_BLOB_PREFIX = 
nil
DEFAULT_AZURE_STORAGE_CONTAINER_PREFIX = 
nil
DEFAULT_AZURE_STORAGE_TABLE_PREFIX = 
nil
DEFAULT_JSON_EXT = 
EXT_EVENT_FORMAT_JSON
DEFAULT_CSV_EXT = 
EXT_EVENT_FORMAT_CSV
DEFAULT_BLOB_SERIALIZATION = 
EXT_EVENT_FORMAT_JSON
DEFAULT_BLOB_MAX_BYTESIZE = 
1 * 1024 * 1024 * 1024
DEFAULT_BLOB_MAX_EVENTS =

256 Kilo events

256 * 1024
DEFAULT_BLOB_MAX_DELAY =

in seconds

60
DEFAULT_BLOB_RETENTION_TIME =

in seconds - one week

60 * 60 * 24 * 7
DEFAULT_BLOB_ACCESS_EXPIRY_TIME =

in seconds - one day

60 * 60 * 24 * 1
DEFAULT_STORAGE_RESURRECT_DELAY = 
10
DEFAULT_NOTIFICATION_ENDPOINT = 
"https://dc.services.visualstudio.com/v2/track"
DEFAULT_NOTIFICATION_VERSION = 
1
DEFAULT_DISABLE_NOTIFICATION = 
false
DEFAULT_DISABLE_BLOB_UPLOAD = 
false
DEFAULT_STOP_ON_UNKNOWN_IO_ERRORS = 
false
DEFAULT_DISABLE_TELEMETRY = 
false
DEFAULT_DISABLE_CLEANUP = 
false
DEFAULT_DELETE_NOT_NOTIFIED_BLOBS = 
false
DEFAULT_SAVE_NOTIFIED_BLOBS_RECORDS = 
false
DEFAULT_CASE_INSENSITIVE = 
false
DEFAULT_LOGGER_FILES = 
[ "logstash-output-application-insights.log" ]
DEFAULT_LOG_LEVEL = 
"INFO"
DEFAULT_LOGGER_PROGNAME = 
"AI"
DEFAULT_LOGGER_SHIFT_AGE =

daily back retension

"daily"
DEFAULT_LOGGER_SHIFT_SIZE =

in bytes - one Mega bytes

1024 * 1024
DEFAULT_IO_RETRY_DELAY =

in seconds

10
DEFAULT_IO_MAX_RETRIES = 
2
DEFAULT_FLOW_CONTROL_SUSPEND_BYTES =

high water mark, -in bytes, based on my laptop experience, without Java to break to NO MEMORY error

50 *1024 * 1024
DEFAULT_FLOW_CONTROL_RESUME_BYTES =

low water mark,

40 *1024 * 1024
DEFAULT_FLOW_CONTROL_DELAY =

in seconds, 1 seconds, can be less than 1 seconds, like 0.5, 0.1

1

Instance Method Summary collapse

Instance Method Details

#close ⇒ Object 



336
337
338
339
340
 
# File 'lib/logstash/outputs/application_insights.rb', line 336

def close
  Telemetry.instance.track_event( "close" )
  Telemetry.instance.flush
  @shutdown.submit
end

#default_configuration ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/logstash/outputs/application_insights/constants.rb', line 24

def default_configuration
  {
    :notification_version => @notification_version || DEFAULT_NOTIFICATION_VERSION,
    :event_separator => @event_separator || DEFAULT_EVENT_SEPARATOR,

    :notification_endpoint => @notification_endpoint || DEFAULT_NOTIFICATION_ENDPOINT,
    :azure_storage_blob_prefix => @azure_storage_blob_prefix || DEFAULT_AZURE_STORAGE_BLOB_PREFIX || Utils.to_storage_name( Socket.gethostname.strip ) || "",
    :azure_storage_container_prefix => @azure_storage_container_prefix || DEFAULT_AZURE_STORAGE_CONTAINER_PREFIX || Utils.to_storage_name( Socket.gethostname.strip ) || "",
    :azure_storage_table_prefix => @azure_storage_table_prefix || DEFAULT_AZURE_STORAGE_TABLE_PREFIX || Utils.to_storage_name( Socket.gethostname.strip ) || "",
    :storage_account_name_key => @storage_account_name_key || [  ],
    :disable_notification => @disable_notification || DEFAULT_DISABLE_NOTIFICATION,
    :disable_blob_upload => @disable_blob_upload || DEFAULT_DISABLE_BLOB_UPLOAD,
    :stop_on_unknown_io_errors => @stop_on_unknown_io_errors || DEFAULT_STOP_ON_UNKNOWN_IO_ERRORS,
    :delete_not_notified_blobs => @delete_not_notified_blobs || DEFAULT_DELETE_NOT_NOTIFIED_BLOBS,
    :save_notified_blobs_records => @save_notified_blobs_records || DEFAULT_SAVE_NOTIFIED_BLOBS_RECORDS,
    :disable_telemetry => @disable_telemetry || DEFAULT_DISABLE_TELEMETRY,
    :disable_cleanup => @disable_cleanup || DEFAULT_DISABLE_CLEANUP,
    :blob_max_bytesize => @blob_max_bytesize || DEFAULT_BLOB_MAX_BYTESIZE,
    :blob_max_events => @blob_max_events || DEFAULT_BLOB_MAX_EVENTS,
    :blob_retention_time => @blob_retention_time || DEFAULT_BLOB_RETENTION_TIME,
    :blob_access_expiry_time => @blob_access_expiry_time || DEFAULT_BLOB_ACCESS_EXPIRY_TIME,

    :resurrect_delay => @resurrect_delay || DEFAULT_STORAGE_RESURRECT_DELAY,
    :io_retry_delay => @io_retry_delay || DEFAULT_IO_RETRY_DELAY,
    :io_max_retries => @io_max_retries || DEFAULT_IO_MAX_RETRIES,

    :logger_level => @logger_level || DEFAULT_LOG_LEVEL,
    :logger_files => @logger_files || DEFAULT_LOGGER_FILES,
    :logger_progname => @logger_progname || DEFAULT_LOGGER_PROGNAME,
    :logger_shift_size => @logger_shift_size || DEFAULT_LOGGER_SHIFT_SIZE,
    :logger_shift_age => @logger_shift_age || DEFAULT_LOGGER_SHIFT_AGE,

    :flow_control_suspend_bytes => @flow_control_suspend_bytes || DEFAULT_FLOW_CONTROL_SUSPEND_BYTES,
    :flow_control_resume_bytes => @flow_control_resume_bytes || DEFAULT_FLOW_CONTROL_RESUME_BYTES,
    :flow_control_delay => @flow_control_delay || DEFAULT_FLOW_CONTROL_DELAY,

    :ca_file => @ca_file || "",

    :tables => @tables || {  },
    :table_id => @table_id || DEFAULT_TABLE_ID,
    :intrumentation_key => @intrumentation_key || DEFAULT_INSTRUMENTATION_KEY,
    :table_columns => @table_columns,
    :case_insensitive_columns => @case_insensitive_columns || DEFAULT_CASE_INSENSITIVE,
    :serialized_event_field => @serialized_event_field,
    :blob_max_delay => @blob_max_delay || DEFAULT_BLOB_MAX_DELAY,
    :blob_serialization => @blob_serialization || DEFAULT_BLOB_SERIALIZATION,
    :csv_separator => @csv_separator || DEFAULT_CSV_SEPARATOR,
    :csv_default_value => @csv_default_value || DEFAULT_CSV_DEFAULT_VALUE,

  }
end

#receive(event) ⇒ Object 



330
331
332
333
334
 
# File 'lib/logstash/outputs/application_insights.rb', line 330

def receive ( event )
  # @codec.encode( event )
  @channels.receive( event, nil )
  return "ok\n"
end

#register ⇒ Object 



291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
 
# File 'lib/logstash/outputs/application_insights.rb', line 291

def register

  # set configuration
  Config.validate_and_adjust_configuration( default_configuration )
  configuration = Config.current

  Multi_io_logger.config( configuration )

  # be careful don't use here @logger, as it will override Logstash @logger, and may create starnge behaviour
  @private_logger = configuration[:logger]

  @private_logger.info { "configuration: #{configuration}" }

  @telemetry = Telemetry.instance
  configuration[:telemetry_channel] = @telemetry.telemetry_channel

  Timer.config( configuration )
  Blob.config( configuration )
  Blob.validate_endpoint if @validate_endpoint
  Blob.validate_storage if @validate_storage

  @shutdown = Shutdown.instance
  @channels = Channels.instance

  @storage_cleanup = Storage_cleanup.start

  @private_logger.info { "plugin registered" }

  # @codec.on_event do |event, encoded_event|
  #   @channels.receive( event, encoded_event )
  # end

  Telemetry.instance.track_event("register", {:properties => configuration})


  return "ok\n"
end