Class: LogStash::Inputs::Kinesis

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/inputs/kinesis.rb

Overview

Receive events through an AWS Kinesis stream.

This input plugin uses the Java Kinesis Client Library underneath, so the documentation at github.com/awslabs/amazon-kinesis-client will be useful.

AWS credentials can be specified either through environment variables, or an IAM instance role. The library uses a DynamoDB table for worker coordination, so you’ll need to grant access to that as well as to the Kinesis stream. The DynamoDB table has the same name as the ‘application_name` configuration option, which defaults to “logstash”.

The library can optionally also send worker statistics to CloudWatch.

Constant Summary collapse

KCL = 
com.amazonaws.services.kinesis.clientlibrary.lib.worker
KCL_PROCESSOR_FACTORY_CLASS = 
com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessorFactory

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(params = {}) ⇒ Kinesis

Returns a new instance of Kinesis.



71
72
73
 
# File 'lib/logstash/inputs/kinesis.rb', line 71

def initialize(params = {})
  super(params)
end

Instance Attribute Details

#kcl_config ⇒ Object (readonly)

Returns the value of attribute kcl_config.



32
33
34
 
# File 'lib/logstash/inputs/kinesis.rb', line 32

def kcl_config
  @kcl_config
end

Instance Method Details

#kcl_builder(output_queue) ⇒ Object 



139
140
141
142
143
144
145
146
147
148
 
# File 'lib/logstash/inputs/kinesis.rb', line 139

def kcl_builder(output_queue)
  KCL::Worker::Builder.new.tap do |builder|
    builder.java_send(:recordProcessorFactory, [KCL_PROCESSOR_FACTORY_CLASS.java_class], worker_factory(output_queue))
    builder.config(@kcl_config)

    if metrics_factory
      builder.metricsFactory(metrics_factory)
    end
  end
end

#register ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
 
# File 'lib/logstash/inputs/kinesis.rb', line 75

def register
  # the INFO log level is extremely noisy in KCL
  lg = org.apache.commons.logging::LogFactory.getLog("com.amazonaws.services.kinesis")
  if lg.java_kind_of?(org.apache.commons.logging.impl::Jdk14Logger)
    kinesis_logger = lg.logger
    if kinesis_logger.java_kind_of?(java.util.logging::Logger)
      kinesis_logger.setLevel(java.util.logging::Level::WARNING)
    else
      kinesis_logger.setLevel(org.apache.log4j::Level::WARN)
    end
  elsif lg.java_kind_of?(org.apache.logging.log4jJcl::Log4jLog)
    logContext = org.apache.logging.log4j::LogManager.getContext(false)
    config = logContext.getConfiguration()
    config.getLoggerConfig("com.amazonaws.services.kinesis").setLevel(org.apache.logging.log4j::Level::WARN)
  else
    raise "Can't configure WARN log level for logger wrapper class #{lg.class}"
  end

  hostname = Socket.gethostname
  uuid = java.util::UUID.randomUUID.to_s
  worker_id = "#{hostname}:#{uuid}"

  # If the AWS profile is set, use the profile credentials provider.
  # Otherwise fall back to the default chain.
  unless @profile.nil?
    creds = com.amazonaws.auth.profile::ProfileCredentialsProvider.new(@profile)
  else
    creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new
  end

  # If a role ARN is set then assume the role as a new layer over the credentials already created
  unless @role_arn.nil?
    kinesis_creds = com.amazonaws.auth::STSAssumeRoleSessionCredentialsProvider.new(creds, @role_arn, @role_session_name)
  else
    kinesis_creds = creds
  end

  initial_position_in_stream = if @initial_position_in_stream == "TRIM_HORIZON"
    KCL::InitialPositionInStream::TRIM_HORIZON
  else
    KCL::InitialPositionInStream::LATEST
  end

  @kcl_config = KCL::KinesisClientLibConfiguration.new(
    @application_name,
    @kinesis_stream_name,
    kinesis_creds, # credential provider for Kinesis, DynamoDB and Cloudwatch access
    worker_id).
      withInitialPositionInStream(initial_position_in_stream).
      withRegionName(@region)

    # Call arbitrary "withX()" functions
    # snake_case => withCamelCase happens automatically
    @additional_settings.each do |key, value|
        fn = "with_#{key}"
        @kcl_config.send(fn, value)
    end
end

#run(output_queue) ⇒ Object 



134
135
136
137
 
# File 'lib/logstash/inputs/kinesis.rb', line 134

def run(output_queue)
  @kcl_worker = kcl_builder(output_queue).build
  @kcl_worker.run
end

#stop ⇒ Object 



150
151
152
 
# File 'lib/logstash/inputs/kinesis.rb', line 150

def stop
  @kcl_worker.shutdown if @kcl_worker
end

#worker_factory(output_queue) ⇒ Object 



154
155
156
 
# File 'lib/logstash/inputs/kinesis.rb', line 154

def worker_factory(output_queue)
  proc { Worker.new(@codec.clone, output_queue, method(:decorate), @checkpoint_interval_seconds, @logger) }
end