Class: LogStash::Inputs::Kinesis

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/inputs/kinesis.rb

Overview

Receive events through an AWS Kinesis stream.

This input plugin uses the Java Kinesis Client Library underneath, so the documentation at github.com/awslabs/amazon-kinesis-client will be useful.

AWS credentials can be specified either through environment variables, or an IAM instance role. The library uses a DynamoDB table for worker coordination, so you’ll need to grant access to that as well as to the Kinesis stream. The DynamoDB table has the same name as the ‘application_name` configuration option, which defaults to “logstash”.

The library can optionally also send worker statistics to CloudWatch.

Constant Summary collapse

KCL = 
com.amazonaws.services.kinesis.clientlibrary.lib.worker
KCL_PROCESSOR_FACTORY_CLASS = 
com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessorFactory

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(params = {}) ⇒ Kinesis

Returns a new instance of Kinesis.



72
73
74
 
# File 'lib/logstash/inputs/kinesis.rb', line 72

def initialize(params = {})
  super(params)
end

Instance Attribute Details

#kcl_config ⇒ Object (readonly)

Returns the value of attribute kcl_config.



33
34
35
 
# File 'lib/logstash/inputs/kinesis.rb', line 33

def kcl_config
  @kcl_config
end

Instance Method Details

#kcl_builder(output_queue) ⇒ Object 



131
132
133
134
135
136
137
138
139
140
 
# File 'lib/logstash/inputs/kinesis.rb', line 131

def kcl_builder(output_queue)
  KCL::Worker::Builder.new.tap do |builder|
    builder.java_send(:recordProcessorFactory, [KCL_PROCESSOR_FACTORY_CLASS.java_class], worker_factory(output_queue))
    builder.config(@kcl_config)

    if metrics_factory
      builder.metricsFactory(metrics_factory)
    end
  end
end

#register ⇒ Object 



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/logstash/inputs/kinesis.rb', line 76

def register
  # the INFO log level is extremely noisy in KCL
  kinesis_logger = org.apache.commons.logging::LogFactory.getLog("com.amazonaws.services.kinesis").logger
  if kinesis_logger.java_kind_of?(java.util.logging::Logger)
    kinesis_logger.setLevel(java.util.logging::Level::WARNING)
  else
    kinesis_logger.setLevel(org.apache.log4j::Level::WARN)
  end

  hostname = Socket.gethostname
  uuid = java.util::UUID.randomUUID.to_s
  worker_id = "#{hostname}:#{uuid}"

  # If the AWS profile is set, use the profile credentials provider.
  # Otherwise fall back to the default chain.
  unless @profile.nil?
    creds = com.amazonaws.auth.profile::ProfileCredentialsProvider.new(@profile)
  else
    creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new
  end

  # If a role ARN is set then assume the role as a new layer over the credentials already created
  unless @role_arn.nil?
    kinesis_creds = com.amazonaws.auth::STSAssumeRoleSessionCredentialsProvider.new(creds, @role_arn, @role_session_name)
  else
    kinesis_creds = creds
  end

  initial_position_in_stream = if @initial_position_in_stream == "TRIM_HORIZON"
    KCL::InitialPositionInStream::TRIM_HORIZON
  else
    KCL::InitialPositionInStream::LATEST
  end

  @kcl_config = KCL::KinesisClientLibConfiguration.new(
    @application_name,
    @kinesis_stream_name,
    kinesis_creds, # credential provider for Kinesis, DynamoDB and Cloudwatch access
    worker_id).
      withInitialPositionInStream(initial_position_in_stream).
      withRegionName(@region)

    # Call arbitrary "withX()" functions
    # snake_case => withCamelCase happens automatically
    @additional_settings.each do |key, value|
        fn = "with_#{key}"
        @kcl_config.send(fn, value)
    end
end

#run(output_queue) ⇒ Object 



126
127
128
129
 
# File 'lib/logstash/inputs/kinesis.rb', line 126

def run(output_queue)
  @kcl_worker = kcl_builder(output_queue).build
  @kcl_worker.run
end

#stop ⇒ Object 



142
143
144
 
# File 'lib/logstash/inputs/kinesis.rb', line 142

def stop
  @kcl_worker.shutdown if @kcl_worker
end

#worker_factory(output_queue) ⇒ Object 



146
147
148
 
# File 'lib/logstash/inputs/kinesis.rb', line 146

def worker_factory(output_queue)
  proc { Worker.new(@codec.clone, output_queue, method(:decorate), @checkpoint_interval_seconds, @logger) }
end