Class: Fluent::BeatsInput

Inherits:
Input
  • Object
show all
Defined in:
lib/fluent/plugin/in_beats.rb

Instance Method Summary collapse

Constructor Details

#initialize ⇒ BeatsInput

Returns a new instance of BeatsInput.



26
27
28
29
30
31
32
 
# File 'lib/fluent/plugin/in_beats.rb', line 26

def initialize
  super

  require "lumberjack/beats"
  require "lumberjack/beats/server"
  require 'concurrent/executor/cached_thread_pool'
end

Instance Method Details

#configure(conf) ⇒ Object 



45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/fluent/plugin/in_beats.rb', line 45

def configure(conf)
  super

  if !@tag && !@metadata_as_tag
    raise ConfigError,  "'tag' or 'metadata_as_tag' parameter is required on beats input"
  end

  @time_parser = Fluent::TextParser::TimeParser.new('%Y-%m-%dT%H:%M:%S.%N%z')
  if @format
    @parser = Plugin.new_parser(@format)
    @parser.configure(conf)
  end
  @connections = []
end

#run ⇒ Object 



80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/fluent/plugin/in_beats.rb', line 80

def run
  until @lumberjack.closed?
    conn = @lumberjack.accept
    next if conn.nil?

    if @max_connections
      @connections.reject! { |c| c.closed? }
      if @connections.size >= @max_connections
        conn.close # close for retry on beats side
        sleep 1
        next
      end          
      @connections << conn
    end

    @thread_pool.post {
      begin
        conn.run { |map|
          tag = @metadata_as_tag ? map['@metadata']['beat'] : @tag

          if map.has_key?('message') && @format
            message = map.delete('message')
            @parser.parse(message) { |time, record|
              record['@timestamp'] = map['@timestamp']
              map.each { |k, v|
                record[k] = v
              }
              router.emit(tag, time, record)
            }
            next
          end

          router.emit(tag, @time_parser.parse(map['@timestamp']), map)
        }
      rescue => e
        log.error "unexpected error", :error => e.to_s
        log.error_backtrace
      end
    }
  end
end

#shutdown ⇒ Object 



72
73
74
75
76
77
78
 
# File 'lib/fluent/plugin/in_beats.rb', line 72

def shutdown
  @lumberjack.close rescue nil
  @thread_pool.shutdown
  @thread.join

  super
end

#start ⇒ Object 



60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/fluent/plugin/in_beats.rb', line 60

def start
  super

  @lumberjack = Lumberjack::Beats::Server.new(
    :address => @bind, :port => @port, :ssl => @use_ssl, :ssl_certificate => @ssl_certificate,
    :ssl_key => @ssl_key, :ssl_key_passphrase => @ssl_key_passphrase)
  # Lumberjack::Beats::Server depends on normal accept so we need to launch thread for each connection.
  # TODO: Re-implement Beats Server with Cool.io for resource control
  @thread_pool = Concurrent::CachedThreadPool.new(:idletime => 15) # idletime setting is based on logstash beats input
  @thread = Thread.new(&method(:run))
end