Class: LogStash::Codecs::Multiline

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/codecs/multiline.rb

Overview

The multiline codec will collapse multiline messages and merge them into a single event.

The original goal of this codec was to allow joining of multiline messages from files into a single event. For example, joining Java exception and stacktrace messages into a single event.

The config looks like this:

source,ruby

input {

stdin {
  codec => multiline {
    pattern => "pattern, a regexp"
    negate => "true" or "false"
    what => "previous" or "next"
  }
}

}

The ‘pattern` should match what you believe to be an indicator that the field is part of a multi-line event.

The ‘what` must be `previous` or `next` and indicates the relation to the multi-line event.

The ‘negate` can be `true` or `false` (defaults to `false`). If `true`, a message not matching the pattern will constitute a match of the multiline filter and the `what` will be applied. (vice-versa is also true)

For example, Java stack traces are multiline and usually have the message starting at the far-left, with each subsequent line indented. Do this:

source,ruby

input {

stdin {
  codec => multiline {
    pattern => "^\s"
    what => "previous"
  }
}

}

This says that any line starting with whitespace belongs to the previous line.

Another example is to merge lines not starting with a date up to the previous line..

source,ruby

input 

file {
  path => "/var/log/someapp.log"
  codec => multiline {
    # Grok pattern names are valid! :)
    pattern => "^%{TIMESTAMP_ISO8601 "
    negate => true
    what => previous
  }
}

}

This says that any line not starting with a timestamp should be merged with the previous line.

One more common example is C line continuations (backslash). Here’s how to do that:

source,ruby

filter {

multiline {
  type => "somefiletype"
  pattern => "\\$"
  what => "next"
}

}

This says that any line ending with a backslash should be combined with the following line.

Instance Method Summary collapse

Instance Method Details

#buffer(text) ⇒ Object

def decode



177
178
179
180
181
 
# File 'lib/logstash/codecs/multiline.rb', line 177

def buffer(text)
  @time = LogStash::Timestamp.now if @buffer.empty?
  @buffer_bytes += text.bytesize
  @buffer << text
end

#buffer_over_limits? ⇒ Boolean

Returns:

  • (Boolean) 


221
222
223
 
# File 'lib/logstash/codecs/multiline.rb', line 221

def buffer_over_limits?
  over_maximun_lines? || over_maximun_bytes?
end

#decode(text, &block) ⇒ Object

def register



163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/logstash/codecs/multiline.rb', line 163

def decode(text, &block)
  text = @converter.convert(text)

  text.split("\n").each do |line|
    match = @grok.match(line)
    @logger.debug("Multiline", :pattern => @pattern, :text => line,
                  :match => !match.nil?, :negate => @negate)

    # Add negate option
    match = (match and !@negate) || (!match and @negate)
    @handler.call(line, match, &block)
  end
end

#do_next(text, matched, &block) ⇒ Object 



203
204
205
206
 
# File 'lib/logstash/codecs/multiline.rb', line 203

def do_next(text, matched, &block)
  buffer(text)
  flush(&block) if !matched || buffer_over_limits?
end

#do_previous(text, matched, &block) ⇒ Object 



208
209
210
211
 
# File 'lib/logstash/codecs/multiline.rb', line 208

def do_previous(text, matched, &block)
  flush(&block) if !matched || buffer_over_limits?
  buffer(text)
end

#encode(event) ⇒ Object 



225
226
227
228
 
# File 'lib/logstash/codecs/multiline.rb', line 225

def encode(event)
  # Nothing to do.
  @on_event.call(event, event)
end

#flush(&block) ⇒ Object 



183
184
185
186
187
188
 
# File 'lib/logstash/codecs/multiline.rb', line 183

def flush(&block)
  if @buffer.any? 
    yield merge_events
    reset_buffer
  end
end

#merge_events ⇒ Object 



190
191
192
193
194
195
196
 
# File 'lib/logstash/codecs/multiline.rb', line 190

def merge_events
  event = LogStash::Event.new(LogStash::Event::TIMESTAMP => @time, "message" => @buffer.join(NL))
  event.tag @multiline_tag if @multiline_tag && @buffer.size > 1
  event.tag "multiline_codec_max_bytes_reached" if over_maximun_bytes?
  event.tag "multiline_codec_max_lines_reached" if over_maximun_lines?
  event
end

#over_maximun_bytes? ⇒ Boolean

Returns:

  • (Boolean) 


217
218
219
 
# File 'lib/logstash/codecs/multiline.rb', line 217

def over_maximun_bytes?
  @buffer_bytes >= @max_bytes
end

#over_maximun_lines? ⇒ Boolean

Returns:

  • (Boolean) 


213
214
215
 
# File 'lib/logstash/codecs/multiline.rb', line 213

def over_maximun_lines?
  @buffer.size > @max_lines 
end

#register ⇒ Object 



130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/logstash/codecs/multiline.rb', line 130

def register
  require "grok-pure" # rubygem 'jls-grok'
  require 'logstash/patterns/core'

  # Detect if we are running from a jarfile, pick the right path.
  patterns_path = []
  patterns_path += [LogStash::Patterns::Core.path]

  @grok = Grok.new

  @patterns_dir = patterns_path.to_a + @patterns_dir
  @patterns_dir.each do |path|
    if File.directory?(path)
      path = File.join(path, "*")
    end

    Dir.glob(path).each do |file|
      @logger.info("Grok loading patterns from file", :path => file)
      @grok.add_patterns_from_file(file)
    end
  end

  @grok.compile(@pattern)
  @logger.debug("Registered multiline plugin", :type => @type, :config => @config)

  reset_buffer

  @handler = method("do_#{@what}".to_sym)

  @converter = LogStash::Util::Charset.new(@charset)
  @converter.logger = @logger
end

#reset_buffer ⇒ Object 



198
199
200
201
 
# File 'lib/logstash/codecs/multiline.rb', line 198

def reset_buffer
  @buffer = []
  @buffer_bytes = 0
end