Class: LogStash::Codecs::Multiline

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/codecs/multiline.rb

Overview

The multiline codec will collapse multiline messages and merge them into a single event.

The original goal of this codec was to allow joining of multiline messages from files into a single event. For example, joining Java exception and stacktrace messages into a single event.

The config looks like this:

source,ruby

input {

stdin {
  codec => multiline {
    pattern => "pattern, a regexp"
    negate => "true" or "false"
    what => "previous" or "next"
  }
}

}

The pattern should match what you believe to be an indicator that the field is part of a multi-line event.

The what must be previous or next and indicates the relation to the multi-line event.

The negate can be true or false (defaults to false). If true, a message not matching the pattern will constitute a match of the multiline filter and the what will be applied. (vice-versa is also true)

For example, Java stack traces are multiline and usually have the message starting at the far-left, with each subsequent line indented. Do this:

source,ruby

input {

stdin {
  codec => multiline {
    pattern => "^\s"
    what => "previous"
  }
}

}

This says that any line starting with whitespace belongs to the previous line.

Another example is to merge lines not starting with a date up to the previous line..

source,ruby

input 

file {
  path => "/var/log/someapp.log"
  codec => multiline {
    # Grok pattern names are valid! :)
    pattern => "^%{TIMESTAMP_ISO8601 "
    negate => true
    what => previous
  }
}

}

This says that any line not starting with a timestamp should be merged with the previous line.

One more common example is C line continuations (backslash). Here’s how to do that:

source,ruby

filter {

multiline {
  type => "somefiletype"
  pattern => "\\$"
  what => "next"
}

}

This says that any line ending with a backslash should be combined with the following line.

Instance Method Summary collapse

Instance Method Details

#buffer(text) ⇒ Object

def decode



163
164
165
166
 
# File 'lib/logstash/codecs/multiline.rb', line 163

def buffer(text)
  @time = LogStash::Timestamp.now if @buffer.empty?
  @buffer << text
end

#decode(text, &block) ⇒ Object 



151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/logstash/codecs/multiline.rb', line 151

def decode(text, &block)
  text = @converter.convert(text)

  match = @grok.match(text)
  @logger.debug("Multiline", :pattern => @pattern, :text => text,
                :match => !match.nil?, :negate => @negate)

  # Add negate option
  match = (match and !@negate) || (!match and @negate)
  @handler.call(text, match, &block)
end

#do_next(text, matched, &block) ⇒ Object 



179
180
181
182
 
# File 'lib/logstash/codecs/multiline.rb', line 179

def do_next(text, matched, &block)
  buffer(text)
  flush(&block) if !matched
end

#do_previous(text, matched, &block) ⇒ Object 



184
185
186
187
 
# File 'lib/logstash/codecs/multiline.rb', line 184

def do_previous(text, matched, &block)
  flush(&block) if !matched
  buffer(text)
end

#encode(event) ⇒ Object 



190
191
192
193
 
# File 'lib/logstash/codecs/multiline.rb', line 190

def encode(event)
  # Nothing to do.
  @on_event.call(event)
end

#flush(&block) ⇒ Object 



168
169
170
171
172
173
174
175
176
177
 
# File 'lib/logstash/codecs/multiline.rb', line 168

def flush(&block)
  if @buffer.any?
    event = LogStash::Event.new(LogStash::Event::TIMESTAMP => @time, "message" => @buffer.join(NL))
    # Tag multiline events
    event.tag @multiline_tag if @multiline_tag && @buffer.size > 1

    yield event
    @buffer = []
  end
end

#register ⇒ Object 



119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/logstash/codecs/multiline.rb', line 119

def register
  require "grok-pure" # rubygem 'jls-grok'
  require 'logstash/patterns/core'
  # Detect if we are running from a jarfile, pick the right path.
  patterns_path = []
  patterns_path += [LogStash::Patterns::Core.path]

  @grok = Grok.new

  @patterns_dir = patterns_path.to_a + @patterns_dir
  @patterns_dir.each do |path|
    if File.directory?(path)
      path = File.join(path, "*")
    end

    Dir.glob(path).each do |file|
      @logger.info("Grok loading patterns from file", :path => file)
      @grok.add_patterns_from_file(file)
    end
  end

  @grok.compile(@pattern)
  @logger.debug("Registered multiline plugin", :type => @type, :config => @config)

  @buffer = []
  @handler = method("do_#{@what}".to_sym)

  @converter = LogStash::Util::Charset.new(@charset)
  @converter.logger = @logger
end