Class: Log2Json::Filters::GrokFilter

Inherits:
Object
  • Object
show all
Defined in:
lib/log2json.rb

Overview

A filter takes a JSON log record, process it by adding, correcting or even removing attributes from it if necessary.

Direct Known Subclasses

NginxAccessLogFilter, SyslogFilter

Constant Summary collapse

DEFAULT_PATTERNS = 
File.join(File.dirname(__FILE__),
'log2json', 'filters', 'base.patterns')
CONFIG = 
{
  NAMED_CAPTURES_ONLY: true,
  KEEP_EMTPY_CAPTURES: false
}

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(type, name, regexps, opts = {}, &filter_block) ⇒ GrokFilter

Returns a new instance of GrokFilter.



146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/log2json.rb', line 146

def initialize(type, name, regexps, opts={}, &filter_block)
  @type = type
  @name = name
  @filter_block = filter_block
  @record_kvs = opts.select { |k,v| k.start_with?('@') }
  @config = opts.select { |k,v| not k.start_with?('@') }.merge CONFIG

  @pile = Grok::Pile.new
  @pile.add_patterns_from_file(@config[:pattern_file] || DEFAULT_PATTERNS)
  regexps.each { |re| @pile.compile(re) }
end

Instance Attribute Details

#name ⇒ Object (readonly)

Returns the value of attribute name.



144
145
146
 
# File 'lib/log2json.rb', line 144

def name
  @name
end

#type ⇒ Object (readonly)

Returns the value of attribute type.



144
145
146
 
# File 'lib/log2json.rb', line 144

def type
  @type
end

Instance Method Details

#filter(record) ⇒ Object

Filter the log record.

This means checking if the record matches the patterns of this filter and add the captured groups as members of the @fields of the record if there’s a match.

Any ‘@’ key-values configured for this filter will also be added to the record after merging the captured groups.

Return the record at the end.



168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
 
# File 'lib/log2json.rb', line 168

def filter(record)
  grok, match = @pile.match(record['@message'])
  if match
    # code stolen and modified from logstash's grok filter.
    fields = record['@fields']
    match.each_capture() do |key, value|
      next if value.nil? and not @config[:KEEP_EMTPY_CAPTURES]
      if key.include?(':')
        pattern_name, key, value_type = key.split(':') # ie, %{pattern_name:key:value_type}
        case value_type 
          when 'int'  ; value = value.to_i
          when 'float'; value = value.to_f
        end
      else
        next if @config[:NAMED_CAPTURES_ONLY]
      end
      if fields[key].nil?
        fields[key] = value
      else # if there already exists a field for the captured value
           # then we aggregate the captured values in an array for the field.
        if not fields[key].is_a?(Array)
          fields[key] = [fields[key]]
        end
        fields[key] << value
      end
    end

    record.merge!(@record_kvs) do |k, oldval, newval|
      if k == '@tags'
        oldval.concat(newval).uniq!
      elsif k == '@fields'
        oldval.merge!(newval)
      end
    end
    (fields['filtered_by'] ||= []) << name
    if @filter_block
      record = @filter_block.call(record)
    end
  end
  record
end