Class: Log2Json::Filters::GrokFilter

Inherits:
Object
  • Object
show all
Defined in:
lib/log2json.rb

Overview

A filter takes a JSON log record, process it by adding, correcting or even removing attributes from it if necessary.

Direct Known Subclasses

NginxAccessLogFilter, SyslogFilter

Constant Summary collapse

DEFAULT_PATTERNS = 
File.join(File.dirname(__FILE__),
'log2json', 'filters', 'base.patterns')
CONFIG = 
{
  NAMED_CAPTURES_ONLY: true,
  KEEP_EMTPY_CAPTURES: false
}

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(type, name, regexps, opts = {}, &filter_block) ⇒ GrokFilter

Returns a new instance of GrokFilter.



138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/log2json.rb', line 138

def initialize(type, name, regexps, opts={}, &filter_block)
  @type = type
  @name = name
  @filter_block = filter_block
  @record_kvs = opts.select { |k,v| k.start_with?('@') }
  @config = opts.select { |k,v| not k.start_with?('@') }.merge CONFIG

  @pile = Grok::Pile.new
  @pile.add_patterns_from_file(@config[:pattern_file] || DEFAULT_PATTERNS)
  regexps.each { |re| @pile.compile(re) }
end

Instance Attribute Details

#name ⇒ Object (readonly)

Returns the value of attribute name.



136
137
138
 
# File 'lib/log2json.rb', line 136

def name
  @name
end

#type ⇒ Object (readonly)

Returns the value of attribute type.



136
137
138
 
# File 'lib/log2json.rb', line 136

def type
  @type
end

Instance Method Details

#filter(record) ⇒ Object

Filter the log record.

This means checking if the record matches the patterns of this filter and add the captured groups as members of the @fields of the record if there’s a match.

Any ‘@’ key-values configured for this filter will also be added to the record after merging the captured groups.

Return the record at the end if there’s a match else return nil. If the ‘@timestamp’ attribute is removed from a record then the record will be dropped.



162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/log2json.rb', line 162

def filter(record)
  grok, match = @pile.match(record['@message'])
  if match
    # code stolen and modified from logstash's grok filter.
    fields = record['@fields']
    match.each_capture() do |key, value|
      next if value.nil? and not @config[:KEEP_EMTPY_CAPTURES]
      if key.include?(':')
        pattern_name, key, value_type = key.split(':') # ie, %{pattern_name:key:value_type}
        case value_type 
          when 'int'  ; value = value.to_i
          when 'float'; value = value.to_f
        end
      else
        next if @config[:NAMED_CAPTURES_ONLY]
      end
      if fields[key].nil?
        fields[key] = value
      else # if there already exists a field for the captured value
           # then we aggregate the captured values in an array for the field.
        if not fields[key].is_a?(Array)
          fields[key] = [fields[key]]
        end
        fields[key] << value
      end
    end

    record.merge!(@record_kvs) do |k, oldval, newval|
      if k == '@tags'
        oldval.concat(newval).uniq!
      elsif k == '@fields'
        oldval.merge!(newval)
      end
    end
    (fields['filtered_by'] ||= []) << name
    if @filter_block
      @filter_block.call(record)
    else
      record
    end
  else
    nil
  end
end