Class: LogStash::Filters::Json

Inherits:
Base
  • Object
show all
Extended by:
PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
Includes:
PluginMixins::ECSCompatibilitySupport, PluginMixins::ECSCompatibilitySupport::TargetCheck
Defined in:
lib/logstash/filters/json.rb

Overview

This is a JSON parsing filter. It takes an existing field which contains JSON and expands it into an actual data structure within the Logstash event.

By default it will place the parsed JSON in the root (top level) of the Logstash event, but this filter can be configured to place the JSON into any arbitrary event field, using the ‘target` configuration.

This plugin has a few fallback scenario when something bad happen during the parsing of the event. If the JSON parsing fails on the data, the event will be untouched and it will be tagged with a ‘_jsonparsefailure` then you can use conditionals to clean the data. You can configured this tag with then `tag_on_failure` option.

If the parsed data contains a ‘@timestamp` field, we will try to use it for the event’s ‘@timestamp`, if the parsing fails, the field will be renamed to `_@timestamp` and the event will be tagged with a `_timestampparsefailure`.

Instance Method Summary collapse

Instance Method Details

#_do_tag_on_failure(event) ⇒ Object 



140
141
142
 
# File 'lib/logstash/filters/json.rb', line 140

def _do_tag_on_failure(event)
  @tag_on_failure.each { |tag| event.tag(tag) }
end

#filter(event) ⇒ Object 



77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/logstash/filters/json.rb', line 77

def filter(event)
  @logger.debug? && @logger.debug("Running json filter", :event => event.to_hash)

  source = event.get(@source)
  return unless source

  begin
    parsed = LogStash::Json.load(source)
  rescue => e
    unless @skip_on_invalid_json
      _do_tag_on_failure(event)
      @logger.warn("Error parsing json", :source => @source, :raw => source, :exception => e)
    end
    return
  end

  if @target
    event.set(@target, parsed)
  else
    unless parsed.is_a?(Hash)
      _do_tag_on_failure(event)
      @logger.warn("Parsed JSON object/hash requires a target configuration option", :source => @source, :raw => source)
      return
    end

    # TODO: (colin) the timestamp initialization should be DRY'ed but exposing the similar code
    # in the Event#init_timestamp method. See https://github.com/elastic/logstash/issues/4293

    # a) since the parsed hash will be set in the event root, first extract any @timestamp field to properly initialized it
    parsed_timestamp = parsed.delete(LogStash::Event::TIMESTAMP)
    begin
      timestamp = parsed_timestamp ? LogStash::Timestamp.coerce(parsed_timestamp) : nil
    rescue LogStash::TimestampParserError => e
      @logger.debug("Failed to coerce timestamp", :timestamp => parsed_timestamp, :message => e.message)
      timestamp = nil
    end

    # b) then set all parsed fields in the event
    parsed.each{|k, v| event.set(k, v)}

    # c) finally re-inject proper @timestamp
    if parsed_timestamp
      if timestamp
        event.timestamp = timestamp
      else
        event.timestamp = LogStash::Timestamp.new
        @logger.warn("Unrecognized #{LogStash::Event::TIMESTAMP} value, setting current time to #{LogStash::Event::TIMESTAMP}, original in #{LogStash::Event::TIMESTAMP_FAILURE_FIELD} field", :value => parsed_timestamp.inspect)
        event.tag(LogStash::Event::TIMESTAMP_FAILURE_TAG)
        event.set(LogStash::Event::TIMESTAMP_FAILURE_FIELD, parsed_timestamp.to_s)
      end
    end
  end

  filter_matched(event)

  @logger.debug? && @logger.debug("Event after json filter", :event => event.to_hash)
rescue => ex
  meta = { :source => @source, :raw => source, :exception => ex }
  meta[:backtrace] = ex.backtrace if @logger.debug?
  @logger.warn('Exception caught in json filter', meta)
  _do_tag_on_failure(event)
end

#register ⇒ Object 



73
74
75
 
# File 'lib/logstash/filters/json.rb', line 73

def register
  # Nothing to do here
end