Class: LogStash::Filters::CSV

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/filters/csv.rb

Overview

The CSV filter takes an event field containing CSV data, parses it, and stores it as individual fields (can optionally specify the names). This filter can also parse data with any separator, not just commas.

Constant Summary collapse

CONVERTERS = 
{
  :integer => lambda do |value|
    CSV::Converters[:integer].call(value)
  end,

  :float => lambda do |value|
    CSV::Converters[:float].call(value)
  end,

  :date => lambda do |value|
    CSV::Converters[:date].call(value)
  end,

  :date_time => lambda do |value|
    CSV::Converters[:date_time].call(value)
  end,

  :boolean => lambda do |value|
     value = value.strip.downcase
     return false if value == "false"
     return true  if value == "true"
     return value
  end
}

Instance Method Summary collapse

Instance Method Details

#filter(event) ⇒ Object 



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/logstash/filters/csv.rb', line 108

def filter(event)
  @logger.debug? && @logger.debug("Running csv filter", :event => event)

  if (source = event.get(@source))
    begin
      values = CSV.parse_line(source, :col_sep => @separator, :quote_char => @quote_char)

      values.each_index do |i|
        unless (@skip_empty_columns && (values[i].nil? || values[i].empty?))
          unless ignore_field?(i)
            field_name = @columns[i] || "column#{i + 1}"
            event.set(field_ref(field_name), transform(field_name, values[i]))
          end
        end
      end

      filter_matched(event)
    rescue => e
      event.tag("_csvparsefailure")
      @logger.warn("Error parsing csv", :field => @source, :source => source, :exception => e)
      return
    end
  end

  @logger.debug? && @logger.debug("Event after csv filter", :event => event)
end

#register ⇒ Object

Raises:

  • (LogStash::ConfigurationError) 


86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/logstash/filters/csv.rb', line 86

def register
  # validate conversion types to be the valid ones.
  bad_types = @convert.values.select do |type|
    !CONVERTERS.has_key?(type.to_sym)
  end.uniq

  raise(LogStash::ConfigurationError, "Invalid conversion types: #{bad_types.join(', ')}") unless bad_types.empty?

  # @convert_symbols contains the symbolized types to avoid symbol conversion in the transform method
  @convert_symbols = @convert.inject({}){|result, (k, v)| result[k] = v.to_sym; result}

  # make sure @target is in the format [field name] if defined, i.e. surrounded by brakets
  @target = "[#{@target}]" if @target && @target !~ /^\[[^\[\]]+\]$/
  
  # if the zero byte character is entered in the config, set the value
  if (@quote_char == "\\x00")
    @quote_char = "\x00"
  end
  
  @logger.debug? && @logger.debug("CSV parsing options", :col_sep => @separator, :quote_char => @quote_char)
end