Class: LogStash::Filters::Translate

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/filters/translate.rb

Overview

A general search and replace tool that uses a configured hash and/or a file to determine replacement values. Currently supported are YAML, JSON, and CSV files.

The dictionary entries can be specified in one of two ways: First, the ‘dictionary` configuration item may contain a hash representing the mapping. Second, an external file (readable by logstash) may be specified in the `dictionary_path` configuration item. These two methods may not be used in conjunction; it will produce an error.

Operationally, if the event field specified in the ‘field` configuration matches the EXACT contents of a dictionary entry key (or matches a regex if `regex` configuration item has been enabled), the field’s value will be substituted with the matched key’s value from the dictionary.

By default, the translate filter will replace the contents of the maching event field (in-place). However, by using the ‘destination` configuration item, you may also specify a target event field to populate with the new translated value.

Alternatively, for simple string search and replacements for just a few values you might consider using the gsub function of the mutate filter.

Instance Method Summary collapse

Instance Method Details

#filter(event) ⇒ Object 



182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
 
# File 'lib/logstash/filters/translate.rb', line 182

def filter(event)
  if @dictionary_path
    if needs_refresh?
      lock_for_write do
        if needs_refresh?
          load_dictionary
          @next_refresh = Time.now + @refresh_interval
          @logger.info("refreshing dictionary file")
        end
      end
    end
  end

  return unless event.include?(@field) # Skip translation in case event does not have @event field.
  return if event.include?(@destination) and not @override # Skip translation in case @destination field already exists and @override is disabled.

  begin
    #If source field is array use first value and make sure source value is string
    source = event.get(@field).is_a?(Array) ? event.get(@field).first.to_s : event.get(@field).to_s
    matched = false
    if @exact
      if @regex
        key = @dictionary.keys.detect{|k| source.match(Regexp.new(k))}
        if key
          event.set(@destination, lock_for_read { @dictionary[key] })
          matched = true
        end
      elsif @dictionary.include?(source)
        event.set(@destination, lock_for_read { @dictionary[source] })
        matched = true
      end
    else
      translation = lock_for_read { source.gsub(Regexp.union(@dictionary.keys), @dictionary) }
      
      if source != translation
        event.set(@destination, translation.force_encoding(Encoding::UTF_8))
        matched = true
      end
    end

    if not matched and @fallback
      event.set(@destination, event.sprintf(@fallback))
      matched = true
    end
    filter_matched(event) if matched or @field == @destination
  rescue Exception => e
    @logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :field => @field, :event => event)
  end
end

#lock_for_read ⇒ Object

def register



164
165
166
167
168
169
170
171
 
# File 'lib/logstash/filters/translate.rb', line 164

def lock_for_read
  @read_lock.lock
  begin
    yield
  ensure
    @read_lock.unlock
  end
end

#lock_for_write ⇒ Object 



173
174
175
176
177
178
179
180
 
# File 'lib/logstash/filters/translate.rb', line 173

def lock_for_write
  @write_lock.lock
  begin
    yield
  ensure
    @write_lock.unlock
  end
end

#register ⇒ Object 



136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/logstash/filters/translate.rb', line 136

def register
  rw_lock = java.util.concurrent.locks.ReentrantReadWriteLock.new
  @read_lock = rw_lock.readLock
  @write_lock = rw_lock.writeLock
  
  if @dictionary_path && !@dictionary.empty?
    raise LogStash::ConfigurationError, I18n.t(
      "logstash.agent.configuration.invalid_plugin_register",
      :plugin => "filter",
      :type => "translate",
      :error => "The configuration options 'dictionary' and 'dictionary_path' are mutually exclusive"
    )
  end

  if @dictionary_path
    @next_refresh = Time.now + @refresh_interval
    raise_exception = true
    lock_for_write { load_dictionary(raise_exception) }
  end

  @logger.debug? and @logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @dictionary)
  if @exact
    @logger.debug? and @logger.debug("#{self.class.name}: Dictionary translation method - Exact")
  else
    @logger.debug? and @logger.debug("#{self.class.name}: Dictionary translation method - Fuzzy")
  end
end