Class: LogStash::Filters::WebServiceMap

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/filters/webservicemap.rb

Overview

A general search and replace tool which uses a Web service with a YAML, CSV or JSON response to determine replacement values.

The map entries can be specified with a Web service who your request produces a YML, CSV or JSON response.

Operationally, if the event field specified in the ‘field` configuration matches the EXACT contents of a map entry key, the field’s value will be substituted with the matched key’s value from the map.

By default, the webservicemap filter will replace the contents of the maching event field (in-place). However, by using the ‘destination` configuration item, you may also specify a target event field to populate with the new mapd value.

Instance Method Summary collapse

Instance Method Details

#csv_loader(data) ⇒ Object 



86
87
88
89
90
91
92
 
# File 'lib/logstash/filters/webservicemap.rb', line 86

def csv_loader(data)
  data = CSV.read(data).inject(Hash.new) do |acc, v|
    acc[v[0]] = v[1]
    acc
  end
  get_map.merge!(data)
end

#download_ws(path, registering = false) ⇒ Object 



125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/logstash/filters/webservicemap.rb', line 125

def download_ws(path, registering=false)
  extension = get_extension(path)
  temp_extension = '_temp'+extension;
  file_name = Digest::SHA1.hexdigest path
  File.open(file_name+temp_extension, 'wb') do |saved_file|
    open(path, 'rb') do |read_file|
      saved_file.write(read_file.read)
    end
  end
  begin
    load_file(registering, extension, file_name+temp_extension)
    FileUtils.mv(file_name+temp_extension, file_name+extension)
  rescue Exception => _
    FileUtils.rm_f(file_name+temp_extension)
  end
end

#filter(event) ⇒ Object 



145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
 
# File 'lib/logstash/filters/webservicemap.rb', line 145

def filter(event)
  if @next_refresh < Time.now
    download_ws(@map_url)
    @next_refresh = Time.now + @refresh_interval
    @logger.info('downloading and refreshing map file')
  end

  return unless event.include?(@field) # Skip mapping in case event does not have @event field.
  return if event.include?(@destination) and not @override # Skip mapping in case @destination field already exists and @override is disabled.

  begin
    source = event[@field].is_a?(Array) ? event[@field].first.to_s : event[@field].to_s
    matched = false
    if get_map.include?(source)
      event[@destination] = get_map[source]
      matched = true
    end

    if not matched and @fallback
      event[@destination] = event.sprintf(@fallback)
      matched = true
    end
    filter_matched(event) if matched or @field == @destination
  rescue Exception => e
    @logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @field, :event => event)
  end
end

#get_extension(path) ⇒ Object 



115
116
117
118
119
120
121
122
 
# File 'lib/logstash/filters/webservicemap.rb', line 115

def get_extension(path)
  if path.end_with?('json')
    return '.json'
  elsif path.end_with?('csv')
    return '.csv'
  end
  '.yml'
end

#get_map ⇒ Object 



62
63
64
 
# File 'lib/logstash/filters/webservicemap.rb', line 62

def get_map
  @my_map
end

#json_loader(data) ⇒ Object

def register



82
83
84
 
# File 'lib/logstash/filters/webservicemap.rb', line 82

def json_loader(data)
  get_map.merge!(JSON.parse(File.read(data)))
end

#load_file(registering, extension, data) ⇒ Object 



98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/logstash/filters/webservicemap.rb', line 98

def load_file(registering, extension, data)
  begin
    if extension.equal?('.json')
      return json_loader(data)
    elsif extension.end_with?('.csv')
      return csv_loader(data)
    end
    yml_loader(data)
  rescue Exception => _
    if registering
      raise "#{self.class.name}: Bad Syntax in map file #{file_name}"
    else
      @logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path => file_name)
    end
  end
end

#register ⇒ Object 



71
72
73
74
75
76
77
78
 
# File 'lib/logstash/filters/webservicemap.rb', line 71

def register
  @my_map = {}
  @next_refresh = Time.now + @refresh_interval
  download_ws(@map_url, true)
  @logger.debug? and @logger.debug("#{self.class.name}: map - ", :map => get_map)
  type = 'Exact'
  @logger.debug? and @logger.debug("#{self.class.name}: map mapping method - "+type)
end

#set_map(map) ⇒ Object 



66
67
68
 
# File 'lib/logstash/filters/webservicemap.rb', line 66

def set_map(map)
  @my_map = map;
end

#yml_loader(data) ⇒ Object 



94
95
96
 
# File 'lib/logstash/filters/webservicemap.rb', line 94

def yml_loader(data)
  get_map.merge!(YAML.load_file(data))
end