Class: LogStash::Filters::WebServiceMap

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/filters/webservicemap.rb

Overview

A general search and replace tool which uses a Web service with a YAML, CSV or JSON response to determine replacement values.

The map entries can be specified with a Web service who your request produces a YML, CSV or JSON response.

Operationally, if the event field specified in the ‘field` configuration matches the EXACT contents of a map entry key, the field’s value will be substituted with the matched key’s value from the map.

By default, the webservicemap filter will replace the contents of the maching event field (in-place). However, by using the ‘destination` configuration item, you may also specify a target event field to populate with the new mapd value.

Instance Method Summary collapse

Instance Method Details

#csv_loader(data) ⇒ Object 



90
91
92
93
94
95
96
 
# File 'lib/logstash/filters/webservicemap.rb', line 90

def csv_loader(data)
  data = CSV.read(data).inject(Hash.new) do |acc, v|
    acc[v[0]] = v[1]
    acc
  end
  get_map.merge!(data)
end

#download_ws(path, registering = false) ⇒ Object 



129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/logstash/filters/webservicemap.rb', line 129

def download_ws(path, registering=false)
  extension = get_extension(path)
  temp_extension = '_temp'+extension;
  file_name = Digest::SHA1.hexdigest path
  begin
    File.open(file_name+temp_extension, 'wb') do |saved_file|
      open(path, 'rb') do |read_file|
        saved_file.write(read_file.read)
      end
    end
  rescue Exception => _
    if registering
      raise "#{self.class.name}: Failed to initialize with #{file_name} and path #{path}"
    end
    @logger.warn("#{self.class.name}: Something happened with URL. Continuing with old map", :map_path => file_name, :path => path)
  end

  begin
    load_file(registering, extension, file_name+temp_extension)
    FileUtils.mv(file_name+temp_extension, file_name+extension)
  rescue Exception => _
    FileUtils.rm_f(file_name+temp_extension)
  end
end

#filter(event) ⇒ Object 



157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/logstash/filters/webservicemap.rb', line 157

def filter(event)
  if @next_refresh < Time.now
    download_ws(@map_url)
    @next_refresh = Time.now + @refresh_interval
    @logger.info('downloading and refreshing map file')
  end

  return unless event.include?(@field) # Skip mapping in case event does not have @event field.
  return if event.include?(@destination) and not @override # Skip mapping in case @destination field already exists and @override is disabled.

  begin
    source = event[@field].is_a?(Array) ? event[@field].first.to_s : event[@field].to_s
    matched = false
    if get_map.include?(source)
      event[@destination] = get_map[source]
      matched = true
    end

    if not matched and @fallback
      event[@destination] = event.sprintf(@fallback)
      matched = true
    end
    filter_matched(event) if matched or @field == @destination
  rescue Exception => e
    @logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @field, :event => event)
  end
end

#get_extension(path) ⇒ Object 



119
120
121
122
123
124
125
126
 
# File 'lib/logstash/filters/webservicemap.rb', line 119

def get_extension(path)
  if path.end_with?('json')
    return '.json'
  elsif path.end_with?('csv')
    return '.csv'
  end
  '.yml'
end

#get_map ⇒ Object 



66
67
68
 
# File 'lib/logstash/filters/webservicemap.rb', line 66

def get_map
  @my_map
end

#json_loader(data) ⇒ Object

def register



86
87
88
 
# File 'lib/logstash/filters/webservicemap.rb', line 86

def json_loader(data)
  get_map.merge!(JSON.parse(File.read(data)))
end

#load_file(registering, extension, data) ⇒ Object 



102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/logstash/filters/webservicemap.rb', line 102

def load_file(registering, extension, data)
  begin
    if extension.equal?('.json')
      return json_loader(data)
    elsif extension.end_with?('.csv')
      return csv_loader(data)
    end
    yml_loader(data)
  rescue Exception => _
    if registering
      raise "#{self.class.name}: Bad Syntax in map file #{file_name}"
    else
      @logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path => file_name)
    end
  end
end

#register ⇒ Object 



75
76
77
78
79
80
81
82
 
# File 'lib/logstash/filters/webservicemap.rb', line 75

def register
  @my_map = {}
  @next_refresh = Time.now + @refresh_interval
  download_ws(@map_url, true)
  @logger.debug? and @logger.debug("#{self.class.name}: map - ", :map => get_map)
  type = 'Exact'
  @logger.debug? and @logger.debug("#{self.class.name}: map mapping method - "+type)
end

#set_map(map) ⇒ Object 



70
71
72
 
# File 'lib/logstash/filters/webservicemap.rb', line 70

def set_map(map)
  @my_map = map;
end

#yml_loader(data) ⇒ Object 



98
99
100
 
# File 'lib/logstash/filters/webservicemap.rb', line 98

def yml_loader(data)
  get_map.merge!(YAML.load_file(data))
end