Class: Fluent::Plugin::MaskingFilter

Inherits:
Filter
  • Object
show all
Includes:
Helpers
Defined in:
lib/fluent/plugin/filter_masking.rb

Constant Summary collapse

MASK_STRING = 
"*******"

Instance Method Summary collapse

Methods included from Helpers

#myDig

Constructor Details

#initialize ⇒ MaskingFilter

Returns a new instance of MaskingFilter.



54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'lib/fluent/plugin/filter_masking.rb', line 54

def initialize
  super
  @fields_to_mask = []
  @fields_to_mask_regex = {}
  @fields_to_mask_keys = {}
  @fieldsToExcludeJSONPathsArray = []

  @handleSpecialEscapedJsonCases = false
  @specialEscapedJsonRegexs = {
    Regexp.new(/,(( *)(\\*)("*)( *)),/m) => "\1,"
  }
end

Instance Method Details

#configure(conf) ⇒ Object

this method only called ones (on startup time)



68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
 
# File 'lib/fluent/plugin/filter_masking.rb', line 68

def configure(conf)
  super
  fieldsToMaskFilePath = conf['fieldsToMaskFilePath']
  fieldsToExcludeJSONPaths = conf['fieldsToExcludeJSONPaths']
  handleSpecialCases = conf['handleSpecialEscapedJsonCases']

  if fieldsToExcludeJSONPaths != nil && fieldsToExcludeJSONPaths.size() > 0 
    fieldsToExcludeJSONPaths.split(",").each do | field |
      # To save splits we'll save the path as an array
      splitArray = field.split(".")
      symArray = []
      splitArray.each do | pathPortion |
        symArray.push(pathPortion.to_sym)
      end
      @fieldsToExcludeJSONPathsArray.push(symArray)
    end
  end

  File.open(fieldsToMaskFilePath, "r") do |f|
    f.each_line do |line|
      value = line.to_s # make sure it's string
      value = value.gsub(/\s+/, "") # remove spaces
      value = value.gsub('\n', '') # remove line breakers

      if value.end_with? "/i"
        # case insensitive
        value = value.delete_suffix('/i')
        hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/mi) # mask element in hash object
        innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/mi) # mask element in json string using capture groups that count the level of escaping inside the json string
      else
        # case sensitive
        hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m) # mask element in hash object
        innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/m) # mask element in json string using capture groups that count the level of escaping inside the json string
      end

      @fields_to_mask.push(value)

      hashObjectRegexStringReplacement = ":#{value}=>\"#{MASK_STRING}\""
      @fields_to_mask_regex[hashObjectRegex] = hashObjectRegexStringReplacement
      @fields_to_mask_keys[hashObjectRegex] = value

      innerJSONStringRegexStringReplacement = "\\1\"#{value}\\1\":\\1\"#{MASK_STRING}\\1\""
      @fields_to_mask_regex[innerJSONStringRegex] = innerJSONStringRegexStringReplacement
      @fields_to_mask_keys[innerJSONStringRegex] = value
    end
  end

  # if true, each record (a json record), will be checked for a special escaped json cases
  # any found case will be 'gsub' with the right solution 
  @handleSpecialEscapedJsonCases = handleSpecialCases != nil && handleSpecialCases.casecmp("true") == 0

  puts "black list fields:"
  puts @fields_to_mask
end

#filter(tag, time, record) ⇒ Object 



123
124
125
126
127
128
 
# File 'lib/fluent/plugin/filter_masking.rb', line 123

def filter(tag, time, record)
  # This method implements the filtering logic for individual filters
  # It is internal to this class and called by filter_stream unless
  # the user overrides filter_stream.
  maskRecord(record)
end

#maskRecord(record) ⇒ Object

returns the masked record error safe method - if any error occurs the original record is returned



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/fluent/plugin/filter_masking.rb', line 18

def maskRecord(record)
  maskedRecord = record
  excludedFields = []
  begin
    @fieldsToExcludeJSONPathsArray.each do | field |
      field_value = myDig(record, field)
      if field_value != nil
        excludedFields = excludedFields + field_value.split(',')
      end
    end
  rescue Exception => e
    $log.error "Failed to find mask exclude record: #{e}"
  end
  begin
    recordStr = record.to_s

    if @handleSpecialEscapedJsonCases == true
      @specialEscapedJsonRegexs.each do | regex, replace |
        recordStr = recordStr.gsub(regex, replace)
      end
    end  
    
    @fields_to_mask_regex.each do | fieldToMaskRegex, fieldToMaskRegexStringReplacement |
      if !(excludedFields.include? @fields_to_mask_keys[fieldToMaskRegex])
        recordStr = recordStr.gsub(fieldToMaskRegex, fieldToMaskRegexStringReplacement) 
      end
    end
    
    maskedRecord = strToHash(recordStr)
  rescue Exception => e
    $log.error "Failed to mask record: #{e}"
  end

  maskedRecord
end

#strToHash(str) ⇒ Object 



12
13
14
 
# File 'lib/fluent/plugin/filter_masking.rb', line 12

def strToHash(str)
  eval(str)
end