Class: Fluent::QueryCombinerOutput

Inherits:
BufferedOutput
  • Object
show all
Defined in:
lib/fluent/plugin/out_query_combiner.rb

Instance Method Summary collapse

Constructor Details

#initialize ⇒ QueryCombinerOutput 



21
22
23
24
25
26
27
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 21

def initialize
  super
  require 'redis'
  require 'msgpack'
  require 'json'
  require 'rubygems'
end

Instance Method Details

#configure(conf) ⇒ Object 



29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 29

def configure(conf)
  super
  @host = conf.has_key?('host') ? conf['host'] : 'localhost'
  @port = conf.has_key?('port') ? conf['port'].to_i : 6379
  @db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil

  @query_identify = @query_identify.split(',').map { |qid| qid.strip }

  # Create functions for each conditions
  @_cond_funcs = {}
  @_replace_keys = {
    'catch' => {},
    'dump' => {},
  }

  def get_arguments(eval_str)
    eval_str.scan(/[\"\']?[a-zA-Z][\w\d\.\-\_]*[\"\']?/).uniq.select{|x|
      not (x.start_with?('\'') or x.start_with?('\"')) and \
      not %w{and or xor not}.include? x
    }
  end

  def parse_replace_expr(element_name, condition_name, str)
    result = {}
    str.split(',').each{|cond|
      before, after = cond.split('=>').map{|var| var.strip}
      result[before] = after
      if not (before.length > 0 and after.length > 0)
        raise Fluent::ConfigError, "SyntaxError at replace condition `#{element_name}`: #{condition_name}"
      end
    }
    if result.none?
      raise Fluent::ConfigError, "SyntaxError at replace condition `#{element_name}`: #{condition_name}"
    end
    result
  end

  def create_func(var, expr)
    begin
      f_argv = get_arguments(expr)
      f = eval('lambda {|' + f_argv.join(',') + '| ' + expr + '}')
      return [f, f_argv]
    rescue SyntaxError
      raise Fluent::ConfigError, "SyntaxError at condition `#{var}`: #{expr}"
    end
  end
  conf.elements.select { |element|
    %w{catch prolong dump release}.include? element.name
  }.each { |element|
    element.each_pair { |var, expr|
      element.has_key?(var)   # to suppress unread configuration warning

      if var == 'condition'
        formula, f_argv = create_func(var, expr)
        @_cond_funcs[element.name] = [f_argv, formula]

      elsif var == 'replace'
        if %w{catch dump}.include? element.name
          @_replace_keys[element.name] = parse_replace_expr(element.name, var, expr)
        else
          raise Fluent::ConfigError, "`replace` configuration in #{element.name}: only allowed in `catch` and `dump`"
        end
      end
    }
  }

  if not (@_cond_funcs.has_key?('catch') and @_cond_funcs.has_key?('dump'))
    raise Fluent::ConfigError, "Must have <catch> and <dump> blocks"
  end
end

#create_func(var, expr) ⇒ Object 



66
67
68
69
70
71
72
73
74
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 66

def create_func(var, expr)
  begin
    f_argv = get_arguments(expr)
    f = eval('lambda {|' + f_argv.join(',') + '| ' + expr + '}')
    return [f, f_argv]
  rescue SyntaxError
    raise Fluent::ConfigError, "SyntaxError at condition `#{var}`: #{expr}"
  end
end

#do_catch(qid, record, time) ⇒ Object 



161
162
163
164
165
166
167
168
169
170
171
172
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 161

def do_catch(qid, record, time)
  # replace record keys
  @_replace_keys['catch'].each_pair { |before, after|
    record[after] = record[before]
    record.delete(before)
  }
  # save record
  tryOnRedis 'set',    @redis_key_prefix + qid, JSON.dump(record)
  # update qid's timestamp
  tryOnRedis 'zadd', @redis_key_prefix, time, qid
  tryOnRedis 'expire', @redis_key_prefix + qid, @query_ttl
end

#do_dump(qid, record) ⇒ Object 



182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 182

def do_dump(qid, record)
  if (tryOnRedis 'exists', @redis_key_prefix + qid)
    # replace record keys
    @_replace_keys['dump'].each_pair { |before, after|
      record[after] = record[before]
      record.delete(before)
    }

    # emit
    catched_record = JSON.load(tryOnRedis('get', @redis_key_prefix + qid))
    combined_record = catched_record.merge(record)
    Fluent::Engine.emit @tag, Fluent::Engine.now, combined_record

    # remove qid
    do_release(qid)
  end
end

#do_prolong(qid, time) ⇒ Object 



174
175
176
177
178
179
180
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 174

def do_prolong(qid, time)
  if (tryOnRedis 'exists', @redis_key_prefix + qid)
    # update qid's timestamp
    tryOnRedis 'zadd', @redis_key_prefix, time, qid
    tryOnRedis 'expire', @redis_key_prefix + qid, @query_ttl
  end
end

#do_release(qid) ⇒ Object 



200
201
202
203
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 200

def do_release(qid)
  tryOnRedis 'del', @redis_key_prefix + qid
  tryOnRedis 'zrem', @redis_key_prefix, qid
end

#exec_func(record, f_argv, formula) ⇒ Object 



109
110
111
112
113
114
115
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 109

def exec_func(record, f_argv, formula)
  argv = []
  f_argv.each {|v|
    argv.push(record[v])
  }
  return formula.call(*argv)
end

#extract_qid(record) ⇒ Object 



205
206
207
208
209
210
211
212
213
214
215
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 205

def extract_qid(record)
  qid = []
  @query_identify.each { |attr|
    if record.has_key?(attr)
      qid.push(record[attr])
    else
      return nil
    end
  }
  qid.join(':')
end

#format(tag, time, record) ⇒ Object 



157
158
159
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 157

def format(tag, time, record)
  [tag, time, record].to_msgpack
end

#get_arguments(eval_str) ⇒ Object 



44
45
46
47
48
49
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 44

def get_arguments(eval_str)
  eval_str.scan(/[\"\']?[a-zA-Z][\w\d\.\-\_]*[\"\']?/).uniq.select{|x|
    not (x.start_with?('\'') or x.start_with?('\"')) and \
    not %w{and or xor not}.include? x
  }
end

#has_all_keys?(record, argv) ⇒ Boolean 



100
101
102
103
104
105
106
107
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 100

def has_all_keys?(record, argv)
  argv.each {|var|
    if not record.has_key?(var)
      return false
    end
  }
  true
end

#parse_replace_expr(element_name, condition_name, str) ⇒ Object 



51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 51

def parse_replace_expr(element_name, condition_name, str)
  result = {}
  str.split(',').each{|cond|
    before, after = cond.split('=>').map{|var| var.strip}
    result[before] = after
    if not (before.length > 0 and after.length > 0)
      raise Fluent::ConfigError, "SyntaxError at replace condition `#{element_name}`: #{condition_name}"
    end
  }
  if result.none?
    raise Fluent::ConfigError, "SyntaxError at replace condition `#{element_name}`: #{condition_name}"
  end
  result
end

#shutdown ⇒ Object 



136
137
138
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 136

def shutdown
  @redis.quit
end

#start ⇒ Object 



117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 117

def start
  super

  begin
    gem "hiredis"
    @redis = Redis.new(
        :host => @host, :port => @port, :driver => :hiredis,
        :thread_safe => true, :db => @db_index
    )
  rescue LoadError
    @redis = Redis.new(
        :host => @host, :port => @port,
        :thread_safe => true, :db => @db_index
    )
  end

  start_watch
end

#start_watch ⇒ Object 



153
154
155
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 153

def start_watch
  @watcher = Thread.new(&method(:watch))
end

#tryOnRedis(method, *args) ⇒ Object 



140
141
142
143
144
145
146
147
148
149
150
151
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 140

def tryOnRedis(method, *args)
  tries = 0
  begin
    @redis.send(method, *args) if @redis.respond_to? method
  rescue Redis::CommandError => e
    tries += 1
    # retry 3 times
    retry if tries <= @redis_retry
    $log.warn %Q[redis command retry failed : #{method}(#{args.join(', ')})]
    raise e.message
  end
end

#watch ⇒ Object 



245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 245

def watch
  @last_checked = Fluent::Engine.now
  tick = @remove_interval
  while true
    sleep 0.5
    if Fluent::Engine.now - @last_checked >= tick
      now = Fluent::Engine.now
      to_expire = now - @query_ttl

      # Delete expired qids
      tryOnRedis 'zremrangebyscore', @redis_key_prefix, '-inf', to_expire

      # Delete buffer_size over qids
      tryOnRedis 'zremrangebyrank', @redis_key_prefix, 0, -@buffer_size

      @last_checked = now
    end
  end
end

#write(chunk) ⇒ Object 



217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
 
# File 'lib/fluent/plugin/out_query_combiner.rb', line 217

def write(chunk)

  begin
    chunk.msgpack_each do |(tag, time, record)|
      if (qid = extract_qid record)

        @_cond_funcs.each_pair { |cond, argv_and_func|
          argv, func = argv_and_func
          if exec_func(record, argv, func)
            case cond
            when "catch"
              do_catch(qid, record, time)
            when "prolong"
              do_prolong(qid, time)
            when "dump"
              do_dump(qid, record)
            when "release"
              do_release(qid)
            end
            break   # very important!
          end
        }
      end
    end

  end
end