Class: Fluent::Plugin::TailInput

Inherits:
Input
  • Object
show all
Defined in:
lib/fluent/plugin/in_tail.rb

Defined Under Namespace

Classes: FilePositionEntry, MemoryPositionEntry, PositionFile, TailWatcher

Constant Summary collapse

FILE_PERMISSION = 
0644

Constants included from Configurable

Configurable::CONFIG_TYPE_REGISTRY

Instance Attribute Summary collapse

Attributes included from Fluent::PluginLoggerMixin

#log

Attributes inherited from Base

#under_plugin_development

Instance Method Summary collapse

Methods inherited from Input

#multi_workers_ready?

Methods included from Fluent::PluginHelper::Mixin

included

Methods included from Fluent::PluginLoggerMixin

included, #terminate

Methods included from Fluent::PluginId

#plugin_id, #plugin_id_configured?, #plugin_id_for_test?, #plugin_root_dir

Methods inherited from Base

#after_shutdown, #after_shutdown?, #after_start, #after_started?, #before_shutdown, #before_shutdown?, #closed?, #configured?, #context_router, #context_router=, #fluentd_worker_id, #has_router?, #inspect, #multi_workers_ready?, #plugin_root_dir, #shutdown?, #started?, #stop, #stopped?, #string_safe_encoding, #terminate, #terminated?

Methods included from SystemConfig::Mixin

#system_config, #system_config_override

Methods included from Configurable

#config, #configure_proxy_generate, #configured_section_create, included, lookup_type, register_type

Constructor Details

#initialize ⇒ TailInput

Returns a new instance of TailInput.



39
40
41
42
43
44
45
46
 
# File 'lib/fluent/plugin/in_tail.rb', line 39

def initialize
  super
  @paths = []
  @tails = {}
  @pf_file = nil
  @pf = nil
  @ignore_list = []
end

Instance Attribute Details

#paths ⇒ Object (readonly)

Returns the value of attribute paths.



88
89
90
 
# File 'lib/fluent/plugin/in_tail.rb', line 88

def paths
  @paths
end

Instance Method Details

#close ⇒ Object 



180
181
182
183
184
 
# File 'lib/fluent/plugin/in_tail.rb', line 180

def close
  super
  # close file handles after all threads stopped (in #close of thread plugin helper)
  close_watcher_handles
end

#close_watcher_handles ⇒ Object 



278
279
280
281
282
283
284
285
 
# File 'lib/fluent/plugin/in_tail.rb', line 278

def close_watcher_handles
  @tails.keys.each do |path|
    tw = @tails.delete(path)
    if tw
      tw.close
    end
  end
end

#configure(conf) ⇒ Object 



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/fluent/plugin/in_tail.rb', line 90

def configure(conf)
  compat_parameters_convert(conf, :parser)
  parser_config = conf.elements('parse').first
  unless parser_config
    raise Fluent::ConfigError, "<parse> section is required."
  end
  unless parser_config["@type"]
    raise Fluent::ConfigError, "parse/@type is required."
  end

  (1..Fluent::Plugin::MultilineParser::FORMAT_MAX_NUM).each do |n|
    parser_config["format#{n}"] = conf["format#{n}"] if conf["format#{n}"]
  end

  super

  @paths = @path.split(',').map {|path| path.strip }
  if @paths.empty?
    raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
  end

  # TODO: Use plugin_root_dir and storage plugin to store positions if available
  unless @pos_file
    $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
    $log.warn "this parameter is highly recommended to save the position to resume tailing."
  end

  configure_tag
  configure_encoding

  @multiline_mode = parser_config["@type"] =~ /multiline/
  @receive_handler = if @multiline_mode
                       method(:parse_multilines)
                     else
                       method(:parse_singleline)
                     end
  @file_perm = system_config.file_permission || FILE_PERMISSION
  @parser = parser_create(conf: parser_config)
end

#configure_encoding ⇒ Object 



140
141
142
143
144
145
146
147
148
149
 
# File 'lib/fluent/plugin/in_tail.rb', line 140

def configure_encoding
  unless @encoding
    if @from_encoding
      raise Fluent::ConfigError, "tail: 'from_encoding' parameter must be specified with 'encoding' parameter."
    end
  end

  @encoding = parse_encoding_param(@encoding) if @encoding
  @from_encoding = parse_encoding_param(@from_encoding) if @from_encoding
end

#configure_tag ⇒ Object 



130
131
132
133
134
135
136
137
138
 
# File 'lib/fluent/plugin/in_tail.rb', line 130

def configure_tag
  if @tag.index('*')
    @tag_prefix, @tag_suffix = @tag.split('*')
    @tag_suffix ||= ''
  else
    @tag_prefix = nil
    @tag_suffix = nil
  end
end

#convert_line_to_event(line, es, tail_watcher) ⇒ Object 



360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
 
# File 'lib/fluent/plugin/in_tail.rb', line 360

def convert_line_to_event(line, es, tail_watcher)
  begin
    line.chomp!  # remove \n
    @parser.parse(line) { |time, record|
      if time && record
        record[@path_key] ||= tail_watcher.path unless @path_key.nil?
        es.add(time, record)
      else
        if @emit_unmatched_lines
          record = {'unmatched_line' => line}
          record[@path_key] ||= tail_watcher.path unless @path_key.nil?
          es.add(Fluent::EventTime.now, record)
        end
        log.warn "pattern not match: #{line.inspect}"
      end
    }
  rescue => e
    log.warn line.dump, error: e.to_s
    log.debug_backtrace(e.backtrace)
  end
end

#detach_watcher(tw, close_io = true) ⇒ Object

TailWatcher#close is called by another thread at shutdown phase. It causes ‘can’t modify string; temporarily locked’ error in IOHandler so adding close_io argument to avoid this problem. At shutdown, IOHandler’s io will be released automatically after detached the event loop



304
305
306
307
308
309
310
311
 
# File 'lib/fluent/plugin/in_tail.rb', line 304

def detach_watcher(tw, close_io = true)
  tw.detach
  tw.close if close_io
  flush_buffer(tw)
  if tw.unwatched && @pf
    @pf[tw.path].update_pos(PositionFile::UNWATCHED_POSITION)
  end
end

#detach_watcher_after_rotate_wait(tw) ⇒ Object 



313
314
315
316
317
 
# File 'lib/fluent/plugin/in_tail.rb', line 313

def detach_watcher_after_rotate_wait(tw)
  timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
    detach_watcher(tw)
  end
end

#expand_paths ⇒ Object 



186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
 
# File 'lib/fluent/plugin/in_tail.rb', line 186

def expand_paths
  date = Time.now
  paths = []

  @paths.each { |path|
    path = date.strftime(path)
    if path.include?('*')
      paths += Dir.glob(path).select { |p|
        is_file = !File.directory?(p)
        if File.readable?(p) && is_file
          if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
            false
          else
            true
          end
        else
          if is_file
            unless @ignore_list.include?(path)
              log.warn "#{p} unreadable. It is excluded and would be examined next time."
              @ignore_list << path if @ignore_repeated_permission_error
            end
          end
          false
        end
      }
    else
      # When file is not created yet, Dir.glob returns an empty array. So just add when path is static.
      paths << path
    end
  }
  excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
  paths - excluded
end

#flush_buffer(tw) ⇒ Object 



319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
 
# File 'lib/fluent/plugin/in_tail.rb', line 319

def flush_buffer(tw)
  if lb = tw.line_buffer
    lb.chomp!
    @parser.parse(lb) { |time, record|
      if time && record
        tag = if @tag_prefix || @tag_suffix
                @tag_prefix + tw.tag + @tag_suffix
              else
                @tag
              end
        record[@path_key] ||= tw.path unless @path_key.nil?
        router.emit(tag, time, record)
      else
        log.warn "got incomplete line at shutdown from #{tw.path}: #{lb.inspect}"
      end
    }
  end
end

#parse_encoding_param(encoding_name) ⇒ Object 



151
152
153
154
155
156
157
 
# File 'lib/fluent/plugin/in_tail.rb', line 151

def parse_encoding_param(encoding_name)
  begin
    Encoding.find(encoding_name) if encoding_name
  rescue ArgumentError => e
    raise Fluent::ConfigError, e.message
  end
end

#parse_multilines(lines, tail_watcher) ⇒ Object 



390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
 
# File 'lib/fluent/plugin/in_tail.rb', line 390

def parse_multilines(lines, tail_watcher)
  lb = tail_watcher.line_buffer
  es = Fluent::MultiEventStream.new
  if @parser.has_firstline?
    tail_watcher.line_buffer_timer_flusher.reset_timer if tail_watcher.line_buffer_timer_flusher
    lines.each { |line|
      if @parser.firstline?(line)
        if lb
          convert_line_to_event(lb, es, tail_watcher)
        end
        lb = line
      else
        if lb.nil?
          if @emit_unmatched_lines
            convert_line_to_event(line, es, tail_watcher)
          end
          log.warn "got incomplete line before first line from #{tail_watcher.path}: #{line.inspect}"
        else
          lb << line
        end
      end
    }
  else
    lb ||= ''
    lines.each do |line|
      lb << line
      @parser.parse(lb) { |time, record|
        if time && record
          convert_line_to_event(lb, es, tail_watcher)
          lb = ''
        end
      }
    end
  end
  tail_watcher.line_buffer = lb
  es
end

#parse_singleline(lines, tail_watcher) ⇒ Object 



382
383
384
385
386
387
388
 
# File 'lib/fluent/plugin/in_tail.rb', line 382

def parse_singleline(lines, tail_watcher)
  es = Fluent::MultiEventStream.new
  lines.each { |line|
    convert_line_to_event(line, es, tail_watcher)
  }
  es
end

#receive_lines(lines, tail_watcher) ⇒ Object

Returns true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError.

Returns:

  • true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError



339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
 
# File 'lib/fluent/plugin/in_tail.rb', line 339

def receive_lines(lines, tail_watcher)
  es = @receive_handler.call(lines, tail_watcher)
  unless es.empty?
    tag = if @tag_prefix || @tag_suffix
            @tag_prefix + tail_watcher.tag + @tag_suffix
          else
            @tag
          end
    begin
      router.emit_stream(tag, es)
    rescue Fluent::Plugin::Buffer::BufferOverflowError
      return false
    rescue
      # ignore non BufferQueueLimitError errors because in_tail can't recover. Engine shows logs and backtraces.
      return true
    end
  end

  return true
end

#refresh_watchers ⇒ Object

in_tail with ‘*’ path doesn’t check rotation file equality at refresh phase. So you should not use ‘*’ path when your logs will be rotated by another tool. It will cause log duplication after updated watch files. In such case, you should separate log directory and specify two paths in path parameter. e.g. path /path/to/dir/*,/path/to/rotated_logs/target_file



225
226
227
228
229
230
231
232
233
234
 
# File 'lib/fluent/plugin/in_tail.rb', line 225

def refresh_watchers
  target_paths = expand_paths
  existence_paths = @tails.keys

  unwatched = existence_paths - target_paths
  added = target_paths - existence_paths

  stop_watchers(unwatched, immediate: false, unwatched: true) unless unwatched.empty?
  start_watchers(added) unless added.empty?
end

#setup_watcher(path, pe) ⇒ Object 



236
237
238
239
240
241
242
243
244
 
# File 'lib/fluent/plugin/in_tail.rb', line 236

def setup_watcher(path, pe)
  line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
  tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
  tw.attach do |watcher|
    watcher.timer_trigger = timer_execute(:in_tail_timer_trigger, 1, &watcher.method(:on_notify)) if watcher.enable_watch_timer
    event_loop_attach(watcher.stat_trigger)
  end
  tw
end

#shutdown ⇒ Object 



172
173
174
175
176
177
178
 
# File 'lib/fluent/plugin/in_tail.rb', line 172

def shutdown
  # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
  stop_watchers(@tails.keys, immediate: true, remove_watcher: false)
  @pf_file.close if @pf_file

  super
end

#start ⇒ Object 



159
160
161
162
163
164
165
166
167
168
169
170
 
# File 'lib/fluent/plugin/in_tail.rb', line 159

def start
  super

  if @pos_file
    @pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY, @file_perm)
    @pf_file.sync = true
    @pf = PositionFile.parse(@pf_file)
  end

  refresh_watchers unless @skip_refresh_on_startup
  timer_execute(:in_tail_refresh_watchers, @refresh_interval, &method(:refresh_watchers))
end

#start_watchers(paths) ⇒ Object 



246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
 
# File 'lib/fluent/plugin/in_tail.rb', line 246

def start_watchers(paths)
  paths.each { |path|
    pe = nil
    if @pf
      pe = @pf[path]
      if @read_from_head && pe.read_inode.zero?
        begin
          pe.update(Fluent::FileWrapper.stat(path).ino, 0)
        rescue Errno::ENOENT
          $log.warn "#{path} not found. Continuing without tailing it."
        end
      end
    end

    @tails[path] = setup_watcher(path, pe)
  }
end

#stop_watchers(paths, immediate: false, unwatched: false, remove_watcher: true) ⇒ Object 



264
265
266
267
268
269
270
271
272
273
274
275
276
 
# File 'lib/fluent/plugin/in_tail.rb', line 264

def stop_watchers(paths, immediate: false, unwatched: false, remove_watcher: true)
  paths.each { |path|
    tw = remove_watcher ? @tails.delete(path) : @tails[path]
    if tw
      tw.unwatched = unwatched
      if immediate
        detach_watcher(tw, false)
      else
        detach_watcher_after_rotate_wait(tw)
      end
    end
  }
end

#update_watcher(path, pe) ⇒ Object

refresh_watchers calls @tails.keys so we don’t use stop_watcher -> start_watcher sequence for safety.



288
289
290
291
292
293
294
295
296
297
298
 
# File 'lib/fluent/plugin/in_tail.rb', line 288

def update_watcher(path, pe)
  if @pf
    unless pe.read_inode == @pf[path].read_inode
      log.trace "Skip update_watcher because watcher has been already updated by other inotify event"
      return
    end
  end
  rotated_tw = @tails[path]
  @tails[path] = setup_watcher(path, pe)
  detach_watcher_after_rotate_wait(rotated_tw) if rotated_tw
end