Class: Fluent::NewTailInput

Inherits:
Input
  • Object
show all
Defined in:
lib/fluent/plugin/in_tail.rb

Defined Under Namespace

Classes: FilePositionEntry, MemoryPositionEntry, PositionFile, TailWatcher

Constant Summary

Constants included from Configurable

Configurable::CONFIG_TYPE_REGISTRY

Instance Attribute Summary collapse

Attributes inherited from Input

#router

Attributes included from PluginLoggerMixin

#log

Instance Method Summary collapse

Methods included from PluginLoggerMixin

included

Methods included from PluginId

#plugin_id

Methods included from Configurable

#config, included, lookup_type, register_type

Constructor Details

#initialize ⇒ NewTailInput

Returns a new instance of NewTailInput.



27
28
29
30
31
 
# File 'lib/fluent/plugin/in_tail.rb', line 27

def initialize
  super
  @paths = []
  @tails = {}
end

Instance Attribute Details

#paths ⇒ Object (readonly)

Returns the value of attribute paths.



60
61
62
 
# File 'lib/fluent/plugin/in_tail.rb', line 60

def paths
  @paths
end

Instance Method Details

#close_watcher(tw, close_io = true) ⇒ Object

TailWatcher#close is called by another thread at shutdown phase. It causes ‘can’t modify string; temporarily locked’ error in IOHandler so adding close_io argument to avoid this problem. At shutdown, IOHandler’s io will be released automatically after detached the event loop



235
236
237
238
239
240
241
 
# File 'lib/fluent/plugin/in_tail.rb', line 235

def close_watcher(tw, close_io = true)
  tw.close(close_io)
  flush_buffer(tw)
  if tw.unwatched && @pf
    @pf[tw.path].update_pos(PositionFile::UNWATCHED_POSITION)
  end
end

#close_watcher_after_rotate_wait(tw) ⇒ Object 



243
244
245
246
 
# File 'lib/fluent/plugin/in_tail.rb', line 243

def close_watcher_after_rotate_wait(tw)
  closer = TailWatcher::Closer.new(@rotate_wait, tw, log, &method(:close_watcher))
  closer.attach(@loop)
end

#configure(conf) ⇒ Object 



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/fluent/plugin/in_tail.rb', line 62

def configure(conf)
  super

  @paths = @path.split(',').map {|path| path.strip }
  if @paths.empty?
    raise ConfigError, "tail: 'path' parameter is required on tail input"
  end

  unless @pos_file
    $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
    $log.warn "this parameter is highly recommended to save the position to resume tailing."
  end

  configure_parser(conf)
  configure_tag
  configure_encoding

  @multiline_mode = conf['format'] =~ /multiline/
  @receive_handler = if @multiline_mode
                       method(:parse_multilines)
                     else
                       method(:parse_singleline)
                     end
end

#configure_encoding ⇒ Object 



102
103
104
105
106
107
108
109
110
111
 
# File 'lib/fluent/plugin/in_tail.rb', line 102

def configure_encoding
  unless @encoding
    if @from_encoding
      raise ConfigError, "tail: 'from_encoding' parameter must be specified with 'encoding' parameter."
    end
  end

  @encoding = parse_encoding_param(@encoding) if @encoding
  @from_encoding = parse_encoding_param(@from_encoding) if @from_encoding
end

#configure_parser(conf) ⇒ Object 



87
88
89
90
 
# File 'lib/fluent/plugin/in_tail.rb', line 87

def configure_parser(conf)
  @parser = Plugin.new_parser(conf['format'])
  @parser.configure(conf)
end

#configure_tag ⇒ Object 



92
93
94
95
96
97
98
99
100
 
# File 'lib/fluent/plugin/in_tail.rb', line 92

def configure_tag
  if @tag.index('*')
    @tag_prefix, @tag_suffix = @tag.split('*')
    @tag_suffix ||= ''
  else
    @tag_prefix = nil
    @tag_suffix = nil
  end
end

#convert_line_to_event(line, es, tail_watcher) ⇒ Object 



303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
 
# File 'lib/fluent/plugin/in_tail.rb', line 303

def convert_line_to_event(line, es, tail_watcher)
  begin
    line.chomp!  # remove \n
    if @encoding
      if @from_encoding
        line.encode!(@encoding, @from_encoding)
      else
        line.force_encoding(@encoding)
      end
    end
    @parser.parse(line) { |time, record|
      if time && record
        record[@path_key] ||= tail_watcher.path unless @path_key.nil?
        es.add(time, record)
      else
        log.warn "pattern not match: #{line.inspect}"
      end
    }
  rescue => e
    log.warn line.dump, error: e.to_s
    log.debug_backtrace(e.backtrace)
  end
end

#expand_paths ⇒ Object 



145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
 
# File 'lib/fluent/plugin/in_tail.rb', line 145

def expand_paths
  date = Time.now
  paths = []

  excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
  @paths.each { |path|
    path = date.strftime(path)
    if path.include?('*')
      paths += Dir.glob(path).select { |p|
        if File.readable?(p)
          true
        else
          log.warn "#{p} unreadable. It is excluded and would be examined next time."
          false
        end
      }
    else
      # When file is not created yet, Dir.glob returns an empty array. So just add when path is static.
      paths << path
    end
  }
  paths - excluded
end

#flush_buffer(tw) ⇒ Object 



248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
 
# File 'lib/fluent/plugin/in_tail.rb', line 248

def flush_buffer(tw)
  if lb = tw.line_buffer
    lb.chomp!
    if @encoding
      if @from_encoding
        lb.encode!(@encoding, @from_encoding)
      else
        lb.force_encoding(@encoding)
      end
    end
    @parser.parse(lb) { |time, record|
      if time && record
        tag = if @tag_prefix || @tag_suffix
                @tag_prefix + tw.tag + @tag_suffix
              else
                @tag
              end
        record[@path_key] ||= tw.path unless @path_key.nil?
        router.emit(tag, time, record)
      else
        log.warn "got incomplete line at shutdown from #{tw.path}: #{lb.inspect}"
      end
    }
  end
end

#parse_encoding_param(encoding_name) ⇒ Object 



113
114
115
116
117
118
119
 
# File 'lib/fluent/plugin/in_tail.rb', line 113

def parse_encoding_param(encoding_name)
  begin
    Encoding.find(encoding_name) if encoding_name
  rescue ArgumentError => e
    raise ConfigError, e.message
  end
end

#parse_multilines(lines, tail_watcher) ⇒ Object 



335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
 
# File 'lib/fluent/plugin/in_tail.rb', line 335

def parse_multilines(lines, tail_watcher)
  lb = tail_watcher.line_buffer
  es = MultiEventStream.new
  if @parser.has_firstline?
    tail_watcher.line_buffer_timer_flusher.reset_timer if tail_watcher.line_buffer_timer_flusher
    lines.each { |line|
      if @parser.firstline?(line)
        if lb
          convert_line_to_event(lb, es, tail_watcher)
        end
        lb = line
      else
        if lb.nil?
          log.warn "got incomplete line before first line from #{tail_watcher.path}: #{line.inspect}"
        else
          lb << line
        end
      end
    }
  else
    lb ||= ''
    lines.each do |line|
      lb << line
      @parser.parse(lb) { |time, record|
        if time && record
          convert_line_to_event(lb, es, tail_watcher)
          lb = ''
        end
      }
    end
  end
  tail_watcher.line_buffer = lb
  es
end

#parse_singleline(lines, tail_watcher) ⇒ Object 



327
328
329
330
331
332
333
 
# File 'lib/fluent/plugin/in_tail.rb', line 327

def parse_singleline(lines, tail_watcher)
  es = MultiEventStream.new
  lines.each { |line|
    convert_line_to_event(line, es, tail_watcher)
  }
  es
end

#receive_lines(lines, tail_watcher) ⇒ Object

Returns true if no error or unrecoverable error happens in emit action. false if got BufferQueueLimitError.

Returns:

  • true if no error or unrecoverable error happens in emit action. false if got BufferQueueLimitError



282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
 
# File 'lib/fluent/plugin/in_tail.rb', line 282

def receive_lines(lines, tail_watcher)
  es = @receive_handler.call(lines, tail_watcher)
  unless es.empty?
    tag = if @tag_prefix || @tag_suffix
            @tag_prefix + tail_watcher.tag + @tag_suffix
          else
            @tag
          end
    begin
      router.emit_stream(tag, es)
    rescue BufferQueueLimitError
      return false
    rescue
      # ignore non BufferQueueLimitError errors because in_tail can't recover. Engine shows logs and backtraces.
      return true
    end
  end

  return true
end

#refresh_watchers ⇒ Object

in_tail with ‘*’ path doesn’t check rotation file equality at refresh phase. So you should not use ‘*’ path when your logs will be rotated by another tool. It will cause log duplication after updated watch files. In such case, you should separate log directory and specify two paths in path parameter. e.g. path /path/to/dir/*,/path/to/rotated_logs/target_file



174
175
176
177
178
179
180
181
182
183
 
# File 'lib/fluent/plugin/in_tail.rb', line 174

def refresh_watchers
  target_paths = expand_paths
  existence_paths = @tails.keys

  unwatched = existence_paths - target_paths
  added = target_paths - existence_paths

  stop_watchers(unwatched, false, true) unless unwatched.empty?
  start_watchers(added) unless added.empty?
end

#run ⇒ Object 



274
275
276
277
278
279
 
# File 'lib/fluent/plugin/in_tail.rb', line 274

def run
  @loop.run
rescue
  log.error "unexpected error", error: $!.to_s
  log.error_backtrace
end

#setup_watcher(path, pe) ⇒ Object 



185
186
187
188
189
190
 
# File 'lib/fluent/plugin/in_tail.rb', line 185

def setup_watcher(path, pe)
  line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
  tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher,  &method(:receive_lines))
  tw.attach(@loop)
  tw
end

#shutdown ⇒ Object 



136
137
138
139
140
141
142
143
 
# File 'lib/fluent/plugin/in_tail.rb', line 136

def shutdown
  @refresh_trigger.detach if @refresh_trigger && @refresh_trigger.attached?

  stop_watchers(@tails.keys, true)
  @loop.stop rescue nil # when all watchers are detached, `stop` raises RuntimeError. We can ignore this exception.
  @thread.join
  @pf_file.close if @pf_file
end

#start ⇒ Object 



121
122
123
124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/fluent/plugin/in_tail.rb', line 121

def start
  if @pos_file
    @pf_file = File.open(@pos_file, File::RDWR|File::CREAT, DEFAULT_FILE_PERMISSION)
    @pf_file.sync = true
    @pf = PositionFile.parse(@pf_file)
  end

  @loop = Coolio::Loop.new
  refresh_watchers

  @refresh_trigger = TailWatcher::TimerWatcher.new(@refresh_interval, true, log, &method(:refresh_watchers))
  @refresh_trigger.attach(@loop)
  @thread = Thread.new(&method(:run))
end

#start_watchers(paths) ⇒ Object 



192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
 
# File 'lib/fluent/plugin/in_tail.rb', line 192

def start_watchers(paths)
  paths.each { |path|
    pe = nil
    if @pf
      pe = @pf[path]
      if @read_from_head && pe.read_inode.zero?
        begin
          pe.update(File::Stat.new(path).ino, 0)
        rescue Errno::ENOENT
          $log.warn "#{path} not found. Continuing without tailing it."
        end
      end
    end

    @tails[path] = setup_watcher(path, pe)
  }
end

#stop_watchers(paths, immediate = false, unwatched = false) ⇒ Object 



210
211
212
213
214
215
216
217
218
219
220
221
222
 
# File 'lib/fluent/plugin/in_tail.rb', line 210

def stop_watchers(paths, immediate = false, unwatched = false)
  paths.each { |path|
    tw = @tails.delete(path)
    if tw
      tw.unwatched = unwatched
      if immediate
        close_watcher(tw, false)
      else
        close_watcher_after_rotate_wait(tw)
      end
    end
  }
end

#update_watcher(path, pe) ⇒ Object

refresh_watchers calls @tails.keys so we don’t use stop_watcher -> start_watcher sequence for safety.



225
226
227
228
229
 
# File 'lib/fluent/plugin/in_tail.rb', line 225

def update_watcher(path, pe)
  rotated_tw = @tails[path]
  @tails[path] = setup_watcher(path, pe)
  close_watcher_after_rotate_wait(rotated_tw) if rotated_tw
end