fluent-plugin-unit-time-filter

Plug-in to aggregate by unit time

Installation

$ gem install fluent-plugin-unit-time-filter

Configuration

<match my.**>
  type unit_time_filter
  filter_path /foo/bar/my_filrer.rb
  unit_sec 10
  #prefix filtered
  #emit_each_tag false
  #pass_hash_row false
  #hash_row_time_key time
  #hash_row_tag_key tag
</match>

<match filtered.my.**>
  type stdout
</match>

Filter

# Count the number of records of every unit time

proc {|records|

  # `records` is an Array such as:
  #   [[tag, time, record], [tag, time, record], ...]
  #
  # e.g.)
  #   [["my.data", 1391820170, {"hoge"=>"fuga"}],
  #    ["my.data", 1391820170, {"hoge"=>"fuga"}],
  #    ["my.data", 1391820170, {"hoge"=>"fuga"}],
  #    ...

  {'count' => records.count}
  # or [{...},{...},{...}, ...]
}

Usage

Run the following command:

while true; do
  echo '{"hoge":"fuga"}' | fluent-cat my.data
done

fluentd outputs the following:

2014-02-08 00:38:40 +0900 filtered.my.data: {"count":42}
2014-02-08 00:38:50 +0900 filtered.my.data: {"count":43}
2014-02-08 00:39:00 +0900 filtered.my.data: {"count":41}
...