Class: Fluent::ElasticsearchOutput

Inherits:
BufferedOutput
  • Object
show all
Includes:
SetTagKeyMixin
Defined in:
lib/fluent/plugin/out_elasticsearch.rb

Instance Method Summary collapse

Constructor Details

#initialize ⇒ ElasticsearchOutput

Returns a new instance of ElasticsearchOutput.



24
25
26
 
# File 'lib/fluent/plugin/out_elasticsearch.rb', line 24

def initialize
  super
end

Instance Method Details

#client ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/fluent/plugin/out_elasticsearch.rb', line 36

def client
  @_es ||= begin
    adapter_conf = lambda {|f| f.adapter :patron }
    transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new({ hosts: get_hosts,
                                                                         options: {
                                                                           reload_connections: true,
                                                                           retry_on_failure: 5
                                                                        }}, &adapter_conf)
    Elasticsearch::Client.new transport: transport
  end
  raise "Can not reach Elasticsearch cluster (#{@host}:#{@port})!" unless @_es.ping
  @_es
end

#configure(conf) ⇒ Object 



28
29
30
 
# File 'lib/fluent/plugin/out_elasticsearch.rb', line 28

def configure(conf)
  super
end

#format(tag, time, record) ⇒ Object 



58
59
60
 
# File 'lib/fluent/plugin/out_elasticsearch.rb', line 58

def format(tag, time, record)
  [tag, time, record].to_msgpack
end

#get_hosts ⇒ Object 



50
51
52
53
54
55
56
 
# File 'lib/fluent/plugin/out_elasticsearch.rb', line 50

def get_hosts
  if @hosts
      @hosts.split(',').map {|x| hp = x.split(':'); { host: hp[0], port: hp[1] || @port } }.compact
   else
     [{host: @host, port: @port }]
   end
end

#send(data) ⇒ Object 



102
103
104
 
# File 'lib/fluent/plugin/out_elasticsearch.rb', line 102

def send(data)
  client.bulk body: data
end

#shutdown ⇒ Object 



62
63
64
 
# File 'lib/fluent/plugin/out_elasticsearch.rb', line 62

def shutdown
  super
end

#start ⇒ Object 



32
33
34
 
# File 'lib/fluent/plugin/out_elasticsearch.rb', line 32

def start
  super
end

#write(chunk) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/fluent/plugin/out_elasticsearch.rb', line 66

def write(chunk)
  bulk_message = []

  chunk.msgpack_each do |tag, time, record|
    if @logstash_format
      record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s}) unless record.has_key?("@timestamp")
      if @utc_index
        target_index = "#{@logstash_prefix}-#{Time.at(time).getutc.strftime("#{@logstash_dateformat}")}"
      else
        target_index = "#{@logstash_prefix}-#{Time.at(time).strftime("#{@logstash_dateformat}")}"
      end
    else
      target_index = @index_name
    end

    if @include_tag_key
      record.merge!(@tag_key => tag)
    end

    meta = { "index" => {"_index" => target_index, "_type" => type_name} }
    if @id_key && record[@id_key]
      meta['index']['_id'] = record[@id_key]
    end

    if @parent_key && record[@parent_key]
      meta['index']['_parent'] = record[@parent_key]
    end

    bulk_message << meta
    bulk_message << record
  end

  send(bulk_message) unless bulk_message.empty?
  bulk_message.clear
end