Class: Fluent::KafkaOutput
- Inherits:
-
Output
- Object
- Output
- Fluent::KafkaOutput
- Defined in:
- lib/fluent/plugin/out_kafka.rb
Instance Attribute Summary collapse
-
#field_separator ⇒ Object
Returns the value of attribute field_separator.
-
#output_data_type ⇒ Object
Returns the value of attribute output_data_type.
Instance Method Summary collapse
- #configure(conf) ⇒ Object
- #emit(tag, es, chain) ⇒ Object
-
#initialize ⇒ KafkaOutput
constructor
A new instance of KafkaOutput.
- #parse_record(record) ⇒ Object
- #refresh_producer ⇒ Object
- #shutdown ⇒ Object
- #start ⇒ Object
Constructor Details
#initialize ⇒ KafkaOutput
Returns a new instance of KafkaOutput.
4 5 6 7 |
# File 'lib/fluent/plugin/out_kafka.rb', line 4 def initialize super require 'poseidon' end |
Instance Attribute Details
#field_separator ⇒ Object
Returns the value of attribute field_separator.
39 40 41 |
# File 'lib/fluent/plugin/out_kafka.rb', line 39 def field_separator @field_separator end |
#output_data_type ⇒ Object
Returns the value of attribute output_data_type.
38 39 40 |
# File 'lib/fluent/plugin/out_kafka.rb', line 38 def output_data_type @output_data_type end |
Instance Method Details
#configure(conf) ⇒ Object
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 |
# File 'lib/fluent/plugin/out_kafka.rb', line 70 def configure(conf) super if @zookeeper require 'zookeeper' require 'yajl' else @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",") log.info "brokers has been set directly: #{@seed_brokers}" end if @compression_codec == 'snappy' require 'snappy' end case @output_data_type when 'json' require 'yajl' when 'ltsv' require 'ltsv' when 'msgpack' require 'msgpack' end @f_separator = case @field_separator when /SPACE/i then ' ' when /COMMA/i then ',' when /SOH/i then "\x01" else "\t" end @custom_attributes = if @output_data_type == 'json' nil elsif @output_data_type == 'ltsv' nil elsif @output_data_type == 'msgpack' nil elsif @output_data_type =~ /^attr:(.*)$/ $1.split(',').map(&:strip).reject(&:empty?) else @formatter = Fluent::Plugin.new_formatter(@output_data_type) @formatter.configure(conf) nil end end |
#emit(tag, es, chain) ⇒ Object
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 |
# File 'lib/fluent/plugin/out_kafka.rb', line 144 def emit(tag, es, chain) begin chain.next es.each do |time,record| record['time'] = time if @output_include_time record['tag'] = tag if @output_include_tag topic = record['topic'] || self.default_topic || tag partition_key = record['partition_key'] || @default_partition_key value = @formatter.nil? ? parse_record(record) : @formatter.format(tag, time, record) log.trace("message send to #{topic} with key: #{partition_key} and value: #{value}.") = Poseidon::MessageToSend.new(topic, value, partition_key) @producer.([]) end rescue Exception => e log.warn("Send exception occurred: #{e}") @producer.close if @producer refresh_producer() raise e end end |
#parse_record(record) ⇒ Object
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 |
# File 'lib/fluent/plugin/out_kafka.rb', line 123 def parse_record(record) if @custom_attributes.nil? case @output_data_type when 'json' Yajl::Encoder.encode(record) when 'ltsv' LTSV.dump(record) when 'msgpack' record.to_msgpack else record.to_s end else @custom_attributes.unshift('time') if @output_include_time @custom_attributes.unshift('tag') if @output_include_tag @custom_attributes.map { |attr| record[attr].nil? ? '' : record[attr].to_s }.join(@f_separator) end end |
#refresh_producer ⇒ Object
47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
# File 'lib/fluent/plugin/out_kafka.rb', line 47 def refresh_producer() if @zookeeper @seed_brokers = [] z = Zookeeper.new(@zookeeper) z.get_children(:path => @zookeeper_path)[:children].each do |id| broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data]) @seed_brokers.push("#{broker['host']}:#{broker['port']}") end z.close log.info "brokers has been refreshed via Zookeeper: #{@seed_brokers}" end begin if @seed_brokers.length > 0 @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms, :compression_codec => @compression_codec.to_sym) log.info "initialized producer #{@client_id}" else log.warn "No brokers found on Zookeeper" end rescue Exception => e log.error e end end |
#shutdown ⇒ Object
119 120 121 |
# File 'lib/fluent/plugin/out_kafka.rb', line 119 def shutdown super end |
#start ⇒ Object
114 115 116 117 |
# File 'lib/fluent/plugin/out_kafka.rb', line 114 def start super refresh_producer() end |