Class: Fluent::KafkaInput
- Inherits:
-
Input
- Object
- Input
- Fluent::KafkaInput
- Defined in:
- lib/fluent/plugin/in_kafka.rb
Defined Under Namespace
Classes: OffsetManager, TopicEntry, TopicWatcher
Instance Method Summary collapse
- #configure(conf) ⇒ Object
-
#initialize ⇒ KafkaInput
constructor
A new instance of KafkaInput.
- #run ⇒ Object
- #shutdown ⇒ Object
- #start ⇒ Object
Constructor Details
#initialize ⇒ KafkaInput
Returns a new instance of KafkaInput.
46 47 48 49 50 |
# File 'lib/fluent/plugin/in_kafka.rb', line 46 def initialize super require 'poseidon' require 'zookeeper' end |
Instance Method Details
#configure(conf) ⇒ Object
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
# File 'lib/fluent/plugin/in_kafka.rb', line 52 def configure(conf) super @topic_list = [] if @topics @topic_list = @topics.split(',').map { |topic| TopicEntry.new(topic.strip, @partition, @offset) } else conf.elements.select { |element| element.name == 'topic' }.each do |element| unless element.has_key?('topic') raise ConfigError, "kafka: 'topic' is a require parameter in 'topic element'." end partition = element.has_key?('partition') ? element['partition'].to_i : 0 offset = element.has_key?('offset') ? element['offset'].to_i : -1 @topic_list.push(TopicEntry.new(element['topic'], partition, offset)) end end if @topic_list.empty? raise ConfigError, "kafka: 'topics' or 'topic element' is a require parameter" end case @format when 'json' require 'yajl' when 'ltsv' require 'ltsv' when 'msgpack' require 'msgpack' end end |
#run ⇒ Object
123 124 125 126 127 128 |
# File 'lib/fluent/plugin/in_kafka.rb', line 123 def run @loop.run rescue $log.error "unexpected error", :error=>$!.to_s $log.error_backtrace end |
#shutdown ⇒ Object
118 119 120 121 |
# File 'lib/fluent/plugin/in_kafka.rb', line 118 def shutdown @loop.stop @zookeeper.close! if @zookeeper end |
#start ⇒ Object
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 |
# File 'lib/fluent/plugin/in_kafka.rb', line 85 def start @loop = Coolio::Loop.new opt = {} opt[:max_bytes] = @max_bytes if @max_bytes opt[:max_wait_ms] = @max_wait_ms if @max_wait_ms opt[:min_bytes] = @min_bytes if @min_bytes opt[:socket_timeout_ms] = @socket_timeout_ms if @socket_timeout_ms @zookeeper = Zookeeper.new(@offset_zookeeper) if @offset_zookeeper @topic_watchers = @topic_list.map {|topic_entry| offset_manager = OffsetManager.new(topic_entry, @zookeeper, @offset_zk_root_node) if @offset_zookeeper TopicWatcher.new( topic_entry, @host, @port, @client_id, interval, @format, @message_key, @add_offset_in_record, @add_prefix, @add_suffix, offset_manager, router, opt) } @topic_watchers.each {|tw| tw.attach(@loop) } @thread = Thread.new(&method(:run)) end |