Class: Kafka::Cluster
- Inherits:
-
Object
- Object
- Kafka::Cluster
- Defined in:
- lib/kafka/cluster.rb
Overview
A cluster represents the state of a Kafka cluster. It needs to be initialized with a non-empty list of seed brokers. The first seed broker that the cluster can connect to will be asked for the cluster metadata, which allows the cluster to map topic partitions to the current leader for those partitions.
Instance Method Summary collapse
-
#add_target_topics(topics) ⇒ nil
Adds a list of topics to the target list.
- #api_info(api_key) ⇒ Object
- #apis ⇒ Object
-
#clear_target_topics ⇒ nil
Clears the list of target topics.
- #create_topic(name, num_partitions: 1, replication_factor: 1, timeout: 30) ⇒ Object
- #disconnect ⇒ Object
- #get_group_coordinator(group_id:) ⇒ Object
-
#get_leader(topic, partition) ⇒ Broker
Finds the broker acting as the leader of the given topic and partition.
-
#initialize(seed_brokers:, broker_pool:, logger:) ⇒ Cluster
constructor
Initializes a Cluster with a set of seed brokers.
- #mark_as_stale! ⇒ Object
- #partitions_for(topic) ⇒ Object
- #refresh_metadata! ⇒ Object
- #refresh_metadata_if_necessary! ⇒ Object
- #resolve_offset(topic, partition, offset) ⇒ Object
- #resolve_offsets(topic, partitions, offset) ⇒ Object
- #topics ⇒ Object
Constructor Details
#initialize(seed_brokers:, broker_pool:, logger:) ⇒ Cluster
Initializes a Cluster with a set of seed brokers.
The cluster will try to fetch cluster metadata from one of the brokers.
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 |
# File 'lib/kafka/cluster.rb', line 19 def initialize(seed_brokers:, broker_pool:, logger:) if seed_brokers.empty? raise ArgumentError, "At least one seed broker must be configured" end @logger = logger @seed_brokers = seed_brokers @broker_pool = broker_pool @cluster_info = nil @stale = true # This is the set of topics we need metadata for. If empty, metadata for # all topics will be fetched. @target_topics = Set.new end |
Instance Method Details
#add_target_topics(topics) ⇒ nil
Adds a list of topics to the target list. Only the topics on this list will be queried for metadata.
40 41 42 43 44 45 46 47 48 49 50 |
# File 'lib/kafka/cluster.rb', line 40 def add_target_topics(topics) new_topics = Set.new(topics) - @target_topics unless new_topics.empty? @logger.info "New topics added to target list: #{new_topics.to_a.join(', ')}" @target_topics.merge(new_topics) end end |
#api_info(api_key) ⇒ Object
52 53 54 |
# File 'lib/kafka/cluster.rb', line 52 def api_info(api_key) apis.find {|api| api.api_key == api_key } end |
#apis ⇒ Object
56 57 58 59 60 61 62 63 64 65 |
# File 'lib/kafka/cluster.rb', line 56 def apis @apis ||= begin response = random_broker.api_versions Protocol.handle_error(response.error_code) response.apis end end |
#clear_target_topics ⇒ nil
Clears the list of target topics.
71 72 73 74 |
# File 'lib/kafka/cluster.rb', line 71 def clear_target_topics @target_topics.clear end |
#create_topic(name, num_partitions: 1, replication_factor: 1, timeout: 30) ⇒ Object
148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 |
# File 'lib/kafka/cluster.rb', line 148 def create_topic(name, num_partitions: 1, replication_factor: 1, timeout: 30) = { topics: { name => { num_partitions: num_partitions, replication_factor: replication_factor, } }, timeout: timeout, } broker = controller_broker @logger.info "Creating topic `#{name}` using controller broker #{broker}" response = broker.create_topics(**) response.errors.each do |topic, error_code| Protocol.handle_error(error_code) end begin partitions_for(name).each do |info| Protocol.handle_error(info.partition_error_code) end rescue Kafka::LeaderNotAvailable @logger.warn "Leader not yet available for `#{name}`, waiting 1s..." sleep 1 retry end @logger.info "Topic `#{name}` was created" end |
#disconnect ⇒ Object
235 236 237 |
# File 'lib/kafka/cluster.rb', line 235 def disconnect @broker_pool.close end |
#get_group_coordinator(group_id:) ⇒ Object
98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 |
# File 'lib/kafka/cluster.rb', line 98 def get_group_coordinator(group_id:) @logger.debug "Getting group coordinator for `#{group_id}`" cluster_info.brokers.each do |broker_info| begin broker = connect_to_broker(broker_info.node_id) response = broker.find_group_coordinator(group_id: group_id) Protocol.handle_error(response.error_code) coordinator_id = response.coordinator_id @logger.debug "Coordinator for group `#{group_id}` is #{coordinator_id}. Connecting..." # It's possible that a new broker is introduced to the cluster and # becomes the coordinator before we have a chance to refresh_metadata. coordinator = begin connect_to_broker(coordinator_id) rescue Kafka::NoSuchBroker @logger.debug "Broker #{coordinator_id} missing from broker cache, refreshing" connect_to_broker(coordinator_id) end @logger.debug "Connected to coordinator: #{coordinator} for group `#{group_id}`" return coordinator rescue GroupCoordinatorNotAvailable @logger.debug "Coordinator not available; retrying in 1s" sleep 1 retry rescue ConnectionError => e @logger.error "Failed to get group coordinator info from #{broker}: #{e}" end end raise Kafka::Error, "Failed to find group coordinator" end |
#get_leader(topic, partition) ⇒ Broker
Finds the broker acting as the leader of the given topic and partition.
94 95 96 |
# File 'lib/kafka/cluster.rb', line 94 def get_leader(topic, partition) connect_to_broker(get_leader_id(topic, partition)) end |
#mark_as_stale! ⇒ Object
76 77 78 |
# File 'lib/kafka/cluster.rb', line 76 def mark_as_stale! @stale = true end |
#partitions_for(topic) ⇒ Object
139 140 141 142 143 144 145 146 |
# File 'lib/kafka/cluster.rb', line 139 def partitions_for(topic) add_target_topics([topic]) cluster_info.partitions_for(topic) rescue Kafka::ProtocolError mark_as_stale! raise end |
#refresh_metadata! ⇒ Object
80 81 82 83 |
# File 'lib/kafka/cluster.rb', line 80 def @cluster_info = nil cluster_info end |
#refresh_metadata_if_necessary! ⇒ Object
85 86 87 |
# File 'lib/kafka/cluster.rb', line 85 def if @stale end |
#resolve_offset(topic, partition, offset) ⇒ Object
226 227 228 |
# File 'lib/kafka/cluster.rb', line 226 def resolve_offset(topic, partition, offset) resolve_offsets(topic, [partition], offset).fetch(partition) end |
#resolve_offsets(topic, partitions, offset) ⇒ Object
183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 |
# File 'lib/kafka/cluster.rb', line 183 def resolve_offsets(topic, partitions, offset) add_target_topics([topic]) partitions_by_broker = partitions.each_with_object({}) {|partition, hsh| broker = get_leader(topic, partition) hsh[broker] ||= [] hsh[broker] << partition } if offset == :earliest offset = -2 elsif offset == :latest offset = -1 end offsets = {} partitions_by_broker.each do |broker, broker_partitions| response = broker.list_offsets( topics: { topic => broker_partitions.map {|partition| { partition: partition, time: offset, max_offsets: 1, } } } ) broker_partitions.each do |partition| offsets[partition] = response.offset_for(topic, partition) end end offsets rescue Kafka::ProtocolError mark_as_stale! raise end |
#topics ⇒ Object
230 231 232 233 |
# File 'lib/kafka/cluster.rb', line 230 def topics cluster_info.topics.map(&:topic_name) end |