Module: Karafka

Defined in:
lib/karafka.rb,
lib/karafka/app.rb,
lib/karafka/cli.rb,
lib/karafka/env.rb,
lib/karafka/pro.rb,
lib/karafka/admin.rb,
lib/karafka/errors.rb,
lib/karafka/runner.rb,
lib/karafka/server.rb,
lib/karafka/status.rb,
lib/karafka/process.rb,
lib/karafka/railtie.rb,
lib/karafka/railtie.rb,
lib/karafka/version.rb,
lib/karafka/cli/base.rb,
lib/karafka/cli/info.rb,
lib/karafka/embedded.rb,
lib/karafka/licenser.rb,
lib/karafka/contracts.rb,
lib/karafka/setup/dsl.rb,
lib/karafka/cli/server.rb,
lib/karafka/cli/topics.rb,
lib/karafka/pro/loader.rb,
lib/karafka/cli/console.rb,
lib/karafka/cli/install.rb,
lib/karafka/pro/cleaner.rb,
lib/karafka/pro/iterator.rb,
lib/karafka/setup/config.rb,
lib/karafka/base_consumer.rb,
lib/karafka/helpers/async.rb,
lib/karafka/messages/seek.rb,
lib/karafka/routing/proxy.rb,
lib/karafka/routing/topic.rb,
lib/karafka/contracts/base.rb,
lib/karafka/pro/encryption.rb,
lib/karafka/routing/router.rb,
lib/karafka/routing/topics.rb,
lib/karafka/contracts/topic.rb,
lib/karafka/messages/parser.rb,
lib/karafka/routing/builder.rb,
lib/karafka/connection/proxy.rb,
lib/karafka/contracts/config.rb,
lib/karafka/helpers/colorize.rb,
lib/karafka/messages/message.rb,
lib/karafka/connection/client.rb,
lib/karafka/messages/messages.rb,
lib/karafka/messages/metadata.rb,
lib/karafka/processing/result.rb,
lib/karafka/processing/worker.rb,
lib/karafka/pro/cleaner/errors.rb,
lib/karafka/pro/contracts/base.rb,
lib/karafka/time_trackers/base.rb,
lib/karafka/time_trackers/poll.rb,
lib/karafka/active_job/consumer.rb,
lib/karafka/connection/listener.rb,
lib/karafka/processing/executor.rb,
lib/karafka/time_trackers/pause.rb,
lib/karafka/processing/jobs/base.rb,
lib/karafka/processing/jobs/idle.rb,
lib/karafka/processing/scheduler.rb,
lib/karafka/setup/attributes_map.rb,
lib/karafka/active_job/dispatcher.rb,
lib/karafka/pro/encryption/cipher.rb,
lib/karafka/pro/encryption/errors.rb,
lib/karafka/pro/iterator/expander.rb,
lib/karafka/processing/jobs_queue.rb,
lib/karafka/routing/features/base.rb,
lib/karafka/instrumentation/logger.rb,
lib/karafka/processing/coordinator.rb,
lib/karafka/processing/partitioner.rb,
lib/karafka/routing/consumer_group.rb,
lib/karafka/helpers/multi_delegator.rb,
lib/karafka/instrumentation/monitor.rb,
lib/karafka/messages/batch_metadata.rb,
lib/karafka/pro/active_job/consumer.rb,
lib/karafka/pro/performance_tracker.rb,
lib/karafka/processing/jobs/consume.rb,
lib/karafka/processing/jobs/revoked.rb,
lib/karafka/processing/jobs_builder.rb,
lib/karafka/routing/consumer_mapper.rb,
lib/karafka/contracts/consumer_group.rb,
lib/karafka/patches/rdkafka/bindings.rb,
lib/karafka/pro/iterator/tpl_builder.rb,
lib/karafka/pro/processing/collapser.rb,
lib/karafka/pro/processing/scheduler.rb,
lib/karafka/processing/jobs/shutdown.rb,
lib/karafka/processing/workers_batch.rb,
lib/karafka/routing/activity_manager.rb,
lib/karafka/active_job/job_extensions.rb,
lib/karafka/connection/pauses_manager.rb,
lib/karafka/messages/builders/message.rb,
lib/karafka/pro/active_job/dispatcher.rb,
lib/karafka/pro/processing/strategies.rb,
lib/karafka/pro/routing/features/base.rb,
lib/karafka/processing/strategies/dlq.rb,
lib/karafka/processing/strategies/mom.rb,
lib/karafka/connection/listeners_batch.rb,
lib/karafka/connection/messages_buffer.rb,
lib/karafka/messages/builders/messages.rb,
lib/karafka/pro/processing/coordinator.rb,
lib/karafka/pro/processing/partitioner.rb,
lib/karafka/processing/strategies/base.rb,
lib/karafka/routing/subscription_group.rb,
lib/karafka/pro/encryption/setup/config.rb,
lib/karafka/pro/processing/filters/base.rb,
lib/karafka/pro/processing/jobs_builder.rb,
lib/karafka/processing/executors_buffer.rb,
lib/karafka/routing/features/active_job.rb,
lib/karafka/connection/rebalance_manager.rb,
lib/karafka/contracts/server_cli_options.rb,
lib/karafka/pro/cleaner/messages/message.rb,
lib/karafka/pro/routing/features/pausing.rb,
lib/karafka/processing/strategies/aj_mom.rb,
lib/karafka/processing/strategy_selector.rb,
lib/karafka/active_job/current_attributes.rb,
lib/karafka/instrumentation/notifications.rb,
lib/karafka/pro/cleaner/messages/messages.rb,
lib/karafka/pro/routing/features/delaying.rb,
lib/karafka/pro/routing/features/expiring.rb,
lib/karafka/pro/routing/features/patterns.rb,
lib/karafka/processing/strategies/default.rb,
lib/karafka/processing/strategies/dlq_mom.rb,
lib/karafka/routing/features/declaratives.rb,
lib/karafka/connection/raw_messages_buffer.rb,
lib/karafka/pro/encryption/messages/parser.rb,
lib/karafka/pro/processing/filters/delayer.rb,
lib/karafka/pro/processing/filters/expirer.rb,
lib/karafka/pro/processing/filters_applier.rb,
lib/karafka/pro/processing/strategies/base.rb,
lib/karafka/pro/routing/features/filtering.rb,
lib/karafka/processing/coordinators_buffer.rb,
lib/karafka/processing/expansions_selector.rb,
lib/karafka/routing/features/base/expander.rb,
lib/karafka/active_job/job_options_contract.rb,
lib/karafka/instrumentation/callbacks/error.rb,
lib/karafka/instrumentation/logger_listener.rb,
lib/karafka/pro/encryption/contracts/config.rb,
lib/karafka/pro/routing/features/active_job.rb,
lib/karafka/pro/routing/features/throttling.rb,
lib/karafka/serialization/json/deserializer.rb,
lib/karafka/messages/builders/batch_metadata.rb,
lib/karafka/pro/contracts/server_cli_options.rb,
lib/karafka/pro/processing/filters/throttler.rb,
lib/karafka/pro/processing/strategies/aj/mom.rb,
lib/karafka/pro/processing/strategies/dlq/vp.rb,
lib/karafka/pro/processing/strategies/ftr/vp.rb,
lib/karafka/pro/processing/strategies/lrj/vp.rb,
lib/karafka/pro/processing/strategies/mom/vp.rb,
lib/karafka/pro/processing/strategy_selector.rb,
lib/karafka/processing/strategies/aj_dlq_mom.rb,
lib/karafka/routing/features/inline_insights.rb,
lib/karafka/pro/processing/strategies/default.rb,
lib/karafka/pro/processing/strategies/dlq/ftr.rb,
lib/karafka/pro/processing/strategies/dlq/lrj.rb,
lib/karafka/pro/processing/strategies/dlq/mom.rb,
lib/karafka/pro/processing/strategies/lrj/ftr.rb,
lib/karafka/pro/processing/strategies/lrj/mom.rb,
lib/karafka/pro/processing/strategies/mom/ftr.rb,
lib/karafka/routing/features/active_job/proxy.rb,
lib/karafka/routing/features/active_job/topic.rb,
lib/karafka/instrumentation/proctitle_listener.rb,
lib/karafka/pro/encryption/messages/middleware.rb,
lib/karafka/pro/routing/features/pausing/topic.rb,
lib/karafka/processing/inline_insights/tracker.rb,
lib/karafka/routing/features/active_job/config.rb,
lib/karafka/routing/features/dead_letter_queue.rb,
lib/karafka/pro/active_job/job_options_contract.rb,
lib/karafka/pro/processing/strategies/aj/mom_vp.rb,
lib/karafka/pro/routing/features/delaying/topic.rb,
lib/karafka/pro/routing/features/expiring/topic.rb,
lib/karafka/pro/routing/features/patterns/topic.rb,
lib/karafka/processing/inline_insights/consumer.rb,
lib/karafka/processing/inline_insights/listener.rb,
lib/karafka/routing/features/active_job/builder.rb,
lib/karafka/routing/features/declaratives/topic.rb,
lib/karafka/routing/subscription_groups_builder.rb,
lib/karafka/instrumentation/callbacks/statistics.rb,
lib/karafka/pro/processing/strategies/aj/dlq_mom.rb,
lib/karafka/pro/processing/strategies/aj/ftr_mom.rb,
lib/karafka/pro/processing/strategies/aj/lrj_mom.rb,
lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb,
lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb,
lib/karafka/pro/processing/strategies/dlq/mom_vp.rb,
lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb,
lib/karafka/pro/processing/strategies/lrj/mom_vp.rb,
lib/karafka/pro/processing/strategies/mom/ftr_vp.rb,
lib/karafka/pro/processing/strategies/vp/default.rb,
lib/karafka/pro/routing/features/delaying/config.rb,
lib/karafka/pro/routing/features/expiring/config.rb,
lib/karafka/pro/routing/features/filtering/topic.rb,
lib/karafka/pro/routing/features/patterns/config.rb,
lib/karafka/pro/routing/features/patterns/topics.rb,
lib/karafka/routing/features/declaratives/config.rb,
lib/karafka/active_job/current_attributes/loading.rb,
lib/karafka/connection/consumer_group_coordinator.rb,
lib/karafka/pro/processing/strategies/dlq/default.rb,
lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb,
lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb,
lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb,
lib/karafka/pro/processing/strategies/ftr/default.rb,
lib/karafka/pro/processing/strategies/lrj/default.rb,
lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb,
lib/karafka/pro/processing/strategies/mom/default.rb,
lib/karafka/pro/processing/virtual_offset_manager.rb,
lib/karafka/pro/routing/features/filtering/config.rb,
lib/karafka/pro/routing/features/long_running_job.rb,
lib/karafka/pro/routing/features/patterns/builder.rb,
lib/karafka/pro/routing/features/patterns/pattern.rb,
lib/karafka/pro/routing/features/throttling/topic.rb,
lib/karafka/pro/processing/filters/virtual_limiter.rb,
lib/karafka/pro/routing/features/dead_letter_queue.rb,
lib/karafka/pro/routing/features/patterns/detector.rb,
lib/karafka/pro/routing/features/patterns/patterns.rb,
lib/karafka/pro/routing/features/throttling/config.rb,
lib/karafka/routing/features/inline_insights/topic.rb,
lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb,
lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb,
lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb,
lib/karafka/pro/routing/features/active_job/builder.rb,
lib/karafka/pro/routing/features/virtual_partitions.rb,
lib/karafka/routing/features/inline_insights/config.rb,
lib/karafka/pro/processing/jobs/consume_non_blocking.rb,
lib/karafka/pro/processing/jobs/revoked_non_blocking.rb,
lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb,
lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb,
lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb,
lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb,
lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb,
lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb,
lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb,
lib/karafka/routing/features/dead_letter_queue/topic.rb,
lib/karafka/active_job/current_attributes/persistence.rb,
lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb,
lib/karafka/routing/features/dead_letter_queue/config.rb,
lib/karafka/routing/features/manual_offset_management.rb,
lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb,
lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb,
lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb,
lib/karafka/pro/routing/features/long_running_job/topic.rb,
lib/karafka/routing/features/active_job/contracts/topic.rb,
lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb,
lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb,
lib/karafka/pro/routing/features/long_running_job/config.rb,
lib/karafka/pro/routing/features/patterns/consumer_group.rb,
lib/karafka/pro/routing/features/pausing/contracts/topic.rb,
lib/karafka/pro/routing/features/delaying/contracts/topic.rb,
lib/karafka/pro/routing/features/expiring/contracts/topic.rb,
lib/karafka/pro/routing/features/patterns/contracts/topic.rb,
lib/karafka/pro/routing/features/virtual_partitions/topic.rb,
lib/karafka/routing/features/declaratives/contracts/topic.rb,
lib/karafka/pro/routing/features/filtering/contracts/topic.rb,
lib/karafka/pro/routing/features/virtual_partitions/config.rb,
lib/karafka/instrumentation/vendors/datadog/logger_listener.rb,
lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb,
lib/karafka/pro/routing/features/patterns/contracts/pattern.rb,
lib/karafka/pro/routing/features/throttling/contracts/topic.rb,
lib/karafka/routing/features/manual_offset_management/topic.rb,
lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb,
lib/karafka/routing/features/inline_insights/contracts/topic.rb,
lib/karafka/routing/features/manual_offset_management/config.rb,
lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb,
lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb,
lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb,
lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb,
lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb,
lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb,
lib/karafka/routing/features/manual_offset_management/contracts/topic.rb

Overview

This Karafka component is a Pro component under a commercial license. This Karafka component is NOT licensed under LGPL.

All of the commercial components are present in the lib/karafka/pro directory of this repository and their usage requires commercial license agreement.

Karafka has also commercial-friendly license, commercial support and commercial components.

By sending a pull request to the pro components, you are agreeing to transfer the copyright of your code to Maciej Mensfeld.

Defined Under Namespace

Modules: ActiveJob, Admin, Connection, Contracts, Embedded, Errors, Helpers, Instrumentation, Messages, Patches, Pro, Processing, Routing, Serialization, Setup, TimeTrackers Classes: App, BaseConsumer, Cli, Env, Licenser, Process, Railtie, Runner, Server, Status

Constant Summary collapse

VERSION =

Current Karafka version

'2.2.7'

Class Method Summary collapse

Class Method Details

.boot_fileString

Note:

By default it is a file called ‘karafka.rb’ but it can be specified as you wish if you have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken. It will be used for console/consumers/etc

Returns path to a default file that contains booting procedure etc.

Examples:

Standard only-Karafka case

Karafka.boot_file #=> '/home/app_path/karafka.rb'

Non standard case

KARAFKA_BOOT_FILE='/home/app_path/app.rb'
Karafka.boot_file #=> '/home/app_path/app.rb'

Returns:

  • (String)

    path to a default file that contains booting procedure etc



95
96
97
# File 'lib/karafka.rb', line 95

def boot_file
  Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
end

.core_rootString

Returns path to Karafka gem root core.

Returns:

  • (String)

    path to Karafka gem root core



64
65
66
# File 'lib/karafka.rb', line 64

def core_root
  Pathname.new(File.expand_path('karafka', __dir__))
end

.envKarafka::Env

Returns env instance that allows us to check environment.

Returns:

  • (Karafka::Env)

    env instance that allows us to check environment



26
27
28
# File 'lib/karafka.rb', line 26

def env
  @env ||= Env.new
end

.env=(environment) ⇒ Karafka::Env

Returns env instance.

Examples:

Assign new environment to Karafka::App

Karafka::App.env = :production

Parameters:

  • environment (String, Symbol)

    new environment that we want to set

Returns:



34
35
36
# File 'lib/karafka.rb', line 34

def env=(environment)
  env.replace(environment.to_s)
end

.gem_rootString

Returns root path of this gem.

Returns:

  • (String)

    root path of this gem



54
55
56
# File 'lib/karafka.rb', line 54

def gem_root
  Pathname.new(File.expand_path('..', __dir__))
end

.loggerLogger

Returns logger that we want to use. Will use ::Karafka::Logger by default.

Returns:

  • (Logger)

    logger that we want to use. Will use ::Karafka::Logger by default



39
40
41
# File 'lib/karafka.rb', line 39

def logger
  @logger ||= App.config.logger
end

.monitor::Karafka::Monitor

Returns monitor that we want to use.

Returns:

  • (::Karafka::Monitor)

    monitor that we want to use



49
50
51
# File 'lib/karafka.rb', line 49

def monitor
  @monitor ||= App.config.monitor
end

.pro?Boolean

Returns true if there is a valid pro token present.

Returns:

  • (Boolean)

    true if there is a valid pro token present



69
70
71
# File 'lib/karafka.rb', line 69

def pro?
  App.config.license.token != false
end

.producerWaterDrop::Producer

Returns waterdrop messages producer.

Returns:

  • (WaterDrop::Producer)

    waterdrop messages producer



44
45
46
# File 'lib/karafka.rb', line 44

def producer
  @producer ||= App.config.producer
end

.rails?Boolean

Returns Do we run within/with Rails. We use this to initialize Railtie and proxy the console invocation to Rails.

Returns:

  • (Boolean)

    Do we run within/with Rails. We use this to initialize Railtie and proxy the console invocation to Rails



75
76
77
78
79
80
81
82
83
84
# File 'lib/karafka.rb', line 75

def rails?
  return @rails if instance_variable_defined?('@rails')

  # Do not load Rails again if already loaded
  Object.const_defined?('Rails::Railtie') || require('rails')

  @rails = true
rescue LoadError
  @rails = false
end

.refresh!Object

We need to be able to overwrite both monitor and logger after the configuration in case they would be changed because those two (with defaults) can be used prior to the setup and their state change should be reflected in the updated setup

This method refreshes the things that might have been altered by the configuration



104
105
106
107
108
109
110
# File 'lib/karafka.rb', line 104

def refresh!
  config = ::Karafka::App.config

  @logger = config.logger
  @producer = config.producer
  @monitor = config.monitor
end

.rootString

Returns Karafka app root path (user application path).

Returns:

  • (String)

    Karafka app root path (user application path)



59
60
61
# File 'lib/karafka.rb', line 59

def root
  Pathname.new(ENV['KARAFKA_ROOT_DIR'] || File.dirname(ENV['BUNDLE_GEMFILE']))
end