Module: Lowkiq

Defined in:
lib/lowkiq.rb,
lib/lowkiq/web.rb,
lib/lowkiq/utils.rb,
lib/lowkiq/script.rb,
lib/lowkiq/server.rb,
lib/lowkiq/worker.rb,
lib/lowkiq/version.rb,
lib/lowkiq/web/api.rb,
lib/lowkiq/queue/keys.rb,
lib/lowkiq/redis_info.rb,
lib/lowkiq/web/action.rb,
lib/lowkiq/queue/fetch.rb,
lib/lowkiq/queue/queue.rb,
lib/lowkiq/option_parser.rb,
lib/lowkiq/queue/actions.rb,
lib/lowkiq/queue/queries.rb,
lib/lowkiq/shard_handler.rb,
lib/lowkiq/extend_tracker.rb,
lib/lowkiq/schedulers/lag.rb,
lib/lowkiq/schedulers/seq.rb,
lib/lowkiq/splitters/by_node.rb,
lib/lowkiq/splitters/default.rb,
lib/lowkiq/queue/queue_metrics.rb,
lib/lowkiq/queue/shard_metrics.rb

Defined Under Namespace

Modules: ExtendTracker, OptionParser, Queue, Schedulers, Script, Splitters, Utils, Web, Worker Classes: RedisInfo, Server, ShardHandler

Constant Summary collapse

VERSION = 
"1.0.6"

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.build_scheduler ⇒ Object

Returns the value of attribute build_scheduler.



40
41
42
 
# File 'lib/lowkiq.rb', line 40

def build_scheduler
  @build_scheduler
end

.build_splitter ⇒ Object

Returns the value of attribute build_splitter.



40
41
42
 
# File 'lib/lowkiq.rb', line 40

def build_splitter
  @build_splitter
end

.client_pool_size ⇒ Object

Returns the value of attribute client_pool_size.



40
41
42
 
# File 'lib/lowkiq.rb', line 40

def client_pool_size
  @client_pool_size
end

.dump_payload ⇒ Object

Returns the value of attribute dump_payload.



40
41
42
 
# File 'lib/lowkiq.rb', line 40

def dump_payload
  @dump_payload
end

.last_words ⇒ Object

Returns the value of attribute last_words.



40
41
42
 
# File 'lib/lowkiq.rb', line 40

def last_words
  @last_words
end

.load_payload ⇒ Object

Returns the value of attribute load_payload.



40
41
42
 
# File 'lib/lowkiq.rb', line 40

def load_payload
  @load_payload
end

.on_server_init ⇒ Object

Returns the value of attribute on_server_init.



40
41
42
 
# File 'lib/lowkiq.rb', line 40

def on_server_init
  @on_server_init
end

.poll_interval ⇒ Object

Returns the value of attribute poll_interval.



40
41
42
 
# File 'lib/lowkiq.rb', line 40

def poll_interval
  @poll_interval
end

.pool_timeout ⇒ Object

Returns the value of attribute pool_timeout.



40
41
42
 
# File 'lib/lowkiq.rb', line 40

def pool_timeout
  @pool_timeout
end

.redis ⇒ Object

Returns the value of attribute redis.



40
41
42
 
# File 'lib/lowkiq.rb', line 40

def redis
  @redis
end

.server_middlewares ⇒ Object

Returns the value of attribute server_middlewares.



40
41
42
 
# File 'lib/lowkiq.rb', line 40

def server_middlewares
  @server_middlewares
end

.threads_per_node ⇒ Object

Returns the value of attribute threads_per_node.



40
41
42
 
# File 'lib/lowkiq.rb', line 40

def threads_per_node
  @threads_per_node
end

Class Method Details

.build_by_node_splitter(number_of_nodes, node_number) ⇒ Object 



93
94
95
96
97
98
99
 
# File 'lib/lowkiq.rb', line 93

def build_by_node_splitter(number_of_nodes, node_number)
  Lowkiq::Splitters::ByNode.new(
    number_of_nodes,
    node_number,
    Lowkiq.threads_per_node,
  )
end

.build_default_splitter ⇒ Object 



89
90
91
 
# File 'lib/lowkiq.rb', line 89

def build_default_splitter
  Lowkiq::Splitters::Default.new Lowkiq.threads_per_node
end

.build_lag_scheduler ⇒ Object 



76
77
78
79
80
81
 
# File 'lib/lowkiq.rb', line 76

def build_lag_scheduler
  Schedulers::Lag.new(
    ->() { sleep Lowkiq.poll_interval },
    Queue::ShardMetrics.new(self.server_redis_pool)
  )
end

.build_seq_scheduler ⇒ Object 



83
84
85
86
87
 
# File 'lib/lowkiq.rb', line 83

def build_seq_scheduler
  Schedulers::Seq.new(
    ->() { sleep Lowkiq.poll_interval }
  )
end

.client_redis_pool ⇒ Object 



51
52
53
 
# File 'lib/lowkiq.rb', line 51

def client_redis_pool
  @client_redis_pool ||= ConnectionPool.new(size: client_pool_size, timeout: pool_timeout, &redis)
end

.server_redis_pool ⇒ Object 



47
48
49
 
# File 'lib/lowkiq.rb', line 47

def server_redis_pool
  @server_redis_pool ||= ConnectionPool.new(size: threads_per_node, timeout: pool_timeout, &redis)
end

.server_wrapper ⇒ Object 



55
56
57
58
59
60
61
62
63
64
 
# File 'lib/lowkiq.rb', line 55

def server_wrapper
  null = -> (worker, batch, &block) { block.call }
  server_middlewares.reduce(null) do |wrapper, m|
    -> (worker, batch, &block) do
      wrapper.call worker, batch do
        m.call worker, batch, &block
      end
    end
  end
end

.shard_handlers ⇒ Object 



70
71
72
73
74
 
# File 'lib/lowkiq.rb', line 70

def shard_handlers
  self.workers.flat_map do |w|
    ShardHandler.build_many w, self.server_wrapper
  end
end

.workers ⇒ Object 



66
67
68
 
# File 'lib/lowkiq.rb', line 66

def workers
  Worker.extended_modules
end