Module: Lowkiq

Defined in:
lib/lowkiq.rb,
lib/lowkiq/web.rb,
lib/lowkiq/utils.rb,
lib/lowkiq/server.rb,
lib/lowkiq/worker.rb,
lib/lowkiq/version.rb,
lib/lowkiq/web/api.rb,
lib/lowkiq/queue/keys.rb,
lib/lowkiq/redis_info.rb,
lib/lowkiq/web/action.rb,
lib/lowkiq/queue/fetch.rb,
lib/lowkiq/queue/queue.rb,
lib/lowkiq/option_parser.rb,
lib/lowkiq/queue/actions.rb,
lib/lowkiq/queue/marshal.rb,
lib/lowkiq/queue/queries.rb,
lib/lowkiq/shard_handler.rb,
lib/lowkiq/extend_tracker.rb,
lib/lowkiq/schedulers/lag.rb,
lib/lowkiq/schedulers/seq.rb,
lib/lowkiq/splitters/by_node.rb,
lib/lowkiq/splitters/default.rb,
lib/lowkiq/queue/queue_metrics.rb,
lib/lowkiq/queue/shard_metrics.rb

Defined Under Namespace

Modules: ExtendTracker, OptionParser, Queue, Schedulers, Splitters, Utils, Web, Worker Classes: RedisInfo, Server, ShardHandler

Constant Summary collapse

VERSION = 
"1.0.4"

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.build_scheduler ⇒ Object

Returns the value of attribute build_scheduler.



39
40
41
 
# File 'lib/lowkiq.rb', line 39

def build_scheduler
  @build_scheduler
end

.build_splitter ⇒ Object

Returns the value of attribute build_splitter.



39
40
41
 
# File 'lib/lowkiq.rb', line 39

def build_splitter
  @build_splitter
end

.client_pool_size ⇒ Object

Returns the value of attribute client_pool_size.



39
40
41
 
# File 'lib/lowkiq.rb', line 39

def client_pool_size
  @client_pool_size
end

.last_words ⇒ Object

Returns the value of attribute last_words.



39
40
41
 
# File 'lib/lowkiq.rb', line 39

def last_words
  @last_words
end

.on_server_init ⇒ Object

Returns the value of attribute on_server_init.



39
40
41
 
# File 'lib/lowkiq.rb', line 39

def on_server_init
  @on_server_init
end

.poll_interval ⇒ Object

Returns the value of attribute poll_interval.



39
40
41
 
# File 'lib/lowkiq.rb', line 39

def poll_interval
  @poll_interval
end

.pool_timeout ⇒ Object

Returns the value of attribute pool_timeout.



39
40
41
 
# File 'lib/lowkiq.rb', line 39

def pool_timeout
  @pool_timeout
end

.redis ⇒ Object

Returns the value of attribute redis.



39
40
41
 
# File 'lib/lowkiq.rb', line 39

def redis
  @redis
end

.server_middlewares ⇒ Object

Returns the value of attribute server_middlewares.



39
40
41
 
# File 'lib/lowkiq.rb', line 39

def server_middlewares
  @server_middlewares
end

.threads_per_node ⇒ Object

Returns the value of attribute threads_per_node.



39
40
41
 
# File 'lib/lowkiq.rb', line 39

def threads_per_node
  @threads_per_node
end

Class Method Details

.build_by_node_splitter(number_of_nodes, node_number) ⇒ Object 



91
92
93
94
95
96
97
 
# File 'lib/lowkiq.rb', line 91

def build_by_node_splitter(number_of_nodes, node_number)
  Lowkiq::Splitters::ByNode.new(
    number_of_nodes,
    node_number,
    Lowkiq.threads_per_node,
  )
end

.build_default_splitter ⇒ Object 



87
88
89
 
# File 'lib/lowkiq.rb', line 87

def build_default_splitter
  Lowkiq::Splitters::Default.new Lowkiq.threads_per_node
end

.build_lag_scheduler ⇒ Object 



74
75
76
77
78
79
 
# File 'lib/lowkiq.rb', line 74

def build_lag_scheduler
  Schedulers::Lag.new(
    ->() { sleep Lowkiq.poll_interval },
    Queue::ShardMetrics.new(self.server_redis_pool)
  )
end

.build_seq_scheduler ⇒ Object 



81
82
83
84
85
 
# File 'lib/lowkiq.rb', line 81

def build_seq_scheduler
  Schedulers::Seq.new(
    ->() { sleep Lowkiq.poll_interval }
  )
end

.client_redis_pool ⇒ Object 



49
50
51
 
# File 'lib/lowkiq.rb', line 49

def client_redis_pool
  @client_redis_pool ||= ConnectionPool.new(size: client_pool_size, timeout: pool_timeout, &redis)
end

.server_redis_pool ⇒ Object 



45
46
47
 
# File 'lib/lowkiq.rb', line 45

def server_redis_pool
  @server_redis_pool ||= ConnectionPool.new(size: threads_per_node, timeout: pool_timeout, &redis)
end

.server_wrapper ⇒ Object 



53
54
55
56
57
58
59
60
61
62
 
# File 'lib/lowkiq.rb', line 53

def server_wrapper
  null = -> (worker, batch, &block) { block.call }
  server_middlewares.reduce(null) do |wrapper, m|
    -> (worker, batch, &block) do
      wrapper.call worker, batch do
        m.call worker, batch, &block
      end
    end
  end
end

.shard_handlers ⇒ Object 



68
69
70
71
72
 
# File 'lib/lowkiq.rb', line 68

def shard_handlers
  self.workers.flat_map do |w|
    ShardHandler.build_many w, self.server_wrapper
  end
end

.workers ⇒ Object 



64
65
66
 
# File 'lib/lowkiq.rb', line 64

def workers
  Worker.extended_modules
end