Class: CI::Queue::Redis::Worker

Inherits:
Base
  • Object
show all
Defined in:
lib/ci/queue/redis/worker.rb

Direct Known Subclasses

Grind, TestTimeRecord

Constant Summary collapse

DEFAULT_SLEEP_SECONDS = 
0.5

Constants inherited from Base

Base::CONNECTION_ERRORS, Base::DEFAULT_TIMEOUT, Base::TEN_MINUTES

Constants included from Common

Common::CONNECTION_ERRORS

Instance Attribute Summary collapse

Attributes included from Common

#config

Instance Method Summary collapse

Methods inherited from Base

#boot_heartbeat_process!, #created_at=, #custom_config, #custom_middlewares, #ensure_heartbeat_thread_alive!, #exhausted?, #expired?, #increment_test_failed, #max_test_failed?, #progress, #queue_initialized?, #queue_initializing?, #reconnect_attempts, #remaining, #running, #size, #stop_heartbeat!, #streaming?, #test_failed, #test_ids, #to_a, #wait_for_master, #with_heartbeat, #workers_count

Methods included from Common

#flaky?, #report_failure!, #report_success!, #rescue_connection_errors

Constructor Details

#initialize(redis, config) ⇒ Worker

Returns a new instance of Worker.



20
21
22
23
24
25
 
# File 'lib/ci/queue/redis/worker.rb', line 20

def initialize(redis, config)
  @reserved_tests = Concurrent::Set.new
  @shutdown_required = false
  @first_reserve_at = nil
  super(redis, config)
end

Instance Attribute Details

#entry_resolver ⇒ Object

Returns the value of attribute entry_resolver.



17
18
19
 
# File 'lib/ci/queue/redis/worker.rb', line 17

def entry_resolver
  @entry_resolver
end

#first_reserve_at ⇒ Object (readonly)

Returns the value of attribute first_reserve_at.



18
19
20
 
# File 'lib/ci/queue/redis/worker.rb', line 18

def first_reserve_at
  @first_reserve_at
end

Instance Method Details

#acknowledge(test_key, error: nil, pipeline: redis) ⇒ Object 



179
180
181
182
183
184
185
186
187
188
189
190
 
# File 'lib/ci/queue/redis/worker.rb', line 179

def acknowledge(test_key, error: nil, pipeline: redis)
  test_id = normalize_test_id(test_key)
  assert_reserved!(test_id)
  entry = reserved_entries.fetch(test_id, queue_entry_for(test_key))
  unreserve_entry(test_id)
  eval_script(
    :acknowledge,
    keys: [key('running'), key('processed'), key('owners'), key('error-reports'), key('requeued-by')],
    argv: [entry, test_id, error.to_s, config.redis_ttl],
    pipeline: pipeline,
  ) == 1
end

#build ⇒ Object 



161
162
163
 
# File 'lib/ci/queue/redis/worker.rb', line 161

def build
  @build ||= CI::Queue::Redis::BuildRecord.new(self, redis, config)
end

#distributed? ⇒ Boolean

Returns:

  • (Boolean) 


27
28
29
 
# File 'lib/ci/queue/redis/worker.rb', line 27

def distributed?
  true
end

#file_loader ⇒ Object 



165
166
167
 
# File 'lib/ci/queue/redis/worker.rb', line 165

def file_loader
  @file_loader ||= CI::Queue::FileLoader.new
end

#master? ⇒ Boolean

Returns:

  • (Boolean) 


100
101
102
 
# File 'lib/ci/queue/redis/worker.rb', line 100

def master?
  @master
end

#poll ⇒ Object 



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/ci/queue/redis/worker.rb', line 106

def poll
  wait_for_master(timeout: config.queue_init_timeout, allow_streaming: true)
  attempt = 0
  until shutdown_required? || config.circuit_breakers.any?(&:open?) || exhausted? || max_test_failed?
    if entry = reserve
      attempt = 0
      yield resolve_entry(entry)
    else
      if still_streaming?
        raise LostMaster, "Streaming stalled for more than #{config.lazy_load_streaming_timeout}s" if streaming_stale?
        sleep 0.1
        next
      end
      # Adding exponential backoff to avoid hammering Redis
      # we just stay online here in case a test gets retried or times out so we can afford to wait
      sleep_time = [DEFAULT_SLEEP_SECONDS * (2 ** attempt), Redis.max_sleep_time].min
      attempt += 1
      sleep sleep_time
    end
  end
  redis.pipelined do |pipeline|
    pipeline.expire(key('worker', worker_id, 'queue'), config.redis_ttl)
    pipeline.expire(key('processed'), config.redis_ttl)
  end
rescue *CONNECTION_ERRORS
end

#populate(tests, random: Random.new) ⇒ Object 



31
32
33
34
35
36
 
# File 'lib/ci/queue/redis/worker.rb', line 31

def populate(tests, random: Random.new)
  @index = tests.map { |t| [t.id, t] }.to_h
  entries = Queue.shuffle(tests, random).map { |test| queue_entry_for(test) }
  push(entries)
  self
end

#populated? ⇒ Boolean

Returns:

  • (Boolean) 


80
81
82
 
# File 'lib/ci/queue/redis/worker.rb', line 80

def populated?
  !!defined?(@index)
end

#release! ⇒ Object 



222
223
224
225
226
227
228
229
 
# File 'lib/ci/queue/redis/worker.rb', line 222

def release!
  eval_script(
    :release,
    keys: [key('running'), key('worker', worker_id, 'queue'), key('owners')],
    argv: [],
  )
  nil
end

#report_worker_error(error) ⇒ Object 



175
176
177
 
# File 'lib/ci/queue/redis/worker.rb', line 175

def report_worker_error(error)
  build.report_worker_error(error)
end

#requeue(test, offset: Redis.requeue_offset) ⇒ Object 



192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
 
# File 'lib/ci/queue/redis/worker.rb', line 192

def requeue(test, offset: Redis.requeue_offset)
  test_id = normalize_test_id(test)
  assert_reserved!(test_id)
  entry = reserved_entries.fetch(test_id, queue_entry_for(test))
  unreserve_entry(test_id)
  global_max_requeues = config.global_max_requeues(total)

  requeued = config.max_requeues > 0 && global_max_requeues > 0 && eval_script(
    :requeue,
    keys: [
      key('processed'),
      key('requeues-count'),
      key('queue'),
      key('running'),
      key('worker', worker_id, 'queue'),
      key('owners'),
      key('error-reports'),
      key('requeued-by'),
    ],
    argv: [config.max_requeues, global_max_requeues, entry, test_id, offset, config.redis_ttl],
  ) == 1

  unless requeued
    reserved_tests << test_id
    reserved_entries[test_id] = entry
    reserved_entry_ids[entry] = test_id
  end
  requeued
end

#retry_queue ⇒ Object 



147
148
149
150
151
152
153
154
155
 
# File 'lib/ci/queue/redis/worker.rb', line 147

def retry_queue
  failures = build.failed_tests.to_set
  log = redis.lrange(key('worker', worker_id, 'queue'), 0, -1)
  log = log.map { |entry| queue_entry_test_id(entry) }
  log.select! { |id| failures.include?(id) }
  log.uniq!
  log.reverse!
  Retry.new(log, config, redis: redis)
end

#retrying? ⇒ Boolean

Returns:

  • (Boolean) 


134
135
136
137
138
 
# File 'lib/ci/queue/redis/worker.rb', line 134

def retrying?
  redis.exists?(key('worker', worker_id, 'queue'))
rescue *CONNECTION_ERRORS
  false
end

#shutdown! ⇒ Object 



92
93
94
 
# File 'lib/ci/queue/redis/worker.rb', line 92

def shutdown!
  @shutdown_required = true
end

#shutdown_required? ⇒ Boolean

Returns:

  • (Boolean) 


96
97
98
 
# File 'lib/ci/queue/redis/worker.rb', line 96

def shutdown_required?
  @shutdown_required
end

#stream_populate(tests, random: Random.new, batch_size: 10_000) ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/ci/queue/redis/worker.rb', line 38

def stream_populate(tests, random: Random.new, batch_size: 10_000)
  batch_size = batch_size.to_i
  batch_size = 1 if batch_size < 1

  value = key('setup', worker_id)
  _, status = redis.pipelined do |pipeline|
    pipeline.set(key('master-status'), value, nx: true)
    pipeline.get(key('master-status'))
  end

  if @master = (value == status)
    @total = 0
    puts "Worker elected as leader, streaming tests to the queue."

    duration = measure do
      start_streaming!
      buffer = []

      tests.each do |test|
        buffer << test

        if buffer.size >= batch_size
          push_batch(buffer, random)
          buffer.clear
        end
      end

      push_batch(buffer, random) unless buffer.empty?
      finalize_streaming
    end

    puts "Streamed #{@total} tests in #{duration.round(2)}s."
    $stdout.flush
  end

  register
  redis.expire(key('workers'), config.redis_ttl)
  self
rescue *CONNECTION_ERRORS
  raise if @master
end

#supervisor ⇒ Object 



157
158
159
 
# File 'lib/ci/queue/redis/worker.rb', line 157

def supervisor
  Supervisor.new(redis_url, config)
end

#total ⇒ Object 



84
85
86
87
88
89
90
 
# File 'lib/ci/queue/redis/worker.rb', line 84

def total
  return @total if defined?(@total) && @total

  redis.get(key('total')).to_i
rescue *CONNECTION_ERRORS
  @total || 0
end

#worker_queue_length ⇒ Object 



169
170
171
172
173
 
# File 'lib/ci/queue/redis/worker.rb', line 169

def worker_queue_length
  redis.llen(key('worker', worker_id, 'queue'))
rescue *CONNECTION_ERRORS
  nil
end