Class: SidekiqUniqueJobs::Orphans::RubyReaper

Inherits:
Reaper
  • Object
show all
Defined in:
lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

Overview

Note:

this is a much slower version of the lua script but does not crash redis

Class DeleteOrphans provides deletion of orphaned digests

rubocop:disable Metrics/ClassLength

Author:

Constant Summary collapse

RUN_SUFFIX =

Returns the suffix for :RUN locks.

Returns:

  • (String) —

    the suffix for :RUN locks

":RUN"

Constants inherited from Reaper

SidekiqUniqueJobs::Orphans::Reaper::REAPERS

Instance Attribute Summary collapse

Attributes inherited from Reaper

#conn

Instance Method Summary collapse

Methods inherited from Reaper

call, #config, #reaper, #reaper_count, #reaper_timeout

Methods included from JSON

dump_json, load_json, safe_load_json

Methods included from Logging

#build_message, included, #log_debug, #log_error, #log_fatal, #log_info, #log_warn, #logger, #logging_context, #with_configured_loggers_context, #with_logging_context

Methods included from Script::Caller

call_script, debug_lua, do_call, extract_args, max_history, now_f, redis_version

Methods included from Connection

included, #redis

Constructor Details

#initialize(conn) ⇒ RubyReaper

Initialize a new instance of DeleteOrphans

Parameters:

  • conn (Redis) —

    a connection to redis



35
36
37
38
39
40
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 35

def initialize(conn)
  super(conn)
  @digests   = SidekiqUniqueJobs::Digests.new
  @scheduled = Redis::SortedSet.new(SCHEDULE)
  @retried   = Redis::SortedSet.new(RETRY)
end

Instance Attribute Details

#digests ⇒ Object (readonly)

Returns the value of attribute digests.



20
21
22
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 20

def digests
  @digests
end

#retried ⇒ Object (readonly)

Returns the value of attribute retried.



28
29
30
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 28

def retried
  @retried
end

#scheduled ⇒ Object (readonly)

Returns the value of attribute scheduled.



24
25
26
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 24

def scheduled
  @scheduled
end

Instance Method Details

#active?(digest) ⇒ Boolean

rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity

Returns:

  • (Boolean) 


138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 138

def active?(digest) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
  Sidekiq.redis do |conn|
    procs = conn.sscan_each("processes").to_a
    return false if procs.empty?

    procs.sort.each do |key|
      valid, workers = conn.pipelined do
        # TODO: Remove the if statement in the future
        if conn.respond_to?(:exists?)
          conn.exists?(key)
        else
          conn.exists(key)
        end
        conn.hgetall("#{key}:workers")
      end

      next unless valid
      next unless workers.any?

      workers.each_pair do |_tid, job|
        next unless (item = safe_load_json(job))

        payload = safe_load_json(item[PAYLOAD])

        return true if match?(digest, payload[LOCK_DIGEST])
        return true if considered_active?(payload[CREATED_AT])
      end
    end

    false
  end
end

#belongs_to_job?(digest) ⇒ true, false

Checks if the digest has a matching job.

1. It checks the scheduled set
2. It checks the retry set
3. It goes through all queues

Parameters:

  • digest (String) —

    the digest to search for

Returns:

  • (true) —

    when either of the checks return true

  • (false) —

    when no job was found for this digest



93
94
95
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 93

def belongs_to_job?(digest)
  scheduled?(digest) || retried?(digest) || enqueued?(digest) || active?(digest)
end

#call ⇒ Integer

Delete orphaned digests

Returns:

  • (Integer) —

    the number of reaped locks



48
49
50
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 48

def call
  BatchDelete.call(orphans, conn)
end

#considered_active?(time_f) ⇒ Boolean

Returns:

  • (Boolean) 


177
178
179
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 177

def considered_active?(time_f)
  (Time.now - reaper_timeout).to_f < time_f
end

#enqueued?(digest) ⇒ true

Checks if the digest exists in a Sidekiq::Queue

Parameters:

  • digest (String) —

    the current digest

Returns:

  • (true) —

    when digest exists in any queue



126
127
128
129
130
131
132
133
134
135
136
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 126

def enqueued?(digest)
  Sidekiq.redis do |conn|
    queues(conn) do |queue|
      entries(conn, queue) do |entry|
        return true if entry.include?(digest)
      end
    end

    false
  end
end

#entries(conn, queue, &block) ⇒ Object

rubocop:disable Metrics/MethodLength



194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 194

def entries(conn, queue, &block) # rubocop:disable Metrics/MethodLength
  queue_key    = "queue:#{queue}"
  initial_size = conn.llen(queue_key)
  deleted_size = 0
  page         = 0
  page_size    = 50

  loop do
    range_start = (page * page_size) - deleted_size
    range_end   = range_start + page_size - 1
    entries     = conn.lrange(queue_key, range_start, range_end)
    page       += 1

    break if entries.empty?

    entries.each(&block)

    deleted_size = initial_size - conn.llen(queue_key)
  end
end

#in_sorted_set?(key, digest) ⇒ true, false

Checks a sorted set for the existance of this digest

Parameters:

  • key (String) —

    the key for the sorted set

  • digest (String) —

    the digest to scan for

Returns:

  • (true) —

    when found

  • (false) —

    when missing



225
226
227
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 225

def in_sorted_set?(key, digest)
  conn.zscan_each(key, match: "*#{digest}*", count: 1).to_a.any?
end

#match?(key_one, key_two) ⇒ Boolean

Returns:

  • (Boolean) 


171
172
173
174
175
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 171

def match?(key_one, key_two)
  return false if key_one.nil? || key_two.nil?

  key_one.delete_suffix(RUN_SUFFIX) == key_two.delete_suffix(RUN_SUFFIX)
end

#orphans ⇒ Array<String>

Find orphaned digests

Returns:

  • (Array<String>) —

    an array of orphaned digests



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 58

def orphans # rubocop:disable Metrics/MethodLength
  page = 0
  per = reaper_count * 2
  orphans = []
  results = conn.zrange(digests.key, page * per, (page + 1) * per)

  while results.size.positive?
    results.each do |digest|
      next if belongs_to_job?(digest)

      orphans << digest
      break if orphans.size >= reaper_count
    end

    break if orphans.size >= reaper_count

    page += 1
    results = conn.zrange(digests.key, page * per, (page + 1) * per)
  end

  orphans
end

#queues(conn) { ... } ⇒ void

This method returns an undefined value.

Loops through all the redis queues and yields them one by one

Parameters:

  • conn (Redis) —

    the connection to use for fetching queues

Yields:

  • queues one at a time



190
191
192
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 190

def queues(conn, &block)
  conn.sscan_each("queues", &block)
end

#retried?(digest) ⇒ true

Checks if the digest exists in the Sidekiq::RetrySet

Parameters:

  • digest (String) —

    the current digest

Returns:

  • (true) —

    when digest exists in retry set



115
116
117
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 115

def retried?(digest)
  in_sorted_set?(RETRY, digest)
end

#scheduled?(digest) ⇒ true

Checks if the digest exists in the Sidekiq::ScheduledSet

Parameters:

  • digest (String) —

    the current digest

Returns:

  • (true) —

    when digest exists in scheduled set



104
105
106
 
# File 'lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb', line 104

def scheduled?(digest)
  in_sorted_set?(SCHEDULE, digest)
end