Class: Wayfarer::Middleware::Dedup

Inherits:
Object
  • Object
show all
Defined in:
lib/wayfarer/middleware/dedup.rb

Instance Method Summary collapse

Instance Method Details

#call(task) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/wayfarer/middleware/dedup.rb', line 6

def call(task)
  return if task.barrier.seen?(task.url)

  begin
    yield if block_given?
  rescue StandardError => e
    task.barrier.unsee(task.url)
    raise e
  end

  staged_urls = task.metadata.staged_urls
  return if staged_urls.none?

  inclusion = task.barrier.peek(staged_urls.to_a)
  unseen = staged_urls.zip(inclusion).reject { |_, seen| seen }.map(&:first)
  task.metadata.staged_urls = SortedSet.new(unseen)
end