Class: Nous::Fetcher::ExtractionRunner

Inherits:
Command
  • Object
show all
Defined in:
lib/nous/fetcher/extraction_runner.rb

Defined Under Namespace

Classes: ExtractionError

Instance Method Summary collapse

Methods inherited from Command

call, #failure, #success

Constructor Details

#initialize(records:, extractor:) ⇒ ExtractionRunner

Returns a new instance of ExtractionRunner.



8
9
10
11
 
# File 'lib/nous/fetcher/extraction_runner.rb', line 8

def initialize(records:, extractor:)
  @records = records
  @extractor = extractor
end

Instance Method Details

#call ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/nous/fetcher/extraction_runner.rb', line 13

def call
  pages = records.each_slice(Nous.configuration.concurrency).each_with_object([]) do |batch, results|
    threads = batch.map { |record| Thread.new { PageExtractor.call(extractor:, record:) } }

    threads.each do |thread|
      result = thread.value
      results << result.payload if result.success?
    end
  end

  success(payload: pages)
end