Class: BraveSearch::PdfDownloader

Inherits:
Object
  • Object
show all
Defined in:
lib/brave_search/pdf_downloader.rb

Instance Method Summary collapse

Constructor Details

#initialize(storage: nil) ⇒ PdfDownloader

Returns a new instance of PdfDownloader.



7
8
9
 
# File 'lib/brave_search/pdf_downloader.rb', line 7

def initialize(storage: nil)
  @storage = storage || default_storage
end

Instance Method Details

#batch_download(urls, folder: "pdfs", &progress_callback) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/brave_search/pdf_downloader.rb', line 18

def batch_download(urls, folder: "pdfs", &progress_callback)
  total = urls.length
  completed = Concurrent::AtomicFixnum.new(0)

  futures = urls.map do |url|
    Concurrent::Future.execute do
      result = download(url, folder: folder)
      current = completed.increment
      progress_callback&.call(current, total)
      result
    end
  end

  futures.map(&:value)
end

#download(url, folder: "pdfs") ⇒ Object 



11
12
13
14
15
16
 
# File 'lib/brave_search/pdf_downloader.rb', line 11

def download(url, folder: "pdfs")
  filename = extract_filename(url)
  key = "#{folder}/#{filename}"

  @storage.download(url, key: key)
end