Class: Nous::Crawler::RecursivePageFetcher

Inherits:
Nous::Command show all
Defined in:
lib/nous/crawler/recursive_page_fetcher.rb

Instance Method Summary collapse

Methods inherited from Nous::Command

call, #failure, #success

Constructor Details

#initialize(seed_url:, http_client: nil) ⇒ RecursivePageFetcher

Returns a new instance of RecursivePageFetcher.



9
10
11
12
13
14
15
 
# File 'lib/nous/crawler/recursive_page_fetcher.rb', line 9

def initialize(seed_url:, http_client: nil)
  @seed_uri = Url.new(seed_url)
  @http_client = http_client
  @records = []
  @queue = [url_filter.canonicalize(seed_uri)]
  @seen = Set.new(queue)
end

Instance Method Details

#call ⇒ Object 



17
18
19
20
21
22
23
24
25
 
# File 'lib/nous/crawler/recursive_page_fetcher.rb', line 17

def call
  suppress_async_warnings unless config.debug?

  open_connection do |client|
    crawl(client)
  end

  success(payload: records)
end