Class: Nous::Crawler

Inherits:
Command show all
Defined in:
lib/nous/crawler.rb,
lib/nous/crawler/url_filter.rb,
lib/nous/crawler/link_extractor.rb,
lib/nous/crawler/redirect_follower.rb,
lib/nous/crawler/async_page_fetcher.rb,
lib/nous/crawler/single_page_fetcher.rb,
lib/nous/crawler/recursive_page_fetcher.rb

Defined Under Namespace

Classes: AsyncPageFetcher, CrawlError, LinkExtractor, RecursivePageFetcher, RedirectFollower, SinglePageFetcher, UrlFilter

Instance Method Summary collapse

Methods inherited from Command

call, #failure, #success

Constructor Details

#initialize(seed_url:, http_client: nil) ⇒ Crawler

Returns a new instance of Crawler.



7
8
9
10
11
 
# File 'lib/nous/crawler.rb', line 7

def initialize(seed_url:, http_client: nil)
  @seed_url = seed_url
  @http_client = http_client
  parse_seed!
end

Instance Method Details

#call ⇒ Object 



13
14
15
16
17
18
19
 
# File 'lib/nous/crawler.rb', line 13

def call
  if config.recursive?
    RecursivePageFetcher.call(seed_url:, http_client:)
  else
    SinglePageFetcher.call(url: seed_url, http_client:)
  end
end