Class: Nous::Crawler::AsyncPageFetcher

Inherits:
Object
  • Object
show all
Defined in:
lib/nous/crawler/async_page_fetcher.rb

Constant Summary collapse

HTML_CONTENT_TYPES = 
%w[text/html application/xhtml+xml].freeze

Instance Method Summary collapse

Constructor Details

#initialize(client:, seed_host:) ⇒ AsyncPageFetcher

Returns a new instance of AsyncPageFetcher.



8
9
10
11
 
# File 'lib/nous/crawler/async_page_fetcher.rb', line 8

def initialize(client:, seed_host:)
  @client = client
  @seed_host = seed_host
end

Instance Method Details

#fetch(url) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/nous/crawler/async_page_fetcher.rb', line 13

def fetch(url)
  Async::Task.current.with_timeout(config.timeout) do
    result = RedirectFollower.call(client:, seed_host:, url:)
    return build_failed_record(url, result.error.message) if result.failure?

    response, final_url = result.payload
    content_type = response.headers["content-type"].to_s
    redirected = final_url.to_s != url

    return build_failed_record(url, "status #{response.status}") unless response.status == 200
    return build_failed_record(url, "non-html content") unless html?(content_type)

    build_success_record(
      url: url,
      final_url: final_url.to_s,
      pathname: final_url.path,
      html: response.read,
      content_type: content_type,
      redirected: redirected
    )
  ensure
    response&.close
  end
rescue Async::TimeoutError
  build_failed_record(url, "timeout after #{config.timeout}s")
rescue IOError, SocketError, Errno::ECONNREFUSED => e
  build_failed_record(url, e.message)
end