Method: Arachnid2::Typhoeus#crawl

Defined in:
lib/arachnid2/typhoeus.rb

#crawl(opts = {}) ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/arachnid2/typhoeus.rb', line 12

def crawl(opts = {})
  preflight(opts)
  typhoeus_preflight

  until @global_queue.empty?
    max_concurrency.times do
      q = @global_queue.shift

      break if time_to_stop?
      @global_visited.insert(q)

      found_in_cache = use_cache(q, opts, &Proc.new)
      return if found_in_cache

      request = ::Typhoeus::Request.new(q, request_options)
      requestable = after_request(request, &Proc.new)
      @hydra.queue(request) if requestable
    end # max_concurrency.times do

    @hydra.run
  end # until @global_queue.empty?
ensure
  @cookie_file.close! if @cookie_file
end