Method: Scruber::Core::Crawler#run

Defined in:
lib/scruber/core/crawler.rb

#run(&block) ⇒ Object

Crawling engine

Parameters:

  • block (Proc) —

    crawler body



60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/scruber/core/crawler.rb', line 60

def run(&block)
  instance_eval &block
  while @queue.has_work? do
    @fetcher.run @queue
    show_progress
    while page = @queue.fetch_downloaded do
      if @callbacks[page.page_type.to_sym]
        processed_page = process_page(page, page.page_type.to_sym)
        instance_exec page, processed_page, &(@callbacks[page.page_type.to_sym])
        page.processed! unless page.sent_to_redownload?
      end
    end
    if @on_page_error_callback
      while page = @queue.fetch_error do
        instance_exec page, &(@on_page_error_callback)
      end
    end
  end
  @on_complete_callbacks.sort_by{|c| -c[0] }.map do |(_,callback)|
    instance_exec &(callback)
  end.first
end