Module: Outlander::Crawler

Defined in:
lib/outlander/crawler.rb

Defined Under Namespace

Modules: ClassMethods

Constant Summary collapse

DEFAULT_OPTIONS = 
{
  num_threads: 3
}

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.included(base) ⇒ Object 



15
16
17
18
19
20
21
22
 
# File 'lib/outlander/crawler.rb', line 15

def included(base)
  base.extend ClassMethods

  base.class_eval do
    @roots = {}
    @handlers = {}
  end
end

Instance Method Details

#initialize(options = {}) ⇒ Object 



46
47
48
49
50
51
52
53
54
55
 
# File 'lib/outlander/crawler.rb', line 46

def initialize(options = {})
  agent.cache_storage = options.delete(:cache_storage)
  @logger = Logger.new(options.fetch(:log_to, STDOUT))
  @options = options.merge DEFAULT_OPTIONS
  @history = {}
  @pool = ThreadsPool.new @options[:num_threads]
  self.class.roots.each do |url, handler|
    enqueue url, handler
  end
end

#run!(&block) ⇒ Object 



57
58
59
60
61
 
# File 'lib/outlander/crawler.rb', line 57

def run!(&block)
  @result_handler = block
  instance_eval &self.class.setup
  @pool.start
end