Module: RCrawler

Defined in:
lib/rcrawler.rb,
lib/rcrawler/cli.rb,
lib/rcrawler/async.rb,
lib/rcrawler/crawl.rb,
lib/rcrawler/driver.rb,
lib/rcrawler/version.rb,
lib/rcrawler/configuration.rb

Defined Under Namespace

Classes: Async, CLI, Configuration, Crawl, Driver

Constant Summary collapse

VERSION = 
"0.0.3"

Class Method Summary collapse

Class Method Details

.async(&block) ⇒ Object 



38
39
40
41
42
 
# File 'lib/rcrawler.rb', line 38

def async(&block)
  async_threads = Async.new
  async_threads.instance_eval &block
  async_threads.execute
end

.configure(&block) ⇒ Object 



30
31
32
33
34
35
36
 
# File 'lib/rcrawler.rb', line 30

def configure(&block)
  if block_given?
    @config.configure &block
  else
    @config
  end
end

.crawl(&block) ⇒ Object 



22
23
24
25
26
27
28
 
# File 'lib/rcrawler.rb', line 22

def crawl(&block)
  begin
    Timeout::timeout(@config.timeout) {Crawl.new.instance_eval &block}
  rescue Timeout::Error => e
    raise if @config.timeout_proc == :raise
  end
end