Module: Polipus

Defined in:
lib/polipus.rb,
lib/polipus/http.rb,
lib/polipus/page.rb,
lib/polipus/plugin.rb,
lib/polipus/robotex.rb,
lib/polipus/storage.rb,
lib/polipus/version.rb,
lib/polipus/url_tracker.rb,
lib/polipus/storage/base.rb,
lib/polipus/plugins/sample.rb,
lib/polipus/queue_overflow.rb,
lib/polipus/signal_handler.rb,
lib/polipus/plugins/cleaner.rb,
lib/polipus/plugins/sleeper.rb,
lib/polipus/storage/dev_null.rb,
lib/polipus/queue_overflow/base.rb,
lib/polipus/storage/mongo_store.rb,
lib/polipus/storage/memory_store.rb,
lib/polipus/queue_overflow/worker.rb,
lib/polipus/storage/rethink_store.rb,
lib/polipus/url_tracker/redis_set.rb,
lib/polipus/queue_overflow/manager.rb,
lib/polipus/url_tracker/bloomfilter.rb,
lib/polipus/queue_overflow/mongo_queue.rb,
lib/polipus/queue_overflow/dev_null_queue.rb,
lib/polipus/queue_overflow/mongo_queue_capped.rb

Defined Under Namespace

Modules: Plugin, QueueOverflow, Storage, UrlTracker Classes: HTTP, Page, PolipusCrawler, Robotex, SignalHandler

Constant Summary collapse

VERSION = 
'0.5.1'
HOMEPAGE = 
'https://github.com/taganaka/polipus'

Class Method Summary collapse

Class Method Details

.crawler(job_name = 'polipus', urls = [], options = {}, &block) ⇒ Object 



18
19
20
 
# File 'lib/polipus.rb', line 18

def self.crawler(job_name = 'polipus', urls = [], options = {}, &block)
  PolipusCrawler.crawl(job_name, urls, options, &block)
end