Module: Polipus

Defined in:
lib/polipus.rb,
lib/polipus/http.rb,
lib/polipus/page.rb,
lib/polipus/plugin.rb,
lib/polipus/storage.rb,
lib/polipus/version.rb,
lib/polipus/url_tracker.rb,
lib/polipus/storage/base.rb,
lib/polipus/plugins/sample.rb,
lib/polipus/queue_overflow.rb,
lib/polipus/plugins/cleaner.rb,
lib/polipus/plugins/sleeper.rb,
lib/polipus/storage/dev_null.rb,
lib/polipus/storage/s3_store.rb,
lib/polipus/queue_overflow/base.rb,
lib/polipus/storage/mongo_store.rb,
lib/polipus/url_tracker/redis_set.rb,
lib/polipus/queue_overflow/manager.rb,
lib/polipus/url_tracker/bloomfilter.rb,
lib/polipus/queue_overflow/mongo_queue.rb,
lib/polipus/queue_overflow/dev_null_queue.rb,
lib/polipus/queue_overflow/mongo_queue_capped.rb

Defined Under Namespace

Modules: Plugin, QueueOverflow, Storage, UrlTracker Classes: HTTP, Page, PolipusCrawler, PolipusSignalHandler

Constant Summary collapse

VERSION = 
"0.1.0"
HOMEPAGE = 
"https://github.com/taganaka/polipus"

Class Method Summary collapse

Class Method Details

.crawler(job_name = 'polipus', urls = [], options = {}, &block) ⇒ Object 



18
19
20
 
# File 'lib/polipus.rb', line 18

def Polipus.crawler(job_name = 'polipus', urls = [], options = {}, &block)
  PolipusCrawler.crawl(job_name, urls, options, &block)
end