Class: Wayfarer::Crawl
- Inherits:
-
Object
- Object
- Wayfarer::Crawl
- Extended by:
- Forwardable
- Includes:
- Observable
- Defined in:
- lib/wayfarer/crawl.rb
Instance Attribute Summary collapse
-
#dispatcher ⇒ Object
readonly
Returns the value of attribute dispatcher.
-
#job ⇒ Object
readonly
The prepared job.
Instance Method Summary collapse
- #execute ⇒ Object
-
#frontier ⇒ Frontier
A frontier with initially pre-staged URIs.
-
#initialize(job, *uris) ⇒ Crawl
constructor
A new instance of Crawl.
Constructor Details
#initialize(job, *uris) ⇒ Crawl
Returns a new instance of Crawl.
21 22 23 24 25 26 |
# File 'lib/wayfarer/crawl.rb', line 21 def initialize(job, *uris) @job = job.prepare @uris = uris @dispatcher = Dispatcher.new(@job) @processor = Processor.new(@job, frontier, @dispatcher) end |
Instance Attribute Details
#dispatcher ⇒ Object (readonly)
Returns the value of attribute dispatcher.
16 17 18 |
# File 'lib/wayfarer/crawl.rb', line 16 def dispatcher @dispatcher end |
#job ⇒ Object (readonly)
The prepared job.
13 14 15 |
# File 'lib/wayfarer/crawl.rb', line 13 def job @job end |
Instance Method Details
#execute ⇒ Object
28 29 30 31 32 33 34 35 36 37 38 |
# File 'lib/wayfarer/crawl.rb', line 28 def execute trap_signals CrawlObserver.new(@processor, @dispatcher, config.logger) @job.run_hook(:before_crawl) @processor.run(*@uris) @job.run_hook(:after_crawl) ensure untrap_signals end |
#frontier ⇒ Frontier
A frontier with initially pre-staged URIs.
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
# File 'lib/wayfarer/crawl.rb', line 42 def frontier return @frontier if @frontier @frontier = case config.frontier when :memory_trie Frontiers::MemoryTrieFrontier.new(config) when :redis Frontiers::RedisFrontier.new(config) when :memory_bloom Frontiers::MemoryBloomfilter.new(config) when :redis_bloom Frontiers::RedisBloomfilter.new(config) else Frontiers::MemoryFrontier.new(config) end @frontier.extend(Frontiers::NormalizeURIs) if config.normalize_uris @frontier.stage(*@uris) # TODO: Test @frontier end |