Class: Wayfarer::Crawl

Inherits:
Object
  • Object
show all
Extended by:
Forwardable
Includes:
Observable
Defined in:
lib/wayfarer/crawl.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(job, *uris) ⇒ Crawl

Returns a new instance of Crawl.


21
22
23
24
25
26
# File 'lib/wayfarer/crawl.rb', line 21

def initialize(job, *uris)
  @job = job.prepare
  @uris = uris
  @dispatcher = Dispatcher.new(@job)
  @processor = Processor.new(@job, frontier, @dispatcher)
end

Instance Attribute Details

#dispatcherObject (readonly)

Returns the value of attribute dispatcher


16
17
18
# File 'lib/wayfarer/crawl.rb', line 16

def dispatcher
  @dispatcher
end

#jobObject (readonly)

The prepared job.


13
14
15
# File 'lib/wayfarer/crawl.rb', line 13

def job
  @job
end

Instance Method Details

#executeObject


28
29
30
31
32
33
34
35
36
37
38
# File 'lib/wayfarer/crawl.rb', line 28

def execute
  trap_signals

  CrawlObserver.new(@processor, @dispatcher, config.logger)

  @job.run_hook(:before_crawl)
  @processor.run(*@uris)
  @job.run_hook(:after_crawl)
ensure
  untrap_signals
end

#frontierFrontier

A frontier with initially pre-staged URIs.

Returns:

  • (Frontier)

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
# File 'lib/wayfarer/crawl.rb', line 42

def frontier
  return @frontier if @frontier

  @frontier = case config.frontier
              when :memory_trie
                Frontiers::MemoryTrieFrontier.new(config)
              when :redis
                Frontiers::RedisFrontier.new(config)
              when :memory_bloom
                Frontiers::MemoryBloomfilter.new(config)
              when :redis_bloom
                Frontiers::RedisBloomfilter.new(config)
              else
                Frontiers::MemoryFrontier.new(config)
              end

  @frontier.extend(Frontiers::NormalizeURIs) if config.normalize_uris

  @frontier.stage(*@uris) # TODO: Test

  @frontier
end