Module: DaimonSkycrawlers::Crawler

Defined in:
lib/daimon_skycrawlers/crawler.rb,
lib/daimon_skycrawlers/crawler/base.rb,
lib/daimon_skycrawlers/crawler/default.rb

Defined Under Namespace

Classes: Base, Default

Class Method Summary collapse

Class Method Details

.config ⇒ DaimonSkycrawlers::Configuration

Shortcut of DaimonSkycrawlers.configuration



38
39
40
 
# File 'lib/daimon_skycrawlers/crawler.rb', line 38

def config
  DaimonSkycrawlers.configuration
end

.default_process_name ⇒ String

Default process name



56
57
58
 
# File 'lib/daimon_skycrawlers/crawler.rb', line 56

def default_process_name
  "#{config.queue_name_prefix}:url"
end

.enqueue_url(url, message = {}) ⇒ void

This method returns an undefined value.

Enqueue a URL to crawler queue



27
28
29
30
31
 
# File 'lib/daimon_skycrawlers/crawler.rb', line 27

def enqueue_url(url, message = {})
  message[:url] = url
  config.logger.debug("#{queue_name}: #{url}")
  SongkickQueue.publish(queue_name, message)
end

.queue_name ⇒ String

Queue name for crawler



47
48
49
 
# File 'lib/daimon_skycrawlers/crawler.rb', line 47

def queue_name
  "#{config.queue_name_prefix}.url"
end

.run(process_name: default_process_name) ⇒ Object

Run registered crawlers



14
15
16
17
18
19
 
# File 'lib/daimon_skycrawlers/crawler.rb', line 14

def run(process_name: default_process_name)
  if config.shutdown_interval > 0
    DaimonSkycrawlers::Timer.setup_shutdown_timer(config.queue_name_prefix, interval: config.shutdown_interval)
  end
  SongkickQueue::Worker.new(process_name, [DaimonSkycrawlers::Consumer::URL]).run
end