Class: DaimonSkycrawlers::Commands::Enqueue

Inherits:
Thor
  • Object
show all
Defined in:
lib/daimon_skycrawlers/commands/enqueue.rb

Instance Method Summary collapse

Instance Method Details

#response(url, *rest) ⇒ Object 



20
21
22
23
24
25
 
# File 'lib/daimon_skycrawlers/commands/enqueue.rb', line 20

def response(url, *rest)
  load_init
  message = rest.map {|arg| arg.split(":") }.to_h
  log.debug("Enqueue URL for processor: #{url} : #{message}")
  DaimonSkycrawlers::Processor.enqueue_http_response(url, message)
end

#sitemap(url) ⇒ Object 



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/daimon_skycrawlers/commands/enqueue.rb', line 31

def sitemap(url)
  load_init
  if options["robots-txt"]
    webrobots = WebRobots.new("DaimonSkycrawlers/#{DaimonSkycrawlers::VERSION}")
    sitemaps = webrobots.sitemaps(url).uniq
  else
    sitemaps = [url]
  end
  sitemap_parser = DaimonSkycrawlers::SitemapParser.new(sitemaps)
  urls = sitemap_parser.parse
  if options["dump"]
    puts urls.join("\n")
    return
  end
  urls.each do |_url|
    DaimonSkycrawlers::Crawler.enqueue_url(_url)
  end
end

#url(url, *rest) ⇒ Object 



12
13
14
15
16
17
 
# File 'lib/daimon_skycrawlers/commands/enqueue.rb', line 12

def url(url, *rest)
  load_init
  message = rest.map {|arg| arg.split(":") }.to_h
  log.debug("Enqueue URL for crawler: #{url} : #{message}")
  DaimonSkycrawlers::Crawler.enqueue_url(url, message)
end