Class: Miteru::Orchestrator

Inherits:
Service
  • Object
show all
Defined in:
lib/miteru/orchestrator.rb

Instance Method Summary collapse

Methods inherited from Service

call, #result, result

Instance Method Details

#call ⇒ Object 



5
6
7
8
9
10
11
12
13
 
# File 'lib/miteru/orchestrator.rb', line 5

def call
  logger.info("#{non_cached_websites.length} websites loaded in total.") if verbose?

  if sidekiq?
    sidekiq_call
  else
    parallel_call
  end
end

#feeds ⇒ Array<Miteru::Feeds::Base>

Returns:



69
70
71
 
# File 'lib/miteru/orchestrator.rb', line 69

def feeds
  @feeds ||= Miteru.feeds.map(&:new)
end

#non_cached_websites ⇒ Array<Miteru::Website>

Returns:



56
57
58
59
60
61
62
63
64
 
# File 'lib/miteru/orchestrator.rb', line 56

def non_cached_websites
  @non_cached_websites ||= [].tap do |out|
    out << if cache?
      websites.reject { |website| cache.cached?(website.url) }
    else
      websites
    end
  end.flatten.uniq(&:url)
end

#parallel_call ⇒ Object 



22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/miteru/orchestrator.rb', line 22

def parallel_call
  logger.info("Use #{threads} thread(s).") if verbose?
  Parallel.each(non_cached_websites, in_threads: threads) do |website|
    logger.info("Website:#{website.truncated_url} crawling started.") if verbose?
    result = Crawler.result(website)
    if result.success?
      logger.info("Crawler:#{website.truncated_url} succeeded.")
    else
      logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
    end
  end
end

#sidekiq_call ⇒ Object 



15
16
17
18
19
20
 
# File 'lib/miteru/orchestrator.rb', line 15

def sidekiq_call
  non_cached_websites.each do |website|
    Jobs::CrawleJob.perform_async(website.url, website.source)
    logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
  end
end

#websites ⇒ Array<Miteru::Website>

Returns:



38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/miteru/orchestrator.rb', line 38

def websites
  @websites ||= [].tap do |out|
    feeds.each do |feed|
      result = feed.result
      if result.success?
        websites = result.value!
        logger.info("Feed:#{feed.source} has #{websites.length} websites.") if verbose?
        out << websites
      else
        logger.warn("Feed:#{feed.source} failed - #{result.failure}")
      end
    end
  end.flatten.uniq(&:url)
end