Class: Miteru::Crawler

Inherits:
Object
  • Object
show all
Defined in:
lib/miteru/crawler.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Crawler

Returns a new instance of Crawler.



11
12
13
14
 
# File 'lib/miteru/crawler.rb', line 11

def initialize
  @downloader = Downloader.new(Miteru.configuration.download_to)
  @feeds = Feeds.new
end

Instance Attribute Details

#downloader ⇒ Object (readonly)

Returns the value of attribute downloader.



9
10
11
 
# File 'lib/miteru/crawler.rb', line 9

def downloader
  @downloader
end

#feeds ⇒ Object (readonly)

Returns the value of attribute feeds.



9
10
11
 
# File 'lib/miteru/crawler.rb', line 9

def feeds
  @feeds
end

Class Method Details

.execute ⇒ Object 



58
59
60
 
# File 'lib/miteru/crawler.rb', line 58

def execute
  new.execute
end

Instance Method Details

#auto_download? ⇒ Boolean

Returns:

  • (Boolean) 


43
44
45
 
# File 'lib/miteru/crawler.rb', line 43

def auto_download?
  Miteru.configuration.auto_download?
end

#crawl(entry) ⇒ Object 



16
17
18
19
20
21
22
 
# File 'lib/miteru/crawler.rb', line 16

def crawl(entry)
  website = Website.new(entry.url, entry.source)
  downloader.download_kits(website.kits) if website.has_kits? && auto_download?
  notify(website) if website.has_kits? || verbose?
rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
  nil
end

#execute ⇒ Object 



24
25
26
27
28
29
30
31
 
# File 'lib/miteru/crawler.rb', line 24

def execute
  suspicious_entries = feeds.suspicious_entries
  Miteru.logger.info "Loaded #{suspicious_entries.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?

  Parallel.each(suspicious_entries, in_threads: threads) do |entry|
    crawl entry
  end
end

#notify(website) ⇒ Object 



37
38
39
40
41
 
# File 'lib/miteru/crawler.rb', line 37

def notify(website)
  Parallel.each(notifiers) do |notifier|
    notifier.notify website
  end
end

#threads ⇒ Object 



33
34
35
 
# File 'lib/miteru/crawler.rb', line 33

def threads
  @threads ||= Miteru.configuration.threads
end

#verbose? ⇒ Boolean

Returns:

  • (Boolean) 


47
48
49
 
# File 'lib/miteru/crawler.rb', line 47

def verbose?
  Miteru.configuration.verbose?
end