Class: Miteru::Crawler
- Inherits:
-
Object
- Object
- Miteru::Crawler
- Defined in:
- lib/miteru/crawler.rb
Instance Attribute Summary collapse
-
#downloader ⇒ Object
readonly
Returns the value of attribute downloader.
-
#feeds ⇒ Object
readonly
Returns the value of attribute feeds.
Class Method Summary collapse
Instance Method Summary collapse
- #auto_download? ⇒ Boolean
- #crawl(url) ⇒ Object
- #execute ⇒ Object
-
#initialize ⇒ Crawler
constructor
A new instance of Crawler.
- #notify(website) ⇒ Object
- #verbose? ⇒ Boolean
Constructor Details
#initialize ⇒ Crawler
Returns a new instance of Crawler.
12 13 14 15 16 17 |
# File 'lib/miteru/crawler.rb', line 12 def initialize @downloader = Downloader.new(Miteru.configuration.download_to) @feeds = Feeds.new @notifier = Notifier.new end |
Instance Attribute Details
#downloader ⇒ Object (readonly)
Returns the value of attribute downloader.
9 10 11 |
# File 'lib/miteru/crawler.rb', line 9 def downloader @downloader end |
#feeds ⇒ Object (readonly)
Returns the value of attribute feeds.
10 11 12 |
# File 'lib/miteru/crawler.rb', line 10 def feeds @feeds end |
Class Method Details
.execute ⇒ Object
37 38 39 |
# File 'lib/miteru/crawler.rb', line 37 def self.execute new.execute end |
Instance Method Details
#auto_download? ⇒ Boolean
45 46 47 |
# File 'lib/miteru/crawler.rb', line 45 def auto_download? Miteru.configuration.auto_download? end |
#crawl(url) ⇒ Object
19 20 21 22 23 24 25 |
# File 'lib/miteru/crawler.rb', line 19 def crawl(url) website = Website.new(url) downloader.download_kits(website.kits) if website.has_kits? && auto_download? notify(website) if website.has_kits? || verbose? rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e nil end |
#execute ⇒ Object
27 28 29 30 31 32 33 34 35 |
# File 'lib/miteru/crawler.rb', line 27 def execute threads = Miteru.configuration.threads suspicious_urls = feeds.suspicious_urls puts "Loaded #{suspicious_urls.length} URLs to crawl. (crawling in #{threads} threads)" if verbose? Parallel.each(suspicious_urls, in_threads: threads) do |url| crawl url end end |
#notify(website) ⇒ Object
41 42 43 |
# File 'lib/miteru/crawler.rb', line 41 def notify(website) @notifier.notify(url: website.url, kits: website.kits, message: website.) end |
#verbose? ⇒ Boolean
49 50 51 |
# File 'lib/miteru/crawler.rb', line 49 def verbose? Miteru.configuration.verbose? end |