Module: DomainsScanner

Defined in:
lib/domains_scanner.rb,
lib/domains_scanner/runner.rb,
lib/domains_scanner/printer.rb,
lib/domains_scanner/results.rb,
lib/domains_scanner/version.rb,
lib/domains_scanner/crawlers.rb,
lib/domains_scanner/result_item.rb,
lib/domains_scanner/crawlers/base.rb,
lib/domains_scanner/crawlers/baidu.rb,
lib/domains_scanner/crawlers/google.rb

Defined Under Namespace

Modules: Crawlers Classes: Printer, ResultItem, Results, Runner

Constant Summary collapse

TOP_LEVEL_DOMAINS = 
%w(com cn com.cn net org ltd cc mobi live io co me hk).freeze
VERSION = 
"0.0.3"

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.engines ⇒ Object

Returns the value of attribute engines.



13
14
15
 
# File 'lib/domains_scanner.rb', line 13

def engines
  @engines
end

.max_page ⇒ Object

Returns the value of attribute max_page.



13
14
15
 
# File 'lib/domains_scanner.rb', line 13

def max_page
  @max_page
end

.verbose ⇒ Object

Returns the value of attribute verbose.



13
14
15
 
# File 'lib/domains_scanner.rb', line 13

def verbose
  @verbose
end

Class Method Details

.analysis_output_queue ⇒ Object 



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/domains_scanner.rb', line 34

def analysis_output_queue
  list = {}
  begin
    while(result = output_queue.pop(non_block = true)) do
      domain, top_level_domain, engine = result.values_at(:domain, :top_level_domain, :engine)
      next unless domain

      group = list[top_level_domain] ||= {}
      group[domain] ||= []

      group[domain] << engine unless group[domain].include?(engine)
    end
  rescue ThreadError # queue is empty
  end

  list
end

.output_queue ⇒ Object 



18
19
20
 
# File 'lib/domains_scanner.rb', line 18

def output_queue
  @output_queue ||= Queue.new
end

.scan(domain_word:, top_level_domains: TOP_LEVEL_DOMAINS) ⇒ Object 



22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/domains_scanner.rb', line 22

def scan(domain_word:, top_level_domains: TOP_LEVEL_DOMAINS)
  top_level_domains ||= TOP_LEVEL_DOMAINS

  runners = top_level_domains.map do |top_level_domain|
    Runner.new(domain_word: domain_word, top_level_domain: top_level_domain)
  end
  runners.each(&:run)

  DomainsScanner::Printer.puts { "Start analysis crawl results...".green }
  analysis_output_queue
end