Class: Wmap::DomainTracker::SubDomain

Inherits:
Wmap::DomainTracker show all
Includes:
Singleton, Utils
Defined in:
lib/wmap/domain_tracker/sub_domain.rb

Overview

Class to differentiate the sub-domain from the top domain for the enterprise. This is needed for better managing of the sub-domains and the associated entities

Constant Summary

Constants included from Utils::UrlMagic

Utils::UrlMagic::Max_http_timeout

Constants included from Utils::DomainRoot

Utils::DomainRoot::File_ccsld, Utils::DomainRoot::File_cctld, Utils::DomainRoot::File_gtld, Utils::DomainRoot::File_tld

Instance Attribute Summary collapse

Attributes inherited from Wmap::DomainTracker

#file_domains, #known_internet_domains

Instance Method Summary collapse

Methods included from Utils

#cidr_2_ips, #file_2_hash, #file_2_list, #get_nameserver, #get_nameservers, #host_2_ip, #host_2_ips, #is_cidr?, #is_fqdn?, #is_ip?, #list_2_file, #reverse_dns_lookup, #sort_ips, #valid_dns_record?, #zone_transferable?

Methods included from Utils::Logger

#wlog

Methods included from Utils::UrlMagic

#create_absolute_url_from_base, #create_absolute_url_from_context, #host_2_url, #is_site?, #is_ssl?, #is_url?, #landing_location, #make_absolute, #normalize_url, #open_page, #redirect_location, #response_code, #url_2_host, #url_2_path, #url_2_port, #url_2_site, #urls_on_same_domain?

Methods included from Utils::DomainRoot

#get_domain_root, #get_domain_root_by_ccsld, #get_domain_root_by_cctld, #get_domain_root_by_tlds, #get_sub_domain, #is_domain_root?, #print_ccsld, #print_cctld, #print_gtld

Methods inherited from Wmap::DomainTracker

#bulk_delete, #count, #delete, #delete_all, #domain_known?, #file_add, #file_delete, #get_domains, #print_known_domains, #print_known_sub_domains, #refresh, #save_domains_to_file!, #search

Constructor Details

#initialize(params = {}) ⇒ SubDomain

Set default instance variables



24
25
26
27
28
29
30
31
32
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 24

def initialize (params = {})
@verbose=params.fetch(:verbose, false)
@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
@max_parallel=params.fetch(:max_parallel, 40)
# Hash table to hold the trusted domains
@file_sub_domains=@data_dir + 'sub_domains'
File.write(@file_sub_domains, "") unless File.exist?(@file_sub_domains)
@known_internet_sub_domains=load_domains_from_file(@file_sub_domains) #unless @known_internet_sub_domains.size>0
end

Instance Attribute Details

#data_dir ⇒ Object

Returns the value of attribute data_dir.



20
21
22
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 20

def data_dir
  @data_dir
end

#domains_file ⇒ Object

Returns the value of attribute domains_file.



20
21
22
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 20

def domains_file
  @domains_file
end

#known_internet_sub_domains ⇒ Object (readonly)

Returns the value of attribute known_internet_sub_domains.



21
22
23
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 21

def known_internet_sub_domains
  @known_internet_sub_domains
end

#max_parallel ⇒ Object

Returns the value of attribute max_parallel.



20
21
22
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 20

def max_parallel
  @max_parallel
end

#verbose ⇒ Object

Returns the value of attribute verbose.



20
21
22
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 20

def verbose
  @verbose
end

Instance Method Details

#add(sub) ⇒ Object

‘setter’ to add sub-domain entry to the cache one at a time



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 35

def add(sub)
	puts "Add entry to the local sub domain cache table: #{sub}" if @verbose
	begin
		record=Hash.new
		sub=sub.strip.downcase
		if @known_internet_sub_domains.key?(sub)
			puts "Skip on known sub-domain: #{sub}" if @verbose
			return nil
		end
		if zone_transferable?(sub)
			record[sub]=true
		else
			record[sub]=false
		end
		puts "Adding new record into the data store: #{record}" if @verbose
		@known_internet_sub_domains.merge!(record)
		return record
	rescue => ee
		puts "Exception on method #{__method__} for #{sub}: #{ee}" if @verbose
	end
end

#bulk_add(list, num = @max_parallel) ⇒ Object Also known as: adds

‘setter’ to add domain entry to the cache in batch (from a list)



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 58

def bulk_add(list, num=@max_parallel)
	puts "Add entries to the local domains cache table from list: #{list}" if @verbose
	begin
		results=Hash.new
		domains=list
		if domains.size > 0
			Parallel.map(list, :in_processes => num) { |target|
				add(target)
			}.each do |process|
				if process.nil?
					next
				elsif process.empty?
					#do nothing
				else
					results.merge!(process)
				end
			end
			@known_internet_sub_domains.merge!(results)
			puts "Done loading entries."
			return results
		else
			puts "Error: no entry is loaded. Please check your list and try again."
		end
		return results
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
	end
end

#save_sub_domains_to_file!(file_domains = @file_sub_domains, domains = @known_internet_sub_domains) ⇒ Object Also known as: save!

Save the current domain hash table into a file



112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 112

def save_sub_domains_to_file!(file_domains=@file_sub_domains, domains=@known_internet_sub_domains)
	puts "Saving the current domains cache table from memory to file: #{file_domains} ..." if @verbose
	begin
		timestamp=Time.now
		f=File.open(file_domains, 'w')
		f.write "# Local domains file created by class #{self.class} method #{__method__} at: #{timestamp}\n"
		f.write "# domain name, free zone transfer detected?\n"
		domains.keys.sort.map do |key|
			if domains[key]
				f.write "#{key}, yes\n"
			else
				f.write "#{key}, no\n"
			end
		end
		f.close
		puts "Domain cache table is successfully saved: #{file_domains}"
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
	end
end

#update_from_host_store! ⇒ Object Also known as: update!

Procedures to identify sub-domain from the hosts store



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 89

def update_from_host_store!
	puts "Invoke internal procedures to update the sub-domain list from the host store."
	begin
		# Step 1 - obtain the latest sub-domains
		my_tracker = Wmap::HostTracker.instance
		my_tracker.data_dir=@data_dir
		subs = my_tracker.dump_sub_domains - [nil,""]
		my_tracker = nil 
		# Step 2 - update the sub-domain list
		unless subs.empty?
			#subs.map { |x| self.add(x) unless domain_known?(x) }
			self.bulk_add(subs,@max_parallel)
		end
		puts "Update discovered sub-domains into the store: #{@known_internet_sub_domains}"
		self.save!(file_domains=@file_sub_domains, domains=@known_internet_sub_domains)
	rescue Exception => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
		return nil
	end
end