Class: Wmap::DomainTracker

Inherits:
Object
  • Object
show all
Includes:
Singleton, Utils
Defined in:
lib/wmap/domain_tracker.rb,
lib/wmap/domain_tracker/sub_domain.rb

Overview

Class to track the known (trusted) Internet domains

Direct Known Subclasses

SubDomain

Defined Under Namespace

Classes: SubDomain

Constant Summary

Constants included from Utils::UrlMagic

Utils::UrlMagic::Max_http_timeout

Constants included from Utils::DomainRoot

Utils::DomainRoot::File_ccsld, Utils::DomainRoot::File_cctld, Utils::DomainRoot::File_gtld, Utils::DomainRoot::File_tld

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Utils

#cidr_2_ips, #file_2_hash, #file_2_list, #get_nameserver, #get_nameservers, #host_2_ip, #host_2_ips, #is_cidr?, #is_fqdn?, #is_ip?, #list_2_file, #reverse_dns_lookup, #sort_ips, #valid_dns_record?, #zone_transferable?

Methods included from Utils::Logger

#wlog

Methods included from Utils::UrlMagic

#create_absolute_url_from_base, #create_absolute_url_from_context, #host_2_url, #is_site?, #is_ssl?, #is_url?, #landing_location, #make_absolute, #normalize_url, #open_page, #redirect_location, #response_code, #url_2_host, #url_2_path, #url_2_port, #url_2_site, #urls_on_same_domain?

Methods included from Utils::DomainRoot

#get_domain_root, #get_domain_root_by_ccsld, #get_domain_root_by_cctld, #get_domain_root_by_tlds, #get_sub_domain, #is_domain_root?, #print_ccsld, #print_cctld, #print_gtld

Constructor Details

#initialize(params = {}) ⇒ DomainTracker

Set default instance variables



22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/wmap/domain_tracker.rb', line 22

def initialize (params = {})
	# Initialize the instance variables
	@verbose=params.fetch(:verbose, false)
	@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
	Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
	@file_domains=params.fetch(:domains_file, @data_dir+'domains')
	@max_parallel=params.fetch(:max_parallel, 40)
	# Hash table to hold the trusted domains
	File.write(@file_domains, "") unless File.exist?(@file_domains)
	@known_internet_domains=load_domains_from_file(@file_domains)
	#@known_internet_sub_domains=Hash.new
end

Instance Attribute Details

#data_dir ⇒ Object

Returns the value of attribute data_dir.



18
19
20
 
# File 'lib/wmap/domain_tracker.rb', line 18

def data_dir
  @data_dir
end

#domains_file ⇒ Object

Returns the value of attribute domains_file.



18
19
20
 
# File 'lib/wmap/domain_tracker.rb', line 18

def domains_file
  @domains_file
end

#file_domains ⇒ Object

Returns the value of attribute file_domains.



18
19
20
 
# File 'lib/wmap/domain_tracker.rb', line 18

def file_domains
  @file_domains
end

#known_internet_domains ⇒ Object (readonly)

Returns the value of attribute known_internet_domains.



19
20
21
 
# File 'lib/wmap/domain_tracker.rb', line 19

def known_internet_domains
  @known_internet_domains
end

#max_parallel ⇒ Object

Returns the value of attribute max_parallel.



18
19
20
 
# File 'lib/wmap/domain_tracker.rb', line 18

def max_parallel
  @max_parallel
end

#verbose ⇒ Object

Returns the value of attribute verbose.



18
19
20
 
# File 'lib/wmap/domain_tracker.rb', line 18

def verbose
  @verbose
end

Instance Method Details

#add(host) ⇒ Object

‘setter’ to add domain entry to the cache one at a time



110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/wmap/domain_tracker.rb', line 110

def add(host)
	begin
		puts "Add entry to the local domains cache table: #{host}" if @verbose
		return nil if host.nil? or host.empty?
		host=host.strip.downcase
		if @known_internet_domains.key?(host)
			puts "Domain is already exist. Skipping: #{host}"
		else
			root=get_domain_root(host)
			sub=get_subdomain(host)
			record=Hash.new
			if host == root
				if zone_transferable?(root)
					record[root]=true
					#@known_internet_domains[root]=true
				else
					record[root]=false
					#@known_internet_domains[root]=false
				end
				puts "Entry loaded: #{record}"
				@known_internet_domains.merge!(record)
				return record
			elsif sub.nil?				# 2/10/2014, additional logic to support sub-domains
				# do nothing
			elsif host != sub
				if zone_transferable?(sub)
					#@known_internet_domains[sub]=true
					record[sub]=true
				else
					#@known_internet_domains[sub]=false
					record[sub]=false
				end
				puts "Entry loaded: #{record}"
				@known_internet_domains.merge!(record)
				return record
			else
				puts "Problem add domain #{host}: unknown domain format - please use legal root domain or sub domain only."
			end
		end
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
	end
end

#bulk_add(list, num = @max_parallel) ⇒ Object Also known as: adds

‘setter’ to add domain entry to the cache in batch (from a list)



168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/wmap/domain_tracker.rb', line 168

def bulk_add(list, num=@max_parallel)
	puts "Add entries to the local domains cache table from list: #{list}" if @verbose
	begin
		results=Hash.new
		domains=list
		if domains.size > 0
			Parallel.map(list, :in_processes => num) { |target|
				add(target)
			}.each do |process|
				if process.nil?
					next
				elsif process.empty?
					#do nothing
				else
					results.merge!(process)
				end
			end
			@known_internet_domains.merge!(results)
			puts "Done loading entries."
			return results
		else
			puts "Error: no entry is loaded. Please check your list and try again."
		end
		return results
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
	end
end

#bulk_delete(list) ⇒ Object Also known as: dels

‘setter’ to delete domain entry to the cache in batch (from a list)



216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/wmap/domain_tracker.rb', line 216

def bulk_delete(list)
	puts "Delete entries to the local domains cache table from list: #{list}" if @verbose
	begin
		domains=list
		changes=Array.new
		if domains.size > 0
			domains.map do |x|
				domain=delete(x)
				changes.push(domain) unless domain.nil?
			end
			puts "Done deleting domains from list: #{list}"
			return changes
		else
			puts "Exception on method bulk_delete: no entry is loaded. Please check your list and try again."
		end
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
	end
end

#count ⇒ Object Also known as: size

Count numbers of entries in the domain cache table



92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/wmap/domain_tracker.rb', line 92

def count
	puts "Counting number of entries in the domain cache table ..."
	begin
		cnt=0
		@known_internet_domains.map do |key|
			unless key =~ /\w+\.\w+/
				cnt=cnt+1
			end
		end
		puts "Current number of entries: #{cnt}"
		return cnt
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
	end
end

#delete(domain) ⇒ Object

‘setter’ to remove entry from the cache one at a time



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
 
# File 'lib/wmap/domain_tracker.rb', line 199

def delete(domain)
	puts "Remove entry from the domains cache table: #{domain} " if @verbose
	begin
		domain=domain.strip.downcase
		if @known_internet_domains.key?(domain)
			@known_internet_domains.delete(domain)
			puts "Entry cleared: #{domain}"
			return domain
		else
			puts "Entry not fund. Skipping: #{domain}"
		end
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
	end
end

#delete_all ⇒ Object

‘setter’ to remove all entries from the store



250
251
252
253
254
255
256
257
258
259
 
# File 'lib/wmap/domain_tracker.rb', line 250

def delete_all
	puts "Delete all entries in the domain store! " if @verbose
	begin
		@known_internet_domains.keys.map do |domain|
			delete(domain)
		end
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
	end
end

#domain_known?(domain) ⇒ Boolean Also known as: is_known?, is_domain_known?

Simple method to check if a domain is already within the domain cache table

Returns:

  • (Boolean) 


281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
 
# File 'lib/wmap/domain_tracker.rb', line 281

def domain_known?(domain)
	begin
		#abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil? or @known_internet_sub_domains.nil?
		domain=domain.strip.downcase unless domain.nil?
		case self.class.name
		when "Wmap::DomainTracker"
			return @known_internet_domains.key?(domain)
		when "Wmap::DomainTracker::SubDomain"
			return @known_internet_sub_domains.key?(domain)
		else
			return nil
		end
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
	end
	return false
end

#file_add(file) ⇒ Object

‘setter’ to add domain entry to the cache in batch (from a file)



155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/wmap/domain_tracker.rb', line 155

def file_add(file)
	begin
		puts "Add entries to the local domains cache table from file: #{file}" if @verbose
		raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
		changes=Array.new
		domains=file_2_list(file)
		changes=bulk_add(domains)
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
	end
end

#file_delete(file) ⇒ Object

‘setter’ to delete domain entry to the cache in batch (from a file)



238
239
240
241
242
243
244
245
246
247
 
# File 'lib/wmap/domain_tracker.rb', line 238

def file_delete(file)
	begin
		puts "Delete entries to the local domains cache table from file: #{file}" if @verbose
		raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
		domains=file_2_list(file)
		changes=bulk_delete(domains)
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
	end
end

#get_domains ⇒ Object Also known as: dump_domains, dump

Dump out the list of known domains



302
303
304
305
306
307
308
309
310
 
# File 'lib/wmap/domain_tracker.rb', line 302

def get_domains
	puts "Retrieve a list of known domain ..." if @verbose
	begin
		return @known_internet_domains.keys
	rescue Exception => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
		return nil
	end
end

#print_known_domains ⇒ Object Also known as: print

Print summary report on all known / trust domains in the domain cache table



334
335
336
337
338
339
340
 
# File 'lib/wmap/domain_tracker.rb', line 334

def print_known_domains
	puts "\nSummary of known Internet Domains:"
	@known_internet_domains.keys.sort.each do |domain|
		puts domain
	end
	puts "End of the summary"
end

#print_known_sub_domains ⇒ Object

Print summary report on all known / trust domains in the domain cache table



136
137
138
139
140
141
142
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 136

def print_known_sub_domains
	puts "\nSummary of known Internet Sub-domains:"
	self.known_internet_sub_domains.keys.sort.each do |domain|
		puts domain
	end
	puts "End of the summary"
end

#refresh(domain) ⇒ Object

Refresh the domain entry one at a time



262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
 
# File 'lib/wmap/domain_tracker.rb', line 262

def refresh(domain)
	begin
		abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil?
		domain=domain.strip.downcase unless domain.nil?
		if domain_known?(domain)
			delete(domain)
			add(domain)
			return domain
		else
			puts "Unknown domain: #{domain}"
			return nil
		end
	rescue => ee
		puts "Exception on method #{__method__} for #{domain}: #{ee}" if @verbose
		return nil
	end
end

#save_domains_to_file!(file_domains = @file_domains, domains = @known_internet_domains) ⇒ Object Also known as: save!

Save the current domain hash table into a file



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/wmap/domain_tracker.rb', line 69

def save_domains_to_file!(file_domains=@file_domains, domains=@known_internet_domains)
	puts "Saving the current domains cache table from memory to file: #{file_domains} ..." if @verbose
	begin
		timestamp=Time.now
		f=File.open(file_domains, 'w')
		f.write "# Local domains file created by class #{self.class} method #{__method__} at: #{timestamp}\n"
		f.write "# domain name, free zone transfer detected?\n"
		domains.keys.sort.map do |key|
			if domains[key]
				f.write "#{key}, yes\n"
			else
				f.write "#{key}, no\n"
			end
		end
		f.close
		puts "Domain cache table is successfully saved: #{file_domains}"
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
	end
end

#search(pattern) ⇒ Object Also known as: find

Search potential matching domains from the domain store by using simple regular expression. Note that any upper-case char in the search string will be automatically converted into lower case



315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
 
# File 'lib/wmap/domain_tracker.rb', line 315

def search (pattern)
	puts "Search domain store for the regular expression: #{pattern}" if @verbose
	begin
		pattern=pattern.strip.downcase
		results=Array.new
		@known_internet_domains.keys.map do |key|
			if key =~ /#{pattern}/i
				results.push(key)
			end
		end
		return results
	rescue Exception => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
		return nil
	end
end