Class: Wmap::DomainTracker

Inherits:
Object
  • Object
show all
Includes:
Singleton, Utils
Defined in:
lib/wmap/domain_tracker.rb,
lib/wmap/domain_tracker/sub_domain.rb

Overview

Class to track the known (trusted) Internet domains

Direct Known Subclasses

SubDomain

Defined Under Namespace

Classes: SubDomain

Constant Summary

Constants included from Utils::UrlMagic

Utils::UrlMagic::Max_http_timeout, Utils::UrlMagic::User_agent

Constants included from Utils::DomainRoot

Utils::DomainRoot::File_ccsld, Utils::DomainRoot::File_cctld, Utils::DomainRoot::File_gtld, Utils::DomainRoot::File_tld

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Utils

#cidr_2_ips, #file_2_hash, #file_2_list, #get_nameserver, #get_nameservers, #host_2_ip, #host_2_ips, #is_cidr?, #is_fqdn?, #is_ip?, #list_2_file, #reverse_dns_lookup, #sort_ips, #valid_dns_record?, #zone_transferable?

Methods included from Utils::Logger

#wlog

Methods included from Utils::UrlMagic

#create_absolute_url_from_base, #create_absolute_url_from_context, #host_2_url, #is_site?, #is_ssl?, #is_url?, #landing_location, #make_absolute, #normalize_url, #open_page, #redirect_location, #response_code, #response_headers, #url_2_host, #url_2_path, #url_2_port, #url_2_site, #urls_on_same_domain?

Methods included from Utils::DomainRoot

#get_domain_root, #get_domain_root_by_ccsld, #get_domain_root_by_cctld, #get_domain_root_by_tlds, #get_sub_domain, #is_domain_root?, #print_ccsld, #print_cctld, #print_gtld

Constructor Details

#initialize(params = {}) ⇒ DomainTracker

Set default instance variables



20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/wmap/domain_tracker.rb', line 20

def initialize (params = {})
	# Initialize the instance variables
	@verbose=params.fetch(:verbose, false)
	@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
	Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
	@domains_file=params.fetch(:domains_file, @data_dir+'domains')
	@max_parallel=params.fetch(:max_parallel, 40)
	# Hash table to hold the trusted domains
	File.write(@domains_file, "") unless File.exist?(@domains_file)
	load_domains_from_file(@domains_file)
end

Instance Attribute Details

#data_dir ⇒ Object

Returns the value of attribute data_dir.



17
18
19
 
# File 'lib/wmap/domain_tracker.rb', line 17

def data_dir
  @data_dir
end

#domains_file ⇒ Object

Returns the value of attribute domains_file.



17
18
19
 
# File 'lib/wmap/domain_tracker.rb', line 17

def domains_file
  @domains_file
end

#known_internet_domains ⇒ Object

Returns the value of attribute known_internet_domains.



17
18
19
 
# File 'lib/wmap/domain_tracker.rb', line 17

def known_internet_domains
  @known_internet_domains
end

#max_parallel ⇒ Object

Returns the value of attribute max_parallel.



17
18
19
 
# File 'lib/wmap/domain_tracker.rb', line 17

def max_parallel
  @max_parallel
end

#verbose ⇒ Object

Returns the value of attribute verbose.



17
18
19
 
# File 'lib/wmap/domain_tracker.rb', line 17

def verbose
  @verbose
end

Instance Method Details

#add(host) ⇒ Object

‘setter’ to add domain entry to the cache one at a time



101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/wmap/domain_tracker.rb', line 101

def add(host)
	puts "Add entry to the local domains cache table: #{host}" if @verbose
	return nil if host.nil? or host.empty?
	host=host.strip.downcase
	if @known_internet_domains.key?(host)
		puts "Domain is already exist. Skipping: #{host}"
	else
		root=get_domain_root(host)
		sub=get_subdomain(host)
		record=Hash.new
		if host == root
			if zone_transferable?(root)
				record[root]=true
				#@known_internet_domains[root]=true
			else
				record[root]=false
				#@known_internet_domains[root]=false
			end
			puts "Entry loaded: #{record}"
			@known_internet_domains.merge!(record)
			return record
		elsif sub.nil?				# 2/10/2014, additional logic to support sub-domains
			# do nothing
		elsif host != sub
			if zone_transferable?(sub)
				#@known_internet_domains[sub]=true
				record[sub]=true
			else
				#@known_internet_domains[sub]=false
				record[sub]=false
			end
			puts "Entry loaded: #{record}"
			@known_internet_domains.merge!(record)
			return record
		else
			puts "Problem add domain #{host}: unknown domain format - please use legal root domain or sub domain only."
		end
	end
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#bulk_add(list, num = @max_parallel) ⇒ Object Also known as: adds

‘setter’ to add domain entry to the cache in batch (from a list)



155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/wmap/domain_tracker.rb', line 155

def bulk_add(list, num=@max_parallel)
	puts "Add entries to the local domains cache table from list: #{list}" if @verbose
	results=Hash.new
	domains=list
	if domains.size > 0
		Parallel.map(list, :in_processes => num) { |target|
			add(target)
		}.each do |process|
			if process.nil?
				next
			elsif process.empty?
				#do nothing
			else
				results.merge!(process)
			end
		end
		@known_internet_domains.merge!(results)
		puts "Done loading entries."
		return results
	else
		puts "Error: no entry is loaded. Please check your list and try again."
	end
	return results
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#bulk_delete(list) ⇒ Object Also known as: dels

‘setter’ to delete domain entry to the cache in batch (from a list)



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
 
# File 'lib/wmap/domain_tracker.rb', line 199

def bulk_delete(list)
	puts "Delete entries to the local domains cache table from list: #{list}" if @verbose
	domains=list
	changes=Array.new
	if domains.size > 0
		domains.map do |x|
			domain=delete(x)
			changes.push(domain) unless domain.nil?
		end
		puts "Done deleting domains from list: #{list}"
		return changes
	else
		puts "Exception on method bulk_delete: no entry is loaded. Please check your list and try again."
	end
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#count ⇒ Object Also known as: size

Count numbers of entries in the domain cache table



85
86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/wmap/domain_tracker.rb', line 85

def count
	puts "Counting number of entries in the domain cache table ..."
	cnt=0
	@known_internet_domains.map do |key|
		unless key =~ /\w+\.\w+/
			cnt=cnt+1
		end
	end
	puts "Current number of entries: #{cnt}"
	return cnt
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#delete(domain) ⇒ Object

‘setter’ to remove entry from the cache one at a time



184
185
186
187
188
189
190
191
192
193
194
195
196
 
# File 'lib/wmap/domain_tracker.rb', line 184

def delete(domain)
	puts "Remove entry from the domains cache table: #{domain} " if @verbose
	domain=domain.strip.downcase
	if @known_internet_domains.key?(domain)
		@known_internet_domains.delete(domain)
		puts "Entry cleared: #{domain}"
		return domain
	else
		puts "Entry not fund. Skipping: #{domain}"
	end
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#delete_all ⇒ Object

‘setter’ to remove all entries from the store



229
230
231
232
233
234
235
236
 
# File 'lib/wmap/domain_tracker.rb', line 229

def delete_all
	puts "Delete all entries in the domain store! " if @verbose
	@known_internet_domains.keys.map do |domain|
		delete(domain)
	end
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#domain_known?(domain) ⇒ Boolean Also known as: is_known?, is_domain_known?

Simple method to check if a domain is already within the domain cache table

Returns:

  • (Boolean) 


256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
 
# File 'lib/wmap/domain_tracker.rb', line 256

def domain_known?(domain)
	#abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil? or @known_internet_sub_domains.nil?
	domain=domain.strip.downcase unless domain.nil?
	case self.class.name
	when "Wmap::DomainTracker"
		return @known_internet_domains.key?(domain)
	when "Wmap::DomainTracker::SubDomain"
		return @known_internet_sub_domains.key?(domain)
	else
		return nil
	end
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
	return false
end

#file_add(file) ⇒ Object

‘setter’ to add domain entry to the cache in batch (from a file)



144
145
146
147
148
149
150
151
152
 
# File 'lib/wmap/domain_tracker.rb', line 144

def file_add(file)
	puts "Add entries to the local domains cache table from file: #{file}" if @verbose
	raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
	changes=Array.new
	domains=file_2_list(file)
	changes=bulk_add(domains)
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#file_delete(file) ⇒ Object

‘setter’ to delete domain entry to the cache in batch (from a file)



219
220
221
222
223
224
225
226
 
# File 'lib/wmap/domain_tracker.rb', line 219

def file_delete(file)
	puts "Delete entries to the local domains cache table from file: #{file}" if @verbose
	raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
	domains=file_2_list(file)
	changes=bulk_delete(domains)
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#get_domains ⇒ Object Also known as: dump_domains, dump

Dump out the list of known domains



275
276
277
278
279
280
281
 
# File 'lib/wmap/domain_tracker.rb', line 275

def get_domains
	puts "Retrieve a list of known domain ..." if @verbose
	return @known_internet_domains.keys
rescue Exception => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
	return nil
end

#load_domains_from_file(file = @domains_file, lc = true) ⇒ Object

‘setter’ to load the known Internet domains into an instance variable



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/wmap/domain_tracker.rb', line 33

def load_domains_from_file (file=@domains_file, lc=true)
	puts "Loading trusted domain file: #{file}"	if @verbose
	@known_internet_domains=Hash.new
	f_domains=File.open(file, 'r')
	f_domains.each_line do |line|
		puts "Processing line: #{line}" if @verbose
		line=line.chomp.strip
		next if line.nil?
		next if line.empty?
		next if line =~ /^\s*#/
		line=line.downcase if lc==true
		entry=line.split(',')
		if @known_internet_domains.key?(entry[0])
			next
		else
			if entry[1] =~ /yes/i
				@known_internet_domains[entry[0]]=true
			else
				@known_internet_domains[entry[0]]=false
			end
		end

	end
	f_domains.close
	return @known_internet_domains
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
	return nil
end

#print_known_domains ⇒ Object Also known as: print

Print summary report on all known / trust domains in the domain cache table



303
304
305
306
307
308
309
 
# File 'lib/wmap/domain_tracker.rb', line 303

def print_known_domains
	puts "\nSummary of known Internet Domains:"
	@known_internet_domains.keys.sort.each do |domain|
		puts domain
	end
	puts "End of the summary"
end

#refresh(domain) ⇒ Object

Refresh the domain entry one at a time



239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
 
# File 'lib/wmap/domain_tracker.rb', line 239

def refresh(domain)
	abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil?
	domain=domain.strip.downcase unless domain.nil?
	if domain_known?(domain)
		delete(domain)
		add(domain)
		return domain
	else
		puts "Unknown domain: #{domain}"
		return nil
	end
rescue => ee
	puts "Exception on method #{__method__} for #{domain}: #{ee}" if @verbose
	return nil
end

#save_domains_to_file!(file_domains = @domains_file, domains = @known_internet_domains) ⇒ Object Also known as: save!

Save the current domain hash table into a file



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/wmap/domain_tracker.rb', line 64

def save_domains_to_file!(file_domains=@domains_file, domains=@known_internet_domains)
	puts "Saving the current domains cache table from memory to file: #{file_domains} ..." if @verbose
	timestamp=Time.now
	f=File.open(file_domains, 'w')
	f.write "# Local domains file created by class #{self.class} method #{__method__} at: #{timestamp}\n"
	f.write "# domain name, free zone transfer detected?\n"
	domains.keys.sort.map do |key|
		if domains[key]
			f.write "#{key}, yes\n"
		else
			f.write "#{key}, no\n"
		end
	end
	f.close
	puts "Domain cache table is successfully saved: #{file_domains}"
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#search(pattern) ⇒ Object Also known as: find

Search potential matching domains from the domain store by using simple regular expression. Note that any upper-case char in the search string will be automatically converted into lower case



286
287
288
289
290
291
292
293
294
295
296
297
298
299
 
# File 'lib/wmap/domain_tracker.rb', line 286

def search (pattern)
	puts "Search domain store for the regular expression: #{pattern}" if @verbose
	pattern=pattern.strip.downcase
	results=Array.new
	@known_internet_domains.keys.map do |key|
		if key =~ /#{pattern}/i
			results.push(key)
		end
	end
	return results
rescue Exception => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
	return nil
end