Class: Wmap::DomainTracker

Inherits:
Object
  • Object
show all
Includes:
Singleton, Utils
Defined in:
lib/wmap/domain_tracker.rb,
lib/wmap/domain_tracker/sub_domain.rb

Overview

Class to track the known (trusted) Internet domains

Direct Known Subclasses

SubDomain

Defined Under Namespace

Classes: SubDomain

Constant Summary

Constants included from Utils::DomainRoot

Utils::DomainRoot::File_ccsld, Utils::DomainRoot::File_cctld, Utils::DomainRoot::File_gtld, Utils::DomainRoot::File_tld

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Utils

#cidr_2_ips, #file_2_hash, #file_2_list, #get_nameserver, #get_nameservers, #host_2_ip, #host_2_ips, #is_cidr?, #is_fqdn?, #is_ip?, #list_2_file, #reverse_dns_lookup, #sort_ips, #valid_dns_record?, #zone_transferable?

Methods included from Utils::Logger

#wlog

Methods included from Utils::UrlMagic

#create_absolute_url_from_base, #create_absolute_url_from_context, #host_2_url, #is_site?, #is_ssl?, #is_url?, #make_absolute, #normalize_url, #url_2_host, #url_2_path, #url_2_port, #url_2_site, #urls_on_same_domain?

Methods included from Utils::DomainRoot

#get_domain_root, #get_sub_domain, #is_domain_root?, #print_ccsld, #print_cctld, #print_gtld

Constructor Details

#initialize(params = {}) ⇒ DomainTracker

Set default instance variables



22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/wmap/domain_tracker.rb', line 22

def initialize (params = {})
  # Initialize the instance variables
  @verbose=params.fetch(:verbose, false)
  @data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
  Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
  @file_domains=params.fetch(:domains_file, @data_dir+'domains')
  @max_parallel=params.fetch(:max_parallel, 40)
  # Hash table to hold the trusted domains
  File.write(@file_domains, "") unless File.exist?(@file_domains)
  @known_internet_domains=load_domains_from_file(@file_domains)
  #@known_internet_sub_domains=Hash.new
end

Instance Attribute Details

#data_dir ⇒ Object

Returns the value of attribute data_dir.



18
19
20
 
# File 'lib/wmap/domain_tracker.rb', line 18

def data_dir
  @data_dir
end

#domains_file ⇒ Object

Returns the value of attribute domains_file.



18
19
20
 
# File 'lib/wmap/domain_tracker.rb', line 18

def domains_file
  @domains_file
end

#file_domains ⇒ Object

Returns the value of attribute file_domains.



18
19
20
 
# File 'lib/wmap/domain_tracker.rb', line 18

def file_domains
  @file_domains
end

#known_internet_domains ⇒ Object (readonly)

Returns the value of attribute known_internet_domains.



19
20
21
 
# File 'lib/wmap/domain_tracker.rb', line 19

def known_internet_domains
  @known_internet_domains
end

#max_parallel ⇒ Object

Returns the value of attribute max_parallel.



18
19
20
 
# File 'lib/wmap/domain_tracker.rb', line 18

def max_parallel
  @max_parallel
end

#verbose ⇒ Object

Returns the value of attribute verbose.



18
19
20
 
# File 'lib/wmap/domain_tracker.rb', line 18

def verbose
  @verbose
end

Instance Method Details

#add(host) ⇒ Object

‘setter’ to add domain entry to the cache one at a time



110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/wmap/domain_tracker.rb', line 110

def add(host)
  begin
    puts "Add entry to the local domains cache table: #{host}" if @verbose
    return nil if host.nil? or host.empty?
    host=host.strip.downcase
    if @known_internet_domains.key?(host)
      puts "Domain is already exist. Skipping: #{host}"
    else
      root=get_domain_root(host)
      sub=get_subdomain(host)
      record=Hash.new
      if host == root
        if zone_transferable?(root)
          record[root]=true
          #@known_internet_domains[root]=true
        else
          record[root]=false
          #@known_internet_domains[root]=false
        end
        puts "Entry loaded: #{record}"
        @known_internet_domains.merge!(record)
        return record
      elsif sub.nil?       # 2/10/2014, additional logic to support sub-domains
        # do nothing
      elsif host != sub
        if zone_transferable?(sub)
          #@known_internet_domains[sub]=true
          record[sub]=true
        else
          #@known_internet_domains[sub]=false
          record[sub]=false
        end
        puts "Entry loaded: #{record}"
        @known_internet_domains.merge!(record)
        return record
      else
        puts "Problem add domain #{host}: unknown domain format - please use legal root domain or sub domain only."
      end
    end
  rescue => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
  end
end

#bulk_add(list, num = @max_parallel) ⇒ Object Also known as: adds

‘setter’ to add domain entry to the cache in batch (from a list)



168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/wmap/domain_tracker.rb', line 168

def bulk_add(list, num=@max_parallel)
  puts "Add entries to the local domains cache table from list: #{list}" if @verbose
  begin
    results=Hash.new
    domains=list
    if domains.size > 0
      Parallel.map(list, :in_processes => num) { |target|
        add(target)
      }.each do |process|
        if process.nil?
          next
        elsif process.empty?
          #do nothing
        else
          results.merge!(process)
        end
      end
      @known_internet_domains.merge!(results)
      puts "Done loading entries."
      return results
    else
      puts "Error: no entry is loaded. Please check your list and try again."
    end
    return results
  rescue => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
  end
end

#bulk_delete(list) ⇒ Object Also known as: dels

‘setter’ to delete domain entry to the cache in batch (from a list)



216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/wmap/domain_tracker.rb', line 216

def bulk_delete(list)
  puts "Delete entries to the local domains cache table from list: #{list}" if @verbose
  begin
    domains=list
    changes=Array.new
    if domains.size > 0
      domains.map do |x|
        domain=delete(x)
        changes.push(domain) unless domain.nil?
      end
      puts "Done deleting domains from list: #{list}"
      return changes
    else
      puts "Exception on method bulk_delete: no entry is loaded. Please check your list and try again."
    end
  rescue => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
  end
end

#count ⇒ Object Also known as: size

Count numbers of entries in the domain cache table



92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/wmap/domain_tracker.rb', line 92

def count
  puts "Counting number of entries in the domain cache table ..."
  begin
    cnt=0
    @known_internet_domains.map do |key|
      unless key =~ /\w+\.\w+/
        cnt=cnt+1
      end
    end
    puts "Current number of entries: #{cnt}"
    return cnt
  rescue => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
  end
end

#delete(domain) ⇒ Object

‘setter’ to remove entry from the cache one at a time



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
 
# File 'lib/wmap/domain_tracker.rb', line 199

def delete(domain)
  puts "Remove entry from the domains cache table: #{domain} " if @verbose
  begin
    domain=domain.strip.downcase
    if @known_internet_domains.key?(domain)
      @known_internet_domains.delete(domain)
      puts "Entry cleared: #{domain}"
      return domain
    else
      puts "Entry not fund. Skipping: #{domain}"
    end
  rescue => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
  end
end

#delete_all ⇒ Object

‘setter’ to remove all entries from the store



250
251
252
253
254
255
256
257
258
259
 
# File 'lib/wmap/domain_tracker.rb', line 250

def delete_all
  puts "Delete all entries in the domain store! " if @verbose
  begin
    @known_internet_domains.keys.map do |domain|
      delete(domain)
    end
  rescue => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
  end
end

#domain_known?(domain) ⇒ Boolean Also known as: is_known?, is_domain_known?

Simple method to check if a domain is already within the domain cache table

Returns:

  • (Boolean) 


281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
 
# File 'lib/wmap/domain_tracker.rb', line 281

def domain_known?(domain)
  begin
    #abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil? or @known_internet_sub_domains.nil?
    domain=domain.strip.downcase unless domain.nil?
    case self.class.name
    when "Wmap::DomainTracker"
      return @known_internet_domains.key?(domain)
    when "Wmap::DomainTracker::SubDomain"
      return @known_internet_sub_domains.key?(domain)
    else
      return nil
    end
  rescue => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
  end
  return false
end

#file_add(file) ⇒ Object

‘setter’ to add domain entry to the cache in batch (from a file)



155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/wmap/domain_tracker.rb', line 155

def file_add(file)
  begin
    puts "Add entries to the local domains cache table from file: #{file}" if @verbose
    raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
    changes=Array.new
    domains=file_2_list(file)
    changes=bulk_add(domains)
  rescue => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
  end
end

#file_delete(file) ⇒ Object

‘setter’ to delete domain entry to the cache in batch (from a file)



238
239
240
241
242
243
244
245
246
247
 
# File 'lib/wmap/domain_tracker.rb', line 238

def file_delete(file)
  begin
    puts "Delete entries to the local domains cache table from file: #{file}" if @verbose
    raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
    domains=file_2_list(file)
    changes=bulk_delete(domains)
  rescue => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
  end
end

#get_domains ⇒ Object Also known as: dump_domains, dump

Dump out the list of known domains



302
303
304
305
306
307
308
309
310
 
# File 'lib/wmap/domain_tracker.rb', line 302

def get_domains
  puts "Retrieve a list of known domain ..." if @verbose
  begin
    return @known_internet_domains.keys
  rescue Exception => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
    return nil
  end
end

#print_known_domains ⇒ Object Also known as: print

Print summary report on all known / trust domains in the domain cache table



334
335
336
337
338
339
340
 
# File 'lib/wmap/domain_tracker.rb', line 334

def print_known_domains
  puts "\nSummary of known Internet Domains:"
  @known_internet_domains.keys.sort.each do |domain|
    puts domain
  end
  puts "End of the summary"
end

#print_known_sub_domains ⇒ Object

Print summary report on all known / trust domains in the domain cache table



136
137
138
139
140
141
142
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 136

def print_known_sub_domains
  puts "\nSummary of known Internet Sub-domains:"
  self.known_internet_sub_domains.keys.sort.each do |domain|
    puts domain
  end
  puts "End of the summary"
end

#refresh(domain) ⇒ Object

Refresh the domain entry one at a time



262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
 
# File 'lib/wmap/domain_tracker.rb', line 262

def refresh(domain)
  begin
    abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil?
    domain=domain.strip.downcase unless domain.nil?
    if domain_known?(domain)
      delete(domain)
      add(domain)
      return domain
    else
      puts "Unknown domain: #{domain}"
      return nil
    end
  rescue => ee
    puts "Exception on method #{__method__} for #{domain}: #{ee}" if @verbose
    return nil
  end
end

#save_domains_to_file!(file_domains = @file_domains, domains = @known_internet_domains) ⇒ Object Also known as: save!

Save the current domain hash table into a file



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/wmap/domain_tracker.rb', line 69

def save_domains_to_file!(file_domains=@file_domains, domains=@known_internet_domains)
  puts "Saving the current domains cache table from memory to file: #{file_domains} ..." if @verbose
  begin
    timestamp=Time.now
    f=File.open(file_domains, 'w')
    f.write "# Local domains file created by class #{self.class} method #{__method__} at: #{timestamp}\n"
    f.write "# domain name, free zone transfer detected?\n"
    domains.keys.sort.map do |key|
      if domains[key]
        f.write "#{key}, yes\n"
      else
        f.write "#{key}, no\n"
      end
    end
    f.close
    puts "Domain cache table is successfully saved: #{file_domains}"
  rescue => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
  end
end

#search(pattern) ⇒ Object Also known as: find

Search potential matching domains from the domain store by using simple regular expression. Note that any upper-case char in the search string will be automatically converted into lower case



315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
 
# File 'lib/wmap/domain_tracker.rb', line 315

def search (pattern)
  puts "Search domain store for the regular expression: #{pattern}" if @verbose
  begin
    pattern=pattern.strip.downcase
    results=Array.new
    @known_internet_domains.keys.map do |key|
      if key =~ /#{pattern}/i
        results.push(key)
      end
    end
    return results
  rescue Exception => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
    return nil
  end
end