Class: Wmap::UrlChecker

Inherits:
Object
  • Object
show all
Includes:
Utils
Defined in:
lib/wmap/url_checker.rb

Overview

A quick checker class to identify / finger-print a URL / site

Constant Summary

Constants included from Wmap::Utils::UrlMagic

Wmap::Utils::UrlMagic::Max_http_timeout

Constants included from Wmap::Utils::DomainRoot

Wmap::Utils::DomainRoot::File_ccsld, Wmap::Utils::DomainRoot::File_cctld, Wmap::Utils::DomainRoot::File_gtld, Wmap::Utils::DomainRoot::File_tld

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Utils

#cidr_2_ips, #file_2_hash, #file_2_list, #get_nameserver, #get_nameservers, #host_2_ip, #host_2_ips, #is_cidr?, #is_fqdn?, #is_ip?, #list_2_file, #reverse_dns_lookup, #sort_ips, #valid_dns_record?, #zone_transferable?

Methods included from Wmap::Utils::Logger

#wlog

Methods included from Wmap::Utils::UrlMagic

#create_absolute_url_from_base, #create_absolute_url_from_context, #host_2_url, #is_site?, #is_ssl?, #is_url?, #landing_location, #make_absolute, #normalize_url, #open_page, #redirect_location, #response_code, #url_2_host, #url_2_path, #url_2_port, #url_2_site, #urls_on_same_domain?

Methods included from Wmap::Utils::DomainRoot

#get_domain_root, #get_domain_root_by_ccsld, #get_domain_root_by_cctld, #get_domain_root_by_tlds, #get_sub_domain, #is_domain_root?, #print_ccsld, #print_cctld, #print_gtld

Constructor Details

#initialize(params = {}) ⇒ UrlChecker

Returns a new instance of UrlChecker.



20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/wmap/url_checker.rb', line 20

def initialize (params = {})
	# Set default instance variables
	@verbose=params.fetch(:verbose, false)
	@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
	@http_timeout=params.fetch(:http_timeout, 5000)
	@max_parallel=params.fetch(:max_parallel, 40)
	@ssl_version=nil
	@url_code={}
	@url_redirection={}
	@url_finger_print={}
	@url_server={}
end

Instance Attribute Details

#data_dir ⇒ Object

Returns the value of attribute data_dir.



18
19
20
 
# File 'lib/wmap/url_checker.rb', line 18

def data_dir
  @data_dir
end

#http_timeout ⇒ Object

Returns the value of attribute http_timeout.



18
19
20
 
# File 'lib/wmap/url_checker.rb', line 18

def http_timeout
  @http_timeout
end

#max_parallel ⇒ Object

Returns the value of attribute max_parallel.



18
19
20
 
# File 'lib/wmap/url_checker.rb', line 18

def max_parallel
  @max_parallel
end

#verbose ⇒ Object

Returns the value of attribute verbose.



18
19
20
 
# File 'lib/wmap/url_checker.rb', line 18

def verbose
  @verbose
end

Instance Method Details

#get_cert_cn(url) ⇒ Object Also known as: get_cn

Retrieve the X509 cert in the clear text from the remote web server, extract and return the common name field within the cert



300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
 
# File 'lib/wmap/url_checker.rb', line 300

def get_cert_cn (url)
	puts "Extract the common name field from a X509 cert: #{cert}" if @verbose
	cert=get_certificate(url)
	subject, cn = ""
	if cert =~ /\n(.+)Subject\:(.+)\n/i
		subject=$2
	end
	if subject =~/CN\=(.+)/i
		cn=$1
	end
	return cn
rescue Exception => ee
	puts "Error on method #{__method__} from #{url}: #{ee}" if @verbose
	return nil
end

#get_certificate(url) ⇒ Object Also known as: get_cert

Retrieve the remote web server certification, open it and return the cert content as a string



283
284
285
286
287
288
289
290
291
292
293
294
295
296
 
# File 'lib/wmap/url_checker.rb', line 283

def get_certificate (url)
	puts "Retrieve the remote web server SSL certificate in clear text: #{url}" if @verbose
	url=url.strip
	raise "Invalid URL string: #{url}" unless is_ssl?(url)
	client = HTTPClient.new
	client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
	response = client.get(url)
	cert = response.peer_cert
	cer = OpenSSL::X509::Certificate.new(cert)
	return cer.to_text
rescue Exception => ee
	puts "Exception on method #{__method__} from #{url}: #{ee}"
	return nil
end

#get_server_header(url) ⇒ Object

Test the URL / site and return the web server type from the HTTP header “server” field



226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/wmap/url_checker.rb', line 226

def get_server_header (url)
	puts "Retrieve the server header field from the url: #{url}" if @verbose
	server=String.new
	raise "Invalid url: #{url}" unless is_url?(url)
	url=url.strip.downcase
	timeo = @http_timeout/1000.0
	uri = URI.parse(url)
	code = response_code (url)
	http = Net::HTTP.new(uri.host, uri.port)
	http.open_timeout = timeo
	http.read_timeout = timeo
	if (url =~ /https\:/i)
		http.use_ssl = true
		# Bypass the remote web server cert validation test
		http.verify_mode = OpenSSL::SSL::VERIFY_NONE
		http.ssl_version = @ssl_version
	end
	request = Net::HTTP::Get.new(uri.request_uri)
	response = http.request(request)
	server=response["server"]
	server=server.gsub(/\,/,' ')
	return server
rescue Exception => ee
	puts "Exception on method get_server_header for URL #{url}: #{ee}" if @verbose
	@url_server[url]=server
	return server
end

#response_body_md5(url) ⇒ Object Also known as: md5

Use MD5 algorithm to fingerprint the URL / site response payload (web page content)



255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
 
# File 'lib/wmap/url_checker.rb', line 255

def response_body_md5(url)
	puts "MD5 finger print page body content: #{url}" if @verbose
	raise "Invalid url: #{url}" unless is_url?(url)
	url=url.strip.downcase
	timeo = @http_timeout/1000.0
	uri = URI.parse(url)
	fp=""
	http = Net::HTTP.new(uri.host, uri.port)
	http.open_timeout = timeo
	http.read_timeout = timeo
	if (url =~ /https\:/i)
		http.use_ssl = true
		# Bypass the remote web server cert validation test
		http.verify_mode = OpenSSL::SSL::VERIFY_NONE
		http.ssl_version = @ssl_version
	end
	request = Net::HTTP::Get.new(uri.request_uri)
	response = http.request(request)
	response_body = response.body.to_s
	fp=Digest::MD5.hexdigest(response_body) unless response_body.nil?
	@url_finger_print[url] = fp
	return fp
rescue Exception => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#url_worker(url) ⇒ Object Also known as: check

Main worker method to perform various checks on the URL / site



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/wmap/url_checker.rb', line 34

def url_worker (url)
	puts "Checking out an unknown URL: #{url}" if @verbose
	url=url.strip.downcase
	raise "Invalid URL format: #{url}" unless is_url?(url)
	timestamp=Time.now
	host=url_2_host(url)
	ip=host_2_ip(host)
	port=url_2_port(url)
	code=10000
	if @url_code.key?(url)
		code=@url_code[url]
	else
		code=response_code(url)
	end
	if code>=300 && code < 400
		loc=landing_location(4,url)
	else
		loc=nil
	end
	if @url_finger_print.key?(url)
		fp=@url_finger_print[url]
	else
		fp=response_body_md5(url)
	end
	if @url_server.key?(url)
		server=@url_server[url]
	else
		server=get_server_header(url)
	end
	# save the data
	checker=Hash.new
	checker['ip']=ip
	checker['port']=port
	checker['url']=url
	checker['code']=code
	checker['redirection']=loc
	checker['md5']=fp
	checker['server']=server
	checker['timestamp']=timestamp
	if Wmap::CidrTracker.new(:data_dir=>@data_dir).ip_trusted?(ip)
		checker['status']="int_hosted"
	else
		checker['status']="ext_hosted"
	end
	return checker
rescue OpenSSL::SSL::SSLError => es  # handler to temporally hold the openssl bug in bay:  SSL_set_session: unable to find ssl method
	checker=Hash.new
	checker['ip']=ip
	checker['port']=port
	checker['url']=url
	checker['code']=20000
	checker['server']="Unknown SSL error: #{es}"
	checker['md']=nil
	checker['redirection']=nil
	checker['timestamp']=timestamp
	return checker
rescue Exception => ee
	puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
	return nil
end

#url_workers(targets, num = @max_parallel) ⇒ Object Also known as: checks

Parallel scanner - by utilizing fork manager ‘parallel’ to spawn numbers of child processes on multiple urls simultaneously



97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/wmap/url_checker.rb', line 97

def url_workers (targets,num=@max_parallel)
	results=Array.new
	targets -= ["", nil]
	if targets.size > 0
		puts "Start the url checker on the targets:\n #{targets}"
		Parallel.map(targets, :in_processes => num) { |target|
			url_worker(target)
		}.each do |process|
			if process.nil?
				next
			elsif process.empty?
				#do nothing
			else
				results << process
			end
		end
	end
	return results
rescue Exception => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
	return nil
end