Class: Rwspider::Client

Inherits:
Object
  • Object
show all
Defined in:
lib/rwspider/client.rb

Constant Summary collapse

DEFAULT_OPTIONS =

follow_HTTP_redirection: Set as true to follow the HTTP redirections

timeout: The timeout of single URL analysis

{
  :useragent => 'RW Spider/' + Rwspider::VERSION,
  :robot_name => 'rwspider',
:scan_documents_limit => 100,
  :scan_domain_limit => nil,
  :scan_images => false,
  :scan_other_files => false,
  :follow_robotstxt_directive => true,
  :follow_HTTP_redirection => true,
			:timeout => 5
}

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ Client

spider = Rwspider::Client::new(opts)



83
84
85
86
87
88
89
90
91
 
# File 'lib/rwspider/client.rb', line 83

def initialize (options = {})
	
    load_options options			
	@robotstxt_cache = Hash.new()			
    @main_hostname = ''
    @scanned_documents = 0
	@queue = Rwspider::Queue.new
	
end

Instance Attribute Details

#opts ⇒ Object

Hash of options for the spider job



31
32
33
 
# File 'lib/rwspider/client.rb', line 31

def opts
  @opts
end

Instance Method Details

#start(start_url) ⇒ Object

Start the crawling from the URL.

Rwspider::Client::start yield an instance of Rwspider::Document Class for each page downloaded. At the end of execution returns an Array of Rwspider::Document instances.

Rwspider::Client::start('http://www.rwspider.com') {do |d|
   puts 'Current URL ' + d.url.normalize.to_s
}

arr = Rwspider::Client::start('http://www.rwspider.com') 
arr.each{do |d|
   puts 'Current URL ' + d.url.normalize.to_s
}



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
 
# File 'lib/rwspider/client.rb', line 107

def start (start_url)
				
	@queue << Rwspider::Document.new(start_url)
	
	@queue.each do |link|
		@main_url = link.url
		if @opts[:scan_documents_limit].nil? || @scanned_documents < @opts[:scan_documents_limit]
			set_as_visited link
			@main_hostname = link.url.host.downcase if @main_hostname.length == 0
			
			t = Thread.new(link) { |link|
				begin
					
					Timeout::timeout(@opts[:timeout]){	
						beginning = Time.now
						response = get_uri(link.url)
					  link.download_time = Time.now - beginning
						link.as_downloaded = true
						link.http_response = response
						
						case response
							when Net::HTTPSuccess then
							
							if  response.content_type == 'text/html' && (@opts[:scan_domain_limit].nil? || link.url.host.downcase.match(@opts[:scan_domain_limit]) )
								
								link.get_links
								link.get_images if @opts[:scan_images]
								link.get_other_files if @opts[:scan_other_files]
								
							  link.documents.each do |doc|
									add_to_queue doc
								end
							end
							
							when Net::HTTPRedirection then
							add_to_queue(Document.new(link.normalize_url(Document.new(response['location']).url))) if @opts[:follow_HTTP_redirection] 
						
						end
					}
				rescue Timeout::Error => e
						link.as_downloaded = false
				rescue StandardError => e
						link.as_downloaded = false
				end
				yield link if block_given?
			} 
			t.join					
		end
	end
	
	return @queue
	
end