Class: Rcrawl::Crawler

Inherits:
Object
  • Object
show all
Defined in:
lib/rcrawl/crawler.rb,
lib/rcrawl/version.rb

Constant Summary collapse

VERSION = 
"0.5.1"

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(site) ⇒ Crawler

Initializes various variables when a new Crawler object is instantiated



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/rcrawl/crawler.rb', line 9

def initialize(site)
	puts "Rcrawl Version #{VERSION} initializing..."
	@links_to_visit = Array.new
	@visited_links = Array.new
	@external_links = Array.new
	@raw_html = Hash.new
	@rules = RobotRules.new('Rcrawl')
	@user_agent = "Rcrawl/#{VERSION} (http://rubyforge.org/projects/rcrawl/)"
	@sites = Hash.new
	@errors = Hash.new
	@meta = Hash.new
	@site = URI.parse(site) || raise("You didn't give me a site to crawl")
	@links_to_visit << site
	puts "Ready to crawl #{site}"
end

Instance Attribute Details

#errors ⇒ Object (readonly)

Returns the value of attribute errors.



6
7
8
 
# File 'lib/rcrawl/crawler.rb', line 6

def errors
  @errors
end

#external_links ⇒ Object (readonly)

Returns the value of attribute external_links.



6
7
8
 
# File 'lib/rcrawl/crawler.rb', line 6

def external_links
  @external_links
end

#links_to_visit ⇒ Object

Returns the value of attribute links_to_visit.



5
6
7
 
# File 'lib/rcrawl/crawler.rb', line 5

def links_to_visit
  @links_to_visit
end

#meta ⇒ Object (readonly)

Returns the value of attribute meta.



6
7
8
 
# File 'lib/rcrawl/crawler.rb', line 6

def meta
  @meta
end

#raw_html ⇒ Object (readonly)

Returns the value of attribute raw_html.



6
7
8
 
# File 'lib/rcrawl/crawler.rb', line 6

def raw_html
  @raw_html
end

#rules ⇒ Object (readonly)

Returns the value of attribute rules.



6
7
8
 
# File 'lib/rcrawl/crawler.rb', line 6

def rules
  @rules
end

#site ⇒ Object

Returns the value of attribute site.



5
6
7
 
# File 'lib/rcrawl/crawler.rb', line 5

def site
  @site
end

#sites ⇒ Object (readonly)

Returns the value of attribute sites.



6
7
8
 
# File 'lib/rcrawl/crawler.rb', line 6

def sites
  @sites
end

#user_agent ⇒ Object

Returns the value of attribute user_agent.



5
6
7
 
# File 'lib/rcrawl/crawler.rb', line 5

def user_agent
  @user_agent
end

#visited_links ⇒ Object (readonly)

Returns the value of attribute visited_links.



6
7
8
 
# File 'lib/rcrawl/crawler.rb', line 6

def visited_links
  @visited_links
end

Instance Method Details

#crawl ⇒ Object

Coordinates the whole crawling process



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/rcrawl/crawler.rb', line 26

def crawl
	until @links_to_visit.empty? do
		begin
			# Get link
			url_server
			next unless robot_safe? @url
			if @url.include? '#'
				print "... Anchor link found, skipping..."
				next
			end
			# Parse robots.txt, then download document if robot_safe
			fetch_http(@url)
			# Store raw HTML in variable to read/reread as needed
			# Then call any processing modules you need for the current document
			ris(@document)
		rescue
			puts ""
			puts "I died on #{@url}"
			$stderr.puts $!
			@errors[@url] = $!
			next
		ensure
			# Stuff you want to make sure gets printed out
			puts " done!"
		end
	end

	puts "Visited #{@visited_links.size} links."
end

#fetch_http(url) ⇒ Object

Download the document



64
65
66
67
68
69
70
71
 
# File 'lib/rcrawl/crawler.rb', line 64

def fetch_http(url)
	# Make sure robots.txt has been parsed for this site first,
	# if not, parse robots.txt then grab document.
	uri = URI.parse(url)
	print "Visiting: #{url}"
	@document = uri.read("User-Agent" => @user_agent, "Referer" => url)
	@visited_links << url
end

#link_extractor(document) ⇒ Object

HTML processing module for extracting links



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/rcrawl/crawler.rb', line 89

def link_extractor(document)
	print "."
	# Parse all links from HTML into an array
	# Set up the scrAPI (http://labnotes.org)
	links = Scraper.define do
		array :urls
		process "a[href]", :urls => "@href"
		result :urls
	end
	
	urls = links.scrape(document)
	
	urls.each { |url|
	uri = URI.parse(url)
	
	# Derelativeize links if necessary
	if uri.relative?
		url = @site.merge(url).to_s
		uri = URI.parse(url)
	end
	
	# Check domain, if in same domain, keep link, else trash it
	if uri.host != @site.host
		@external_links << url
		@external_links.uniq!
		next
	end

	# Find out if we've seen this link already
	if (@visited_links.include? url) || (@links_to_visit.include? url)
		next
	end

	@links_to_visit << url
}
end

#page_meta(document) ⇒ Object 



138
139
140
 
# File 'lib/rcrawl/crawler.rb', line 138

def page_meta(document)
	@meta[@url] = document.meta
end

#process_html(document) ⇒ Object

HTML processing module for raw HTML storage



127
128
129
130
131
132
133
134
135
136
 
# File 'lib/rcrawl/crawler.rb', line 127

def process_html(document)
	
	# Add link and raw HTML to a hash as key/value
	# for later storage in database
	unless @raw_html.has_value?(document)
  		print "."
  		@raw_html[@document.base_uri.to_s] = document
	end
		
end

#ris(document) ⇒ Object

Rewind Input Stream, for storing and reading of raw HTML



74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/rcrawl/crawler.rb', line 74

def ris(document)
	print "."
	# Store raw HTML into local variable
	# Based on MIME type, invoke the proper processing modules
	case document.content_type
		when "text/html"
			link_extractor(document)
			process_html(document)
			page_meta(document)
		else
			print "... not HTML, skipping..."
	end
end

#robot_safe?(url) ⇒ Boolean

robots.txt parsing

Returns:

  • (Boolean) 


143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/rcrawl/crawler.rb', line 143

def robot_safe?(url)
	uri = URI.parse(url)
	location = "#{uri.host}:#{uri.port}"

	return true unless %w{http https}.include?(uri.scheme)

	unless @sites.include? location
		@sites[location] = true

		robot_url = "http://#{location}/robots.txt"
		begin
			robot_file = open(robot_url) { |page| page.read }
		rescue
			return true
		end
		@rules.parse(robot_url, robot_file)
	end

	@rules.allowed? url
end

#url_server ⇒ Object

Authoritative list of URLs to be processed by Rcrawl



57
58
59
60
61
 
# File 'lib/rcrawl/crawler.rb', line 57

def url_server
	unless @links_to_visit.empty?
		@url = @links_to_visit.pop
	end
end