Module: CrawlerHelper

Included in:
Crawler
Defined in:
lib/crawler_lib.rb

Instance Method Summary collapse

Instance Method Details

#do_not_ignore?(each_link, scraped) ⇒ Boolean

end

Returns:

  • (Boolean) 


52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'lib/crawler_lib.rb', line 52

def do_not_ignore?(each_link, scraped)
	# This checks if the passed link should be
	# scraped or not based on:
	# Has it already been scraped, is it bad_link?
	# puts each_link
	# puts scraped.class
	if scraped.include?(each_link)
		return false
	elsif bad_link?(each_link)
		return false
	else
		return true
	end
end

#fix_scheme(url) ⇒ Object 



10
11
12
13
14
15
16
17
18
19
 
# File 'lib/crawler_lib.rb', line 10

def fix_scheme(url)
	puts "- No scheme provided for #{url}, trying to fix it."
	driver = Selenium::WebDriver.for :firefox
	driver.get("http://"+url) #assumes redirect to https is setup if it exists.
	url_tmp = driver.current_url
	scheme = URI.parse(url_tmp).scheme
	driver.quit
	puts "scheme is: #{scheme}"
	return scheme+"://"+url
end

#sanitize(link) ⇒ Object 



22
23
24
25
26
27
28
29
30
31
 
# File 'lib/crawler_lib.rb', line 22

def sanitize(link)
	# puts link
	name = link.gsub(":", "")
	name = name.gsub("/", "")
	name = name.gsub("%", "")
	name = name.gsub('\\', "")
	name = name.gsub('.', "")
	# puts name
	return name
end

#test ⇒ Object 



6
7
8
 
# File 'lib/crawler_lib.rb', line 6

def test
	puts "Test call"
end