Class: Google
- Inherits:
-
Object
- Object
- Includes:
- Searchy
- Defined in:
- lib/esearchy/SearchEngines/google.rb
Instance Attribute Summary collapse
-
#emails ⇒ Object
Returns the value of attribute emails.
Instance Method Summary collapse
-
#initialize(maxhits = 0, start = 0) ⇒ Google
constructor
A new instance of Google.
- #parse(html) ⇒ Object
- #search(query) ⇒ Object
Methods included from Searchy
#clean, #fix, #hash_url, #maxhits=, #print_emails, #search_depth, #search_docs, #search_emails, #search_office_xml, #search_pdfs, #search_txts
Constructor Details
#initialize(maxhits = 0, start = 0) ⇒ Google
Returns a new instance of Google.
8 9 10 11 12 13 14 15 16 17 18 19 |
# File 'lib/esearchy/SearchEngines/google.rb', line 8 def initialize(maxhits = 0, start = 0) @start = start @totalhits = maxhits @emails = [] @r_urls = Queue.new @r_docs = Queue.new @r_pdfs = Queue.new @r_txts = Queue.new @r_officexs = Queue.new @lock = Mutex.new @threads = [] end |
Instance Attribute Details
#emails ⇒ Object
Returns the value of attribute emails.
20 21 22 |
# File 'lib/esearchy/SearchEngines/google.rb', line 20 def emails @emails end |
Instance Method Details
#parse(html) ⇒ Object
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 |
# File 'lib/esearchy/SearchEngines/google.rb', line 54 def parse(html) @totalhits= html.scan(/<\/b> of about <b>(.*)<\/b> for /)[0][0].gsub(",","").to_i if @totalhits == 0 html.scan(/<div class=g><span class="b w xsm">\[([A-Z]+)\]<\/span> <h2 class=r><a href="\ ([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)"|<h2 class=r><a href="\ ([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)"/).each do |result| case result[0] when /PDF/ @r_pdfs << result[1] when /DOC|XLS|PPT/ case result[1] when /.doc$/i @r_docs << result[1] when /.docx$|.xlsx$|.pptx$|.odt$|.odp$|.ods$|.odb$/i @r_officexs << result[1] end when nil case result[2] when /.pdf$/i @r_pdfs << result[2] when /.doc$/i @r_docs << result[2] when /.docx$|.xlsx$|.pptx$|.odt$|.odp$|.ods$|.odb$/i @r_officexs << result[2] when /.txt$|.rtf$|ans$/i @r_txts << result[2] else @r_urls << result[2] end else ESearchy::LOG.puts "I do not parse the #{result[0]} filetype yet:)" end end end |
#search(query) ⇒ Object
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 |
# File 'lib/esearchy/SearchEngines/google.rb', line 22 def search(query) @query = query http = Net::HTTP.new("www.google.com",80) begin http.start do |http| request = Net::HTTP::Get.new( "/cse?&safe=off&num=100&site=" + "&q=" + CGI.escape(query) + "&btnG=Search&start=#{@start}", {'User-Agent' => UserAgent::fetch}) response = http.request(request) case response when Net::HTTPSuccess, Net::HTTPRedirection parse(response.body) @start = @start + 100 if @totalhits > @start ESearchy::LOG.puts "Searching #{self.class} from #{@start-100} to #{@start}" search_emails(response.body.gsub(/<em>|<\/em>/,"")) sleep(ESearchy::DELAY) search(query) else ESearchy::LOG.puts "Searching #{self.class} from #{@start-100} to #{@start}" search_emails(response.body.gsub(/<em>|<\/em>/,"")) end else return response.error! end end rescue Net::HTTPFatalError ESearchy::LOG.puts "Error: Something went wrong with the HTTP request" end end |