Class: GoogleGroups
- Inherits:
-
Object
- Object
- GoogleGroups
- Includes:
- Searchy
- Defined in:
- lib/esearchy/OtherEngines/googlegroups.rb
Instance Attribute Summary collapse
-
#emails ⇒ Object
Returns the value of attribute emails.
Instance Method Summary collapse
-
#initialize(maxhits = nil, start = nil) ⇒ GoogleGroups
constructor
A new instance of GoogleGroups.
- #parse(html) ⇒ Object
- #search(query) ⇒ Object
Methods included from Searchy
#clean, #fix, #hash_url, #maxhits=, #print_emails, #search_depth, #search_docs, #search_emails, #search_office_xml, #search_pdfs, #search_txts
Constructor Details
#initialize(maxhits = nil, start = nil) ⇒ GoogleGroups
Returns a new instance of GoogleGroups.
8 9 10 11 12 13 14 15 16 17 18 19 |
# File 'lib/esearchy/OtherEngines/googlegroups.rb', line 8 def initialize(maxhits = nil, start = nil) @start = start || 0 @totalhits = maxhits || 0 @r_urls = Queue.new @r_docs = Queue.new @r_pdfs = Queue.new @r_txts = Queue.new @r_officexs = Queue.new @emails = [] @lock = Mutex.new @threads = [] end |
Instance Attribute Details
#emails ⇒ Object
Returns the value of attribute emails.
20 21 22 |
# File 'lib/esearchy/OtherEngines/googlegroups.rb', line 20 def emails @emails end |
Instance Method Details
#parse(html) ⇒ Object
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
# File 'lib/esearchy/OtherEngines/googlegroups.rb', line 55 def parse(html) @totalhits=html.scan(/<\/b> of about <b>(.*)<\/b> for /)[0][0].gsub(",","").to_i if @totalhits == 0 html.scan(/<div class=g align="left">\ <a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" target=""/).each do |result| case result[0] when /.pdf$/i @r_pdfs << result[0] when /.doc$/i @r_docs << result[0] when /.docx$|.xlsx$|.pptx$/i @r_officexs << result[0] when /.txt$|.asn$/i @r_txts << result[0] else @r_urls << result[0] end end end |
#search(query) ⇒ Object
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
# File 'lib/esearchy/OtherEngines/googlegroups.rb', line 22 def search(query) @query = query http = Net::HTTP.new("groups.google.com",80) begin http.start do |http| request = Net::HTTP::Get.new( "/groups/search?&safe=off&num=100" + "&q=" + CGI.escape(query) + "&btnG=Search&start=#{@start}", {'User-Agent' => UserAgent::fetch}) response = http.request(request) case response when Net::HTTPSuccess, Net::HTTPRedirection parse(response.body) @start = @start + 100 if @totalhits > @start ESearchy::LOG.puts "Searching #{self.class} from #{@start-100} to #{@start}" search_emails(response.body) sleep(ESearchy::DELAY) search(query) else ESearchy::LOG.puts "Searching #{self.class} from #{@start-100} to #{@start}" search_emails(response.body) end else return response.error! end end rescue Net::HTTPFatalError ESearchy::LOG.puts "Error: Something went wrong with the HTTP request" end @start = 0 end |