Class: Yahoo
- Inherits:
-
Object
- Object
- Yahoo
- Includes:
- Searchy
- Defined in:
- lib/esearchy/SearchEngines/yahoo.rb
Instance Attribute Summary collapse
-
#appid ⇒ Object
Returns the value of attribute appid.
-
#emails ⇒ Object
Returns the value of attribute emails.
Instance Method Summary collapse
-
#initialize(maxhits = 0, appid = nil, start = 0) ⇒ Yahoo
constructor
A new instance of Yahoo.
- #parse(json) ⇒ Object
- #search(query) ⇒ Object
Methods included from Searchy
#clean, #fix, #hash_url, #maxhits=, #print_emails, #search_depth, #search_docs, #search_emails, #search_office_xml, #search_pdfs, #search_txts
Constructor Details
#initialize(maxhits = 0, appid = nil, start = 0) ⇒ Yahoo
Returns a new instance of Yahoo.
8 9 10 11 12 13 14 15 16 17 18 19 20 |
# File 'lib/esearchy/SearchEngines/yahoo.rb', line 8 def initialize(maxhits=0, appid = nil, start=0) @appid = appid || Keys::YAHOO_APP_KEY @start = start @totalhits = maxhits @emails = [] @r_urls = Queue.new @r_docs = Queue.new @r_pdfs = Queue.new @r_officexs = Queue.new @r_txts = Queue.new @threads = [] @lock = Mutex.new end |
Instance Attribute Details
#appid ⇒ Object
Returns the value of attribute appid.
21 22 23 |
# File 'lib/esearchy/SearchEngines/yahoo.rb', line 21 def appid @appid end |
#emails ⇒ Object
Returns the value of attribute emails.
21 22 23 |
# File 'lib/esearchy/SearchEngines/yahoo.rb', line 21 def emails @emails end |
Instance Method Details
#parse(json) ⇒ Object
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 |
# File 'lib/esearchy/SearchEngines/yahoo.rb', line 56 def parse(json) doc = JSON.parse(json) @totalhits = doc["ysearchresponse"]["totalhits"].to_i if @totalhits == 0 doc["ysearchresponse"]["resultset_web"].each do |result| case result["url"] when /.pdf$/i @r_pdfs << result["url"] when /.docx$|.xlsx$|.pptx$|.odt$|.odp$|.ods$|.odb$/i @r_officexs << result["url"] when /.doc$/i @r_docs << result["url"] when /.txt$|.rtf$|ans$/i @r_txts << result["url"] else @r_urls << result["url"] end end end |
#search(query) ⇒ Object
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 |
# File 'lib/esearchy/SearchEngines/yahoo.rb', line 23 def search(query) @query = query begin http = Net::HTTP.new("boss.yahooapis.com",80) http.start do |http| request = Net::HTTP::Get.new("/ysearch/web/v1/" + CGI.escape(query) + "?appid="+ @appid + "&format=json&count=50"+ "&start=#{@start}", {'User-Agent' => UserAgent::fetch} ) response = http.request(request) case response when Net::HTTPSuccess, Net::HTTPRedirection parse(response.body) @start = @start + 50 if @totalhits > @start ESearchy::LOG.puts "Searching #{self.class} from #{@start-50} to #{@start}" search_emails(response.body) sleep(ESearchy::DELAY) search(@query) else ESearchy::LOG.puts "Searching #{self.class} from #{@start-50} to #{@start}" search_emails(response.body) end else return response.error! end end rescue Net::HTTPFatalError ESearchy::LOG.puts "Error: Something went wrong with the HTTP request" end @start = 0 end |