Class: Rspider::HDBContentStorage

Inherits:
Object
  • Object
show all
Defined in:
lib/rspider/ContentStorage.rb

Overview

this class store the content in tokyocabinet database so she can get perfect performance and uses little memory

Instance Method Summary collapse

Constructor Details

#initialize(path) ⇒ HDBContentStorage

the file path to hold the HDB file



37
38
39
40
41
42
43
 
# File 'lib/rspider/ContentStorage.rb', line 37

def initialize(path)
	@hdb = TokyoCabinet::HDB::new
	if(!@hdb.open(path, TokyoCabinet::HDB::OWRITER | TokyoCabinet::HDB::OCREAT))
		ecode = @hdb.ecode
		STDERR.printf("open error: %s\n", @hdb.errmsg(ecode))
	end
end

Instance Method Details

#add(url, content) ⇒ Object

store an url and content of the url



45
46
47
 
# File 'lib/rspider/ContentStorage.rb', line 45

def add(url,content)
	@hdb.put(url,content)
end

#close ⇒ Object

close the db



49
50
51
 
# File 'lib/rspider/ContentStorage.rb', line 49

def close
	@hdb.close
end

#get(url) ⇒ Object

fetch the content of specificed url



62
63
64
 
# File 'lib/rspider/ContentStorage.rb', line 62

def get(url)
	return @hdb.get(url)	
end

#urls ⇒ Object

list all the urls



53
54
55
56
57
58
59
60
 
# File 'lib/rspider/ContentStorage.rb', line 53

def urls
	@hdb.iterinit
	keys=[]
	while(key = @hdb.iternext)
		keys<< key
	end
	keys
end