Class: LSH::Storage::RedisBackend

Inherits:
Object
  • Object
show all
Defined in:
lib/lsh/storage/redis_backend.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(params = { :redis => { :host => '127.0.0.1', :port => 6379 }, :data_dir => 'data' }) ⇒ RedisBackend

Returns a new instance of RedisBackend.



28
29
30
31
32
33
34
35
 
# File 'lib/lsh/storage/redis_backend.rb', line 28

def initialize(params = { :redis => { :host => '127.0.0.1', :port => 6379 }, :data_dir => 'data' })
  @redis = Redis.new(params[:redis])
  @data_dir = params[:data_dir]
  unless File.exists?(@data_dir)
    Dir.mkdir(@data_dir)
    Dir.mkdir(File.join(@data_dir, 'projections'))
  end
end

Instance Attribute Details

#data_dir ⇒ Object (readonly)

Returns the value of attribute data_dir.



26
27
28
 
# File 'lib/lsh/storage/redis_backend.rb', line 26

def data_dir
  @data_dir
end

#redis ⇒ Object (readonly)

Returns the value of attribute redis.



26
27
28
 
# File 'lib/lsh/storage/redis_backend.rb', line 26

def redis
  @redis
end

Instance Method Details

#add_vector_id(vector, id) ⇒ Object 



126
127
128
129
130
 
# File 'lib/lsh/storage/redis_backend.rb', line 126

def add_vector_id(vector, id)
  save_vector(vector) # Writing vector to disk if not already there
  @redis.set "lsh:vector_to_id:#{vector.hash}", id
  @redis.set "lsh:id_to_vector:#{id}", vector.hash.to_s
end

#add_vector_to_bucket(bucket, hash, vector) ⇒ Object 



121
122
123
124
 
# File 'lib/lsh/storage/redis_backend.rb', line 121

def add_vector_to_bucket(bucket, hash, vector)
  save_vector(vector) # Writing vector to disk if not already there
  @redis.sadd "#{bucket}:#{hash}", vector.hash.to_s # Only storing vector's hash in Redis
end

#clear_data! ⇒ Object 



42
43
44
45
46
 
# File 'lib/lsh/storage/redis_backend.rb', line 42

def clear_data!
  keys = @redis.keys("lsh:bucket:*")
  @redis.del(keys) unless keys.empty?
  delete_dat_files_in_dir(@data_dir)
end

#clear_projections! ⇒ Object 



48
49
50
51
52
 
# File 'lib/lsh/storage/redis_backend.rb', line 48

def clear_projections!
  @redis.del("lsh:parameters")
  @redis.del("lsh:buckets")
  delete_dat_files_in_dir(File.join(@data_dir, 'projections'))
end

#create_new_bucket ⇒ Object 



106
107
108
 
# File 'lib/lsh/storage/redis_backend.rb', line 106

def create_new_bucket
  @redis.incr "lsh:buckets"
end

#delete_dat_files_in_dir(dir) ⇒ Object 



54
55
56
 
# File 'lib/lsh/storage/redis_backend.rb', line 54

def delete_dat_files_in_dir(dir)
  Dir.foreach(dir) {|f| File.delete(File.join(dir, f)) if f != '.' and f != '..' and f.end_with?('.dat')}
end

#find_bucket(i) ⇒ Object 



141
142
143
 
# File 'lib/lsh/storage/redis_backend.rb', line 141

def find_bucket(i)
  "lsh:bucket:#{i}"
end

#has_index? ⇒ Boolean

Returns:

  • (Boolean) 


58
59
60
 
# File 'lib/lsh/storage/redis_backend.rb', line 58

def has_index?
  parameters and projections and number_of_buckets > 0
end

#id_to_vector(id) ⇒ Object 



136
137
138
139
 
# File 'lib/lsh/storage/redis_backend.rb', line 136

def id_to_vector(id)
  vector_hash = @redis.get "lsh:id_to_vector:#{id}"
  load_vector(vector_hash)
end

#load_vector(hash) ⇒ Object 



115
116
117
118
119
 
# File 'lib/lsh/storage/redis_backend.rb', line 115

def load_vector(hash)
  vector = MathUtil.zeros(1, parameters[:dim])
  vector.load(File.join(@data_dir, hash+'.dat'))
  vector
end

#number_of_buckets ⇒ Object 



62
63
64
 
# File 'lib/lsh/storage/redis_backend.rb', line 62

def number_of_buckets
  @redis.get("lsh:buckets").to_i || 0
end

#parameters ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/lsh/storage/redis_backend.rb', line 93

def parameters
  begin
    @parms ||= (
      parms = JSON.parse(@redis.get "lsh:parameters")
      parms.keys.each { |k| parms[k.to_sym] = parms[k]; parms.delete(k) }
      parms[:window] = Float::INFINITY if parms[:window] == 'Infinity'
      parms
    )
  rescue TypeError
    nil
  end 
end

#parameters=(parms) ⇒ Object 



88
89
90
91
 
# File 'lib/lsh/storage/redis_backend.rb', line 88

def parameters=(parms)
  parms[:window] = 'Infinity' if parms[:window] == Float::INFINITY
  @redis.set "lsh:parameters", parms.to_json
end

#projections ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/lsh/storage/redis_backend.rb', line 75

def projections
  return unless parameters
  @projections ||= (
    projections = []
    parameters[:number_of_independent_projections].times do |i|
      m = MathUtil.zeros(parameters[:dim], parameters[:number_of_random_vectors])
      m.load(File.join(@data_dir, 'projections', "projection_#{i}.dat"))
      projections << m
    end
    projections
  )
end

#projections=(projections) ⇒ Object 



66
67
68
69
70
71
72
73
 
# File 'lib/lsh/storage/redis_backend.rb', line 66

def projections=(projections)
  # Saving the projections to disk
  # (too slow to serialize and store in Redis for
  # large number of dimensions/projections)
  projections.each_with_index do |projection, i|
    projection.save(File.join(@data_dir, 'projections', "projection_#{i}.dat"))
  end
end

#query_buckets(hashes) ⇒ Object 



145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/lsh/storage/redis_backend.rb', line 145

def query_buckets(hashes)
  vector_hashes = []
  hashes.each_with_index do |hash, i|
    bucket = find_bucket(i)
    results = @redis.smembers("#{bucket}:#{hash}")
    vector_hashes += results if results
  end
  # Making sure we don't load the same vectors twice if they match
  # in different random projections
  vector_hashes.uniq!
  results = []
  vector_hashes.each do |vector_hash|
    vector = load_vector(vector_hash)
    results << vector
  end
  results
end

#reset! ⇒ Object 



37
38
39
40
 
# File 'lib/lsh/storage/redis_backend.rb', line 37

def reset!
  clear_data!
  clear_projections!
end

#save_vector(vector) ⇒ Object 



110
111
112
113
 
# File 'lib/lsh/storage/redis_backend.rb', line 110

def save_vector(vector)
  path = File.join(@data_dir, vector.hash.to_s+'.dat')
  vector.save(path) unless File.exists?(path)
end

#vector_to_id(vector) ⇒ Object 



132
133
134
 
# File 'lib/lsh/storage/redis_backend.rb', line 132

def vector_to_id(vector)
  @redis.get "lsh:vector_to_id:#{vector.hash}"
end