Class: LSH::Storage::RedisBackend

Inherits:
Object
  • Object
show all
Defined in:
lib/lsh/storage/redis_backend.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(params = { :redis => { :host => '127.0.0.1', :port => 6379 }, :data_dir => 'data' }) ⇒ RedisBackend

Returns a new instance of RedisBackend.



28
29
30
31
32
33
 
# File 'lib/lsh/storage/redis_backend.rb', line 28

def initialize(params = { :redis => { :host => '127.0.0.1', :port => 6379 }, :data_dir => 'data' })
  @redis = Redis.new(params[:redis])
  @data_dir = params[:data_dir]
  Dir.mkdir(@data_dir) unless File.exists?(@data_dir)
  Dir.mkdir(File.join(@data_dir, 'projections')) unless File.exists?(File.join(@data_dir, 'projections'))
end

Instance Attribute Details

#data_dir ⇒ Object (readonly)

Returns the value of attribute data_dir.



26
27
28
 
# File 'lib/lsh/storage/redis_backend.rb', line 26

def data_dir
  @data_dir
end

#redis ⇒ Object (readonly)

Returns the value of attribute redis.



26
27
28
 
# File 'lib/lsh/storage/redis_backend.rb', line 26

def redis
  @redis
end

Instance Method Details

#add_vector(vector, vector_hash) ⇒ Object 



123
124
125
 
# File 'lib/lsh/storage/redis_backend.rb', line 123

def add_vector(vector, vector_hash)
  save_vector(vector, vector_hash) # Writing vector to disk if not already there
end

#add_vector_hash_to_bucket(bucket, hash, vector_hash) ⇒ Object 



127
128
129
 
# File 'lib/lsh/storage/redis_backend.rb', line 127

def add_vector_hash_to_bucket(bucket, hash, vector_hash)
  @redis.sadd "#{bucket}:#{hash}", vector_hash.to_s # Only storing vector's hash in Redis
end

#add_vector_id(vector_hash, id) ⇒ Object 



131
132
133
134
 
# File 'lib/lsh/storage/redis_backend.rb', line 131

def add_vector_id(vector_hash, id)
  @redis.set "lsh:vector_to_id:#{vector_hash}", id
  @redis.set "lsh:id_to_vector:#{id}", vector_hash.to_s
end

#clear_data! ⇒ Object 



40
41
42
43
44
45
46
47
48
 
# File 'lib/lsh/storage/redis_backend.rb', line 40

def clear_data!
  keys = @redis.keys("lsh:bucket:*")
  @redis.del(keys) unless keys.empty?
  keys = @redis.keys("lsh:vector_to_id:*")
  @redis.del(keys) unless keys.empty?
  keys = @redis.keys("lsh:id_to_vector:*")
  @redis.del(keys) unless keys.empty?
  delete_dat_files_in_dir(@data_dir)
end

#clear_projections! ⇒ Object 



50
51
52
53
54
 
# File 'lib/lsh/storage/redis_backend.rb', line 50

def clear_projections!
  @redis.del("lsh:parameters")
  @redis.del("lsh:buckets")
  delete_dat_files_in_dir(File.join(@data_dir, 'projections'))
end

#create_new_bucket ⇒ Object 



108
109
110
 
# File 'lib/lsh/storage/redis_backend.rb', line 108

def create_new_bucket
  @redis.incr "lsh:buckets"
end

#delete_dat_files_in_dir(dir) ⇒ Object 



56
57
58
 
# File 'lib/lsh/storage/redis_backend.rb', line 56

def delete_dat_files_in_dir(dir)
  Dir.foreach(dir) {|f| File.delete(File.join(dir, f)) if f != '.' and f != '..' and f.end_with?('.dat')}
end

#find_bucket(i) ⇒ Object 



145
146
147
 
# File 'lib/lsh/storage/redis_backend.rb', line 145

def find_bucket(i)
  "lsh:bucket:#{i}"
end

#has_index? ⇒ Boolean

Returns:

  • (Boolean) 


60
61
62
 
# File 'lib/lsh/storage/redis_backend.rb', line 60

def has_index?
  parameters and projections and number_of_buckets > 0
end

#id_to_vector(id) ⇒ Object 



140
141
142
143
 
# File 'lib/lsh/storage/redis_backend.rb', line 140

def id_to_vector(id)
  vector_hash = @redis.get "lsh:id_to_vector:#{id}"
  load_vector(vector_hash)
end

#load_vector(hash) ⇒ Object 



117
118
119
120
121
 
# File 'lib/lsh/storage/redis_backend.rb', line 117

def load_vector(hash)
  vector = MathUtil.zeros(1, parameters[:dim])
  vector.load(File.join(@data_dir, hash+'.dat'))
  vector
end

#number_of_buckets ⇒ Object 



64
65
66
 
# File 'lib/lsh/storage/redis_backend.rb', line 64

def number_of_buckets
  @redis.get("lsh:buckets").to_i || 0
end

#parameters ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/lsh/storage/redis_backend.rb', line 95

def parameters
  begin
    @parms ||= (
      parms = JSON.parse(@redis.get "lsh:parameters")
      parms.keys.each { |k| parms[k.to_sym] = parms[k]; parms.delete(k) }
      parms[:window] = Float::INFINITY if parms[:window] == 'Infinity'
      parms
    )
  rescue TypeError
    nil
  end 
end

#parameters=(parms) ⇒ Object 



90
91
92
93
 
# File 'lib/lsh/storage/redis_backend.rb', line 90

def parameters=(parms)
  parms[:window] = 'Infinity' if parms[:window] == Float::INFINITY
  @redis.set "lsh:parameters", parms.to_json
end

#projections ⇒ Object 



77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/lsh/storage/redis_backend.rb', line 77

def projections
  return unless parameters
  @projections ||= (
    projections = []
    parameters[:number_of_independent_projections].times do |i|
      m = MathUtil.zeros(parameters[:number_of_random_vectors], parameters[:dim])
      m.load(File.join(@data_dir, 'projections', "projection_#{i}.dat"))
      projections << m
    end
    projections
  )
end

#projections=(projections) ⇒ Object 



68
69
70
71
72
73
74
75
 
# File 'lib/lsh/storage/redis_backend.rb', line 68

def projections=(projections)
  # Saving the projections to disk
  # (too slow to serialize and store in Redis for
  # large number of dimensions/projections)
  projections.each_with_index do |projection, i|
    projection.save(File.join(@data_dir, 'projections', "projection_#{i}.dat"))
  end
end

#query_buckets(hashes) ⇒ Object 



149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
 
# File 'lib/lsh/storage/redis_backend.rb', line 149

def query_buckets(hashes)
  results_hashes = {}
  hashes.each_with_index do |hash, i|
    bucket = find_bucket(i)
    vector_hashes_in_bucket = @redis.smembers("#{bucket}:#{hash}")
    if vector_hashes_in_bucket
      vector_hashes_in_bucket.each do |vector_hash|
        results_hashes[vector_hash] = true
      end
    end
  end
  results_hashes.keys.map do |vector_hash|
    {
      :data => load_vector(vector_hash),
      :hash => vector_hash.to_i,
      :id => vector_hash_to_id(vector_hash)
    }
  end
end

#reset! ⇒ Object 



35
36
37
38
 
# File 'lib/lsh/storage/redis_backend.rb', line 35

def reset!
  clear_data!
  clear_projections!
end

#save_vector(vector, vector_hash) ⇒ Object 



112
113
114
115
 
# File 'lib/lsh/storage/redis_backend.rb', line 112

def save_vector(vector, vector_hash)
  path = File.join(@data_dir, vector_hash.to_s+'.dat')
  vector.save(path) unless File.exists?(path)
end

#vector_hash_to_id(vector_hash) ⇒ Object 



136
137
138
 
# File 'lib/lsh/storage/redis_backend.rb', line 136

def vector_hash_to_id(vector_hash)
  @redis.get "lsh:vector_to_id:#{vector_hash}"
end