Class: Soulheart::Loader

Inherits:
Base
  • Object
show all
Defined in:
lib/soulheart/loader.rb

Instance Attribute Summary

Attributes inherited from Base

#type

Instance Method Summary collapse

Methods inherited from Base

#base_id, #cache_duration, #cache_id, #categories_id, #category_combos, #category_combos_id, #category_id, #combinatored_category_array, #no_query_id, #redis, #results_hashes_id, #set_category_combos_array, #sorted_category_array

Methods included from Helpers

#normalize, #prefixes_for_phrase

Instance Method Details

#add_item(item, category_base_id = nil, cleaned = false) ⇒ Object 



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/soulheart/loader.rb', line 96

def add_item(item, category_base_id=nil, cleaned=false)
  item = clean(item) unless cleaned
  category_base_id ||= category_id(item['category'])

  priority = (-item['priority'])
  redis.pipelined do
    redis.zadd(no_query_id(category_base_id), priority, item['term']) # Add to master set for queryless searches
    # store the raw data in a separate key to reduce memory usage, if it's cleaned it's done
    redis.hset(results_hashes_id, item['term'], MultiJson.encode(item['data'])) unless cleaned
    phrase = ([item['term']] + (item['aliases'] || [])).join(' ')
    # Store all the prefixes
    prefixes_for_phrase(phrase).each do |p|
      redis.sadd(base_id, p) unless cleaned # remember prefix in a master set
      # store the normalized term in the index for each of the categories
      redis.zadd("#{category_base_id}#{p}", priority, item['term'])
    end
  end
  item
end

#clean(item) ⇒ Object 



85
86
87
88
89
90
91
92
93
94
 
# File 'lib/soulheart/loader.rb', line 85

def clean(item)
  item = clean_hash(item)
  item.keys.select{ |k| !%w(category priority term aliases data).include?(k) }.each do |key|
    item['data'].merge!({"#{key}" => item.delete(key)})
  end
  unless redis.smembers(categories_id).include?(item['category'])
    redis.sadd categories_id, item['category']
  end  
  item
end

#clean_hash(item) ⇒ Object 



76
77
78
79
80
81
82
83
 
# File 'lib/soulheart/loader.rb', line 76

def clean_hash(item)
  item['aliases'] = item['aliases'].split(',').map(&:strip) if item['aliases'] && !item['aliases'].kind_of?(Array)
  fail ArgumentError, 'Items must have text' unless item['text']
  default_items_hash(item.delete('text'), item.delete('category'))
    .tap { |i| i['data'].merge!(item.delete('data')) if item['data'] }
    .tap { |i| i['priority'] = item.delete('priority').to_f if item['priority'] }
    .merge item
end

#clear(remove_results = false) ⇒ Object 



52
53
54
55
56
57
 
# File 'lib/soulheart/loader.rb', line 52

def clear(remove_results=false)
  category_combos.each {|cat| delete_data(category_id(cat)) }
  delete_categories
  delete_data
  remove_results_hash if remove_results
end

#default_items_hash(text, category) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
 
# File 'lib/soulheart/loader.rb', line 3

def default_items_hash(text, category)
  category ||= 'default'
  {
    'category' => normalize(category),
    'priority' => 100,
    'term' => normalize(text),
    'aliases' => [],
    'data' => {
      'text' => text,
      'category' => category
    }
  }
end

#delete_categories ⇒ Object 



17
18
19
20
 
# File 'lib/soulheart/loader.rb', line 17

def delete_categories
  redis.expire category_combos_id, 0
  redis.expire categories_id, 0
end

#delete_data(id = "#{base_id}:") ⇒ Object 



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/soulheart/loader.rb', line 27

def delete_data(id="#{base_id}:")
  # delete the sorted sets for this type
  phrases = redis.smembers(base_id)
  redis.pipelined do
    phrases.each do |p|
      redis.del("#{id}#{p}")
    end
    redis.del(id)
  end

  # Redis can continue serving cached requests while the reload is
  # occurring. Some requests may be cached incorrectly as empty set (for requests
  # which come in after the above delete, but before the loading completes). But
  # everything will work itself out as soon as the cache expires again.
end

#load(items) ⇒ Object 



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/soulheart/loader.rb', line 59

def load(items)
  # Replace with item return so we know we have category_id
  i = 0
  items.each do |item|
    item.replace(add_item(item))
    i += 1
  end
  set_category_combos_array.each do |category_combo|
    items.each do |item|
      next unless category_combo.match(item['category']) || category_combo == 'all'
      add_item(item, category_id(category_combo), true) # send it base
      i += 1
    end
  end
  puts "Total items (including combinatored categories):    #{i}"
end

#remove_results_hash ⇒ Object 



43
44
45
46
47
48
49
50
 
# File 'lib/soulheart/loader.rb', line 43

def remove_results_hash
  # delete the data store
  # We don't do this every time we clear because because it breaks the caching feature. 
  # The option to clear this is only called in testing right now. 
  # There should be an option to clear it other times though.
  redis.expire results_hashes_id, 0
  redis.del(results_hashes_id)
end

#reset_categories(categories) ⇒ Object 



22
23
24
25
 
# File 'lib/soulheart/loader.rb', line 22

def reset_categories(categories)
  delete_categories
  redis.sadd categories_id, categories
end