Class: FtsLite::Tokenizer::Trigram

Inherits:
Object
  • Object
show all
Defined in:
lib/fts_lite/tokenizer.rb

Instance Method Summary collapse

Instance Method Details

#query(text, options = {}) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
 
# File 'lib/fts_lite/tokenizer.rb', line 64

def query(text, options = {})
  fuzzy = options.key?(:fuzzy) ? options[:fuzzy] : false
  near = fuzzy ? NEAR2 : NEAR0
  text = Tokenizer.normalize(text)
  text.split(QUERY_DELIMITER).map {|segment|
    segment.split(SIMPLE_DELIMITER).map {|word|
      0.upto(word.size - 3).map {|i| word[i, 3] }
    }.join(near)
  }.flatten.join(" ")
end

#split(text) ⇒ Object 



77
78
79
80
81
82
 
# File 'lib/fts_lite/tokenizer.rb', line 77

def split(text)
  text = Tokenizer.normalize(text)
  text.split(SIMPLE_DELIMITER).map {|word|
    0.upto(word.size - 3).map {|i| word[i, 3] }
  }.flatten
end

#vector(text) ⇒ Object 



74
75
76
 
# File 'lib/fts_lite/tokenizer.rb', line 74

def vector(text)
  split(text).join(" ")
end