Class: FtsLite::Tokenizer::Trigram

Inherits:
Object
  • Object
show all
Defined in:
lib/fts_lite/tokenizer.rb

Instance Method Summary collapse

Instance Method Details

#split(text) ⇒ Object 



51
52
53
54
55
56
 
# File 'lib/fts_lite/tokenizer.rb', line 51

def split(text)
  text = Tokenizer.normalize(text)
  text.split(SIMPLE_DELIMITER).map {|word|
    0.upto(word.size - 3).map {|i| word[i, 3] }
  }.flatten
end

#vector(text) ⇒ Object 



48
49
50
 
# File 'lib/fts_lite/tokenizer.rb', line 48

def vector(text)
  split(text).join(" ")
end