Class: FtsLite::Tokenizer::Simple

Inherits:
Object
  • Object
show all
Defined in:
lib/fts_lite/tokenizer.rb

Instance Method Summary collapse

Instance Method Details

#split(text) ⇒ Object 



32
33
34
 
# File 'lib/fts_lite/tokenizer.rb', line 32

def split(text)
  Tokenizer.normalize(text).gsub(/[\.,\?!;:]/, ' ').split(SIMPLE_DELIMITER)
end

#vector(text) ⇒ Object 



29
30
31
 
# File 'lib/fts_lite/tokenizer.rb', line 29

def vector(text)
  split(text).join(" ")
end