Class: FtsLite::Tokenizer::Wakachi

Inherits:
Object
  • Object
show all
Defined in:
lib/fts_lite/tokenizer.rb

Instance Method Summary collapse

Instance Method Details

#query(text, options = {}) ⇒ Object 



85
86
87
88
89
90
91
92
93
94
 
# File 'lib/fts_lite/tokenizer.rb', line 85

def query(text, options = {})
  fuzzy = options.key?(:fuzzy) ? options[:fuzzy] : false
  near = fuzzy ? NEAR2 : NEAR0
  text = Tokenizer.normalize(text)
  text.split(QUERY_DELIMITER).map {|segment|
    BimyouSegmenter.segment(segment,
                            :white_space => false,
                            :symbol => false).join(near)
  }.join(" ")
end

#split(text) ⇒ Object 



98
99
100
101
102
 
# File 'lib/fts_lite/tokenizer.rb', line 98

def split(text)
  BimyouSegmenter.segment(Tokenizer.normalize(text),
                          :white_space => false,
                          :symbol => false)
end

#vector(text) ⇒ Object 



95
96
97
 
# File 'lib/fts_lite/tokenizer.rb', line 95

def vector(text)
  split(text).join(" ")
end