Module: FtsLite::Tokenizer
- Defined in:
- lib/fts_lite/tokenizer.rb
Defined Under Namespace
Classes: Bigram, Simple, Trigram, Wakachi, WakachiBigram
Constant Summary
collapse
- QUERY_DELIMITER =
/[\s ]+/
- SIMPLE_DELIMITER =
/[\s \.\*"',\?!;\(\)。、.,?!「」『』()]+/
- NEAR0 =
" NEAR/0 "
- NEAR2 =
" NEAR/2 "
Class Method Summary
collapse
Class Method Details
.create(name) ⇒ Object
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
|
# File 'lib/fts_lite/tokenizer.rb', line 12
def self.create(name)
case name.to_sym
when :simple
Simple.new
when :bigram
Bigram.new
when :trigram
Trigram.new
when :wakachi
Wakachi.new
when :wakachi_bigram
WakachiBigram.new
else
raise ArgumentError
end
end
|
.normalize(text) ⇒ Object
28
29
30
|
# File 'lib/fts_lite/tokenizer.rb', line 28
def self.normalize(text)
NKF::nkf('-wZX', text).downcase
end
|