Class: TwitterCldr::Tokenizers::SegmentationTokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/twitter_cldr/tokenizers/segmentation/segmentation_tokenizer.rb

Instance Method Summary collapse

Instance Method Details

#tokenize(pattern) ⇒ Object 



10
11
12
13
14
15
 
# File 'lib/twitter_cldr/tokenizers/segmentation/segmentation_tokenizer.rb', line 10

def tokenize(pattern)
  # according to the spec, whitespace should be ignored
  tokenizer.tokenize(pattern).reject do |token|
    token.value.strip.empty?
  end
end