Class: TwitterCldr::Tokenizers::PatternTokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/twitter_cldr/tokenizers/pattern_tokenizer.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(data_reader, tokenizer) ⇒ PatternTokenizer

Returns a new instance of PatternTokenizer.



12
13
14
15
 
# File 'lib/twitter_cldr/tokenizers/pattern_tokenizer.rb', line 12

def initialize(data_reader, tokenizer)
  @data_reader = data_reader
  @tokenizer = tokenizer
end

Instance Attribute Details

#data_reader ⇒ Object (readonly)

Returns the value of attribute data_reader.



10
11
12
 
# File 'lib/twitter_cldr/tokenizers/pattern_tokenizer.rb', line 10

def data_reader
  @data_reader
end

#tokenizer ⇒ Object (readonly)

Returns the value of attribute tokenizer.



10
11
12
 
# File 'lib/twitter_cldr/tokenizers/pattern_tokenizer.rb', line 10

def tokenizer
  @tokenizer
end

Instance Method Details

#tokenize(pattern) ⇒ Object 



17
18
19
 
# File 'lib/twitter_cldr/tokenizers/pattern_tokenizer.rb', line 17

def tokenize(pattern)
  tokenizer.tokenize(expand(pattern))
end