Class: CldrPlurals::Compiler::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/cldr-plurals/compiler/tokenizer.rb

Constant Summary collapse

TOKENS = 
{
  /@integer/ => :int_sample,
  /@decimal/ => :dec_sample,
  /\u2026/   => :infinite_set,
  /~/        => :sample_range,
  /and/      => :and,
  /or/       => :or,
  /[niftvw]/ => :operand,
  /,/        => :comma,
  /\.\./     => :range,
  /%/        => :modulo,
  /=/        => :equals,
  /\!=/      => :not_equals,
  /[\d]+/    => :number
}
ALL_TOKENS = 
Regexp.compile(
  TOKENS.map { |r, _| r.source }.join('|')
)

Class Method Summary collapse

Class Method Details

.tokenize(text) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/cldr-plurals/compiler/tokenizer.rb', line 36

def self.tokenize(text)
  text.scan(ALL_TOKENS).each_with_object([]) do |token, ret|
    found_type = TOKENS.each_pair do |regex, token_type|
      break token_type if token =~ regex
    end

    if found_type.is_a?(Symbol)
      ret << make_token(token, found_type)
    end
  end
end