Class: TBMX::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/tbmx.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(text) ⇒ Tokenizer

Returns a new instance of Tokenizer.



172
173
174
175
 
# File 'lib/tbmx.rb', line 172

def initialize(text)
  @text = text
  tokenize
end

Instance Attribute Details

#text ⇒ Object (readonly)

Returns the value of attribute text.



171
172
173
 
# File 'lib/tbmx.rb', line 171

def text
  @text
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



171
172
173
 
# File 'lib/tbmx.rb', line 171

def tokens
  @tokens
end

Instance Method Details

#tokenize ⇒ Object 



177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/tbmx.rb', line 177

def tokenize
  @tokens = []
  rest = text.gsub("\r", "")
  while rest.length > 0
    if result =     BackslashToken.matches?(rest) or # Single Character Tokens
       result =     LeftBraceToken.matches?(rest) or
       result =    RightBraceToken.matches?(rest) or
       result = EmptyNewlinesToken.matches?(rest) or # String Tokens
       result =    WhitespaceToken.matches?(rest) or
       result =          WordToken.matches?(rest)
    then
      @tokens << result[0]
      rest = result[1]
    else
      raise RuntimeError, "Couldn't tokenize the remaining text."
    end
  end
  return @tokens
end