Class: TBMX::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/tbmx.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(text) ⇒ Tokenizer

Returns a new instance of Tokenizer.



208
209
210
211
 
# File 'lib/tbmx.rb', line 208

def initialize(text)
  @text = text
  tokenize
end

Instance Attribute Details

#text ⇒ Object (readonly)

Returns the value of attribute text.



207
208
209
 
# File 'lib/tbmx.rb', line 207

def text
  @text
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



207
208
209
 
# File 'lib/tbmx.rb', line 207

def tokens
  @tokens
end

Instance Method Details

#tokenize ⇒ Object 



213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
 
# File 'lib/tbmx.rb', line 213

def tokenize
  @tokens = []
  rest = text.gsub("\r", "")
  while rest.length > 0
    if result =     BackslashToken.matches?(rest) or # Single Character Tokens
       result =     LeftBraceToken.matches?(rest) or
       result =    RightBraceToken.matches?(rest) or
       result = EmptyNewlinesToken.matches?(rest) or # String Tokens
       result =    WhitespaceToken.matches?(rest) or
       result =        NumberToken.matches?(rest) or
       result =          WordToken.matches?(rest)
    then
      @tokens << result[0]
      rest = result[1]
    else
      raise RuntimeError, "Couldn't tokenize the remaining text."
    end
  end
  return @tokens
end