Class: Sie::Parser::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/sie/parser/tokenizer.rb,
lib/sie/parser/tokenizer/token.rb

Defined Under Namespace

Classes: ArrayToken, BeginArrayToken, EndArrayToken, EntryToken, StringToken, Token

Instance Method Summary collapse

Instance Method Details

#tokenize ⇒ Object 



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/sie/parser/tokenizer.rb', line 9

def tokenize
  tokens = []
  check_for_control_characters

  loop do
    case
    when whitespace?
      next
    when match = find_entry
      tokens << EntryToken.new(match)
    when begin_array?
      tokens << BeginArrayToken.new
    when end_array?
      tokens << EndArrayToken.new
    when match = find_string
      tokens << StringToken.new(match)
    when end_of_string?
      break
    else
      # We shouldn't get here, but if we do we need to bail out, otherwise we get an infinite loop.
      fail "Unhandled character in line at position #{scanner.pos}: " + scanner.string
    end
  end

  tokens
end