Class: Sie::Parser::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/sie/parser/tokenizer.rb,
lib/sie/parser/tokenizer/token.rb,
lib/sie/parser/tokenizer/character.rb

Defined Under Namespace

Classes: ArrayToken, BeginArrayToken, Character, EndArrayToken, EntryToken, StringToken, Token

Instance Method Summary collapse

Instance Method Details

#tokenize ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/sie/parser/tokenizer.rb', line 10

def tokenize
  @tokens = []
  @consume = false
  @quoted = false

  loop do
    move_to_next_character
    break unless current_character.value

    if consume?
      if quoted?
        consume_quoted_value
      else
        consume_unquoted_value
      end
    else
      add_new_token
    end
  end

  tokens
end