Class: Krackle::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/krackle.rb

Instance Method Summary collapse

Constructor Details

#initialize(str) ⇒ Tokenizer

Returns a new instance of Tokenizer.



16
17
18
 
# File 'lib/krackle.rb', line 16

def initialize(str)
  @str = str
end

Instance Method Details

#tokenize ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/krackle.rb', line 20

def tokenize
  scanner = StringScanner.new(@str)
  tokens = []
  until scanner.empty?
    case 
      when match = scanner.scan(/\[(\d+)?\]/)
        tokens << [:COLLECTION, (scanner[1] ? scanner[1].to_i : nil)]
      when match =scanner.scan(/\.?(\w+)/)
        tokens << [:KEY, scanner[1]]
      else
        raise "Say what? <#{scanner.peek(10).inspect}> at pos #{scanner.pos}"
      end
  end
  tokens
end