Class: Dentaku::Tokenizer
- Inherits:
-
Object
- Object
- Dentaku::Tokenizer
- Defined in:
- lib/dentaku/tokenizer.rb
Constant Summary collapse
- LPAREN =
TokenMatcher.new(:grouping, [:open, :fopen])
- RPAREN =
TokenMatcher.new(:grouping, :close)
Instance Method Summary collapse
Instance Method Details
#scan(string, scanner) ⇒ Object
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
# File 'lib/dentaku/tokenizer.rb', line 27 def scan(string, scanner) if tokens = scanner.scan(string) tokens.each do |token| raise "unexpected zero-width match (:#{ token.category }) at '#{ string }'" if token.length == 0 @nesting += 1 if LPAREN == token @nesting -= 1 if RPAREN == token raise "too many closing parentheses" if @nesting < 0 @tokens << token unless token.is?(:whitespace) end match_length = tokens.map(&:length).reduce(:+) [true, string[match_length..-1]] else [false, string] end end |
#tokenize(string) ⇒ Object
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 |
# File 'lib/dentaku/tokenizer.rb', line 10 def tokenize(string) @nesting = 0 @tokens = [] input = string.to_s.dup until input.empty? raise "parse error at: '#{ input }'" unless TokenScanner.scanners.any? do |scanner| scanned, input = scan(input, scanner) scanned end end raise "too many opening parentheses" if @nesting > 0 @tokens end |