Class: JSGF::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/jsgf/tokenizer.rb

Constant Summary collapse

TOKENS = 
{
    /grammar/				=> :GRAMMAR,
    /#JSGF/				=> :HEADER,
    /import/				=> :IMPORT,
    /public/				=> :PUBLIC,
    /\{(\\.|[^\}]+)*\}/			=> :TAG,
    /[^ \t\n;=|*+<>\(\)\[\]{}*\/]+/	=> :TOKEN,
    %r{/\d*(\.\d+)?/}			=> :WEIGHT,
}

Instance Method Summary collapse

Constructor Details

#initialize(io) ⇒ Tokenizer

Returns a new instance of Tokenizer.

Parameters:

  • io (IO) —

    the IO stream to read from



16
17
18
19
 
# File 'lib/jsgf/tokenizer.rb', line 16

def initialize(io)
    io = io.read unless io.is_a?(String)
    @scanner = StringScanner.new(io)
end

Instance Method Details

#next_token ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/jsgf/tokenizer.rb', line 21

def next_token
    return if @scanner.eos?

    @scanner.skip(/\s+/)	# Skip all leading whitespace
    @scanner.skip(%r{//.*\n})	# Skip single-line comments

    # Skip C-style comments
    if @scanner.scan(%r{/\*})
	# Look for the end of the block, and skip any trailing whitespace
	@scanner.skip_until(%r{\*/\s*})
    end

    TOKENS.each do |(pattern, token)|
	text = @scanner.scan(pattern)
	return [token, text] if text
    end
    x = @scanner.getch
    [x, x] unless x.nil?
end