Class: SDL4R::Parser::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/sdl4r/parser/tokenizer.rb

Overview

Tokenizer of the SDL parser

Constant Summary collapse

TOKEN_TYPES =

:nodoc: all

[
:IDENTIFIER,

# punctuation
:COLON, :SEMICOLON, :EQUALS, :START_BLOCK, :END_BLOCK,

# literals
:STRING, :CHARACTER, :BOOLEAN, :NUMBER, :DATE, :TIME, :BINARY, :NULL ]

Instance Method Summary collapse

Constructor Details

#initialize(io) ⇒ Tokenizer

Creates an SDL tokenizer on the specified IO.

Raises:

  • (ArgumentError) 


43
44
45
46
47
48
49
50
51
 
# File 'lib/sdl4r/parser/tokenizer.rb', line 43

def initialize(io)
  raise ArgumentError, "io == nil" if io.nil?

  @reader = Parser::Reader.new(io)
  @token_start = 0
  @startEscapedQuoteLine = false
  @tokens = nil
  @tokenText = nil
end

Instance Method Details

#close ⇒ Object

Closes this Tokenizer and its underlying Reader.



54
55
56
 
# File 'lib/sdl4r/parser/tokenizer.rb', line 54

def close
  @reader.close
end

#expecting_but_got(expecting, got, line, position) ⇒ Object

Close the reader and throw a SdlParseError using the format Was expecting X but got Y.



77
78
79
 
# File 'lib/sdl4r/parser/tokenizer.rb', line 77

def expecting_but_got(expecting, got, line, position)
  parse_error("Was expecting #{expecting} but got #{got}", line, position)
end

#line ⇒ Object 



102
103
104
 
# File 'lib/sdl4r/parser/tokenizer.rb', line 102

def line
  @reader.line
end

#line_no ⇒ Object 



94
95
96
 
# File 'lib/sdl4r/parser/tokenizer.rb', line 94

def line_no
  @reader.line_no
end

#parse_error(description, line_no = nil, position = nil) ⇒ Object

Close the reader and throw a SdlParseError.

Raises:



59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/sdl4r/parser/tokenizer.rb', line 59

def parse_error(description, line_no = nil, position = nil)
  begin
    @reader.close()
  rescue IOError
    # no recourse
  end

  line_no = @reader.line_no if line_no.nil?
  position = @reader.pos if position.nil?

  # We add one because editors typically start with line 1 and position 1
  # rather than 0...
  raise SdlParseError.new(description, line_no + 1, position + 1, @reader.line)
end

#pos ⇒ Object 



98
99
100
 
# File 'lib/sdl4r/parser/tokenizer.rb', line 98

def pos
  @reader.pos
end

#read_line_tokens ⇒ Object

Returns the next line as tokens or nil if the end of the stream has been reached. This method handles line continuations both within and outside String literals. The line of tokens is assigned to @tokens.

Returns a logical line as a list of Tokens.



87
88
89
90
91
92
 
# File 'lib/sdl4r/parser/tokenizer.rb', line 87

def read_line_tokens
  begin
    read_line_tokens_even_if_empty()
  end until @tokens.nil? or !@tokens.empty?
  return @tokens
end