Class: CTokenizer::Lexer

Inherits:
StringScanner
  • Object
show all
Includes:
CTokenizer
Defined in:
lib/dbc/ctokenizer.rb

Constant Summary

Constants included from CTokenizer

EOF_TOKEN

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from CTokenizer

check_token, #collect, #each, #error, error, line_count, #parse_error, #to_a, #token_error, #warning, whitespace?

Constructor Details

#initialize(str, file = nil, line = 1) ⇒ Lexer

Returns a new instance of Lexer.



127
128
129
130
131
132
 
# File 'lib/dbc/ctokenizer.rb', line 127

def initialize(str, file=nil, line=1)
	str.freeze
	super(str, false) # DO NOT dup str
	@file = file
	@line = line
end

Instance Attribute Details

#file ⇒ Object (readonly)

Returns the value of attribute file.



134
135
136
 
# File 'lib/dbc/ctokenizer.rb', line 134

def file
  @file
end

#line ⇒ Object (readonly)

Returns the value of attribute line.



134
135
136
 
# File 'lib/dbc/ctokenizer.rb', line 134

def line
  @line
end

Instance Method Details

#shift ⇒ Object 



138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
 
# File 'lib/dbc/ctokenizer.rb', line 138

def shift
	# don't need \A in regexp's cause StringScanner does this automatically.
	t = case
		when m = scan(Expression::SPACE)
			@line += CTokenizer.line_count(m)
			[:SPACE, m]
		when m = scan(Expression::IDENTIFIER)
			[:IDENTIFIER, m]
		when m = scan(Expression::COMMENT)
			@line += CTokenizer.line_count(m)
			[:COMMENT, m]
		# SYMBOL should come before INTEGER and FLOAT
		when m = scan(Expression::SYMBOL)
			[:SYMBOL, m]
		when m = scan(Expression::NEWLINE)
			@line += CTokenizer.line_count(m)
			[:NEWLINE, m]
		# FLOAT should come before INTEGER
		when m = scan(Expression::FLOAT)
			[:FLOAT, m]
		when m = scan(Expression::INTEGER)
			[:INTEGER, m]
		when m = scan(Expression::CHARACTER)
			[:CHARACTER, m]
		when m = scan(Expression::STRING)
			[:STRING, m]
		when eos?
			EOF_TOKEN # end of file, \Z don't work with StringScanner
		when m = getch
			@line += CTokenizer.line_count(m)
			[:UNKNOWN, m]
		else
			raise "shouldn't get here!"
	end # case
	m.freeze
	t.freeze
end