21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
# File 'lib/scanner.rb', line 21
def getToken
unless @needToken
@needToken = true
return @lastToken
end
state = 0
foundOne = false
c = @istream.getc()
if @istream.eof() then
@lastToken = Token.new(:eof,@lineCount,@colCount)
return @lastToken
end
until foundOne
@colCount = @colCount + 1
case state
when 0
lex = ""
column = @colCount
line = @lineCount
if isLetter(c) then state=1
elsif isDigit(c) then state=2
elsif c == ?+ then state = 3
elsif c == ?- then state = 4
elsif c == ?* then state = 5
elsif c == ?/ then state = 6
elsif c == ?( then state = 7
elsif c == ?) then state = 8
elsif c == ?% then state = 9
elsif c == ?= then state = 10
elsif c == ?\n then
@colCount = -1
@lineCount = @lineCount+1
elsif isWhiteSpace(c) then state = state
elsif @istream.eof() then
@foundOne = true
type = :eof
else
raise UnrecognizedTokenException
end
when 1
if isLetter(c) or isDigit(c) then state = 1
else
if @keywords.include?(lex) then
foundOne = true
type = :keyword
else
foundOne = true
type = :identifier
end
end
when 2
if isDigit(c) then state = 2
else
type = :number
foundOne = true
end
when 3
type = :add
foundOne = true
when 4
type = :sub
foundOne = true
when 5
type = :times
foundOne = true
when 6
type = :divide
foundOne = true
when 7
type = :lparen
foundOne = true
when 8
type = :rparen
foundOne = true
when 9
type = :module
foundOne = true
when 10
type = :equ
foundOne = true
end
if !foundOne then
lex.concat(c)
if @istream.eof() and state == 0 then
@lastToken = Token.new(:eof,@lineCount,@colCount)
return @lastToken
end
c = @istream.getc()
end
end
@istream.ungetc(c)
@colCount = @colCount - 1
if type == :number or type == :identifier or type == :keyword then
t = LexicalToken.new(type,lex,line,column)
else
t = Token.new(type,line,column)
end
@lastToken = t
return t
end
|