35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
|
# File 'app/yarv.rb', line 35
def scan_tokens(tokens, options)
state = :initial
number_expected = true
until eos?
kind = nil
match = nil
case state
when :initial
if match = scan(/^\s*/)
tokens << [match, :space] unless match.empty?
end
state =
if match = scan(/^$/)
:initial
else
:expect_label
end
when :expect_label
state =
if match = scan(/\d+/x)
tokens << [match, :label]
:expect_opcode
else
match = scan(/^.*$/)
tokens << [match, :error]
:initial
end
when :expect_opcode
state =
if match = scan(/(\s+)(\S+)/)
match =~ /(\s+)(\S+)/
tokens << [$1, :space]
opcode = $2
tokens << [opcode, :reserved]
:expect_operand
else
match = scan(/^(.*)$/)
tokens << [match, :error]
:initial
end
when :expect_literal
space_parse(tokens)
if match = scan(/^#<.+>/)
tokens << [match, :content]
elsif match = scan(/^(\d+)/)
tokens << [match, :integer]
elsif match = scan(/^([:][^: ,\n]+)/)
tokens << [match, :symbol]
elsif string_parse(tokens)
elsif match = scan(/nil|true|false/)
tokens << [match, :pre_constant]
elsif match = scan(/\/.*\//)
tokens << [match, :entity]
else
match = scan(/^.*$/)
tokens << [match, :error] unless match.empty?
end
state = :expect_opt_lineno
when :expect_operand
space_parse(tokens)
state = :expect_another_operand
if match = scan(/^(\d+)/)
tokens << [match, :integer]
if match = scan(/^[.]{2}\d+/)
tokens << ['..', :operator]
tokens << [match[2..-1], :integer]
end
elsif match = scan(/\/.*\//)
tokens << [match, :entity]
elsif match = scan(/^([:][^: ,\n]+)/)
tokens << [match, :symbol]
elsif match = scan(/^([$][^ ,\n]+)/)
tokens << [match, :global_variable]
elsif match = scan(/nil|true|false/)
tokens << [match, :pre_constant]
elsif match = scan(/nil|true|false/)
tokens << [match, :pre_constant]
elsif match = scan(/block in (?:<.+>|[^,]+)/)
tokens << ["block in ", :variable]
tokens << [match['block in '.size..-1], :content]
elsif match = scan(/[A-Za-z_][_A-Za-z0-9?!]*/)
tokens << [match, :variable]
elsif match = scan(/^#[^, \n]*/)
tokens << [match, :content]
elsif match = scan(/^<.+>/)
tokens << [match, :content]
elsif string_parse(tokens)
else
state = :expect_opt_lineno
end
when :expect_another_operand
state =
if match = scan(/^,/)
tokens << [match, :operator]
:expect_operand
else
:expect_opt_lineno
end
when :expect_opt_lineno
space_parse(tokens)
if match = scan(/^\(\s+\d+\)$/)
tokens << [match, :comment]
else
match = scan(/^.*$/)
tokens << [match, :error] unless match.empty?
end
state = :initial
end
end
tokens
end
|