Class: Antlr4::Runtime::Lexer

Inherits:
Recognizer show all
Defined in:
lib/antlr4/runtime/lexer.rb

Constant Summary collapse

DEFAULT_MODE =
0
MORE =
-2
SKIP =
-3
DEFAULT_TOKEN_CHANNEL =
Token::DEFAULT_CHANNEL
HIDDEN =
Token::HIDDEN_CHANNEL
MIN_CHAR_VALUE =
0x0000
MAX_CHAR_VALUE =
0x10FFFF

Constants inherited from Recognizer

Recognizer::EOF

Instance Attribute Summary collapse

Attributes inherited from Recognizer

#_interp, #_state_number

Instance Method Summary collapse

Methods inherited from Recognizer

#action, #add_error_listener, #error_header, #error_listener_dispatch, #get_rule_index_map, #get_serialized_atn, #get_token_type, #get_token_type_map, #get_vocabulary, #parse_info, #precpred, #remove_error_listener, #remove_error_listeners, #rule_names, #sempred, #token_error_display, #token_names

Constructor Details

#initialize(input = nil) ⇒ Lexer

Returns a new instance of Lexer



45
46
47
48
49
50
51
52
53
54
55
56
# File 'lib/antlr4/runtime/lexer.rb', line 45

def initialize(input = nil)
  super()
  unless input.nil?
    @_input = input
    @_token_factory_source_pair = OpenStruct.new
    @_token_factory_source_pair.a = self
    @_token_factory_source_pair.b = input
  end
  @_mode_stack = []
  reset
  @_factory = CommonTokenFactory.instance
end

Instance Attribute Details

#_channelObject

Returns the value of attribute _channel



19
20
21
# File 'lib/antlr4/runtime/lexer.rb', line 19

def _channel
  @_channel
end

#_hit_eofObject

Returns the value of attribute _hit_eof



18
19
20
# File 'lib/antlr4/runtime/lexer.rb', line 18

def _hit_eof
  @_hit_eof
end

#_inputObject

Returns the value of attribute _input



13
14
15
# File 'lib/antlr4/runtime/lexer.rb', line 13

def _input
  @_input
end

#_modeObject

Returns the value of attribute _mode



22
23
24
# File 'lib/antlr4/runtime/lexer.rb', line 22

def _mode
  @_mode
end

#_mode_stackObject

Returns the value of attribute _mode_stack



21
22
23
# File 'lib/antlr4/runtime/lexer.rb', line 21

def _mode_stack
  @_mode_stack
end

#_textObject

Returns the value of attribute _text



23
24
25
# File 'lib/antlr4/runtime/lexer.rb', line 23

def _text
  @_text
end

#_token_start_char_indexObject

Returns the value of attribute _token_start_char_index



15
16
17
# File 'lib/antlr4/runtime/lexer.rb', line 15

def _token_start_char_index
  @_token_start_char_index
end

#_token_start_char_position_in_lineObject

Returns the value of attribute _token_start_char_position_in_line



17
18
19
# File 'lib/antlr4/runtime/lexer.rb', line 17

def _token_start_char_position_in_line
  @_token_start_char_position_in_line
end

#_token_start_lineObject

Returns the value of attribute _token_start_line



16
17
18
# File 'lib/antlr4/runtime/lexer.rb', line 16

def _token_start_line
  @_token_start_line
end

#_typeObject

Returns the value of attribute _type



20
21
22
# File 'lib/antlr4/runtime/lexer.rb', line 20

def _type
  @_type
end

#tokenObject

Returns the value of attribute token



14
15
16
# File 'lib/antlr4/runtime/lexer.rb', line 14

def token
  @token
end

Instance Method Details

#all_tokensObject



192
193
194
195
196
197
198
199
200
# File 'lib/antlr4/runtime/lexer.rb', line 192

def all_tokens
  tokens = []
  t = next_token
  while t.type != Token::EOF
    tokens << t
    t = next_token
  end
  tokens
end

#char_error_display(c) ⇒ Object



243
244
245
246
# File 'lib/antlr4/runtime/lexer.rb', line 243

def char_error_display(c)
  s = error_display_char(c)
  "'" + s + "'"
end

#char_indexObject



182
183
184
# File 'lib/antlr4/runtime/lexer.rb', line 182

def char_index
  @_input.index
end

#char_position_in_lineObject



170
171
172
# File 'lib/antlr4/runtime/lexer.rb', line 170

def char_position_in_line
  @_interp.char_position_in_line
end

#emit(token = nil) ⇒ Object



151
152
153
154
155
156
157
# File 'lib/antlr4/runtime/lexer.rb', line 151

def emit(token = nil)
  if !token.nil?
    @_token = token
  else
    @_token = @_factory.create(@_token_factory_source_pair, @_type, @_text, @_channel, @_token_start_char_index, char_index - 1, @_token_start_line, @_token_start_char_position_in_line)
  end
end

#emit_eofObject



159
160
161
162
163
164
# File 'lib/antlr4/runtime/lexer.rb', line 159

def emit_eof
  cpos = char_position_in_line
  eof = @_factory.create(@_token_factory_source_pair, Token::EOF, nil, Token::DEFAULT_CHANNEL, @_input.index, @_input.index - 1, line, cpos)
  emit(eof)
  eof
end

#error_display(s) ⇒ Object



217
218
219
220
221
222
223
# File 'lib/antlr4/runtime/lexer.rb', line 217

def error_display(s)
  buf = ''
  s.chars.each do |c|
    buf << error_display_char(c)
  end
  buf
end

#error_display_char(c) ⇒ Object



225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
# File 'lib/antlr4/runtime/lexer.rb', line 225

def error_display_char(c)
  s = ''
  s << c
  case c
  when Token::EOF
    s = '<EOF>'
  when '\n'
    s = '\\n'
  when '\t'
    s = '\\t'
  when '\r'
    s = '\\r'
  else
    # type code here
  end
  s
end

#input_stream(input) ⇒ Object



136
137
138
139
140
141
142
143
144
145
# File 'lib/antlr4/runtime/lexer.rb', line 136

def input_stream(input)
  @_input = nil
  @_token_factory_source_pair = OpenStruct.new
  @_token_factory_source_pair.a = self
  @_token_factory_source_pair.b = @_input
  reset
  @_input = input
  @_token_factory_source_pair.a = self
  @_token_factory_source_pair.b = @_input
end

#lineObject



166
167
168
# File 'lib/antlr4/runtime/lexer.rb', line 166

def line
  @_interp.line
end

#mode(m) ⇒ Object



118
119
120
# File 'lib/antlr4/runtime/lexer.rb', line 118

def mode(m)
  @_mode = m
end

#moreObject



114
115
116
# File 'lib/antlr4/runtime/lexer.rb', line 114

def more
  @_type = MORE
end

#next_tokenObject



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
# File 'lib/antlr4/runtime/lexer.rb', line 58

def next_token
  if @_input.nil?
    raise IllegalStateException, 'nextToken requires a non-nil input stream.'
  end

  # Mark start location in char stream so unbuffered streams are
  # guaranteed at least have text of current token
  token_start_marker = @_input.mark
  begin
    repeat_outer = true
    repeat_outer = next_token_inner while repeat_outer
    return @_token
  ensure # make sure we release marker after match or
    # unbuffered char stream will keep buffering
    @_input.release(token_start_marker)
  end
end

#next_token_innerObject



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
# File 'lib/antlr4/runtime/lexer.rb', line 76

def next_token_inner
  loop do
    if @_hit_eof
      emit_eof
      return false
    end

    @_token = nil
    @_channel = Token::DEFAULT_CHANNEL
    @_token_start_char_index = @_input.index
    @_token_start_char_position_in_line = @_interp.char_position_in_line
    @_token_start_line = @_interp.line
    @_text = nil
    loop do
      @_type = Token::INVALID_TYPE

      begin
        ttype = @_interp.match(@_input, @_mode)
      rescue LexerNoViableAltException => e
        notify_listeners(e) # report error
        recover1(e)
        ttype = SKIP
      end
      @_hit_eof = true if @_input.la(1) == IntStream::EOF
      @_type = ttype if @_type == Token::INVALID_TYPE
      return true if @_type == SKIP
      break if @_type != MORE
    end

    emit if @_token.nil?
    return false
  end
end

#notify_listeners(e) ⇒ Object



209
210
211
212
213
214
215
# File 'lib/antlr4/runtime/lexer.rb', line 209

def notify_listeners(e)
  text = @_input.text(Interval.of(@_token_start_char_index, @_input.index))
  msg = "token recognition error at: '" + error_display(text) + "'"

  listener = error_listener_dispatch
  listener.syntax_error(self, nil, @_token_start_line, @_token_start_char_position_in_line, msg, e)
end

#pop_modeObject

Raises:

  • (EmptyStackException)


128
129
130
131
132
133
134
# File 'lib/antlr4/runtime/lexer.rb', line 128

def pop_mode
  raise EmptyStackException if @_mode_stack.empty?

  puts('popMode back to ' + @_mode_stack[-1]) if LexerATNSimulator.debug
  mode(@_mode_stack.pop)
  @_mode
end

#push_mode(m) ⇒ Object



122
123
124
125
126
# File 'lib/antlr4/runtime/lexer.rb', line 122

def push_mode(m)
  puts('pushMode ' + m) if LexerATNSimulator.debug
  @_mode_stack.push(@_mode)
  mode(m)
end

#recover1(_e) ⇒ Object



202
203
204
205
206
207
# File 'lib/antlr4/runtime/lexer.rb', line 202

def recover1(_e)
  if @_input.la(1) != IntStream::EOF
    # skip a char and begin again
    @_interp.consume(@_input)
  end
end

#recover2(_re) ⇒ Object



248
249
250
# File 'lib/antlr4/runtime/lexer.rb', line 248

def recover2(_re)
  @_input.consume
end

#resetObject



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
# File 'lib/antlr4/runtime/lexer.rb', line 25

def reset
  # wack Lexer state variables
  unless @_input.nil?
    @_input.seek(0) # rewind the input
  end
  @_token = nil
  @_type = Token::INVALID_TYPE
  @_channel = Token::DEFAULT_CHANNEL
  @_token_start_char_index = -1
  @_token_start_char_position_in_line = -1
  @_token_start_line = -1
  @_text = nil

  @_hit_eof = false
  @_mode = DEFAULT_MODE
  @_mode_stack.clear

  @_interp.reset unless @_interp.nil?
end

#set_char_position_in_line(char_position_in_line) ⇒ Object



178
179
180
# File 'lib/antlr4/runtime/lexer.rb', line 178

def set_char_position_in_line(char_position_in_line)
  @_interp.set_char_position_in_line(char_position_in_line)
end

#set_line(line) ⇒ Object



174
175
176
# File 'lib/antlr4/runtime/lexer.rb', line 174

def set_line(line)
  @_interp.set_line(line)
end

#skipObject



110
111
112
# File 'lib/antlr4/runtime/lexer.rb', line 110

def skip
  @_type = SKIP
end

#source_nameObject



147
148
149
# File 'lib/antlr4/runtime/lexer.rb', line 147

def source_name
  @_input.get_source_name
end

#textObject



186
187
188
189
190
# File 'lib/antlr4/runtime/lexer.rb', line 186

def text
  return @_text unless @_text.nil?

  @_interp.text(@_input)
end