Class: Dhaka::LexerRun

Inherits:
Object
  • Object
show all
Includes:
Enumerable
Defined in:
lib/lexer/lexer_run.rb

Overview

Represents a run of a lexer on a given input string.

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(lexer, input) ⇒ LexerRun

Returns a new instance of LexerRun.



7
8
9
10
11
 
# File 'lib/lexer/lexer_run.rb', line 7

def initialize lexer, input
  @lexer, @input          = lexer, input
  @input_position         = 0
  @not_yet_accepted_chars = []
end

Instance Attribute Details

#current_lexeme ⇒ Object (readonly)

Returns the value of attribute current_lexeme.



6
7
8
 
# File 'lib/lexer/lexer_run.rb', line 6

def current_lexeme
  @current_lexeme
end

Instance Method Details

#create_token(symbol_name) ⇒ Object

Constructs a token of type symbol_name from the current_lexeme.



14
15
16
 
# File 'lib/lexer/lexer_run.rb', line 14

def create_token(symbol_name)
  Token.new(symbol_name, @current_lexeme.characters.join, @current_lexeme.input_position)
end

#each {|Token.new(END_SYMBOL_NAME, nil, nil)| ... } ⇒ Object

Yields each token as it is recognized. Returns a TokenizerErrorResult if an error occurs during tokenization.

Yields:



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/lexer/lexer_run.rb', line 19

def each
  reset_and_rewind
  loop do
    c = curr_char
    break if (c == "\0" && @not_yet_accepted_chars.empty? && !@current_lexeme.accepted?)
    dest_state  = @curr_state.transitions[c]
    unless dest_state
      return TokenizerErrorResult.new(@input_position) unless @current_lexeme.accepted?
      token = get_token
      yield token if token
      reset_and_rewind
    else
      @curr_state = dest_state
      if @curr_state.accepting?
        @current_lexeme.pattern = @curr_state.pattern
        @current_lexeme.concat @not_yet_accepted_chars
        @not_yet_accepted_chars = []
        @current_lexeme << c
      else
        @not_yet_accepted_chars << c
      end
      advance
    end
  end
  yield Token.new(END_SYMBOL_NAME, nil, nil)
end