Class: Collie::Parser::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/collie/parser/lexer.rb

Overview

Lexer for .y grammar files

Constant Summary collapse

KEYWORDS = 
%w[
  %token %type %left %right %nonassoc %prec %union %start
  %rule %inline %code %expect %define %param %parse-param
  %lex-param %initial-action %destructor %printer
].freeze

Instance Method Summary collapse

Constructor Details

#initialize(source, filename: "<input>") ⇒ Lexer

Returns a new instance of Lexer.



30
31
32
33
34
35
36
37
 
# File 'lib/collie/parser/lexer.rb', line 30

def initialize(source, filename: "<input>")
  @source = source
  @filename = filename
  @pos = 0
  @line = 1
  @column = 1
  @tokens = []
end

Instance Method Details

#tokenize ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/collie/parser/lexer.rb', line 39

def tokenize
  until eof?
    skip_whitespace
    break if eof?

    if current_char == "/" && peek_char == "/"
      skip_line_comment
    elsif current_char == "/" && peek_char == "*"
      skip_block_comment
    elsif current_char == "%" && peek_char == "{"
      @tokens << tokenize_prologue
    elsif current_char == "%" && peek_char == "}"
      advance(2)
      @tokens << make_token(:PROLOGUE_END, "%}")
    elsif current_char == "%" && peek_char == "%"
      advance(2)
      @tokens << make_token(:SECTION_SEPARATOR, "%%")
    elsif current_char == "%" && alpha?(peek_char)
      @tokens << tokenize_directive
    elsif current_char == "{"
      @tokens << tokenize_action
    elsif current_char == "'"
      @tokens << tokenize_char_literal
    elsif current_char == '"'
      @tokens << tokenize_string_literal
    elsif current_char == "|"
      advance
      @tokens << make_token(:PIPE, "|")
    elsif current_char == ":"
      advance
      @tokens << make_token(:COLON, ":")
    elsif current_char == ";"
      advance
      @tokens << make_token(:SEMICOLON, ";")
    elsif current_char == "("
      advance
      @tokens << make_token(:LPAREN, "(")
    elsif current_char == ")"
      advance
      @tokens << make_token(:RPAREN, ")")
    elsif current_char == "["
      advance
      @tokens << make_token(:LBRACKET, "[")
    elsif current_char == "]"
      advance
      @tokens << make_token(:RBRACKET, "]")
    elsif current_char == ","
      advance
      @tokens << make_token(:COMMA, ",")
    elsif current_char == "<"
      @tokens << tokenize_type_tag
    elsif alpha?(current_char) || current_char == "_"
      @tokens << tokenize_identifier
    else
      advance
    end
  end

  @tokens << make_token(:EOF, "")
  @tokens
end