Class: MVinl::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/mvinl/lexer.rb

Constant Summary collapse

ID_REGEX = 
/[a-zA-Z_][a-zA-Z0-9_]*/
TOKENS = 
{
  KEYWORD: Regexp.union(Context::RESERVED),
  OPEN_PAREN: /\(/,
  CLOSE_PAREN: /\)/,
  NEW_LINE: /(?:[ \t]*(?:\r?\n)[ \t]*)+/,
  WHITESPACE: /[ \t]/,
  KEYWORD_ARG: /(#{ID_REGEX}):/,
  ID: ID_REGEX,
  GROUP: /@(#{ID_REGEX})/,
  CONSTANT: /!([A-Z][A-Z0-9_]*)/,
  VARIABLE: /!(#{ID_REGEX})/,
  FLOAT: /[+-]?\d+\.\d+/,
  NUMBER: /[+-]?\d+/,
  MULTILINE_STRING: /"((?:\\.|[^"\\])*)"\s*\\\s*/,
  STRING: /"((?:\\.|[^"\\])*)"/,
  SYMBOL: /:(#{ID_REGEX})/,
  COMMENT: /#/,
  OPER: %r{[+\-*/%]},
  END_TAG: /\./
}.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(context, input = '') ⇒ Lexer

Returns a new instance of Lexer.



40
41
42
43
44
45
46
 
# File 'lib/mvinl/lexer.rb', line 40

def initialize(context, input = '')
  @context = context
  @ss = StringScanner.new(input)
  @args_n = 0
  @in_group = false
  @eos = false
end

Instance Attribute Details

#eos ⇒ Object (readonly)

Returns the value of attribute eos.



16
17
18
 
# File 'lib/mvinl/lexer.rb', line 16

def eos
  @eos
end

Instance Method Details

#feed(input) ⇒ Object 



48
49
50
 
# File 'lib/mvinl/lexer.rb', line 48

def feed(input)
  @ss = StringScanner.new(input)
end

#next_token ⇒ Object 



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/mvinl/lexer.rb', line 52

def next_token
  return process_eos if @ss.eos?

  # Check if variable name been used
  MVinl::Context::CONSTANTS.each_key do |const_name|
    return [:CONSTANT_CALL, const_name] if @ss.scan(/\A#{Regexp.escape const_name.to_s}\b/)
  end
  @context.variables.each_key do |var_name|
    return [:VARIABLE_CALL, var_name] if @ss.scan(/\A#{Regexp.escape var_name.to_s}\b/)
  end

  TOKENS.each do |type, regex|
    if @ss.scan regex
      @last_type = type
      break
    end
  end
  case @last_type
  when :NEW_LINE
    @context.state[:lines] += 1
    return next_token if continuation_line?

    [:END_TAG, "\n"]
  when :WHITESPACE
    # Whitespace (' ' and '\t') does nothing in mvnl 0.1. I handle them
    # though as indentation plays a role in 0.2 to predict position of elements
    next_token
  when :COMMENT
    skip_to_next_line
    next_token
  when :OPEN_PAREN then [:OPEN_PAREN, '(']
  when :CLOSE_PAREN
    unless @context.state[:depth].positive?
      raise UnexpectedTokenError, 'CLOSE_PARAM found with no matching OPEN_PARAM'
    end

    [:CLOSE_PAREN, ')']
  when :OPER
    unless @context.state[:depth].positive?
      raise UnexpectedTokenError, 'OPER found with no matching OPEN_PARAM'
    end

    [:OPER, @ss.matched]
  when :KEYWORD
    return [:DEF, @ss.matched] if @ss.matched == 'def'

    [:KEYWORD, @ss.matched]
  when :KEYWORD_ARG
    if !@context.state[:in_prop]
      raise UnexpectedTokenError, 'Looking for identifier but found KEYWORD_ARG'
    elsif @context.state[:in_keyword_arg]
      raise UnexpectedTokenError, 'Looking for a keyword argument value but found KEYWORD_ARG'
    end

    [:KEYWORD_ARG, @ss[1]]
  when :GROUP
    # Group gets canceled whenever encountered another group id or a matching end tag
    if @context.state[:in_keyword_arg]
      raise UnexpectedTokenError, 'Looking for a keyword argument value but found GROUP'
    end

    @in_group = true
    [:GROUP, @ss[1]]
  when :CONSTANT then [:CONSTANT, @ss[1]]
  when :VARIABLE then [:VARIABLE, @ss[1]]
  when :ID then [:ID, @ss.matched]
  when :NUMBER, :FLOAT, :STRING, :SYMBOL, :MULTILINE_STRING
    # Values can't be used outside an property or a lambda
    if !@context.state[:in_prop] && !@context.state[:depth].positive? && !@context.state[:in_var]
      raise UnexpectedTokenError, "Looking for ID or OPEN_PAREN but found #{@last_type}"
    elsif !@context.state[:in_keyword_arg] && @context.state[:keyword_arg_depth].positive? &&
          !@context.state[:depth].positive? && !@context.state[:in_var]
      raise UnexpectedTokenError, "Looking for END_TAG or KEYWORD_ARG but found #{@last_type}"
    end

    [@last_type, @ss[1] || @ss.matched]
  when :END_TAG then [:END_TAG, '.']
  else
    raise UnexpectedTokenError, "Unexpected character: '#{@ss.getch}'"
  end
rescue LexerError => e
  warn "Syntax error at #{@ss.charpos - @ss.matched_size}: #{e}"
end

#tokenize ⇒ Object 



136
137
138
139
140
141
142
143
 
# File 'lib/mvinl/lexer.rb', line 136

def tokenize
  tokens = []
  while (token = next_token)
    warn "Tokenize: #{token}"
    tokens << token
  end
  tokens
end