Class: Liquid::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/liquid/lexer.rb

Constant Summary collapse

SPECIALS = 
{
  '|' => :pipe,
  '.' => :dot,
  ':' => :colon,
  ',' => :comma,
  '[' => :open_square,
  ']' => :close_square,
  '(' => :open_round,
  ')' => :close_round,
  '?' => :question,
  '-' => :dash,
}.freeze
IDENTIFIER = 
/[a-zA-Z_][\w-]*\??/
SINGLE_STRING_LITERAL = 
/'[^\']*'/
DOUBLE_STRING_LITERAL = 
/"[^\"]*"/
NUMBER_LITERAL = 
/-?\d+(\.\d+)?/
DOTDOT = 
/\.\./
COMPARISON_OPERATOR = 
/==|!=|<>|<=?|>=?|contains(?=\s)/
WHITESPACE_OR_NOTHING = 
/\s*/

Instance Method Summary collapse

Constructor Details

#initialize(input) ⇒ Lexer

Returns a new instance of Lexer.



26
27
28
 
# File 'lib/liquid/lexer.rb', line 26

def initialize(input)
  @ss = StringScanner.new(input)
end

Instance Method Details

#tokenize ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/liquid/lexer.rb', line 30

def tokenize
  @output = []

  until @ss.eos?
    @ss.skip(WHITESPACE_OR_NOTHING)
    break if @ss.eos?
    tok      = if (t = @ss.scan(COMPARISON_OPERATOR))
      [:comparison, t]
    elsif (t = @ss.scan(SINGLE_STRING_LITERAL))
      [:string, t]
    elsif (t = @ss.scan(DOUBLE_STRING_LITERAL))
      [:string, t]
    elsif (t = @ss.scan(NUMBER_LITERAL))
      [:number, t]
    elsif (t = @ss.scan(IDENTIFIER))
      [:id, t]
    elsif (t = @ss.scan(DOTDOT))
      [:dotdot, t]
    else
      c     = @ss.getch
      if (s = SPECIALS[c])
        [s, c]
      else
        raise SyntaxError, "Unexpected character #{c}"
      end
    end
    @output << tok
  end

  @output << [:end_of_string]
end