Class: Twig::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/twig/lexer.rb

Constant Summary collapse

TAG_COMMENT = 
%w[{# #}].freeze
TAG_BLOCK = 
%w[{% %}].freeze
TAG_VARIABLE = 
%w[{{ }}].freeze
WHITESPACE_TRIM = 
'-'
WHITESPACE_LINE_TRIM = 
'~'
WHITESPACE_LINE_CHARS = 
" \t\0\x0B"
INTERPOLATION = 
%w[#{ }].freeze
OPENING_BRACKET = 
'([{'.chars
CLOSING_BRACKET = 
')]}'.chars
PUNCTUATION = 
OPENING_BRACKET + CLOSING_BRACKET + '?:.,|'.chars
REGEX_LNUM = 
/[0-9]+(_[0-9]+)*/
REGEX_FRAC = 
/\.#{REGEX_LNUM}/
REGEX_EXPONENT = 
/[eE][+-]?#{REGEX_LNUM}/
REGEX_DNUM = 
/#{REGEX_LNUM}(?:#{REGEX_FRAC})?/
REGEX_NAME = 
/[a-zA-Z_][a-zA-Z0-9_]*/
REGEX_SYMBOL = 
/:#{REGEX_NAME}/
REGEX_STRING = 
/\A"([^#"\\]*(?:\\\\.[^#"\\]*)*)"|'([^'\\]*(?:\\\\.[^'\\]*)*)'/s
REGEX_NUMBER = 
/\A(?:#{REGEX_DNUM}(?:#{REGEX_EXPONENT})?)/x
STATE_DATA = 
0
STATE_BLOCK = 
1
STATE_VAR = 
2
STATE_STRING = 
3
STATE_INTERPOLATION = 
4

Instance Method Summary collapse

Constructor Details

#initialize(environment) ⇒ Lexer

Returns a new instance of Lexer.

Parameters:



33
34
35
 
# File 'lib/twig/lexer.rb', line 33

def initialize(environment)
  @environment = environment
end

Instance Method Details

#tokenize(source) ⇒ Object

Parameters:



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/twig/lexer.rb', line 38

def tokenize(source)
  @source = source
  @code = source.code.tr("\r\n", "\n")
  @cursor = 0
  @lineno = 1
  @end = @code.length
  @tokens = []
  @state = STATE_DATA
  @states = []
  @brackets = []
  @position = -1
  @positions = @code.to_enum(:scan, lex_tokens_start).map { Regexp.last_match }

  while @cursor < @end
    case @state
    when STATE_DATA
      lex_data
    when STATE_BLOCK
      lex_block
    when STATE_VAR
      lex_var
    else
      raise "Unknown state: #{@state}"
    end
  end

  push_token(Token::EOF_TYPE)

  TokenStream.new(@tokens, @source)
end