Class: Liquid2::Parser

Inherits:
Object
  • Object
show all
Defined in:
lib/liquid2/parser.rb

Overview

Liquid template parser.

Defined Under Namespace

Classes: Precedence

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(env, tokens, length) ⇒ Parser

Returns a new instance of Parser.

Parameters:

  • env (Environment)
  • tokens (Array[[Symbol, String?, Integer]])
  • length (Integer) —

    Length of the source string.



41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/liquid2/parser.rb', line 41

def initialize(env, tokens, length)
  @env = env
  @tokens = tokens
  @pos = 0
  @eof = [:token_eof, nil, length - 1]
  @whitespace_carry = nil

  # If both tags and output statements share the same end delimiter, we expect
  # `:token_tag_end` to close an output statement as the scanner scans for tags first.
  @output_end = @env.universal_markup_end ? :token_tag_end : :token_output_end
end

Class Method Details

.parse(env, source, scanner: nil) ⇒ Array[Node | String]

Parse Liquid template text into a syntax tree.

Parameters:

Returns:

  • (Array[Node | String]) 


32
33
34
35
36
 
# File 'lib/liquid2/parser.rb', line 32

def self.parse(env, source, scanner: nil)
  new(env,
      Liquid2::Scanner.tokenize(env, source, scanner || StringScanner.new("")),
      source.length).parse
end

Instance Method Details

#carry_whitespace_control ⇒ Object

Advance the pointer if the current token is a whitespace control token, and remember the token's value for the next text node.



160
161
162
 
# File 'lib/liquid2/parser.rb', line 160

def carry_whitespace_control
  @whitespace_carry = current_kind == :token_whitespace_control ? self.next[1] : nil
end

#current ⇒ Object

Return the current token without advancing the pointer. An EOF token is returned if there are no tokens left.



55
 
# File 'lib/liquid2/parser.rb', line 55

def current = @tokens[@pos] || @eof

#current_kind ⇒ Object

Return the kind of the current token without advancing the pointer.



58
 
# File 'lib/liquid2/parser.rb', line 58

def current_kind = current.first

#eat(kind, message = nil) ⇒ Token

Consume the next token if its kind matches kind, raise an error if it does not.

Parameters:

  • kind (Symbol)
  • message (String?) (defaults to: nil) —

    An error message to use if the next token kind does not match kind.

Returns:

  • (Token) —

    The consumed token.



83
84
85
86
87
88
89
90
 
# File 'lib/liquid2/parser.rb', line 83

def eat(kind, message = nil)
  token = self.next
  unless token.first == kind
    raise LiquidSyntaxError.new(message || "unexpected #{token.first}", token)
  end

  token
end

#eat_empty_tag(name) ⇒ Object

Returns The :token_tag_name token.

Parameters:

  • name (String)

Returns:

  • The :token_tag_name token.



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/liquid2/parser.rb', line 106

def eat_empty_tag(name)
  eat(:token_tag_start, "expected tag #{name}")
  @pos += 1 if current_kind == :token_whitespace_control
  name_token = eat(:token_tag_name, "expected tag #{name}")

  unless name == name_token[1]
    raise LiquidSyntaxError.new(
      "unexpected tag #{name_token[1]}", name_token
    )
  end

  carry_whitespace_control
  eat(:token_tag_end, "expected tag #{name}")
  name_token
end

#eat_one_of(*kinds) ⇒ Token

Consume the next token if its kind is in kinds, raise an error if it does not.

Parameters:

  • kind (Symbol)

Returns:

  • (Token) —

    The consumed token.



95
96
97
98
99
100
101
102
 
# File 'lib/liquid2/parser.rb', line 95

def eat_one_of(*kinds)
  token = self.next
  unless kinds.include? token.first
    raise LiquidSyntaxError.new("unexpected #{token.first}", token)
  end

  token
end

#expect_expression ⇒ Object

Raises:



164
165
166
167
168
169
 
# File 'lib/liquid2/parser.rb', line 164

def expect_expression
  return unless TERMINATE_EXPRESSION.include?(current_kind)

  raise LiquidSyntaxError.new("missing expression",
                              current)
end

#next ⇒ Object

Return the next token and advance the pointer.



61
62
63
64
65
66
67
68
 
# File 'lib/liquid2/parser.rb', line 61

def next
  if (token = @tokens[@pos])
    @pos += 1
    token
  else
    @eof
  end
end

#next_kind ⇒ Object

Return the kind of the next token and advance the pointer



71
 
# File 'lib/liquid2/parser.rb', line 71

def next_kind = self.next.first

#parse ⇒ Array[Node | String]

Returns:

  • (Array[Node | String]) 


172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
 
# File 'lib/liquid2/parser.rb', line 172

def parse
  nodes = [] # : Array[Node | String]

  loop do
    kind, value = self.next
    @pos += 1 if current_kind == :token_whitespace_control

    case kind
    when :token_other
      rstrip = peek[1] if peek_kind == :token_whitespace_control
      @env.trim(value || raise, @whitespace_carry, rstrip)
      nodes << (value || raise)
    when :token_output_start
      nodes << parse_output
    when :token_tag_start
      nodes << parse_tag
    when :token_comment_start
      nodes << parse_comment
    when :token_eof
      return nodes
    else
      raise LiquidSyntaxError.new("unexpected #{kind}", previous)
    end
  end
end

#parse_arguments ⇒ [Array[untyped], Array[KeywordArgument]]

Parse mixed positional and keyword arguments. Leading commas should be consumed by the caller, if allowed.

Returns:



523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
 
# File 'lib/liquid2/parser.rb', line 523

def parse_arguments
  args = [] # : Array[untyped]
  kwargs = [] # : Array[KeywordArgument]

  loop do
    break if TERMINATE_EXPRESSION.member?(current_kind)

    case current_kind
    when :token_word
      if KEYWORD_ARGUMENT_DELIMITERS.include?(peek_kind)
        token = self.next
        @pos += 1 # = or :
        kwargs << KeywordArgument.new(token, token[1] || raise, parse_primary)
      else
        # A positional argument
        args << parse_primary
      end
    else
      # A positional argument
      args << parse_primary
    end

    break unless current_kind == :token_comma

    @pos += 1
  end

  [args, kwargs]
end

#parse_block(end_block) ⇒ Block

Parse Liquid markup until we find a tag token in end_block.

Parameters:

  • end_block (responds to include?) —

    An array or set of tag names that will indicate the end of the block.

Returns:



202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
 
# File 'lib/liquid2/parser.rb', line 202

def parse_block(end_block)
  token = current
  nodes = [] # : Array[Node | String]

  loop do
    kind, value = self.next

    case kind
    when :token_other
      rstrip = peek[1] if peek_kind == :token_whitespace_control
      @env.trim(value || raise, @whitespace_carry, rstrip)
      nodes << (value || raise)
    when :token_output_start
      @pos += 1 if current_kind == :token_whitespace_control
      nodes << parse_output
    when :token_tag_start
      if end_block.include?(peek_tag_name)
        @pos -= 1
        break
      end

      @pos += 1 if current_kind == :token_whitespace_control
      nodes << parse_tag
    when :token_comment_start
      nodes << parse_comment
    when :token_eof
      break
    else
      raise LiquidSyntaxError.new("unexpected token: #{token.inspect}", previous)
    end
  end

  Block.new(token, nodes)
end

#parse_filtered_expression ⇒ FilteredExpression|TernaryExpression

Returns:



238
239
240
241
242
243
244
245
246
247
248
249
250
 
# File 'lib/liquid2/parser.rb', line 238

def parse_filtered_expression
  token = current
  left = parse_primary
  left = parse_implicit_array(left) if current_kind == :token_comma
  filters = parse_filters if current_kind == :token_pipe
  expr = FilteredExpression.new(token, left, filters)

  if current_kind == :token_if
    parse_ternary_expression(expr)
  else
    expr
  end
end

#parse_identifier(trailing_question: true) ⇒ Object 



426
427
428
429
430
431
432
433
434
 
# File 'lib/liquid2/parser.rb', line 426

def parse_identifier(trailing_question: true)
  token = eat(:token_word)

  if PATH_PUNCTUATION.include?(current_kind)
    raise LiquidSyntaxError.new("expected an identifier, found a path", current)
  end

  Identifier.new(token)
end

#parse_keyword_arguments ⇒ Array<KeywordArgument>

Parse comma separated name/value pairs. Leading commas should be consumed by the caller, if allowed.

Returns:



473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
 
# File 'lib/liquid2/parser.rb', line 473

def parse_keyword_arguments
  args = [] # : Array[KeywordArgument]

  loop do
    break if TERMINATE_EXPRESSION.member?(current_kind)

    word = eat(:token_word)
    eat_one_of(:token_assign, :token_colon)
    args << KeywordArgument.new(word, word[1] || raise, parse_primary)

    break unless current_kind == :token_comma

    @pos += 1
  end

  args
end

#parse_line_statements ⇒ Object 



314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
 
# File 'lib/liquid2/parser.rb', line 314

def parse_line_statements
  token = previous
  nodes = [] # : Array[Node]

  loop do
    case current_kind
    when :token_tag_start
      @pos += 1
      nodes << parse_tag
    when :token_whitespace_control, :token_tag_end
      break
    else
      raise LiquidSyntaxError.new("unexpected #{current_kind}", current)
    end
  end

  Block.new(token, nodes)
end

#parse_loop_expression ⇒ LoopExpression

Returns:



253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
 
# File 'lib/liquid2/parser.rb', line 253

def parse_loop_expression
  identifier = parse_identifier
  eat(:token_in, "missing 'in'")
  expect_expression
  enum = parse_primary

  reversed = false
  offset = nil # : (Expression | nil)
  limit = nil # : (Expression | nil)
  cols = nil # : (Expression | nil)

  if current_kind == :token_comma
    unless LOOP_KEYWORDS.member?(peek[1] || raise)
      enum = parse_implicit_array(enum)
      return LoopExpression.new(identifier.token, identifier, enum,
                                limit: limit, offset: offset, reversed: reversed, cols: cols)
    end

    # A comma between the iterable and the first argument is OK.
    @pos += 1 if current_kind == :token_comma
  end

  loop do
    token = current
    case token.first
    when :token_word
      case token[1]
      when "reversed"
        @pos += 1
        reversed = true
      when "limit"
        @pos += 1
        eat_one_of(:token_colon, :token_assign)
        limit = parse_primary
      when "cols"
        @pos += 1
        eat_one_of(:token_colon, :token_assign)
        cols = parse_primary
      when "offset"
        @pos += 1
        eat_one_of(:token_colon, :token_assign)
        offset_token = current
        offset = if offset_token.first == :token_word && offset_token[1] == "continue"
                   Identifier.new(self.next)
                 else
                   parse_primary
                 end
      else
        raise LiquidSyntaxError.new("expected 'reversed', 'offset' or 'limit'", token)
      end
    when :token_comma
      @pos += 1
    else
      break
    end
  end

  LoopExpression.new(identifier.token, identifier, enum,
                     limit: limit, offset: offset, reversed: reversed, cols: cols)
end

#parse_name ⇒ String

Parse a string literals or unquoted word.

Returns:

  • (String) 


438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
 
# File 'lib/liquid2/parser.rb', line 438

def parse_name
  case current_kind
  when :token_word
    parse_identifier.name
  when :token_single_quote_string, :token_double_quote_string
    node = parse_string_literal
    unless node.is_a?(String)
      raise LiquidSyntaxError.new("names can't be template strings", node.token)
    end

    node
  else
    raise LiquidSyntaxError.new("expected a string literal or unquoted word", current)
  end
end

#parse_parameters ⇒ Hash[String, Parameter]

Parse comma separated parameter names with optional default expressions. Leading commas should be consumed by the caller, if allowed.

Returns:



494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
 
# File 'lib/liquid2/parser.rb', line 494

def parse_parameters
  args = {} # : Hash[String, Parameter]

  loop do
    break if TERMINATE_EXPRESSION.member?(current_kind)

    word = eat(:token_word)
    name = word[1] || raise

    case current_kind
    when :token_assign, :token_colon
      @pos += 1
      args[name] = Parameter.new(word, name, parse_primary)
      @pos += 1 if current_kind == :token_comma
    when :comma
      args[name] = Parameter.new(word, name, :undefined)
      @pos += 1
    else
      args[name] = Parameter.new(word, name, :undefined)
      break
    end
  end

  args
end

#parse_positional_arguments ⇒ Array<Expression>

Parse comma separated expression. Leading commas should be consumed by the caller.

Returns:



457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'lib/liquid2/parser.rb', line 457

def parse_positional_arguments
  args = [] # : Array[untyped]

  loop do
    args << parse_primary
    break unless current_kind == :token_comma

    @pos += 1
  end

  args
end

#parse_primary(precedence: Precedence::LOWEST, infix: true) ⇒ Node

Parse a primary expression. A primary expression is a literal, a path (to a variable), or a logical expression composed of other primary expressions.

Returns:



337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
 
# File 'lib/liquid2/parser.rb', line 337

def parse_primary(precedence: Precedence::LOWEST, infix: true)
  # Keywords followed by a dot or square bracket are parsed as paths.
  looks_like_a_path = PATH_PUNCTUATION.include?(peek_kind)

  # @type var kind: Symbol
  kind = current_kind

  left = case kind
         when :token_true
           if looks_like_a_path
             parse_path
           else
             self.next
             true
           end
         when :token_false
           if looks_like_a_path
             parse_path
           else
             self.next
             false
           end
         when :token_nil
           if looks_like_a_path
             parse_path
           else
             self.next
             nil
           end
         when :token_int
           Liquid2.to_liquid_int(self.next[1])
         when :token_float
           Float(self.next[1])
         when :token_blank
           looks_like_a_path ? parse_path : Blank.new(self.next)
         when :token_empty
           looks_like_a_path ? parse_path : Empty.new(self.next)
         when :token_single_quote_string, :token_double_quote_string
           parse_string_literal
         when :token_word
           parse_path
         when :token_lbracket
           parse_array_or_path
         when :token_lbrace
           parse_object_literal
         when :token_lparen
           parse_range_lambda_or_grouped_expression
         when :token_not, :token_plus, :token_minus
           parse_prefix_expression
         else
           unless looks_like_a_path && RESERVED_WORDS.include?(kind)
             raise LiquidSyntaxError.new("unexpected #{current_kind}", current)
           end

           parse_path
         end

  return left unless infix

  loop do
    kind = current_kind

    if kind == :token_unknown
      raise LiquidSyntaxError.new("unexpected #{current[1]&.inspect || kind}",
                                  current)
    end

    if kind == :token_eof ||
       (PRECEDENCES[kind] || Precedence::LOWEST) < precedence ||
       !BINARY_OPERATORS.member?(kind)
      break
    end

    left = parse_infix_expression(left)
  end

  left
end

#parse_string ⇒ String

Parse a string literal without interpolation..

Returns:

  • (String)

Raises:



419
420
421
422
423
424
 
# File 'lib/liquid2/parser.rb', line 419

def parse_string
  node = parse_primary
  raise LiquidTypeError, "expected a string literal" unless node.is_a?(String)

  node
end

#peek(offset = 1) ⇒ Object 



73
 
# File 'lib/liquid2/parser.rb', line 73

def peek(offset = 1) = @tokens[@pos + offset] || @eof

#peek_kind(offset = 1) ⇒ Object 



75
 
# File 'lib/liquid2/parser.rb', line 75

def peek_kind(offset = 1) = peek(offset).first

#peek_tag_name ⇒ String

Return the next tag name without advancing the pointer. Assumes the current token is :token_tag_start.

Returns:

  • (String) 


142
143
144
145
146
147
148
149
150
151
 
# File 'lib/liquid2/parser.rb', line 142

def peek_tag_name
  token = current # Whitespace control or tag name
  token = peek if token.first == :token_whitespace_control
  unless token.first == :token_tag_name
    raise LiquidSyntaxError.new("missing tag name #{token}",
                                token)
  end

  token[1] || raise
end

#previous ⇒ Object 



77
 
# File 'lib/liquid2/parser.rb', line 77

def previous = @tokens[@pos - 1] || raise

#skip_whitespace_control ⇒ Object

Advance the pointer if the current token is a whitespace control token.



154
155
156
 
# File 'lib/liquid2/parser.rb', line 154

def skip_whitespace_control
  @pos += 1 if current_kind == :token_whitespace_control
end

#tag?(name) ⇒ bool

Return true if we're at the start of a tag named name.

Parameters:

  • name (String)

Returns:

  • (bool) 


125
126
127
128
129
 
# File 'lib/liquid2/parser.rb', line 125

def tag?(name)
  token = peek # Whitespace control or tag name
  token = peek(2) if token.first == :token_whitespace_control
  token.first == :token_tag_name && token[1] == name
end

#word?(text) ⇒ bool

Return true if the current token is a word matching text.

Parameters:

  • text (String)

Returns:

  • (bool) 


134
135
136
137
 
# File 'lib/liquid2/parser.rb', line 134

def word?(text)
  token = current
  token.first == :token_word && token[1] == text
end