Class: Syntax::YAML

Inherits:
Tokenizer show all
Defined in:
lib/syntax/lang/yaml.rb

Overview

A simple implementation of an YAML lexer. It handles most cases. It is not a validating lexer.

Instance Attribute Summary

Attributes inherited from Tokenizer

#chunk, #group

Instance Method Summary collapse

Methods inherited from Tokenizer

#finish, #option, #set, #setup, #start, #teardown, #tokenize

Instance Method Details

#step ⇒ Object

Step through a single iteration of the tokenization process. This will yield (potentially) many tokens, and possibly zero tokens.



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/syntax/lang/yaml.rb', line 11

def step
  if bol?
    case
      when scan(/---(\s*.+)?$/)
        start_group :document, matched
      when scan(/(\s*)([a-zA-Z][-\w]*)(\s*):/)
        start_group :normal, subgroup(1)
        start_group :key, subgroup(2)
        start_group :normal, subgroup(3)
        start_group :punct, ":"
      when scan(/(\s*)-/)
        start_group :normal, subgroup(1)
        start_group :punct, "-"
      when scan(/\s*$/)
        start_group :normal, matched
      when scan(/#.*$/)
        start_group :comment, matched
      else
        append getch
    end
  else
    case
      when scan(/[\n\r]+/)
        start_group :normal, matched
      when scan(/[ \t]+/)
        start_group :normal, matched
      when scan(/!+(.*?^)?\S+/)
        start_group :type, matched
      when scan(/&\S+/)
        start_group :anchor, matched
      when scan(/\*\S+/)
        start_group :ref, matched
      when scan(/\d\d:\d\d:\d\d/)
        start_group :time, matched
      when scan(/\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d(\.\d+)? [-+]\d\d:\d\d/)
        start_group :date, matched
      when scan(/['"]/)
        start_group :punct, matched
        scan_string matched
      when scan(/:\w+/)
        start_group :symbol, matched
      when scan(/[:]/)
        start_group :punct, matched
      when scan(/#.*$/)
        start_group :comment, matched
      when scan(/>-?/)
        start_group :punct, matched
        start_group :normal, scan(/.*$/)
        append getch until eos? || bol?
        return if eos?
        indent = check(/ */)
        start_group :string
        loop do
          break if eos?
          line = check_until(/[\n\r]|\Z/)
          break if line.nil?
          if line.chomp.length > 0
            this_indent = line.chomp.match( /^\s*/ )[0]
            break if this_indent.length < indent.length
          end
          append scan_until(/[\n\r]|\Z/)
        end
      else
        start_group :normal, scan_until(/(?=$|#)/)
    end
  end
end