Class: Syntax::Ruby

Inherits:
Tokenizer show all
Defined in:
lib/syntax/ruby.rb

Overview

A tokenizer for the Ruby language. It recognizes all common syntax (and some less common syntax) but because it is not a true lexer, it will make mistakes on some ambiguous cases.

Constant Summary collapse

KEYWORDS =

The list of all identifiers recognized as keywords.

%w{if then elsif else end begin do rescue ensure while for
class module def yield raise until unless and or not when
case super undef break next redo retry in return alias
defined?}

Instance Method Summary collapse

Methods inherited from Tokenizer

delegate, #finish, #start, #teardown, #tokenize

Instance Method Details

#setup ⇒ Object

Perform ruby-specific setup



18
19
20
 
# File 'lib/syntax/ruby.rb', line 18

def setup
  @selector = false
end

#step ⇒ Object

Step through a single iteration of the tokenization process.



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
# File 'lib/syntax/ruby.rb', line 23

def step
  case
    when bol? && check( /=begin/ )
      start_group( :comment, scan_until( /^=end$/ ) )
    when bol? && check( /__END__$/ )
      start_group( :comment, scan_until( /\Z/ ) )
  else
    case
      when check( /def\s+/ )
        start_group :keyword, scan( /def\s+/ )
        start_group :method,  scan_until( /$|(?=[;(\s])/ )
      when check( /class\s+/ )
        start_group :keyword, scan( /class\s+/ )
        start_group :class,  scan_until( /$|(?=[;\s<])/ )
      when check( /module\s+/ )
        start_group :keyword, scan( /module\s+/ )
        start_group :module,  scan_until( /$|(?=[;\s])/ )
      when check( /::/ )
        start_group :punct, scan(/::/)
      when check( /:"/ )
        start_group :symbol, scan(/:/)
        scan_delimited_region :symbol, :symbol, "", true
      when check( /:'/ )
        start_group :symbol, scan(/:/)
        scan_delimited_region :symbol, :symbol, "", false
      when check( /:\w/ )
        start_group :symbol, scan(/:\w+[!?]?/)
      when check( /\?\\?./ )
        start_group :char, scan(/\?\\?./)
      when check( /(__FILE__|__LINE__|true|false|nil|self)[?!]?/ )
        if @selector || matched[-1] == ?? || matched[-1] == ?!
          start_group :ident,
            scan(/(__FILE__|__LINE__|true|false|nil|self)[?!]?/)
        else
          start_group :constant,
            scan(/(__FILE__|__LINE__|true|false|nil|self)/)
        end
        @selector = false
      else
        case peek(2)
          when "%r"
            scan_delimited_region :punct, :regex, scan( /../ ), true
          when "%w", "%q"
            scan_delimited_region :punct, :string, scan( /../ ), false
          when "%s"
            scan_delimited_region :punct, :symbol, scan( /../ ), false
          when "%W", "%Q", "%x"
            scan_delimited_region :punct, :string, scan( /../ ), true
          when /%[^\sa-zA-Z0-9]/
            scan_delimited_region :punct, :string, scan( /./ ), true
          when "<<"
            start_group :punct, scan( /<</ )
            float_right = scan( /-/ )
            append "-" if float_right
            if ( type = scan( /['"]/ ) )
              append type
              delim = scan_until( /(?=#{type})/ )
              if delim.nil?
                append scan_until( /\Z/ )
                return
              end
            else
              delim = scan( /\w+/ ) or return
            end
            start_group :constant, delim
            start_group :punct, scan( /#{type}/ ) if type
            scan_delimited_region :constant, :string, "", ( type != "'" ),
              delim, true, float_right
          else
            case peek(1)
              when /\s/
                start_group :normal, scan( /\s+/ )
              when "#"
                start_group :comment, scan( /#.*$/ )
              when /[A-Z]/
                start_group :constant, scan( /\w+/ )
              when /[a-z_]/
                word = scan( /\w+[?!]?/ )
                if !@selector && KEYWORDS.include?( word )
                  start_group :keyword, word
                elsif
                  start_group :ident, word
                end
                @selector = false
              when /\d/
                start_group :number,
                  scan( /[\d_]+(\.[\d_]+)?([eE][\d_]+)?/ )
              when '"'
                scan_delimited_region :punct, :string, "", true
              when '/'
                scan_delimited_region :punct, :regex, "", true
              when "'"
                scan_delimited_region :punct, :string, "", false
              when "."
                dots = scan( /\.{1,3}/ )
                start_group :punct, dots
                @selector = ( dots.length == 1 )
              when /[@]/
                start_group :attribute, scan( /@{1,2}\w*/ )
              when /[$]/
                start_group :global, scan(/\$/)
                start_group :global, scan( /\w+|./ ) if check(/./)
              when /[-!?*\/+=<>()\[\]\{}:;,&|%]/
                start_group :punct,
                  scan(/[-!?*\/+=<>()\[\]\{}:;,&|%]/)
              else
                # all else just falls through this, to prevent
                # infinite loops...
                append getch
            end
        end
    end
  end
end