Class: CodeRay::Scanners::Python

Inherits:
Scanner
  • Object
show all
Includes:
CodeRay::Streamable
Defined in:
lib/coderay/scanners/python.rb

Overview

Bases on pygments’ PythonLexer, see dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.

Constant Summary collapse

KEYWORDS = 
[
  'and', 'as', 'assert', 'break', 'class', 'continue', 'def',
  'del', 'elif', 'else', 'except', 'finally', 'for',
  'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not',
  'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',
  'nonlocal',  # new in Python 3
]
OLD_KEYWORDS = 
[
  'exec', 'print',  # gone in Python 3
]
PREDEFINED_METHODS_AND_TYPES = 
%w[
  __import__ abs all any apply basestring bin bool buffer
  bytearray bytes callable chr classmethod cmp coerce compile
  complex delattr dict dir divmod enumerate eval execfile exit
  file filter float frozenset getattr globals hasattr hash hex id
  input int intern isinstance issubclass iter len list locals
  long map max min next object oct open ord pow property range
  raw_input reduce reload repr reversed round set setattr slice
  sorted staticmethod str sum super tuple type unichr unicode
  vars xrange zip
]
PREDEFINED_EXCEPTIONS = 
%w[
  ArithmeticError AssertionError AttributeError
  BaseException DeprecationWarning EOFError EnvironmentError
  Exception FloatingPointError FutureWarning GeneratorExit IOError
  ImportError ImportWarning IndentationError IndexError KeyError
  KeyboardInterrupt LookupError MemoryError NameError
  NotImplemented NotImplementedError OSError OverflowError
  OverflowWarning PendingDeprecationWarning ReferenceError
  RuntimeError RuntimeWarning StandardError StopIteration
  SyntaxError SyntaxWarning SystemError SystemExit TabError
  TypeError UnboundLocalError UnicodeDecodeError
  UnicodeEncodeError UnicodeError UnicodeTranslateError
  UnicodeWarning UserWarning ValueError Warning ZeroDivisionError
]
PREDEFINED_VARIABLES_AND_CONSTANTS = 
[
  'False', 'True', 'None', # "keywords" since Python 3
  'self', 'Ellipsis', 'NotImplemented',
]
IDENT_KIND = 
WordList.new(:ident).
add(KEYWORDS, :keyword).
add(OLD_KEYWORDS, :old_keyword).
add(PREDEFINED_METHODS_AND_TYPES, :predefined).
add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant).
add(PREDEFINED_EXCEPTIONS, :exception)
NAME = 
/ [^\W\d] \w* /x
ESCAPE = 
/ [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
UNICODE_ESCAPE = 
/ u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x
OPERATOR = 
/
  \.\.\. |          # ellipsis
  \.(?!\d) |        # dot but not decimal point
  [,;:()\[\]{}] |   # simple delimiters
  \/\/=? | \*\*=? | # special math
  [-+*\/%&|^]=? |   # ordinary math and binary logic
  [~`] |            # binary complement and inspection
  <<=? | >>=? | [<>=]=? | !=  # comparison and assignment
/x
STRING_DELIMITER_REGEXP = 
Hash.new do |h, delimiter|
  h[delimiter] = Regexp.union delimiter
end
STRING_CONTENT_REGEXP = 
Hash.new do |h, delimiter|
  h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x
end
DEF_NEW_STATE = 
WordList.new(:initial).
add(%w(def), :def_expected).
add(%w(import from), :include_expected).
add(%w(class), :class_expected)
DESCRIPTOR = 
/
  #{NAME}
  (?: \. #{NAME} )*
  | \*
/x

Constants inherited from Scanner

Scanner::DEFAULT_OPTIONS, Scanner::KINDS_NOT_LOC, Scanner::ScanError

Instance Method Summary collapse

Methods inherited from Scanner

#column, #each, file_extension, #initialize, #lang, #line, #marshal_dump, #marshal_load, normify, #reset, streamable?, #streaming?, #string=, #tokenize, #tokens

Methods included from Plugin

#helper, #included, #plugin_host, #plugin_id, #register_for, #title

Constructor Details

This class inherits a constructor from CodeRay::Scanners::Scanner

Instance Method Details

#scan_tokens(tokens, options) ⇒ Object 



97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
 
# File 'lib/coderay/scanners/python.rb', line 97

def scan_tokens tokens, options
  
  state = :initial
  string_delimiter = nil
  string_raw = false
  last_token_dot = false
  unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
  from_import_state = []
  
  until eos?
    
    kind = nil
    match = nil
    
    if state == :string
      if scan(STRING_DELIMITER_REGEXP[string_delimiter])
        tokens << [matched, :delimiter]
        tokens << [:close, :string]
        state = :initial
        next
      elsif string_delimiter.size == 3 && scan(/\n/)
        kind = :content
      elsif scan(STRING_CONTENT_REGEXP[string_delimiter])
        kind = :content
      elsif !string_raw && scan(/ \\ #{ESCAPE} /ox)
        kind = :char
      elsif scan(/ \\ #{UNICODE_ESCAPE} /ox)
        kind = :char
      elsif scan(/ \\ . /x)
        kind = :content
      elsif scan(/ \\ | $ /x)
        tokens << [:close, :string]
        kind = :error
        state = :initial
      else
        raise_inspect "else case \" reached; %p not handled." % peek(1), tokens, state
      end
    
    elsif match = scan(/ [ \t]+ | \\\n /x)
      tokens << [match, :space]
      next
    
    elsif match = scan(/\n/)
      tokens << [match, :space]
      state = :initial if state == :include_expected
      next
    
    elsif match = scan(/ \# [^\n]* /mx)
      tokens << [match, :comment]
      next
    
    elsif state == :initial
      
      if scan(/#{OPERATOR}/o)
        kind = :operator
      
      elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
        tokens << [:open, :string]
        string_delimiter = self[2]
        string_raw = false
        modifiers = self[1]
        unless modifiers.empty?
          string_raw = !!modifiers.index(?r)
          tokens << [modifiers, :modifier]
          match = string_delimiter
        end
        state = :string
        kind = :delimiter
      
      # TODO: backticks
      
      elsif match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
        kind = IDENT_KIND[match]
        # TODO: keyword arguments
        kind = :ident if last_token_dot
        if kind == :old_keyword
          kind = check(/\(/) ? :ident : :keyword
        elsif kind == :predefined && check(/ *=/)
          kind = :ident
        elsif kind == :keyword
          state = DEF_NEW_STATE[match]
          from_import_state << match.to_sym if state == :include_expected
        end
      
      elsif scan(/@[a-zA-Z0-9_.]+[lL]?/)
        kind = :decorator
      
      elsif scan(/0[xX][0-9A-Fa-f]+[lL]?/)
        kind = :hex
      
      elsif scan(/0[bB][01]+[lL]?/)
        kind = :bin
      
      elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
        kind = :float
        if scan(/[jJ]/)
          match << matched
          kind = :imaginary
        end
      
      elsif scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)
        kind = :oct
      
      elsif match = scan(/\d+([lL])?/)
        kind = :integer
        if self[1] == nil && scan(/[jJ]/)
          match << matched
          kind = :imaginary
        end
      
      else
        getch
        kind = :error
      
      end
        
    elsif state == :def_expected
      state = :initial
      if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
        kind = :method
      else
        next
      end
    
    elsif state == :class_expected
      state = :initial
      if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
        kind = :class
      else
        next
      end
      
    elsif state == :include_expected
      if match = scan(unicode ? /#{DESCRIPTOR}/uo : /#{DESCRIPTOR}/o)
        kind = :include
        if match == 'as'
          kind = :keyword
          from_import_state << :as
        elsif from_import_state.first == :from && match == 'import'
          kind = :keyword
          from_import_state << :import
        elsif from_import_state.last == :as
          # kind = match[0,1][unicode ? /[[:upper:]]/u : /[[:upper:]]/] ? :class : :method
          kind = :ident
          from_import_state.pop
        elsif IDENT_KIND[match] == :keyword
          unscan
          match = nil
          state = :initial
          next
        end
      elsif match = scan(/,/)
        from_import_state.pop if from_import_state.last == :as
        kind = :operator
      else
        from_import_state = []
        state = :initial
        next
      end
      
    else
      raise_inspect 'Unknown state', tokens, state
      
    end
    
    match ||= matched
    if $CODERAY_DEBUG and not kind
      raise_inspect 'Error token %p in line %d' %
        [[match, kind], line], tokens, state
    end
    raise_inspect 'Empty token', tokens, state unless match
    
    last_token_dot = match == '.'
    
    tokens << [match, kind]
    
  end
  
  if state == :string
    tokens << [:close, :string]
  end
  
  tokens
end