Module: LangScan::Python

Defined in:
lib/langscan/python.rb,
ext/langscan/python/python/python.c

Defined Under Namespace

Classes: Tokenizer

Constant Summary collapse

Keywords = 
%w(
  and assert break class continue def del elif else except exec finally
  for from global if import in is lambda not or pass print raise return
  try while yield
)
FutureKeywords = 
%w(
  as None
)
KeywordsHash = 
{}

Class Method Summary collapse

Class Method Details

.abbrev ⇒ Object 



22
23
24
 
# File 'lib/langscan/python.rb', line 22

def abbrev
  "python"
end

.each_fragment(input) ⇒ Object

LangScan::Python.each_fragment iterates over Python-language fragments in input. The fragments contains tokens and inter-token spaces and comments.

If a String is specified, the String itself is assumed as a Python-program. If a IO is specified, the content of the IO is assumed as a Python-program.



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/langscan/python.rb', line 69

def each_fragment(input) # :yields: token
  begin
    tokenizer = Tokenizer.new(input)
    while token_info = tokenizer.get_token
      type, text, beg_lineno, beg_columnno, beg_byteno, end_lineno, end_columnno, end_byteno = token_info
      token = Fragment.new(type, text, beg_lineno, beg_byteno)
      if token.type == :ident
        if KeywordsHash[token.text]
          token.type = :keyword 
        end
      end
      yield token
    end
  ensure
    tokenizer.close
  end
end

.extnames ⇒ Object 



26
27
28
 
# File 'lib/langscan/python.rb', line 26

def extnames
  [".py"]
end

.name ⇒ Object 



18
19
20
 
# File 'lib/langscan/python.rb', line 18

def name
  "Python"
end

.scan(input, &block) ⇒ Object

LangScan::Python.scan iterates over Python program. It yields for each Fragment.



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/langscan/python.rb', line 32

def scan(input, &block)
  last_token = nil
  each_fragment(input) {|t|
    if t.type == :space
      yield t
      next
    end
    if last_token
      if t.type == :ident and last_token.type == :keyword
        case last_token.text
        when 'def'
          t.type = :fundef
        when 'class'
          t.type = :classdef
        end
      elsif t.type == :punct and t.text == '(' and last_token.type == :ident
        last_token.type = :funcall
      end
      yield last_token
      last_token = nil
    end
    if t.type == :ident or t.type == :keyword
      last_token = t
    else
      yield t
    end
  }
  if last_token
    yield last_token
  end
end