Class: LangScan::OCaml::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/langscan/ocaml.rb

Constant Summary collapse

SYMBOL_TBL = 
{
  "text" => :text,
  "ident" => :ident,
  "punct" => :punct,
  "keyword" => :keyword,
  "comment" => :comment,
  "integer" => :integer,
  "float" => :float,
  "string" => :string,
  "character" => :character,
  "funcdef" => :funcdef         # not implemented yet
}

Instance Method Summary collapse

Constructor Details

#initialize(input) ⇒ Tokenizer

Returns a new instance of Tokenizer.



37
38
39
40
41
42
43
44
45
 
# File 'lib/langscan/ocaml.rb', line 37

def initialize(input)
  @io = IO.popen(CAMLEXER_PATH, "r+")
  @tin = Thread.start {
    input.each {|l|
      @io.puts(l)
    }
    @io.close_write()
  }
end

Instance Method Details

#denormalize(str) ⇒ Object 



52
53
54
 
# File 'lib/langscan/ocaml.rb', line 52

def denormalize(str)
  str.gsub(/([^\\])\\o/,'\1'+"\n")
end

#dispose ⇒ Object 



47
48
49
50
 
# File 'lib/langscan/ocaml.rb', line 47

def dispose()
  @tin.join()
  @io.close()
end

#get_token ⇒ Object 



56
57
58
59
60
61
62
63
 
# File 'lib/langscan/ocaml.rb', line 56

def get_token()
  if @io.eof? 
    nil
  else
    lno, cno, tp, wd = @io.gets().chomp().split(":",4)
    Fragment.new(SYMBOL_TBL[tp], denormalize(wd), lno.to_i(), cno.to_i())
  end
end