Class: LXL::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/lxl.rb

Overview

Based on John Carter’s LittleLexer (littlelexer.rubyforge.org).

Defined Under Namespace

Classes: LexerJammed

Instance Method Summary collapse

Constructor Details

#initialize(re_to_chr, skip_whitespace = true) ⇒ Lexer

Returns a new instance of Lexer.



569
570
571
572
 
# File 'lib/lxl.rb', line 569

def initialize(re_to_chr, skip_whitespace=true)
  @re_to_chr = re_to_chr
  @skip_whitespace = skip_whitespace
end

Instance Method Details

#scan(string, string_tokens = nil) ⇒ Object 



574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
 
# File 'lib/lxl.rb', line 574

def scan(string, string_tokens=nil)
  types = String.new
  tokens = Array.new
  if string_tokens
    next_token(string) do |t,token,tail|
      types << t
      tokens << [string_tokens[0...tail], string[0...tail]]
      string = string[tail..-1]
      string_tokens = string_tokens[tail..-1]
    end
  else
    next_token(string) do |t,token,tail|
      types << t
      tokens << token
    end
  end
  return types,tokens
end