Class: Keisan::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/keisan/tokenizer.rb

Constant Summary collapse

TOKEN_CLASSES = 
[
  Tokens::Null,
  Tokens::Boolean,
  Tokens::Word,
  Tokens::Number,
  Tokens::Assignment,
  Tokens::BitwiseShift,
  Tokens::LogicalOperator,
  Tokens::ArithmeticOperator,
  Tokens::BitwiseOperator,
  Tokens::Comma,
  Tokens::Colon,
  Tokens::Dot,
  Tokens::LineSeparator,
  Tokens::Unknown
]
TOKEN_REGEX = 
Regexp::new(
  TOKEN_CLASSES.map(&:regex).join("|")
)

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(expression) ⇒ Tokenizer

Returns a new instance of Tokenizer.



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/keisan/tokenizer.rb', line 26

def initialize(expression)
  @expression = self.class.normalize_expression(expression)

  portions = StringAndGroupParser.new(@expression).portions

  @tokens = portions.inject([]) do |tokens, portion|
    case portion
    when StringAndGroupParser::StringPortion
      tokens << Tokens::String.new(portion.to_s)
    when StringAndGroupParser::GroupPortion
      tokens << Tokens::Group.new(portion.to_s)
    when StringAndGroupParser::OtherPortion
      scan = portion.to_s.scan(TOKEN_REGEX)
      tokens += tokenize!(scan)
    end

    tokens
  end
end

Instance Attribute Details

#expression ⇒ Object (readonly)

Returns the value of attribute expression.



24
25
26
 
# File 'lib/keisan/tokenizer.rb', line 24

def expression
  @expression
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



24
25
26
 
# File 'lib/keisan/tokenizer.rb', line 24

def tokens
  @tokens
end

Class Method Details

.normalize_expression(expression) ⇒ Object 



46
47
48
49
 
# File 'lib/keisan/tokenizer.rb', line 46

def self.normalize_expression(expression)
  expression = normalize_line_delimiters(expression)
  expression = remove_comments(expression)
end