Class: Keisan::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/keisan/tokenizer.rb

Constant Summary collapse

TOKEN_CLASSES = 
[
  Tokens::Group,
  Tokens::String,
  Tokens::Null,
  Tokens::Boolean,
  Tokens::Word,
  Tokens::Number,
  Tokens::ArithmeticOperator,
  Tokens::LogicalOperator,
  Tokens::BitwiseOperator,
  Tokens::Comma
]
TOKEN_REGEX = 
Regexp::new(
  TOKEN_CLASSES.map(&:regex).join("|")
)

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(expression) ⇒ Tokenizer

Returns a new instance of Tokenizer.



22
23
24
25
26
 
# File 'lib/keisan/tokenizer.rb', line 22

def initialize(expression)
  @expression = self.class.strip_whitespace(expression)
  @scan = @expression.scan(TOKEN_REGEX)
  @tokens = tokenize!
end

Instance Attribute Details

#expression ⇒ Object (readonly)

Returns the value of attribute expression.



20
21
22
 
# File 'lib/keisan/tokenizer.rb', line 20

def expression
  @expression
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



20
21
22
 
# File 'lib/keisan/tokenizer.rb', line 20

def tokens
  @tokens
end

Class Method Details

.strip_whitespace(expression) ⇒ Object

Raises:



28
29
30
31
32
33
34
35
 
# File 'lib/keisan/tokenizer.rb', line 28

def self.strip_whitespace(expression)
  # Do not allow whitespace between variables, numbers, and the like; they must be joined by operators
  raise Keisan::Exceptions::TokenizingError.new if expression.gsub(Tokens::String.regex, "").match /\w\s+\w/

  # Only strip whitespace outside of strings, e.g.
  # "1 + 2 + 'hello world'" => "1+2+'hello world'"
  expression.split(Keisan::Tokens::String.regex).map.with_index {|s,i| i.even? ? s.gsub(/\s+/, "") : s}.join
end