Class: EyeOfNewt::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/eye_of_newt/tokenizer.rb

Constant Summary collapse

NO_MATCH =

should never match anything

/a^/
WHITESPACE = 
/\s+/
ANYTHING = 
/[^()]+/
WORD = 
/\w+(-\w+)*/
NUMBER = 
/\d+/
OF = 
/of/
A = 
/an?/
TO_TASTE = 
/to taste/

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(string, units: EyeOfNewt.units.all, unit_modifiers: EyeOfNewt.units.unit_modifiers) ⇒ Tokenizer

Returns a new instance of Tokenizer.



17
18
19
20
21
22
 
# File 'lib/eye_of_newt/tokenizer.rb', line 17

def initialize(string, units: EyeOfNewt.units.all, unit_modifiers: EyeOfNewt.units.unit_modifiers)
  @string = string
  @units = units
  @unit_modifiers = unit_modifiers
  @ss = StringScanner.new(string)
end

Instance Attribute Details

#string ⇒ Object (readonly)

Returns the value of attribute string.



15
16
17
 
# File 'lib/eye_of_newt/tokenizer.rb', line 15

def string
  @string
end

#unit_modifiers ⇒ Object (readonly)

Returns the value of attribute unit_modifiers.



15
16
17
 
# File 'lib/eye_of_newt/tokenizer.rb', line 15

def unit_modifiers
  @unit_modifiers
end

#units ⇒ Object (readonly)

Returns the value of attribute units.



15
16
17
 
# File 'lib/eye_of_newt/tokenizer.rb', line 15

def units
  @units
end

Instance Method Details

#next_token ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/eye_of_newt/tokenizer.rb', line 24

def next_token
  return if @ss.eos?

  @ss.scan(WHITESPACE)

  case
  when @scan_text && text = @ss.scan(/#{ANYTHING}\b/)
    @scan_text = false
    [:TEXT, text]
  when text = @ss.scan(NUMBER)
    [:NUMBER, text]
  when text = @ss.scan(/#{OF}\b/)
    [:OF, text]
  when text = @ss.scan(/#{A}\b/)
    [:A, text]
  when text = @ss.scan(/#{TO_TASTE}\b/)
    [:TO_TASTE, text]
  when text = @ss.scan(/#{unit_matcher}\b/)
    [:UNIT, text]
  when text = @ss.scan(/#{unit_modifier}\b/)
    [:UNIT_MODIFIER, text]
  when text = @ss.scan(/#{WORD}\b/)
    [:WORD, text]
  else
    x = @ss.getch
    @scan_text = true if x == ',' || x == '('
    [x, x]
  end
end