Class: Cooklang::TokenStream

Inherits:
Object
  • Object
show all
Extended by:
Forwardable
Includes:
Enumerable
Defined in:
lib/cooklang/token_stream.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(tokens) ⇒ TokenStream

Returns a new instance of TokenStream.



15
16
17
18
 
# File 'lib/cooklang/token_stream.rb', line 15

def initialize(tokens)
  @tokens = tokens
  @position = 0
end

Instance Attribute Details

#position ⇒ Object

Returns the value of attribute position.



13
14
15
 
# File 'lib/cooklang/token_stream.rb', line 13

def position
  @position
end

#tokens ⇒ Object (readonly)

Public interface methods to avoid instance_variable_get/set



119
120
121
 
# File 'lib/cooklang/token_stream.rb', line 119

def tokens
  @tokens
end

Instance Method Details

#advance_to(new_position) ⇒ Object 



126
127
128
 
# File 'lib/cooklang/token_stream.rb', line 126

def advance_to(new_position)
  self.position = new_position
end

#check(type) ⇒ Object 



53
54
55
 
# File 'lib/cooklang/token_stream.rb', line 53

def check(type)
  current&.type == type
end

#consume(expected_type = nil) ⇒ Object 



28
29
30
31
32
33
34
35
 
# File 'lib/cooklang/token_stream.rb', line 28

def consume(expected_type = nil)
  return nil if eof?
  return nil if expected_type && current&.type != expected_type

  token = current
  @position += 1
  token
end

#consume_until(&block) ⇒ Object 



70
71
72
73
74
75
76
 
# File 'lib/cooklang/token_stream.rb', line 70

def consume_until(&block)
  result = []
  until eof? || block.call(current)
    result << consume
  end
  result
end

#consume_while(&block) ⇒ Object

Convenience methods for complex parsing



62
63
64
65
66
67
68
 
# File 'lib/cooklang/token_stream.rb', line 62

def consume_while(&block)
  result = []
  while !eof? && block.call(current)
    result << consume
  end
  result
end

#current ⇒ Object 



20
21
22
 
# File 'lib/cooklang/token_stream.rb', line 20

def current
  @tokens[@position]
end

#each ⇒ Object

Ruby Enumerable support



42
43
44
45
46
 
# File 'lib/cooklang/token_stream.rb', line 42

def each
  while !eof?
    yield consume
  end
end

#each_with_lookahead ⇒ Object

Advanced iteration with lookahead



83
84
85
86
87
88
89
 
# File 'lib/cooklang/token_stream.rb', line 83

def each_with_lookahead
  return enum_for(:each_with_lookahead) unless block_given?

  (0...(@tokens.length - 1)).each do |i|
    yield @tokens[i], @tokens[i + 1]
  end
end

#eof? ⇒ Boolean

Returns:

  • (Boolean) 


37
38
39
 
# File 'lib/cooklang/token_stream.rb', line 37

def eof?
  @position >= @tokens.length
end

#find_next(type) ⇒ Object

Utility methods



105
106
107
 
# File 'lib/cooklang/token_stream.rb', line 105

def find_next(type)
  (@position...@tokens.length).find { |i| @tokens[i].type == type }
end

#find_next_matching(&block) ⇒ Object 



109
110
111
 
# File 'lib/cooklang/token_stream.rb', line 109

def find_next_matching(&block)
  (@position...@tokens.length).find { |i| block.call(@tokens[i]) }
end

#peek(offset = 1) ⇒ Object 



24
25
26
 
# File 'lib/cooklang/token_stream.rb', line 24

def peek(offset = 1)
  @tokens[@position + offset]
end

#reset ⇒ Object 



96
97
98
 
# File 'lib/cooklang/token_stream.rb', line 96

def reset
  @position = 0
end

#rest ⇒ Object

StringScanner-inspired position methods



92
93
94
 
# File 'lib/cooklang/token_stream.rb', line 92

def rest
  @tokens[@position..]
end

#rewind(steps = 1) ⇒ Object 



100
101
102
 
# File 'lib/cooklang/token_stream.rb', line 100

def rewind(steps = 1)
  @position = [@position - steps, 0].max
end

#scan(type) ⇒ Object

StringScanner-inspired methods



49
50
51
 
# File 'lib/cooklang/token_stream.rb', line 49

def scan(type)
  consume if check(type)
end

#skip(type) ⇒ Object 



57
58
59
 
# File 'lib/cooklang/token_stream.rb', line 57

def skip(type)
  @position += 1 if check(type)
end

#skip_whitespace ⇒ Object 



78
79
80
 
# File 'lib/cooklang/token_stream.rb', line 78

def skip_whitespace
  consume_while { |token| token.type == :whitespace }
end

#slice_from_current ⇒ Object

Create a new stream starting from current position



114
115
116
 
# File 'lib/cooklang/token_stream.rb', line 114

def slice_from_current
  TokenStream.new(@tokens[@position..])
end