Class: Ast::Tokens

Inherits:
Array
  • Object
show all
Defined in:
lib/ast_ast/tokens.rb

Overview

An Array of Token instances basically, but with added methods which add StringScanner type capabilities.

Defined Under Namespace

Classes: Error

Instance Attribute Summary collapse

Scanning Tokens collapse

Enumeration collapse

Instance Method Summary collapse

Constructor Details

#initialize(args = []) ⇒ Tokens

Creates tokens for each item given if not already and sets pointer.



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/ast_ast/tokens.rb', line 12

def initialize(args=[])
  @pos = 0
  return self if args == []
  if args[0].is_a? Token
    args.each_token do |i|
      self << i
    end
  else
    args.each do |i|
      if i.size > 0
        self << Token.new(i[0], i[1])
      else
        self << Token.new(i[0], nil)
      end
    end
  end
  self
end

Instance Attribute Details

#pos ⇒ Object

Returns the value of attribute pos.



6
7
8
 
# File 'lib/ast_ast/tokens.rb', line 6

def pos
  @pos
end

#prev_pos ⇒ Object

Returns the value of attribute prev_pos.



6
7
8
 
# File 'lib/ast_ast/tokens.rb', line 6

def prev_pos
  @prev_pos
end

Instance Method Details

#<<(val) ⇒ Tokens

Adds val to self, if a Token is given it is added as expected. If an Array is given and it is valid, it will be converted to a Token and added, if invalid an error is raised.

Parameters:

Returns:



38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/ast_ast/tokens.rb', line 38

def <<(val)
  raise "value given #{val} is invalid" unless Token.valid?(val)
  if val.is_a? Array
    if val.size > 0
      self << Token.new(val[0], val[1])
    else
      self << Token.new(val[0], nil)
    end
  else
    super
  end
end

#_each ⇒ Object 



266
 
# File 'lib/ast_ast/tokens.rb', line 266

alias_method :_each, :each

#check(type = nil) ⇒ Token

Reads the current token, but does not advance pointer. If a type is given it will throw an error if types do not match.

Parameters:

  • type (Symbol) (defaults to: nil)

Returns:

Raises:



138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/ast_ast/tokens.rb', line 138

def check(type=nil)
  if type.nil?
    pointer
  else
    if pointing_at?(type)
      pointer
    else
      raise Error, "wrong type: #{type} for #{self.pointer}"
    end
  end
end

#check_until(type) ⇒ Tokens

Reads the tokens until a token of type is found. Returns tokens upto and including the matched token, but does not advance the pointer.

Parameters:

  • type (Symbol)

Returns:

See Also:



207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/ast_ast/tokens.rb', line 207

def check_until(type)
  r = Tokens.new
  a = 0
  until pointing_at?(type) || self.eot?
    r << scan
    a += 1
  end
  r << scan
  @pos -= a + 1
  r
end

#clear ⇒ Object

Set the scan pointer to the end of the tokens.



246
247
248
 
# File 'lib/ast_ast/tokens.rb', line 246

def clear
  @pos = self.size-1
end

#dec ⇒ Integer?

Decrement the pointer unless at first token.

Returns:

  • (Integer, nil) —

    New position



85
86
87
 
# File 'lib/ast_ast/tokens.rb', line 85

def dec
  @pos -= 1 unless @pos == 0
end

#each {|Symbol, Object| ... } ⇒ Object

Loops through the types and contents of each tag separately, passing them to the block given.

Examples:


tokens = Ast::Tokens.new
tokens << [:a, 1] << [:b, 2] << [:c, 3] << [:d, 4]

sa.each do |t, v|
  puts "#{t} -> #{v}"
end
#=> a -> 1
#=> b -> 2
#=> c -> 3
#=> d -> 4

Yields:

  • (Symbol, Object) —

    gives the type and content of each block in turn



286
287
288
289
290
291
 
# File 'lib/ast_ast/tokens.rb', line 286

def each(&blck)
  self._each do |i|
    yield(i.type, i.value)
  end
  self
end

#each_token {|Token| ... } ⇒ Object

Loops through the tokens, passing them to block given.

Examples:


tokens = Ast::Tokens.new
tokens << [:a, 1] << [:b, 2] << [:c, 3] << [:d, 4]

sa.each_token do |token|
  puts token
end
#=> <:a, 1>
#=> <:b, 2>
#=> <:c, 3>
#=> <:d, 4>

Yields:

See Also:



362
363
364
365
366
 
# File 'lib/ast_ast/tokens.rb', line 362

def each_token(&blck)
  self._each do |i|
    yield(i)
  end
end

#each_type {|Symbol| ... } ⇒ Object

Loops through the types of each tag, passing them to block given.

Examples:


tokens = Ast::Tokens.new
tokens << [:a, 1] << [:b, 2] << [:c, 3] << [:d, 4]

sa.each_type do |t|
  puts t
end
#=> a
#=> b
#=> c
#=> d

Yields:

  • (Symbol)

See Also:



311
312
313
314
315
 
# File 'lib/ast_ast/tokens.rb', line 311

def each_type(&blck)
  self._each do |i|
    yield(i.type)
  end
end

#each_value {|Object| ... } ⇒ Object

Loops through the values of each tag, passing them to block given.

Examples:


tokens = Ast::Tokens.new
tokens << [:a, 1] << [:b, 2] << [:c, 3] << [:d, 4]

sa.each_value do |v|
  puts v
end
#=> 1
#=> 2
#=> 3
#=> 4

Yields:

  • (Object)

See Also:



335
336
337
338
339
 
# File 'lib/ast_ast/tokens.rb', line 335

def each_value(&blck)
  self._each do |i|
    yield(i.value)
  end
end

#eot? ⇒ boolean

Returns whether at end of tokens.

Returns:

  • (boolean) —

    whether at end of tokens



173
174
175
 
# File 'lib/ast_ast/tokens.rb', line 173

def eot?
  @pos >= self.size-1
end

#inc ⇒ Integer?

Increment the pointer unless at end of tokens.

Returns:

  • (Integer, nil) —

    New position



76
77
78
 
# File 'lib/ast_ast/tokens.rb', line 76

def inc
  @pos += 1 unless eot?
end

#inspect ⇒ Object 



59
60
61
 
# File 'lib/ast_ast/tokens.rb', line 59

def inspect
  "#< #{@pos}/#{self.size-1} #{self.to_s[1..-2]} >"
end

#peek(len) ⇒ Tokens

Gets a list of tokens len from current position, without advancing pointer.

Parameters:

  • len (Integer)

Returns:



111
112
113
 
# File 'lib/ast_ast/tokens.rb', line 111

def peek(len)
  self[@pos..(@pos+len-1)]
end

#pointer ⇒ Token Also known as: curr_item

Returns the current token being ‘pointed’ to.

Returns:

  • (Token) —

    the current token being ‘pointed’ to



66
67
68
 
# File 'lib/ast_ast/tokens.rb', line 66

def pointer
  self[@pos]
end

#pointing_at ⇒ Symbol

Gets the type of the current token.

Returns:

  • (Symbol) 


101
102
103
 
# File 'lib/ast_ast/tokens.rb', line 101

def pointing_at
  pointer.type
end

#pointing_at?(type) ⇒ true, false

Checks whether the pointer is at a token with type type

Returns:

  • (true, false) 


93
94
95
 
# File 'lib/ast_ast/tokens.rb', line 93

def pointing_at?(type)
  pointing_at == type
end

#rest ⇒ Tokens

Returns All tokens after the current token.

Returns:

  • (Tokens) —

    All tokens after the current token.



240
241
242
 
# File 'lib/ast_ast/tokens.rb', line 240

def rest
  self[pos..-1]
end

#scan(type = nil) ⇒ Token

Reads the current token and advances the pointer. If a type is given it will throw an error if types do not match.

Parameters:

  • type (Symbol) (defaults to: nil)

Returns:

Raises:



123
124
125
126
127
128
 
# File 'lib/ast_ast/tokens.rb', line 123

def scan(type=nil)
  @prev_pos = @pos
  a = check(type)
  inc
  a
end

#scan_until(type) ⇒ Tokens

Reads the tokens until a token of type is found. Return tokens upto and including the matched token, also advances pointer.

Parameters:

  • type (Symbol)

Returns:

See Also:



188
189
190
191
192
193
194
195
196
 
# File 'lib/ast_ast/tokens.rb', line 188

def scan_until(type)
  @prev_pos = @pos
  r = Tokens.new
  until pointing_at?(type) || self.eot?
    r << scan
  end
  r << scan
  r
end

#skip(type = nil) ⇒ Integer

Attempts to skip the current token. If type is given will only skip a token of that type, will raise error for anything else.

Parameters:

  • type (Symbol) (defaults to: nil)

Returns:

  • (Integer) —

    The new pointer position

Raises:

  • (Error) —

    if type of next token does not match type



159
160
161
162
163
164
165
166
167
168
169
170
 
# File 'lib/ast_ast/tokens.rb', line 159

def skip(type=nil)
  @prev_pos = @pos
  if type.nil?
    inc
  else
    if pointing_at?(type)
      inc
    else
      raise Error, "wrong type: #{type} for #{self.pointer}"
    end
  end
end

#skip_until(type) ⇒ Integer

Advances the pointer until token of type is found.

Parameters:

  • type (Symbol)

Returns:

  • (Integer) —

    Number of tokens advanced, including match



225
226
227
228
229
230
231
232
233
234
235
 
# File 'lib/ast_ast/tokens.rb', line 225

def skip_until(type)
  @prev_pos = @pos
  r = 0
  until pointing_at?(type) || self.eot?
    inc
    r += 1
  end
  inc
  r += 1
  r
end

#to_a ⇒ Array

Turns the Tokens, and Token instances inside into arrays.

Returns:

  • (Array) 


55
56
57
 
# File 'lib/ast_ast/tokens.rb', line 55

def to_a
  self.collect {|i| i.to_a }
end

#unscan ⇒ Object Also known as: unskip

Sets the pointer to the previous remembered position. Only one previous position is remembered, which is updated every scan or skip.



254
255
256
257
258
259
 
# File 'lib/ast_ast/tokens.rb', line 254

def unscan
  if @prev_pos
    @pos = @prev_pos
    @prev_pos = nil
  end
end