Class: LanguageParser::TokenStream

Inherits:
Array
  • Object
show all
Defined in:
lib/cgialib/lp/Tokenizer.rb

Overview

class : TokenStream

An array of tokens with many useful helper methods

Instance Method Summary collapse

Constructor Details

#initialize ⇒ TokenStream

initialize()

Initalizes the array



95
96
97
98
99
 
# File 'lib/cgialib/lp/Tokenizer.rb', line 95

def initialize()
  
  super()
  
end

Instance Method Details

#code_only ⇒ Object

code_only()

Returns a new token stream with the code tokens only



132
133
134
135
136
137
138
139
140
 
# File 'lib/cgialib/lp/Tokenizer.rb', line 132

def code_only()
  
  out = TokenStream.new()
  
  each { |tok| out.push( tok ) if ( tok.is_a?( CodeToken ) ) }
  
  out
  
end

#comments_only ⇒ Object

comments_only()

Returns a new token stream with the comments only



146
147
148
149
150
151
152
153
154
 
# File 'lib/cgialib/lp/Tokenizer.rb', line 146

def comments_only()
  
  out = TokenStream.new()
  
  each { |tok| out.push( tok ) if ( tok.is_a?( CommentToken ) ) }
  
  out
  
end

#find(code_value) ⇒ Object

find( code_value )

code_value - The code token text as a string

This finds a CodeToken with the same text as the input (case insensitive) and returns the index.



250
251
252
253
254
255
256
257
258
259
260
261
262
 
# File 'lib/cgialib/lp/Tokenizer.rb', line 250

def find( code_value )
  
  each_index { |index|
  
    next unless ( at( index ).is_a?( CodeToken ) )
  
    return index if ( at( index ).to_s.downcase == code_value.to_s.downcase )
  
  }
  
  nil
  
end

#find_and_remove(code_value) ⇒ Object

find_and_remove( code_value )

code_value - The code token text as a string

This is the same as find, but it removes the item. It returns true if it found something and false if not.



271
272
273
274
275
276
277
278
279
 
# File 'lib/cgialib/lp/Tokenizer.rb', line 271

def find_and_remove( code_value )
  
  index = find( code_value )
  
  delete_at( index ) if ( index != nil )
  
  ( index == nil ) ? false : true
  
end

#find_pattern(pattern) ⇒ Object

find_pattern( pattern )

pattern - A pattern array

This searches the set of tokens for a pattern of strings. The array should contain a series of strings and lambdas. In the first pass the pattern finder looks for the strings. If a match is found the lambdas are called with the token in the original string for that position.

For example:

find_pattern( “primary”, “key”, “(”, lambda { |name| print name }, “)” )

Would print “myname” if the original sequence was “primary key ( myname )”



171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
 
# File 'lib/cgialib/lp/Tokenizer.rb', line 171

def find_pattern( pattern )
  
  code = code_only
  
  delta = ( code.length - pattern.length ) + 1
  
  delta.times { |start|
  
    found = true
  
    pattern.each_index { |index|
  
      if ( pattern[ index ].is_a?( String ) )
  
        unless ( pattern[ index ].downcase == code[ start + index ].to_s.downcase )
          found = false 
        end
  
      end
  
    }
  
    next unless ( found )
  
    pattern.each_index { |index|
  
      unless ( pattern[ index ].is_a?( String ) )
  
        pattern[index].call( code[ start + index ].to_s.downcase )
  
      end
  
    }
  
    return true
  
  }
  
  false
  
end

#get_comments(start) ⇒ Object

get_comments( start )

start - The start index

This method looks backwards from the starting index to find all of the comments and to put them together into a comment stream. It stops if it finds new CodeTokens.



221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/cgialib/lp/Tokenizer.rb', line 221

def get_comments( start )
  
  commentStream = TokenStream.new()
  
  index = start - 1
  
  while ( index > -1 )
  
    break if ( at( index ).is_a?( CodeToken ) )
  
    commentStream.unshift( at( index ) )
  
    index -= 1
  
  end
  
  comments = commentStream.map { |tok| tok.to_s }.join( "" )
  
  comments
  
end

#strip! ⇒ Object

strip!()

Deletes any leading or trailing whitespace



117
118
119
120
121
122
123
124
125
126
 
# File 'lib/cgialib/lp/Tokenizer.rb', line 117

def strip!()
  
  while( first.is_a?( WhitespaceToken ) )
    shift
  end
  while( last.is_a?( WhitespaceToken ) )
    pop
  end
  
end

#to_s ⇒ Object

to_s()

Converts all of the tokens back to text



105
106
107
108
109
110
111
 
# File 'lib/cgialib/lp/Tokenizer.rb', line 105

def to_s()
  
  text = ""
  each { |tok| text += tok.to_s }
  text
  
end