Class: ListTokenSource

Inherits:
TokenSource show all
Defined in:
lib/antlr4/ListTokenSource.rb

Overview

Provides an implementation of TokenSource as a wrapper around a list of Token objects.

<p>If the final token in the list is an Token#EOF token, it will be used as the EOF token for every call to #nextToken after the end of the list is reached. Otherwise, an EOF token will be created.</p>

Instance Attribute Summary collapse

Attributes inherited from Recognizer

#interp, #listeners, #ruleIndexMapCache, #state, #tokenTypeMapCache

Instance Method Summary collapse

Methods inherited from Recognizer

#addErrorListener, #checkVersion, #extractVersion, #getErrorHeader, #getErrorListenerDispatch, #getRuleIndexMap, #getState, #getTokenErrorDisplay, #getTokenType, #getTokenTypeMap, #precpred, #sempred

Constructor Details

#initialize(_tokens, source_name = nil) ⇒ ListTokenSource

Returns a new instance of ListTokenSource.

Raises:

  • (ReferenceError) 


23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/antlr4/ListTokenSource.rb', line 23

def initialize(_tokens, source_name=nil)
    raise ReferenceError.new("tokens cannot be null") if tokens.nil? 
    @tokens = _tokens
    @sourceName = source_name
    # The index into {@link #tokens} of token to return by the next call to
    # {@link #nextToken}. The end of the input is indicated by this value
    # being greater than or equal to the number of items in {@link #tokens}.
    @pos = 0
    # This field caches the EOF token for the token source.
    @eofToken = nil
    # This is the backing field for {@link #getTokenFactory} and
    @factory = CommonTokenFactory.DEFAULT
end

Instance Attribute Details

#eofToken ⇒ Object

Constructs a new ListTokenSource instance from the specified collection of Token objects and source name.

TokenSource. null, #getSourceName will attempt to infer the name from the next Token (or the previous token if the end of the input has been reached).

Parameters:

  • tokens —

    The collection of Token objects to provide as a

  • sourceName —

    The name of the TokenSource. If this value is



22
23
24
 
# File 'lib/antlr4/ListTokenSource.rb', line 22

def eofToken
  @eofToken
end

#factory ⇒ Object

Constructs a new ListTokenSource instance from the specified collection of Token objects and source name.

TokenSource. null, #getSourceName will attempt to infer the name from the next Token (or the previous token if the end of the input has been reached).

Parameters:

  • tokens —

    The collection of Token objects to provide as a

  • sourceName —

    The name of the TokenSource. If this value is



22
23
24
 
# File 'lib/antlr4/ListTokenSource.rb', line 22

def factory
  @factory
end

#pos ⇒ Object

Constructs a new ListTokenSource instance from the specified collection of Token objects and source name.

TokenSource. null, #getSourceName will attempt to infer the name from the next Token (or the previous token if the end of the input has been reached).

Parameters:

  • tokens —

    The collection of Token objects to provide as a

  • sourceName —

    The name of the TokenSource. If this value is



22
23
24
 
# File 'lib/antlr4/ListTokenSource.rb', line 22

def pos
  @pos
end

#sourceName ⇒ Object

Constructs a new ListTokenSource instance from the specified collection of Token objects and source name.

TokenSource. null, #getSourceName will attempt to infer the name from the next Token (or the previous token if the end of the input has been reached).

Parameters:

  • tokens —

    The collection of Token objects to provide as a

  • sourceName —

    The name of the TokenSource. If this value is



22
23
24
 
# File 'lib/antlr4/ListTokenSource.rb', line 22

def sourceName
  @sourceName
end

#tokens ⇒ Object

Constructs a new ListTokenSource instance from the specified collection of Token objects and source name.

TokenSource. null, #getSourceName will attempt to infer the name from the next Token (or the previous token if the end of the input has been reached).

Parameters:

  • tokens —

    The collection of Token objects to provide as a

  • sourceName —

    The name of the TokenSource. If this value is



22
23
24
 
# File 'lib/antlr4/ListTokenSource.rb', line 22

def tokens
  @tokens
end

Instance Method Details

#column ⇒ Object

@inheritDoc



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/antlr4/ListTokenSource.rb', line 41

def column
    if self.pos < self.tokens.length
        return self.tokens[self.pos].column
    elsif not self.eofToken.nil? 
        return self.eofToken.column
    elsif self.tokens.length > 0
        # have to calculate the result from the line/column of the previous
        # token, along with the text of the token.
        lastToken = self.tokens[-1]
        tokenText = lastToken.getText()
        if not tokenText.nil? then
            lastNewLine = tokenText.rfind('\n')
            if lastNewLine >= 0 
                return tokenText.length - lastNewLine - 1
            end
        end
        return lastToken.column + lastToken.stopIndex - lastToken.startIndex + 1
    end
    # only reach this if tokens is empty, meaning EOF occurs at the first
    # position in the input
    return 0
end

#getInputStream ⇒ Object

@inheritDoc



118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/antlr4/ListTokenSource.rb', line 118

def getInputStream
    if self.pos < self.tokens.length
        return self.tokens[self.pos].getInputStream()
    elsif not self.eofToken.nil?
        return self.eofToken.getInputStream()
    elsif self.tokens.length > 0
        return self.tokens[-1].getInputStream()
    else
        # no input stream information is available
        return nil
    end
end

#getSourceName ⇒ Object 



131
132
133
134
135
136
137
138
139
 
# File 'lib/antlr4/ListTokenSource.rb', line 131

def getSourceName
    return self.sourceName unless self.sourceName.nil?
    inputStream = self.getInputStream()
    if not inputStream.nil?
        return inputStream.getSourceName()
    else
        return "List"
    end
end

#line ⇒ Object 



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/antlr4/ListTokenSource.rb', line 90

def line
    if self.pos < self.tokens.length
        return self.tokens[self.pos].line
    elsif not self.eofToken.nil? 
        return self.eofToken.line
    elsif self.tokens.length > 0
        # have to calculate the result from the line/column of the previous
        # token, along with the text of the token.
        lastToken = self.tokens[-1]
        line = lastToken.line
        tokenText = lastToken.text
        if not tokenText.nil? then
            for c in tokenText do
                if c  == '\n'
                    line = line + 1
                end
            end
        end
        # if no text is available, assume the token did not contain any newline characters.
        return line
    end
    # only reach this if tokens is empty, meaning EOF occurs at the first
    # position in the input
    return 1
end

#nextToken ⇒ Object

@inheritDoc



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
 
# File 'lib/antlr4/ListTokenSource.rb', line 66

def nextToken
    if self.pos >= self.tokens.length then
        if self.eofToken.nil? then
            start = -1
            if self.tokens.length > 0 then
                previousStop = self.tokens[-1].stopIndex
                if previousStop != -1 then
                    start = previousStop + 1
                end
            end
            stop = [-1, start - 1].max
            self.eofToken = self.factory.create([self, self.getInputStream()],
                        Token::EOF, "EOF", Token::DEFAULT_CHANNEL, start, stop, self.line, self.column)
            stop = [-1, start - 1].max
        end
        return self.eofToken
    end
    t = self.tokens[self.pos]
    if self.pos == self.tokens.length - 1 and t.type == Token::EOF
        eofToken = t
    end
    self.pos = self.pos + 1
    return t
end