blob: 431df0c34b9631fdc53dbfd87f44f8f4dfcdc279 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
|
/* ANTLR Translator Generator
* Project led by Terence Parr at http://www.jGuru.com
* Software rights: http://www.antlr.org/license.html
*
* $Id$
*/
#include "antlr/TokenStreamHiddenTokenFilter.hpp"
#include "antlr/CommonHiddenStreamToken.hpp"
#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
namespace antlr {
#endif
/**This object filters a token stream coming from a lexer
* or another TokenStream so that only certain token channels
* get transmitted to the parser.
*
* Any of the channels can be filtered off as "hidden" channels whose
* tokens can be accessed from the parser.
*/
TokenStreamHiddenTokenFilter::TokenStreamHiddenTokenFilter(TokenStream& input)
: TokenStreamBasicFilter(input)
{
}
void TokenStreamHiddenTokenFilter::consume()
{
nextMonitoredToken = input->nextToken();
}
void TokenStreamHiddenTokenFilter::consumeFirst()
{
consume();
// Handle situation where hidden or discarded tokens
// appear first in input stream
RefToken p;
// while hidden or discarded scarf tokens
while ( hideMask.member(LA(1)->getType()) || discardMask.member(LA(1)->getType()) ) {
if ( hideMask.member(LA(1)->getType()) ) {
if ( !p ) {
p = LA(1);
}
else {
static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1));
static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p); // double-link
p = LA(1);
}
lastHiddenToken = p;
if (!firstHidden)
firstHidden = p; // record hidden token if first
}
consume();
}
}
BitSet TokenStreamHiddenTokenFilter::getDiscardMask() const
{
return discardMask;
}
/** Return a ptr to the hidden token appearing immediately after
* token t in the input stream.
*/
RefToken TokenStreamHiddenTokenFilter::getHiddenAfter(RefToken t)
{
return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenAfter();
}
/** Return a ptr to the hidden token appearing immediately before
* token t in the input stream.
*/
RefToken TokenStreamHiddenTokenFilter::getHiddenBefore(RefToken t)
{
return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenBefore();
}
BitSet TokenStreamHiddenTokenFilter::getHideMask() const
{
return hideMask;
}
/** Return the first hidden token if one appears
* before any monitored token.
*/
RefToken TokenStreamHiddenTokenFilter::getInitialHiddenToken()
{
return firstHidden;
}
void TokenStreamHiddenTokenFilter::hide(int m)
{
hideMask.add(m);
}
void TokenStreamHiddenTokenFilter::hide(const BitSet& mask)
{
hideMask = mask;
}
RefToken TokenStreamHiddenTokenFilter::LA(int)
{
return nextMonitoredToken;
}
/** Return the next monitored token.
* Test the token following the monitored token.
* If following is another monitored token, save it
* for the next invocation of nextToken (like a single
* lookahead token) and return it then.
* If following is unmonitored, nondiscarded (hidden)
* channel token, add it to the monitored token.
*
* Note: EOF must be a monitored Token.
*/
RefToken TokenStreamHiddenTokenFilter::nextToken()
{
// handle an initial condition; don't want to get lookahead
// token of this splitter until first call to nextToken
if ( !LA(1) ) {
consumeFirst();
}
// we always consume hidden tokens after monitored, thus,
// upon entry LA(1) is a monitored token.
RefToken monitored = LA(1);
// point to hidden tokens found during last invocation
static_cast<CommonHiddenStreamToken*>(monitored.get())->setHiddenBefore(lastHiddenToken);
lastHiddenToken = nullToken;
// Look for hidden tokens, hook them into list emanating
// from the monitored tokens.
consume();
RefToken p = monitored;
// while hidden or discarded scarf tokens
while ( hideMask.member(LA(1)->getType()) || discardMask.member(LA(1)->getType()) ) {
if ( hideMask.member(LA(1)->getType()) ) {
// attach the hidden token to the monitored in a chain
// link forwards
static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1));
// link backwards
if (p != monitored) { //hidden cannot point to monitored tokens
static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p);
}
p = lastHiddenToken = LA(1);
}
consume();
}
return monitored;
}
#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
#endif
|
