blob: 5832ef6e7ffc4c6c45e0bd0876cd97d830f71aa7 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
#ifndef INC_TokenBuffer_h__
#define INC_TokenBuffer_h__
/* ANTLR Translator Generator
* Project led by Terence Parr at http://www.jGuru.com
* Software rights: http://www.antlr.org/license.html
*
* $Id$
*/
#include <antlr/config.h>
#include <antlr/TokenStream.h>
#include <antlr/CircularQueue.h>
#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
namespace antlr {
#endif
/**A Stream of Token objects fed to the parser from a TokenStream that can
* be rewound via mark()/rewind() methods.
* <p>
* A dynamic array is used to buffer up all the input tokens. Normally,
* "k" tokens are stored in the buffer. More tokens may be stored during
* guess mode (testing syntactic predicate), or when LT(i>k) is referenced.
* Consumption of tokens is deferred. In other words, reading the next
* token is not done by conume(), but deferred until needed by LA or LT.
* <p>
*
* @todo: see if we can integrate this one with InputBuffer into one template
* or so.
*
* @see antlr.Token
* @see antlr.TokenStream
* @see antlr.TokenQueue
*/
class ANTLR_API TokenBuffer {
public:
/** Create a token buffer */
TokenBuffer(TokenStream& input_);
virtual ~TokenBuffer();
/// Reset the input buffer to empty state
inline void reset( void )
{
nMarkers = 0;
markerOffset = 0;
numToConsume = 0;
queue.clear();
}
/** Get a lookahead token value */
int LA( unsigned int i );
/** Get a lookahead token */
RefToken LT( unsigned int i );
/** Return an integer marker that can be used to rewind the buffer to
* its current state.
*/
unsigned int mark();
/**Rewind the token buffer to a marker.
* @param mark Marker returned previously from mark()
*/
void rewind(unsigned int mark);
/** Mark another token for deferred consumption */
inline void consume()
{
numToConsume++;
}
/// Return the number of entries in the TokenBuffer
virtual unsigned int entries() const;
private:
/** Ensure that the token buffer is sufficiently full */
void fill(unsigned int amount);
/** Sync up deferred consumption */
void syncConsume();
protected:
/// Token source
TokenStream& input;
/// Number of active markers
unsigned int nMarkers;
/// Additional offset used when markers are active
unsigned int markerOffset;
/// Number of calls to consume() since last LA() or LT() call
unsigned int numToConsume;
/// Circular queue with Tokens
CircularQueue<RefToken> queue;
private:
TokenBuffer(const TokenBuffer& other);
const TokenBuffer& operator=(const TokenBuffer& other);
};
/** Sync up deferred consumption */
inline void TokenBuffer::syncConsume()
{
if (numToConsume > 0)
{
if (nMarkers > 0)
markerOffset += numToConsume;
else
queue.removeItems( numToConsume );
numToConsume = 0;
}
}
#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
#endif
#endif //INC_TokenBuffer_h__
|
