mirrors/cdt
1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-04-29 19:45:01 +02:00
Code Issues Wiki Activity

Cleaned up token storage in the Parser to reduce peak memory usage upon parsing large expressions/initializers.

Browse source
This commit is contained in:
John Camelon 2004-05-19 21:50:52 +00:00
parent d6606182b1
commit b87794dd73
5 changed files with 70 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/NullSourceElementRequestor.java
View file

@ -475,8 +475,7 @@ public class NullSourceElementRequestor implements ISourceElementRequestor
/* (non-Javadoc)
* @see org.eclipse.cdt.core.parser.ISourceElementRequestor#parserTimeout()
*/
public boolean parserTimeout() {
// TODO Auto-generated method stub
public boolean parserTimeout() {
return false;
}

35
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java
View file

@ -230,6 +230,7 @@ public class ExpressionParser implements IExpressionParser, IParserData {
* @throws BacktrackException request a backtrack
*/
protected IToken consumeTemplateParameters(IToken previousLast) throws EndOfFileException, BacktrackException {
if( language != ParserLanguage.CPP ) return previousLast;
IToken last = previousLast;
if (LT(1) == IToken.tLT)
{
@ -524,6 +525,7 @@ public class ExpressionParser implements IExpressionParser, IParserData {
* @throws BacktrackException
*/
protected IToken consumeTemplateArguments(IASTScope scope, IToken last, TemplateParameterManager argumentList) throws EndOfFileException, BacktrackException {
if( language != ParserLanguage.CPP ) return last;
if( LT(1) == IToken.tLT ){
IToken secondMark = mark();
consume( IToken.tLT );
@ -1206,11 +1208,9 @@ public class ExpressionParser implements IExpressionParser, IParserData {
case IToken.tLTEQUAL :
case IToken.tGTEQUAL :
IToken mark = mark();
IToken t = consume();
IToken next = LA(1);
IASTExpression secondExpression =
shiftExpression(scope,kind, key);
if (next == LA(1))
int t = consume().getType();
IASTExpression secondExpression = shiftExpression(scope,kind, key);
if (LA(1) == mark.getNext() )
{
// we did not consume anything
// this is most likely an error
@ -1218,7 +1218,7 @@ public class ExpressionParser implements IExpressionParser, IParserData {
return firstExpression;
}
IASTExpression.Kind expressionKind = null;
switch (t.getType())
switch (t)
{
case IToken.tGT :
expressionKind =
@ -1476,8 +1476,17 @@ public class ExpressionParser implements IExpressionParser, IParserData {
// If this isn't a type name, then we shouldn't be here
try
{
typeId = typeId(scope, false, getCastExpressionKind(kind));
consume(IToken.tRPAREN);
try
{
typeId = typeId(scope, false, getCastExpressionKind(kind));
consume(IToken.tRPAREN);
}
catch( BacktrackException bte )
{
backup( mark );
throw bte;
}
mark = null; // clean up mark so that we can garbage collect
if( templateIdScopes != null ){ templateIdScopes.pop(); popped = true;}
IASTExpression castExpression = castExpression(scope,kind,key);
try
@ -1502,7 +1511,6 @@ public class ExpressionParser implements IExpressionParser, IParserData {
}
catch (BacktrackException b)
{
backup(mark);
if( templateIdScopes != null && !popped ){ templateIdScopes.pop(); }
}
}
@ -1526,7 +1534,6 @@ public class ExpressionParser implements IExpressionParser, IParserData {
*/
public IASTTypeId typeId(IASTScope scope, boolean skipArrayModifiers, CompletionKind completionKind) throws EndOfFileException, BacktrackException {
IToken mark = mark();
IToken start = mark;
ITokenDuple name = null;
boolean isConst = false, isVolatile = false;
boolean isSigned = false, isUnsigned = false;
@ -1703,8 +1710,8 @@ public class ExpressionParser implements IExpressionParser, IParserData {
try
{
String signature = "";//$NON-NLS-1$
if( start != null && lastToken != null )
signature = TokenDuple.createStringRepresentation(start, lastToken);
if( lastToken != null )
signature = TokenDuple.createStringRepresentation(mark, lastToken);
return astFactory.createTypeId( scope, kind, isConst, isVolatile, isShort, isLong, isSigned, isUnsigned, isTypename, name, id.getPointerOperators(), id.getArrayModifiers(), signature);
}
catch (ASTSemanticException e)
@ -2136,6 +2143,7 @@ public class ExpressionParser implements IExpressionParser, IParserData {
throw bt;
backup( current );
}
current = null;
consume( IToken.tLPAREN );
if( templateIdScopes != null ){ templateIdScopes.push( new Integer( IToken.tLPAREN ) ); }
IASTExpression expressionList = expression( scope, CompletionKind.TYPE_REFERENCE, key );
@ -2706,13 +2714,14 @@ public class ExpressionParser implements IExpressionParser, IParserData {
case IToken.tCOMPL:
ITokenDuple duple = null;
IToken mark = mark();
try
{
duple = name(scope, kind, key);
}
catch( BacktrackException bt )
{
IToken mark = mark();
Declarator d = new Declarator( new DeclarationWrapper(scope, mark.getOffset(), mark.getLineNumber(), null) );
if (LT(1) == IToken.tCOLONCOLON || LT(1) == IToken.tIDENTIFIER)

79
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java
View file

@ -142,14 +142,14 @@ public abstract class Parser extends ExpressionParser implements IParser
}
IToken lastBacktrack = null;
IToken checkToken = null;
while (true)
{
try
{
checkToken = LA(1);
int checkOffset = LA(1).getOffset();
declaration(compilationUnit, null, null);
if (LA(1) == checkToken)
if (LA(1).getOffset() == checkOffset)
errorHandling();
}
catch (EndOfFileException e)
@ -355,7 +355,7 @@ public abstract class Parser extends ExpressionParser implements IParser
linkage.enterScope( requestor );
linkageDeclarationLoop : while (LT(1) != IToken.tRBRACE)
{
IToken checkToken = LA(1);
int checkToken = LA(1).getOffset();
switch (LT(1))
{
case IToken.tRBRACE :
@ -369,11 +369,11 @@ public abstract class Parser extends ExpressionParser implements IParser
catch (BacktrackException bt)
{
failParse();
if (checkToken == LA(1))
if (checkToken == LA(1).getOffset())
errorHandling();
}
}
if (checkToken == LA(1))
if (checkToken == LA(1).getOffset())
errorHandling();
}
// consume the }
@ -891,7 +891,7 @@ public abstract class Parser extends ExpressionParser implements IParser
setCompletionValues(scope,CompletionKind.VARIABLE_TYPE, Key.DECLARATION );
namespaceDeclarationLoop : while (LT(1) != IToken.tRBRACE)
{
IToken checkToken = LA(1);
int checkToken = LA(1).getOffset();
switch (LT(1))
{
case IToken.tRBRACE :
@ -905,11 +905,11 @@ public abstract class Parser extends ExpressionParser implements IParser
catch (BacktrackException bt)
{
failParse();
if (checkToken == LA(1))
if (checkToken == LA(1).getOffset())
errorHandling();
}
}
if (checkToken == LA(1))
if (checkToken == LA(1).getOffset())
errorHandling();
}
setCompletionValues(scope, CompletionKind.NO_SUCH_KIND,Key.EMPTY );
@ -1001,6 +1001,7 @@ public abstract class Parser extends ExpressionParser implements IParser
sdw.isComplex(),
sdw.isImaginary(),
sdw.isGloballyQualified(), sdw.getExtensionParameters()));
sdw.setTypeName( null );
}
catch (Exception e1)
{
@ -1837,40 +1838,42 @@ public abstract class Parser extends ExpressionParser implements IParser
{
consume(IToken.tASSIGN);
setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY);
simpleDeclarationMark = null;
throwAwayMarksForInitializerClause(d);
IASTInitializerClause clause = initializerClause(scope,constructInitializers);
d.setInitializerClause(clause);
setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY);
}
else if (LT(1) == IToken.tLPAREN )
{
IToken mark = mark();
// initializer in constructor
try
{
consume(IToken.tLPAREN); // EAT IT!
setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY);
IASTExpression astExpression = null;
astExpression = expression(scope, CompletionKind.SINGLE_NAME_REFERENCE, Key.EXPRESSION);
setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY);
consume(IToken.tRPAREN);
d.setConstructorExpression(astExpression);
} catch( BacktrackException bt )
{
backup( mark );
throw bt;
}
consume(IToken.tLPAREN); // EAT IT!
setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY);
IASTExpression astExpression = null;
astExpression = expression(scope, CompletionKind.SINGLE_NAME_REFERENCE, Key.EXPRESSION);
setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY);
consume(IToken.tRPAREN);
d.setConstructorExpression(astExpression);
}
}
protected void optionalCInitializer( Declarator d, boolean constructInitializers ) throws EndOfFileException, BacktrackException
/**
* @param d
*/
protected void throwAwayMarksForInitializerClause(Declarator d) {
simpleDeclarationMark = null;
if( d.getNameDuple() != null )
d.getNameDuple().getLastToken().setNext( null );
}
protected void optionalCInitializer( Declarator d, boolean constructInitializers ) throws EndOfFileException, BacktrackException
{
final IASTScope scope = d.getDeclarationWrapper().getScope();
setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY);
if( LT(1) == IToken.tASSIGN )
{
consume( IToken.tASSIGN );
simpleDeclarationMark = null;
throwAwayMarksForInitializerClause(d);
setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY);
d.setInitializerClause( cInitializerClause(scope, Collections.EMPTY_LIST, constructInitializers ) );
setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY);
@ -1887,11 +1890,11 @@ public abstract class Parser extends ExpressionParser implements IParser
{
if (LT(1) == IToken.tLBRACE)
{
IToken mark = consume(IToken.tLBRACE);
consume(IToken.tLBRACE);
List initializerList = new ArrayList();
for (;;)
{
IToken checkToken = LA(1);
int checkOffset = LA(1).getOffset();
// required at least one initializer list
// get designator list
List newDesignators = designatorList(scope);
@ -1909,11 +1912,9 @@ public abstract class Parser extends ExpressionParser implements IParser
consume(IToken.tCOMMA);
if (LT(1) == IToken.tRBRACE)
break;
if( checkToken == LA(1))
{
backup( mark );
if( checkOffset == LA(1).getOffset())
throw backtrack;
}
// otherwise, its another initializer in the list
}
// consume the closing brace
@ -2648,7 +2649,7 @@ public abstract class Parser extends ExpressionParser implements IParser
handleClassSpecifier( astClassSpecifier );
memberDeclarationLoop : while (LT(1) != IToken.tRBRACE)
{
IToken checkToken = LA(1);
int checkToken = LA(1).getOffset();
switch (LT(1))
{
case IToken.t_public :
@ -2678,11 +2679,11 @@ public abstract class Parser extends ExpressionParser implements IParser
catch (BacktrackException bt)
{
failParse();
if (checkToken == LA(1))
if (checkToken == LA(1).getOffset())
errorHandling();
}
}
if (checkToken == LA(1))
if (checkToken == LA(1).getOffset())
errorHandling();
}
// consume the }
@ -3083,7 +3084,7 @@ public abstract class Parser extends ExpressionParser implements IParser
}
newScope.enterScope( requestor );
}
IToken checkToken = null;
setCompletionValues(
(createNewScope ? newScope : scope ),
CompletionKind.SINGLE_NAME_REFERENCE,
@ -3091,7 +3092,7 @@ public abstract class Parser extends ExpressionParser implements IParser
while (LT(1) != IToken.tRBRACE)
{
checkToken = LA(1);
int checkToken = LA(1).getOffset();
try
{
statement((IASTCodeScope) (createNewScope ? newScope : scope) );
@ -3099,7 +3100,7 @@ public abstract class Parser extends ExpressionParser implements IParser
catch( BacktrackException b )
{
failParse();
if( LA(1) == checkToken )
if( LA(1).getOffset() == checkToken )
errorHandling();
}
setCompletionValues(((createNewScope ? newScope : scope )), CompletionKind.SINGLE_NAME_REFERENCE,

3
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java
View file

@ -68,6 +68,7 @@ import org.eclipse.cdt.internal.core.parser.util.TraceUtil;
public class Scanner implements IScanner {
protected static final EndOfFileException EOF = new EndOfFileException();
static ScannerStringBuffer strbuff = new ScannerStringBuffer(100);
protected static final String HEX_PREFIX = "0x"; //$NON-NLS-1$
private static final ObjectMacroDescriptor CPLUSPLUS_MACRO = new ObjectMacroDescriptor( __CPLUSPLUS, "199711L"); //$NON-NLS-1$
@ -2302,7 +2303,7 @@ public class Scanner implements IScanner {
if( node == null )
{
if( offsetLimit == NO_OFFSET_LIMIT )
throw new EndOfFileException();
throw EOF;
if( finalToken != null && finalToken.getEndOffset() == offsetLimit )
throw new OffsetLimitReachedException(finalToken);

6
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenDuple.java
View file

@ -192,6 +192,7 @@ public class TokenDuple implements ITokenDuple {
private static final Integer LPAREN = new Integer( IToken.tLPAREN );
private String [] qualifiedName = null;
private static final String EMPTY_STRING = ""; //$NON-NLS-1$
private String stringRepresentation = null;
public IToken consumeTemplateIdArguments( IToken name, Iterator iter ){
IToken token = name;
@ -298,7 +299,9 @@ public class TokenDuple implements ITokenDuple {
public String toString()
{
return createStringRepresentation(firstToken, lastToken);
if( stringRepresentation == null )
stringRepresentation = createStringRepresentation(firstToken, lastToken);
return stringRepresentation;
}
public boolean isIdentifier()
@ -540,5 +543,6 @@ public class TokenDuple implements ITokenDuple {
qualifiedName = (String[]) qn.toArray( qualifiedName );
}
}