From b87794dd732c3769d9613b2180ea61aaa0897974 Mon Sep 17 00:00:00 2001 From: John Camelon Date: Wed, 19 May 2004 21:50:52 +0000 Subject: [PATCH] Cleaned up token storage in the Parser to reduce peak memory usage upon parsing large expressions/initializers. --- .../parser/NullSourceElementRequestor.java | 3 +- .../core/parser/ExpressionParser.java | 35 +++++--- .../cdt/internal/core/parser/Parser.java | 79 ++++++++++--------- .../internal/core/parser/scanner/Scanner.java | 3 +- .../core/parser/token/TokenDuple.java | 6 +- 5 files changed, 70 insertions(+), 56 deletions(-) diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/NullSourceElementRequestor.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/NullSourceElementRequestor.java index 1594727460d..a9b1b909ebd 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/NullSourceElementRequestor.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/NullSourceElementRequestor.java @@ -475,8 +475,7 @@ public class NullSourceElementRequestor implements ISourceElementRequestor /* (non-Javadoc) * @see org.eclipse.cdt.core.parser.ISourceElementRequestor#parserTimeout() */ - public boolean parserTimeout() { - // TODO Auto-generated method stub + public boolean parserTimeout() { return false; } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java index e77e524e20f..e056f89b709 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java @@ -230,6 +230,7 @@ public class ExpressionParser implements IExpressionParser, IParserData { * @throws BacktrackException request a backtrack */ protected IToken consumeTemplateParameters(IToken previousLast) throws EndOfFileException, BacktrackException { + if( language != ParserLanguage.CPP ) return previousLast; IToken last = previousLast; if (LT(1) == IToken.tLT) { @@ -524,6 +525,7 @@ public class ExpressionParser implements IExpressionParser, IParserData { * @throws BacktrackException */ protected IToken consumeTemplateArguments(IASTScope scope, IToken last, TemplateParameterManager argumentList) throws EndOfFileException, BacktrackException { + if( language != ParserLanguage.CPP ) return last; if( LT(1) == IToken.tLT ){ IToken secondMark = mark(); consume( IToken.tLT ); @@ -1206,11 +1208,9 @@ public class ExpressionParser implements IExpressionParser, IParserData { case IToken.tLTEQUAL : case IToken.tGTEQUAL : IToken mark = mark(); - IToken t = consume(); - IToken next = LA(1); - IASTExpression secondExpression = - shiftExpression(scope,kind, key); - if (next == LA(1)) + int t = consume().getType(); + IASTExpression secondExpression = shiftExpression(scope,kind, key); + if (LA(1) == mark.getNext() ) { // we did not consume anything // this is most likely an error @@ -1218,7 +1218,7 @@ public class ExpressionParser implements IExpressionParser, IParserData { return firstExpression; } IASTExpression.Kind expressionKind = null; - switch (t.getType()) + switch (t) { case IToken.tGT : expressionKind = @@ -1476,8 +1476,17 @@ public class ExpressionParser implements IExpressionParser, IParserData { // If this isn't a type name, then we shouldn't be here try { - typeId = typeId(scope, false, getCastExpressionKind(kind)); - consume(IToken.tRPAREN); + try + { + typeId = typeId(scope, false, getCastExpressionKind(kind)); + consume(IToken.tRPAREN); + } + catch( BacktrackException bte ) + { + backup( mark ); + throw bte; + } + mark = null; // clean up mark so that we can garbage collect if( templateIdScopes != null ){ templateIdScopes.pop(); popped = true;} IASTExpression castExpression = castExpression(scope,kind,key); try @@ -1502,7 +1511,6 @@ public class ExpressionParser implements IExpressionParser, IParserData { } catch (BacktrackException b) { - backup(mark); if( templateIdScopes != null && !popped ){ templateIdScopes.pop(); } } } @@ -1526,7 +1534,6 @@ public class ExpressionParser implements IExpressionParser, IParserData { */ public IASTTypeId typeId(IASTScope scope, boolean skipArrayModifiers, CompletionKind completionKind) throws EndOfFileException, BacktrackException { IToken mark = mark(); - IToken start = mark; ITokenDuple name = null; boolean isConst = false, isVolatile = false; boolean isSigned = false, isUnsigned = false; @@ -1703,8 +1710,8 @@ public class ExpressionParser implements IExpressionParser, IParserData { try { String signature = "";//$NON-NLS-1$ - if( start != null && lastToken != null ) - signature = TokenDuple.createStringRepresentation(start, lastToken); + if( lastToken != null ) + signature = TokenDuple.createStringRepresentation(mark, lastToken); return astFactory.createTypeId( scope, kind, isConst, isVolatile, isShort, isLong, isSigned, isUnsigned, isTypename, name, id.getPointerOperators(), id.getArrayModifiers(), signature); } catch (ASTSemanticException e) @@ -2136,6 +2143,7 @@ public class ExpressionParser implements IExpressionParser, IParserData { throw bt; backup( current ); } + current = null; consume( IToken.tLPAREN ); if( templateIdScopes != null ){ templateIdScopes.push( new Integer( IToken.tLPAREN ) ); } IASTExpression expressionList = expression( scope, CompletionKind.TYPE_REFERENCE, key ); @@ -2706,13 +2714,14 @@ public class ExpressionParser implements IExpressionParser, IParserData { case IToken.tCOMPL: ITokenDuple duple = null; - IToken mark = mark(); + try { duple = name(scope, kind, key); } catch( BacktrackException bt ) { + IToken mark = mark(); Declarator d = new Declarator( new DeclarationWrapper(scope, mark.getOffset(), mark.getLineNumber(), null) ); if (LT(1) == IToken.tCOLONCOLON || LT(1) == IToken.tIDENTIFIER) diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java index 1e5e1c43e42..5747339c467 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java @@ -142,14 +142,14 @@ public abstract class Parser extends ExpressionParser implements IParser } IToken lastBacktrack = null; - IToken checkToken = null; + while (true) { try { - checkToken = LA(1); + int checkOffset = LA(1).getOffset(); declaration(compilationUnit, null, null); - if (LA(1) == checkToken) + if (LA(1).getOffset() == checkOffset) errorHandling(); } catch (EndOfFileException e) @@ -355,7 +355,7 @@ public abstract class Parser extends ExpressionParser implements IParser linkage.enterScope( requestor ); linkageDeclarationLoop : while (LT(1) != IToken.tRBRACE) { - IToken checkToken = LA(1); + int checkToken = LA(1).getOffset(); switch (LT(1)) { case IToken.tRBRACE : @@ -369,11 +369,11 @@ public abstract class Parser extends ExpressionParser implements IParser catch (BacktrackException bt) { failParse(); - if (checkToken == LA(1)) + if (checkToken == LA(1).getOffset()) errorHandling(); } } - if (checkToken == LA(1)) + if (checkToken == LA(1).getOffset()) errorHandling(); } // consume the } @@ -891,7 +891,7 @@ public abstract class Parser extends ExpressionParser implements IParser setCompletionValues(scope,CompletionKind.VARIABLE_TYPE, Key.DECLARATION ); namespaceDeclarationLoop : while (LT(1) != IToken.tRBRACE) { - IToken checkToken = LA(1); + int checkToken = LA(1).getOffset(); switch (LT(1)) { case IToken.tRBRACE : @@ -905,11 +905,11 @@ public abstract class Parser extends ExpressionParser implements IParser catch (BacktrackException bt) { failParse(); - if (checkToken == LA(1)) + if (checkToken == LA(1).getOffset()) errorHandling(); } } - if (checkToken == LA(1)) + if (checkToken == LA(1).getOffset()) errorHandling(); } setCompletionValues(scope, CompletionKind.NO_SUCH_KIND,Key.EMPTY ); @@ -1001,6 +1001,7 @@ public abstract class Parser extends ExpressionParser implements IParser sdw.isComplex(), sdw.isImaginary(), sdw.isGloballyQualified(), sdw.getExtensionParameters())); + sdw.setTypeName( null ); } catch (Exception e1) { @@ -1837,40 +1838,42 @@ public abstract class Parser extends ExpressionParser implements IParser { consume(IToken.tASSIGN); setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY); - simpleDeclarationMark = null; + throwAwayMarksForInitializerClause(d); IASTInitializerClause clause = initializerClause(scope,constructInitializers); d.setInitializerClause(clause); setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY); } else if (LT(1) == IToken.tLPAREN ) { - IToken mark = mark(); // initializer in constructor - try - { - consume(IToken.tLPAREN); // EAT IT! - setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY); - IASTExpression astExpression = null; - astExpression = expression(scope, CompletionKind.SINGLE_NAME_REFERENCE, Key.EXPRESSION); - setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY); - consume(IToken.tRPAREN); - d.setConstructorExpression(astExpression); - } catch( BacktrackException bt ) - { - backup( mark ); - throw bt; - } + consume(IToken.tLPAREN); // EAT IT! + setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY); + IASTExpression astExpression = null; + astExpression = expression(scope, CompletionKind.SINGLE_NAME_REFERENCE, Key.EXPRESSION); + setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY); + consume(IToken.tRPAREN); + d.setConstructorExpression(astExpression); } } - protected void optionalCInitializer( Declarator d, boolean constructInitializers ) throws EndOfFileException, BacktrackException + /** + * @param d + */ + protected void throwAwayMarksForInitializerClause(Declarator d) { + simpleDeclarationMark = null; + if( d.getNameDuple() != null ) + d.getNameDuple().getLastToken().setNext( null ); + } + + + protected void optionalCInitializer( Declarator d, boolean constructInitializers ) throws EndOfFileException, BacktrackException { final IASTScope scope = d.getDeclarationWrapper().getScope(); setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY); if( LT(1) == IToken.tASSIGN ) { consume( IToken.tASSIGN ); - simpleDeclarationMark = null; + throwAwayMarksForInitializerClause(d); setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY); d.setInitializerClause( cInitializerClause(scope, Collections.EMPTY_LIST, constructInitializers ) ); setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY); @@ -1887,11 +1890,11 @@ public abstract class Parser extends ExpressionParser implements IParser { if (LT(1) == IToken.tLBRACE) { - IToken mark = consume(IToken.tLBRACE); + consume(IToken.tLBRACE); List initializerList = new ArrayList(); for (;;) { - IToken checkToken = LA(1); + int checkOffset = LA(1).getOffset(); // required at least one initializer list // get designator list List newDesignators = designatorList(scope); @@ -1909,11 +1912,9 @@ public abstract class Parser extends ExpressionParser implements IParser consume(IToken.tCOMMA); if (LT(1) == IToken.tRBRACE) break; - if( checkToken == LA(1)) - { - backup( mark ); + if( checkOffset == LA(1).getOffset()) throw backtrack; - } + // otherwise, its another initializer in the list } // consume the closing brace @@ -2648,7 +2649,7 @@ public abstract class Parser extends ExpressionParser implements IParser handleClassSpecifier( astClassSpecifier ); memberDeclarationLoop : while (LT(1) != IToken.tRBRACE) { - IToken checkToken = LA(1); + int checkToken = LA(1).getOffset(); switch (LT(1)) { case IToken.t_public : @@ -2678,11 +2679,11 @@ public abstract class Parser extends ExpressionParser implements IParser catch (BacktrackException bt) { failParse(); - if (checkToken == LA(1)) + if (checkToken == LA(1).getOffset()) errorHandling(); } } - if (checkToken == LA(1)) + if (checkToken == LA(1).getOffset()) errorHandling(); } // consume the } @@ -3083,7 +3084,7 @@ public abstract class Parser extends ExpressionParser implements IParser } newScope.enterScope( requestor ); } - IToken checkToken = null; + setCompletionValues( (createNewScope ? newScope : scope ), CompletionKind.SINGLE_NAME_REFERENCE, @@ -3091,7 +3092,7 @@ public abstract class Parser extends ExpressionParser implements IParser while (LT(1) != IToken.tRBRACE) { - checkToken = LA(1); + int checkToken = LA(1).getOffset(); try { statement((IASTCodeScope) (createNewScope ? newScope : scope) ); @@ -3099,7 +3100,7 @@ public abstract class Parser extends ExpressionParser implements IParser catch( BacktrackException b ) { failParse(); - if( LA(1) == checkToken ) + if( LA(1).getOffset() == checkToken ) errorHandling(); } setCompletionValues(((createNewScope ? newScope : scope )), CompletionKind.SINGLE_NAME_REFERENCE, diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java index a8997ddb550..37ad4e0745a 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java @@ -68,6 +68,7 @@ import org.eclipse.cdt.internal.core.parser.util.TraceUtil; public class Scanner implements IScanner { + protected static final EndOfFileException EOF = new EndOfFileException(); static ScannerStringBuffer strbuff = new ScannerStringBuffer(100); protected static final String HEX_PREFIX = "0x"; //$NON-NLS-1$ private static final ObjectMacroDescriptor CPLUSPLUS_MACRO = new ObjectMacroDescriptor( __CPLUSPLUS, "199711L"); //$NON-NLS-1$ @@ -2302,7 +2303,7 @@ public class Scanner implements IScanner { if( node == null ) { if( offsetLimit == NO_OFFSET_LIMIT ) - throw new EndOfFileException(); + throw EOF; if( finalToken != null && finalToken.getEndOffset() == offsetLimit ) throw new OffsetLimitReachedException(finalToken); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenDuple.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenDuple.java index 05e1a47afd1..1cd05446542 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenDuple.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenDuple.java @@ -192,6 +192,7 @@ public class TokenDuple implements ITokenDuple { private static final Integer LPAREN = new Integer( IToken.tLPAREN ); private String [] qualifiedName = null; private static final String EMPTY_STRING = ""; //$NON-NLS-1$ + private String stringRepresentation = null; public IToken consumeTemplateIdArguments( IToken name, Iterator iter ){ IToken token = name; @@ -298,7 +299,9 @@ public class TokenDuple implements ITokenDuple { public String toString() { - return createStringRepresentation(firstToken, lastToken); + if( stringRepresentation == null ) + stringRepresentation = createStringRepresentation(firstToken, lastToken); + return stringRepresentation; } public boolean isIdentifier() @@ -540,5 +543,6 @@ public class TokenDuple implements ITokenDuple { qualifiedName = (String[]) qn.toArray( qualifiedName ); } + }