mirrors/cdt
1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-04-29 19:45:01 +02:00
Code Issues Wiki Activity

Cleaned up token storage in the Parser to reduce peak memory usage upon parsing large expressions/initializers.

Browse source
This commit is contained in:
John Camelon 2004-05-19 21:50:52 +00:00
parent d6606182b1
commit b87794dd73
5 changed files with 70 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/NullSourceElementRequestor.java
View file

@ -476,7 +476,6 @@ public class NullSourceElementRequestor implements ISourceElementRequestor
* @see org.eclipse.cdt.core.parser.ISourceElementRequestor#parserTimeout() * @see org.eclipse.cdt.core.parser.ISourceElementRequestor#parserTimeout()
*/ */
public boolean parserTimeout() { public boolean parserTimeout() {
// TODO Auto-generated method stub
return false; return false;
} }

35
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java
View file

@ -230,6 +230,7 @@ public class ExpressionParser implements IExpressionParser, IParserData {
* @throws BacktrackException request a backtrack * @throws BacktrackException request a backtrack
*/ */
protected IToken consumeTemplateParameters(IToken previousLast) throws EndOfFileException, BacktrackException { protected IToken consumeTemplateParameters(IToken previousLast) throws EndOfFileException, BacktrackException {
if( language != ParserLanguage.CPP ) return previousLast;
IToken last = previousLast; IToken last = previousLast;
if (LT(1) == IToken.tLT) if (LT(1) == IToken.tLT)
{ {
@ -524,6 +525,7 @@ public class ExpressionParser implements IExpressionParser, IParserData {
* @throws BacktrackException * @throws BacktrackException
*/ */
protected IToken consumeTemplateArguments(IASTScope scope, IToken last, TemplateParameterManager argumentList) throws EndOfFileException, BacktrackException { protected IToken consumeTemplateArguments(IASTScope scope, IToken last, TemplateParameterManager argumentList) throws EndOfFileException, BacktrackException {
if( language != ParserLanguage.CPP ) return last;
if( LT(1) == IToken.tLT ){ if( LT(1) == IToken.tLT ){
IToken secondMark = mark(); IToken secondMark = mark();
consume( IToken.tLT ); consume( IToken.tLT );
@ -1206,11 +1208,9 @@ public class ExpressionParser implements IExpressionParser, IParserData {
case IToken.tLTEQUAL : case IToken.tLTEQUAL :
case IToken.tGTEQUAL : case IToken.tGTEQUAL :
IToken mark = mark(); IToken mark = mark();
IToken t = consume(); int t = consume().getType();
IToken next = LA(1); IASTExpression secondExpression = shiftExpression(scope,kind, key);
IASTExpression secondExpression = if (LA(1) == mark.getNext() )
shiftExpression(scope,kind, key);
if (next == LA(1))
{ {
// we did not consume anything // we did not consume anything
// this is most likely an error // this is most likely an error
@ -1218,7 +1218,7 @@ public class ExpressionParser implements IExpressionParser, IParserData {
return firstExpression; return firstExpression;
} }
IASTExpression.Kind expressionKind = null; IASTExpression.Kind expressionKind = null;
switch (t.getType()) switch (t)
{ {
case IToken.tGT : case IToken.tGT :
expressionKind = expressionKind =
@ -1476,8 +1476,17 @@ public class ExpressionParser implements IExpressionParser, IParserData {
// If this isn't a type name, then we shouldn't be here // If this isn't a type name, then we shouldn't be here
try try
{ {
typeId = typeId(scope, false, getCastExpressionKind(kind)); try
consume(IToken.tRPAREN); {
typeId = typeId(scope, false, getCastExpressionKind(kind));
consume(IToken.tRPAREN);
}
catch( BacktrackException bte )
{
backup( mark );
throw bte;
}
mark = null; // clean up mark so that we can garbage collect
if( templateIdScopes != null ){ templateIdScopes.pop(); popped = true;} if( templateIdScopes != null ){ templateIdScopes.pop(); popped = true;}
IASTExpression castExpression = castExpression(scope,kind,key); IASTExpression castExpression = castExpression(scope,kind,key);
try try
@ -1502,7 +1511,6 @@ public class ExpressionParser implements IExpressionParser, IParserData {
} }
catch (BacktrackException b) catch (BacktrackException b)
{ {
backup(mark);
if( templateIdScopes != null && !popped ){ templateIdScopes.pop(); } if( templateIdScopes != null && !popped ){ templateIdScopes.pop(); }
} }
} }
@ -1526,7 +1534,6 @@ public class ExpressionParser implements IExpressionParser, IParserData {
*/ */
public IASTTypeId typeId(IASTScope scope, boolean skipArrayModifiers, CompletionKind completionKind) throws EndOfFileException, BacktrackException { public IASTTypeId typeId(IASTScope scope, boolean skipArrayModifiers, CompletionKind completionKind) throws EndOfFileException, BacktrackException {
IToken mark = mark(); IToken mark = mark();
IToken start = mark;
ITokenDuple name = null; ITokenDuple name = null;
boolean isConst = false, isVolatile = false; boolean isConst = false, isVolatile = false;
boolean isSigned = false, isUnsigned = false; boolean isSigned = false, isUnsigned = false;
@ -1703,8 +1710,8 @@ public class ExpressionParser implements IExpressionParser, IParserData {
try try
{ {
String signature = "";//$NON-NLS-1$ String signature = "";//$NON-NLS-1$
if( start != null && lastToken != null ) if( lastToken != null )
signature = TokenDuple.createStringRepresentation(start, lastToken); signature = TokenDuple.createStringRepresentation(mark, lastToken);
return astFactory.createTypeId( scope, kind, isConst, isVolatile, isShort, isLong, isSigned, isUnsigned, isTypename, name, id.getPointerOperators(), id.getArrayModifiers(), signature); return astFactory.createTypeId( scope, kind, isConst, isVolatile, isShort, isLong, isSigned, isUnsigned, isTypename, name, id.getPointerOperators(), id.getArrayModifiers(), signature);
} }
catch (ASTSemanticException e) catch (ASTSemanticException e)
@ -2136,6 +2143,7 @@ public class ExpressionParser implements IExpressionParser, IParserData {
throw bt; throw bt;
backup( current ); backup( current );
} }
current = null;
consume( IToken.tLPAREN ); consume( IToken.tLPAREN );
if( templateIdScopes != null ){ templateIdScopes.push( new Integer( IToken.tLPAREN ) ); } if( templateIdScopes != null ){ templateIdScopes.push( new Integer( IToken.tLPAREN ) ); }
IASTExpression expressionList = expression( scope, CompletionKind.TYPE_REFERENCE, key ); IASTExpression expressionList = expression( scope, CompletionKind.TYPE_REFERENCE, key );
@ -2706,13 +2714,14 @@ public class ExpressionParser implements IExpressionParser, IParserData {
case IToken.tCOMPL: case IToken.tCOMPL:
ITokenDuple duple = null; ITokenDuple duple = null;
IToken mark = mark();
try try
{ {
duple = name(scope, kind, key); duple = name(scope, kind, key);
} }
catch( BacktrackException bt ) catch( BacktrackException bt )
{ {
IToken mark = mark();
Declarator d = new Declarator( new DeclarationWrapper(scope, mark.getOffset(), mark.getLineNumber(), null) ); Declarator d = new Declarator( new DeclarationWrapper(scope, mark.getOffset(), mark.getLineNumber(), null) );
if (LT(1) == IToken.tCOLONCOLON || LT(1) == IToken.tIDENTIFIER) if (LT(1) == IToken.tCOLONCOLON || LT(1) == IToken.tIDENTIFIER)

79
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java
View file

@ -142,14 +142,14 @@ public abstract class Parser extends ExpressionParser implements IParser
} }
IToken lastBacktrack = null; IToken lastBacktrack = null;
IToken checkToken = null;
while (true) while (true)
{ {
try try
{ {
checkToken = LA(1); int checkOffset = LA(1).getOffset();
declaration(compilationUnit, null, null); declaration(compilationUnit, null, null);
if (LA(1) == checkToken) if (LA(1).getOffset() == checkOffset)
errorHandling(); errorHandling();
} }
catch (EndOfFileException e) catch (EndOfFileException e)
@ -355,7 +355,7 @@ public abstract class Parser extends ExpressionParser implements IParser
linkage.enterScope( requestor ); linkage.enterScope( requestor );
linkageDeclarationLoop : while (LT(1) != IToken.tRBRACE) linkageDeclarationLoop : while (LT(1) != IToken.tRBRACE)
{ {
IToken checkToken = LA(1); int checkToken = LA(1).getOffset();
switch (LT(1)) switch (LT(1))
{ {
case IToken.tRBRACE : case IToken.tRBRACE :
@ -369,11 +369,11 @@ public abstract class Parser extends ExpressionParser implements IParser
catch (BacktrackException bt) catch (BacktrackException bt)
{ {
failParse(); failParse();
if (checkToken == LA(1)) if (checkToken == LA(1).getOffset())
errorHandling(); errorHandling();
} }
} }
if (checkToken == LA(1)) if (checkToken == LA(1).getOffset())
errorHandling(); errorHandling();
} }
// consume the } // consume the }
@ -891,7 +891,7 @@ public abstract class Parser extends ExpressionParser implements IParser
setCompletionValues(scope,CompletionKind.VARIABLE_TYPE, Key.DECLARATION ); setCompletionValues(scope,CompletionKind.VARIABLE_TYPE, Key.DECLARATION );
namespaceDeclarationLoop : while (LT(1) != IToken.tRBRACE) namespaceDeclarationLoop : while (LT(1) != IToken.tRBRACE)
{ {
IToken checkToken = LA(1); int checkToken = LA(1).getOffset();
switch (LT(1)) switch (LT(1))
{ {
case IToken.tRBRACE : case IToken.tRBRACE :
@ -905,11 +905,11 @@ public abstract class Parser extends ExpressionParser implements IParser
catch (BacktrackException bt) catch (BacktrackException bt)
{ {
failParse(); failParse();
if (checkToken == LA(1)) if (checkToken == LA(1).getOffset())
errorHandling(); errorHandling();
} }
} }
if (checkToken == LA(1)) if (checkToken == LA(1).getOffset())
errorHandling(); errorHandling();
} }
setCompletionValues(scope, CompletionKind.NO_SUCH_KIND,Key.EMPTY ); setCompletionValues(scope, CompletionKind.NO_SUCH_KIND,Key.EMPTY );
@ -1001,6 +1001,7 @@ public abstract class Parser extends ExpressionParser implements IParser
sdw.isComplex(), sdw.isComplex(),
sdw.isImaginary(), sdw.isImaginary(),
sdw.isGloballyQualified(), sdw.getExtensionParameters())); sdw.isGloballyQualified(), sdw.getExtensionParameters()));
sdw.setTypeName( null );
} }
catch (Exception e1) catch (Exception e1)
{ {
@ -1837,40 +1838,42 @@ public abstract class Parser extends ExpressionParser implements IParser
{ {
consume(IToken.tASSIGN); consume(IToken.tASSIGN);
setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY); setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY);
simpleDeclarationMark = null; throwAwayMarksForInitializerClause(d);
IASTInitializerClause clause = initializerClause(scope,constructInitializers); IASTInitializerClause clause = initializerClause(scope,constructInitializers);
d.setInitializerClause(clause); d.setInitializerClause(clause);
setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY); setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY);
} }
else if (LT(1) == IToken.tLPAREN ) else if (LT(1) == IToken.tLPAREN )
{ {
IToken mark = mark();
// initializer in constructor // initializer in constructor
try consume(IToken.tLPAREN); // EAT IT!
{ setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY);
consume(IToken.tLPAREN); // EAT IT! IASTExpression astExpression = null;
setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY); astExpression = expression(scope, CompletionKind.SINGLE_NAME_REFERENCE, Key.EXPRESSION);
IASTExpression astExpression = null; setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY);
astExpression = expression(scope, CompletionKind.SINGLE_NAME_REFERENCE, Key.EXPRESSION); consume(IToken.tRPAREN);
setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY); d.setConstructorExpression(astExpression);
consume(IToken.tRPAREN);
d.setConstructorExpression(astExpression);
} catch( BacktrackException bt )
{
backup( mark );
throw bt;
}
} }
} }
protected void optionalCInitializer( Declarator d, boolean constructInitializers ) throws EndOfFileException, BacktrackException /**
* @param d
*/
protected void throwAwayMarksForInitializerClause(Declarator d) {
simpleDeclarationMark = null;
if( d.getNameDuple() != null )
d.getNameDuple().getLastToken().setNext( null );
}
protected void optionalCInitializer( Declarator d, boolean constructInitializers ) throws EndOfFileException, BacktrackException
{ {
final IASTScope scope = d.getDeclarationWrapper().getScope(); final IASTScope scope = d.getDeclarationWrapper().getScope();
setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY); setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY);
if( LT(1) == IToken.tASSIGN ) if( LT(1) == IToken.tASSIGN )
{ {
consume( IToken.tASSIGN ); consume( IToken.tASSIGN );
simpleDeclarationMark = null; throwAwayMarksForInitializerClause(d);
setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY); setCompletionValues(scope,CompletionKind.SINGLE_NAME_REFERENCE,Key.EMPTY);
d.setInitializerClause( cInitializerClause(scope, Collections.EMPTY_LIST, constructInitializers ) ); d.setInitializerClause( cInitializerClause(scope, Collections.EMPTY_LIST, constructInitializers ) );
setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY); setCompletionValues(scope,CompletionKind.NO_SUCH_KIND,Key.EMPTY);
@ -1887,11 +1890,11 @@ public abstract class Parser extends ExpressionParser implements IParser
{ {
if (LT(1) == IToken.tLBRACE) if (LT(1) == IToken.tLBRACE)
{ {
IToken mark = consume(IToken.tLBRACE); consume(IToken.tLBRACE);
List initializerList = new ArrayList(); List initializerList = new ArrayList();
for (;;) for (;;)
{ {
IToken checkToken = LA(1); int checkOffset = LA(1).getOffset();
// required at least one initializer list // required at least one initializer list
// get designator list // get designator list
List newDesignators = designatorList(scope); List newDesignators = designatorList(scope);
@ -1909,11 +1912,9 @@ public abstract class Parser extends ExpressionParser implements IParser
consume(IToken.tCOMMA); consume(IToken.tCOMMA);
if (LT(1) == IToken.tRBRACE) if (LT(1) == IToken.tRBRACE)
break; break;
if( checkToken == LA(1)) if( checkOffset == LA(1).getOffset())
{
backup( mark );
throw backtrack; throw backtrack;
}
// otherwise, its another initializer in the list // otherwise, its another initializer in the list
} }
// consume the closing brace // consume the closing brace
@ -2648,7 +2649,7 @@ public abstract class Parser extends ExpressionParser implements IParser
handleClassSpecifier( astClassSpecifier ); handleClassSpecifier( astClassSpecifier );
memberDeclarationLoop : while (LT(1) != IToken.tRBRACE) memberDeclarationLoop : while (LT(1) != IToken.tRBRACE)
{ {
IToken checkToken = LA(1); int checkToken = LA(1).getOffset();
switch (LT(1)) switch (LT(1))
{ {
case IToken.t_public : case IToken.t_public :
@ -2678,11 +2679,11 @@ public abstract class Parser extends ExpressionParser implements IParser
catch (BacktrackException bt) catch (BacktrackException bt)
{ {
failParse(); failParse();
if (checkToken == LA(1)) if (checkToken == LA(1).getOffset())
errorHandling(); errorHandling();
} }
} }
if (checkToken == LA(1)) if (checkToken == LA(1).getOffset())
errorHandling(); errorHandling();
} }
// consume the } // consume the }
@ -3083,7 +3084,7 @@ public abstract class Parser extends ExpressionParser implements IParser
} }
newScope.enterScope( requestor ); newScope.enterScope( requestor );
} }
IToken checkToken = null;
setCompletionValues( setCompletionValues(
(createNewScope ? newScope : scope ), (createNewScope ? newScope : scope ),
CompletionKind.SINGLE_NAME_REFERENCE, CompletionKind.SINGLE_NAME_REFERENCE,
@ -3091,7 +3092,7 @@ public abstract class Parser extends ExpressionParser implements IParser
while (LT(1) != IToken.tRBRACE) while (LT(1) != IToken.tRBRACE)
{ {
checkToken = LA(1); int checkToken = LA(1).getOffset();
try try
{ {
statement((IASTCodeScope) (createNewScope ? newScope : scope) ); statement((IASTCodeScope) (createNewScope ? newScope : scope) );
@ -3099,7 +3100,7 @@ public abstract class Parser extends ExpressionParser implements IParser
catch( BacktrackException b ) catch( BacktrackException b )
{ {
failParse(); failParse();
if( LA(1) == checkToken ) if( LA(1).getOffset() == checkToken )
errorHandling(); errorHandling();
} }
setCompletionValues(((createNewScope ? newScope : scope )), CompletionKind.SINGLE_NAME_REFERENCE, setCompletionValues(((createNewScope ? newScope : scope )), CompletionKind.SINGLE_NAME_REFERENCE,

3
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java
View file

@ -68,6 +68,7 @@ import org.eclipse.cdt.internal.core.parser.util.TraceUtil;
public class Scanner implements IScanner { public class Scanner implements IScanner {
protected static final EndOfFileException EOF = new EndOfFileException();
static ScannerStringBuffer strbuff = new ScannerStringBuffer(100); static ScannerStringBuffer strbuff = new ScannerStringBuffer(100);
protected static final String HEX_PREFIX = "0x"; //$NON-NLS-1$ protected static final String HEX_PREFIX = "0x"; //$NON-NLS-1$
private static final ObjectMacroDescriptor CPLUSPLUS_MACRO = new ObjectMacroDescriptor( __CPLUSPLUS, "199711L"); //$NON-NLS-1$ private static final ObjectMacroDescriptor CPLUSPLUS_MACRO = new ObjectMacroDescriptor( __CPLUSPLUS, "199711L"); //$NON-NLS-1$
@ -2302,7 +2303,7 @@ public class Scanner implements IScanner {
if( node == null ) if( node == null )
{ {
if( offsetLimit == NO_OFFSET_LIMIT ) if( offsetLimit == NO_OFFSET_LIMIT )
throw new EndOfFileException(); throw EOF;
if( finalToken != null && finalToken.getEndOffset() == offsetLimit ) if( finalToken != null && finalToken.getEndOffset() == offsetLimit )
throw new OffsetLimitReachedException(finalToken); throw new OffsetLimitReachedException(finalToken);

6
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenDuple.java
View file

@ -192,6 +192,7 @@ public class TokenDuple implements ITokenDuple {
private static final Integer LPAREN = new Integer( IToken.tLPAREN ); private static final Integer LPAREN = new Integer( IToken.tLPAREN );
private String [] qualifiedName = null; private String [] qualifiedName = null;
private static final String EMPTY_STRING = ""; //$NON-NLS-1$ private static final String EMPTY_STRING = ""; //$NON-NLS-1$
private String stringRepresentation = null;
public IToken consumeTemplateIdArguments( IToken name, Iterator iter ){ public IToken consumeTemplateIdArguments( IToken name, Iterator iter ){
IToken token = name; IToken token = name;
@ -298,7 +299,9 @@ public class TokenDuple implements ITokenDuple {
public String toString() public String toString()
{ {
return createStringRepresentation(firstToken, lastToken); if( stringRepresentation == null )
stringRepresentation = createStringRepresentation(firstToken, lastToken);
return stringRepresentation;
} }
public boolean isIdentifier() public boolean isIdentifier()
@ -541,4 +544,5 @@ public class TokenDuple implements ITokenDuple {
} }
} }