From c24ed6aa2aa3e663b177a3f10149d82a3756ceeb Mon Sep 17 00:00:00 2001 From: John Camelon Date: Tue, 27 Jul 2004 19:18:19 +0000 Subject: [PATCH] Further Scanner2 work regarding macro expansions. --- .../core/parser/scanner2/Scanner2.java | 140 ++++++++++-------- 1 file changed, 82 insertions(+), 58 deletions(-) diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java index f3a68b29f10..053efd4bbda 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java @@ -44,6 +44,7 @@ import org.eclipse.cdt.internal.core.parser.scanner.IScannerData; import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility; import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionDirective; import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionParseException; +import org.eclipse.cdt.internal.core.parser.token.ImagedExpansionToken; import org.eclipse.cdt.internal.core.parser.token.ImagedToken; import org.eclipse.cdt.internal.core.parser.token.SimpleToken; @@ -323,7 +324,7 @@ public class Scanner2 implements IScanner, IScannerData { int tokenType = IToken.tSTRING; if( lastToken.getType() == IToken.tLSTRING || nextToken.getType() == IToken.tLSTRING ) tokenType = IToken.tLSTRING; - lastToken = new ImagedToken(tokenType, CharArrayUtils.concat( lastToken.getCharImage(), nextToken.getCharImage() ), nextToken.getEndOffset(), getCurrentFilename() ); + lastToken = newToken(tokenType, CharArrayUtils.concat( lastToken.getCharImage(), nextToken.getCharImage() ) ); if (oldToken != null) oldToken.setNext(lastToken); nextToken = fetchToken(); @@ -473,20 +474,20 @@ public class Scanner2 implements IScanner, IScannerData { if (pos + 2 < limit) { if (buffer[pos + 2] == '.') { bufferPos[bufferStackPos] += 2; - return new SimpleToken(IToken.tELLIPSIS, bufferPos[bufferStackPos] + 1 , getCurrentFilename() ); + return newToken(IToken.tELLIPSIS ); } } case '*': ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tDOTSTAR, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tDOTSTAR ); } } - return new SimpleToken(IToken.tDOT, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tDOT); case '#': if (pos + 1 < limit && buffer[pos + 1] == '#') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tPOUNDPOUND, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tPOUNDPOUND); } // Should really check to make sure this is the first @@ -495,49 +496,49 @@ public class Scanner2 implements IScanner, IScannerData { continue; case '{': - return new SimpleToken(IToken.tLBRACE, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tLBRACE ); case '}': - return new SimpleToken(IToken.tRBRACE, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tRBRACE ); case '[': - return new SimpleToken(IToken.tLBRACKET, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tLBRACKET ); case ']': - return new SimpleToken(IToken.tRBRACKET, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tRBRACKET ); case '(': - return new SimpleToken(IToken.tLPAREN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tLPAREN ); case ')': - return new SimpleToken(IToken.tRPAREN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tRPAREN ); case ';': - return new SimpleToken(IToken.tSEMI, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tSEMI ); case ':': if (pos + 1 < limit) { if (buffer[pos + 1] == ':') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tCOLONCOLON, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tCOLONCOLON ); } } - return new SimpleToken(IToken.tCOLON, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tCOLON ); case '?': - return new SimpleToken(IToken.tQUESTION, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tQUESTION ); case '+': if (pos + 1 < limit) { if (buffer[pos + 1] == '+') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tINCR, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tINCR); } else if (buffer[pos + 1] == '=') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tPLUSASSIGN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tPLUSASSIGN ); } } - return new SimpleToken(IToken.tPLUS, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tPLUS ); case '-': if (pos + 1 < limit) { @@ -545,140 +546,140 @@ public class Scanner2 implements IScanner, IScannerData { if (pos + 2 < limit) { if (buffer[pos + 2] == '*') { bufferPos[bufferStackPos] += 2; - return new SimpleToken(IToken.tARROWSTAR, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tARROWSTAR ); } } ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tARROW, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tARROW); } else if (buffer[pos + 1] == '-') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tDECR, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tDECR ); } else if (buffer[pos + 1] == '=') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tMINUSASSIGN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tMINUSASSIGN ); } } - return new SimpleToken(IToken.tMINUS, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tMINUS ); case '*': if (pos + 1 < limit) { if (buffer[pos + 1] == '=') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tSTARASSIGN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tSTARASSIGN ); } } - return new SimpleToken(IToken.tSTAR, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tSTAR); case '/': if (pos + 1 < limit) { if (buffer[pos + 1] == '=') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tDIVASSIGN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tDIVASSIGN ); } } - return new SimpleToken(IToken.tDIV, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tDIV ); case '%': if (pos + 1 < limit) { if (buffer[pos + 1] == '=') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tMODASSIGN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tMODASSIGN ); } } - return new SimpleToken(IToken.tMOD, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tMOD ); case '^': if (pos + 1 < limit) { if (buffer[pos + 1] == '=') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tXORASSIGN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tXORASSIGN ); } } - return new SimpleToken(IToken.tXOR, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tXOR ); case '&': if (pos + 1 < limit) { if (buffer[pos + 1] == '&') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tAND, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tAND ); } else if (buffer[pos + 1] == '=') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tAMPERASSIGN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tAMPERASSIGN ); } } - return new SimpleToken(IToken.tAMPER, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tAMPER ); case '|': if (pos + 1 < limit) { if (buffer[pos + 1] == '|') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tOR, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tOR ); } else if (buffer[pos + 1] == '=') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tBITORASSIGN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tBITORASSIGN ); } } - return new SimpleToken(IToken.tBITOR, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tBITOR ); case '~': - return new SimpleToken(IToken.tCOMPL, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tCOMPL ); case '!': if (pos + 1 < limit) { if (buffer[pos + 1] == '=') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tNOTEQUAL, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tNOTEQUAL ); } } - return new SimpleToken(IToken.tNOT, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tNOT ); case '=': if (pos + 1 < limit) { if (buffer[pos + 1] == '=') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tEQUAL, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tEQUAL ); } } - return new SimpleToken(IToken.tASSIGN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tASSIGN ); case '<': if (pos + 1 < limit) { if (buffer[pos + 1] == '=') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tLTEQUAL, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tLTEQUAL ); } else if (buffer[pos + 1] == '<') { if (pos + 2 < limit) { if (buffer[pos + 2] == '=') { bufferPos[bufferStackPos] += 2; - return new SimpleToken(IToken.tSHIFTLASSIGN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tSHIFTLASSIGN ); } } ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tSHIFTL, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tSHIFTL ); } } - return new SimpleToken(IToken.tLT, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tLT ); case '>': if (pos + 1 < limit) { if (buffer[pos + 1] == '=') { ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tGTEQUAL, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tGTEQUAL ); } else if (buffer[pos + 1] == '>') { if (pos + 2 < limit) { if (buffer[pos + 2] == '=') { bufferPos[bufferStackPos] += 2; - return new SimpleToken(IToken.tSHIFTRASSIGN, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tSHIFTRASSIGN ); } } ++bufferPos[bufferStackPos]; - return new SimpleToken(IToken.tSHIFTR, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tSHIFTR); } } - return new SimpleToken(IToken.tGT, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tGT ); case ',': - return new SimpleToken(IToken.tCOMMA, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + return newToken(IToken.tCOMMA ); default: // skip over anything we don't handle @@ -689,6 +690,29 @@ public class Scanner2 implements IScanner, IScannerData { return null; } + /** + * @return + */ + private IToken newToken( int signal ) { + return new SimpleToken(signal, bufferPos[bufferStackPos] + 1 , getCurrentFilename() ); + } + + private IToken newToken( int signal, char [] buffer ) + { + if( bufferData[bufferStackPos] instanceof ObjectStyleMacro || + bufferData[bufferStackPos] instanceof FunctionStyleMacro ) + { + int mostRelevant; + for( mostRelevant = bufferStackPos; mostRelevant >= 0; --mostRelevant ) + if( bufferData[mostRelevant] instanceof InclusionData || bufferData[mostRelevant] instanceof CodeReader ) + break; + if( bufferData[bufferStackPos] instanceof ObjectStyleMacro ) + return new ImagedExpansionToken( signal, buffer, bufferPos[mostRelevant], ((ObjectStyleMacro)bufferData[bufferStackPos]).name.length, getCurrentFilename() ); + return new ImagedExpansionToken( signal, buffer, bufferPos[mostRelevant], ((FunctionStyleMacro)bufferData[bufferStackPos]).name.length, getCurrentFilename() ); + } + return new ImagedToken(signal, buffer, bufferPos[bufferStackPos] + 1 , getCurrentFilename() ); + } + private IToken scanIdentifier() { char[] buffer = bufferStack[bufferStackPos]; int start = bufferPos[bufferStackPos]; @@ -748,8 +772,8 @@ public class Scanner2 implements IScanner, IScannerData { int tokenType = keywords.get(buffer, start, len); char [] result = removedEscapedNewline( CharArrayUtils.extract( buffer, start, len ) ); if (tokenType == keywords.undefined) - return new ImagedToken(IToken.tIDENTIFIER, result, bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); - return new SimpleToken(tokenType, start + len, getCurrentFilename() ); + return newToken(IToken.tIDENTIFIER, result ); + return newToken(tokenType); } private IToken scanString() { @@ -783,7 +807,7 @@ public class Scanner2 implements IScanner, IScannerData { // We should really throw an exception if we didn't get the terminating // quote before the end of buffer - return new ImagedToken(tokenType, CharArrayUtils.extract(buffer, stringStart, stringLen), stringStart + stringLen+ 1, getCurrentFilename() ); + return newToken(tokenType, CharArrayUtils.extract(buffer, stringStart, stringLen)); } private IToken scanCharLiteral(boolean b) { @@ -800,7 +824,7 @@ public class Scanner2 implements IScanner, IScannerData { } if (start >= limit) { - return new ImagedToken(tokenType, emptyCharArray, start, getCurrentFilename() ); + return newToken(tokenType, emptyCharArray ); } @@ -823,7 +847,7 @@ public class Scanner2 implements IScanner, IScannerData { ? CharArrayUtils.extract(buffer, start, length) : emptyCharArray; - return new ImagedToken(tokenType, image, start + length+ 1 , getCurrentFilename() ); + return newToken(tokenType, image ); } private IToken scanNumber() { @@ -999,9 +1023,9 @@ public class Scanner2 implements IScanner, IScannerData { --bufferPos[bufferStackPos]; - return new ImagedToken(isFloat ? IToken.tFLOATINGPT : IToken.tINTEGER, + return newToken( isFloat ? IToken.tFLOATINGPT : IToken.tINTEGER, CharArrayUtils.extract(buffer, start, - bufferPos[bufferStackPos] - start + 1), bufferPos[bufferStackPos]+ 1, getCurrentFilename() ); + bufferPos[bufferStackPos] - start + 1) ); } private void handlePPDirective(int pos) throws ScannerException {