diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java index e062aff6f14..a895464a853 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java @@ -200,25 +200,25 @@ public class LexerTests extends BaseTestCase { public void testLineSplicingHeaderName() throws Exception { init("p\"a\\\nb\""); - fLexer.setInsideIncludeDirective(); + fLexer.setInsideIncludeDirective(true); id("p"); token(Lexer.tQUOTE_HEADER_NAME, "\"ab\""); eof(); init("p\"a\\\r\nb\""); - fLexer.setInsideIncludeDirective(); + fLexer.setInsideIncludeDirective(true); id("p"); token(Lexer.tQUOTE_HEADER_NAME, "\"ab\""); eof(); init("p"); - fLexer.setInsideIncludeDirective(); + fLexer.setInsideIncludeDirective(true); id("p"); token(Lexer.tSYSTEM_HEADER_NAME, ""); eof(); init("p"); - fLexer.setInsideIncludeDirective(); + fLexer.setInsideIncludeDirective(true); id("p"); token(Lexer.tSYSTEM_HEADER_NAME, ""); eof(); @@ -280,13 +280,13 @@ public class LexerTests extends BaseTestCase { public void testHeaderName() throws Exception { init("p\"'/*//\\\""); - fLexer.setInsideIncludeDirective(); + fLexer.setInsideIncludeDirective(true); id("p"); token(Lexer.tQUOTE_HEADER_NAME, "\"'/*//\\\""); eof(); init("p<'\"/*//>"); - fLexer.setInsideIncludeDirective(); + fLexer.setInsideIncludeDirective(true); id("p"); token(Lexer.tSYSTEM_HEADER_NAME, "<'\"/*//>"); eof(); @@ -316,8 +316,7 @@ public class LexerTests extends BaseTestCase { init(ident, false, true); final int idxDollar = ident.indexOf('$'); id(ident.substring(0, idxDollar)); - problem(IProblem.SCANNER_BAD_CHARACTER, "$"); - ws(); + token(IToken.tOTHER_CHARACTER, "$"); id(ident.substring(idxDollar+1)); } @@ -437,7 +436,7 @@ public class LexerTests extends BaseTestCase { IToken.tBITORASSIGN, IToken.tSHIFTL, IToken.tSHIFTR, IToken.tSHIFTLASSIGN, IToken.tSHIFTRASSIGN, IToken.tEQUAL, IToken.tNOTEQUAL, IToken.tLTEQUAL, IToken.tGTEQUAL, IToken.tAND, IToken.tOR, IToken.tINCR, IToken.tDECR, IToken.tCOMMA, IToken.tARROWSTAR, - IToken.tARROW, IGCCToken.tMIN, IGCCToken.tMAX, IToken.tBACKSLASH, + IToken.tARROW, IGCCToken.tMIN, IGCCToken.tMAX, IToken.tOTHER_CHARACTER, }; for (int splices=0; splices<9; splices++) { diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LocationMapTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LocationMapTests.java index 2bddc6ea32e..6383005cbd1 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LocationMapTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LocationMapTests.java @@ -466,38 +466,38 @@ public class LocationMapTests extends BaseTestCase { public void testContexts() { init(DIGITS); assertEquals(FN, fLocationMap.getTranslationUnitPath()); - assertEquals(FN, fLocationMap.getCurrentFilename()); + assertEquals(FN, fLocationMap.getCurrentFilePath()); // number: [30,36)[46,50) ILocationCtx pre1= fLocationMap.pushPreInclusion("0102030405".toCharArray(), 0, false); - assertEquals(FN, fLocationMap.getCurrentFilename()); + assertEquals(FN, fLocationMap.getCurrentFilePath()); // number: [0,6)[26,30) ILocationCtx pre2= fLocationMap.pushPreInclusion("a1a2a3a4a5".toCharArray(), 0, true); - assertEquals(FN, fLocationMap.getCurrentFilename()); + assertEquals(FN, fLocationMap.getCurrentFilePath()); fLocationMap.encounteredComment(0,2,true); // number: [6,15)[25,26) ILocationCtx i1= fLocationMap.pushInclusion(0, 2, 4, 6, "b1b2b3b4b5".toCharArray(), "pre1", "pre1".toCharArray(), false); - assertEquals("pre1", fLocationMap.getCurrentFilename()); + assertEquals("pre1", fLocationMap.getCurrentFilePath()); fLocationMap.encounteredComment(2,4,true); // number: [15,25) ILocationCtx i2= fLocationMap.pushInclusion(6, 7, 8, 9, "c1c2c3c4c5".toCharArray(), "pre11", "pre11".toCharArray(), false); - assertEquals("pre11", fLocationMap.getCurrentFilename()); + assertEquals("pre11", fLocationMap.getCurrentFilePath()); fLocationMap.encounteredComment(2,6,true); fLocationMap.popContext(i2); // add a comment before the include fLocationMap.encounteredComment(4,6,false); - assertEquals("pre1", fLocationMap.getCurrentFilename()); + assertEquals("pre1", fLocationMap.getCurrentFilePath()); fLocationMap.popContext(i1); - assertEquals(FN, fLocationMap.getCurrentFilename()); + assertEquals(FN, fLocationMap.getCurrentFilePath()); fLocationMap.popContext(pre2); - assertEquals(FN, fLocationMap.getCurrentFilename()); + assertEquals(FN, fLocationMap.getCurrentFilePath()); // number [36, 46) ILocationCtx i3= fLocationMap.pushInclusion(0, 2, 4, 6, "d1d2d3d4d5".toCharArray(), "pre2", "pre2".toCharArray(), false); - assertEquals("pre2", fLocationMap.getCurrentFilename()); + assertEquals("pre2", fLocationMap.getCurrentFilePath()); fLocationMap.encounteredComment(0,2,true); fLocationMap.popContext(i3); fLocationMap.popContext(pre1); - assertEquals(FN, fLocationMap.getCurrentFilename()); + assertEquals(FN, fLocationMap.getCurrentFilePath()); IASTComment[] comments= fLocationMap.getComments(); diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PortedScannerTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PortedScannerTest.java new file mode 100644 index 00000000000..0f6fbf02644 --- /dev/null +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PortedScannerTest.java @@ -0,0 +1,2532 @@ +/******************************************************************************* + * Copyright (c) 2004, 2007 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM - Initial API and implementation + * Markus Schorn (Wind River Systems) + *******************************************************************************/ +package org.eclipse.cdt.core.parser.tests.scanner; + +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import junit.framework.ComparisonFailure; +import junit.framework.TestSuite; + +import org.eclipse.cdt.core.dom.ICodeReaderFactory; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorIncludeStatement; +import org.eclipse.cdt.core.dom.ast.IASTProblem; +import org.eclipse.cdt.core.dom.ast.IMacroBinding; +import org.eclipse.cdt.core.dom.parser.IScannerExtensionConfiguration; +import org.eclipse.cdt.core.dom.parser.c.GCCScannerExtensionConfiguration; +import org.eclipse.cdt.core.dom.parser.cpp.GPPScannerExtensionConfiguration; +import org.eclipse.cdt.core.parser.CodeReader; +import org.eclipse.cdt.core.parser.EndOfFileException; +import org.eclipse.cdt.core.parser.IParserLogService; +import org.eclipse.cdt.core.parser.IProblem; +import org.eclipse.cdt.core.parser.IScannerInfo; +import org.eclipse.cdt.core.parser.ISourceElementRequestor; +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.NullLogService; +import org.eclipse.cdt.core.parser.NullSourceElementRequestor; +import org.eclipse.cdt.core.parser.ParserLanguage; +import org.eclipse.cdt.core.parser.ParserMode; +import org.eclipse.cdt.core.parser.ScannerInfo; +import org.eclipse.cdt.core.parser.util.CharArrayUtils; +import org.eclipse.cdt.core.testplugin.util.BaseTestCase; +import org.eclipse.cdt.internal.core.parser.scanner.CPreprocessor; +import org.eclipse.cdt.internal.core.parser.scanner.ILocationResolver; +import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory; + +/** + * Scanner2Tests ported to use the CPreprocessor + */ +public class PortedScannerTest extends BaseTestCase { + private static final IParserLogService NULL_LOG = new NullLogService(); + + public static TestSuite suite() { + return suite(PortedScannerTest.class); + } + + private CPreprocessor fScanner; + private ILocationResolver fLocationResolver; + + public PortedScannerTest() { + super(); + } + + public PortedScannerTest(String name) { + super(name); + } + + protected void initializeScanner(String input) throws IOException { + initializeScanner(input, ParserMode.COMPLETE_PARSE); + } + + protected void initializeScanner(String input, ParserMode mode) throws IOException { + initializeScanner(input, ParserLanguage.CPP, mode); + } + + protected void initializeScanner(String input, ParserLanguage lang) throws IOException { + initializeScanner(input, lang, ParserMode.COMPLETE_PARSE); + } + + protected void initializeScanner(String input, ParserLanguage lang, ParserMode mode) throws IOException { + ICodeReaderFactory readerFactory= FileCodeReaderFactory.getInstance(); + CodeReader reader= new CodeReader(input.toCharArray()); + IScannerExtensionConfiguration scannerConfig; + IScannerInfo scannerInfo= new ScannerInfo(); + + if (lang == ParserLanguage.C) { + scannerConfig= new GCCScannerExtensionConfiguration(); + } + else { + scannerConfig= new GPPScannerExtensionConfiguration(); + } + + fScanner= new CPreprocessor(reader, scannerInfo, lang, NULL_LOG, scannerConfig, readerFactory); + fLocationResolver= (ILocationResolver) fScanner.getAdapter(ILocationResolver.class); + } + + + protected int fullyTokenize() throws Exception { + try { + for(;;) { + IToken t= fScanner.nextToken(); + assertTrue(t.getType() <= IToken.tLAST); + } + } + catch ( EndOfFileException e){ + } + return fScanner.getCount(); + } + + + protected void validateToken(int tokenType) throws Exception { + IToken t= fScanner.nextToken(); + assertEquals(tokenType, t.getType()); + } + + protected void validateToken(int tokenType, String image) throws Exception { + IToken t= fScanner.nextToken(); + assertEquals(tokenType, t.getType()); + assertEquals(image, t.getImage()); + } + + protected void validateInteger(String expectedImage) throws Exception { + validateToken(IToken.tINTEGER, expectedImage); + } + + protected void validateIdentifier(String expectedImage) throws Exception { + validateToken(IToken.tIDENTIFIER, expectedImage); + } + + protected void validateString(String expectedImage) throws Exception { + validateToken(IToken.tSTRING, "\"" + expectedImage + "\""); + } + + protected void validateChar(String expectedImage) throws Exception { + validateToken(IToken.tCHAR, "'" + expectedImage + "'"); + } + + protected void validateWideChar(String expectedImage) throws Exception { + validateToken(IToken.tLCHAR, "L'" + expectedImage + "'"); + } + + protected void validateLString(String expectedImage) throws Exception { + validateToken(IToken.tLSTRING, "L\"" + expectedImage + "\""); + } + + protected void validateFloatingPointLiteral(String expectedImage) throws Exception { + validateToken(IToken.tFLOATINGPT, expectedImage); + } + + protected void validateEOF() throws Exception { + try { + IToken t= fScanner.nextToken(); + fail("superfluous token " + t); + } + catch(EndOfFileException e) { + } + } + + private void assertCharArrayEquals(char[] expected, char[] actual) { + if (!CharArrayUtils.equals(expected, actual)) + throw new ComparisonFailure(null, new String(expected), new String(actual)); + } + + protected void validateDefinition(String name, String value) { + Object expObject = fScanner.getRealDefinitions().get(name.toCharArray()); + assertNotNull(expObject); + assertTrue(expObject instanceof IMacroBinding); + assertCharArrayEquals(value.toCharArray(), ((IMacroBinding)expObject).getExpansion()); + } + + protected void validateDefinition(String name, int value) { + validateDefinition(name, String.valueOf(value)); + } + + protected void validateAsUndefined(String name) { + assertNull(fScanner.getDefinitions().get(name.toCharArray())); + } + + protected void validateProblemCount(int count) throws Exception { + assertEquals(count, fLocationResolver.getScannerProblems().length); + } + + public void testBug102825_1() throws Exception { + StringBuffer buffer = new StringBuffer( + "#define CURLOPTTYPE_OBJECTPOINT 10000\n"); + buffer.append("#define CINIT = CURLOPTTYPE_##OBJECTPOINT + 1\n"); + buffer.append("CINIT\n"); + initializeScanner(buffer.toString()); + validateToken(IToken.tASSIGN); + validateInteger("10000"); + } + + public void testBug102825_2() throws Exception { + StringBuffer buffer = new StringBuffer( + "#define CURLOPTTYPE_OBJECTPOINT 10000\n"); + buffer + .append("#define CINIT(name,type,number) = CURLOPTTYPE_##type + number\n"); + buffer.append("CINIT(FILE, OBJECTPOINT, 1)\n"); + initializeScanner(buffer.toString()); + validateToken(IToken.tASSIGN); + validateInteger("10000"); + } + + public void testBug102825_3() throws Exception { + StringBuffer buffer = new StringBuffer( + "#define CURLOPTTYPE_OBJECTPOINT 10000\n"); + buffer + .append("#define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number\n"); + buffer.append("CINIT(FILE, OBJECTPOINT, 1)\n"); + initializeScanner(buffer.toString()); + validateIdentifier("CURLOPT_FILE"); + validateToken(IToken.tASSIGN); + validateInteger("10000"); + } + + public void testBug102825_4() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define glue( a, b ) a ## b\n"); + buffer.append("#define HIGHLOW \"hello\"\n"); + buffer.append("glue( HIGH, LOW )\n"); + + initializeScanner(buffer.toString(), ParserMode.QUICK_PARSE); + validateString("hello"); + validateProblemCount(0); + } + + public void testBug195610_1() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define glue(x, y, z) x ## y ## z\n"); + buffer.append("glue(, b, c)\n"); + + initializeScanner(buffer.toString(), ParserMode.QUICK_PARSE); + validateIdentifier("bc"); + validateProblemCount(0); + } + + public void testBug195610_2() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define glue(x, y, z) x ## y ## z\n"); + buffer.append("glue(a, , c)\n"); + + initializeScanner(buffer.toString(), ParserMode.QUICK_PARSE); + validateIdentifier("ac"); + validateProblemCount(0); + } + + public void testBug195610_3() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define glue(x, y, z) x ## y ## z\n"); + buffer.append("glue(a, b, )\n"); + + initializeScanner(buffer.toString(), ParserMode.QUICK_PARSE); + validateIdentifier("ab"); + validateProblemCount(0); + } + + public class TableRow { + private int[] values; + private int length; + + public TableRow(int[] v) { + length = v.length; + values = new int[length]; + System.arraycopy(v, 0, values, 0, length); + } + + public String toString() { + StringBuffer s = new StringBuffer(); + for (int i = 0; i < length; ++i) { + s + .append("var").append(i).append("=").append(values[i]).append(" "); //$NON-NLS-3$ + } + return s.toString(); + } + + public String symbolName(int index) { + return "DEFINITION" + index; + } + + public int symbolValue(int index) { + return new Long(Math.round(Math.pow(index, index))).intValue(); + } + + public String generateCode() { + if (length < 2) { + return "Array must have at least 2 elements"; + } + int numberOfElsifs = length - 1; + StringBuffer buffer = new StringBuffer(); + buffer.append("#if ").append(values[0]).append("\n#\tdefine "); + buffer.append(symbolName(0)).append(" ").append(symbolValue(0)); + for (int i = 0; i < numberOfElsifs; ++i) + buffer.append("\n#elif ") + .append(values[1 + i]).append("\n#\tdefine ") + .append(symbolName(i + 1)).append(" ") + .append(symbolValue(i + 1)); + buffer.append("\n#else \n#\tdefine ") + .append(symbolName(length)).append(" ") + .append(symbolValue(length)).append("\n#endif"); + return buffer.toString(); + } + + public int selectWinner() { + for (int i = 0; i < values.length; ++i) { + if (values[i] != 0) { + return i; + } + } + return length; + } + + /** + * Returns the length. + * @return int + */ + public int getLength() { + return length; + } + + } + + public class TruthTable { + private int numberOfVariables; + private int numberOfRows; + public TableRow[] rows; + + public TruthTable(int n) { + numberOfVariables = n; + numberOfRows = new Long(Math.round(Math.pow(2, n))).intValue(); + + rows = new TableRow[numberOfRows]; + for (int i = 0; i < numberOfRows; ++i) { + String Z = Integer.toBinaryString(i); + + int[] input = new int[numberOfVariables]; + for (int j = 0; j < numberOfVariables; ++j) { + int padding = numberOfVariables - Z.length(); + int k = 0; + for (; k < padding; ++k) { + input[k] = 0; + } + for (int l = 0; l < Z.length(); ++l) { + char c = Z.charAt(l); + int value = Character.digit(c, 10); + input[k++] = value; + } + } + rows[i] = new TableRow(input); + } + } + + /** + * Returns the numberOfRows. + * @return int + */ + public int getNumberOfRows() { + return numberOfRows; + } + + } + + public final static int SIZEOF_TRUTHTABLE = 10; + + public void testWeirdStrings() throws Exception { + initializeScanner("Living Life L\"LONG\""); + validateIdentifier("Living"); + validateIdentifier("Life"); + validateLString("LONG"); + validateEOF(); + + } + + public void testNumerics() throws Exception { + initializeScanner("3.0 0.9 .5 3. 4E5 2.01E-03 ..."); + validateFloatingPointLiteral("3.0"); + validateFloatingPointLiteral("0.9"); + validateFloatingPointLiteral(".5"); + validateFloatingPointLiteral("3."); + validateFloatingPointLiteral("4E5"); + validateFloatingPointLiteral("2.01E-03"); + validateToken(IToken.tELLIPSIS); + validateEOF(); + + } + + public void testPreprocessorDefines() throws Exception { + initializeScanner("#define SIMPLE_NUMERIC 5\nint x = SIMPLE_NUMERIC"); + validateToken(IToken.t_int); + validateDefinition("SIMPLE_NUMERIC", "5"); + validateIdentifier("x"); + validateToken(IToken.tASSIGN); + validateInteger("5"); + validateEOF(); + + initializeScanner("#define SIMPLE_STRING \"This is a simple string.\"\n\nconst char * myVariable = SIMPLE_STRING;"); + validateToken(IToken.t_const); + validateDefinition("SIMPLE_STRING", "\"This is a simple string.\""); + validateToken(IToken.t_char); + validateToken(IToken.tSTAR); + validateIdentifier("myVariable"); + validateToken(IToken.tASSIGN); + validateString("This is a simple string."); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define FOOL 5 \n int tryAFOOL = FOOL + FOOL;"); + validateToken(IToken.t_int); + validateIdentifier("tryAFOOL"); + validateToken(IToken.tASSIGN); + validateInteger("5"); + validateToken(IToken.tPLUS); + validateInteger("5"); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define FOOL 5 \n int FOOLer = FOOL;"); + validateToken(IToken.t_int); + validateIdentifier("FOOLer"); + validateToken(IToken.tASSIGN); + validateInteger("5"); + validateToken(IToken.tSEMI); + validateEOF(); + + // the case we were failing against in ctype.h + // this is a definition, not a macro! + initializeScanner("#define _ALPHA (0x0100|_UPPER|_LOWER)"); + validateEOF(); + validateDefinition("_ALPHA", "(0x0100|_UPPER|_LOWER)"); + + // test for comments after the macro + initializeScanner("#define NO_COMMENT// ignore me"); + validateEOF(); + validateDefinition("NO_COMMENT", ""); + + initializeScanner("#define NO_COMMENT/* ignore me*/"); + validateEOF(); + validateDefinition("NO_COMMENT", ""); + + initializeScanner("#define ANSWER 42 // i think"); + validateEOF(); + validateDefinition("ANSWER", "42"); + + initializeScanner("#define ANSWER 42 /* i think */"); + validateEOF(); + validateDefinition("ANSWER", "42"); + + initializeScanner("#define MULTILINE 3 /* comment \n that goes more than one line */"); + validateEOF(); + validateDefinition("MULTILINE", "3"); + + initializeScanner("#define MULTICOMMENT X /* comment1 */ + Y /* comment 2 */"); + validateEOF(); + validateDefinition("MULTICOMMENT", "X + Y"); + + initializeScanner("#define SIMPLE_STRING This is a simple string.\n"); + validateEOF(); + validateDefinition("SIMPLE_STRING", + "This is a simple string."); + + initializeScanner("# define SIMPLE_NUMERIC 5\n"); + validateEOF(); + validateDefinition("SIMPLE_NUMERIC", "5"); + + initializeScanner("# define SIMPLE_NUMERIC 5\n"); + validateEOF(); + validateDefinition("SIMPLE_NUMERIC", "5"); + + initializeScanner("#define SIMPLE_STRING \"This is a simple string.\"\n"); + validateEOF(); + validateDefinition("SIMPLE_STRING", + "\"This is a simple string.\""); + + initializeScanner("#define SIMPLE_STRING This is a simple string.\n"); + validateEOF(); + validateDefinition("SIMPLE_STRING", "This is a simple string."); + + initializeScanner("#define FLAKE\n\nFLAKE"); + validateEOF(); + validateDefinition("FLAKE", ""); + + initializeScanner("#define SIMPLE_STRING This is a simple string.\\\n Continue please."); + validateEOF(); + validateDefinition("SIMPLE_STRING", "This is a simple string. Continue please."); + } + + public void testBug67834() throws Exception { + initializeScanner("#if ! BAR\n" + + "foo\n" + + "#else\n" + + "bar\n" + + "#endif\n" + ); + validateIdentifier("foo"); + validateEOF(); + initializeScanner("#if ! (BAR)\n" + + "foo\n" + + "#else\n" + + "bar\n" + + "#endif\n" + ); + validateIdentifier("foo"); + validateEOF(); + } + + public void testConcatenation() throws Exception { + initializeScanner("#define F1 3\n#define F2 F1##F1\nint x=F2;"); + validateToken(IToken.t_int); + validateDefinition("F1", "3"); + validateDefinition("F2", "F1##F1"); + validateIdentifier("x"); + validateToken(IToken.tASSIGN); + validateIdentifier("F1F1"); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define PREFIX RT_\n#define RUN PREFIX##Run"); + validateEOF(); + validateDefinition("PREFIX", "RT_"); + validateDefinition("RUN", "PREFIX##Run"); + + initializeScanner("#define DECLARE_HANDLE(name) struct name##__ { int unused; }; typedef struct name##__ *name\n DECLARE_HANDLE( joe )"); + validateToken(IToken.t_struct); + validateIdentifier("joe__"); + validateToken(IToken.tLBRACE); + validateToken(IToken.t_int); + validateIdentifier("unused"); + validateToken(IToken.tSEMI); + validateToken(IToken.tRBRACE); + validateToken(IToken.tSEMI); + validateToken(IToken.t_typedef); + validateToken(IToken.t_struct); + validateIdentifier("joe__"); + validateToken(IToken.tSTAR); + validateIdentifier("joe"); + validateEOF(); + } + + public void testSimpleIfdef() throws Exception { + initializeScanner("#define SYMBOL 5\n#ifdef SYMBOL\nint counter(SYMBOL);\n#endif"); + validateToken(IToken.t_int); + validateIdentifier("counter"); + validateToken(IToken.tLPAREN); + validateInteger("5"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define SYMBOL 5\n#ifndef SYMBOL\nint counter(SYMBOL);\n#endif"); + validateEOF(); + + initializeScanner("#ifndef DEFINED\n#define DEFINED 100\n#endif\nint count = DEFINED;"); + validateToken(IToken.t_int); + validateDefinition("DEFINED", "100"); + + validateIdentifier("count"); + validateToken(IToken.tASSIGN); + validateInteger("100"); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#ifndef DEFINED\n#define DEFINED 100\n#endif\nint count = DEFINED;"); + addDefinition("DEFINED", "101"); + validateDefinition("DEFINED", "101"); + validateToken(IToken.t_int); + validateIdentifier("count"); + validateToken(IToken.tASSIGN); + validateInteger("101"); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("/* NB: This is #if 0'd out */"); + validateEOF(); + } + + /** + * @param string + * @param string2 + */ + private void addDefinition(String string, String string2) { + fScanner.addMacroDefinition(string.toCharArray(), string2.toCharArray()); + } + + public void testMultipleLines() throws Exception { + Writer code = new StringWriter(); + code.write("#define COMPLEX_MACRO 33 \\\n"); + code.write(" + 44\n\nCOMPLEX_MACRO"); + initializeScanner(code.toString()); + validateInteger("33"); + validateToken(IToken.tPLUS); + validateInteger("44"); + } + + public void testSlightlyComplexIfdefStructure() throws Exception { + initializeScanner("#ifndef BASE\n#define BASE 10\n#endif\n#ifndef BASE\n#error BASE is defined\n#endif"); + validateEOF(); + + initializeScanner("#ifndef ONE\n#define ONE 1\n#ifdef TWO\n#define THREE ONE + TWO\n#endif\n#endif\nint three(THREE);"); + + validateToken(IToken.t_int); + validateDefinition("ONE", "1"); + validateAsUndefined("TWO"); + validateAsUndefined("THREE"); + validateIdentifier("three"); + validateToken(IToken.tLPAREN); + validateIdentifier("THREE"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + initializeScanner( + "#ifndef ONE\n" + + "#define ONE 1\n" + + "#ifdef TWO\n" + + "#define THREE ONE + TWO\n" + + "#endif\n"+ + "#endif\n"+ + "int three(THREE);"); + addDefinition("TWO", "2"); + validateToken(IToken.t_int); + validateDefinition("ONE", "1"); + validateDefinition("TWO", "2"); + validateDefinition("THREE", "ONE + TWO"); + + validateIdentifier("three"); + validateToken(IToken.tLPAREN); + validateInteger("1"); + validateToken(IToken.tPLUS); + validateInteger("2"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + initializeScanner("#ifndef FOO\n#define FOO 4\n#else\n#undef FOO\n#define FOO 6\n#endif"); + validateEOF(); + validateDefinition("FOO", "4"); + + initializeScanner("#ifndef FOO\n#define FOO 4\n#else\n#undef FOO\n#define FOO 6\n#endif"); + addDefinition("FOO", "2"); + validateEOF(); + validateDefinition("FOO", "6"); + + initializeScanner("#ifndef ONE\n# define ONE 1\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#else\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#endif\n"); + validateEOF(); + validateDefinition("ONE", "1"); + validateDefinition("TWO", "ONE + ONE"); + + initializeScanner("#ifndef ONE\r\n" + + "# define ONE 1\n" + + "# ifndef TWO\n" + + "# define TWO ONE + ONE \n" + + "# else\n" + + "# undef TWO\n" + + "# define TWO 2 \n" + + "# endif\n" + + "#else\n" + + "# ifndef TWO\n" + + "# define TWO ONE + ONE \n" + + "# else\n" + + "# undef TWO\n" + + "# define TWO 2 \n" + + "# endif\n" + + "#endif\n"); + + addDefinition("ONE", "one"); + validateEOF(); + validateDefinition("ONE", "one"); + validateDefinition("TWO", "ONE + ONE"); + + initializeScanner("#ifndef ONE\n# define ONE 1\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#else\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#endif\n"); + addDefinition("ONE", "one"); + addDefinition("TWO", "two"); + validateEOF(); + validateDefinition("ONE", "one"); + validateDefinition("TWO", "2"); + + initializeScanner("#ifndef ONE\n# define ONE 1\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#else\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#endif\n"); + addDefinition("TWO", "two"); + validateEOF(); + validateDefinition("ONE", "1"); + validateDefinition("TWO", "2"); + } + + public void testIfs() throws Exception { + initializeScanner("#if 0\n#error NEVER\n#endif\n"); + validateEOF(); + initializeScanner("#define X 5\n#define Y 7\n#if (X < Y)\n#define Z X + Y\n#endif"); + validateEOF(); + validateDefinition("X", "5"); + validateDefinition("Y", "7"); + validateDefinition("Z", "X + Y"); + + initializeScanner("#if T < 20\n#define Z T + 1\n#endif"); + addDefinition("X", "5"); + addDefinition("Y", "7"); + addDefinition("T", "X + Y"); + validateEOF(); + validateDefinition("X", "5"); + validateDefinition("Y", "7"); + validateDefinition("T", "X + Y"); + validateDefinition("Z", "T + 1"); + + initializeScanner("#if ( 10 / 5 ) != 2\n#error 10/5 seems to not equal 2 anymore\n#endif\n"); + validateEOF(); + + initializeScanner("#ifndef FIVE \n" + + "#define FIVE 5\n" + + "#endif \n" + + "#ifndef TEN\n" + + "#define TEN 2 * FIVE\n" + + "#endif\n" + + "#if TEN != 10\n" + + "#define MISTAKE 1\n" + + "#error Five does not equal 10\n" + + "#endif\n", ParserMode.QUICK_PARSE); + addDefinition("FIVE", "55"); + validateEOF(); + validateDefinition("FIVE", "55"); + validateDefinition("TEN", "2 * FIVE"); + validateDefinition("MISTAKE", "1"); + validateProblemCount(1); + + initializeScanner("#if ((( FOUR / TWO ) * THREE )< FIVE )\n#error 6 is not less than 5 \n#endif\n#if ( ( FIVE * ONE ) != (( (FOUR) + ONE ) * ONE ) )\n#error 5 should equal 5\n#endif \n"); + + addDefinition("ONE", "1"); + addDefinition("TWO", "(ONE + ONE)"); + addDefinition("THREE", "(TWO + ONE)"); + addDefinition("FOUR", "(TWO * TWO)"); + addDefinition("FIVE", "(THREE + TWO)"); + + validateEOF(); + validateDefinition("ONE", "1"); + validateDefinition("TWO", "(ONE + ONE)"); + validateDefinition("THREE", "(TWO + ONE)"); + validateDefinition("FOUR", "(TWO * TWO)"); + validateDefinition("FIVE", "(THREE + TWO)"); + + TruthTable table = new TruthTable(SIZEOF_TRUTHTABLE); + int numberOfRows = table.getNumberOfRows(); + TableRow[] rows = table.rows; + + for (int i = 0; i < numberOfRows; ++i) { + TableRow row = rows[i]; + String code = row.generateCode(); + initializeScanner(code); + validateEOF(); + validateAllDefinitions(row); + } + + initializeScanner("#if ! 0\n#error Correct!\n#endif"); + validateEOF(); + } + + public void testPreprocessorMacros() throws Exception { + initializeScanner("#define GO(x) x+1\nint y(5);\ny = GO(y);"); + validateToken(IToken.t_int); + validateIdentifier("y"); + validateToken(IToken.tLPAREN); + validateInteger("5"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + + validateIdentifier("y"); + validateToken(IToken.tASSIGN); + validateIdentifier("y"); + validateToken(IToken.tPLUS); + validateInteger("1"); + validateToken(IToken.tSEMI); + validateEOF(); + initializeScanner("#define ONE 1\n" + + "#define SUM(a,b,c,d,e,f,g) ( a + b + c + d + e + f + g )\n" + + "int daSum = SUM(ONE,3,5,7,9,11,13);"); + validateToken(IToken.t_int); + validateIdentifier("daSum"); + validateToken(IToken.tASSIGN); + validateToken(IToken.tLPAREN); + validateInteger("1"); + validateToken(IToken.tPLUS); + validateInteger("3"); + validateToken(IToken.tPLUS); + validateInteger("5"); + validateToken(IToken.tPLUS); + validateInteger("7"); + validateToken(IToken.tPLUS); + validateInteger("9"); + validateToken(IToken.tPLUS); + validateInteger("11"); + validateToken(IToken.tPLUS); + validateInteger("13"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define LOG( format, var1) printf( format, var1 )\nLOG( \"My name is %s\", \"Bogdan\" );\n"); + validateIdentifier("printf"); + validateToken(IToken.tLPAREN); + validateString("My name is %s"); + validateToken(IToken.tCOMMA); + validateString("Bogdan"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define INCR( x ) ++x\nint y(2);\nINCR(y);"); + validateToken(IToken.t_int); + validateIdentifier("y"); + validateToken(IToken.tLPAREN); + validateInteger("2"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateToken(IToken.tINCR); + validateIdentifier("y"); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define CHECK_AND_SET( x, y, z ) if( x ) { \\\n y = z; \\\n }\n\nCHECK_AND_SET( 1, balance, 5000 );\nCHECK_AND_SET( confused(), you, dumb );"); + validateToken(IToken.t_if); + validateToken(IToken.tLPAREN); + validateInteger("1"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tLBRACE); + validateIdentifier("balance"); + validateToken(IToken.tASSIGN); + validateInteger("5000"); + validateToken(IToken.tSEMI); + validateToken(IToken.tRBRACE); + validateToken(IToken.tSEMI); + + validateToken(IToken.t_if); + validateToken(IToken.tLPAREN); + validateIdentifier("confused"); + validateToken(IToken.tLPAREN); + validateToken(IToken.tRPAREN); + validateToken(IToken.tRPAREN); + validateToken(IToken.tLBRACE); + validateIdentifier("you"); + validateToken(IToken.tASSIGN); + validateIdentifier("dumb"); + validateToken(IToken.tSEMI); + validateToken(IToken.tRBRACE); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner( + "#define ON 7\n"+ + "#if defined(ON)\n"+ + "int itsOn = ON;\n"+ + "#endif"); + validateToken(IToken.t_int); + validateIdentifier("itsOn"); + validateToken(IToken.tASSIGN); + validateInteger("7"); + validateToken(IToken.tSEMI); + validateEOF(); + initializeScanner("#if defined( NOTHING ) \nint x = NOTHING;\n#endif"); + validateEOF(); + } + + public void testQuickScan() throws Exception { + initializeScanner("#if X + 5 < 7\n int found = 1;\n#endif", ParserMode.QUICK_PARSE); + validateToken(IToken.t_int); + validateIdentifier("found"); + validateToken(IToken.tASSIGN); + validateInteger("1"); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#if 0\n int error = 666;\n#endif"); + validateEOF(); + + } + + public void testOtherPreprocessorCommands() throws Exception { + initializeScanner("#\n#\t\n#define MAX_SIZE 1024\n#\n# "); + validateEOF(); + validateDefinition("MAX_SIZE", "1024"); + + for (int i = 0; i < 4; ++i) { + switch (i) { + case 0: + initializeScanner("# ape"); + break; + case 1: + initializeScanner("# #"); + break; + case 2: + initializeScanner("# 32"); + break; + case 3: + initializeScanner("# defines"); + break; + } + + validateEOF(); + // These are no longer fScanner exceptions, the are simply ignored. + } + } + + public void validateAllDefinitions(TableRow row) { + int winner = row.selectWinner(); + int rowLength = row.getLength(); + for (int i = 0; i <= rowLength; ++i) { + if (i == winner) + validateDefinition(row.symbolName(i), row.symbolValue(i)); + else + validateAsUndefined(row.symbolName(i)); + } + } + + public void testBug36287() throws Exception { + initializeScanner("X::X( const X & rtg_arg ) : U( rtg_arg ) , Z( rtg_arg.Z ) , er( rtg_arg.er ){}"); + validateIdentifier("X"); + validateToken(IToken.tCOLONCOLON); + validateIdentifier("X"); + validateToken(IToken.tLPAREN); + validateToken(IToken.t_const); + validateIdentifier("X"); + validateToken(IToken.tAMPER); + validateIdentifier("rtg_arg"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tCOLON); + validateIdentifier("U"); + validateToken(IToken.tLPAREN); + validateIdentifier("rtg_arg"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tCOMMA); + validateIdentifier("Z"); + validateToken(IToken.tLPAREN); + validateIdentifier("rtg_arg"); + validateToken(IToken.tDOT); + validateIdentifier("Z"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tCOMMA); + validateIdentifier("er"); + validateToken(IToken.tLPAREN); + validateIdentifier("rtg_arg"); + validateToken(IToken.tDOT); + validateIdentifier("er"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tLBRACE); + validateToken(IToken.tRBRACE); + validateEOF(); + + initializeScanner("foo.*bar"); + validateIdentifier("foo"); + validateToken(IToken.tDOTSTAR); + validateIdentifier("bar"); + validateEOF(); + + initializeScanner("foo...bar"); + validateIdentifier("foo"); + validateToken(IToken.tELLIPSIS); + validateIdentifier("bar"); + validateEOF(); + } + + public void testBug35892() throws Exception { + initializeScanner("'c'"); + validateChar("c"); + validateEOF(); + } + + public void testBug36045() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append('"'); + buffer.append('\\'); + buffer.append('"'); + buffer.append('"'); + + buffer.append('"'); + buffer.append('\\'); + buffer.append('\\'); + buffer.append('"'); + buffer.append("\n\n"); + initializeScanner(buffer.toString()); + validateString("\\\"\\\\"); + } + + public void testConditionalWithBraces() throws Exception { + for (int i = 0; i < 4; ++i) { + initializeScanner( + "int foobar(int a) { if(a == 0) {\n"+ + "#ifdef THIS\n"+ + "} else {}\n"+ + "#elif THAT\n" + + "} else {}\n"+ + "#endif\n"+ + "return 0;}"); + switch (i) { + case 0: + addDefinition("THIS", "1"); + addDefinition("THAT", "1"); + break; + case 1: + addDefinition("THIS", "1"); + addDefinition("THAT", "0"); + break; + case 2: + addDefinition("THAT", "1"); + break; + case 3: + addDefinition("THAT", "0"); + break; + } + + validateToken(IToken.t_int); + validateIdentifier("foobar"); + validateToken(IToken.tLPAREN); + validateToken(IToken.t_int); + validateIdentifier("a"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tLBRACE); + validateToken(IToken.t_if); + validateToken(IToken.tLPAREN); + validateIdentifier("a"); + validateToken(IToken.tEQUAL); + validateInteger("0"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tLBRACE); + + if (i <= 1) { + validateToken(IToken.tRBRACE); + validateToken(IToken.t_else); + validateToken(IToken.tLBRACE); + validateToken(IToken.tRBRACE); + } + + if (i == 2) { + validateToken(IToken.tRBRACE); + validateToken(IToken.t_else); + validateToken(IToken.tLBRACE); + validateToken(IToken.tRBRACE); + } + + validateToken(IToken.t_return); + validateInteger("0"); + validateToken(IToken.tSEMI); + validateToken(IToken.tRBRACE); + validateEOF(); + } + + } + + public void testNestedRecursiveDefines() throws Exception { + initializeScanner("#define C B A\n#define B C C\n#define A B\nA"); + + validateIdentifier("B"); + validateDefinition("A", "B"); + validateDefinition("B", "C C"); + validateDefinition("C", "B A"); + validateIdentifier("A"); + validateIdentifier("B"); + validateIdentifier("A"); + validateEOF(); + } + + public void testBug36316() throws Exception { + initializeScanner("#define A B->A\nA"); + + validateIdentifier("B"); + validateDefinition("A", "B->A"); + validateToken(IToken.tARROW); + validateIdentifier("A"); + validateEOF(); + } + + public void testBug36434() throws Exception { + initializeScanner("#define X(Y)\nX(55)"); + validateEOF(); + /*IMacroDescriptor macro = fScanner.getDefinition( "X" ); + assertNotNull( macro ); + assertEquals( macro.getParameters().length, 1 ); + assertEquals( macro.getParameters()[0], "Y" ); + assertEquals( macro.getTokenizedExpansion().length, 0 );*/ + } + + public void testBug36047() throws Exception { + StringWriter writer = new StringWriter(); + writer.write("# define MAD_VERSION_STRINGIZE(str) #str\n"); + writer + .write("# define MAD_VERSION_STRING(num) MAD_VERSION_STRINGIZE(num)\n"); + writer + .write("# define MAD_VERSION MAD_VERSION_STRING(MAD_VERSION_MAJOR) \".\" \\\n"); + writer + .write(" MAD_VERSION_STRING(MAD_VERSION_MINOR) \".\" \\\n"); + writer + .write(" MAD_VERSION_STRING(MAD_VERSION_PATCH) \".\" \\\n"); + writer + .write(" MAD_VERSION_STRING(MAD_VERSION_EXTRA)\n"); + writer.write("# define MAD_VERSION_MAJOR 2\n"); + writer.write("# define MAD_VERSION_MINOR 1\n"); + writer.write("# define MAD_VERSION_PATCH 3\n"); + writer.write("# define MAD_VERSION_EXTRA boo\n"); + writer.write("MAD_VERSION\n"); + initializeScanner(writer.toString()); + + validateString("2.1.3.boo"); + + validateEOF(); + } + + public void testBug36475() throws Exception { + StringWriter writer = new StringWriter(); + writer.write(" \"A\" \"B\" \"C\" "); + + initializeScanner(writer.toString()); + + validateString("ABC"); + validateEOF(); + } + + public void testBug36509() throws Exception { + StringWriter writer = new StringWriter(); + writer + .write("#define debug(s, t) printf(\"x\" # s \"= %d, x\" # t \"= %s\", \\\n"); + writer.write(" x ## s, x ## t) \n"); + writer.write("debug(1, 2);"); + + initializeScanner(writer.toString()); + //printf("x1=%d, x2= %s", x1, x2); + validateIdentifier("printf"); + validateToken(IToken.tLPAREN); + validateString("x1= %d, x2= %s"); + validateToken(IToken.tCOMMA); + validateIdentifier("x1"); + validateToken(IToken.tCOMMA); + validateIdentifier("x2"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + } + + public void testBug36695() throws Exception { + StringWriter writer = new StringWriter(); + writer.write("\'\\4\' \'\\n\'"); + initializeScanner(writer.toString()); + + validateChar("\\4"); + validateChar("\\n"); + validateEOF(); + } + + public void testBug36521() throws Exception { + StringWriter writer = new StringWriter(); + writer.write("#define str(s) # s\n"); + writer.write("fputs(str(strncmp(\"abc\\0d\", \"abc\", \'\\4\')\n"); + writer.write(" == 0), s);\n"); + + initializeScanner(writer.toString()); + validateIdentifier("fputs"); + validateToken(IToken.tLPAREN); + + //TODO as in 36701B, whitespace is not properly replaced inside the string, ok for now. + validateString("strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') == 0"); + //validateString("strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') == 0"); + + validateToken(IToken.tCOMMA); + validateIdentifier("s"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + } + + public void testBug36770() throws Exception { + StringWriter writer = new StringWriter(); + writer.write("#define A 0\n"); + writer.write("#if ( A == 1 )\n"); + writer.write("# define foo 1\n"); + writer.write("#else\n"); + writer.write("# define foo 2\n"); + writer.write("#endif\n"); + writer.write("foo\n"); + initializeScanner(writer.toString()); + validateInteger("2"); + validateEOF(); + } + + public void testBug36816() throws Exception { + initializeScanner("#include \"foo.h", ParserMode.QUICK_PARSE); + validateEOF(); + validateProblemCount(2); + + initializeScanner("#include \n"); + + initializeScanner(buffer.toString()); + validateEOF(); + validateProblemCount(1); // file does not exist + IASTPreprocessorIncludeStatement[] includes= fLocationResolver.getIncludeDirectives(); + assertEquals(1, includes.length); + assertEquals("stdio.h", includes[0].getName().toString()); + } + + public void testBug46402() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define X 5\n"); + buffer.append("#if defined( X )\n"); + buffer.append("// blah\n"); + buffer.append("#elif Y > 5 \n"); + buffer.append("// coo\n"); + buffer.append("#endif\n"); + initializeScanner(buffer.toString(), ParserMode.COMPLETE_PARSE); + validateEOF(); + } + + public void testBug50821() throws Exception { + initializeScanner("\'\n\n\n", ParserMode.QUICK_PARSE); + fScanner.nextToken(); + validateProblemCount(1); + } + + public void test54778() throws Exception { + initializeScanner("#if 1 || 0 < 3 \n printf \n #endif\n"); + validateIdentifier("printf"); + validateEOF(); + initializeScanner("#if !defined FOO || FOO > 3\nprintf\n#endif\n"); + validateIdentifier("printf"); + validateEOF(); + initializeScanner("#if !defined FOO || FOO < 3\nprintf\n#endif\n"); + validateIdentifier("printf"); + validateEOF(); + } + + public void test68229() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define COUNT 0 \n"); + writer.write("1 \n"); + writer.write("#if COUNT \n"); + writer.write(" 2 \n"); + writer.write("#endif \n"); + writer.write("3 \n"); + + initializeScanner(writer.toString()); + + IToken t1 = fScanner.nextToken(); + IToken t3 = fScanner.nextToken(); + + assertEquals(t1.getImage(), "1"); + assertEquals(t3.getImage(), "3"); + assertEquals(t1.getNext(), t3); + validateEOF(); + + writer = new StringWriter(); + writer.write("#define FOO( x ) x \n"); + writer.write("1 FOO( 2 ) 3 \n"); + + initializeScanner(writer.toString()); + t1 = fScanner.nextToken(); + IToken t2 = fScanner.nextToken(); + t3 = fScanner.nextToken(); + validateEOF(); + + assertEquals(t1.getImage(), "1"); + assertEquals(t2.getImage(), "2"); + assertEquals(t3.getImage(), "3"); + + assertEquals(t1.getNext(), t2); + } + + public void testBug56517() throws Exception { + Writer writer = new StringWriter(); + writer.write("#if 0 \n"); + writer.write("char * x = \"#boo\";\n"); + writer.write("#endif\n"); + initializeScanner(writer.toString()); + validateEOF(); + } + + public void testBug36770B() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define A 0\n"); + writer.write("#if ( A == 1 )\n"); + writer.write("# define foo\n"); + writer.write("#else\n"); + writer.write("# define bar\n"); + writer.write("#endif\n"); + initializeScanner(writer.toString(), ParserMode.QUICK_PARSE); + validateEOF(); + validateDefinition("A", 0); + validateDefinition("bar", ""); + + } + + public void testBug47797() throws Exception { + initializeScanner("\"\\uABCD\" \'\\uABCD\' \\uABCD_ident \\u001A01BC_ident ident\\U01AF ident\\u01bc00AF"); + validateString("\\uABCD"); + validateChar("\\uABCD"); + validateIdentifier("\\uABCD_ident"); + validateIdentifier("\\u001A01BC_ident"); + validateIdentifier("ident\\U01AF"); + validateIdentifier("ident\\u01bc00AF"); + validateEOF(); + } + + public void testBug59768() throws Exception { + initializeScanner("#define A A\nA"); + validateIdentifier("A"); + validateEOF(); + /*IMacroDescriptor d = fScanner.getDefinition( "A"); + assertTrue( d.isCircular() );*/ + } + + public void testBug60764() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define P a,b\n"); + writer.write("#define M(x) M1(x)\n"); + writer.write("#define M1(x,y) #x #y\n"); + writer.write("M(P)\n"); + initializeScanner(writer.toString()); + validateString("ab"); + validateEOF(); + } + + public void testBug62042() throws Exception { + initializeScanner("0x", ParserMode.QUICK_PARSE); + validateInteger("0x"); // to me this is a valid number + validateEOF(); + } + + public void testBug61968() throws Exception { + Writer writer = new StringWriter(); + writer.write("unsigned int ui = 2172748163; //ok \n"); + writer.write("int big = 999999999999999;//ok \n"); + writer.write("void main() { \n"); + writer.write("caller(4); //ok\n"); + writer + .write("caller(2172748163);//causes java.lang.NumberFormatException \n"); + writer + .write("caller(999999999999999); //also causes NumberFormatException \n"); + writer.write("}\n"); + initializeScanner(writer.toString(), ParserMode.QUICK_PARSE); + fullyTokenize(); + validateProblemCount(0); + } + + public void testBug62378() throws Exception { + initializeScanner("\"\\?\\?<\""); + validateString("\\?\\?<"); + } + + public void testBug62384() throws Exception { + initializeScanner("18446744073709551615LL"); + validateInteger("18446744073709551615LL"); + } + + public void testBug62390() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define f(x) x\n"); + writer.write("#if f(\n"); + writer.write("5) == 5\n"); + writer.write("true1\n"); + writer.write("#endif\n"); + writer.write("#if A\n"); + writer.write("#elif f(\n"); + writer.write("5) == 5\n"); + writer.write("true2\n"); + writer.write("#endif\n"); + writer.write("#undef f\n"); + writer.write("#define f(x) \"A0I70_001.h\"\n"); + writer.write("#include f(\n"); + writer.write("5\n"); + writer.write(")\n"); + writer.write("#undef f\n"); + writer.write("#define f(x) 1467\n"); + writer.write("#line f(\n"); + writer.write("5\n"); + writer.write(")\n"); + writer.write("#pragma f(\n"); + writer.write("5\n"); + writer.write(")\n"); + writer.write("}\n"); + initializeScanner(writer.toString(), ParserMode.QUICK_PARSE); + fullyTokenize(); + } + + public void testBug62009() throws Exception { + initializeScanner("#define def(x) (x#)\ndef(orange)\n", ParserMode.QUICK_PARSE); + fullyTokenize(); + validateProblemCount(1); + } + + public void testBug61972() throws Exception { + initializeScanner("#define DEF1(A1) A1\n#define DEF2 DEF1(DEF2)\nDEF2;"); + validateIdentifier("DEF2"); + validateToken(IToken.tSEMI); + validateEOF(); + } + + public void testBug64268() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define BODY \\\n"); + writer.write(" { \\\n"); + writer.write(" /* this multi-line comment messes \\\n"); + writer.write(" up the parser. */ }\n"); + writer.write("BODY "); + initializeScanner(writer.toString()); + validateToken(IToken.tLBRACE); + validateToken(IToken.tRBRACE); + validateEOF(); + } + + public void testUndef() throws Exception { + initializeScanner("#define A 5\n" + + "#define B 10\n" + + "#undef A\n" + + "A B"); + validateIdentifier("A"); + validateInteger("10"); + validateEOF(); + } + + public void testWackyFunctionMacros() throws Exception { + initializeScanner("#define A(X) hi##X\n" + + "#define B(Y) A(Y)\n" + + "B(there)"); + validateIdentifier("hithere"); + validateEOF(); + } + + public void testSlashes() throws Exception { + initializeScanner("__q / __n"); + validateIdentifier("__q"); + validateToken(IToken.tDIV); + validateIdentifier("__n"); + validateEOF(); + } + + public void testStringify() throws Exception { + initializeScanner("#define xS(s) #s\n#define S(s) xS(s)\n#define X hi\nS(X)"); + validateString("hi"); + validateEOF(); + } + + public void testWideToNarrowConcatenation() throws Exception { + initializeScanner("\"ONE\" L\"TWO\""); + validateLString("ONETWO"); + validateEOF(); + } + + public void testEmptyIncludeDirective() throws Exception { + initializeScanner("#include \n#include \n"); + validateEOF(); + IASTPreprocessorIncludeStatement[] includes= fLocationResolver.getIncludeDirectives(); + assertEquals(1, includes.length); + assertEquals("foo.h", includes[0].getName().toString()); + } + + public void testBug69412() throws Exception { + initializeScanner("\'\\\\\'", ParserMode.COMPLETE_PARSE); + validateChar("\\\\"); + validateEOF(); + validateProblemCount(0); + } + + public void testBug70072() throws Exception { + initializeScanner("#if 1/0\nint i;\n#elif 2/0\nint j;\n#endif\nint k;\n"); + fullyTokenize(); + } + + public void testBug70261() throws Exception { + initializeScanner("0X0"); + validateInteger("0X0"); + } + + public void testBug62571() throws Exception { + StringBuffer buffer = new StringBuffer("#define J(X,Y) X##Y\n"); + buffer.append("J(A,1Xxyz)\n"); + buffer.append("J(B,1X1X1Xxyz)\n"); + buffer.append("J(C,0Xxyz)\n"); + buffer.append("J(CC,0Xxyz)\n"); + buffer.append("J(D,0xxyz)\n"); + buffer.append("J(E,0x0x0xxyz)\n"); + initializeScanner(buffer.toString()); + validateIdentifier("A1Xxyz"); + validateIdentifier("B1X1X1Xxyz"); + validateIdentifier("C0Xxyz"); + validateIdentifier("CC0Xxyz"); + validateIdentifier("D0xxyz"); + validateIdentifier("E0x0x0xxyz"); + } + + public void testBug69134() throws Exception { + Writer writer = new StringWriter(); + writer.write("# ifdef YYDEBUG\n"); + writer.write("if (yyDebug) {\n"); + writer.write("(void) fprintf (yyTrace,\n"); + writer.write("\" # |Position|State|Mod|Lev|Action |Terminal and Lookahead or Rule\\n\");\n"); + writer.write("yyNl ();\n"); + writer.write("}\n"); + writer.write("# endif\n"); + initializeScanner(writer.toString()); + fullyTokenize(); + validateProblemCount(0); + } + + public void testBug70073() throws Exception { + StringBuffer buffer = new StringBuffer( + "#if CONST \n #endif \n #elif CONST \n int"); + final List problems = new ArrayList(); + ISourceElementRequestor requestor = new NullSourceElementRequestor() { + public boolean acceptProblem(IProblem problem) { + problems.add(problem); + return super.acceptProblem(problem); + } + }; + initializeScanner(buffer.toString()); + validateToken(IToken.t_int); + validateProblemCount(1); + } + + public void testBug73652() throws Exception { + StringWriter writer = new StringWriter(); + writer.write("#define DoSuperMethodA IDoSuperMethodA\n"); + writer + .write("#define IDoSuperMethodA(a,b,c) IIntuition->IDoSuperMethodA(a,b,c)\n"); + writer.write("DoSuperMethodA(0,0,0);\n"); + + initializeScanner(writer.toString()); + + validateIdentifier("IIntuition"); + validateToken(IToken.tARROW); + validateIdentifier("IDoSuperMethodA"); + validateToken(IToken.tLPAREN); + validateInteger("0"); + validateToken(IToken.tCOMMA); + validateInteger("0"); + validateToken(IToken.tCOMMA); + validateInteger("0"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + } + + public void testBug73652_2() throws Exception { + StringWriter writer = new StringWriter(); + writer.write("#define DoSuperMethodA DoSuperMethodB //doobalie\n"); + writer.write("#define DoSuperMethodB DoSuperMethodC /*oogalie*/ \n"); + writer.write("#define DoSuperMethodC IDoSuperMethodA \\\n\n"); + writer + .write("#define IDoSuperMethodA(a,b,c) IIntuition->IDoSuperMethodA(a,b,c)\n"); + writer.write("DoSuperMethodA (0,0,0);\n"); + + initializeScanner(writer.toString()); + + validateIdentifier("IIntuition"); + validateToken(IToken.tARROW); + validateIdentifier("IDoSuperMethodA"); + validateToken(IToken.tLPAREN); + validateInteger("0"); + validateToken(IToken.tCOMMA); + validateInteger("0"); + validateToken(IToken.tCOMMA); + validateInteger("0"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + + } + + public void testBug72997() throws Exception { + initializeScanner("'\\\\'"); + validateChar("\\\\"); + validateEOF(); + } + + public void testBug72725() throws Exception { + for (int i = 0; i < 2; ++i) { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define a \\"); + if (i == 0) + buffer.append("\r"); + buffer.append("\n"); + buffer.append("long macro stuff"); + if (i == 0) + buffer.append("\r"); + buffer.append("\n"); + + initializeScanner(buffer.toString()); + validateEOF(); + validateProblemCount(0); + } + } + + public void testBug72506() throws Exception { + StringWriter writer = new StringWriter(); + writer.write("#define INCFILE(x) ver ## x\n"); + writer.write("#define xstr(x) str(x)\n"); + writer.write("#define str(x) #x\n"); + writer.write("xstr(INCFILE(2).h)\n"); + + initializeScanner(writer.toString()); + validateString("ver2.h"); + validateEOF(); + } + + public void testBug72506_2() throws Exception { + StringWriter writer = new StringWriter(); + writer.write("#define str(x) #x\n"); + writer.write("#define A B\n"); + writer.write("#define B A\n"); + writer.write("str(B)\n"); + + initializeScanner(writer.toString()); + validateString("B"); + validateEOF(); + } + + public void testMacroPastingError() throws Exception { + StringWriter writer = new StringWriter(); + writer.write("#define m(expr) \\\r\n"); + writer.write(" foo( #expr ) \r\n"); + + initializeScanner(writer.toString()); + validateEOF(); + validateProblemCount(0); + } + + public void testBug74176() throws Exception { + initializeScanner("#define MYSTRING \"X Y Z "); + validateEOF(); + + initializeScanner("#define m(b) #"); + validateEOF(); + + initializeScanner("#define m(foo,b) #b"); + validateEOF(); + } + + public void testBug74180() throws Exception { + initializeScanner("true false", ParserLanguage.C); + validateIdentifier("true"); + validateIdentifier("false"); + + initializeScanner("true false", ParserLanguage.CPP); + validateToken(IToken.t_true); + validateToken(IToken.t_false); + } + + public void testBug73492() throws Exception { + String code = "#define PTR void *\n" + + "PTR;\n"; + + int offset = code.indexOf("PTR;") + 3; + initializeScanner(code); + + IToken t = fScanner.nextToken(); + assertEquals(t.getType(), IToken.t_void); + assertEquals(offset, t.getOffset()); + + t = fScanner.nextToken(); + assertEquals(t.getType(), IToken.tSTAR); + assertEquals(offset + 1, t.getOffset()); + + t = fScanner.nextToken(); + assertEquals(t.getType(), IToken.tSEMI); + assertEquals(offset + 2, t.getOffset()); + } + + public void testBug74328() throws Exception { + initializeScanner("\"\";\n"); + validateString(""); + validateToken(IToken.tSEMI); + validateEOF(); + } + + public void testBug72537() throws Exception { + initializeScanner("FOO BAR(boo)"); + fScanner.addMacroDefinition("FOO".toCharArray(), "foo".toCharArray()); + fScanner.addMacroDefinition("BAR(x)".toCharArray(), "x".toCharArray()); + + validateIdentifier("foo"); + validateIdentifier("boo"); + validateEOF(); + } + + public void testBug75083() throws Exception { + String code = "#define blah() { extern foo\n blah()\n"; + initializeScanner(code); + + int idx = code.indexOf("\n blah()") + 2 + 6; + IToken t = fScanner.nextToken(); + assertEquals(IToken.tLBRACE, t.getType()); + assertEquals(idx, t.getOffset()); + assertEquals(idx + 1, t.getEndOffset()); + + t = fScanner.nextToken(); + assertEquals(IToken.t_extern, t.getType()); + assertEquals(idx + 1, t.getOffset()); + assertEquals(idx + 2, t.getEndOffset()); + + t = fScanner.nextToken(); + assertEquals(IToken.tIDENTIFIER, t.getType()); + assertEquals(idx + 2, t.getOffset()); + assertEquals(idx + 3, t.getEndOffset()); + } + + // when fixing 75532 several IProblems were added to ExpressionEvaluator and one to Scanner2, this is to test them + // the problems currently don't get reported when using the DOMScanner + public void testBug75532IProblems() throws Exception { + Writer writer = new StringWriter(); + writer.write("#if 09 == 9\n#endif\n"); // malformed octal + writer.write("#if 1A == 0x1A\n#endif\n"); // malformed decimal + writer.write("#if 0x == 0x0\n#endif\n"); // malformed hex + writer.write("#if 0xAX == 0xA\n#endif\n"); // malformed hex + writer.write("#if 1/0 == 1\n#endif\n"); // division by zero + writer.write("#if defined ( sadf a\n#endif\n"); // missing ')' in defined + writer.write("#if defined ( sadf\n#endif\n"); // missing ')' in defined + writer.write("#if defined ( 2sadf )\n#endif\n"); // illegal identifier in defined + writer.write("#if ( 1 == 1 ? 1\n#endif\n"); // bad conditional expression + writer.write("#if ( \n#endif\n"); // expression syntax error + writer.write("#if @\n#endif\n"); // expression syntax error + writer.write("#if \n#endif\n"); // expression syntax error + writer.write("#if -\n#endif\n"); // expression syntax error + writer.write("#if ( 1 == 1\n#endif\n"); // missing ')' + writer.write("#if 1 = 1\n#endif\n"); // assignment not allowed + + writer.write("int main(int argc, char **argv) {\n"); + writer.write("if ( 09 == 9 )\n"); // added while fixing this bug, IProblem on invalid octal number + writer.write("return 1;\nreturn 0;\n}\n"); + + initializeScanner(writer.toString()); + fullyTokenize(); + IASTProblem[] problems= fLocationResolver.getScannerProblems(); + assertEquals(16, problems.length); + int i= 0; + assertEquals(IProblem.SCANNER_BAD_OCTAL_FORMAT, problems[i].getID() ); + assertEquals(IProblem.SCANNER_BAD_DECIMAL_FORMAT, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_BAD_HEX_FORMAT, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_BAD_HEX_FORMAT, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_DIVIDE_BY_ZERO, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_MISSING_R_PAREN, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_MISSING_R_PAREN, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_ILLEGAL_IDENTIFIER, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_BAD_CONDITIONAL_EXPRESSION,problems[++i].getID() ); + assertEquals(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_MISSING_R_PAREN, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_ASSIGNMENT_NOT_ALLOWED, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_BAD_OCTAL_FORMAT, problems[++i].getID() ); + } + + public void testExpressionEvalProblems() throws Exception { + Writer writer = new StringWriter(); + writer.write(" #if 1 == 1L \n"); + writer.write(" #endif \n"); + + initializeScanner(writer.toString()); + validateEOF(); + validateProblemCount(0); + } + + public void testExpressionEvalProblems_2() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define FOO( a, b ) a##b \n"); + writer.write("#if FOO ( 1, 0 ) == 10 \n"); + writer.write("1 \n"); + writer.write("#endif \n"); + + initializeScanner(writer.toString()); + validateInteger("1"); + validateEOF(); + validateProblemCount(0); + } + + public void testUnExpandedFunctionMacro() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define foo( a ) #a \n"); + writer.write("foo( 1 ) foo \n"); + + initializeScanner(writer.toString()); + validateString("1"); + validateIdentifier("foo"); + validateEOF(); + } + + public void testBug39688A() throws Exception { // test valid IProblems + Writer writer = new StringWriter(); + writer.write("#define decl1(type, ... \\\n ) type var;\n"); + writer.write("decl1(int, x, y, z)\n"); + writer.write("#define decl2(type, args...) type args;"); + writer.write("decl2(int, a, b, c, x, y, z)\n"); + writer.write("#define decl3(type, args...) \\\n type args;"); + writer.write("decl3(int, a, b, c, x, y)\n"); + writer.write("#define decl4(type, args... \\\n ) type args;"); + writer.write("decl4(int, a, b, z)\n"); + writer.write("#define decl5(type, ...) type __VA_ARGS__;"); + writer.write("decl5(int, z)\n"); + writer.write("#define decl6(type, ... \\\n) type __VA_ARGS__;"); + writer.write("decl6(int, a, b, c, x)\n"); + writer.write("#define foo(a) a __VA_ARGS__;\n"); // C99: 6.10.3.5 this should produce an IProblem + writer.write("#define foo2(a) a #__VA_ARGS__;\n"); // C99: 6.10.3.5 this should produce an IProblem + + initializeScanner(writer.toString()); + fullyTokenize(); + IASTProblem[] problems= fLocationResolver.getScannerProblems(); + assertEquals(2, problems.length); + assertTrue(problems[0].getID() == IProblem.PREPROCESSOR_INVALID_VA_ARGS); + assertTrue(problems[1].getID() == IProblem.PREPROCESSOR_MACRO_PASTING_ERROR); + } + + public void testBug39688B() throws Exception { // test C99 + Writer writer = new StringWriter(); + writer.write("#define debug(...) fprintf(stderr, __VA_ARGS__)\n"); + writer.write("#define showlist(...) puts(#__VA_ARGS__)\n"); + writer + .write("#define report(test, ...) ((test)?puts(#test):\\\n printf(__VA_ARGS__))\n"); + writer.write("int main() {\n"); + writer.write("debug(\"Flag\");\n"); + writer.write("debug(\"X = %d\\n\", x);\n"); + writer.write("showlist(The first, second, and third items.);\n"); + writer.write("report(x>y, \"x is %d but y is %d\", x, y);\n"); + writer.write("return 0; }\n"); + + initializeScanner(writer.toString()); + fullyTokenize(); + validateProblemCount(0); + + Map defs = fScanner.getDefinitions(); + assertTrue(defs.containsKey("debug")); + assertTrue(defs.containsKey("showlist")); + assertTrue(defs.containsKey("report")); + IMacroBinding debug = (IMacroBinding) defs.get("debug"); + assertTrue(new String(debug.getParameterPlaceholderList()[0]).equals("__VA_ARGS__")); + assertEquals("fprintf(stderr, __VA_ARGS__)", new String(debug.getExpansion())); + + IMacroBinding showlist = (IMacroBinding) defs.get("showlist"); + assertTrue(new String(showlist.getParameterPlaceholderList()[0]).equals("__VA_ARGS__")); + assertTrue(new String(showlist.getExpansion()) + .equals("puts(#__VA_ARGS__)")); + IMacroBinding report = (IMacroBinding) defs.get("report"); + assertTrue(new String(report.getParameterPlaceholderList()[0]).equals("test")); + assertTrue(new String(report.getParameterPlaceholderList()[1]).equals("__VA_ARGS__")); + assertTrue(new String(report.getExpansion()) + .equals("((test)?puts(#test): printf(__VA_ARGS__))")); + + check39688Tokens(writer); + } + + public void testBug39688C() throws Exception { // test GCC + Writer writer = new StringWriter(); + writer.write("#define debug(vars...) fprintf(stderr, vars)\n"); + writer.write("#define showlist(vars...) puts(#vars)\n"); + writer + .write("#define report(test, vars...) ((test)?puts(#test):\\\n printf(vars))\n"); + writer.write("int main() {\n"); + writer.write("debug(\"Flag\");\n"); + writer.write("debug(\"X = %d\\n\", x);\n"); + writer.write("showlist(The first, second, and third items.);\n"); + writer.write("report(x>y, \"x is %d but y is %d\", x, y);\n"); + writer.write("return 0; }\n"); + + initializeScanner(writer.toString()); + fullyTokenize(); + validateProblemCount(0); + + Map defs = fScanner.getDefinitions(); + assertTrue(defs.containsKey("debug")); + assertTrue(defs.containsKey("showlist")); + assertTrue(defs.containsKey("report")); + IMacroBinding debug = (IMacroBinding) defs.get("debug"); + assertTrue(new String(debug.getParameterPlaceholderList()[0]).equals("vars")); + assertTrue(new String(debug.getExpansion()) + .equals("fprintf(stderr, vars)")); + IMacroBinding showlist = (IMacroBinding) defs.get("showlist"); + assertTrue(new String(showlist.getParameterPlaceholderList()[0]).equals("vars")); + assertTrue(new String(showlist.getExpansion()).equals("puts(#vars)")); + IMacroBinding report = (IMacroBinding) defs.get("report"); + assertTrue(new String(report.getParameterPlaceholderList()[0]).equals("test")); + assertTrue(new String(report.getParameterPlaceholderList()[1]).equals("vars")); + assertTrue(new String(report.getExpansion()) + .equals("((test)?puts(#test): printf(vars))")); + + check39688Tokens(writer); + } + + private void check39688Tokens(Writer writer) throws Exception { + initializeScanner(writer.toString()); + + validateToken(IToken.t_int); + validateIdentifier("main"); + validateToken(IToken.tLPAREN); + validateToken(IToken.tRPAREN); + validateToken(IToken.tLBRACE); + + validateIdentifier("fprintf"); + validateToken(IToken.tLPAREN); + validateIdentifier("stderr"); + validateToken(IToken.tCOMMA); + validateString("Flag"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + + validateIdentifier("fprintf"); + validateToken(IToken.tLPAREN); + validateIdentifier("stderr"); + validateToken(IToken.tCOMMA); + validateString("X = %d\\n"); + validateToken(IToken.tCOMMA); + validateIdentifier("x"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + + validateIdentifier("puts"); + validateToken(IToken.tLPAREN); + validateString("The first, second, and third items."); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + + validateToken(IToken.tLPAREN); + validateToken(IToken.tLPAREN); + validateIdentifier("x"); + validateToken(IToken.tGT); + validateIdentifier("y"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tQUESTION); + validateIdentifier("puts"); + validateToken(IToken.tLPAREN); + validateString("x>y"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tCOLON); + validateIdentifier("printf"); + validateToken(IToken.tLPAREN); + validateString("x is %d but y is %d"); + validateToken(IToken.tCOMMA); + validateIdentifier("x"); + validateToken(IToken.tCOMMA); + validateIdentifier("y"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + + validateToken(IToken.t_return); + validateInteger("0"); + validateToken(IToken.tSEMI); + validateToken(IToken.tRBRACE); + + validateEOF(); + } + + public void testMacroArgumentExpansion() throws Exception { + Writer writer = new StringWriter(); + writer + .write("#define g_return( expr ) ( expr ) \n"); + writer + .write("#define ETH( obj ) ( CHECK( (obj), boo ) ) \n"); + writer + .write("#define CHECK CHECK_INSTANCE \n"); + writer + .write("#define CHECK_INSTANCE( instance, type ) (foo((instance), (type))) \n"); + writer + .write("g_return( ETH(ooga) ) \n"); + + initializeScanner(writer.toString()); + + validateToken(IToken.tLPAREN); + validateToken(IToken.tLPAREN); + validateToken(IToken.tLPAREN); + validateIdentifier("foo"); + validateToken(IToken.tLPAREN); + validateToken(IToken.tLPAREN); + validateToken(IToken.tLPAREN); + validateIdentifier("ooga"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tRPAREN); + validateToken(IToken.tCOMMA); + validateToken(IToken.tLPAREN); + validateIdentifier("boo"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tRPAREN); + validateToken(IToken.tRPAREN); + validateToken(IToken.tRPAREN); + validateToken(IToken.tRPAREN); + } + + public void testBug75956() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define ROPE( name ) name##Alloc\n"); + writer.write("#define _C 040 \n"); + writer.write("ROPE( _C ) \n"); + + initializeScanner(writer.toString()); + validateIdentifier("_CAlloc"); + validateEOF(); + + writer = new StringWriter(); + writer.write("#define ROPE( name ) Alloc ## name \n"); + writer.write("#define _C 040 \n"); + writer.write("ROPE( _C ) \n"); + + initializeScanner(writer.toString()); + validateIdentifier("Alloc_C"); + validateEOF(); + + writer = new StringWriter(); + writer.write("#define ROPE( name ) name##Alloc\n"); + writer.write("#define _C 040 \n"); + writer.write("#define _CAlloc ooga \n"); + writer.write("ROPE( _C ) \n"); + + initializeScanner(writer.toString()); + validateIdentifier("ooga"); + validateEOF(); + } + + public void testUnExpandedFunctionMacros() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define ETH(x) x \n"); + writer.write("#define E ETH \n"); + writer.write("ETH( c ), ETH, E; \n"); + initializeScanner(writer.toString()); + validateIdentifier("c"); + validateToken(IToken.tCOMMA); + validateIdentifier("ETH"); + validateToken(IToken.tCOMMA); + validateIdentifier("ETH"); + validateToken(IToken.tSEMI); + validateEOF(); + } + + public void testBug79490A() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define TEST 'n'\n"); + writer.write("#if TEST == 'y'\n"); + writer.write("#define TRUE 1\n"); + writer.write("#else\n"); + writer.write("#define FALSE 1\n"); + writer.write("#endif\n"); + initializeScanner(writer.toString()); + validateEOF(); + validateDefinition("TEST", "'n'"); + validateDefinition("FALSE", "1"); + } + + public void testBug79490B() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define TEST 'y'\n"); + writer.write("#if TEST == 'y'\n"); + writer.write("#define TRUE 1\n"); + writer.write("#else\n"); + writer.write("#define FALSE 1\n"); + writer.write("#endif\n"); + initializeScanner(writer.toString()); + validateEOF(); + validateDefinition("TEST", "'y'"); + validateDefinition("TRUE", "1"); + } + + public void testBug102568A() throws Exception { + initializeScanner("///*\r\nint x;\r\n"); + validateToken(IToken.t_int); + validateIdentifier("x"); + validateToken(IToken.tSEMI); + validateEOF(); + } + + public void testBug102568B() throws Exception { + initializeScanner("// bla some thing /* ... \r\nint x;\r\n"); + validateToken(IToken.t_int); + validateIdentifier("x"); + validateToken(IToken.tSEMI); + validateEOF(); + } + + public void testbug84270() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define h g( ~\n"); + writer.write("#define g f\n"); + writer.write("#define f(a) f(x * (a))\n"); + writer.write("h 5) \n"); + initializeScanner(writer.toString()); + fullyTokenize(); + } + + public void testBug107150() throws Exception { + Writer writer = new StringWriter(); + writer.write("#define FUNC_PROTOTYPE_PARAMS(list) list\r\n"); + writer.write("int func2 FUNC_PROTOTYPE_PARAMS\r\n"); + writer.write("((int arg1)){\r\n"); + writer.write(" return 0;\r\n"); + writer.write("}\r\n"); + initializeScanner(writer.toString()); + validateToken(IToken.t_int); + validateIdentifier("func2"); + validateToken(IToken.tLPAREN); + validateToken(IToken.t_int); + validateIdentifier("arg1"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tLBRACE); + validateToken(IToken.t_return); + validateInteger("0"); + validateToken(IToken.tSEMI); + validateToken(IToken.tRBRACE); + validateEOF(); + + writer = new StringWriter(); + writer.write("#define FUNC_PROTOTYPE_PARAMS(list) list\n"); + writer.write("int func2 FUNC_PROTOTYPE_PARAMS\n"); + writer.write("((int arg1)){\n"); + writer.write(" return 0;\n"); + writer.write("}\n"); + initializeScanner(writer.toString()); + validateToken(IToken.t_int); + validateIdentifier("func2"); + validateToken(IToken.tLPAREN); + validateToken(IToken.t_int); + validateIdentifier("arg1"); + validateToken(IToken.tRPAREN); + validateToken(IToken.tLBRACE); + validateToken(IToken.t_return); + validateInteger("0"); + validateToken(IToken.tSEMI); + validateToken(IToken.tRBRACE); + validateEOF(); + } + + public void testBug126136() throws Exception { + StringBuffer buffer = new StringBuffer("#define C C\n"); + buffer.append("#if !C\n"); + buffer.append("true\n"); + buffer.append("#endif\n"); + initializeScanner(buffer.toString(), ParserLanguage.CPP); + fullyTokenize(); + } + + public void testBug156137() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#if (3 % 2 == 1) \n"); + buffer.append("C \n"); + buffer.append("#endif \n"); + + initializeScanner(buffer.toString()); + validateIdentifier("C"); + validateEOF(); + } + + public void testBug162214() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#ifdef xxx // is not defined \n"); + buffer.append("A \n"); + buffer.append("#endif \n"); + buffer.append("B \n"); + + initializeScanner(buffer.toString()); + validateIdentifier("B"); + validateEOF(); + + buffer.setLength(0); + buffer.append("#ifdef xxx //* is not defined \n"); + buffer.append("A \n"); + buffer.append("#endif \n"); + buffer.append("B \n"); + + initializeScanner(buffer.toString()); + validateIdentifier("B"); + validateEOF(); + } + + public void testBug156988() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define a \n"); + buffer.append("#define b \" \n"); + buffer.append("#define c < \n"); + buffer.append("#define d \"\" \n"); + buffer.append("#define e <> \n"); + buffer.append("#define f f \n"); + buffer.append("#define g gg \n"); + buffer.append("#include a \n"); + buffer.append("#include b \n"); + buffer.append("#include c \n"); + buffer.append("#include d \n"); + buffer.append("#include e \n"); + buffer.append("#include f \n"); + buffer.append("#include g \n"); + buffer.append("A \n"); + + initializeScanner(buffer.toString()); + validateIdentifier("A"); + validateEOF(); + } + + public void testBug156988_1() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define a(x) x \n"); + buffer.append("#define b \" \n"); + buffer.append("#define c < \n"); + buffer.append("#define d \"\" \n"); + buffer.append("#define e <> \n"); + buffer.append("#define f f \n"); + buffer.append("#define g gg \n"); + buffer.append("#include a() \n"); + buffer.append("#include a(<) \n"); + buffer.append("#include a(\"\") \n"); + buffer.append("#include a(<>) \n"); + buffer.append("#include a(f) \n"); + buffer.append("#include a(gg) \n"); + buffer.append("#include a(g\\\ng) \n"); + buffer.append("A \n"); + + initializeScanner(buffer.toString()); + validateIdentifier("A"); + validateEOF(); + } + + public void testBug162410() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#pragma message (\"test\") \n"); + buffer.append("a \n"); + initializeScanner(buffer.toString()); + validateIdentifier("a"); + } + + // https://bugs.eclipse.org/bugs/show_bug.cgi?id=180172 + public void testBug180172() throws Exception { + StringBuffer buffer = new StringBuffer(); + String value = "\"https://bugs.eclipse.org/bugs/show_bug.cgi?id=180172\""; + buffer.append("#define bug180172 ").append(value).append(" // bla \n"); + initializeScanner(buffer.toString()); + fullyTokenize(); + validateDefinition("bug180172", value); + } + + // https://bugs.eclipse.org/bugs/show_bug.cgi?id=182180 + public void testBug182180_1() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#ifdef _bug_182180_\n").append( + "printf(\"Hello World /*.ap\\n\");\n").append("#endif\n") + .append("bug182180\n"); + initializeScanner(buffer.toString()); + validateIdentifier("bug182180"); + } + + // https://bugs.eclipse.org/bugs/show_bug.cgi?id=182180 + public void testBug182180_2() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#ifdef _bug_182180_\n").append( + "char c='\"'; printf(\"Hello World /*.ap\\n\");\n").append( + "#endif\n").append("bug182180\n"); + initializeScanner(buffer.toString()); + validateIdentifier("bug182180"); + } + + // https://bugs.eclipse.org/bugs/show_bug.cgi?id=182180 + public void testBug182180_3() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#ifdef _bug_182180_\n").append( + "char c1='\\'',c2='\\\"'; printf(\"Hello World /*.ap\\n\");\n") + .append("#endif\n").append("bug182180\n"); + initializeScanner(buffer.toString()); + validateIdentifier("bug182180"); + } + + // https://bugs.eclipse.org/bugs/show_bug.cgi?id=182180 + public void testBug182180_4() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#ifdef _bug_182180_\n").append( + "printf(\"Hello '\"'World /*.ap\\n\");\n").append("#endif\n") + .append("bug182180\n"); + initializeScanner(buffer.toString()); + validateIdentifier("bug182180"); + } + + // https://bugs.eclipse.org/bugs/show_bug.cgi?id=182180 + public void testBug182180_5() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#ifdef _bug_182180_\n").append( + "printf(\"Hello \\\"World /*.ap\\n\");\n").append("#endif\n") + .append("bug182180\n"); + initializeScanner(buffer.toString()); + validateIdentifier("bug182180"); + } + + // https://bugs.eclipse.org/bugs/show_bug.cgi?id=200830 + public void testBug200830_1() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define string BROKEN\r\n"); + buffer.append("#define macro(inst) (char*)inst\r\n"); + buffer.append("macro(\"string\");\r\n"); + initializeScanner(buffer.toString()); + validateToken(IToken.tLPAREN); + validateToken(IToken.t_char); + validateToken(IToken.tSTAR); + validateToken(IToken.tRPAREN); + validateString("string"); + } + + // https://bugs.eclipse.org/bugs/show_bug.cgi?id=200830 + public void testBug200830_2() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define string BROKEN\r\n"); + buffer.append("#define macro(inst) (char*)inst\r\n"); + buffer.append("macro(\" string \");\r\n"); + initializeScanner(buffer.toString()); + validateToken(IToken.tLPAREN); + validateToken(IToken.t_char); + validateToken(IToken.tSTAR); + validateToken(IToken.tRPAREN); + validateString(" string "); + } + + // https://bugs.eclipse.org/bugs/show_bug.cgi?id=200830 + public void testBug200830_3() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define string BROKEN\r\n"); + buffer.append("#define macro(inst) (char*)inst\r\n"); + buffer.append("macro(\"\\\"string \");\r\n"); + initializeScanner(buffer.toString()); + validateToken(IToken.tLPAREN); + validateToken(IToken.t_char); + validateToken(IToken.tSTAR); + validateToken(IToken.tRPAREN); + validateString("\\\"string "); + } + + // https://bugs.eclipse.org/bugs/show_bug.cgi?id=200830 + public void testBug200830_4() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define s B\r\n"); + buffer.append("#define macro(inst) (char*)inst\r\n"); + buffer.append("macro('s');\r\n"); + initializeScanner(buffer.toString()); + validateToken(IToken.tLPAREN); + validateToken(IToken.t_char); + validateToken(IToken.tSTAR); + validateToken(IToken.tRPAREN); + validateChar("s"); + } + + public void testBug185120_1() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define TEST_DEFINE 1UL\n"); + buffer.append("#if TEST_DEFINE != 1UL\n"); + buffer.append("-\n"); + buffer.append("#else\n"); + buffer.append("+\n"); + buffer.append("#endif\n"); + initializeScanner(buffer.toString()); + validateToken(IToken.tPLUS); + } + + public void testBug185120_2() throws Exception { + StringBuffer buffer = new StringBuffer(); + buffer.append("#define TEST_DEFINE 1LLU\n"); + buffer.append("#if TEST_DEFINE != 1ULL\n"); + buffer.append("-\n"); + buffer.append("#else\n"); + buffer.append("+\n"); + buffer.append("#endif\n"); + initializeScanner(buffer.toString()); + validateToken(IToken.tPLUS); + } +} diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/ScannerTestSuite.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/ScannerTestSuite.java index 2a05f7af5e5..484ae0c42ea 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/ScannerTestSuite.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/ScannerTestSuite.java @@ -6,24 +6,20 @@ * http://www.eclipse.org/legal/epl-v10.html * * Contributors: - * IBM - Initial API and implementation * Markus Schorn (Wind River Systems) - * Emanuel Graf (IFS) *******************************************************************************/ package org.eclipse.cdt.core.parser.tests.scanner; import junit.framework.Test; import junit.framework.TestSuite; -/** - * @author jcamelon - */ public class ScannerTestSuite extends TestSuite { public static Test suite() { TestSuite suite= new ScannerTestSuite(); suite.addTest(LexerTests.suite()); suite.addTest(LocationMapTests.suite()); + suite.addTest(PortedScannerTest.suite()); return suite; } } diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/TestMacro.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/TestMacro.java index de99af2b453..ad256195b6a 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/TestMacro.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/TestMacro.java @@ -66,8 +66,11 @@ final class TestMacro implements IMacroBinding { return result; } - public String getFilename() { - // mstodo Auto-generated method stub - return null; + public char[] getExpansionImage() { + return getExpansion(); + } + + public char[][] getParameterPlaceholderList() { + return getParameterList(); } } \ No newline at end of file diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/IMacroBinding.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/IMacroBinding.java index 4a7a5b836e6..9665faa8631 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/IMacroBinding.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/ast/IMacroBinding.java @@ -16,12 +16,6 @@ package org.eclipse.cdt.core.dom.ast; *

This interface is not intended to be implemented by clients.

*/ public interface IMacroBinding extends IBinding { - - /** - * Returns the expansion of this macro definition. - * @since 5.0 - */ - char[] getExpansion(); /** * Returns true if this is a function-style macro. @@ -33,4 +27,25 @@ public interface IMacroBinding extends IBinding { * Returns the parameter names or null if this is not a function style macro. */ char[][] getParameterList(); + + /** + * Returns the expansion of this macro definition. + * @since 5.0 + */ + char[] getExpansion(); + + /** + * Returns the parameter list where the name of the last parameter is changed if this is a variadic macro, + * or null if this is not a function style macro. + * The parameter '...' will be changed to '__VA_ARGS__' + * Parameters like 'a...' will be changed to 'a'. + * @since 5.0 + */ + char[][] getParameterPlaceholderList(); + + /** + * Returns the image of the expansion (also containing comments). + * @since 5.0 + */ + char[] getExpansionImage(); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java index ed27b812d61..7c44405dbde 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java @@ -21,24 +21,44 @@ public interface IToken { public int getType(); public String getImage(); public char [] getCharImage(); - public char [] getFilename(); public int getOffset(); public int getLength(); public int getEndOffset(); - // NOTE:if the token spans lines due to escaped newlines then - // the line number returned is the last one - public int getLineNumber(); public IToken getNext(); + public void setNext(IToken t); public void setType(int i); - // queries - public boolean looksLikeExpression(); - public boolean isPointer(); public boolean isOperator(); - public boolean canBeAPrefix(); + + /** + * @deprecated semantics is unclear, depends on the parser. + * will be removed in 5.0 + */ + public boolean isPointer(); + /** + * @deprecated semantics is unclear, depends on the parser. + * will be removed in 5.0 + */ + public boolean canBeAPrefix(); + + /** + * @deprecated semantics is unclear, depends on the parser. + * will be removed in 5.0 + */ + public boolean looksLikeExpression(); + + /** + * @deprecated will be removed in 5.0 + */ + public int getLineNumber(); + + /** + * @deprecated will be removed in 5.0 + */ + public char [] getFilename(); // Token types int FIRST_RESERVED_PREPROCESSOR= -200; @@ -84,6 +104,7 @@ public interface IToken { static public final int tNOT = 36; static public final int tEQUAL = 37; static public final int tASSIGN = 38; + static public final int tUNKNOWN_CHAR= 39; static public final int tSHIFTL = 40; static public final int tLTEQUAL = 41; static public final int tLT = 42; @@ -97,7 +118,7 @@ public interface IToken { static public final int tDOT = 50; static public final int tDIVASSIGN = 51; static public final int tDIV = 52; - static public final int tBACKSLASH= 53; + static public final int tOTHER_CHARACTER= 53; /** @deprecated use {@link #tAND} */ static public final int t_and = 54; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/Keywords.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/Keywords.java index 99600032cb9..c477fd56554 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/Keywords.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/Keywords.java @@ -8,6 +8,7 @@ * Contributors: * IBM Rational Software - Initial API and implementation * Anton Leherbauer (Wind River Systems) + * Markus Schorn (Wind River Systems) *******************************************************************************/ package org.eclipse.cdt.core.parser; @@ -233,7 +234,6 @@ public class Keywords { public static final char[] cpDIV = "/".toCharArray(); //$NON-NLS-1$ public static final char[] cpPOUND = "#".toCharArray(); //$NON-NLS-1$ public static final char[] cpPOUNDPOUND = "##".toCharArray(); //$NON-NLS-1$ - public static final char[] cpBACKSLASH = "\\".toCharArray(); //$NON-NLS-1$ // gcc extensions public static final char[] cpMIN = "= 0 && fCurrentContext == fRootContext) { throw new OffsetLimitReachedException(ORIGIN_PREPROCESSOR_DIRECTIVE, null); } headerName= new char[buf.length()]; @@ -965,22 +992,29 @@ public class CPreprocessor implements ILexerLog, IScanner { return; } - CodeReader reader= null; + String path= null; + boolean reported= false; if (active) { final File currentDir= userInclude || include_next ? new File(String.valueOf(getCurrentFilename())).getParentFile() : null; - reader= findInclusion(new String(headerName), userInclude, include_next, currentDir); + final CodeReader reader= findInclusion(new String(headerName), userInclude, include_next, currentDir); if (reader != null) { - ILocationCtx ctx= fLocationMap.pushInclusion(poundOffset, nameOffset, nameEndOffset, endOffset, reader.buffer, new String(reader.filename), headerName, userInclude); - ScannerContextFile fctx= new ScannerContextFile(ctx, fCurrentContext, new Lexer(reader.buffer, fLexOptions, this)); - fCurrentContext= fctx; + path= new String(reader.filename); + if (!isCircularInclusion(path)) { + reported= true; + fAllIncludedFiles.add(path); + ILocationCtx ctx= fLocationMap.pushInclusion(poundOffset, nameOffset, nameEndOffset, endOffset, reader.buffer, path, headerName, userInclude); + ScannerContextFile fctx= new ScannerContextFile(ctx, fCurrentContext, new Lexer(reader.buffer, fLexOptions, this)); + fCurrentContext= fctx; + } } else { handleProblem(IProblem.PREPROCESSOR_INCLUSION_NOT_FOUND, headerName, poundOffset, endOffset); } } else { + // test if the include is inactive just because it was included before (bug 167100) final File currentDir= userInclude || include_next ? new File(String.valueOf(getCurrentFilename())).getParentFile() : null; - String path= (String) findInclusion(new String(headerName), userInclude, include_next, currentDir, createPathTester); + path= (String) findInclusion(new String(headerName), userInclude, include_next, currentDir, createPathTester); if (path != null) { if (fCodeReaderFactory instanceof IIndexBasedCodeReaderFactory) { // fast indexer @@ -990,23 +1024,37 @@ public class CPreprocessor implements ILexerLog, IScanner { } else { // full indexer - if (!includedFiles.containsKey(path.toCharArray())) { + if (!fAllIncludedFiles.contains(path)) { path= null; } } } + } + if (!reported) { fLocationMap.encounterPoundInclude(poundOffset, nameOffset, nameEndOffset, endOffset, headerName, path, !userInclude, active); } } + + private boolean isCircularInclusion(String filename) { + ILocationCtx checkContext= fCurrentContext.getLocationCtx(); + while (checkContext != null) { + if (filename.equals(checkContext.getFilePath())) { + return true; + } + checkContext= checkContext.getParent(); + } + return false; + } + private void executeDefine(final Lexer lexer, int startOffset) throws OffsetLimitReachedException { try { ObjectStyleMacro macrodef = fMacroDefinitionParser.parseMacroDefinition(lexer, this); fMacroDictionary.put(macrodef.getNameCharArray(), macrodef); final Token name= fMacroDefinitionParser.getNameToken(); + final int endOffset= lexer.currentToken().getEndOffset(); fLocationMap.encounterPoundDefine(startOffset, name.getOffset(), name.getEndOffset(), - fMacroDefinitionParser.getExpansionOffset(), fMacroDefinitionParser.getExpansionEndOffset(), - macrodef); + fMacroDefinitionParser.getExpansionOffset(), endOffset, macrodef); } catch (InvalidMacroDefinitionException e) { int end= lexer.consumeLine(ORIGIN_PREPROCESSOR_DIRECTIVE); handleProblem(IProblem.PREPROCESSOR_INVALID_MACRO_DEFN, e.fName, startOffset, end); @@ -1062,13 +1110,15 @@ public class CPreprocessor implements ILexerLog, IScanner { private void executeIf(Lexer lexer, int startOffset) throws OffsetLimitReachedException { boolean isActive= false; - Token condition= new SimpleToken(0,0,0); + TokenList condition= new TokenList(); final int condOffset= lexer.nextToken().getOffset(); final int condEndOffset= getPreprocessedTokensOfLine(lexer, condition); final int endOffset= lexer.currentToken().getEndOffset(); - condition= (Token) condition.getNext(); - if (condition != null) { + if (condition.first() == null) { + handleProblem(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, null, startOffset, endOffset); + } + else { try { isActive= fExpressionEvaluator.evaluate(condition, fMacroDictionary); } catch (EvalException e) { @@ -1089,15 +1139,16 @@ public class CPreprocessor implements ILexerLog, IScanner { * Macro expansion is reported to the location map. * Returns the end-offset of the last token used from the input. */ - private int getPreprocessedTokensOfLine(Lexer lexer, Token resultHolder) throws OffsetLimitReachedException { + private int getPreprocessedTokensOfLine(Lexer lexer, TokenList result) throws OffsetLimitReachedException { final ScannerContext sctx= fCurrentContext; final ScannerContextPPDirective ppdCtx= new ScannerContextPPDirective(lexer, true); fCurrentContext= ppdCtx; + boolean cn= fCheckNumbers; + fCheckNumbers= false; try { Token t= fetchTokenFromPreprocessor(); while (t.getType() != Lexer.tEND_OF_INPUT) { - resultHolder.setNext(t); - resultHolder= t; + result.append(t); t= fetchTokenFromPreprocessor(); } // make sure an exception is thrown if we are running content assist at the end of the line @@ -1106,6 +1157,7 @@ public class CPreprocessor implements ILexerLog, IScanner { } finally { fCurrentContext= sctx; + fCheckNumbers= cn; } } @@ -1181,10 +1233,9 @@ public class CPreprocessor implements ILexerLog, IScanner { boolean isActive= false; int condOffset= lexer.nextToken().getOffset(); if (nesting == 0 && takeElseBranch) { - Token condition= new SimpleToken(0,0,0); + TokenList condition= new TokenList(); condEndOffset= getPreprocessedTokensOfLine(lexer, condition); - condition= (Token) condition.getNext(); - if (condition != null) { + if (condition.first() != null) { try { isActive= fExpressionEvaluator.evaluate(condition, fMacroDictionary); } catch (EvalException e) { @@ -1231,7 +1282,7 @@ public class CPreprocessor implements ILexerLog, IScanner { } /** - * The method assumes that the identifier is not yet consumed. + * The method assumes that the identifier is consumed. *

* Checks whether the identifier causes a macro expansion. May advance the current lexer * to check for the opening bracket succeeding the identifier. @@ -1242,27 +1293,31 @@ public class CPreprocessor implements ILexerLog, IScanner { * @return * @throws OffsetLimitReachedException */ - private boolean expandMacro(final Token identifier, boolean multiline) throws OffsetLimitReachedException { + private boolean expandMacro(final Token identifier) throws OffsetLimitReachedException { final char[] name= identifier.getCharImage(); PreprocessorMacro macro= (PreprocessorMacro) fMacroDictionary.get(name); if (macro == null) { return false; } - Token bracket= null; if (macro instanceof FunctionStyleMacro) { - bracket= fCurrentContext.nextPPToken(); - if (multiline) { - while(bracket.getType() == Lexer.tNEWLINE) { - bracket= fCurrentContext.nextPPToken(); - } - } - if (bracket.getType() != IToken.tLPAREN) { + if (!findLParenthesisInContext()) { return false; } } - - // mstodo really expand the macro + fExpandingMacro= true; + final boolean contentAssist = fContentAssistLimit>=0 && fCurrentContext == fRootContext; + TokenList replacement= new TokenList(); + final int endOffset= fMacroExpander.expand(macro, identifier, contentAssist, replacement); + fExpandingMacro= false; + + final ImageLocationInfo[] ili= fMacroExpander.createImageLocations(replacement); + final IASTName[] expansions= fMacroExpander.createImplicitExpansions(); + final int length= fMacroExpander.adjustOffsets(replacement); + ILocationCtx ctx= fLocationMap.pushMacroExpansion( + identifier.getOffset(), identifier.getEndOffset(), endOffset, length, macro, expansions, ili); + fCurrentContext= new ScannerContextMacroExpansion(ctx, fCurrentContext, replacement); + return true; } @@ -1274,6 +1329,9 @@ public class CPreprocessor implements ILexerLog, IScanner { } // stuff to be removed + public void addDefinition(IMacro macro) { + addMacroDefinition(macro.getSignature(), macro.getExpansion()); + } public IMacro addDefinition(char[] key, char[] value) { throw new UnsupportedOperationException(); } @@ -1283,9 +1341,6 @@ public class CPreprocessor implements ILexerLog, IScanner { public char[] getMainFilename() { throw new UnsupportedOperationException(); } - public void addDefinition(IMacro macro) { - throw new UnsupportedOperationException(); - } public IMacro addDefinition(char[] name, char[][] params, char[] expansion) { throw new UnsupportedOperationException(); } @@ -1299,717 +1354,4 @@ public class CPreprocessor implements ILexerLog, IScanner { throw new UnsupportedOperationException(); } -// private int skipOverMacroArg() { -// char[] buffer = bufferStack[bufferStackPos]; -// int limit = bufferLimit[bufferStackPos]; -// int argEnd = bufferPos[bufferStackPos]--; -// int nesting = 0; -// while (++bufferPos[bufferStackPos] < limit) { -// switch (buffer[bufferPos[bufferStackPos]]) { -// case '(': -// ++nesting; -// break; -// case ')': -// if (nesting == 0) { -// --bufferPos[bufferStackPos]; -// return argEnd; -// } -// --nesting; -// break; -// case ',': -// if (nesting == 0) { -// --bufferPos[bufferStackPos]; -// return argEnd; -// } -// break; -// // fix for 95119 -// case '\'': -// boolean escapedChar = false; -// loop: while (++bufferPos[bufferStackPos] < bufferLimit[bufferStackPos]) { -// switch (buffer[bufferPos[bufferStackPos]]) { -// case '\\': -// escapedChar = !escapedChar; -// continue; -// case '\'': -// if (escapedChar) { -// escapedChar = false; -// continue; -// } -// break loop; -// default: -// escapedChar = false; -// } -// } -// break; -// case '"': -// boolean escaped = false; -// loop: while (++bufferPos[bufferStackPos] < bufferLimit[bufferStackPos]) { -// switch (buffer[bufferPos[bufferStackPos]]) { -// case '\\': -// escaped = !escaped; -// continue; -// case '"': -// if (escaped) { -// escaped = false; -// continue; -// } -// break loop; -// default: -// escaped = false; -// } -// } -// break; -// } -// argEnd = bufferPos[bufferStackPos]; -// skipOverWhiteSpace(); -// } -// --bufferPos[bufferStackPos]; -// // correct argEnd when reaching limit, (bug 179383) -// if (argEnd==limit) { -// argEnd--; -// } -// return argEnd; -// } -// -// -// private char[] expandFunctionStyleMacro(FunctionStyleMacro macro) { -// char[] buffer = bufferStack[bufferStackPos]; -// int limit = bufferLimit[bufferStackPos]; -// int start = bufferPos[bufferStackPos] - macro.name.length + 1; -// skipOverWhiteSpace(); -// while (bufferPos[bufferStackPos] < limit -// && buffer[bufferPos[bufferStackPos]] == '\\' -// && bufferPos[bufferStackPos] + 1 < buffer.length -// && buffer[bufferPos[bufferStackPos] + 1] == '\n') { -// bufferPos[bufferStackPos] += 2; -// skipOverWhiteSpace(); -// } -// -// if (++bufferPos[bufferStackPos] >= limit) { -// //allow a macro boundary cross here, but only if the caller was -// // prepared to accept a bufferStackPos change -// if (pushContext) { -// int idx = -1; -// int stackpPos = bufferStackPos; -// while (bufferData[stackpPos] != null -//// && bufferData[stackpPos] instanceof MacroData -// ) { -// stackpPos--; -// if (stackpPos < 0) -// return EMPTY_CHAR_ARRAY; -// idx = indexOfNextNonWhiteSpace(bufferStack[stackpPos], -// bufferPos[stackpPos], bufferLimit[stackpPos]); -// if (idx >= bufferLimit[stackpPos]) -// continue; -// if (idx > 0 && bufferStack[stackpPos][idx] == '(') -// break; -// bufferPos[bufferStackPos]--; -// return null; -// } -// if (idx == -1) { -// bufferPos[bufferStackPos]--; -// return null; -// } -// -//// MacroData data; -// IMacro popMacro= macro; -// do { -//// data= (MacroData) bufferData[bufferStackPos]; -// popContextForFunctionMacroName(popMacro); -// popMacro= data.macro; -// } while (bufferStackPos > stackpPos); -// -// bufferPos[bufferStackPos] = idx; -// buffer = bufferStack[bufferStackPos]; -// limit = bufferLimit[bufferStackPos]; -// start = data.startOffset; -// } else { -// bufferPos[bufferStackPos]--; -// return null; -// } -// } -// -// // fix for 107150: the scanner stops at the \n or \r after skipOverWhiteSpace() take that into consideration -// while (bufferPos[bufferStackPos] + 1 < limit && (buffer[bufferPos[bufferStackPos]] == '\n' || buffer[bufferPos[bufferStackPos]] == '\r')) { -// bufferPos[bufferStackPos]++; // skip \n or \r -// skipOverWhiteSpace(); // skip any other spaces after the \n -// -// if (bufferPos[bufferStackPos] + 1 < limit && buffer[bufferPos[bufferStackPos]] != '(' && buffer[bufferPos[bufferStackPos] + 1] == '(') -// bufferPos[bufferStackPos]++; // advance to ( if necessary -// } -// -// if (buffer[bufferPos[bufferStackPos]] != '(') { -// bufferPos[bufferStackPos]--; -// return null; -// } -// -// char[][] arglist = macro.arglist; -// int currarg = 0; -// CharArrayObjectMap argmap = new CharArrayObjectMap(arglist.length); -// -// while (bufferPos[bufferStackPos] < limit) { -// skipOverWhiteSpace(); -// -// if (bufferPos[bufferStackPos] + 1 >= limit) -// break; -// -// if (buffer[++bufferPos[bufferStackPos]] == ')') { -// if (currarg > 0 && argmap.size() <= currarg) { -// argmap.put(arglist[currarg], EMPTY_CHAR_ARRAY); -// } -// break; // end of macro -// } -// if (buffer[bufferPos[bufferStackPos]] == ',') { -// if (argmap.size() <= currarg) { -// argmap.put(arglist[currarg], EMPTY_CHAR_ARRAY); -// } -// currarg++; -// continue; -// } -// -// if ((currarg >= arglist.length || arglist[currarg] == null) -// && !macro.hasVarArgs() && !macro.hasGCCVarArgs()) { -// // too many args and no variable argument -// handleProblem(IProblem.PREPROCESSOR_MACRO_USAGE_ERROR, -// bufferPos[bufferStackPos], macro.name); -// break; -// } -// -// int argstart = bufferPos[bufferStackPos]; -// -// int argend = -1; -// if ((macro.hasGCCVarArgs() || macro.hasVarArgs()) && currarg == macro.getVarArgsPosition()) { -// // there are varargs and the other parameters have been accounted -// // for, the rest will replace __VA_ARGS__ or name where -// // "name..." is the parameter -// for (;;) { -// argend= skipOverMacroArg(); -// skipOverWhiteSpace(); -// // to continue we need at least a comma and another char. -// if (bufferPos[bufferStackPos]+2 >= limit) { -// break; -// } -// if (buffer[++bufferPos[bufferStackPos]] == ')') { -// bufferPos[bufferStackPos]--; -// break; -// } -// // it's a comma -// bufferPos[bufferStackPos]++; -// } -// } else { -// argend = skipOverMacroArg(); -// } -// -// char[] arg = EMPTY_CHAR_ARRAY; -// int arglen = argend - argstart + 1; -// if (arglen > 0) { -// arg = new char[arglen]; -// System.arraycopy(buffer, argstart, arg, 0, arglen); -// } -// -// argmap.put(arglist[currarg], arg); -// } -// -// int numRequiredArgs = arglist.length; -// for (int i = 0; i < arglist.length; i++) { -// if (arglist[i] == null) { -// numRequiredArgs = i; -// break; -// } -// } -// -// /* Don't require a match for the vararg placeholder */ -// /* Workaround for bugzilla 94365 */ -// if (macro.hasGCCVarArgs()|| macro.hasVarArgs()) -// numRequiredArgs--; -// -// if (argmap.size() < numRequiredArgs) { -// handleProblem(IProblem.PREPROCESSOR_MACRO_USAGE_ERROR, -// bufferPos[bufferStackPos], macro.name); -// } -// -// char[] result = null; -// if (macro instanceof DynamicFunctionStyleMacro) { -// result = ((DynamicFunctionStyleMacro) macro).execute(argmap); -// } else { -// CharArrayObjectMap replacedArgs = new CharArrayObjectMap(argmap -// .size()); -// int size = expandFunctionStyleMacro(macro.getExpansion(), argmap, -// replacedArgs, null); -// result = new char[size]; -// expandFunctionStyleMacro(macro.getExpansion(), argmap, replacedArgs, -// result); -// } -// if (pushContext) -// { -//// pushContext(result, new FunctionMacroData(start, bufferPos[bufferStackPos] + 1, -//// macro, argmap)); -// } -// return result; -// } -// -// private char[] replaceArgumentMacros(char[] arg) { -// int limit = arg.length; -// int start = -1, end = -1; -// Object expObject = null; -// for (int pos = 0; pos < limit; pos++) { -// char c = arg[pos]; -// if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' -// || Character.isLetter(c) -// || (support$Initializers && c == '$')) { -// start = pos; -// while (++pos < limit) { -// c = arg[pos]; -// if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') -// || c == '_' || (c >= '0' && c <= '9') -// || (support$Initializers && c == '$') -// || Character.isUnicodeIdentifierPart(c)) { -// continue; -// } -// break; -// } -// end = pos - 1; -// } -// else if (c == '"') { -// boolean escaped= false; -// while (++pos < limit) { -// c = arg[pos]; -// if (!escaped && c == '"') { -// break; -// } -// if (c == '\\') { -// escaped= !escaped; -// } -// else { -// escaped= false; -// } -// } -// } -// else if (c == '\'') { -// boolean escaped= false; -// while (++pos < limit) { -// c = arg[pos]; -// if (!escaped && c == '\'') { -// break; -// } -// if (c == '\\') { -// escaped= !escaped; -// } -// else { -// escaped= false; -// } -// } -// } -// -// if (start != -1 && end >= start) { -// //Check for macro expansion -// expObject = fMacroDictionary.get(arg, start, (end - start + 1)); -// if (expObject == null || !shouldExpandMacro((IMacro) expObject)) { -// expObject = null; -// start = -1; -// continue; -// } -// //else, break and expand macro -// break; -// } -// } -// -// if (expObject == null) -// { -// return arg; -// } -// -// -// char[] expansion = null; -// if (expObject instanceof FunctionStyleMacro) { -// FunctionStyleMacro expMacro = (FunctionStyleMacro) expObject; -// pushContext((start == 0) ? arg : CharArrayUtils.extract(arg, start, -// arg.length - start)); -// bufferPos[bufferStackPos] += end - start + 1; -// expansion = handleFunctionStyleMacro(expMacro, false); -// end = bufferPos[bufferStackPos] + start; -// popContext(); -// } else if (expObject instanceof ObjectStyleMacro) { -// ObjectStyleMacro expMacro = (ObjectStyleMacro) expObject; -// expansion = expMacro.getExpansion(); -// } else if (expObject instanceof char[]) { -// expansion = (char[]) expObject; -// } else if (expObject instanceof DynamicStyleMacro) { -// DynamicStyleMacro expMacro = (DynamicStyleMacro) expObject; -// expansion = expMacro.execute(); -// } -// -// if (expansion != null) { -// int newlength = start + expansion.length + (limit - end - 1); -// char[] result = new char[newlength]; -// System.arraycopy(arg, 0, result, 0, start); -// System.arraycopy(expansion, 0, result, start, expansion.length); -// if (arg.length > end + 1) -// System.arraycopy(arg, end + 1, result, -// start + expansion.length, limit - end - 1); -// -// -// beforeReplaceAllMacros(); -// //we need to put the macro on the context stack in order to detect -// // recursive macros -//// pushContext(EMPTY_CHAR_ARRAY, -//// new MacroData(start, start -//// + ((IMacro) expObject).getName().length, -//// (IMacro) expObject) -//// ); -// arg = replaceArgumentMacros(result); //rescan for more macros -// popContext(); -// afterReplaceAllMacros(); -// } -// -// return arg; -// } -// -// -// private int expandFunctionStyleMacro(char[] expansion, -// CharArrayObjectMap argmap, CharArrayObjectMap replacedArgs, -// char[] result) { -// -// // The current position in the expansion string that we are looking at -// int pos = -1; -// // The last position in the expansion string that was copied over -// int lastcopy = -1; -// // The current write offset in the result string - also tells us the -// // length of the result string -// int outpos = 0; -// // The first character in the current block of white space - there are -// // times when we don't -// // want to copy over the whitespace -// int wsstart = -1; -// //whether or not we are on the second half of the ## operator -// boolean prevConcat = false; -// //for handling ## -// char[] prevArg = null; -// int prevArgStart = -1; -// int prevArgLength = -1; -// int prevArgTarget = 0; -// -// int limit = expansion.length; -// -// while (++pos < limit) { -// char c = expansion[pos]; -// -// if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' -// || (c >= '0' && c < '9') -// || Character.isUnicodeIdentifierPart(c)) { -// -// wsstart = -1; -// int idstart = pos; -// while (++pos < limit) { -// c = expansion[pos]; -// if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') -// || (c >= '0' && c <= '9') || c == '_' || Character -// .isUnicodeIdentifierPart(c))) { -// break; -// } -// } -// --pos; -// -// char[] repObject = (char[]) argmap.get(expansion, idstart, pos -// - idstart + 1); -// -// int next = indexOfNextNonWhiteSpace(expansion, pos, limit); -// boolean nextIsPoundPound = (next + 1 < limit -// && expansion[next] == '#' && expansion[next + 1] == '#'); -// -// if (prevConcat && prevArgStart > -1 && prevArgLength > 0) { -// int l1 = prevArg != null ? prevArg.length : prevArgLength; -// int l2 = repObject != null ? repObject.length : pos -// - idstart + 1; -// char[] newRep = new char[l1 + l2]; -// if (prevArg != null) -// System.arraycopy(prevArg, 0, newRep, 0, l1); -// else -// System -// .arraycopy(expansion, prevArgStart, newRep, 0, -// l1); -// -// if (repObject != null) -// System.arraycopy(repObject, 0, newRep, l1, l2); -// else -// System.arraycopy(expansion, idstart, newRep, l1, l2); -// idstart = prevArgStart; -// repObject = newRep; -// } -// if (repObject != null) { -// // copy what we haven't so far -// if (++lastcopy < idstart) { -// int n = idstart - lastcopy; -// if (result != null) { -// // the outpos may be set back when prevConcat is true, so make sure we -// // stay in bounds. -// if (prevConcat && outpos+n > result.length) { -// n= result.length- outpos; -// } -// System.arraycopy(expansion, lastcopy, result, -// outpos, n); -// } -// outpos += n; -// } -// -// if (prevConcat) -// outpos = prevArgTarget; -// -// if (!nextIsPoundPound) { -// //16.3.1 completely macro replace the arguments before -// // substituting them in -// char[] rep = (char[]) ((replacedArgs != null) ? replacedArgs -// .get(repObject) -// : null); -// -// if (rep != null) -// repObject = rep; -// else { -// rep = replaceArgumentMacros(repObject); -// if (replacedArgs != null) -// replacedArgs.put(repObject, rep); -// repObject = rep; -// } -// -// if (result != null ) -// System.arraycopy(repObject, 0, result, outpos, repObject.length); -// } -// outpos += repObject.length; -// -// lastcopy = pos; -// } -// -// prevArg = repObject; -// prevArgStart = idstart; -// prevArgLength = pos - idstart + 1; -// prevArgTarget = repObject != null ? outpos - repObject.length -// : outpos + idstart - lastcopy - 1; -// prevConcat = false; -// } else if (c == '"') { -// -// // skip over strings -// wsstart = -1; -// boolean escaped = false; -// while (++pos < limit) { -// c = expansion[pos]; -// if (c == '"') { -// if (!escaped) -// break; -// } else if (c == '\\') { -// escaped = !escaped; -// } -// escaped = false; -// } -// prevConcat = false; -// } else if (c == '\'') { -// -// // skip over character literals -// wsstart = -1; -// boolean escaped = false; -// while (++pos < limit) { -// c = expansion[pos]; -// if (c == '\'') { -// if (!escaped) -// break; -// } else if (c == '\\') { -// escaped = !escaped; -// } -// escaped = false; -// } -// prevConcat = false; -// } else if (c == ' ' || c == '\t') { -// // obvious whitespace -// if (wsstart < 0) -// wsstart = pos; -// } else if (c == '/' && pos + 1 < limit) { -// -// // less than obvious, comments are whitespace -// c = expansion[pos+1]; -// if (c == '/') { -// // copy up to here or before the last whitespace -// ++pos; -// ++lastcopy; -// int n = wsstart < 0 ? pos - 1 - lastcopy : wsstart -// - lastcopy; -// if (result != null) -// System -// .arraycopy(expansion, lastcopy, result, outpos, -// n); -// outpos += n; -// -// // skip the rest -// lastcopy = expansion.length - 1; -// } else if (c == '*') { -// ++pos; -// if (wsstart < 1) -// wsstart = pos - 1; -// while (++pos < limit) { -// if (expansion[pos] == '*' && pos + 1 < limit -// && expansion[pos + 1] == '/') { -// ++pos; -// break; -// } -// } -// } else -// wsstart = -1; -// -// } else if (c == '\\' && pos + 1 < limit -// && expansion[pos + 1] == 'n') { -// // skip over this -// ++pos; -// -// } else if (c == '#') { -// -// if (pos + 1 < limit && expansion[pos + 1] == '#') { -// prevConcat = true; -// ++pos; -// // skip whitespace -// if (wsstart < 0) -// wsstart = pos - 1; -// while (++pos < limit) { -// switch (expansion[pos]) { -// case ' ': -// case '\t': -// continue; -// -// case '/': -// if (pos + 1 < limit) { -// c = expansion[pos + 1]; -// if (c == '/') -// // skip over everything -// pos = expansion.length; -// else if (c == '*') { -// ++pos; -// while (++pos < limit) { -// if (expansion[pos] == '*' -// && pos + 1 < limit -// && expansion[pos + 1] == '/') { -// ++pos; -// break; -// } -// } -// continue; -// } -// } -// } -// break; -// } -// --pos; -// } else { -// prevConcat = false; -// // stringify -// -// // copy what we haven't so far -// if (++lastcopy < pos) { -// int n = pos - lastcopy; -// if (result != null) -// System.arraycopy(expansion, lastcopy, result, -// outpos, n); -// outpos += n; -// } -// -// // skip whitespace -// while (++pos < limit) { -// switch (expansion[pos]) { -// case ' ': -// case '\t': -// continue; -// case '/': -// if (pos + 1 < limit) { -// c = expansion[pos + 1]; -// if (c == '/') -// // skip over everything -// pos = expansion.length; -// else if (c == '*') { -// ++pos; -// while (++pos < limit) { -// if (expansion[pos] == '*' -// && pos + 1 < limit -// && expansion[pos + 1] == '/') { -// ++pos; -// break; -// } -// } -// continue; -// } -// } -// //TODO handle comments -// } -// break; -// } -// -// // grab the identifier -// c = expansion[pos]; -// int idstart = pos; -// if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'X') -// || c == '_' || Character.isUnicodeIdentifierPart(c)) { -// while (++pos < limit) { -// c = expansion[pos]; -// if (!((c >= 'a' && c <= 'z') -// || (c >= 'A' && c <= 'X') -// || (c >= '0' && c <= '9') || c == '_' || Character -// .isUnicodeIdentifierPart(c))) -// break; -// } -// } // else TODO something -// --pos; -// int idlen = pos - idstart + 1; -// char[] argvalue = (char[]) argmap.get(expansion, idstart, -// idlen); -// if (argvalue != null) { -// //16.3.2-2 ... a \ character is inserted before each " -// // and \ -// // character -// //of a character literal or string literal -// -// //technically, we are also supposed to replace each -// // occurence -// // of whitespace -// //(including comments) in the argument with a single -// // space. -// // But, at this time -// //we don't really care what the contents of the string -// // are, -// // just that we get the string -// //so we won't bother doing that -// if (result != null) { -// result[outpos++] = '"'; -// for (int i = 0; i < argvalue.length; i++) { -// if (argvalue[i] == '"' || argvalue[i] == '\\') -// result[outpos++] = '\\'; -// if (argvalue[i] == '\r' || argvalue[i] == '\n') -// result[outpos++] = ' '; -// else -// result[outpos++] = argvalue[i]; -// } -// result[outpos++] = '"'; -// } else { -// for (int i = 0; i < argvalue.length; i++) { -// if (argvalue[i] == '"' || argvalue[i] == '\\') -// ++outpos; -// ++outpos; -// } -// outpos += 2; -// } -// } -// lastcopy = pos; -// wsstart = -1; -// } -// } else { -// prevConcat = false; -// // not sure what it is but it sure ain't whitespace -// wsstart = -1; -// } -// -// } -// -// if (wsstart < 0 && ++lastcopy < expansion.length) { -// int n = expansion.length - lastcopy; -// if (result != null) -// System.arraycopy(expansion, lastcopy, result, outpos, n); -// outpos += n; -// } -// -// return outpos; -// } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java index 3f58c8a7682..219a00713c9 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java @@ -19,7 +19,6 @@ import org.eclipse.cdt.core.parser.util.CharArrayObjectMap; /** * Used to evaluate expressions in preprocessor directives. - * @since 5.0 */ class ExpressionEvaluator { static class EvalException extends Exception { @@ -28,6 +27,7 @@ class ExpressionEvaluator { public EvalException(int problemID, char[] problemArg) { fProblemID= problemID; + fProblemArg= problemArg; } public int getProblemID() { @@ -42,8 +42,8 @@ class ExpressionEvaluator { private Token fTokens; private CharArrayObjectMap fDictionary; - public boolean evaluate(Token condition, CharArrayObjectMap dictionary) throws EvalException { - fTokens= condition; + public boolean evaluate(TokenList condition, CharArrayObjectMap dictionary) throws EvalException { + fTokens= condition.first(); fDictionary= dictionary; return expression() != 0; } @@ -136,7 +136,7 @@ class ExpressionEvaluator { private long relationalExpression() throws EvalException { long r1 = shiftExpression(); for (int t = LA(); t == IToken.tLT || t == IToken.tLTEQUAL || t == IToken.tGT - || t == IToken.tGTEQUAL; t = LA()) { + || t == IToken.tGTEQUAL || t == IToken.tASSIGN; t = LA()) { consume(); long r2 = shiftExpression(); switch (t) { @@ -152,6 +152,8 @@ class ExpressionEvaluator { case IToken.tGTEQUAL: r1 = (r1 >= r2) ? 1 : 0; break; + case IToken.tASSIGN: + throw new EvalException(IProblem.SCANNER_ASSIGNMENT_NOT_ALLOWED, null); } } return r1; @@ -235,7 +237,8 @@ class ExpressionEvaluator { } throw new EvalException(IProblem.SCANNER_MISSING_R_PAREN, null); case IToken.tIDENTIFIER: - return 1; + consume(); + return 0; default: throw new EvalException(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, null); } @@ -350,14 +353,26 @@ class ExpressionEvaluator { } private long getNumber(char[] tokenImage, int from, int to, int base, int problemID) throws EvalException { + if (from == to) { + throw new EvalException(problemID, tokenImage); + } long result= 0; - for (int i = from; i < to; i++) { + int i= from; + for (; i < to; i++) { int digit= getDigit(tokenImage[i]); if (digit >= base) { - throw new EvalException(problemID, tokenImage); + break; } result= result*base + digit; } + for (; i < to; i++) { + switch(tokenImage[i]) { + case 'u' : case 'l': case 'U': case 'L': + break; + default: + throw new EvalException(problemID, tokenImage); + } + } return result; } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILexerLog.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILexerLog.java index 53243a31cd6..5e5053e2fed 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILexerLog.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILexerLog.java @@ -17,6 +17,10 @@ import org.eclipse.cdt.core.parser.IProblem; * @since 5.0 */ public interface ILexerLog { + ILexerLog NULL = new ILexerLog() { + public void handleComment(boolean isBlockComment, int offset, int endOffset) {} + public void handleProblem(int problemID, char[] info, int offset, int endOffset) {} + }; /** * A problem has been detected diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationCtx.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationCtx.java index 0cc3dd2ee76..ac52538bea4 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationCtx.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationCtx.java @@ -12,9 +12,22 @@ package org.eclipse.cdt.internal.core.parser.scanner; /** - * Interface for modeling contexts that can deal with offsets. These are: + * Interface between location map and preprocessor for modeling contexts that can deal with offsets. + * These are: * synthetic contexts used for pre-included files, file-contexts, macro-expansions. * @since 5.0 */ public interface ILocationCtx { + + /** + * If this is a file context the filename of this context is returned, + * otherwise the filename of the first enclosing context that is a file context is returned. + */ + String getFilePath(); + + /** + * Returns the enclosing context or null if this is the translation unit context. + */ + ILocationCtx getParent(); + } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationResolver.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationResolver.java index 04f86ed453f..0e5f29dc643 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationResolver.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationResolver.java @@ -65,6 +65,7 @@ public interface ILocationResolver extends org.eclipse.cdt.internal.core.parser. /** * @see IASTTranslationUnit#getContainingFilename() + * mstodo- scanner removal should be renamed */ public String getContainingFilename(int offset); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ImageLocationInfo.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ImageLocationInfo.java index 5066a405018..e2668e3d545 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ImageLocationInfo.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ImageLocationInfo.java @@ -18,4 +18,6 @@ package org.eclipse.cdt.internal.core.parser.scanner; */ public class ImageLocationInfo { + public static final ImageLocationInfo[] NO_LOCATION_INFOS= {}; + } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java index 2252135b303..6c2a465bd7b 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java @@ -10,7 +10,6 @@ *******************************************************************************/ package org.eclipse.cdt.internal.core.parser.scanner; -import org.eclipse.cdt.core.dom.ast.IASTProblem; import org.eclipse.cdt.core.parser.IGCCToken; import org.eclipse.cdt.core.parser.IProblem; import org.eclipse.cdt.core.parser.IToken; @@ -23,7 +22,7 @@ import org.eclipse.cdt.core.parser.OffsetLimitReachedException; * Returns preprocessor tokens. *

* In addition to the preprocessor tokens the following tokens may also be returned: - * {@link #tEND_OF_INPUT}, {@link IToken#tCOMPLETION}. + * {@link #tBEFORE_INPUT}, {@link #tEND_OF_INPUT}, {@link IToken#tCOMPLETION}. *

* Number literals are split up into {@link IToken#tINTEGER} and {@link IToken#tFLOATINGPT}. * No checks are done on the number literals. @@ -67,6 +66,7 @@ final public class Lexer { // the input to the lexer private final char[] fInput; + private int fStart; private int fLimit; // after phase 3 (newline, trigraph, line-splice) @@ -75,7 +75,7 @@ final public class Lexer { private int fCharPhase3; private boolean fInsideIncludeDirective= false; - private Token fToken= new SimpleToken(tBEFORE_INPUT, 0, 0); + private Token fToken; // for the few cases where we have to lookahead more than one character private int fMarkOffset; @@ -85,18 +85,16 @@ final public class Lexer { public Lexer(char[] input, LexerOptions options, ILexerLog log) { - fInput= input; - fLimit= input.length; - fOptions= options; - fLog= log; - nextCharPhase3(); + this(input, 0, input.length, options, log); } - public Lexer(char[] input, int limit, LexerOptions options, ILexerLog log) { + public Lexer(char[] input, int start, int end, LexerOptions options, ILexerLog log) { fInput= input; - fLimit= limit; + fStart= fOffset= fEndOffset= start; + fLimit= end; fOptions= options; fLog= log; + fToken= new SimpleToken(tBEFORE_INPUT, start, start); nextCharPhase3(); } @@ -107,7 +105,7 @@ final public class Lexer { fOptions.fSupportContentAssist= true; fLimit= Math.min(fLimit, fInput.length); // re-initialize - fOffset= fEndOffset= 0; + fOffset= fEndOffset= fStart; nextCharPhase3(); } @@ -115,8 +113,8 @@ final public class Lexer { * Call this before consuming the name-token in the include directive. It causes the header-file * tokens to be created. */ - public void setInsideIncludeDirective() { - fInsideIncludeDirective= true; + public void setInsideIncludeDirective(boolean val) { + fInsideIncludeDirective= val; } /** @@ -131,7 +129,8 @@ final public class Lexer { * @throws OffsetLimitReachedException when completion is requested in a literal or a header-name. */ public Token nextToken() throws OffsetLimitReachedException { - fFirstTokenAfterNewline= fToken.getType() == tNEWLINE; + final int t= fToken.getType(); + fFirstTokenAfterNewline= t == tNEWLINE || t == tBEFORE_INPUT; return fToken= fetchToken(); } @@ -169,6 +168,33 @@ final public class Lexer { } } + /** + * Advances to the next newline. + * @return the list of tokens found on this line. + * @param origin parameter for the {@link OffsetLimitReachedException} when it has to be thrown. + */ + public final void getTokensOfLine(int origin, TokenList result) throws OffsetLimitReachedException { + Token t= fToken; + while(true) { + switch(t.getType()) { + case IToken.tCOMPLETION: + fToken= t; + throw new OffsetLimitReachedException(origin, t); + case Lexer.tEND_OF_INPUT: + fToken= t; + if (fOptions.fSupportContentAssist) { + throw new OffsetLimitReachedException(origin, null); + } + return; + case Lexer.tNEWLINE: + fToken= t; + return; + } + result.append(t); + t= fetchToken(); + } + } + /** * Advances to the next pound token that starts a preprocessor directive. * @return pound token of the directive or end-of-input. @@ -196,7 +222,6 @@ final public class Lexer { break; } } - t= fetchToken(); } fToken= t; return t; @@ -264,7 +289,7 @@ final public class Lexer { nextCharPhase3(); return identifier(start, 2); } - return newToken(IToken.tBACKSLASH, start); + return newToken(IToken.tOTHER_CHARACTER, start, 1); case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -511,9 +536,8 @@ final public class Lexer { } break; } - - handleProblem(IASTProblem.SCANNER_BAD_CHARACTER, new char[] {(char) c}, start); - // loop is continued, character is treated as white-space. + // handles for instance @ + return newToken(IToken.tOTHER_CHARACTER, start, 1); } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LocationCtx.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LocationCtx.java index 067ebc4276b..504501c6277 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LocationCtx.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LocationCtx.java @@ -37,11 +37,11 @@ abstract class LocationCtx implements ILocationCtx { } } - public String getFilename() { - return fParent.getFilename(); + public String getFilePath() { + return fParent.getFilePath(); } - final public LocationCtx getParent() { + final public ILocationCtx getParent() { return fParent; } /** @@ -262,7 +262,7 @@ class FileLocationCtx extends ContainerLocationCtx { } } - public final String getFilename() { + public final String getFilePath() { return fFilename; } @@ -331,5 +331,5 @@ class MacroExpansionCtx extends LocationCtx { return fLength; } - // mstodo once image locations are supported we need to handle those in here + // mstodo- image locations } \ No newline at end of file diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LocationMap.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LocationMap.java index f3807021269..fd72c14c28f 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LocationMap.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LocationMap.java @@ -183,12 +183,11 @@ public class LocationMap implements ILocationResolver { public void popContext(ILocationCtx locationCtx) { assert fCurrentContext == locationCtx; final LocationCtx child= fCurrentContext; - final LocationCtx parent= fCurrentContext.getParent(); + final LocationCtx parent= (LocationCtx) fCurrentContext.getParent(); if (parent != null) { - fCurrentContext= child.getParent(); + fCurrentContext= parent; fLastChildInsertionOffset= child.fParentEndOffset; parent.addChildSequenceLength(child.getSequenceLength()); - fCurrentContext= parent; } } @@ -328,8 +327,8 @@ public class LocationMap implements ILocationResolver { * Returns the filename of the current context. If the context is a macro-expansion the filename of * the enclosing file is returned. */ - public String getCurrentFilename() { - return fCurrentContext.getFilename(); + public String getCurrentFilePath() { + return fCurrentContext.getFilePath(); } /** @@ -337,13 +336,13 @@ public class LocationMap implements ILocationResolver { *

* You must insert all child contexts before the given offset before conversion. */ - private int getSequenceNumberForOffset(int offset) { + int getSequenceNumberForOffset(int offset) { return fCurrentContext.getSequenceNumberForOffset(offset, offset < fLastChildInsertionOffset); } public String getContainingFilename(int sequenceNumber) { LocationCtx ctx= fRootContext.ctxForNumberRange(sequenceNumber, 1); - return new String(ctx.getFilename()); + return new String(ctx.getFilePath()); } public IASTFileLocation getMappedFileLocation(int sequenceNumber, int length) { @@ -423,21 +422,21 @@ public class LocationMap implements ILocationResolver { public void cleanup() { throw new UnsupportedOperationException(); } - // mstodo get rid of IASTNodeLocation + // mstodo- locations public IASTFileLocation flattenLocations(IASTNodeLocation[] locations) { if (locations.length != 1 || !(locations[0] instanceof IASTFileLocation)) { throw new IllegalArgumentException(); } return (IASTFileLocation) locations[0]; } - // mstodo get rid of IASTNodeLocation + // mstodo- locations public IASTNodeLocation[] getLocations(int offset, int length) { return new IASTNodeLocation[] {getMappedFileLocation(offset, length)}; } public ASTPreprocessorSelectionResult getPreprocessorNode(String path, int offset, int length) { throw new UnsupportedOperationException(); } - // mstodo get rid of IASTNodeLocation + // mstodo- locations public char[] getUnpreprocessedSignature(IASTNodeLocation[] locations) { switch(locations.length) { case 0: return CharArrayUtils.EMPTY; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/MacroDefinitionParser.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/MacroDefinitionParser.java index d874b21033b..6ca270fe9dc 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/MacroDefinitionParser.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/MacroDefinitionParser.java @@ -53,14 +53,6 @@ class MacroDefinitionParser { return fExpansionOffset; } - /** - * In case the expansion was successfully parsed, the end offset is returned. - * Otherwise the return value is undefined. - */ - public int getExpansionEndOffset() { - return fExpansionEndOffset; - } - /** * Parses an entire macro definition. Name must be the next token of the lexer. */ @@ -70,7 +62,8 @@ class MacroDefinitionParser { final char[] source= lexer.getInput(); final char[] nameChars= name.getCharImage(); final char[][] paramList= parseParamList(lexer, name); - final Token replacement= parseExpansion(lexer, log, nameChars, paramList, fHasVarArgs); + final TokenList replacement= new TokenList(); + parseExpansion(lexer, log, nameChars, paramList, replacement); if (paramList == null) { return new ObjectStyleMacro(nameChars, fExpansionOffset, fExpansionEndOffset, replacement, source); } @@ -177,7 +170,13 @@ class MacroDefinitionParser { paramList.add(Keywords.cVA_ARGS); next= lex.nextToken(); break; - + + case IToken.tRPAREN: + if (next == null) { + next= param; + break; + } + // no break; default: throw new InvalidMacroDefinitionException(name.getCharImage()); } @@ -191,16 +190,14 @@ class MacroDefinitionParser { return (char[][]) paramList.toArray(new char[paramList.size()][]); } - private Token parseExpansion(final Lexer lexer, final ILexerLog log, final char[] name, final char[][] paramList, final int hasVarArgs) - throws OffsetLimitReachedException { - final boolean allowVarArgsArray= hasVarArgs==FunctionStyleMacro.VAARGS; + public void parseExpansion(final Lexer lexer, final ILexerLog log, final char[] name, final char[][] paramList, + TokenList result) throws OffsetLimitReachedException { boolean needParam= false; + boolean isFirst= true; Token needAnotherToken= null; Token candidate= lexer.currentToken(); - fExpansionOffset= candidate.getOffset(); - Token last= new SimpleToken(Lexer.tNEWLINE, fExpansionOffset, fExpansionOffset); - final Token resultHolder= last; + fExpansionOffset= fExpansionEndOffset= candidate.getOffset(); loop: while(true) { switch(candidate.getType()) { @@ -210,20 +207,31 @@ class MacroDefinitionParser { case Lexer.tNEWLINE: break loop; case IToken.tIDENTIFIER: - if (!allowVarArgsArray && CharArrayUtils.equals(Keywords.cVA_ARGS, candidate.getCharImage())) { - log.handleProblem(IProblem.PREPROCESSOR_INVALID_VA_ARGS, null, fExpansionOffset, candidate.getEndOffset()); + if (paramList != null) { + // convert the parameters to special tokens + final char[] image = candidate.getCharImage(); + int idx= CharArrayUtils.indexOf(image, paramList); + if (idx >= 0) { + candidate= new PlaceHolderToken(CPreprocessor.tMACRO_PARAMETER, idx, candidate.getOffset(), candidate.getEndOffset(), paramList[idx]); + needParam= false; + } + else { + if (needParam) { + log.handleProblem(IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, name, fExpansionOffset, candidate.getEndOffset()); + } + else if (CharArrayUtils.equals(Keywords.cVA_ARGS, image)) { + log.handleProblem(IProblem.PREPROCESSOR_INVALID_VA_ARGS, null, fExpansionOffset, candidate.getEndOffset()); + } + needParam= false; + } } - if (needParam && CharArrayUtils.indexOf(candidate.getCharImage(), paramList) == -1) { - log.handleProblem(IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, name, fExpansionOffset, candidate.getEndOffset()); - } - needParam= false; needAnotherToken= null; break; case IToken.tPOUND: needParam= paramList != null; break; case IToken.tPOUNDPOUND: - if (needParam || resultHolder == last) { + if (needParam || isFirst) { log.handleProblem(IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, name, fExpansionOffset, candidate.getEndOffset()); } needAnotherToken= candidate; @@ -237,13 +245,13 @@ class MacroDefinitionParser { needAnotherToken= null; break; } - last.setNext(candidate); last=candidate; + isFirst= false; + fExpansionEndOffset= candidate.getEndOffset(); + result.append(candidate); candidate= lexer.nextToken(); } if (needAnotherToken != null) { log.handleProblem(IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, name, needAnotherToken.getOffset(), needAnotherToken.getEndOffset()); } - fExpansionEndOffset= last.getEndOffset(); - return (Token) resultHolder.getNext(); } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/MacroExpander.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/MacroExpander.java new file mode 100644 index 00000000000..b89ee1674ab --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/MacroExpander.java @@ -0,0 +1,539 @@ +/******************************************************************************* + * Copyright (c) 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import java.util.ArrayList; +import java.util.IdentityHashMap; + +import org.eclipse.cdt.core.dom.ast.IASTName; +import org.eclipse.cdt.core.parser.IProblem; +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.OffsetLimitReachedException; +import org.eclipse.cdt.core.parser.util.CharArrayObjectMap; +import org.eclipse.cdt.core.parser.util.CharArrayUtils; +import org.eclipse.cdt.internal.core.parser.scanner.Lexer.LexerOptions; + +/** + * Utility class to perform macro expansion. + * @since 5.0 + */ +public class MacroExpander { + private static final int ORIGIN = OffsetLimitReachedException.ORIGIN_MACRO_EXPANSION; + + /** + * Marks the beginning and the end of the scope of a macro expansion. Necessary to properly + * handle recursive expansions and to figure out whether spaces are required during a stringify + * operation across such boundaries. + */ + public static final class ExpansionBoundary extends Token { + private PreprocessorMacro fScope; + private boolean fIsStart; + + ExpansionBoundary(PreprocessorMacro scope, int offset, boolean isStart) { + super(CPreprocessor.tSCOPE_MARKER, offset, offset); + fScope= scope; + fIsStart= isStart; + } + + public char[] getCharImage() { + return CharArrayUtils.EMPTY; + } + + public String toString() { + return "{" + (fIsStart ? '+' : '-') + //$NON-NLS-1$ + (fScope == null ? String.valueOf(getOffset()) : fScope.getName()) + '}'; + } + + public void execute(IdentityHashMap forbidden) { + if (fIsStart) { + forbidden.put(fScope, fScope); + } + else { + forbidden.remove(fScope); + } + fScope= null; + } + + public Object clone() { + // when cloned for the purpose of argument substitution, the boundaries no longer prevent a + // recursive macro expansion. + ExpansionBoundary t= (ExpansionBoundary) super.clone(); + t.fScope= null; + return t; + } + } + + /** + * Combines a list of tokens with the preprocessor to form the input for macro expansion. + */ + private class TokenSource extends TokenList { + private boolean fUseCpp; + + public TokenSource(boolean useCpp) { + fUseCpp= true; + } + + public Token fetchFirst() throws OffsetLimitReachedException { + Token t= removeFirst(); + if (t == null && fUseCpp) { + t= fCpp.fetchTokenFromPreprocessor(); + fEndOffset= t.getEndOffset(); + } + return t; + } + + public boolean findLParenthesis(IdentityHashMap forbidden) throws OffsetLimitReachedException { + Token t= first(); + while (t != null) { + switch (t.getType()) { + case Lexer.tNEWLINE: + break; + case CPreprocessor.tSCOPE_MARKER: + ((ExpansionBoundary) t).execute(forbidden); + break; + case IToken.tLPAREN: + return true; + default: + return false; + } + removeFirst(); + t= first(); + } + + if (fUseCpp) { + return fCpp.findLParenthesisInContext(); + } + return false; + } + } + + private final MacroDefinitionParser fDefinitionParser; + private final CharArrayObjectMap fDictionary; + private final LocationMap fLocationMap; + private final CPreprocessor fCpp; + private final LexerOptions fLexOptions; + private int fEndOffset; + private ArrayList fImplicitMacroExpansions= new ArrayList(); + private boolean fCompletionMode; + private int fStartOffset; + + public MacroExpander(CPreprocessor cpp, CharArrayObjectMap dict, LocationMap locationMap, MacroDefinitionParser mdp, LexerOptions lexOptions) { + fCpp= cpp; + fDictionary= dict; + fLocationMap= locationMap; + fDefinitionParser= mdp; + fLexOptions= lexOptions; + } + + /** + * Expects that the identifier has been consumed, stores the result in the list provided and returns the + * end offset of the last token read from the preprocessor input. + */ + public int expand(PreprocessorMacro macro, Token identifier, boolean completionMode, TokenList expansion) throws OffsetLimitReachedException { + fStartOffset= identifier.getOffset(); + fEndOffset= identifier.getEndOffset(); + fCompletionMode= completionMode; + + IdentityHashMap forbidden= new IdentityHashMap(); + + // setup input sequence + TokenSource input= new TokenSource(true); + TokenList firstExpansion= expandOne(macro, forbidden, input, fStartOffset, fEndOffset); + input.prepend(firstExpansion); + + expandAll(input, forbidden, expansion); + return fEndOffset; + } + + /** + * Expects that the identifier of the macro expansion has been consumed. + */ + private TokenList expandOne(PreprocessorMacro macro, IdentityHashMap forbidden, TokenSource input, int offset, int endOffset) + throws OffsetLimitReachedException { + TokenList result= new TokenList(); + result.append(new ExpansionBoundary(macro, offset, true)); + if (macro.isFunctionStyle()) { + final TokenSource[] argInputs= new TokenSource[macro.getParameterPlaceholderList().length]; + endOffset= parseArguments(input, (FunctionStyleMacro) macro, argInputs); + TokenList[] clonedArgs= new TokenList[argInputs.length]; + TokenList[] expandedArgs= new TokenList[argInputs.length]; + for (int i = 0; i < argInputs.length; i++) { + final TokenSource argInput = argInputs[i]; + clonedArgs[i]= argInput.cloneTokens(); + final TokenList expandedArg= new TokenList(); + expandAll(argInput, forbidden, expandedArg); + expandedArgs[i]= expandedArg; + } + replaceArgs(macro, clonedArgs, expandedArgs, result); + } + else { + objStyleTokenPaste(macro, macro.getTokens(fDefinitionParser, fLexOptions), result); + } + result.append(new ExpansionBoundary(macro, endOffset, false)); + return result; + } + + private void expandAll(TokenSource input, IdentityHashMap forbidden, TokenList result) throws OffsetLimitReachedException { + Token t= input.removeFirst(); + while(t != null) { + switch(t.getType()) { + case CPreprocessor.tSCOPE_MARKER: + ((ExpansionBoundary) t).execute(forbidden); + result.append(t); + break; + case IToken.tIDENTIFIER: + PreprocessorMacro macro= (PreprocessorMacro) fDictionary.get(t.getCharImage()); + if (macro != null && !forbidden.containsKey(macro)) { + final boolean isFunctionStyle= macro.isFunctionStyle(); + if (!isFunctionStyle || input.findLParenthesis(forbidden)) { + // mstodo- image location + fImplicitMacroExpansions.add(fLocationMap.encounterImplicitMacroExpansion(macro, null)); + TokenList replacement= expandOne(macro, forbidden, input, t.getOffset(), t.getEndOffset()); + input.prepend(replacement); + t= null; + } + } + if (t != null) { + t.setType(CPreprocessor.tEXPANDED_IDENTIFIER); // prevent any further expansion + result.append(t); + } + break; + default: + result.append(t); + break; + } + t= input.removeFirst(); + } + } + + /** + * Expects that the identifier has been consumed. + * @throws OffsetLimitReachedException + */ + private int parseArguments(TokenSource input, FunctionStyleMacro macro, TokenSource[] result) throws OffsetLimitReachedException { + final int argCount= macro.getParameterPlaceholderList().length; + final boolean hasVarargs= macro.hasVarArgs() != FunctionStyleMacro.NO_VAARGS; + final int requiredArgs= hasVarargs ? argCount-1 : argCount; + int endOffset= 0; + int idx= 0; + int nesting = -1; + for (int i = 0; i < result.length; i++) { + result[i]= new TokenSource(false); + } + + loop: while (true) { + Token t= input.fetchFirst(); + if (t == null) { + break loop; + } + endOffset= t.getEndOffset(); + switch(t.getType()) { + case Lexer.tEND_OF_INPUT: + if (fCompletionMode) { + throw new OffsetLimitReachedException(ORIGIN, null); + } + break loop; + case IToken.tCOMPLETION: + throw new OffsetLimitReachedException(ORIGIN, t); + + case Lexer.tNEWLINE: + assert false; // we should not get any newlines from macros or the preprocessor. + break; + + case IToken.tLPAREN: + if (++nesting > 0) { + result[idx].append(t); + } + break; + case IToken.tRPAREN: + if (--nesting < 0) { + idx++; + break loop; + } + result[idx].append(t); + break; + + case IToken.tCOMMA: + if (nesting == 0) { + if (idx < argCount-1) { // next argument + idx++; + break; + } + else if (!hasVarargs) { + // too many arguments + handleProblem(IProblem.PREPROCESSOR_MACRO_USAGE_ERROR, macro.getNameCharArray()); + break loop; + } + } + // part of argument + result[idx].append(t); + break; + + default: + if (nesting < 0) { + assert false; // no leading parenthesis, which is checked before the method is called. + break loop; + } + result[idx].append(t); + break; + } + } + + if (idx < requiredArgs) { + handleProblem(IProblem.PREPROCESSOR_MACRO_USAGE_ERROR, macro.getNameCharArray()); + } + return endOffset; + } + + private void handleProblem(int problemID, char[] arg) { + fCpp.handleProblem(problemID, arg, fStartOffset, fEndOffset); + } + + private void replaceArgs(PreprocessorMacro macro, TokenList[] args, TokenList[] expandedArgs, TokenList result) { + TokenList input= macro.getTokens(fDefinitionParser, fLexOptions); + + Token n; + Token pasteArg1= null; + for (Token t= input.first(); t != null; t=n) { + n= (Token) t.getNext(); + boolean pasteNext= n != null && n.getType() == IToken.tPOUNDPOUND; + + switch(t.getType()) { + case CPreprocessor.tMACRO_PARAMETER: + int idx= ((PlaceHolderToken) t).getIndex(); + if (idx < args.length) { // be defensive + TokenList arg= pasteNext ? args[idx] : expandedArgs[idx]; + pasteArg1= cloneAndAppend(arg.first(), result, pasteNext); + } + break; + + case IToken.tPOUND: + StringBuffer buf= new StringBuffer(); + buf.append('"'); + if (n != null && n.getType() == CPreprocessor.tMACRO_PARAMETER) { + idx= ((PlaceHolderToken) n).getIndex(); + if (idx < args.length) { // be defensive + stringify(args[idx], buf); + } + n= (Token) n.getNext(); + pasteNext= n != null && n.getType() == IToken.tPOUNDPOUND; + } + buf.append('"'); + final int length= buf.length(); + final char[] image= new char[length]; + buf.getChars(0, length, image, 0); + + pasteArg1= appendToResult(new ImageToken(IToken.tSTRING, 0, 0, image), result, pasteNext); + break; + + case IToken.tPOUNDPOUND: + if (pasteArg1 != null) { + Token pasteArg2= null; + Token rest= null; + if (n != null) { + if (n.getType() == CPreprocessor.tMACRO_PARAMETER) { + idx= ((PlaceHolderToken) n).getIndex(); + if (idx < args.length) { // be defensive + TokenList arg= args[idx]; + pasteArg2= arg.first(); + if (pasteArg2 != null) { + rest= (Token) pasteArg2.getNext(); + } + } + } + else { + pasteArg2= n; + } + n= (Token) n.getNext(); + pasteNext= n != null && n.getType() == IToken.tPOUNDPOUND; + } + + t= tokenpaste(pasteArg1, pasteArg2, macro); + if (t != null) { + pasteArg1= appendToResult((Token) t.clone(), result, pasteNext && rest == null); + } + if (rest != null) { + pasteArg1= cloneAndAppend(rest, result, pasteNext); + } + } + break; + + default: + pasteArg1= appendToResult((Token) t.clone(), result, pasteNext); + break; + } + } + } + + private void objStyleTokenPaste(PreprocessorMacro macro, TokenList input, TokenList result) { + Token n; + Token pasteArg1= null; + for (Token t= input.first(); t != null; t=n) { + n= (Token) t.getNext(); + boolean pasteNext= n != null && n.getType() == IToken.tPOUNDPOUND; + + switch(t.getType()) { + case IToken.tPOUNDPOUND: + if (pasteArg1 != null) { + Token pasteArg2= null; + if (n != null) { + pasteArg2= n; + n= (Token) n.getNext(); + pasteNext= n != null && n.getType() == IToken.tPOUNDPOUND; + } + + t= tokenpaste(pasteArg1, pasteArg2, macro); + if (t != null) { + pasteArg1= appendToResult((Token) t.clone(), result, pasteNext); + } + } + break; + + default: + pasteArg1= appendToResult((Token) t.clone(), result, pasteNext); + break; + } + } + } + + private Token appendToResult(Token t, TokenList result, boolean pasteNext) { + if (pasteNext) { + return t; + } + result.append(t); + return null; + } + + private Token cloneAndAppend(Token tokens, TokenList result, boolean pasteNext) { + Token t= tokens; + Token r= t == null ? null : (Token) t.getNext(); + while (r != null) { + result.append((Token) t.clone()); + t= r; + r= (Token) r.getNext(); + } + if (t != null && !pasteNext) { + result.append((Token) t.clone()); + return null; + } + return t; + } + + private Token tokenpaste(Token arg1, Token arg2, PreprocessorMacro macro) { + if (arg2 == null) { + if (arg1.getType() == IToken.tCOMMA) { // gcc-extension for variadic macros + return null; + } + return arg1; + } + + final char[] image1= arg1.getCharImage(); + final char[] image2= arg2.getCharImage(); + final int l1 = image1.length; + final int l2 = image2.length; + final char[] image= new char[l1+l2]; + System.arraycopy(image1, 0, image, 0, l1); + System.arraycopy(image2, 0, image, l1, l2); + Lexer lex= new Lexer(image, fLexOptions, ILexerLog.NULL); + try { + Token t1= lex.nextToken(); + Token t2= lex.nextToken(); + if (t1.getType() != Lexer.tEND_OF_INPUT && t2.getType() == Lexer.tEND_OF_INPUT) { + t1.setOffset(arg1.getOffset(), arg2.getEndOffset()); + return t1; + } + } catch (OffsetLimitReachedException e) { + } + handleProblem(IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, macro.getNameCharArray()); + return null; + } + + private void stringify(TokenList tokenList, StringBuffer buf) { + Token t= tokenList.first(); + if (t == null) { + return; + } + int endOffset= t.getOffset(); + for (; t != null; t= (Token) t.getNext()) { + switch(t.getType()) { + case IToken.tSTRING: + case IToken.tLSTRING: + case IToken.tCHAR: + case IToken.tLCHAR: + if (endOffset < t.getOffset()) { + buf.append(' '); + } + endOffset= t.getEndOffset(); + final char[] image= t.getCharImage(); + for (int i = 0; i < image.length; i++) { + final char c = image[i]; + if (c == '"' || c == '\\') { + buf.append('\\'); + } + buf.append(c); + } + break; + + case CPreprocessor.tSCOPE_MARKER: + ExpansionBoundary sm= (ExpansionBoundary) t; + if (sm.fIsStart) { + if (endOffset < t.getOffset()) { + buf.append(' '); + } + endOffset= Integer.MAX_VALUE; + } + else { + endOffset= t.getEndOffset(); + } + break; + + default: + if (endOffset < t.getOffset()) { + buf.append(' '); + } + endOffset= t.getEndOffset(); + buf.append(t.getCharImage()); + break; + } + + } + } + + public IASTName[] createImplicitExpansions() { + IASTName[] result= (IASTName[]) fImplicitMacroExpansions.toArray(new IASTName[fImplicitMacroExpansions.size()]); + fImplicitMacroExpansions.clear(); + return result; + } + + public ImageLocationInfo[] createImageLocations(TokenList replacement) { + // mstodo- image locations + return ImageLocationInfo.NO_LOCATION_INFOS; + } + + + public int adjustOffsets(TokenList replacement) { + int offset= 0; + Token l= null; + for (Token t= replacement.first(); t!=null; t= (Token) t.getNext()) { + switch(t.getType()) { + case CPreprocessor.tEXPANDED_IDENTIFIER: + t.setType(IToken.tIDENTIFIER); + break; + case CPreprocessor.tSCOPE_MARKER: + replacement.removeBehind(l); + continue; + } + t.setOffset(offset, ++offset); + l= t; + } + return offset; + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/PreprocessorMacro.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/PreprocessorMacro.java index 7d950fc5912..29d9e8d8ef7 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/PreprocessorMacro.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/PreprocessorMacro.java @@ -14,8 +14,10 @@ import org.eclipse.cdt.core.dom.ILinkage; import org.eclipse.cdt.core.dom.ast.IMacroBinding; import org.eclipse.cdt.core.dom.ast.IScope; import org.eclipse.cdt.core.parser.Keywords; +import org.eclipse.cdt.core.parser.OffsetLimitReachedException; import org.eclipse.cdt.core.parser.util.CharArrayUtils; import org.eclipse.cdt.internal.core.dom.Linkage; +import org.eclipse.cdt.internal.core.parser.scanner.Lexer.LexerOptions; /** * Models macros used by the preprocessor @@ -51,10 +53,33 @@ abstract class PreprocessorMacro implements IMacroBinding { public char[][] getParameterList() { return null; } + + public char[][] getParameterPlaceholderList() { + return null; + } public Object getAdapter(Class clazz) { return null; } + + public String toString() { + char[][] p= getParameterList(); + if (p == null) { + return getName(); + } + StringBuffer buf= new StringBuffer(); + buf.append(getNameCharArray()); + buf.append('('); + for (int i = 0; i < p.length; i++) { + if (i>0) { + buf.append(','); + } + buf.append(p[i]); + } + buf.append(')'); + return buf.toString(); + } + public abstract TokenList getTokens(MacroDefinitionParser parser, LexerOptions lexOptions); } abstract class DynamicStyleMacro extends PreprocessorMacro { @@ -63,29 +88,36 @@ abstract class DynamicStyleMacro extends PreprocessorMacro { super(name); } public char[] getExpansion() { + return getExpansionImage(); + } + public char[] getExpansionImage() { return execute().getCharImage(); } public abstract Token execute(); + + public TokenList getTokens(MacroDefinitionParser mdp, LexerOptions lexOptions) { + TokenList result= new TokenList(); + result.append(execute()); + return result; + } } class ObjectStyleMacro extends PreprocessorMacro { - private static final Token NOT_INITIALIZED = new SimpleToken(0,0,0); - private final char[] fExpansion; final int fExpansionOffset; final int fEndOffset; -// private Token fExpansionTokens; + private TokenList fExpansionTokens; public ObjectStyleMacro(char[] name, char[] expansion) { - this(name, 0, expansion.length, NOT_INITIALIZED, expansion); + this(name, 0, expansion.length, null, expansion); } - public ObjectStyleMacro(char[] name, int expansionOffset, int endOffset, Token expansion, char[] source) { + public ObjectStyleMacro(char[] name, int expansionOffset, int endOffset, TokenList expansion, char[] source) { super(name); fExpansionOffset= expansionOffset; fEndOffset= endOffset; fExpansion= source; -// fExpansionTokens= expansion; + fExpansionTokens= expansion; } public int findParameter(char[] tokenImage) { @@ -93,14 +125,47 @@ class ObjectStyleMacro extends PreprocessorMacro { } public char[] getExpansion() { + TokenList tl= getTokens(new MacroDefinitionParser(), new LexerOptions()); + StringBuffer buf= new StringBuffer(); + Token t= tl.first(); + if (t == null) { + return CharArrayUtils.EMPTY; + } + int endOffset= t.getOffset(); + for (; t != null; t= (Token) t.getNext()) { + if (endOffset < t.getOffset()) { + buf.append(' '); + } + buf.append(t.getCharImage()); + endOffset= t.getEndOffset(); + } + final int length= buf.length(); + final char[] expansion= new char[length]; + buf.getChars(0, length, expansion, 0); + return expansion; + } + + public char[] getExpansionImage() { final int length = fEndOffset - fExpansionOffset; if (length == fExpansion.length) { return fExpansion; } char[] result= new char[length]; - System.arraycopy(fExpansion, fEndOffset, result, 0, length); + System.arraycopy(fExpansion, fExpansionOffset, result, 0, length); return result; } + + public TokenList getTokens(MacroDefinitionParser mdp, LexerOptions lexOptions) { + if (fExpansionTokens == null) { + fExpansionTokens= new TokenList(); + Lexer lex= new Lexer(fExpansion, fExpansionOffset, fEndOffset, lexOptions, ILexerLog.NULL); + try { + mdp.parseExpansion(lex, ILexerLog.NULL, getNameCharArray(), getParameterPlaceholderList(), fExpansionTokens); + } catch (OffsetLimitReachedException e) { + } + } + return fExpansionTokens; + } } @@ -120,13 +185,35 @@ class FunctionStyleMacro extends ObjectStyleMacro { } public FunctionStyleMacro(char[] name, char[][] paramList, int hasVarArgs, int expansionFileOffset, int endFileOffset, - Token expansion, char[] source) { + TokenList expansion, char[] source) { super(name, expansionFileOffset, endFileOffset, expansion, source); fParamList = paramList; fHasVarArgs= hasVarArgs; } public char[][] getParameterList() { + final int length = fParamList.length; + if (fHasVarArgs == NO_VAARGS || length==0) { + return fParamList; + } + char[][] result= new char[length][]; + System.arraycopy(fParamList, 0, result, 0, length-1); + if (fHasVarArgs == VAARGS) { + result[length-1] = Keywords.cVA_ARGS; + } + else { + final char[] param= fParamList[length-1]; + final int plen= param.length; + final int elen = Keywords.cpELLIPSIS.length; + final char[] rp= new char[plen+elen]; + System.arraycopy(param, 0, rp, 0, plen); + System.arraycopy(Keywords.cpELLIPSIS, 0, rp, plen, elen); + result[length-1]= rp; + } + return result; + } + + public char[][] getParameterPlaceholderList() { return fParamList; } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContext.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContext.java index 49e1fed4b5d..f1d586ec97c 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContext.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContext.java @@ -74,6 +74,15 @@ abstract class ScannerContext { public abstract boolean changeBranch(Integer state); - public abstract Token currentPPToken(); + /** + * Returns the current token from this context. When called before calling {@link #nextPPToken()} + * a token of type {@link Lexer#tBEFORE_INPUT} will be returned. + * @since 5.0 + */ + public abstract Token currentLexerToken(); + + /** + * Returns the next token from this context. + */ public abstract Token nextPPToken() throws OffsetLimitReachedException; } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextFile.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextFile.java index 33643e4d4d5..0c959e88d0a 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextFile.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextFile.java @@ -33,7 +33,7 @@ public class ScannerContextFile extends ScannerContext { fLexer= lexer; } - public Token currentPPToken() { + public Token currentLexerToken() { return fLexer.currentToken(); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextMacroExpansion.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextMacroExpansion.java new file mode 100644 index 00000000000..cd457e351f8 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextMacroExpansion.java @@ -0,0 +1,49 @@ +/******************************************************************************* + * Copyright (c) 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + + + +public class ScannerContextMacroExpansion extends ScannerContext { + private static final Token END_TOKEN = new SimpleToken(Lexer.tEND_OF_INPUT, 0, 0); + + private Token fTokens; + + public ScannerContextMacroExpansion(ILocationCtx ctx, ScannerContext parent, TokenList tokens) { + super(ctx, parent); + fTokens= tokens.first(); + } + + public boolean changeBranch(Integer state) { + return false; + } + + public Token currentLexerToken() { + Token t= fTokens; + if (t == null) { + return END_TOKEN; + } + return t; + } + + public Lexer getLexerForPPDirective() { + return null; + } + + public Token nextPPToken() { + fTokens= (Token) fTokens.getNext(); + return currentLexerToken(); + } + + public boolean expandsMacros() { + return false; + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextPPDirective.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextPPDirective.java index e4d02087686..aca3be7c140 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextPPDirective.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextPPDirective.java @@ -36,13 +36,14 @@ public final class ScannerContextPPDirective extends ScannerContext { public ScannerContextPPDirective(Lexer lexer, boolean convertDefinedToken) { super(null, null); fLexer= lexer; + fConvertDefinedToken= convertDefinedToken; + final Token currentToken = lexer.currentToken(); fLastEndOffset= currentToken.getOffset(); fToken= convertToken(currentToken); - fConvertDefinedToken= convertDefinedToken; } - public Token currentPPToken() { + public Token currentLexerToken() { return fToken; } @@ -69,10 +70,10 @@ public final class ScannerContextPPDirective extends ScannerContext { private Token convertToken(Token t) { switch (t.getType()) { case Lexer.tNEWLINE: - t= new SimpleToken(Lexer.tEND_OF_INPUT, fToken.getEndOffset(), fToken.getEndOffset()); + t= new SimpleToken(Lexer.tEND_OF_INPUT, t.getEndOffset(), t.getEndOffset()); break; case IToken.tIDENTIFIER: - if (fConvertDefinedToken && CharArrayUtils.equals(Keywords.cDEFINED, fToken.getCharImage())) { + if (fConvertDefinedToken && CharArrayUtils.equals(Keywords.cDEFINED, t.getCharImage())) { t.setType(CPreprocessor.tDEFINED); fPreventMacroExpansion= STATE_DEFINED; } @@ -109,10 +110,6 @@ public final class ScannerContextPPDirective extends ScannerContext { return fPreventMacroExpansion == 0; } - public void setInsideIncludeDirective() { - fLexer.setInsideIncludeDirective(); - } - public int getLastEndOffset() { return fLastEndOffset; } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Token.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Token.java index 9f68d6d5c41..1fbab02d0ae 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Token.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Token.java @@ -17,11 +17,10 @@ import org.eclipse.cdt.core.parser.IToken; * them on to the parsers. * @since 5.0 */ -public abstract class Token implements IToken { +public abstract class Token implements IToken, Cloneable { private int fKind; private int fOffset; private int fEndOffset; - private IToken fNextToken; Token(int kind, int offset, int endOffset) { @@ -59,9 +58,18 @@ public abstract class Token implements IToken { fNextToken= t; } + public void setOffset(int offset, int endOffset) { + fOffset= offset; + fEndOffset= endOffset; + } + public abstract char[] getCharImage(); - + + public String toString() { + return getImage(); + } + public boolean isOperator() { return TokenUtil.isOperator(fKind); } @@ -70,29 +78,36 @@ public abstract class Token implements IToken { return new String(getCharImage()); } + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException e) { + return null; + } + } public char[] getFilename() { - // mstodo + // mstodo- parser removal throw new UnsupportedOperationException(); } public boolean looksLikeExpression() { - // mstodo + // mstodo- parser removal throw new UnsupportedOperationException(); } public boolean canBeAPrefix() { - // mstodo + // mstodo- parser removal throw new UnsupportedOperationException(); } public int getLineNumber() { - // mstodo + // mstodo- parser removal throw new UnsupportedOperationException(); } public boolean isPointer() { - // mstodo + // mstodo- parser removal throw new UnsupportedOperationException(); } } @@ -107,6 +122,23 @@ class SimpleToken extends Token { } } +class PlaceHolderToken extends ImageToken { + private final int fIndex; + + public PlaceHolderToken(int type, int idx, int offset, int endOffset, char[] name) { + super(type, offset, endOffset, name); + fIndex= idx; + } + + public int getIndex() { + return fIndex; + } + + public String toString() { + return "[" + fIndex + "]"; //$NON-NLS-1$ //$NON-NLS-2$ + } +} + class DigraphToken extends Token { public DigraphToken(int kind, int offset, int endOffset) { super(kind, offset, endOffset); @@ -148,6 +180,11 @@ class SourceImageToken extends Token { } return fImage; } + + public void setOffset(int offset, int endOffset) { + getCharImage(); + super.setOffset(offset, endOffset); + } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenList.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenList.java new file mode 100644 index 00000000000..a5eaf11d494 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenList.java @@ -0,0 +1,85 @@ +/******************************************************************************* + * Copyright (c) 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +class TokenList { + private Token fFirst; + private Token fLast; + + final Token removeFirst() { + final Token first= fFirst; + if (first == fLast) { + fFirst= fLast= null; + return first; + } + else { + fFirst= (Token) first.getNext(); + return first; + } + } + + final public void append(Token t) { + if (fFirst == null) { + fFirst= fLast= t; + } + else { + fLast.setNext(t); + fLast= t; + } + t.setNext(null); + } + + final public void prepend(TokenList prepend) { + final Token first= prepend.fFirst; + if (first != null) { + final Token last= prepend.fLast; + last.setNext(fFirst); + fFirst= first; + if (fLast == null) { + fLast= last; + } + } + } + + final public TokenList cloneTokens() { + TokenList result= new TokenList(); + for (Token t= fFirst; t != null; t= (Token) t.getNext()) { + result.append((Token) t.clone()); + } + return result; + } + + final public Token first() { + return fFirst; + } + + public void removeBehind(Token l) { + if (l == null) { + Token t= fFirst; + if (t != null) { + t= (Token) t.getNext(); + fFirst= t; + if (t == null) { + fLast= null; + } + } + } + else { + final Token r= (Token) l.getNext(); + if (r != null) { + l.setNext(r.getNext()); + if (r == fLast) { + fLast= l; + } + } + } + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenUtil.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenUtil.java index a0412ec98cd..6b2576b5c3b 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenUtil.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenUtil.java @@ -118,7 +118,6 @@ public class TokenUtil { case IToken.tDOT: return Keywords.cpDOT; case IToken.tDIVASSIGN: return Keywords.cpDIVASSIGN; case IToken.tDIV: return Keywords.cpDIV; - case IToken.tBACKSLASH: return Keywords.cpBACKSLASH; case IGCCToken.tMIN: return Keywords.cpMIN; case IGCCToken.tMAX: return Keywords.cpMAX; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/MacroBinding.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/MacroBinding.java index f3c1b23e221..17a0f40f1c9 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/MacroBinding.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/MacroBinding.java @@ -69,4 +69,12 @@ public class MacroBinding extends PlatformObject implements IMacroBinding { } return null; } + + public char[] getExpansionImage() { + throw new UnsupportedOperationException(); + } + + public char[][] getParameterPlaceholderList() { + throw new UnsupportedOperationException(); + } }