From 8a05f34b0bfbee3b3b2aba6c6605e7aeb52d6d5f Mon Sep 17 00:00:00 2001 From: Doug Schaefer Date: Thu, 17 Jun 2004 21:27:37 +0000 Subject: [PATCH] Scanner2 - lots of little things trying to properly scan windows/stdio/iostream. --- .../parser/tests/scanner2/Scanner2Test.java | 9 +++ .../parser/scanner2/ExpressionEvaluator.java | 79 +++++++++++++------ .../core/parser/scanner2/Scanner2.java | 70 +++++++++++----- 3 files changed, 112 insertions(+), 46 deletions(-) diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java index 0877a7a8ee8..de9b196648f 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java @@ -1659,4 +1659,13 @@ public class Scanner2Test extends BaseScanner2Test validateInteger("10"); validateEOF(); } + + public void testWackyFunctionMacros() throws Exception { + initializeScanner( + "#define A(X) hi##X\n" + + "#define B(Y) A(Y)\n" + + "B(there)"); + validateIdentifier("hithere"); + validateEOF(); + } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/ExpressionEvaluator.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/ExpressionEvaluator.java index 7395acb80f7..71d835f0edd 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/ExpressionEvaluator.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/ExpressionEvaluator.java @@ -199,16 +199,16 @@ public class ExpressionEvaluator { switch (LA()) { case tPLUS: consume(); - return expression(); + return unaryExpression(); case tMINUS: consume(); - return - expression(); + return - unaryExpression(); case tNOT: consume(); - return expression() == 0 ? 1 : 0; + return unaryExpression() == 0 ? 1 : 0; case tCOMPL: consume(); - return ~ expression(); + return ~ unaryExpression(); case tNUMBER: return consume(); case t_defined: @@ -228,20 +228,26 @@ public class ExpressionEvaluator { } private long handleDefined() throws EvalException { - consume(); // the defined keyword - - if (LA() != tLPAREN) - throw new EvalException("missing ( after defined"); - - // Now we need to do some special handline to get the identifier without it being + // We need to do some special handline to get the identifier without it being // expanded by macro expansion skipWhiteSpace(); char[] buffer = bufferStack[bufferStackPos]; int limit = bufferLimit[bufferStackPos]; + if (++bufferPos[bufferStackPos] >= limit) + return 0; // check first character - char c = buffer[++bufferPos[bufferStackPos]]; + char c = buffer[bufferPos[bufferStackPos]]; + boolean inParens = false; + if (c == '(') { + inParens = true; + skipWhiteSpace(); + if (++bufferPos[bufferStackPos] >= limit) + return 0; + c = buffer[bufferPos[bufferStackPos]]; + } + if (!((c >= 'A' && c <= 'Z') || c == '_' || (c >= 'a' && c <= 'z'))) { throw new EvalException("illegal identifier in defined()"); } @@ -260,14 +266,16 @@ public class ExpressionEvaluator { --bufferPos[bufferStackPos]; // consume to the closing paren; - while (true) { + if (inParens) { skipWhiteSpace(); - if (++bufferPos[bufferStackPos] >= limit) + if (++bufferPos[bufferStackPos] < limit + && buffer[bufferPos[bufferStackPos]] != ')') throw new EvalException("missing ) on defined"); - if (buffer[bufferPos[bufferStackPos]] == ')') - break; } + // Set up the lookahead to whatever comes next + nextToken(); + return definitions.get(buffer, idstart, idlen) != null ? 1 : 0; } @@ -380,8 +388,7 @@ public class ExpressionEvaluator { // Check for defined( pos = bufferPos[bufferStackPos]; - if (pos + 1 < limit && buffer[pos + 1] == '(' - && CharArrayUtils.equals(buffer, start, len, _defined)) { + if (CharArrayUtils.equals(buffer, start, len, _defined)) { tokenType = t_defined; return; } @@ -443,23 +450,34 @@ public class ExpressionEvaluator { } while (++bufferPos[bufferStackPos] < limit) { - int c = buffer[bufferPos[bufferStackPos]]; + char c = buffer[bufferPos[bufferStackPos]]; if (c >= '0' && c <= '9') { tokenValue *= (isHex ? 16 : 10); tokenValue += c - '0'; + continue; } else if (isHex) { if (c >= 'a' && c <= 'f') { tokenValue *= 16; tokenValue += c - 'a'; + continue; } else if (c >= 'A' && c <= 'F') { tokenValue *= 16; tokenValue += c - 'A'; + continue; } } + + // end of number + if (c == 'L') { + // eat the long + ++bufferPos[bufferStackPos]; + } + + // done + break; } --bufferPos[bufferStackPos]; return; - case '(': tokenType = tLPAREN; return; @@ -501,10 +519,20 @@ public class ExpressionEvaluator { return; case '&': + if (pos + 1 < limit && buffer[pos + 1] == '&') { + ++bufferPos[bufferStackPos]; + tokenType = tAND; + return; + } tokenType = tBITAND; return; case '|': + if (pos + 1 < limit && buffer[pos + 1] == '|') { + ++bufferPos[bufferStackPos]; + tokenType = tOR; + return; + } tokenType = tBITOR; return; @@ -517,11 +545,11 @@ public class ExpressionEvaluator { return; case '=': - if (pos + 1 < limit) - if (buffer[pos + 1] == '=') { - tokenType = tEQUAL; - return; - } + if (pos + 1 < limit && buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + tokenType = tEQUAL; + return; + } throw new EvalException("assignment not allowed"); case '<': @@ -712,9 +740,10 @@ public class ExpressionEvaluator { break; } } + continue; } } - continue; + break; case '\\': if (pos + 1 < limit && buffer[pos + 1] == '\n') { // \n is a whitespace diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java index b87fbd4597f..b3b530d202c 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java @@ -628,11 +628,16 @@ public class Scanner2 implements IScanner, IScannerData { --bufferPos[bufferStackPos]; // Check for macro expansion + + // First look up the stack at the function style macro expansion args Object expObject = null; - if (bufferData[bufferStackPos] instanceof FunctionStyleMacro.Expansion) { - // first check if name is a macro arg - expObject = ((FunctionStyleMacro.Expansion)bufferData[bufferStackPos]) + int tmpPos = bufferStackPos + 1; + while (--tmpPos >= 0 + && bufferData[tmpPos] instanceof FunctionStyleMacro.Expansion) { + expObject = ((FunctionStyleMacro.Expansion)bufferData[tmpPos]) .definitions.get(buffer, start, len); + if (expObject != null) + break; } if (expObject == null) { @@ -964,8 +969,11 @@ public class Scanner2 implements IScanner, IScannerData { start = bufferPos[bufferStackPos]; skipToNewLine(); len = bufferPos[bufferStackPos] - start; - if (expressionEvaluator.evaluate(buffer, start, len, definitions) == 0) + if (expressionEvaluator.evaluate(buffer, start, len, definitions) == 0) { + //System.out.println("#if " + new String(buffer,start+1,len-1)); skipOverConditionalCode(true); + } //else + //System.out.println("#if " + new String(buffer,start+1,len-1)); return; case ppElse: case ppElif: @@ -1101,6 +1109,7 @@ public class Scanner2 implements IScanner, IScannerData { --bufferPos[bufferStackPos]; char[] name = new char[idlen]; System.arraycopy(buffer, idstart, name, 0, idlen); + //System.out.println("#define " + new String(buffer, idstart, idlen)); // Now check for function style macro to store the arguments char[][] arglist = null; @@ -1229,11 +1238,16 @@ public class Scanner2 implements IScanner, IScannerData { --bufferPos[bufferStackPos]; skipToNewLine(); - - if ((definitions.get(buffer, idstart, idlen) != null) == positive) + + if ((definitions.get(buffer, idstart, idlen) != null) == positive) { + //System.out.println((positive ? "#ifdef" : "#ifndef") + // + " " + new String(buffer, idstart, idlen)); // continue on return; - + } + + //System.out.println((positive ? "#ifdef" : "#ifndef") + // + " " + new String(buffer, idstart, idlen)); // skip over this group skipOverConditionalCode(true); } @@ -1273,6 +1287,7 @@ public class Scanner2 implements IScanner, IScannerData { switch (type) { case ppIfdef: case ppIfndef: + case ppIf: ++nesting; break; case ppElse: @@ -1510,6 +1525,8 @@ public class Scanner2 implements IScanner, IScannerData { char[] buffer = bufferStack[bufferStackPos]; int limit = bufferLimit[bufferStackPos]; + skipOverWhiteSpace(); + if (++bufferPos[bufferStackPos] >= limit || buffer[bufferPos[bufferStackPos]] != '(') return; @@ -1578,31 +1595,43 @@ public class Scanner2 implements IScanner, IScannerData { pushContext(expText, exp); } - // gcc built-ins + // standard built-ins private static final ObjectStyleMacro __cplusplus = new ObjectStyleMacro("__cplusplus".toCharArray(), "1".toCharArray()); - private static final ObjectStyleMacro __STDC_HOSTED__ - = new ObjectStyleMacro("__STDC_HOSTED__".toCharArray(), "0".toCharArray()); - private static final ObjectStyleMacro __STDC_VERSION__ - = new ObjectStyleMacro("__STDC_VERSION__".toCharArray(), "199001L".toCharArray()); + private static final ObjectStyleMacro __STDC__ + = new ObjectStyleMacro("__STDC__".toCharArray(), "1".toCharArray()); + + // gcc built-ins + private static final ObjectStyleMacro __inline__ + = new ObjectStyleMacro("__inline__".toCharArray(), "inline".toCharArray()); + private static final ObjectStyleMacro __extension__ + = new ObjectStyleMacro("__extension__".toCharArray(), emptyCharArray); + private static final ObjectStyleMacro __asm__ + = new ObjectStyleMacro("__asm__".toCharArray(), "asm".toCharArray()); + private static final ObjectStyleMacro __restrict__ + = new ObjectStyleMacro("__restrict__".toCharArray(), "restrict".toCharArray()); + private static final ObjectStyleMacro __restrict + = new ObjectStyleMacro("__restrict".toCharArray(), "restrict".toCharArray()); private static final FunctionStyleMacro __attribute__ = new FunctionStyleMacro( "__attribute__".toCharArray(), emptyCharArray, new char[][] { "arg".toCharArray() }); - // standard built-ins - private static final ObjectStyleMacro __STDC__ - = new ObjectStyleMacro("__STDC__".toCharArray(), "1".toCharArray()); - protected void setupBuiltInMacros() { - - // gcc extensions + + definitions.put(__STDC__.name, __STDC__); if( language == ParserLanguage.CPP ) definitions.put(__cplusplus.name, __cplusplus); - definitions.put(__STDC_HOSTED__.name, __STDC_HOSTED__); - definitions.put(__STDC_VERSION__.name, __STDC_VERSION__); + + // gcc extensions + definitions.put(__inline__.name, __inline__); + definitions.put(__extension__.name, __extension__); definitions.put(__attribute__.name, __attribute__); + definitions.put(__restrict__.name, __restrict__); + definitions.put(__restrict.name, __restrict); + if( language == ParserLanguage.CPP ) + definitions.put(__asm__.name, __asm__); /* @@ -1639,7 +1668,6 @@ public class Scanner2 implements IScanner, IScannerData { */ // standard extensions - definitions.put(__STDC__.name, __STDC__); /* if( getDefinition(__STDC__) == null )