Scanner2 - lots of little things trying to properly scan windows/stdio/iostream.

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java

@@ -1659,4 +1659,13 @@ public class Scanner2Test extends BaseScanner2Test
     	validateInteger("10");
     	validateEOF();
     }
+    
+    public void testWackyFunctionMacros() throws Exception {
+    	initializeScanner(
+    			"#define A(X) hi##X\n" +
+				"#define B(Y) A(Y)\n" +
+				"B(there)");
+    	validateIdentifier("hithere");
+    	validateEOF();
+    }
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/ExpressionEvaluator.java

@@ -199,16 +199,16 @@ public class ExpressionEvaluator {
 		switch (LA()) {
 			case tPLUS:
 				consume();
-				return expression();
+				return unaryExpression();
 			case tMINUS:
 				consume();
-				return - expression();
+				return - unaryExpression();
 			case tNOT:
 				consume();
-				return expression() == 0 ? 1 : 0;
+				return unaryExpression() == 0 ? 1 : 0;
 			case tCOMPL:
 				consume();
-				return ~ expression();
+				return ~ unaryExpression();
 			case tNUMBER:
 				return consume();
 			case t_defined:
@@ -228,20 +228,26 @@ public class ExpressionEvaluator {
 	}
 	
 	private long handleDefined() throws EvalException {
-		consume(); // the defined keyword
+		// We need to do some special handline to get the identifier without it being
-		
-		if (LA() != tLPAREN)
-			throw new EvalException("missing ( after defined");
-
-		// Now we need to do some special handline to get the identifier without it being
 		// expanded by macro expansion
 		skipWhiteSpace();
 		
 		char[] buffer = bufferStack[bufferStackPos];
 		int limit = bufferLimit[bufferStackPos];
+		if (++bufferPos[bufferStackPos] >= limit)
+			return 0;
 
 		// check first character
-		char c = buffer[++bufferPos[bufferStackPos]];
+		char c = buffer[bufferPos[bufferStackPos]];
+		boolean inParens = false;
+		if (c == '(') {
+			inParens = true;
+			skipWhiteSpace();
+			if (++bufferPos[bufferStackPos] >= limit)
+				return 0;
+			c = buffer[bufferPos[bufferStackPos]];
+		}
+
 		if (!((c >= 'A' && c <= 'Z') || c == '_' || (c >= 'a' && c <= 'z'))) {
 			throw new EvalException("illegal identifier in defined()");
 		}
@@ -260,14 +266,16 @@ public class ExpressionEvaluator {
 		--bufferPos[bufferStackPos];
 		
 		// consume to the closing paren;
-		while (true) {
+		if (inParens) {
 			skipWhiteSpace();
-			if (++bufferPos[bufferStackPos] >= limit)
+			if (++bufferPos[bufferStackPos] < limit
+					&& buffer[bufferPos[bufferStackPos]] != ')')
 				throw new EvalException("missing ) on defined");
-			if (buffer[bufferPos[bufferStackPos]] == ')')
-				break;
 		}
 
+		// Set up the lookahead to whatever comes next
+		nextToken();
+		
 		return definitions.get(buffer, idstart, idlen) != null ? 1 : 0;
 	}
 	
@@ -380,8 +388,7 @@ public class ExpressionEvaluator {
 						
 						// Check for defined(
 						pos = bufferPos[bufferStackPos];
-						if (pos + 1 < limit && buffer[pos + 1] == '('
+						if (CharArrayUtils.equals(buffer, start, len, _defined)) {
-								&& CharArrayUtils.equals(buffer, start, len, _defined)) {
 							tokenType = t_defined;
 							return;
 						}
@@ -443,23 +450,34 @@ public class ExpressionEvaluator {
 						}
 						
 						while (++bufferPos[bufferStackPos] < limit) {
-							int c = buffer[bufferPos[bufferStackPos]];
+							char c = buffer[bufferPos[bufferStackPos]];
 							if (c >= '0' && c <= '9') {
 								tokenValue *= (isHex ? 16 : 10);
 								tokenValue += c - '0';
+								continue;
 							} else if (isHex) {
 								if (c >= 'a' && c <= 'f') {
 									tokenValue *= 16;
 									tokenValue += c - 'a';
+									continue;
 								} else if (c >= 'A' && c <= 'F') {
 									tokenValue *= 16;
 									tokenValue += c - 'A';
+									continue;
 								}
 							}
+							
+							// end of number
+							if (c == 'L') {
+								// eat the long
+								++bufferPos[bufferStackPos];
+							}
+							
+							// done
+							break;
 						}
 						--bufferPos[bufferStackPos];
 						return;
-
 					case '(':
 						tokenType = tLPAREN;
 						return;
@@ -501,10 +519,20 @@ public class ExpressionEvaluator {
 						return;
 					
 					case '&':
+						if (pos + 1 < limit && buffer[pos + 1] == '&') {
+							++bufferPos[bufferStackPos];
+							tokenType = tAND;
+							return;
+						}
 						tokenType = tBITAND;
 						return;
 					
 					case '|':
+						if (pos + 1 < limit && buffer[pos + 1] == '|') {
+							++bufferPos[bufferStackPos];
+							tokenType = tOR;
+							return;
+						}
 						tokenType = tBITOR;
 						return;
 					
@@ -517,11 +545,11 @@ public class ExpressionEvaluator {
 						return;
 					
 					case '=':
-						if (pos + 1 < limit)
+						if (pos + 1 < limit && buffer[pos + 1] == '=') {
-							if (buffer[pos + 1] == '=') {
+							++bufferPos[bufferStackPos];
-								tokenType = tEQUAL;
+							tokenType = tEQUAL;
-								return;
+							return;
-							}
+						}
 						throw new EvalException("assignment not allowed");
 					
 					case '<':
@@ -712,9 +740,10 @@ public class ExpressionEvaluator {
 									break;
 								}
 							}
+							continue;
 						}
 					}
-					continue;
+					break;
 				case '\\':
 					if (pos + 1 < limit && buffer[pos + 1] == '\n') {
 						// \n is a whitespace

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java

@@ -628,11 +628,16 @@ public class Scanner2 implements IScanner, IScannerData {
 		--bufferPos[bufferStackPos];
 		
 		// Check for macro expansion
+		
+		// First look up the stack at the function style macro expansion args
 		Object expObject = null;
-		if (bufferData[bufferStackPos] instanceof FunctionStyleMacro.Expansion) {
+		int tmpPos = bufferStackPos + 1;
-			// first check if name is a macro arg
+		while (--tmpPos >= 0
-			expObject = ((FunctionStyleMacro.Expansion)bufferData[bufferStackPos])
+				&& bufferData[tmpPos] instanceof FunctionStyleMacro.Expansion) {
+			expObject = ((FunctionStyleMacro.Expansion)bufferData[tmpPos])
 				.definitions.get(buffer, start, len);
+			if (expObject != null)
+				break;
 		}
 
 		if (expObject == null) {
@@ -964,8 +969,11 @@ public class Scanner2 implements IScanner, IScannerData {
 						start = bufferPos[bufferStackPos];
 						skipToNewLine();
 						len = bufferPos[bufferStackPos] - start;
-						if (expressionEvaluator.evaluate(buffer, start, len, definitions) == 0)
+						if (expressionEvaluator.evaluate(buffer, start, len, definitions) == 0) {
+							//System.out.println("#if <FALSE> " + new String(buffer,start+1,len-1));
 							skipOverConditionalCode(true);
+						} //else
+							//System.out.println("#if <TRUE> " + new String(buffer,start+1,len-1));
 						return;
 					case ppElse:
 					case ppElif:
@@ -1101,6 +1109,7 @@ public class Scanner2 implements IScanner, IScannerData {
 		--bufferPos[bufferStackPos];
 		char[] name = new char[idlen];
 		System.arraycopy(buffer, idstart, name, 0, idlen);
+		//System.out.println("#define " + new String(buffer, idstart, idlen));
 		
 		// Now check for function style macro to store the arguments
 		char[][] arglist = null;
@@ -1230,10 +1239,15 @@ public class Scanner2 implements IScanner, IScannerData {
 
 		skipToNewLine();
 
-		if ((definitions.get(buffer, idstart, idlen) != null) == positive)
+		if ((definitions.get(buffer, idstart, idlen) != null) == positive) {
+			//System.out.println((positive ? "#ifdef" : "#ifndef")
+			//		+ " <TRUE> " + new String(buffer, idstart, idlen));
 			// continue on
 			return;
+		}
 		
+		//System.out.println((positive ? "#ifdef" : "#ifndef")
+		//		+ " <FALSE> " + new String(buffer, idstart, idlen));
 		// skip over this group
 		skipOverConditionalCode(true);
 	}
@@ -1273,6 +1287,7 @@ public class Scanner2 implements IScanner, IScannerData {
 						switch (type) {
 							case ppIfdef:
 							case ppIfndef:
+							case ppIf:
 								++nesting;
 								break;
 							case ppElse:
@@ -1510,6 +1525,8 @@ public class Scanner2 implements IScanner, IScannerData {
 		char[] buffer = bufferStack[bufferStackPos];
 		int limit = bufferLimit[bufferStackPos];
 		
+		skipOverWhiteSpace();
+		
 		if (++bufferPos[bufferStackPos] >= limit
 				|| buffer[bufferPos[bufferStackPos]] != '(')
 			return;
@@ -1578,31 +1595,43 @@ public class Scanner2 implements IScanner, IScannerData {
 			pushContext(expText, exp);
 	}
 
-	// gcc built-ins
+	// standard built-ins
 	private static final ObjectStyleMacro __cplusplus
 		= new ObjectStyleMacro("__cplusplus".toCharArray(), "1".toCharArray());
-	private static final ObjectStyleMacro __STDC_HOSTED__
+	private static final ObjectStyleMacro __STDC__
-		= new ObjectStyleMacro("__STDC_HOSTED__".toCharArray(), "0".toCharArray());
+		= new ObjectStyleMacro("__STDC__".toCharArray(), "1".toCharArray());
-	private static final ObjectStyleMacro __STDC_VERSION__
+	
-		= new ObjectStyleMacro("__STDC_VERSION__".toCharArray(), "199001L".toCharArray());
+	// gcc built-ins
+	private static final ObjectStyleMacro __inline__
+		= new ObjectStyleMacro("__inline__".toCharArray(), "inline".toCharArray());
+	private static final ObjectStyleMacro __extension__
+		= new ObjectStyleMacro("__extension__".toCharArray(), emptyCharArray);
+	private static final ObjectStyleMacro __asm__
+		= new ObjectStyleMacro("__asm__".toCharArray(), "asm".toCharArray());
+	private static final ObjectStyleMacro __restrict__
+		= new ObjectStyleMacro("__restrict__".toCharArray(), "restrict".toCharArray());
+	private static final ObjectStyleMacro __restrict
+		= new ObjectStyleMacro("__restrict".toCharArray(), "restrict".toCharArray());
 	private static final FunctionStyleMacro __attribute__
 		= new FunctionStyleMacro(
 				"__attribute__".toCharArray(),
 				emptyCharArray,
 				new char[][] { "arg".toCharArray() });
 	
-	// standard built-ins
-	private static final ObjectStyleMacro __STDC__
-		= new ObjectStyleMacro("__STDC__".toCharArray(), "1".toCharArray());
-	
 	protected void setupBuiltInMacros() {
 
-		// gcc extensions
+		definitions.put(__STDC__.name, __STDC__);
 		if( language == ParserLanguage.CPP )
 			definitions.put(__cplusplus.name, __cplusplus);
-		definitions.put(__STDC_HOSTED__.name, __STDC_HOSTED__);
+
-		definitions.put(__STDC_VERSION__.name, __STDC_VERSION__);
+		// gcc extensions
+		definitions.put(__inline__.name, __inline__);
+		definitions.put(__extension__.name, __extension__);
 		definitions.put(__attribute__.name, __attribute__);
+		definitions.put(__restrict__.name, __restrict__);
+		definitions.put(__restrict.name, __restrict);
+		if( language == ParserLanguage.CPP )
+			definitions.put(__asm__.name, __asm__);
 		
 		/*
 		
@@ -1639,7 +1668,6 @@ public class Scanner2 implements IScanner, IScannerData {
 		*/
 		
 		// standard extensions
-		definitions.put(__STDC__.name, __STDC__);
 
 		/*
 		if( getDefinition(__STDC__) == null )