Patch for Devin Steffler

Fixed 75532 [Scanner] Wrong compare if the two declarations are in other number system.
- Octal support has been added along side Hex support in ExpressionEvaluator.java.
- SourceIndexerRequestor.java now handles IProblem.SCANNER_RELATED IProblems
- IProblems can now be thrown easily from ExpressionEvaluator.java
- IProblems are reported for malformed Decimal/Hex/Octal numbers as well as the existing EvalExceptions that occur in ExpressionEvaluator.java
- tests were added to check Octal support and that the IProblems are raised properly within ExpressionEvaluator.java

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/CompleteParseASTTest.java

@@ -2247,20 +2247,30 @@ public class CompleteParseASTTest extends CompleteParseBaseTest
 	{
     	try {
         	Writer writer = new StringWriter();
-        	writer.write( "#if 2147483647 == 0x7fffffff\n");
+        	writer.write( "#if 2147483647 == 0x7fffffff\n"); // check reported hex problem
         	writer.write( "#error This was equal, but not for the eclipse.\n");
         	writer.write( "#endif\n");
+        	writer.write( "#if 010 == 8\n"); // check octal
+        	writer.write( "#error octal test\n");
+        	writer.write( "#endif\n");
+        	
         	parse( writer.toString() );
 
         	assertTrue(false);
     	} catch (ParserException pe) {
     		// expected IProblem
     	} finally {
-        	assertTrue( callback.getProblems().hasNext() );
+        	Iterator probs = callback.getProblems();
-        	Object ipo = callback.getProblems().next();
+    		assertTrue( probs.hasNext() );
+        	Object ipo = probs.next();
         	assertTrue( ipo instanceof IProblem );
-        	IProblem ip = (IProblem)callback.getProblems().next();
+        	IProblem ip = (IProblem)ipo;
         	assertTrue(ip.getArguments().indexOf("This was equal, but not for the eclipse") > 0);
+        	assertTrue( probs.hasNext() );
+        	ipo = probs.next();
+        	assertTrue( ipo instanceof IProblem );
+        	ip = (IProblem)ipo;
+        	assertTrue(ip.getArguments().indexOf("octal test") > 0);
     	}
 	}
     

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java

@@ -1927,4 +1927,50 @@ public class Scanner2Test extends BaseScanner2Test
         assertEquals( t.getEndOffset(), idx + 6 );
     }
     
+    // when fixing 75532 several IProblems were added to ExpressionEvaluator and one to Scanner2, this is to test them
+    public void testBug75532IProblems() throws Exception {
+    	Writer writer = new StringWriter();
+    	writer.write("#if 09 == 9\n#endif\n"); // malformed octal
+    	writer.write("#if 1A == 0x1A\n#endif\n"); // malformed decimal
+    	writer.write("#if 0x == 0x0\n#endif\n"); // malformed hex
+    	writer.write("#if 0xAX == 0xA\n#endif\n"); // malformed hex
+    	writer.write("#if 1/0 == 1\n#endif\n"); // division by zero 
+    	writer.write("#if defined ( sadf a\n#endif\n"); // missing ')' in defined 
+    	writer.write("#if defined ( sadf\n#endif\n"); // missing ')' in defined 
+    	writer.write("#if defined ( 2sadf )\n#endif\n"); // illegal identifier in defined 
+    	writer.write("#if ( 1 == 1 ? 1\n#endif\n"); // bad conditional expression  
+    	writer.write("#if (  \n#endif\n"); // expression syntax error 
+    	writer.write("#if @\n#endif\n"); // expression syntax error 
+    	writer.write("#if \n#endif\n"); // expression syntax error 
+    	writer.write("#if -\n#endif\n"); // expression syntax error 
+    	writer.write("#if ( 1 == 1\n#endif\n"); // missing ')' 
+    	writer.write("#if 1 = 1\n#endif\n"); // assignment not allowed
+    	
+    	writer.write("int main(int argc, char **argv) {\n");
+		writer.write("if ( 09 == 9 )\n"); // added while fixing this bug, IProblem on invalid octal number
+		writer.write("return 1;\nreturn 0;\n}\n");
+				
+    	Callback callback = new Callback( ParserMode.COMPLETE_PARSE );
+    	initializeScanner( writer.toString(), ParserMode.COMPLETE_PARSE, callback );
+    	fullyTokenize();
+    	assertTrue( callback.problems.size() == 16 );
+    	Iterator probs = callback.problems.iterator();
+    	assertTrue( probs.hasNext() );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_BAD_OCTAL_FORMAT );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_BAD_DECIMAL_FORMAT );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_BAD_HEX_FORMAT );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_BAD_HEX_FORMAT );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_DIVIDE_BY_ZERO );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_MISSING_R_PAREN );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_MISSING_R_PAREN );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_ILLEGAL_IDENTIFIER );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_BAD_CONDITIONAL_EXPRESSION );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_MISSING_R_PAREN );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_ASSIGNMENT_NOT_ALLOWED );
+    	assertTrue(((IProblem)probs.next()).getID() ==  IProblem.SCANNER_BAD_OCTAL_FORMAT );
+    }
 }

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/SourceIndexerRequestor.java

@@ -781,7 +781,7 @@ public class SourceIndexerRequestor implements ISourceElementRequestor, IIndexCo
 		boolean semantics = ( problemMarkersEnabled & IndexManager.SEMANTIC_PROBLEMS_BIT ) != 0;
 		boolean syntax = ( problemMarkersEnabled & IndexManager.SYNTACTIC_PROBLEMS_BIT ) != 0;
 		
-		if( problem.checkCategory( IProblem.PREPROCESSOR_RELATED ) )
+		if( problem.checkCategory( IProblem.PREPROCESSOR_RELATED ) || problem.checkCategory( IProblem.SCANNER_RELATED ) )
 			return preprocessor && problem.getID() != IProblem.PREPROCESSOR_CIRCULAR_INCLUSION;
 		else if( problem.checkCategory( IProblem.SEMANTICS_RELATED ) )
 			return semantics;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IProblem.java

@@ -271,6 +271,55 @@ public interface IProblem
 	 */
 	public final static int SCANNER_UNEXPECTED_EOF = SCANNER_RELATED | 0x006;
 	
+	/** 
+	 * Bad octal encountered by Scanner. 
+	 * Required attributes: none.  
+	 */
+	public final static int SCANNER_BAD_OCTAL_FORMAT = SCANNER_RELATED | 0x007;
+
+	/** 
+	 * Bad decimal encountered by Scanner. 
+	 * Required attributes: none.  
+	 */
+	public final static int SCANNER_BAD_DECIMAL_FORMAT = SCANNER_RELATED | 0x008;
+
+	/** 
+	 * Assignment '=' encountered in macro by Scanner. 
+	 * Required attributes: none.  
+	 */
+	public final static int SCANNER_ASSIGNMENT_NOT_ALLOWED = SCANNER_RELATED | 0x009;
+
+	/** 
+	 * Division by 0 encountered in macro by Scanner. 
+	 * Required attributes: none.  
+	 */
+	public final static int SCANNER_DIVIDE_BY_ZERO = SCANNER_RELATED | 0x00A;
+	
+	/** 
+	 * Missing ')' encountered in macro by Scanner. 
+	 * Required attributes: none.  
+	 */
+	public final static int SCANNER_MISSING_R_PAREN = SCANNER_RELATED | 0x00B;	
+
+	/** 
+	 * Expression syntax error encountered in macro by Scanner. 
+	 * Required attributes: none.  
+	 */
+	public final static int SCANNER_EXPRESSION_SYNTAX_ERROR = SCANNER_RELATED | 0x00C;
+	
+	/** 
+	 * Expression syntax error encountered in macro by Scanner. 
+	 * Required attributes: none.  
+	 */
+	public final static int SCANNER_ILLEGAL_IDENTIFIER = SCANNER_RELATED | 0x00D;
+
+	/** 
+	 * Division by 0 encountered in macro by Scanner. 
+	 * Required attributes: none.  
+	 */
+	public final static int SCANNER_BAD_CONDITIONAL_EXPRESSION = SCANNER_RELATED | 0x00E;
+	
+
 	/*
 	 * Preprocessor Problems
 	 */

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ParserMessages.properties

@@ -38,6 +38,14 @@ ScannerProblemFactory.error.scanner.invalidEscapeChar=Invalid escape character e
 ScannerProblemFactory.error.scanner.unboundedString=Unbounded string encountered 
 ScannerProblemFactory.error.scanner.badFloatingPoint=Invalid floating point format encountered 
 ScannerProblemFactory.error.scanner.badHexFormat=Invalid hexidecimal format encountered 
+ScannerProblemFactory.error.scanner.badOctalFormat=Invalid octal format encountered 
+ScannerProblemFactory.error.scanner.badDecimalFormat=Invalid decimal format encountered 
+ScannerProblemFactory.error.scanner.assignmentNotAllowed=Assignment not allowed 
+ScannerProblemFactory.error.scanner.divideByZero=Division by zero not allowed 
+ScannerProblemFactory.error.scanner.missingRParen=Missing right parenthesis ) 
+ScannerProblemFactory.error.scanner.expressionSyntaxError=Expression syntax error 
+ScannerProblemFactory.error.scanner.illegalIdentifier=Illegal identifier in defined() 
+ScannerProblemFactory.error.scanner.badConditionalExpression=Bad conditional expression 
 ScannerProblemFactory.error.scanner.unexpectedEOF=Unexpected End Of File encountered 
 ScannerProblemFactory.error.scanner.badCharacter=Bad character sequence encountered : {0}
 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/problem/Problem.java

@@ -193,6 +193,30 @@ public class Problem implements IProblem {
 		errorMessages.put(
 			new Integer(IProblem.SCANNER_BAD_HEX_FORMAT),
 			ParserMessages.getString("ScannerProblemFactory.error.scanner.badHexFormat")); //$NON-NLS-1$
+		errorMessages.put(
+				new Integer(IProblem.SCANNER_BAD_OCTAL_FORMAT),
+				ParserMessages.getString("ScannerProblemFactory.error.scanner.badOctalFormat")); //$NON-NLS-1$
+		errorMessages.put(
+				new Integer(IProblem.SCANNER_BAD_DECIMAL_FORMAT),
+				ParserMessages.getString("ScannerProblemFactory.error.scanner.badDecimalFormat")); //$NON-NLS-1$
+		errorMessages.put(
+				new Integer(IProblem.SCANNER_ASSIGNMENT_NOT_ALLOWED),
+				ParserMessages.getString("ScannerProblemFactory.error.scanner.assignmentNotAllowed")); //$NON-NLS-1$		
+		errorMessages.put(
+				new Integer(IProblem.SCANNER_DIVIDE_BY_ZERO),
+				ParserMessages.getString("ScannerProblemFactory.error.scanner.divideByZero")); //$NON-NLS-1$
+		errorMessages.put(
+				new Integer(IProblem.SCANNER_MISSING_R_PAREN),
+				ParserMessages.getString("ScannerProblemFactory.error.scanner.missingRParen")); //$NON-NLS-1$
+		errorMessages.put(
+				new Integer(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR),
+				ParserMessages.getString("ScannerProblemFactory.error.scanner.expressionSyntaxError")); //$NON-NLS-1$
+		errorMessages.put(
+				new Integer(IProblem.SCANNER_ILLEGAL_IDENTIFIER),
+				ParserMessages.getString("ScannerProblemFactory.error.scanner.illegalIdentifier")); //$NON-NLS-1$
+		errorMessages.put(
+				new Integer(IProblem.SCANNER_BAD_CONDITIONAL_EXPRESSION),
+				ParserMessages.getString("ScannerProblemFactory.error.scanner.badConditionalExpression")); //$NON-NLS-1$		
 		errorMessages.put(
 			new Integer(IProblem.SCANNER_UNEXPECTED_EOF),
 			ParserMessages.getString("ScannerProblemFactory.error.scanner.unexpectedEOF")); //$NON-NLS-1$

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/ExpressionEvaluator.java

@@ -10,6 +10,8 @@
  ******************************************************************************/
 package org.eclipse.cdt.internal.core.parser.scanner2;
 
+import org.eclipse.cdt.core.parser.IProblem;
+import org.eclipse.cdt.core.parser.ISourceElementRequestor;
 import org.eclipse.cdt.core.parser.util.CharArrayObjectMap;
 import org.eclipse.cdt.core.parser.util.CharArrayUtils;
 
@@ -28,13 +30,36 @@ public class ExpressionEvaluator {
 	private int[] bufferPos = new int[bufferInitialSize];
 	private int[] bufferLimit = new int[bufferInitialSize];
 		
+	private ISourceElementRequestor requestor = null;
+	private ScannerProblemFactory spf = null;
+
+	private int lineNumber = 1;
+	private char[] fileName = null;
+	
+	private int pos = 0;
+	
 	// The macros
 	CharArrayObjectMap definitions;
 
-	public long evaluate(char[] buffer, int pos, int length, CharArrayObjectMap defs) {
+	public ExpressionEvaluator() {
+		super();
+	}
+	
+	public ExpressionEvaluator(ISourceElementRequestor requestor, ScannerProblemFactory spf) {
+		this.requestor = requestor;
+		this.spf = spf;
+	}
+	
+	public long evaluate(char[] buffer, int p, int length, CharArrayObjectMap defs) {
+		return evaluate(buffer, p, length, defs, 0, "".toCharArray()); //$NON-NLS-1$
+	}
+	
+	public long evaluate(char[] buffer, int p, int length, CharArrayObjectMap defs, int ln, char[] fn) {
+		this.lineNumber = ln;
+		this.fileName = fn;
 		bufferStack[++bufferStackPos] = buffer;
-		bufferPos[bufferStackPos] = pos - 1;
+		bufferPos[bufferStackPos] = p - 1;
-		bufferLimit[bufferStackPos] = pos + length;
+		bufferLimit[bufferStackPos] = p + length;
 		this.definitions = defs;
 		tokenType = 0;
 	
@@ -67,8 +92,10 @@ public class ExpressionEvaluator {
 			long r2 = expression();
 			if (LA() == tCOLON)
 				consume();
-			else
+			else {
+				handleProblem(IProblem.SCANNER_BAD_CONDITIONAL_EXPRESSION, pos);
 				throw new EvalException("bad conditional expression"); //$NON-NLS-1$
+			}
 			long r3 = conditionalExpression();
 			return r1 != 0 ? r2 : r3;
 		} 
@@ -186,14 +213,17 @@ public class ExpressionEvaluator {
 	private long multiplicativeExpression() throws EvalException {
 		long r1 = unaryExpression();
 		for (int t = LA(); t == tMULT|| t == tDIV; t = LA()) {
+			int position = pos; // for IProblem /0 below, need position before consume()
 			consume();
 			long r2 = unaryExpression();
 			if (t == tMULT)
 				r1 = r1 * r2;
 			else if( r2 != 0 )// t == tDIV;
 				r1 = r1 / r2;
-			else
+			else {
+				handleProblem(IProblem.SCANNER_DIVIDE_BY_ZERO, position);
 				throw new EvalException( "Divide by 0 encountered"); //$NON-NLS-1$
+			}
 		}
 		return r1;
 	}
@@ -223,10 +253,11 @@ public class ExpressionEvaluator {
 					consume();
 					return r1;
 				} 
+				handleProblem(IProblem.SCANNER_MISSING_R_PAREN, pos);
 				throw new EvalException("missing )"); //$NON-NLS-1$ 
 			default:
+				handleProblem(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, pos);
 				throw new EvalException("expression syntax error"); //$NON-NLS-1$ 
-
 		}
 	}
 	
@@ -252,6 +283,7 @@ public class ExpressionEvaluator {
 		}
 
 		if (!((c >= 'A' && c <= 'Z') || c == '_' || (c >= 'a' && c <= 'z'))) {
+			handleProblem(IProblem.SCANNER_ILLEGAL_IDENTIFIER, pos);
 			throw new EvalException("illegal identifier in defined()"); //$NON-NLS-1$ 
 		}
 
@@ -271,9 +303,11 @@ public class ExpressionEvaluator {
 		// consume to the closing paren;
 		if (inParens) {
 			skipWhiteSpace();
-			if (++bufferPos[bufferStackPos] < limit
+			if (++bufferPos[bufferStackPos] <= limit
-					&& buffer[bufferPos[bufferStackPos]] != ')')
+					&& buffer[bufferPos[bufferStackPos]] != ')') {
+				handleProblem(IProblem.SCANNER_MISSING_R_PAREN, pos);
 				throw new EvalException("missing ) on defined"); //$NON-NLS-1$
+			}
 		}
 
 		// Set up the lookahead to whatever comes next
@@ -302,6 +336,10 @@ public class ExpressionEvaluator {
 	private static char[] _defined = "defined".toCharArray(); //$NON-NLS-1$
 	
 	private void nextToken() throws EvalException {
+		boolean isHex = false;
+		boolean isOctal = false;
+		boolean isDecimal = false;
+		
 		contextLoop:
 			while (bufferStackPos >= 0) {
 				
@@ -317,7 +355,19 @@ public class ExpressionEvaluator {
 				// Tokens don't span buffers, stick to our current one
 				char[] buffer = bufferStack[bufferStackPos];
 				int limit = bufferLimit[bufferStackPos];
-				int pos = bufferPos[bufferStackPos];
+				pos = bufferPos[bufferStackPos];
+
+				if (buffer[pos] >= '1' && buffer[pos] <= '9')
+					isDecimal = true;
+				else if (buffer[pos] == '0' && pos + 1 < limit)
+					if (buffer[pos + 1] == 'x' || buffer[pos + 1] == 'X') {
+						isHex = true;
+						++bufferPos[bufferStackPos];
+						if (pos + 2 < limit )
+							if ( (buffer[pos + 2] < '0' || buffer[pos + 2] > '9') && (buffer[pos + 2] < 'a' || buffer[pos + 2] > 'f') && (buffer[pos + 2] < 'A' || buffer[pos + 2] > 'F'))
+								handleProblem(IProblem.SCANNER_BAD_HEX_FORMAT, pos);					
+					} else 
+						isOctal = true;
 				
 				switch (buffer[pos]) {
 					case 'a':
@@ -441,24 +491,14 @@ public class ExpressionEvaluator {
 						tokenValue = buffer[pos] - '0';
 						tokenType = tNUMBER;
 						
-						boolean isHex = false;
-						if (buffer[pos] == '0' && pos + 1 < limit) {
-							switch (buffer[pos + 1]) {
-								case 'x':
-								case 'X':
-									isHex = true;
-									++bufferPos[bufferStackPos];
-							}
-						}
-						
 						while (++bufferPos[bufferStackPos] < limit) {
 							char c = buffer[bufferPos[bufferStackPos]];
-							if (c >= '0' && c <= '9') {
+							if (isHex) {
-								tokenValue *= (isHex ? 16 : 10);
+								if (c >= '0' && c <= '9') {
-								tokenValue += c - '0';
+									tokenValue *= 16;
-								continue;
+									tokenValue += c - '0';
-							} else if (isHex) {
+									continue;
-								if (c >= 'a' && c <= 'f') {
+								} else if (c >= 'a' && c <= 'f') {
 									tokenValue = (tokenValue == 0 ? 10 : (tokenValue * 16) + 10);
 									tokenValue += c - 'a';
 									continue;
@@ -466,12 +506,34 @@ public class ExpressionEvaluator {
 									tokenValue = (tokenValue == 0 ? 10 : (tokenValue * 16) + 10);
 									tokenValue += c - 'A';
 									continue;
+								} else {
+									if (bufferPos[bufferStackPos] + 1 < limit)
+										if (!isValidTokenSeparator(c, buffer[bufferPos[bufferStackPos] + 1]))
+											handleProblem(IProblem.SCANNER_BAD_HEX_FORMAT, pos);
 								}
+							} else if (isOctal) {
+								if (c >= '0' && c <= '7') {
+									tokenValue *= 8;
+									tokenValue += c - '0';
+									continue;
+								} 
+								if (bufferPos[bufferStackPos] + 1 < limit)
+								if (!isValidTokenSeparator(c, buffer[bufferPos[bufferStackPos] + 1])) 
+									handleProblem(IProblem.SCANNER_BAD_OCTAL_FORMAT, pos);
+							} else if (isDecimal) {
+								if (c >= '0' && c <= '9') {
+									tokenValue *= 10;
+									tokenValue += c - '0';
+									continue;
+								} 
+								if (bufferPos[bufferStackPos] + 1 < limit)
+									if (!isValidTokenSeparator(c, buffer[bufferPos[bufferStackPos] + 1])) 
+										handleProblem(IProblem.SCANNER_BAD_DECIMAL_FORMAT, pos);
 							}
 							
 							// end of number
-							if (c == 'L') {
+							if (c == 'L' || c =='l' || c == 'U' || c =='u') {
-								// eat the long
+								// eat the long/unsigned
 								++bufferPos[bufferStackPos];
 							}
 							
@@ -557,6 +619,7 @@ public class ExpressionEvaluator {
 							tokenType = tEQUAL;
 							return;
 						}
+						handleProblem(IProblem.SCANNER_ASSIGNMENT_NOT_ALLOWED, pos);
 						throw new EvalException("assignment not allowed"); //$NON-NLS-1$ 
 					
 					case '<':
@@ -619,8 +682,8 @@ public class ExpressionEvaluator {
 
 			skipWhiteSpace();
 			
-			int pos = ++bufferPos[bufferStackPos];
+			int p = ++bufferPos[bufferStackPos];
-			char c = buffer[pos];
+			char c = buffer[p];
 			if (c == ')') {
 				if (parens == 0)
 					// end of macro
@@ -637,7 +700,7 @@ public class ExpressionEvaluator {
 			}
 
 			// peel off the arg
-			int argstart = pos;
+			int argstart = p;
 			int argend = argstart - 1;
 			
 			// Loop looking for end of argument
@@ -679,15 +742,19 @@ public class ExpressionEvaluator {
 				case ' ':
 				case '\t':
 				case '\r':
-				case '\n':
 				case ',':
 				case ')':
 					--bufferPos[bufferStackPos];
 					return;
+				case '\n':
+					lineNumber++;
+					--bufferPos[bufferStackPos];
+					return;
 				case '\\':
-					int pos = bufferPos[bufferStackPos];
+					int p = bufferPos[bufferStackPos];
-					if (pos + 1 < limit && buffer[pos + 1] == '\n') {
+					if (p + 1 < limit && buffer[p + 1] == '\n') {
 						// \n is whitespace
+						lineNumber++;
 						--bufferPos[bufferStackPos];
 						return;
 					}
@@ -721,26 +788,26 @@ public class ExpressionEvaluator {
 		int limit = bufferLimit[bufferStackPos];
 		
 		while (++bufferPos[bufferStackPos] < limit) {
-			int pos = bufferPos[bufferStackPos];
+			int p = bufferPos[bufferStackPos];
-			switch (buffer[pos]) {
+			switch (buffer[p]) {
 				case ' ':
 				case '\t':
 				case '\r':
 					continue;
 				case '/':
-					if (pos + 1 < limit) {
+					if (p + 1 < limit) {
-						if (buffer[pos + 1] == '/') {
+						if (buffer[p + 1] == '/') {
 							// C++ comment, skip rest of line
 							return;
-						} else if (buffer[pos + 1] == '*') {
+						} else if (buffer[p + 1] == '*') {
 							// C comment, find closing */
 							for (bufferPos[bufferStackPos] += 2;
 									bufferPos[bufferStackPos] < limit;
 									++bufferPos[bufferStackPos]) {
-								pos = bufferPos[bufferStackPos];
+								p = bufferPos[bufferStackPos];
-								if (buffer[pos] == '*'
+								if (buffer[p] == '*'
-										&& pos + 1 < limit
+										&& p + 1 < limit
-										&& buffer[pos + 1] == '/') {
+										&& buffer[p + 1] == '/') {
 									++bufferPos[bufferStackPos];
 									break;
 								}
@@ -750,8 +817,9 @@ public class ExpressionEvaluator {
 					}
 					break;
 				case '\\':
-					if (pos + 1 < limit && buffer[pos + 1] == '\n') {
+					if (p + 1 < limit && buffer[p + 1] == '\n') {
 						// \n is a whitespace
+						lineNumber++;
 						++bufferPos[bufferStackPos];
 						continue;
 					}
@@ -761,6 +829,10 @@ public class ExpressionEvaluator {
 			--bufferPos[bufferStackPos];
 			return;
 		}
+		
+		// fell out of while without continuing, we're done
+		--bufferPos[bufferStackPos];
+		return;		
 	}
 	
 	private static final int tNULL		= 0;
@@ -825,4 +897,40 @@ public class ExpressionEvaluator {
 		--bufferStackPos;
 	}
 
+	private void handleProblem(int id, int startOffset) {
+		if (requestor != null && spf != null)
+			requestor.acceptProblem(spf.createProblem( id, startOffset, bufferPos[(bufferStackPos == -1 ? 0 : bufferStackPos)], lineNumber, (fileName == null ? "".toCharArray() : fileName), emptyCharArray, false, true )); //$NON-NLS-1$
+	}
+	
+	private boolean isValidTokenSeparator(char c, char c2) throws EvalException {
+		switch (c) {
+			case '\t':
+			case '\r':
+			case '\n':
+			case ' ':	
+			case '(':
+			case ')':
+			case ':':
+			case '?':
+			case '+':
+			case '-':
+			case '*':
+			case '/':
+			case '%':
+			case '^':
+			case '&':
+			case '|':
+			case '~':
+			case '!':
+			case '<':
+			case '>':
+				return true;
+			case '=':
+				if (c2 == '=')
+					return true;
+				return false;
+		}
+		
+		return false;
+	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java

@@ -159,6 +159,7 @@ public class Scanner2 implements IScanner, IScannerData {
 		this.language = language;
 		this.log = log;
 		this.workingCopies = workingCopies;
+		this.expressionEvaluator = new ExpressionEvaluator(requestor, spf);
 
 		if( language == ParserLanguage.C )
 		    keywords = ckeywords;
@@ -465,7 +466,7 @@ public class Scanner2 implements IScanner, IScannerData {
 				nextToken = null;
 				return nextToken();
 			}
-		} else if (lastToken.getType() == IToken.tSTRING || lastToken.getType() ==IToken.tLSTRING ) {
+		} else if (lastToken != null && (lastToken.getType() == IToken.tSTRING || lastToken.getType() ==IToken.tLSTRING )) {
 			while (nextToken != null && ( nextToken.getType() == IToken.tSTRING || nextToken.getType() == IToken.tLSTRING )) {
 				// Concatenate the adjacent strings
 				int tokenType = IToken.tSTRING; 
@@ -1177,12 +1178,23 @@ public class Scanner2 implements IScanner, IScannerData {
 		boolean hasExponent = false;
 		
 		boolean isHex = false;
+		boolean isOctal = false;
+		boolean isMalformedOctal = false;
+		
 		if (buffer[start] == '0' && start + 1 < limit) {
 			switch (buffer[start + 1]) {
 				case 'x':
 				case 'X':
 					isHex = true;
 					++bufferPos[bufferStackPos];
+					break;
+                default :
+                	if (buffer[start + 1] > '0' && buffer[start + 1] < '7')
+                		isOctal = true;
+                	else if (buffer[start + 1] == '8' || buffer[start + 1] == '9') {
+                		isOctal = true;
+                		isMalformedOctal = true;
+                	}
 			}
 		}
 		
@@ -1199,6 +1211,11 @@ public class Scanner2 implements IScanner, IScannerData {
 				case '7':
 				case '8':
 				case '9':
+					if ((buffer[pos] == '8' || buffer[pos] == '9') && isOctal) {
+						isMalformedOctal = true;
+						break;
+					}
+					
 					continue;
 				
 				case '.':
@@ -1349,8 +1366,11 @@ public class Scanner2 implements IScanner, IScannerData {
 		
 		char[] result = CharArrayUtils.extract( buffer, start, bufferPos[bufferStackPos] - start + 1);
 		int tokenType = isFloat ? IToken.tFLOATINGPT : IToken.tINTEGER;
+		
 		if( tokenType == IToken.tINTEGER && isHex && result.length == 2 ){
 		    handleProblem( IProblem.SCANNER_BAD_HEX_FORMAT, start, result );
+		} else if( tokenType == IToken.tINTEGER && isOctal && isMalformedOctal ){
+		    handleProblem( IProblem.SCANNER_BAD_OCTAL_FORMAT, start, result );
 		}  
 		
 		return newToken( tokenType, result );
@@ -1450,7 +1470,7 @@ public class Scanner2 implements IScanner, IScannerData {
 						if( isLimitReached() )
 							handleCompletionOnExpression( CharArrayUtils.extract( buffer, start, len ) );
 						branchState( BRANCH_IF );
-						if (expressionEvaluator.evaluate(buffer, start, len, definitions) == 0) {
+						if (expressionEvaluator.evaluate(buffer, start, len, definitions, getLineNumber( bufferPos[bufferStackPos] ), getCurrentFilename()) == 0) {
 							if (dlog != null) dlog.println("#if <FALSE> " + new String(buffer,start+1,len-1)); //$NON-NLS-1$
 							skipOverConditionalCode(true);
 							if( isLimitReached() )
@@ -2051,7 +2071,7 @@ public class Scanner2 implements IScanner, IScannerData {
 										start = bufferPos[bufferStackPos] + 1;
 										skipToNewLine();
 										len = bufferPos[bufferStackPos] - start;
-										if (expressionEvaluator.evaluate(buffer, start, len, definitions) != 0)
+										if (expressionEvaluator.evaluate(buffer, start, len, definitions, getLineNumber( bufferPos[bufferStackPos] ), getCurrentFilename()) != 0)
 											// condition passed, we're good
 											return;
 									}