Patch for Andrew Niefer:

This patch fixes 2 scanner bugs:
   Bug36816 Scanner infinite loop on incomplete #include
   Bug36255 Parser hangs/goes infinite
 
This patch also introduces a new automated test, 
FractionalAutomatedTest, which tests the parser on incomplete files.

core/org.eclipse.cdt.core/parser/ChangeLog

@@ -7,6 +7,10 @@
 	Fixed bug36811 - The using directive causes parser to fail
 	Fixed bug36794 - ClassCastException for A template with no parameters 
 
+2003-04-24 Andrew Niefer
+	Fixed Bug36816		Scanner infinite loop on incomplete #include
+	Fixed Bug36255		Parser hangs/goes infinite
+
 2003-04-21 John Camelon
 	Revisited bug36247	Parser confused by symbols #defined elsewhere 
 	Fixed Bug36708		Problem parsing Loki's Reference TypeTraits.h 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java

@@ -832,7 +832,11 @@ public class Scanner implements IScanner {
 							// get the rest of the line		
 							String expression = getRestOfPreprocessorLine();
 							
-							boolean expressionEvalResult =	evaluateExpression(expression);
+							boolean expressionEvalResult = false;
+							try{
+								expressionEvalResult = evaluateExpression(expression);
+							} catch( ScannerException e ){}
+							
 							passOnToClient = branches.poundif( expressionEvalResult ); 
 							c = getChar();
 							continue;
@@ -893,8 +897,10 @@ public class Scanner implements IScanner {
 								if (throwExceptionOnBadPreprocessorSyntax)
 									throw new ScannerException("Malformed #elsif clause");
 
-							boolean elsifResult =
+							boolean elsifResult = false;
-									evaluateExpression(elsifExpression );
+							try{
+								elsifResult = evaluateExpression(elsifExpression );
+							} catch( ScannerException e ){}
 
 							passOnToClient = branches.poundelif( elsifResult ); 
 							c = getChar();
@@ -1556,14 +1562,22 @@ public class Scanner implements IScanner {
 		boolean useIncludePath = true;
 		if (c == '<') {
 			c = getChar();
-			while ((c != '>')) {
+			while (c != '>') {
+				if( c == NOCHAR ){
+					//don't attempt an include if we hit the end of file before closing the brackets
+					return;
+				}
 				fileName.append((char) c);
 				c = getChar();
 			}
 		}
 		else if (c == '"') {
 			c = getChar();
-			while ((c != '"')) {
+			while (c != '"') {
+				if( c == NOCHAR ){
+					//don't attempt an include if we hit the end of file before closing the quotes
+					return;
+				}
 				fileName.append((char) c);
 				c = getChar();
 			}
@@ -1613,6 +1627,9 @@ public class Scanner implements IScanner {
 			StringBuffer buffer = new StringBuffer();
 			c = getChar();
 			while (c != ')') {
+				if( c == NOCHAR ){
+					return;	//don't attempt #define if we don't hit the closing bracket
+				}
 				buffer.append((char) c);
 				c = getChar();
 			}
@@ -1748,7 +1765,7 @@ public class Scanner implements IScanner {
 					else if (c == ')')
 						--bracketCount;
 
-					if (bracketCount == 0)
+					if(bracketCount == 0 || c == NOCHAR)
 						break;
 					buffer.append((char) c);
 					c = getChar( true );

core/org.eclipse.cdt.ui.tests/ChangeLog

@@ -2,6 +2,13 @@
 	Moved fixed tests from FailedTests to DOMTests. 
 	Added LokiFailures.java to failed tests directory.
 
+2003-04-24 Andrew Niefer
+	Added AutomatedFramework.java
+	Added FractionalAutomatedTest (which derives from AutomatedFramework)
+	Refactored AutomatedTest to derive from AutomatdFramework
+	Added ScannerTestCase.testBug36816
+	Added ScannerTestCase.testBug36255	
+
 2003-04-24 John Camelon
 	Fixed Java 1.3 compliance issue w/AutomatedTest.java
 	Fixed False failure in HelloWorld.java.  

core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/resources/.cvsignore

@@ -1 +1,2 @@
 AutomatedTest.properties
+FractionalAutomatedTest.properties

core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/AutomatedFramework.java

@@ -0,0 +1,185 @@
+/*******************************************************************************
+ * Copyright (c) 2001 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors:
+ *     IBM Corp. - Rational Software - initial implementation
+ ******************************************************************************/
+ 
+package org.eclipse.cdt.core.parser.tests;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Properties;
+import java.util.Set;
+
+import org.eclipse.cdt.internal.core.parser.IParserCallback;
+import org.eclipse.cdt.internal.core.parser.NullParserCallback;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * @author aniefer
+ *
+ * To change the template for this generated type comment go to
+ * Window>Preferences>Java>Code Generation>Code and Comments
+ */
+public abstract class AutomatedFramework extends TestCase {
+
+	public AutomatedFramework() {
+		super();
+	}
+
+	public AutomatedFramework(String name) {
+		super(name);
+	}
+
+	protected abstract AutomatedFramework newTest( String name );
+	protected abstract void loadProperties() throws Exception;
+	public	  abstract void doFile() throws Throwable;
+
+	private void fillSuite( TestSuite suite, File path ){
+		File files[] = null;
+		if( path.isFile() ){
+			files = new File[ 1 ];
+			files[0] = path;
+		}
+		else
+			files = path.listFiles();
+	
+		File file = null;
+		String filePath = null;
+		int i = 0;
+		try{
+			file = files[ i++ ];
+			while( file != null )
+			{
+				if( file.isDirectory() )
+					fillSuite( suite, file );
+				else if( file.isFile() && nameFilter.accept( file.getParentFile(), file.getName() ) ){
+					try{
+						filePath = file.getCanonicalPath();
+					} catch ( Exception e ){
+						continue;
+					}
+					
+					if(	filePath.endsWith(".cpp") || filePath.endsWith(".hpp") || 
+						filePath.endsWith(".hxx") || filePath.endsWith(".hh") )
+					{
+						AutomatedTest.natures.put( filePath, "cpp" );
+					} else if( filePath.endsWith(".c") ){ 
+						AutomatedTest.natures.put( filePath, "c" );
+					} else {
+						AutomatedTest.natures.put( filePath, AutomatedTest.defaultNature );
+					}
+					
+					AutomatedTest.fileList.add( file );
+					suite.addTest( newTest( file.getName().replace(',', '_') ) );
+				}				
+				file = files[ i++ ];
+			}
+		} catch( ArrayIndexOutOfBoundsException e ){
+			//done
+		}
+	}
+
+	public void reportFailed() {
+		fail( "Unable to open " + outputFile + "for output of results." );
+	}
+
+	public void propertiesFailed() {
+		fail( "Unable to load properties file." );
+	}
+	
+	protected void runTest() throws Throwable {
+		String name = getName();
+		
+		if( name.equals("propertiesFailed") )
+			propertiesFailed();
+		else if ( name.equals("reportFailed") )
+			reportFailed();
+		else
+			doFile();
+	}
+
+	public Test createSuite() {
+		TestSuite suite = new TestSuite();
+		
+		try{
+			loadProperties();
+		} catch( Exception e ){
+			suite.addTest( newTest( "propertiesFailed") );
+		}
+		
+		if( outputFile != null && !outputFile.equals("") ){
+			try{
+				
+				File output = new File( outputFile );
+				
+				if( output.exists() ){
+					output.delete();
+				}
+				
+				output.createNewFile();
+			
+				report = new FileOutputStream( output );
+			
+			} catch( Exception e ) {
+				suite.addTest( newTest( "reportFailed" ) );
+			}
+		}
+		
+		Set keys = testSources.keySet();
+		Iterator iter = keys.iterator();
+		int size = keys.size();
+		String item = null;
+		for( int i = size; i > 0; i-- )
+		{
+			item = (String) iter.next();
+			File file = new File( item );
+			if( file.exists() ){
+				defaultNature = (String) testSources.get( item );
+				fillSuite( suite, file );		
+			}
+		}
+	
+		return suite;
+	}
+
+	protected static IParserCallback nullCallback = new NullParserCallback();
+	protected static Properties properties = new Properties();
+	protected static String defaultNature;
+	protected static String outputFile = null;
+	protected static HashMap testSources = new HashMap();
+	protected static HashMap natures = new HashMap();
+	protected static LinkedList fileList = new LinkedList();
+	private static FilenameFilter nameFilter = new Filter();
+	protected static FileOutputStream report = null;
+	
+	static private class Filter implements FilenameFilter
+	{
+		public boolean accept(File dir, String name) {
+			if( name.endsWith(".cpp") 	|| 
+				name.endsWith(".c") 	|| 
+				name.endsWith(".cc") 	|| 
+				name.endsWith(".h") 	||
+				name.endsWith(".hh")	||
+				name.endsWith(".hxx"))
+			{
+				return true;
+			}
+			else
+				return false;
+		}
+	}
+
+}

core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/AutomatedTest.java

@@ -14,27 +14,17 @@ package org.eclipse.cdt.core.parser.tests;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
+
-import java.io.FilenameFilter;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.NoSuchElementException;
-import java.util.Properties;
-import java.util.Set;
 import java.util.StringTokenizer;
 
 import org.eclipse.core.runtime.Path;
 
 import org.eclipse.cdt.internal.core.parser.IParser;
-import org.eclipse.cdt.internal.core.parser.IParserCallback;
-import org.eclipse.cdt.internal.core.parser.NullParserCallback;
 import org.eclipse.cdt.internal.core.parser.Parser;
 
 import junit.framework.AssertionFailedError;
 import junit.framework.Test;
-import junit.framework.TestCase;
-import junit.framework.TestSuite;
 
 
 
@@ -44,7 +34,10 @@ import junit.framework.TestSuite;
  * To change the template for this generated type comment go to
  * Window>Preferences>Java>Code Generation>Code and Comments
  */
-public class AutomatedTest extends TestCase {
+public class AutomatedTest extends AutomatedFramework {
+
+	public AutomatedTest() {
+	}
 	public AutomatedTest(String name){
 		super(name);
 	}
@@ -88,112 +81,15 @@ public class AutomatedTest extends TestCase {
 		}
 	}
 	
-	public void reportFailed(){
+	protected AutomatedFramework newTest( String name ){
-		fail( "Unable to open " + outputFile + "for output of results." );
+		return new AutomatedTest( name );
-	}
-	
-	public void propertiesFailed(){
-		fail( "Unable to load properties file." );
-	}
-	
-	protected void runTest() throws Throwable {
-		String name = getName();
-		
-		if( name.equals("propertiesFailed") )
-			propertiesFailed();
-		else if ( name.equals("reportFailed") )
-			reportFailed();
-		else
-			doFile();
 	}
 	
 	public static Test suite()
 	{
-		TestSuite suite = new TestSuite();
+		AutomatedFramework frame = new AutomatedTest();
 		
-		try{
+		return frame.createSuite();
-			loadProperties();
-		} catch( Exception e ){
-			suite.addTest( new AutomatedTest( "propertiesFailed") );
-		}
-		
-		if( outputFile != null && !outputFile.equals("") ){
-			try{
-				
-				File output = new File( outputFile );
-				
-				if( output.exists() ){
-					output.delete();
-				}
-				
-				output.createNewFile();
-			
-				report = new FileOutputStream( output );
-			
-			} catch( Exception e ) {
-				suite.addTest( new AutomatedTest( "reportFailed" ) );
-			}
-		}
-		
-		Set keys = testSources.keySet();
-		Iterator iter = keys.iterator();
-		int size = keys.size();
-		String item = null;
-		for( int i = size; i > 0; i-- )
-		{
-			item = (String) iter.next();
-			File file = new File( item );
-			if( file.exists() ){
-				defaultNature = (String) testSources.get( item );
-				fillSuite( suite, file );		
-			}
-		}
-
-		return suite;
-	}
-	private static void fillSuite( TestSuite suite, File path ){
-		File files[] = null;
-		if( path.isFile() ){
-			files = new File[ 1 ];
-			files[0] = path;
-		}
-		else
-			files = path.listFiles();
-
-		File file = null;
-		String filePath = null;
-		int i = 0;
-		try{
-			file = files[ i++ ];
-			while( file != null )
-			{
-				if( file.isDirectory() )
-					fillSuite( suite, file );
-				else if( file.isFile() && nameFilter.accept( file.getParentFile(), file.getName() ) ){
-					try{
-						filePath = file.getCanonicalPath();
-					} catch ( Exception e ){
-						continue;
-					}
-					
-					if(	filePath.endsWith(".cpp") || filePath.endsWith(".hpp") || 
-						filePath.endsWith(".hxx") || filePath.endsWith(".hh") )
-					{
-						natures.put( filePath, "cpp" );
-					} else if( filePath.endsWith(".c") ){ 
-						natures.put( filePath, "c" );
-					} else {
-						natures.put( filePath, defaultNature );
-					}
-					
-					fileList.add( file );
-					suite.addTest( new AutomatedTest( file.getName().replace(',', '_') ) );
-				}				
-				file = files[ i++ ];
-			}
-		} catch( ArrayIndexOutOfBoundsException e ){
-			//done
-		}
 	}
 
 	protected void tearDown () throws Exception	{
@@ -203,7 +99,7 @@ public class AutomatedTest extends TestCase {
 		}
 	}
 	
-	static private void loadProperties() throws Exception{
+	protected void loadProperties() throws Exception{
 		String resourcePath = org.eclipse.core.runtime.Platform.getPlugin("org.eclipse.cdt.ui.tests").find(new Path("/")).getFile();
 		resourcePath += "/parser/org/eclipse/cdt/core/parser/resources";
 		
@@ -237,30 +133,4 @@ public class AutomatedTest extends TestCase {
 		}
 	}
 
-	private static LinkedList fileList = new LinkedList();
-	private static FilenameFilter nameFilter = new Filter();
-	private static FileOutputStream report = null;
-	private static IParserCallback nullCallback = new NullParserCallback();
-	private static Properties properties = new Properties();
-	private static String defaultNature;
-	private static String outputFile = null;
-	private static HashMap testSources = new HashMap();
-	private static HashMap natures = new HashMap();
-	
-	static private class Filter implements FilenameFilter
-	{
-		public boolean accept(File dir, String name) {
-			if( name.endsWith(".cpp") 	|| 
-				name.endsWith(".c") 	|| 
-				name.endsWith(".cc") 	|| 
-				name.endsWith(".h") 	||
-				name.endsWith(".hh")	||
-				name.endsWith(".hxx"))
-			{
-				return true;
-			}
-			else
-				return false;
-		}
-	}
 }

core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/FractionalAutomatedTest.java

@@ -0,0 +1,186 @@
+/*******************************************************************************
+ * Copyright (c) 2001 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors:
+ *     IBM Corp. - Rational Software - initial implementation
+ ******************************************************************************/
+package org.eclipse.cdt.core.parser.tests;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.StringWriter;
+import java.util.NoSuchElementException;
+import java.util.StringTokenizer;
+
+import junit.framework.AssertionFailedError;
+import junit.framework.Test;
+
+import org.eclipse.core.runtime.Path;
+
+import org.eclipse.cdt.internal.core.parser.IParser;
+import org.eclipse.cdt.internal.core.parser.Parser;
+
+/**
+ * @author aniefer
+ *
+ * To change the template for this generated type comment go to
+ * Window>Preferences>Java>Code Generation>Code and Comments
+ */
+public class FractionalAutomatedTest extends AutomatedFramework {
+
+	public FractionalAutomatedTest() {
+		super();
+	}
+
+	public FractionalAutomatedTest(String name) {
+		super(name);
+	}
+	
+	protected AutomatedFramework newTest( String name ){
+		return new FractionalAutomatedTest( name );
+	}
+	protected void loadProperties() throws Exception{
+		String resourcePath = org.eclipse.core.runtime.Platform.getPlugin("org.eclipse.cdt.ui.tests").find(new Path("/")).getFile();
+		resourcePath += "/parser/org/eclipse/cdt/core/parser/resources";
+	
+		try{
+			FileInputStream propertiesIn = new FileInputStream( resourcePath + "/FractionalAutomatedTest.properties");
+			properties.load( propertiesIn );
+		
+			outputFile = properties.getProperty( "outputFile", "" );
+			String sourceInfo = properties.getProperty( "source", "" );
+
+			stepSize = Integer.parseInt( properties.getProperty( "stepSize", "50" ) );
+			windowSize = Integer.parseInt( properties.getProperty( "windowSize", "200" ) );
+			
+			tempFile = properties.getProperty( "tempFile", "" );
+			
+			if( sourceInfo.equals("") )
+				throw new FileNotFoundException();
+			else{
+				StringTokenizer tokenizer = new StringTokenizer( sourceInfo, "," );
+				String str = null, val = null;
+				try{
+					while( tokenizer.hasMoreTokens() ){
+						str = tokenizer.nextToken().trim();
+						val = tokenizer.nextToken().trim();
+					
+						testSources.put( str, val );
+					}
+				} catch ( NoSuchElementException e ){
+					//only way to get here is to have a missing val, assume cpp for that str
+					testSources.put( str, "cpp" );
+				}
+			
+			}
+		} catch ( FileNotFoundException e ){
+			testSources.put( resourcePath + "/cppFiles", "cpp" );
+			testSources.put( resourcePath + "/cFiles", "c" );
+		}
+	}
+	
+	public static Test suite()
+	{
+		AutomatedFramework frame = new FractionalAutomatedTest();
+		
+		return frame.createSuite();
+	}
+	
+	static private void outputTempFile( String code ) {
+		if( tempFile == null || tempFile.equals("") )
+			return;
+			
+		File output = new File( tempFile );
+				
+		try{
+			if( output.exists() ){
+				output.delete();
+			}
+			
+			output.createNewFile();
+			FileOutputStream stream = new FileOutputStream( output );
+			stream.write( code.getBytes() );
+			stream.flush();
+			stream.close();
+		} catch ( Exception e )
+		{}
+	}
+	
+	public void doFile() throws Throwable {
+		assertNotNull( fileList );
+		
+		File file = null;
+		IParser parser = null;
+		
+		try{
+			file = (File)fileList.removeFirst();
+			FileInputStream stream = new FileInputStream( file );
+
+			String filePath = file.getCanonicalPath();
+			String nature = (String)natures.get( filePath );
+
+			boolean cppNature = nature.equalsIgnoreCase("cpp");
+			
+			StringWriter code = new StringWriter(); 
+			
+			byte b[] = new byte[stepSize]; 
+			int n = stream.read( b );
+			while( n != -1 ){
+				code.write( new String( b ) );
+				parser = new Parser( code.toString(), nullCallback, true);
+				parser.setCppNature( cppNature );
+				parser.mapLineNumbers(true);
+			
+				outputTempFile( code.toString() );
+				parser.parse();
+				
+				n = stream.read( b );
+			}
+			
+			String fullCode = code.toString();
+			String windowedCode = null;
+			int length = fullCode.length();
+			int curPos = 0;
+			
+			while( curPos + windowSize < length){
+				windowedCode = fullCode.substring( 0, curPos );
+				windowedCode += "\n" + fullCode.substring( curPos + windowSize, length );
+				
+				parser = new Parser( windowedCode, nullCallback, true );
+				parser.setCppNature( cppNature );
+				parser.mapLineNumbers(true);
+
+				outputTempFile( windowedCode );
+				parser.parse();
+				
+				curPos += stepSize;
+			}
+		} 
+		catch( Throwable e )
+		{
+			String output = null;
+			if( e instanceof AssertionFailedError ){
+				output = file.getCanonicalPath() + ": Parse failed on line ";
+				output += parser.getLineNumberForOffset(parser.getLastErrorOffset()) + "\n";
+			} else {
+				output = file.getCanonicalPath() + ": " + e.getClass().toString();
+				output += " on line " + parser.getLineNumberForOffset(parser.getLastErrorOffset()) + "\n";
+			}
+			if( report != null ){
+				report.write( output.getBytes() );
+			}
+
+			fail( output );
+		}
+	}
+
+	static private int stepSize = 50;
+	static private int windowSize = 200;
+	static private String tempFile = null;
+}

core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java

@@ -1433,4 +1433,31 @@ public class ScannerTestCase extends TestCase
 		validateEOF();
 	}
 		
+	public void testBug36816() throws Exception
+	{
+		initializeScanner( "#include \"foo.h" );
+		validateEOF();
+		
+		initializeScanner( "#include <foo.h" );
+		validateEOF();
+		
+		initializeScanner( "#define FOO(A" );
+		validateEOF();
+		
+		initializeScanner( "#define FOO(A) 1\n FOO(foo" );
+		validateInteger("1");
+	}
+	
+	public void testBug36255() throws Exception
+	{
+		StringWriter writer = new StringWriter();
+		writer.write( "#if defined ( A ) \n" );
+		writer.write( "   #if defined ( B ) && ( B != 0 ) \n" );
+		writer.write( "      boo\n" );
+		writer.write( "   #endif /*B*/\n" );
+		writer.write( "#endif /*A*/" );
+		
+		initializeScanner( writer.toString() );
+		validateEOF();
+	}
 }