Patch for Andrew Niefer.

core:
  Fixed Bug 36287 - Parser failure with new CDT 1.1 parser
  Fixed Bug 37011 - Scanner: #define A "//" not properly handled
 
UI.tests
  Updated FractionalAutomatedTest to use threads
  Modified ScannerTestCase::testBug36287
  Added ScannerTestCase::testBug37011
  Added ScannerTestCase::testOtherPreprocessorDefines

core/org.eclipse.cdt.core/parser/ChangeLog

@@ -1,3 +1,7 @@
+2003-05-01 Andrew Niefer
+	Fixed Bug 36287 - Parser failure with new CDT 1.1 parser
+	Fixed Bug 37011 - Scanner: #define A "//" not properly handled
+	
 2003-04-30 John Camelon
 	Added some Javadoc to IParser.java and Parser.java.  
 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java

@@ -120,17 +120,28 @@ public class Scanner implements IScanner {
 		StringBuffer buffer = new StringBuffer();
 		skipOverWhitespace();
 		int c = getChar();
-
+		boolean inString = false;
+		boolean inChar = false;
 		while (true) {
 			while ((c != '\n')
 				&& (c != '\r')
 				&& (c != '\\')
 				&& (c != '/')
+				&& (c != '"' || ( c == '"' && inChar ) )
+				&& (c != '\'' || ( c == '\'' && inString ) )
 				&& (c != NOCHAR)) {
 				buffer.append((char) c);
-				c = getChar();
+				c = getChar( true );
 			}
+			
 			if (c == '/') {
+				//only care about comments outside of a quote
+				if( inString || inChar ){
+					buffer.append( (char) c );
+					c = getChar( true );
+					continue;
+				}
+				
 				// we need to peek ahead at the next character to see if 
 				// this is a comment or not
 				int next = getChar();
@@ -143,7 +154,7 @@ public class Scanner implements IScanner {
 					if (skipOverMultilineComment())
 						break;
 					else
-						c = getChar();
+						c = getChar( true );
 					continue;
 				} else {
 					// we are not in a comment
@@ -151,12 +162,35 @@ public class Scanner implements IScanner {
 					c = next;
 					continue;
 				}
-			} else {
-				if (c != '\\') {
-					ungetChar(c);
+			} else if( c == '"' ){
+				inString = !inString;
+				buffer.append((char) c);
+				c = getChar( true );
+				continue;
+			} else if( c == '\'' ){
+				inChar = !inChar;
+				buffer.append((char) c);
+				c = getChar( true );
+				continue;
+			} else if( c == '\\' ){
+				c = getChar(true);
+				if( c == '\r' ){
+					c = getChar(true);
+					if( c == '\n' ){
+						c = getChar(true);		
+					}
+				} else if( c == '\n' ){ 
+					c = getChar(true);
 				} else {
-					c = getChar();
+					buffer.append('\\');
+					if( c == '"' || c == '\'' ){
+						buffer.append((char)c);
+						c = getChar( true );
+					}
 				}
+				continue;
+			} else {
+				ungetChar(c);
 				break;
 			}
 		}
@@ -645,13 +679,23 @@ public class Scanner implements IScanner {
 
 				c = getChar();
 				
-				if (c == 'x') {
-					if( ! firstCharZero && floatingPoint )
-					{
-						ungetChar( c ); 
-						return newToken( Token.tDOT, ".", contextStack.getCurrentContext() ); 
+				if( ! firstCharZero && floatingPoint && !(c >= '0' && c <= '9') ){
+					//if pasting, there could actually be a float here instead of just a .
+					if( buff.toString().equals( "." ) ){
+						if( c == '*' ){
+							return newToken( Token.tDOTSTAR, ".*", contextStack.getCurrentContext() );
+						} else if( c == '.' ){
+							if( getChar() == '.' )
+								return newToken( Token.tELIPSE, "..." );
+							else
+								throw new ScannerException( "Invalid floating point @ offset " + contextStack.getCurrentContext().getOffset() ); 
+						} else {
+							ungetChar( c );
+							return newToken( Token.tDOT, ".", contextStack.getCurrentContext() );
+						}
 					}
-					else if( ! firstCharZero ) 
+				} else if (c == 'x') {
+					if( ! firstCharZero ) 
 						throw new ScannerException( "Invalid Hexidecimal @ offset " + contextStack.getCurrentContext().getOffset() );
 					
 					hex = true;
@@ -668,11 +712,6 @@ public class Scanner implements IScanner {
 				if( c == '.' )
 				{
 					buff.append( (char)c);
-					if( floatingPoint || hex ) 	{
-						if( buff.toString().equals( "..") && getChar() == '.' ) 
-							return newToken( Token.tELIPSE, "..." ); 
-						throw new ScannerException( "Invalid floating point @ offset " + contextStack.getCurrentContext().getOffset() );						
-					} 
 					
 					floatingPoint = true;
 					c= getChar(); 

core/org.eclipse.cdt.ui.tests/ChangeLog

@@ -1,3 +1,9 @@
+2003-05-01 Andrew Niefer
+	Updated FractionalAutomatedTest to use threads
+	Modified ScannerTestCase::testBug36287
+	Added ScannerTestCase::testBug37011
+	Added ScannerTestCase::testOtherPreprocessorDefines
+
 2003-04-28 John Camelon
 	Moved testBug36730() & testBug37019() from DOMFailedTests to DOMTests.
 

core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java

@@ -18,8 +18,6 @@ import junit.framework.Test;
 import junit.framework.TestSuite;
 
 import org.eclipse.cdt.core.parser.tests.BaseScannerTest;
-import org.eclipse.cdt.internal.core.parser.ScannerException;
-import org.eclipse.cdt.internal.core.parser.Token;
 
 /**
  * @author aniefer
@@ -38,7 +36,6 @@ public class ScannerFailedTest extends BaseScannerTest  {
 		TestSuite suite = new TestSuite();
 		 
 		suite.addTest( new ScannerFailedTest( "testBug36701" ) );
-		suite.addTest( new ScannerFailedTest( "testBug37011" ) );
 
 		return suite;
 	}
@@ -64,27 +61,4 @@ public class ScannerFailedTest extends BaseScannerTest  {
 			fail( "The expected error did not occur." );
 	}
 	
-	public void testBug37011() throws Exception
-	{
-		boolean testPassed = false;
-		try{
-			StringWriter writer = new StringWriter();
-			writer.write( "#define A \"//\"\n" );
-			writer.write( " { A };\n" );
-			
-			initializeScanner( writer.toString() );
-			validateToken( Token.tLBRACE );
-			validateString( "//" );
-			validateToken( Token.tRBRACE );
-			validateToken( Token.tSEMI );
-			validateEOF();
-			
-			testPassed = true;
-		} catch( ScannerException e ){
-			//expected failure
-		}
-				
-		if( testPassed )
-			fail( "The expected error did not occur." );
-	}
 }

core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/FractionalAutomatedTest.java

@@ -14,14 +14,13 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.StringWriter;
 import java.util.NoSuchElementException;
 import java.util.StringTokenizer;
 
-import junit.framework.AssertionFailedError;
 import junit.framework.Test;
 
-import org.eclipse.cdt.internal.core.parser.IParser;
 import org.eclipse.cdt.internal.core.parser.Parser;
 import org.eclipse.core.runtime.Path;
 
@@ -57,8 +56,8 @@ public class FractionalAutomatedTest extends AutomatedFramework {
 
 			stepSize = Integer.parseInt( properties.getProperty( "stepSize", "50" ) );
 			windowSize = Integer.parseInt( properties.getProperty( "windowSize", "200" ) );
-			
-			tempFile = properties.getProperty( "tempFile", "" );
+			timeOut = Integer.parseInt( properties.getProperty( "timeOut", "5000" ));
+			outputDir = properties.getProperty( "outDir", "" );
 			
 			if( sourceInfo.equals("") )
 				throw new FileNotFoundException();
@@ -91,95 +90,163 @@ public class FractionalAutomatedTest extends AutomatedFramework {
 		return frame.createSuite();
 	}
 	
-	static private void outputTempFile( String code ) {
-		if( tempFile == null || tempFile.equals("") )
-			return;
+	static private String outputFile( String code ) {
+		if( outputDir == null || outputDir.equals("") )
+			return "";
 			
-		File output = new File( tempFile );
+		File output = new File( outputDir );
 				
 		try{
 			if( output.exists() ){
-				output.delete();
+				if( output.isFile() ){
+					output.delete();
+					output.createNewFile();
+					FileOutputStream stream = new FileOutputStream( output );
+					stream.write( code.getBytes() );
+					stream.flush();
+					stream.close();
+					return outputDir;
+				}
+			} else {
+				output.mkdir();
 			}
-			
-			output.createNewFile();
-			FileOutputStream stream = new FileOutputStream( output );
+			File file = new File( outputDir + "/" + failures++ + ".tmp" );
+			if( file.exists() )
+				file.delete();
+			file.createNewFile();
+			FileOutputStream stream = new FileOutputStream( file );
 			stream.write( code.getBytes() );
 			stream.flush();
 			stream.close();
+			
+			return file.getCanonicalPath();
+			
 		} catch ( Exception e )
 		{}
+		return "";
+	}
+	
+	static private void reportHang( String code, String file ){
+		String output = outputFile( code.toString() );
+		if( output.equals("") )
+			output = "Parser hang while parsing " + file + "\n";
+		else 
+			output = "Parser hang while parsing " + output + "\n";
+	 
+		if( report != null ){
+			try{
+				report.write( output.getBytes() );
+			} catch ( IOException e ) {}
+		}
+
+		fail( output );
+	}
+	
+	static private void reportException( String code, String file, String exception ){
+		String output = outputFile( code.toString() );
+
+		if( output.equals("") )
+			output = exception.getClass().toString() + " encountered in " + file + "\n";
+		else 
+			output = exception.getClass().toString() + " encountered in " + output + "\n";
+	 
+		if( report != null ){
+			try{
+				report.write( output.getBytes() );
+			} catch ( IOException e ) {}
+		}
+		
+		fail( output );
 	}
 	
 	public void doFile() throws Throwable {
 		assertNotNull( fileList );
 		
-		File file = null;
-		IParser parser = null;
+		File file = (File)fileList.removeFirst();
+		FileInputStream stream = new FileInputStream( file );
+
+		String filePath = file.getCanonicalPath();
+		String nature = (String)natures.get( filePath );
+
+		boolean cppNature = nature.equalsIgnoreCase("cpp");
 		
-		try{
-			file = (File)fileList.removeFirst();
-			FileInputStream stream = new FileInputStream( file );
+		StringWriter code = new StringWriter(); 
+		
+		ParseThread thread = new ParseThread();
+		
+		byte b[] = new byte[stepSize]; 
+		int n = stream.read( b );
+		while( n != -1 ){
+			code.write( new String( b ) );
 
-			String filePath = file.getCanonicalPath();
-			String nature = (String)natures.get( filePath );
-
-			boolean cppNature = nature.equalsIgnoreCase("cpp");
+			thread.code = code.toString();
+			thread.cppNature = cppNature;
+			thread.start();
+			thread.join( timeOut );
 			
-			StringWriter code = new StringWriter(); 
-			
-			byte b[] = new byte[stepSize]; 
-			int n = stream.read( b );
-			while( n != -1 ){
-				code.write( new String( b ) );
-				parser = new Parser( code.toString(), nullCallback, true);
-				parser.setCppNature( cppNature );
-				parser.mapLineNumbers(true);
-			
-				outputTempFile( code.toString() );
-				parser.parse();
-				
-				n = stream.read( b );
+			if( thread.isAlive() ){
+				//Use deprecated Thread.stop() for now
+				//alternative is to create a callback which could stop the parse on a flag
+				//by throwing something, but that has the disadvantage of being unable to 
+				//stop any loops that don't involve callbacks.
+				thread.stop();
+				reportHang( code.toString(), filePath );
+			} else if( thread.result != null ) {
+				reportException( code.toString(), filePath, thread.result );
 			}
 			
-			String fullCode = code.toString();
-			String windowedCode = null;
-			int length = fullCode.length();
-			int curPos = 0;
+			n = stream.read( b );
+		}
+		
+		String fullCode = code.toString();
+		String windowedCode = null;
+		int length = fullCode.length();
+		int curPos = 0;
+		
+		while( curPos + windowSize < length){
+			windowedCode = fullCode.substring( 0, curPos );
+			windowedCode += "\n" + fullCode.substring( curPos + windowSize, length );
 			
-			while( curPos + windowSize < length){
-				windowedCode = fullCode.substring( 0, curPos );
-				windowedCode += "\n" + fullCode.substring( curPos + windowSize, length );
-				
-				parser = new Parser( windowedCode, nullCallback, true );
-				parser.setCppNature( cppNature );
-				parser.mapLineNumbers(true);
+			thread.code = windowedCode;
+			thread.cppNature = cppNature;
+			thread.file = filePath;
+			thread.start();
+			thread.join( timeOut );
 
-				outputTempFile( windowedCode );
-				parser.parse();
-				
-				curPos += stepSize;
-			}
-		} 
-		catch( Throwable e )
-		{
-			String output = null;
-			if( e instanceof AssertionFailedError ){
-				output = file.getCanonicalPath() + ": Parse failed on line ";
-				output += parser.getLineNumberForOffset(parser.getLastErrorOffset()) + "\n";
-			} else {
-				output = file.getCanonicalPath() + ": " + e.getClass().toString();
-				output += " on line " + parser.getLineNumberForOffset(parser.getLastErrorOffset()) + "\n";
-			}
-			if( report != null ){
-				report.write( output.getBytes() );
+			if( thread.isAlive() )
+			{
+				thread.stop();
+				reportHang( windowedCode, filePath );	
+			} else if( thread.result != null ) {
+				reportException( windowedCode, filePath, thread.result );
 			}
 
-			fail( output );
+			curPos += stepSize;
 		}
 	}
 
+	static class ParseThread extends Thread{
+		public String code;
+		public boolean cppNature;
+		public String file;
+		public String result;
+		
+		public void run(){
+			try{
+				result = null;
+				Parser parser = new Parser( code, nullCallback, true);
+				parser.setCppNature( cppNature );
+				parser.mapLineNumbers(true);
+				parser.parse();
+			} catch ( Exception e ){
+				result = e.getClass().toString();
+			}
+		}
+	}
+	
 	static private int stepSize = 50;
 	static private int windowSize = 200;
-	static private String tempFile = null;
+	static private int timeOut = 5000;
+	static private String outputDir = null;
+	static private int failures = 0;
 }

core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java

@@ -995,7 +995,7 @@ public class ScannerTestCase extends BaseScannerTest
 	
 	public void testBug36287() throws Exception
 	{
-		initializeScanner( "X::X( const X & rtg_arg ) : U( rtg_arg ) , Z( rtg_arg.Z ) , br( rtg_arg.br ){}" );
+		initializeScanner( "X::X( const X & rtg_arg ) : U( rtg_arg ) , Z( rtg_arg.Z ) , er( rtg_arg.er ){}" );
 		validateIdentifier("X");
 		validateToken( Token.tCOLONCOLON);
 		validateIdentifier("X");
@@ -1018,15 +1018,27 @@ public class ScannerTestCase extends BaseScannerTest
 		validateIdentifier( "Z");
 		validateToken( Token.tRPAREN );
 		validateToken( Token.tCOMMA );
-		validateIdentifier( "br");
+		validateIdentifier( "er");
 		validateToken( Token.tLPAREN );
 		validateIdentifier( "rtg_arg");
 		validateToken( Token.tDOT );
-		validateIdentifier( "br");
+		validateIdentifier( "er");
 		validateToken( Token.tRPAREN );
 		validateToken( Token.tLBRACE);
 		validateToken( Token.tRBRACE);
 		validateEOF();
+		
+		initializeScanner( "foo.*bar");
+		validateIdentifier("foo");
+		validateToken( Token.tDOTSTAR );
+		validateIdentifier("bar");
+		validateEOF();
+		
+		initializeScanner( "foo...bar");
+		validateIdentifier("foo");
+		validateToken( Token.tELIPSE );
+		validateIdentifier("bar");
+		validateEOF();
 	}
 
 	public void testBug35892()
@@ -1309,4 +1321,39 @@ public class ScannerTestCase extends BaseScannerTest
 		initializeScanner( writer.toString() );
 		validateEOF();
 	}
+	
+	public void testBug37011() throws Exception{
+		StringWriter writer = new StringWriter();
+		writer.write( "#define A \"//\"");
+		
+		initializeScanner( writer.toString() );
+		
+		validateEOF();
+		validateDefinition("A", "\"//\"");
+	}
+
+	public void testOtherPreprocessorDefines() throws Exception{
+		StringWriter writer = new StringWriter();
+		writer.write( "#define A a//boo\n" );
+		writer.write( "#define B a /*boo*/ a\n" );
+		writer.write( "#define C a \" //boo \"\n" );
+		writer.write( "#define D a \\\"//boo\n" );
+		writer.write( "#define E a \\n \"\\\"\"\n" );
+		writer.write( "#define F a\\\n b\n" );
+		writer.write( "#define G a '\"'//boo\n" );
+		writer.write( "#define H a '\\'//b'\"/*bo\\o*/\" b\n" );
+		 
+		initializeScanner( writer.toString() );
+		
+		validateEOF();
+		
+		validateDefinition("A", "a");
+		validateDefinition("B", "a  a");
+		validateDefinition("C", "a \" //boo \"");
+		validateDefinition("D", "a \\\"");
+		validateDefinition("E", "a \\n \"\\\"\"");
+		validateDefinition("F", "a b");
+		validateDefinition("G", "a '\"'");
+		validateDefinition("H", "a '\\'//b'\"/*bo\\o*/\" b");
+	}
 }