Patch for Andrew Niefer:

Fixed Bug36475 - Scanner does not concatenate strings
Fixed Bug36509 - Scanner turns strings into identifiers when expanding macros
Fixed Bug36521 - Scanner gets confused over commas in function like macros
Fixed Bug36695 - Scanner looses escaping on chars (ie '\4' to '4')
 
 
Moved ScannerFailedTest::testBug36475 to ScannerTestCase::testBug36475() 
Moved ScannerFailedTest::testBug36509 to ScannerTestCase::testBug36509()
Moved ScannerFailedTest::testBug36521 to ScannerTestCase::testBug36521()
Added ScannerTestCase::testBug36695()
 
Updated ScannerTestCase::testBug36047
Updated ScannerTestCase::testBug36045

core/org.eclipse.cdt.core/parser/ChangeLog

@@ -1,3 +1,9 @@
+2003-04-21 Andrew Niefer
+	Fixed Bug36475 - Scanner does not concatenate strings
+	Fixed Bug36509 - Scanner turns strings into identifiers when expanding macros
+	Fixed Bug36521 - Scanner gets confused over commas in function like macros
+	Fixed Bug36695 - Scanner looses escaping on chars (ie '\4' to '4')
+
 2003-04-17 John Camelon
 	Fixed error in Elaborated Enumeration Types.  
 	Fixed Bug36559  - Parsing Templates... 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java

@@ -300,6 +300,7 @@ public class Scanner implements IScanner {
 	private static HashMap ppDirectives = new HashMap();
 
 	private Token currentToken = null;
+	private Token cachedToken = null;
 
 	private boolean passOnToClient = true; 
 	private BranchTracker branches = new BranchTracker();
@@ -436,6 +437,11 @@ public class Scanner implements IScanner {
 
 	protected Token nextToken( boolean pasting ) throws ScannerException, Parser.EndOfFile
 	{
+		if( cachedToken != null ){
+			setCurrentToken( cachedToken );
+			cachedToken = null;
+			return currentToken;	
+		}
 		
 		count++;
 		boolean madeMistake = false; 
@@ -510,10 +516,33 @@ public class Scanner implements IScanner {
 				if (c != NOCHAR ) 
 				{
 					int type = wideString ? Token.tLSTRING : Token.tSTRING;
-					return newToken(
+					
-						type,
+					//If the next token is going to be a string as well, we need to concatenate
-						buff.toString(),
+					//it with this token.
-						contextStack.getCurrentContext());
+					Token returnToken = newToken( type, buff.toString(), contextStack.getCurrentContext());
+					Token next = null;
+					try{
+						next = nextToken( true );
+					} catch( Parser.EndOfFile e ){ 
+						next = null;
+					}
+					
+					while( next != null && next.type == returnToken.type ){
+						returnToken.image += next.image;
+						returnToken.setNext( null );
+						currentToken = returnToken; 
+						try{ 
+							next = nextToken( true ); 
+						} catch( Parser.EndOfFile e ){ 
+							next = null;
+						}
+					}
+					
+					cachedToken = next;
+					currentToken = returnToken;
+					returnToken.setNext( null );
+									
+					return returnToken; 
 	
 				} else {
 					if (throwExceptionOnUnboundedString)
@@ -916,9 +945,16 @@ public class Scanner implements IScanner {
 			} else {
 				switch (c) {
 					case '\'' :
-						c = getChar(); 
+						c = getChar( true ); 
-						int next = getChar(); 
+						int next = getChar( true );
-						if( next == '\'' )
+						if( c == '\\' ){
+							c = next;
+							next = getChar( true );
+							if( next == '\'' )
+								return newToken( Token.tCHAR, '\\' + new Character( (char)c ).toString(), contextStack.getCurrentContext() );
+							else if( throwExceptionOnBadCharacterRead )
+								throw new ScannerException( "Invalid character '" + (char)c + "' read @ offset " + contextStack.getCurrentContext().getOffset() + " of file " + contextStack.getCurrentContext().getFilename() );
+						} else if( next == '\'' )
 							return newToken( Token.tCHAR, new Character( (char)c ).toString(), contextStack.getCurrentContext() ); 
 						else
 							if( throwExceptionOnBadCharacterRead )
@@ -1714,7 +1750,7 @@ public class Scanner implements IScanner {
 					if (bracketCount == 0)
 						break;
 					buffer.append((char) c);
-					c = getChar();
+					c = getChar( true );
 				}
 				
 				String betweenTheBrackets = buffer.toString().trim();
@@ -1724,11 +1760,15 @@ public class Scanner implements IScanner {
 				Token t = null;
 				String str = new String();
 				boolean space = false;
+				int nParen = 0;
 				try{
 					while (true) {
 						t = tokenizer.nextToken(false);
-						if( t.type == Token.tCOMMA )
+						if( t.type == Token.tLPAREN ){
-						{
+							nParen++;
+						} else if ( t.type == Token.tRPAREN ){
+							nParen--;
+						} else if( t.type == Token.tCOMMA && nParen == 0 ){
 							parameterValues.add( str );
 							str = "";
 							space = false;
@@ -1738,11 +1778,13 @@ public class Scanner implements IScanner {
 						if( space )
 							str += ' ';
 							
-						if( t.type == Token.tSTRING )	
+						switch( t.type )
-							str += '\"' + t.image + '\"';							
+						{
-						else 
+							case Token.tSTRING:  str += '\"' + t.image + '\"';  break;
-							str += t.image;
+							case Token.tLSTRING: str += "L\"" + t.image + '\"';	break;
-							
+							case Token.tCHAR:	 str += '\'' + t.image + '\''; 	break;
+							default:			 str += t.image;				break;
+						}
 						space = true;
 					}
 				} catch (Parser.EndOfFile e) {
@@ -1812,7 +1854,13 @@ public class Scanner implements IScanner {
 							buffer.append('\"');
 						}
 					} else {
-						buffer.append(t.image);
+						switch( t.type )
+						{
+							case Token.tSTRING:  buffer.append('\"' + t.image + '\"');  break;
+							case Token.tLSTRING: buffer.append("L\"" + t.image + '\"');	break;
+							case Token.tCHAR:	 buffer.append('\'' + t.image + '\''); 	break;
+							default:			 buffer.append(t.image);				break;
+						}
 					}
 					
 					boolean pastingNext = false;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java

@@ -17,6 +17,10 @@ public class Token {
 		image = i;
 		filename = context.getFilename();
 		offset = context.getOffset() - image.length() - context.undoStackSize();
+		
+		if( type == tLSTRING || type == tSTRING || type == tCHAR ){
+			offset--;
+		}
 	}
 	
 	public Token(int t, String i) {

core/org.eclipse.cdt.ui.tests/ChangeLog

@@ -1,3 +1,11 @@
+2003-04-17 Andrew Niefer
+	Added ScannerTestCase::testBug36695()
+	Moved ScannerFailedTest::testBug36521 to ScannerTestCase::testBug36521()
+	Moved ScannerFailedTest::testBug36509 to ScannerTestCase::testBug36509()
+	Moved ScannerFailedTest::testBug36475 to ScannerTestCase::testBug36475() 
+	Updated ScannerTestCase::testBug36047
+	Updated ScannerTestCase::testBug36045
+
 2003-04-17 John Camelon
 	Updated DOMTests::testBug36600().
 	Updated LineNumberTest::testDOMLineNos().  

core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java

@@ -37,102 +37,26 @@ public class ScannerFailedTest extends ScannerTestCase {
 	{
 		TestSuite suite = new TestSuite();
 		 
-		suite.addTest( new ScannerFailedTest( "testBug36475" ) );
+		suite.addTest( new ScannerFailedTest( "testBug36701" ) );
-               suite.addTest( new ScannerFailedTest( "testBug36509" ) ); 
-               suite.addTest( new ScannerFailedTest( "testBug36521" ) ); 
 
 		return suite;
 	}
 	
-	public void testBug36475() throws Exception
+	public void testBug36701() throws Exception
 	{
 		boolean testPassed = false;
-		try
+		try{
-		{
 			StringWriter writer = new StringWriter();
-			writer.write( " \"A\" \"B\" \"C\" " ); 
+			writer.write( "#define str(s) # s\n" );
+			writer.write( "str( @ \n )\n");
 			
 			initializeScanner( writer.toString() );
-			  
+			validateString( "@ \\\\n" );
-			validateString( "ABC" );
 			validateEOF();
 			
 			testPassed = true;
-		
+		} catch( AssertionFailedError e ){
-		}
+			//expected failure
-		catch( Throwable e )
-		{
-			if( !(e instanceof AssertionFailedError) ){
-				fail( "Unexpected Error: " + e.getMessage() );
-			}
-               } 
-       
-               if( testPassed ) 
-                       fail( "The expected error did not occur." ); 
-       } 
-       public void testBug36509() throws Exception 
-       { 
-               boolean testPassed = false; 
-               
-               try{ 
-                       StringWriter writer = new StringWriter(); 
-                       writer.write("#define debug(s, t) printf(\"x\" # s \"= %d, x\" # t \"= %s\", \\\n"); 
-                       writer.write("                    x ## s, x ## t) \n"); 
-                       
-                       initializeScanner( writer.toString() ); 
-                       //printf("x" "1" "=%d, x" "2" "= %s", x1, x2); 
-                       validateIdentifier( "printf" ); 
-                       validateToken( Token.tLPAREN ); 
-                       validateString("x"); 
-                       validateString("1"); 
-                       validateString("= %d, x"); 
-                       validateString("2"); 
-                       validateString("= %s"); 
-                       validateToken(Token.tCOMMA); 
-                       validateIdentifier("x1"); 
-                       validateToken(Token.tCOMMA); 
-                       validateIdentifier("x2"); 
-                       validateToken(Token.tRPAREN); 
-                       validateToken(Token.tSEMI); 
-                       
-                       testPassed = true; 
-               } 
-               catch( Throwable e ) 
-               { 
-                       if( !(e instanceof AssertionFailedError) ){ 
-                               fail( "Unexpected Error: " + e.getMessage() ); 
-                       } 
-               } 
-       
-               if( testPassed ) 
-                       fail( "The expected error did not occur." ); 
-       } 
-       public void testBug36521() throws Exception 
-       { 
-               boolean testPassed = false; 
-               
-               try{ 
-                       StringWriter writer = new StringWriter(); 
-                       writer.write("#define str(s)      # s\n"); 
-                       writer.write("fputs(str(strncmp(\"abc\\0d\", \"abc\", \'\\4\')\n"); 
-                       writer.write("        == 0) str(: @\\n), s);\n"); 
-                                               
-                       initializeScanner( writer.toString() ); 
-                       validateIdentifier("fputs"); 
-                       validateToken(Token.tLPAREN); 
-                       validateString("strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') == 0"); 
-                       validateString(": @\\n"); 
-                       validateToken(Token.tCOMMA); 
-                       validateIdentifier("s"); 
-                       validateToken(Token.tRPAREN); 
-                       validateToken(Token.tSEMI); 
-                       
-                       testPassed = true; 
-               } 
-               catch( ScannerException e ) 
-               { 
-                       if( !e.getMessage().equals( "Improper use of macro str" ) ) 
-                               fail( "Unexpected Error: " + e.getMessage() ); 
 		}
 		
 		if( testPassed )

core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java

@@ -1072,6 +1072,16 @@ public class ScannerTestCase extends TestCase
 			assertTrue(false);
 		}		
 	}
+	public void validateChar( String expected ) throws ScannerException
+	{
+		try {
+			Token t= scanner.nextToken();
+			assertTrue(t.getType() == Token.tCHAR );
+			assertEquals( t.getImage(), expected ); 
+		} catch (Parser.EndOfFile e) {
+			assertTrue(false);
+		}		
+	}
 
 	public void validateString( String expectedImage ) throws ScannerException
 	{
@@ -1219,8 +1229,7 @@ public class ScannerTestCase extends TestCase
 		buffer.append( '"');
 		buffer.append( "\n\n");
 		initializeScanner( buffer.toString());
-		validateString( "\\\"");
+		validateString( "\\\"\\\\");
-		validateString( "\\\\");
 	}
 
 	public void testConditionalWithBraces()
@@ -1343,15 +1352,68 @@ public class ScannerTestCase extends TestCase
 		writer.write( "MAD_VERSION\n" );
 		initializeScanner( writer.toString() );
 		  
-		validateString( "2" );
+		validateString( "2.1.3.boo" );
-		validateString( "." );
-		validateString( "1" );
-		validateString( "." );
-		validateString( "3" );
-		validateString( "." );
-		validateString( "boo" );
 		
 		validateEOF(); 
 	}
 	
+	public void testBug36475() throws Exception
+	{
+		StringWriter writer = new StringWriter(); 
+		writer.write( " \"A\" \"B\" \"C\" " ); 
+		
+		initializeScanner( writer.toString() );
+		  
+		validateString( "ABC" );
+		validateEOF(); 
+	}
+	
+	public void testBug36509() throws Exception 
+	{ 
+		StringWriter writer = new StringWriter(); 
+		writer.write("#define debug(s, t) printf(\"x\" # s \"= %d, x\" # t \"= %s\", \\\n"); 
+		writer.write("                    x ## s, x ## t) \n"); 
+		writer.write("debug(1, 2);");
+		   
+		initializeScanner( writer.toString() ); 
+		//printf("x1=%d, x2= %s", x1, x2); 
+		validateIdentifier( "printf" ); 
+		validateToken( Token.tLPAREN ); 
+		validateString("x1= %d, x2= %s"); 
+		validateToken(Token.tCOMMA); 
+		validateIdentifier("x1"); 
+		validateToken(Token.tCOMMA); 
+		validateIdentifier("x2"); 
+		validateToken(Token.tRPAREN); 
+		validateToken(Token.tSEMI); 
+		validateEOF();
+	}
+	
+	public void testBug36695() throws Exception
+	{
+		StringWriter writer = new StringWriter();
+		writer.write("\'\\4\'  \'\\n\'");
+		initializeScanner( writer.toString() );
+		
+		validateChar( "\\4" );
+		validateChar( "\\n" );
+		validateEOF();
+	}
+	
+	public void testBug36521() throws Exception 
+	{ 
+		StringWriter writer = new StringWriter(); 
+		writer.write("#define str(s)      # s\n"); 
+		writer.write("fputs(str(strncmp(\"abc\\0d\", \"abc\", \'\\4\')\n"); 
+		writer.write("        == 0), s);\n"); 
+		                         
+		initializeScanner( writer.toString() ); 
+		validateIdentifier("fputs"); 
+		validateToken(Token.tLPAREN); 
+		validateString("strncmp ( \\\"abc\\\\0d\\\" , \\\"abc\\\" , '\\\\4' ) == 0"); 
+		validateToken(Token.tCOMMA); 
+		validateIdentifier("s"); 
+		validateToken(Token.tRPAREN); 
+		validateToken(Token.tSEMI); 
+	}
 }