Patch for Victor Mozgin.

Fix for PR36701 (Preprocessor stringizing)

core/org.eclipse.cdt.core/parser/ChangeLog

@@ -2,6 +2,9 @@
 	Fixes for templated constructor/destructor/operator declarations.
 	This fixed PR 36766, 36767, 36769 (STL parsing problems).
 
+2003-06-09 Victor Mozgin
+	Fixed Bug 36701 - Scanner looses non-token chars while macro stringizing
+
 2003-06-06 Victor Mozgin
 	Fixed Bug 38065 - Scanner skipped backslashes inside the code
 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java

@@ -1360,6 +1360,120 @@ public class Scanner implements IScanner {
 		throw Parser.endOfFile;
 	}
 
+
+    protected static class endOfMacroTokenException extends Exception {};
+    // the static instance we always use
+    protected static endOfMacroTokenException endOfMacroToken = new endOfMacroTokenException();
+    
+    protected Token nextTokenForStringizing() throws ScannerException, Parser.EndOfFile
+    {     
+        int c = getChar();
+        StringBuffer tokenImage = new StringBuffer();
+
+        try {
+        while (c != NOCHAR) {
+
+            if ((c == ' ') || (c == '\r') || (c == '\t') || (c == '\n')) {
+                
+                if (tokenImage.length() > 0) throw endOfMacroToken;                
+                c = getChar();
+                continue;
+                
+            } else if (c == '"') {
+       
+                if (tokenImage.length() > 0) throw endOfMacroToken;
+                 
+                // string
+                StringBuffer buff = new StringBuffer(); 
+                int beforePrevious = NOCHAR;
+                int previous = c;
+                c = getChar(true);
+
+                for( ; ; )
+                {
+                    if ( c =='"' ) break;
+                    if( c == NOCHAR) break;  
+                    buff.append((char) c);
+                    beforePrevious = previous;
+                    previous = c;
+                    c = getChar(true);
+                }
+
+                if (c != NOCHAR ) 
+                {
+                    return newToken( Token.tSTRING, buff.toString(), contextStack.getCurrentContext());
+    
+                } else {
+                    if (throwExceptionOnUnboundedString)
+                        throw new ScannerException(
+                            "Unbounded string" );
+                }
+        
+            } else {
+                switch (c) {
+                    case '\'' :
+                    if (tokenImage.length() > 0) throw endOfMacroToken;
+                        c = getChar( true ); 
+                        int next = getChar( true );
+                        if( c == '\\' ){
+                            c = next;
+                            next = getChar( true );
+                            if( next == '\'' )
+                                return newToken( Token.tCHAR, '\\' + new Character( (char)c ).toString(), contextStack.getCurrentContext() );
+                            else if( throwExceptionOnBadCharacterRead )
+                                throw new ScannerException( "Invalid character '" + (char)c + "' read @ offset " + contextStack.getCurrentContext().getOffset() + " of file " + contextStack.getCurrentContext().getFilename() );
+                        } else if( next == '\'' )
+                            return newToken( Token.tCHAR, new Character( (char)c ).toString(), contextStack.getCurrentContext() ); 
+                        else
+                            if( throwExceptionOnBadCharacterRead )
+                                throw new ScannerException( "Invalid character '" + (char)c + "' read @ offset " + contextStack.getCurrentContext().getOffset() + " of file " + contextStack.getCurrentContext().getFilename() );
+                    case ',' :
+                        if (tokenImage.length() > 0) throw endOfMacroToken;
+                        return newToken(Token.tCOMMA, ",", contextStack.getCurrentContext());
+                    case '(' :
+                        if (tokenImage.length() > 0) throw endOfMacroToken;
+                        return newToken(Token.tLPAREN, "(", contextStack.getCurrentContext());
+                    case ')' :
+                        if (tokenImage.length() > 0) throw endOfMacroToken;
+                        return newToken(Token.tRPAREN, ")", contextStack.getCurrentContext());
+                    case '/' :
+                        if (tokenImage.length() > 0) throw endOfMacroToken;
+                        c = getChar();
+                        switch (c) {
+                            case '/' :
+                                c = getChar();
+                                while (c != '\n' && c != NOCHAR)
+                                    c = getChar();
+                                continue;
+                            case '*' :
+                                skipOverMultilineComment();
+                                c = getChar();
+                                continue;
+                            default:
+                                tokenImage.append('/');
+                                continue;
+                        }
+                    default :
+                        tokenImage.append((char)c);
+                        c = getChar();
+                }
+            }
+        }
+        } catch (endOfMacroTokenException e) {
+            // unget the first character after the end of token
+            ungetChar(c);            
+        }
+        
+        // return completed token
+        if (tokenImage.length() > 0) {
+            return newToken(Token.tIDENTIFIER, tokenImage.toString(), contextStack.getCurrentContext());
+        }
+        
+        // we're done
+        throw Parser.endOfFile;
+    }
+
+
 	static {
 		cppKeywords.put("and", new Integer(Token.t_and));
 		cppKeywords.put("and_eq", new Integer(Token.t_and_eq));
@@ -1840,6 +1954,48 @@ public class Scanner implements IScanner {
 		}
 	}
     
+    protected Vector getMacroParameters (String params, boolean forStringizing) throws ScannerException {
+        
+        Scanner tokenizer = new Scanner(new StringReader(params), TEXT, definitions);
+        Vector parameterValues = new Vector();
+        Token t = null;
+        String str = new String();
+        boolean space = false;
+        int nParen = 0;
+        
+        try {
+            while (true) {
+                t = forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false);
+                if (t.type == Token.tLPAREN) {
+                    nParen++;
+                } else if (t.type == Token.tRPAREN) {
+                    nParen--;
+                } else if (t.type == Token.tCOMMA && nParen == 0) {
+                    parameterValues.add(str);
+                    str = "";
+                    space = false;
+                    continue;
+                }
+
+                if (space)
+                    str += ' ';
+
+                switch (t.type) {
+                    case Token.tSTRING :  str += '\"' + t.image + '\"'; break;
+                    case Token.tLSTRING : str += "L\"" + t.image + '\"'; break;
+                    case Token.tCHAR :    str += '\'' + t.image + '\'';  break;
+                    default :             str += t.image; break;
+                }
+                space = true;
+            }
+        } catch (Parser.EndOfFile e) {
+            // Good
+            parameterValues.add(str);
+        }
+        
+        return parameterValues;
+    }
+    	
 	protected void expandDefinition(String symbol, Object expansion)
 		throws ScannerException {
 		if (expansion instanceof String ) {
@@ -1869,42 +2025,10 @@ public class Scanner implements IScanner {
 				
 				String betweenTheBrackets = buffer.toString().trim();
                 
-				Scanner tokenizer = new Scanner( new StringReader(betweenTheBrackets), TEXT, definitions );
+                Vector parameterValues = getMacroParameters(betweenTheBrackets, false);
-				Vector parameterValues = new Vector();
+                Vector parameterValuesForStringizing = getMacroParameters(betweenTheBrackets, true);
-				Token t = null;
+                Token t = null;
-				String str = new String();
-				boolean space = false;
-				int nParen = 0;
-				try{
-					while (true) {
-						t = tokenizer.nextToken(false);
-						if( t.type == Token.tLPAREN ){
-							nParen++;
-						} else if ( t.type == Token.tRPAREN ){
-							nParen--;
-						} else if( t.type == Token.tCOMMA && nParen == 0 ){
-							parameterValues.add( str );
-							str = "";
-							space = false;
-							continue;
-						}
                 
-						if( space )
-							str += ' ';
-							
-						switch( t.type )
-						{
-							case Token.tSTRING:  str += '\"' + t.image + '\"';  break;
-							case Token.tLSTRING: str += "L\"" + t.image + '\"';	break;
-							case Token.tCHAR:	 str += '\'' + t.image + '\''; 	break;
-							default:			 str += t.image;				break;
-						}
-						space = true;
-					}
-				} catch (Parser.EndOfFile e) {
-					// Good
-					parameterValues.add( str );
-				}
 				// create a string that represents what needs to be tokenized
 				buffer = new StringBuffer();
 				List tokens = macro.getTokenizedExpansion();
@@ -1944,7 +2068,7 @@ public class Scanner implements IScanner {
 								throw new ScannerException(	"Improper use of the # preprocessing token." );	
 						} else {
 							buffer.append('\"');
-							String value = (String)parameterValues.elementAt(index);
+							String value = (String)parameterValuesForStringizing.elementAt(index);
 							char val [] = value.toCharArray();
 							char ch;
 							int length = value.length();
@@ -1987,7 +2111,8 @@ public class Scanner implements IScanner {
 					}
 					
 					if( t.getType() != tPOUNDPOUND && ! pastingNext )
-						buffer.append( " " ); 
+						if (i < (numberOfTokens-1)) // Do not append to the last one 
+                            buffer.append( " " ); 
 				}
 				String finalString = buffer.toString(); 
 				contextStack.updateContext(

core/org.eclipse.cdt.ui.tests/ChangeLog

@@ -3,6 +3,11 @@
 	Renamed them to testBug36766and36769x(), as they cover both PRs.
 	Added testBug36766and36769D() - test for templated destructor.
 
+2003-06-09 Victor Mozgin
+	Moved testBug36701() from ScannerFailedTests.java to ScannerTestCase.java.
+	Renamed it to testBug36701A() and fixed it.
+	Added ScannerTestCase.testBug36701B().
+
 2003-06-05 John Camelon
 	Moved testBug23478A() & testBug23478B() from failed tests to TranslationUnitTests.java.
 	Removed TranslationUnitFailedTests.java as it was empty. 

core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java

@@ -1,64 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2001 IBM Corporation and others.
- * All rights reserved. This program and the accompanying materials 
- * are made available under the terms of the Common Public License v0.5 
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/cpl-v05.html
- * 
- * Contributors:
- *     IBM Corp. - Rational Software - initial implementation
- ******************************************************************************/
-
-package org.eclipse.cdt.core.parser.failedTests;
-
-import java.io.StringWriter;
-
-import junit.framework.AssertionFailedError;
-import junit.framework.Test;
-import junit.framework.TestSuite;
-
-import org.eclipse.cdt.core.parser.tests.BaseScannerTest;
-
-/**
- * @author aniefer
- *
- * To change the template for this generated type comment go to
- * Window>Preferences>Java>Code Generation>Code and Comments
- */
-public class ScannerFailedTest extends BaseScannerTest  {
-
-	public ScannerFailedTest(String name){
-		super(name);
-	}
-	
-	public static Test suite()
-	{
-		TestSuite suite = new TestSuite();
-		 
-		suite.addTest( new ScannerFailedTest( "testBug36701" ) );
-
-		return suite;
-	}
-	
-	public void testBug36701() throws Exception
-	{
-		boolean testPassed = false;
-		try{
-			StringWriter writer = new StringWriter();
-			writer.write( "#define str(s) # s\n" );
-			writer.write( "str( @ \n )\n");
-			
-			initializeScanner( writer.toString() );
-			validateString( "@ \\\\n" );
-			validateEOF();
-			
-			testPassed = true;
-		} catch( AssertionFailedError e ){
-			//expected failure
-		}
-		
-		if( testPassed )
-			fail( "The expected error did not occur." );
-	}
-	
-}

core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java

@@ -1378,4 +1378,26 @@ public class ScannerTestCase extends BaseScannerTest
 			assertTrue(se.getMessage().equals("Invalid character '\\' read @ offset 5 of file TEXT"));
 		}
 	}
+    
+    public void testBug36701A() throws Exception
+    {
+        StringWriter writer = new StringWriter();
+        writer.write("#define str(s) # s\n");
+        writer.write("str( @ \\n )\n");
+
+        initializeScanner(writer.toString());
+        validateString("@ \\\\n");
+        validateEOF();
+    }
+    
+    public void testBug36701B() throws Exception 
+    {
+        StringWriter writer = new StringWriter();
+        writer.write("#define str(s) # s\n");
+        writer.write("str( @ /*ff*/  \\n  hh  \"aa\"  )\n");
+
+        initializeScanner(writer.toString());
+        validateString("@ \\\\n hh \\\"aa\\\"");
+        validateEOF();
+    }
 }

core/org.eclipse.cdt.ui.tests/suite/org/eclipse/cdt/core/suite/AutomatedIntegrationSuite.java

@@ -80,7 +80,6 @@ public class AutomatedIntegrationSuite extends TestSuite implements TestListener
 		suite.addTestSuite(ACEFailedTest.class);
 		suite.addTestSuite(DOMFailedTest.class);
 		suite.addTestSuite(LokiFailures.class);
-		suite.addTestSuite(ScannerFailedTest.class);
 		suite.addTestSuite(STLFailedTests.class);
 		suite.addTestSuite(CModelElementsFailedTests.class);