- Added support for digraphs and trigraphs. Both scanner and UI syntax

highlighting are updated.
-  Added support for hex floating point literals.
- Fixed stack overflow problem with string literals concatenation.
- Fixed problem with token pasting in macros.
- This solves PR 39523, 39550, 39552.

core/org.eclipse.cdt.core.tests/ChangeLog

@@ -19,6 +19,13 @@
 	Removed DOMTests, BaseDOMTest, DOMFailedTests after methods were migrated to QuickParseASTTests & ASTFailedTests.  
 	Made sure every parser failed test had a defect number associated with it. 
 	
+2003-07-17 Victor Mozgin
+	Added PerformanceTests.java (not included into AutomatedIntegrationSuite).
+	Moved testBug39523() from DOMFailedTest.java to PerformanceTests.java.
+	Moved testBug39550() from DOMFailedTest.java to DOMTests.java.
+	Moved testBug39552A() and testBug39552B()from DOMFailedTest.java to DOMTests.java.
+	TortureTest overrides timeout value for a very time-consuming test ('concat1.C').
+
 2003-07-15 Victor Mozgin
 	Moved testBug39349() from DOMFailedTest.java to DOMTests.java.
 	Moved testBug39544() from DOMFailedTest.java to DOMTests.java.

core/org.eclipse.cdt.core.tests/failures/org/eclipse/cdt/core/parser/failedTests/ASTFailedTests.java

@@ -60,36 +60,7 @@ public class ASTFailedTests extends BaseASTTest
     {
         assertCodeFailsParse("int* gp_down = static_cast<int*>(gp_stat);");
     }
-
+   
-    public void testBug39523()
-    {
-    	if( ! debugging )
-    	{
-			Writer code = new StringWriter();
-			try	{ 
-				code.write("#define e0	\"a\"\n");
-				code.write("#define e1	e0 e0 e0 e0 e0 e0 e0 e0 e0 e0\n");
-				code.write("#define e2	e1 e1 e1 e1 e1 e1 e1 e1 e1 e1\n");
-				code.write("#define e3	e2 e2 e2 e2 e2 e2 e2 e2 e2 e2\n");
-				code.write("#define e4	e3 e3 e3 e3 e3 e3 e3 e3 e3 e3\n");
-				code.write("#define e5	e4 e4 e4 e4 e4 e4 e4 e4 e4 e4\n");
-				code.write("void foo() { (void)(e5); }\n");
-			} catch( IOException ioe ){}
-			
-			boolean testPassed = false;
-			try {
-				parse(code.toString());
-				testPassed = true;
-				fail( "We should not reach this point");
-			} catch (Throwable e) {
-				if (!(e instanceof StackOverflowError))
-					fail("Unexpected Error: " + e.getMessage());
-			}
-			if (testPassed)
-				fail("The expected error did not occur.");
-    	}
-    }
-    
     public void testBug39525() throws Exception
     {
         assertCodeFailsParse("C &(C::*DD)(const C &x) = &C::operator=;");
@@ -161,80 +132,19 @@ public class ASTFailedTests extends BaseASTTest
     {
         assertCodeFailsParse("struct X x = { .b = 40, .z = {} };");
     }
-    public void testBug39550() throws Exception
+
-    {
-        assertCodeFailsParse("double x = 0x1.fp1;");
-    }
     public void testBug39551A() throws Exception
     {
         IASTFunction function = (IASTFunction)parse("extern float _Complex conjf (float _Complex);").getDeclarations().next();
         assertEquals( function.getName(), "conjf");
     }
+
     public void testBug39551B() throws Exception
     {
         IASTVariable variable = (IASTVariable)parse("_Imaginary double id = 99.99 * __I__;").getDeclarations().next();
         assertEquals( variable.getName(), "id");
     }
     
-    public void testBug39552A() throws Exception
-    {
-        Writer code = new StringWriter();
-        try
-        {
-            code.write(
-                "%:define glue(x, y) x %:%: y	/* #define glue(x, y) x ## y. */\n");
-            code.write("#ifndef glue\n");
-            code.write("#error glue not defined!\n");
-            code.write("#endif\n");
-            code.write("%:define str(x) %:x		/* #define str(x) #x */\n");
-            code.write("int main (int argc, char *argv<::>) /* argv[] */\n");
-            code.write("glue (<, %) /* { */\n");
-            code.write("			 /* di_str[] = */\n");
-            code.write(
-                "  const char di_str glue(<, :)glue(:, >) = str(%:%:<::><%%>%:);\n");
-            code.write(
-                "  /* Check the glue macro actually pastes, and that the spelling of\n");
-            code.write("	 all digraphs is preserved.  */\n");
-            code.write("  if (glue(str, cmp) (di_str, \"%:%:<::><%%>%:\"))\n");
-            code.write("	err (\"Digraph spelling not preserved!\");\n");
-            code.write("  return 0;\n");
-            code.write("glue (%, >) /* } */\n");
-        }
-        catch (IOException ioe)
-        {
-        }
-        assertCodeFailsParse(code.toString());
-    }
-    public void testBug39552B() throws Exception
-    {
-        Writer code = new StringWriter();
-        try
-        {
-            code.write("??=include <stdio.h>\n");
-            code.write("??=define TWELVE 1??/\n");
-            code.write("2\n");
-            code.write("static const char str??(??) = \"0123456789??/n\";\n");
-            code.write("int\n");
-            code.write("main(void)\n");
-            code.write("??<\n");
-            code.write("  unsigned char x = 5;\n");
-            code.write("  if (sizeof str != TWELVE)\n");
-            code.write("	abort ();\n");
-            code.write(
-                "  /* Test ^=, the only multi-character token to come from trigraphs.  */\n");
-            code.write("  x ??'= 3;\n");
-            code.write("  if (x != 6)\n");
-            code.write("	abort ();\n");
-            code.write("  if ((5 ??! 3) != 7)\n");
-            code.write("	abort ();\n");
-            code.write("  return 0;\n");
-            code.write("??>\n");
-        }
-        catch (IOException ioe)
-        {
-        }
-        assertCodeFailsParse(code.toString());
-    }
     public void testBug39553() throws Exception
     {
         parse("#define COMP_INC \"foobar.h\"  \n" + "#include COMP_INC");

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/PerformanceTests.java

@@ -0,0 +1,45 @@
+/**********************************************************************
+ * Copyright (c) 2002,2003 Rational Software Corporation and others.
+ * All rights reserved.   This program and the accompanying materials
+ * are made available under the terms of the Common Public License v0.5
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors: 
+ * IBM Rational Software - Initial API and implementation
+***********************************************************************/
+package org.eclipse.cdt.core.parser.tests;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+
+import org.eclipse.cdt.core.parser.tests.BaseASTTest;
+
+/**
+ * @author vmozgin
+ *
+ */
+public class PerformanceTests extends BaseASTTest
+{
+	public PerformanceTests(String name)
+	{
+		super(name);
+	}
+
+	public void testBug39523() throws Exception
+	{
+		Writer code = new StringWriter();
+		try	{ 
+			code.write("#define e0	\"a\"\n");
+			code.write("#define e1	e0 e0 e0 e0 e0 e0 e0 e0 e0 e0\n");
+			code.write("#define e2	e1 e1 e1 e1 e1 e1 e1 e1 e1 e1\n");
+			code.write("#define e3	e2 e2 e2 e2 e2 e2 e2 e2 e2 e2\n");
+			code.write("#define e4	e3 e3 e3 e3 e3 e3 e3 e3 e3 e3\n");
+			code.write("#define e5	e4 e4 e4 e4 e4 e4 e4 e4 e4 e4\n");
+			code.write("void foo() { (void)(e5); }\n");
+		} catch( IOException ioe ){}
+			
+		parse(code.toString());
+	}    
+}

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java

@@ -1646,6 +1646,61 @@ public class QuickParseASTTests extends BaseASTTest
 		assertEquals( parm.getName(), "" );
 	}
      
+	public void testBug39550() throws Exception
+	{
+		parse("double x = 0x1.fp1;").getDeclarations().next();
+	}
 	
-	
+	public void testBug39552A() throws Exception
+	{
+		Writer code = new StringWriter();
+
+		code.write("%:define glue(x, y) x %:%: y	/* #define glue(x, y) x ## y. */\n");
+		code.write("#ifndef glue\n");
+		code.write("#error glue not defined!\n");
+		code.write("#endif\n");
+        
+		code.write("%:define str(x) %:x		/* #define str(x) #x */\n");
+        
+		code.write("int main (int argc, char *argv<::>) /* argv[] */\n");
+		code.write("glue (<, %) /* { */\n");
+		code.write("			 /* di_str[] = */\n");
+		code.write("  const char di_str glue(<, :)glue(:, >) = str(%:%:<::><%%>%:);\n");
+		code.write("  /* Check the glue macro actually pastes, and that the spelling of\n");
+		code.write("	 all digraphs is preserved.  */\n");
+		code.write("  if (glue(str, cmp) (di_str, \"%:%:<::><%%>%:\"))\n");
+		code.write("	err (\"Digraph spelling not preserved!\");\n");
+		code.write("  return 0;\n");
+		code.write("glue (%, >) /* } */\n");
+
+		parse(code.toString());
+	}
+    
+	public void testBug39552B() throws Exception
+	{
+		Writer code = new StringWriter();
+
+		code.write("??=include <stdio.h>\n");
+		code.write("??=define TWELVE 1??/\n");
+		code.write("2\n");
+        
+		code.write("static const char str??(??) = \"0123456789??/n\";\n");
+        
+		code.write("int\n");
+		code.write("main(void)\n");
+		code.write("??<\n");
+		code.write("  unsigned char x = 5;\n");
+		code.write("  if (sizeof str != TWELVE)\n");
+		code.write("	abort ();\n");
+		code.write("  /* Test ^=, the only multi-character token to come from trigraphs.  */\n");
+		code.write("  x ??'= 3;\n");
+		code.write("  if (x != 6)\n");
+		code.write("	abort ();\n");
+		code.write("  if ((5 ??! 3) != 7)\n");
+		code.write("	abort ();\n");
+		code.write("  return 0;\n");
+		code.write("??>\n");
+		
+		parse(code.toString());
+	}
 }

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/TortureTest.java

@@ -211,6 +211,8 @@ public class TortureTest extends FractionalAutomatedTest {
 	
 	
 	public void doFile() throws Throwable {
+		int timeOut = FractionalAutomatedTest.timeOut;
+		
 		assertNotNull (fileList);
 		
 		File file = (File)fileList.removeFirst();
@@ -230,6 +232,12 @@ public class TortureTest extends FractionalAutomatedTest {
 		
 		String testCode = code.toString();
 		
+		if ( file.getName().equals("concat1.C")) {
+			// This is a really time-consuming test,
+			// override timeout
+			timeOut = 600000;
+		}
+		
 		if (isExpectedToPass(testCode, file)) {
 			ParseThread thread = new ParseThread();
 

core/org.eclipse.cdt.core/parser/ChangeLog

@@ -29,6 +29,13 @@
 	Restructured AST class hierarchy.  
 	Removed the old IParserCallback return Objects from every Parser method.
 	
+2003-07-17 Victor Mozgin
+	Added support for digraphs and trigraphs.
+	Added support for hex floating point literals.
+	Fixed stack overflow problem with string literals concatenation.
+	Fixed problem with token pasting in macros.
+	This solves PR 39523, 39550, 39552.
+
 2003-07-15 Victor Mozgin
 	Fixed PR 39349 : Scanner fails on long long literals.
 	Fixed PR 39544 : Scanner fails on wide char literals.

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java

@@ -370,6 +370,9 @@ public class Scanner implements IScanner {
 	// these are scanner configuration aspects that we perhaps want to tweak
 	// eventually, these should be configurable by the client, but for now
 	// we can just leave it internal
+	private boolean enableDigraphReplacement = true;
+	private boolean enableTrigraphReplacement = true;
+	private boolean enableTrigraphReplacementInStrings = true;
 	private boolean throwExceptionOnBadPreprocessorSyntax = true;
 	private boolean throwExceptionOnInclusionNotFound = true;
 	private boolean throwExceptionOnBadMacroExpansion = true;
@@ -427,8 +430,69 @@ public class Scanner implements IScanner {
 				}
 			}
 		} while (!done);
-
+		
-		if( ! insideString )
+		int baseOffset = lastContext.getOffset() - lastContext.undoStackSize() - 1;
+		
+		if (enableTrigraphReplacement && (!insideString || enableTrigraphReplacementInStrings)) {
+			// Trigraph processing
+			enableTrigraphReplacement = false;
+			if (c == '?') {
+				c = getChar(insideString);
+				if (c == '?') {
+					c = getChar(insideString);
+					switch (c) {
+						case '(':
+							expandDefinition("??(", "[", baseOffset);
+							c = getChar(insideString);
+							break;
+						case ')':
+							expandDefinition("??)", "]", baseOffset);
+							c = getChar(insideString);
+							break;
+						case '<':
+							expandDefinition("??<", "{", baseOffset);
+							c = getChar(insideString);
+							break;
+						case '>':
+							expandDefinition("??>", "}", baseOffset);
+							c = getChar(insideString);
+							break;
+						case '=':
+							expandDefinition("??=", "#", baseOffset);
+							c = getChar(insideString);
+							break;
+						case '/':
+							expandDefinition("??/", "\\", baseOffset);
+							c = getChar(insideString);
+							break;
+						case '\'':
+							expandDefinition("??\'", "^", baseOffset);
+							c = getChar(insideString);
+							break;
+						case '!':
+							expandDefinition("??!", "|", baseOffset);
+							c = getChar(insideString);
+							break;
+						case '-':
+							expandDefinition("??-", "~", baseOffset);
+							c = getChar(insideString);
+							break;
+						default:
+							// Not a trigraph
+							ungetChar(c);
+							ungetChar('?');
+							c = '?';	
+					}
+				} else {
+					// Not a trigraph
+					ungetChar(c);
+					c = '?';
+				}
+			}
+			enableTrigraphReplacement = true;
+		} 
+		
+		if (!insideString)
 		{
 			if (c == '\\') {
 				c = getChar(false);
@@ -447,8 +511,48 @@ public class Scanner implements IScanner {
 					ungetChar(c);
 					c = '\\';
 				}
+			} else if (enableDigraphReplacement) {
+				enableDigraphReplacement = false;
+				// Digraph processing
+				if (c == '<') {
+					c = getChar(false);
+					if (c == '%') {
+						expandDefinition("<%", "{", baseOffset);
+						c = getChar(false);
+					} else if (c == ':') {
+						expandDefinition("<:", "[", baseOffset);
+						c = getChar(false);
+					} else {
+						// Not a digraph
+						ungetChar(c);
+						c = '<';
+					}
+				} else if (c == ':') {
+					c = getChar(false);
+					if (c == '>') {
+						expandDefinition(":>", "]", baseOffset);
+						c = getChar(false);
+					} else {
+						// Not a digraph
+						ungetChar(c);
+						c = ':';
+					}
+				} else if (c == '%') {
+					c = getChar(false);
+					if (c == '>') {
+						expandDefinition("%>", "}", baseOffset);
+						c = getChar(false);
+					} else if (c == ':') {
+						expandDefinition("%:", "#", baseOffset);
+						c = getChar(false);
+					} else {
+						// Not a digraph
+						ungetChar(c);
+						c = '%';
+					}
+				}
+				enableDigraphReplacement = true;
 			}
-
 		}
 			
 		return c;
@@ -483,10 +587,14 @@ public class Scanner implements IScanner {
 	
 
 	public IToken nextToken() throws ScannerException, EndOfFile {
-		return nextToken( true ); 
+		return nextToken( true, false ); 
 	}
 
-	public IToken nextToken( boolean pasting ) throws ScannerException, EndOfFile
+	public IToken nextToken(boolean pasting) throws ScannerException, EndOfFile {
+		return nextToken( pasting, false ); 
+	}
+
+	public IToken nextToken( boolean pasting, boolean lookingForNextAlready ) throws ScannerException, EndOfFile
 	{
 		if( cachedToken != null ){
 			setCurrentToken( cachedToken );
@@ -564,29 +672,34 @@ public class Scanner implements IScanner {
 				if (c != NOCHAR ) 
 				{
 					int type = wideLiteral ? IToken.tLSTRING : IToken.tSTRING;
-					
+										
 					//If the next token is going to be a string as well, we need to concatenate
 					//it with this token.
 					IToken returnToken = newToken( type, buff.toString(), contextStack.getCurrentContext());
-					IToken next = null;
-					try{
-						next = nextToken( true );
-					} catch( EndOfFile e ){ 
-						next = null;
-					}
 					
-					while( next != null && next.getType()  == returnToken.getType() ){
+					if (!lookingForNextAlready) {
-						returnToken.setImage( returnToken.getImage() + next.getImage() ); 
+						IToken next = null;
-						returnToken.setNext( null );
+						try{
-						currentToken = returnToken; 
+							next = nextToken( true, true );
-						try{ 
-							next = nextToken( true ); 
 						} catch( EndOfFile e ){ 
 							next = null;
 						}
+						
+						while( next != null && next.getType()  == returnToken.getType() ){
+							returnToken.setImage( returnToken.getImage() + next.getImage() ); 
+							returnToken.setNext( null );
+							currentToken = returnToken; 
+							try{
+								next = nextToken( true, true );
+							} catch( EndOfFile e ){ 
+								next = null;
+							}
+						}
+						
+						cachedToken = next;
+					
 					}
 					
-					cachedToken = next;
 					currentToken = returnToken;
 					returnToken.setNext( null );
 									
@@ -730,7 +843,9 @@ public class Scanner implements IScanner {
 					
 					floatingPoint = true;
 					c= getChar(); 
-					while ((c >= '0' && c <= '9') )
+					while ((c >= '0' && c <= '9')
+					|| (hex
+						&& ((c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))))
 					{
 						buff.append((char) c);
 						c = getChar();
@@ -738,10 +853,10 @@ public class Scanner implements IScanner {
 				}
 				
 
-				if( c == 'e' || c == 'E' )
+				if (c == 'e' || c == 'E' || (hex && (c == 'p' || c == 'P')))
 				{
 					if( ! floatingPoint ) floatingPoint = true; 
-					// exponent type for flaoting point 
+					// exponent type for floating point 
 					buff.append((char)c);
 					c = getChar(); 
 					
@@ -2138,8 +2253,10 @@ public class Scanner implements IScanner {
 					if( i != numberOfTokens - 1)
 					{
 						IToken t2 = (IToken) tokens.get(i+1);
-						if( t2.getType() == tPOUNDPOUND )
+						if( t2.getType() == tPOUNDPOUND ) {
-							pastingNext = true;  
+							pastingNext = true;
+							i++;
+						}  
 					}
 					
 					if( t.getType() != tPOUNDPOUND && ! pastingNext )

core/org.eclipse.cdt.ui/ChangeLog

@@ -11,6 +11,9 @@
 2003-07-17  John Camelon
 	Partially converted DOM to ISourceElementRequestor (requires refactoring of CModelBuilder & StuctureComparator modules in near future).  
 
+2003-07-17 Victor Mozgin
+	Added support for di- and trigraph notation of preprocessor directives.
+
 2003-07-16 Alain Magloire
 
 	Patch from Alex chapiro.

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/PreprocessorRule.java

@@ -52,6 +52,7 @@ public class PreprocessorRule extends WordRule implements IRule {
 	public IToken evaluate(ICharacterScanner scanner) {
 		int c;
 		int nCharsToRollback = 0;
+		boolean hashSignDetected = false;
 
 		if (scanner.getColumn() > 0)
 			return Token.UNDEFINED;
@@ -60,8 +61,30 @@ public class PreprocessorRule extends WordRule implements IRule {
 			c = scanner.read();
 			nCharsToRollback++;
 		} while (Character.isWhitespace((char) c));
-
+		
+		
+		// Di- and trigraph support
 		if (c == '#') {
+			hashSignDetected = true;
+		} else if (c == '%') {
+			c = scanner.read();
+			nCharsToRollback++;
+			if (c == ':') {
+				hashSignDetected = true;
+			}
+		} else if (c == '?') {
+			c = scanner.read();
+			nCharsToRollback++;
+			if (c == '?') {
+				c = scanner.read();
+				nCharsToRollback++;
+				if (c == '=') {
+					hashSignDetected = true;
+				}
+			}
+		}
+
+		if (hashSignDetected) {
 
 			do {
 				c = scanner.read();