From 329664223a168d7d69779ac24eb0bd598e07c0f8 Mon Sep 17 00:00:00 2001 From: John Camelon Date: Tue, 22 Jul 2003 00:44:18 +0000 Subject: [PATCH] - Added support for digraphs and trigraphs. Both scanner and UI syntax highlighting are updated. - Added support for hex floating point literals. - Fixed stack overflow problem with string literals concatenation. - Fixed problem with token pasting in macros. - This solves PR 39523, 39550, 39552. --- core/org.eclipse.cdt.core.tests/ChangeLog | 7 + .../parser/failedTests/ASTFailedTests.java | 96 +--------- .../core/parser/tests/PerformanceTests.java | 45 +++++ .../core/parser/tests/QuickParseASTTests.java | 57 +++++- .../cdt/core/parser/tests/TortureTest.java | 8 + core/org.eclipse.cdt.core/parser/ChangeLog | 7 + .../cdt/internal/core/parser/Scanner.java | 165 +++++++++++++++--- core/org.eclipse.cdt.ui/ChangeLog | 3 + .../internal/ui/text/PreprocessorRule.java | 25 ++- 9 files changed, 294 insertions(+), 119 deletions(-) create mode 100644 core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/PerformanceTests.java diff --git a/core/org.eclipse.cdt.core.tests/ChangeLog b/core/org.eclipse.cdt.core.tests/ChangeLog index 3fa1b1ae91c..00a2c53cc10 100644 --- a/core/org.eclipse.cdt.core.tests/ChangeLog +++ b/core/org.eclipse.cdt.core.tests/ChangeLog @@ -19,6 +19,13 @@ Removed DOMTests, BaseDOMTest, DOMFailedTests after methods were migrated to QuickParseASTTests & ASTFailedTests. Made sure every parser failed test had a defect number associated with it. +2003-07-17 Victor Mozgin + Added PerformanceTests.java (not included into AutomatedIntegrationSuite). + Moved testBug39523() from DOMFailedTest.java to PerformanceTests.java. + Moved testBug39550() from DOMFailedTest.java to DOMTests.java. + Moved testBug39552A() and testBug39552B()from DOMFailedTest.java to DOMTests.java. + TortureTest overrides timeout value for a very time-consuming test ('concat1.C'). + 2003-07-15 Victor Mozgin Moved testBug39349() from DOMFailedTest.java to DOMTests.java. Moved testBug39544() from DOMFailedTest.java to DOMTests.java. diff --git a/core/org.eclipse.cdt.core.tests/failures/org/eclipse/cdt/core/parser/failedTests/ASTFailedTests.java b/core/org.eclipse.cdt.core.tests/failures/org/eclipse/cdt/core/parser/failedTests/ASTFailedTests.java index 2015feee558..a4006397ef9 100644 --- a/core/org.eclipse.cdt.core.tests/failures/org/eclipse/cdt/core/parser/failedTests/ASTFailedTests.java +++ b/core/org.eclipse.cdt.core.tests/failures/org/eclipse/cdt/core/parser/failedTests/ASTFailedTests.java @@ -60,36 +60,7 @@ public class ASTFailedTests extends BaseASTTest { assertCodeFailsParse("int* gp_down = static_cast(gp_stat);"); } - - public void testBug39523() - { - if( ! debugging ) - { - Writer code = new StringWriter(); - try { - code.write("#define e0 \"a\"\n"); - code.write("#define e1 e0 e0 e0 e0 e0 e0 e0 e0 e0 e0\n"); - code.write("#define e2 e1 e1 e1 e1 e1 e1 e1 e1 e1 e1\n"); - code.write("#define e3 e2 e2 e2 e2 e2 e2 e2 e2 e2 e2\n"); - code.write("#define e4 e3 e3 e3 e3 e3 e3 e3 e3 e3 e3\n"); - code.write("#define e5 e4 e4 e4 e4 e4 e4 e4 e4 e4 e4\n"); - code.write("void foo() { (void)(e5); }\n"); - } catch( IOException ioe ){} - - boolean testPassed = false; - try { - parse(code.toString()); - testPassed = true; - fail( "We should not reach this point"); - } catch (Throwable e) { - if (!(e instanceof StackOverflowError)) - fail("Unexpected Error: " + e.getMessage()); - } - if (testPassed) - fail("The expected error did not occur."); - } - } - + public void testBug39525() throws Exception { assertCodeFailsParse("C &(C::*DD)(const C &x) = &C::operator=;"); @@ -161,80 +132,19 @@ public class ASTFailedTests extends BaseASTTest { assertCodeFailsParse("struct X x = { .b = 40, .z = {} };"); } - public void testBug39550() throws Exception - { - assertCodeFailsParse("double x = 0x1.fp1;"); - } + public void testBug39551A() throws Exception { IASTFunction function = (IASTFunction)parse("extern float _Complex conjf (float _Complex);").getDeclarations().next(); assertEquals( function.getName(), "conjf"); } + public void testBug39551B() throws Exception { IASTVariable variable = (IASTVariable)parse("_Imaginary double id = 99.99 * __I__;").getDeclarations().next(); assertEquals( variable.getName(), "id"); } - public void testBug39552A() throws Exception - { - Writer code = new StringWriter(); - try - { - code.write( - "%:define glue(x, y) x %:%: y /* #define glue(x, y) x ## y. */\n"); - code.write("#ifndef glue\n"); - code.write("#error glue not defined!\n"); - code.write("#endif\n"); - code.write("%:define str(x) %:x /* #define str(x) #x */\n"); - code.write("int main (int argc, char *argv<::>) /* argv[] */\n"); - code.write("glue (<, %) /* { */\n"); - code.write(" /* di_str[] = */\n"); - code.write( - " const char di_str glue(<, :)glue(:, >) = str(%:%:<::><%%>%:);\n"); - code.write( - " /* Check the glue macro actually pastes, and that the spelling of\n"); - code.write(" all digraphs is preserved. */\n"); - code.write(" if (glue(str, cmp) (di_str, \"%:%:<::><%%>%:\"))\n"); - code.write(" err (\"Digraph spelling not preserved!\");\n"); - code.write(" return 0;\n"); - code.write("glue (%, >) /* } */\n"); - } - catch (IOException ioe) - { - } - assertCodeFailsParse(code.toString()); - } - public void testBug39552B() throws Exception - { - Writer code = new StringWriter(); - try - { - code.write("??=include \n"); - code.write("??=define TWELVE 1??/\n"); - code.write("2\n"); - code.write("static const char str??(??) = \"0123456789??/n\";\n"); - code.write("int\n"); - code.write("main(void)\n"); - code.write("??<\n"); - code.write(" unsigned char x = 5;\n"); - code.write(" if (sizeof str != TWELVE)\n"); - code.write(" abort ();\n"); - code.write( - " /* Test ^=, the only multi-character token to come from trigraphs. */\n"); - code.write(" x ??'= 3;\n"); - code.write(" if (x != 6)\n"); - code.write(" abort ();\n"); - code.write(" if ((5 ??! 3) != 7)\n"); - code.write(" abort ();\n"); - code.write(" return 0;\n"); - code.write("??>\n"); - } - catch (IOException ioe) - { - } - assertCodeFailsParse(code.toString()); - } public void testBug39553() throws Exception { parse("#define COMP_INC \"foobar.h\" \n" + "#include COMP_INC"); diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/PerformanceTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/PerformanceTests.java new file mode 100644 index 00000000000..e625ec00f2c --- /dev/null +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/PerformanceTests.java @@ -0,0 +1,45 @@ +/********************************************************************** + * Copyright (c) 2002,2003 Rational Software Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Rational Software - Initial API and implementation +***********************************************************************/ +package org.eclipse.cdt.core.parser.tests; + +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; + +import org.eclipse.cdt.core.parser.tests.BaseASTTest; + +/** + * @author vmozgin + * + */ +public class PerformanceTests extends BaseASTTest +{ + public PerformanceTests(String name) + { + super(name); + } + + public void testBug39523() throws Exception + { + Writer code = new StringWriter(); + try { + code.write("#define e0 \"a\"\n"); + code.write("#define e1 e0 e0 e0 e0 e0 e0 e0 e0 e0 e0\n"); + code.write("#define e2 e1 e1 e1 e1 e1 e1 e1 e1 e1 e1\n"); + code.write("#define e3 e2 e2 e2 e2 e2 e2 e2 e2 e2 e2\n"); + code.write("#define e4 e3 e3 e3 e3 e3 e3 e3 e3 e3 e3\n"); + code.write("#define e5 e4 e4 e4 e4 e4 e4 e4 e4 e4 e4\n"); + code.write("void foo() { (void)(e5); }\n"); + } catch( IOException ioe ){} + + parse(code.toString()); + } +} diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java index a5c6cdce1ca..5cd9dc0b0f3 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java @@ -1646,6 +1646,61 @@ public class QuickParseASTTests extends BaseASTTest assertEquals( parm.getName(), "" ); } + public void testBug39550() throws Exception + { + parse("double x = 0x1.fp1;").getDeclarations().next(); + } - + public void testBug39552A() throws Exception + { + Writer code = new StringWriter(); + + code.write("%:define glue(x, y) x %:%: y /* #define glue(x, y) x ## y. */\n"); + code.write("#ifndef glue\n"); + code.write("#error glue not defined!\n"); + code.write("#endif\n"); + + code.write("%:define str(x) %:x /* #define str(x) #x */\n"); + + code.write("int main (int argc, char *argv<::>) /* argv[] */\n"); + code.write("glue (<, %) /* { */\n"); + code.write(" /* di_str[] = */\n"); + code.write(" const char di_str glue(<, :)glue(:, >) = str(%:%:<::><%%>%:);\n"); + code.write(" /* Check the glue macro actually pastes, and that the spelling of\n"); + code.write(" all digraphs is preserved. */\n"); + code.write(" if (glue(str, cmp) (di_str, \"%:%:<::><%%>%:\"))\n"); + code.write(" err (\"Digraph spelling not preserved!\");\n"); + code.write(" return 0;\n"); + code.write("glue (%, >) /* } */\n"); + + parse(code.toString()); + } + + public void testBug39552B() throws Exception + { + Writer code = new StringWriter(); + + code.write("??=include \n"); + code.write("??=define TWELVE 1??/\n"); + code.write("2\n"); + + code.write("static const char str??(??) = \"0123456789??/n\";\n"); + + code.write("int\n"); + code.write("main(void)\n"); + code.write("??<\n"); + code.write(" unsigned char x = 5;\n"); + code.write(" if (sizeof str != TWELVE)\n"); + code.write(" abort ();\n"); + code.write(" /* Test ^=, the only multi-character token to come from trigraphs. */\n"); + code.write(" x ??'= 3;\n"); + code.write(" if (x != 6)\n"); + code.write(" abort ();\n"); + code.write(" if ((5 ??! 3) != 7)\n"); + code.write(" abort ();\n"); + code.write(" return 0;\n"); + code.write("??>\n"); + + parse(code.toString()); + } } \ No newline at end of file diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/TortureTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/TortureTest.java index 5d41adbd142..942cb1a8717 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/TortureTest.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/TortureTest.java @@ -211,6 +211,8 @@ public class TortureTest extends FractionalAutomatedTest { public void doFile() throws Throwable { + int timeOut = FractionalAutomatedTest.timeOut; + assertNotNull (fileList); File file = (File)fileList.removeFirst(); @@ -230,6 +232,12 @@ public class TortureTest extends FractionalAutomatedTest { String testCode = code.toString(); + if ( file.getName().equals("concat1.C")) { + // This is a really time-consuming test, + // override timeout + timeOut = 600000; + } + if (isExpectedToPass(testCode, file)) { ParseThread thread = new ParseThread(); diff --git a/core/org.eclipse.cdt.core/parser/ChangeLog b/core/org.eclipse.cdt.core/parser/ChangeLog index 682f439c734..fcc012b234c 100644 --- a/core/org.eclipse.cdt.core/parser/ChangeLog +++ b/core/org.eclipse.cdt.core/parser/ChangeLog @@ -29,6 +29,13 @@ Restructured AST class hierarchy. Removed the old IParserCallback return Objects from every Parser method. +2003-07-17 Victor Mozgin + Added support for digraphs and trigraphs. + Added support for hex floating point literals. + Fixed stack overflow problem with string literals concatenation. + Fixed problem with token pasting in macros. + This solves PR 39523, 39550, 39552. + 2003-07-15 Victor Mozgin Fixed PR 39349 : Scanner fails on long long literals. Fixed PR 39544 : Scanner fails on wide char literals. diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java index 1c08c23dd0b..ddcbc208505 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java @@ -370,6 +370,9 @@ public class Scanner implements IScanner { // these are scanner configuration aspects that we perhaps want to tweak // eventually, these should be configurable by the client, but for now // we can just leave it internal + private boolean enableDigraphReplacement = true; + private boolean enableTrigraphReplacement = true; + private boolean enableTrigraphReplacementInStrings = true; private boolean throwExceptionOnBadPreprocessorSyntax = true; private boolean throwExceptionOnInclusionNotFound = true; private boolean throwExceptionOnBadMacroExpansion = true; @@ -427,8 +430,69 @@ public class Scanner implements IScanner { } } } while (!done); - - if( ! insideString ) + + int baseOffset = lastContext.getOffset() - lastContext.undoStackSize() - 1; + + if (enableTrigraphReplacement && (!insideString || enableTrigraphReplacementInStrings)) { + // Trigraph processing + enableTrigraphReplacement = false; + if (c == '?') { + c = getChar(insideString); + if (c == '?') { + c = getChar(insideString); + switch (c) { + case '(': + expandDefinition("??(", "[", baseOffset); + c = getChar(insideString); + break; + case ')': + expandDefinition("??)", "]", baseOffset); + c = getChar(insideString); + break; + case '<': + expandDefinition("??<", "{", baseOffset); + c = getChar(insideString); + break; + case '>': + expandDefinition("??>", "}", baseOffset); + c = getChar(insideString); + break; + case '=': + expandDefinition("??=", "#", baseOffset); + c = getChar(insideString); + break; + case '/': + expandDefinition("??/", "\\", baseOffset); + c = getChar(insideString); + break; + case '\'': + expandDefinition("??\'", "^", baseOffset); + c = getChar(insideString); + break; + case '!': + expandDefinition("??!", "|", baseOffset); + c = getChar(insideString); + break; + case '-': + expandDefinition("??-", "~", baseOffset); + c = getChar(insideString); + break; + default: + // Not a trigraph + ungetChar(c); + ungetChar('?'); + c = '?'; + } + } else { + // Not a trigraph + ungetChar(c); + c = '?'; + } + } + enableTrigraphReplacement = true; + } + + if (!insideString) { if (c == '\\') { c = getChar(false); @@ -447,8 +511,48 @@ public class Scanner implements IScanner { ungetChar(c); c = '\\'; } + } else if (enableDigraphReplacement) { + enableDigraphReplacement = false; + // Digraph processing + if (c == '<') { + c = getChar(false); + if (c == '%') { + expandDefinition("<%", "{", baseOffset); + c = getChar(false); + } else if (c == ':') { + expandDefinition("<:", "[", baseOffset); + c = getChar(false); + } else { + // Not a digraph + ungetChar(c); + c = '<'; + } + } else if (c == ':') { + c = getChar(false); + if (c == '>') { + expandDefinition(":>", "]", baseOffset); + c = getChar(false); + } else { + // Not a digraph + ungetChar(c); + c = ':'; + } + } else if (c == '%') { + c = getChar(false); + if (c == '>') { + expandDefinition("%>", "}", baseOffset); + c = getChar(false); + } else if (c == ':') { + expandDefinition("%:", "#", baseOffset); + c = getChar(false); + } else { + // Not a digraph + ungetChar(c); + c = '%'; + } + } + enableDigraphReplacement = true; } - } return c; @@ -483,10 +587,14 @@ public class Scanner implements IScanner { public IToken nextToken() throws ScannerException, EndOfFile { - return nextToken( true ); + return nextToken( true, false ); } - public IToken nextToken( boolean pasting ) throws ScannerException, EndOfFile + public IToken nextToken(boolean pasting) throws ScannerException, EndOfFile { + return nextToken( pasting, false ); + } + + public IToken nextToken( boolean pasting, boolean lookingForNextAlready ) throws ScannerException, EndOfFile { if( cachedToken != null ){ setCurrentToken( cachedToken ); @@ -564,29 +672,34 @@ public class Scanner implements IScanner { if (c != NOCHAR ) { int type = wideLiteral ? IToken.tLSTRING : IToken.tSTRING; - + //If the next token is going to be a string as well, we need to concatenate //it with this token. IToken returnToken = newToken( type, buff.toString(), contextStack.getCurrentContext()); - IToken next = null; - try{ - next = nextToken( true ); - } catch( EndOfFile e ){ - next = null; - } - while( next != null && next.getType() == returnToken.getType() ){ - returnToken.setImage( returnToken.getImage() + next.getImage() ); - returnToken.setNext( null ); - currentToken = returnToken; - try{ - next = nextToken( true ); + if (!lookingForNextAlready) { + IToken next = null; + try{ + next = nextToken( true, true ); } catch( EndOfFile e ){ next = null; } + + while( next != null && next.getType() == returnToken.getType() ){ + returnToken.setImage( returnToken.getImage() + next.getImage() ); + returnToken.setNext( null ); + currentToken = returnToken; + try{ + next = nextToken( true, true ); + } catch( EndOfFile e ){ + next = null; + } + } + + cachedToken = next; + } - cachedToken = next; currentToken = returnToken; returnToken.setNext( null ); @@ -730,7 +843,9 @@ public class Scanner implements IScanner { floatingPoint = true; c= getChar(); - while ((c >= '0' && c <= '9') ) + while ((c >= '0' && c <= '9') + || (hex + && ((c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')))) { buff.append((char) c); c = getChar(); @@ -738,10 +853,10 @@ public class Scanner implements IScanner { } - if( c == 'e' || c == 'E' ) + if (c == 'e' || c == 'E' || (hex && (c == 'p' || c == 'P'))) { if( ! floatingPoint ) floatingPoint = true; - // exponent type for flaoting point + // exponent type for floating point buff.append((char)c); c = getChar(); @@ -2138,8 +2253,10 @@ public class Scanner implements IScanner { if( i != numberOfTokens - 1) { IToken t2 = (IToken) tokens.get(i+1); - if( t2.getType() == tPOUNDPOUND ) - pastingNext = true; + if( t2.getType() == tPOUNDPOUND ) { + pastingNext = true; + i++; + } } if( t.getType() != tPOUNDPOUND && ! pastingNext ) diff --git a/core/org.eclipse.cdt.ui/ChangeLog b/core/org.eclipse.cdt.ui/ChangeLog index a18de1c7222..0c1db86dc02 100644 --- a/core/org.eclipse.cdt.ui/ChangeLog +++ b/core/org.eclipse.cdt.ui/ChangeLog @@ -11,6 +11,9 @@ 2003-07-17 John Camelon Partially converted DOM to ISourceElementRequestor (requires refactoring of CModelBuilder & StuctureComparator modules in near future). +2003-07-17 Victor Mozgin + Added support for di- and trigraph notation of preprocessor directives. + 2003-07-16 Alain Magloire Patch from Alex chapiro. diff --git a/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/PreprocessorRule.java b/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/PreprocessorRule.java index f9417124646..39839c9ba6e 100644 --- a/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/PreprocessorRule.java +++ b/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/PreprocessorRule.java @@ -52,6 +52,7 @@ public class PreprocessorRule extends WordRule implements IRule { public IToken evaluate(ICharacterScanner scanner) { int c; int nCharsToRollback = 0; + boolean hashSignDetected = false; if (scanner.getColumn() > 0) return Token.UNDEFINED; @@ -60,8 +61,30 @@ public class PreprocessorRule extends WordRule implements IRule { c = scanner.read(); nCharsToRollback++; } while (Character.isWhitespace((char) c)); - + + + // Di- and trigraph support if (c == '#') { + hashSignDetected = true; + } else if (c == '%') { + c = scanner.read(); + nCharsToRollback++; + if (c == ':') { + hashSignDetected = true; + } + } else if (c == '?') { + c = scanner.read(); + nCharsToRollback++; + if (c == '?') { + c = scanner.read(); + nCharsToRollback++; + if (c == '=') { + hashSignDetected = true; + } + } + } + + if (hashSignDetected) { do { c = scanner.read();