From bff475c9681bb23b0bcc70290bda76b0a5af6ef3 Mon Sep 17 00:00:00 2001 From: Doug Schaefer Date: Mon, 21 Apr 2003 18:04:53 +0000 Subject: [PATCH] Patch for Andrew Niefer: Fixed Bug36475 - Scanner does not concatenate strings Fixed Bug36509 - Scanner turns strings into identifiers when expanding macros Fixed Bug36521 - Scanner gets confused over commas in function like macros Fixed Bug36695 - Scanner looses escaping on chars (ie '\4' to '4') Moved ScannerFailedTest::testBug36475 to ScannerTestCase::testBug36475() Moved ScannerFailedTest::testBug36509 to ScannerTestCase::testBug36509() Moved ScannerFailedTest::testBug36521 to ScannerTestCase::testBug36521() Added ScannerTestCase::testBug36695() Updated ScannerTestCase::testBug36047 Updated ScannerTestCase::testBug36045 --- core/org.eclipse.cdt.core/parser/ChangeLog | 6 + .../cdt/internal/core/parser/Scanner.java | 82 +++++++++++--- .../cdt/internal/core/parser/Token.java | 4 + core/org.eclipse.cdt.ui.tests/ChangeLog | 8 ++ .../parser/failedTests/ScannerFailedTest.java | 104 +++--------------- .../core/parser/tests/ScannerTestCase.java | 82 ++++++++++++-- 6 files changed, 169 insertions(+), 117 deletions(-) diff --git a/core/org.eclipse.cdt.core/parser/ChangeLog b/core/org.eclipse.cdt.core/parser/ChangeLog index e25244f9923..4be9219e7c5 100644 --- a/core/org.eclipse.cdt.core/parser/ChangeLog +++ b/core/org.eclipse.cdt.core/parser/ChangeLog @@ -1,3 +1,9 @@ +2003-04-21 Andrew Niefer + Fixed Bug36475 - Scanner does not concatenate strings + Fixed Bug36509 - Scanner turns strings into identifiers when expanding macros + Fixed Bug36521 - Scanner gets confused over commas in function like macros + Fixed Bug36695 - Scanner looses escaping on chars (ie '\4' to '4') + 2003-04-17 John Camelon Fixed error in Elaborated Enumeration Types. Fixed Bug36559 - Parsing Templates... diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java index 47f89912a1e..6b0ba953181 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java @@ -300,6 +300,7 @@ public class Scanner implements IScanner { private static HashMap ppDirectives = new HashMap(); private Token currentToken = null; + private Token cachedToken = null; private boolean passOnToClient = true; private BranchTracker branches = new BranchTracker(); @@ -436,7 +437,12 @@ public class Scanner implements IScanner { protected Token nextToken( boolean pasting ) throws ScannerException, Parser.EndOfFile { - + if( cachedToken != null ){ + setCurrentToken( cachedToken ); + cachedToken = null; + return currentToken; + } + count++; boolean madeMistake = false; int c = getChar(); @@ -510,10 +516,33 @@ public class Scanner implements IScanner { if (c != NOCHAR ) { int type = wideString ? Token.tLSTRING : Token.tSTRING; - return newToken( - type, - buff.toString(), - contextStack.getCurrentContext()); + + //If the next token is going to be a string as well, we need to concatenate + //it with this token. + Token returnToken = newToken( type, buff.toString(), contextStack.getCurrentContext()); + Token next = null; + try{ + next = nextToken( true ); + } catch( Parser.EndOfFile e ){ + next = null; + } + + while( next != null && next.type == returnToken.type ){ + returnToken.image += next.image; + returnToken.setNext( null ); + currentToken = returnToken; + try{ + next = nextToken( true ); + } catch( Parser.EndOfFile e ){ + next = null; + } + } + + cachedToken = next; + currentToken = returnToken; + returnToken.setNext( null ); + + return returnToken; } else { if (throwExceptionOnUnboundedString) @@ -916,9 +945,16 @@ public class Scanner implements IScanner { } else { switch (c) { case '\'' : - c = getChar(); - int next = getChar(); - if( next == '\'' ) + c = getChar( true ); + int next = getChar( true ); + if( c == '\\' ){ + c = next; + next = getChar( true ); + if( next == '\'' ) + return newToken( Token.tCHAR, '\\' + new Character( (char)c ).toString(), contextStack.getCurrentContext() ); + else if( throwExceptionOnBadCharacterRead ) + throw new ScannerException( "Invalid character '" + (char)c + "' read @ offset " + contextStack.getCurrentContext().getOffset() + " of file " + contextStack.getCurrentContext().getFilename() ); + } else if( next == '\'' ) return newToken( Token.tCHAR, new Character( (char)c ).toString(), contextStack.getCurrentContext() ); else if( throwExceptionOnBadCharacterRead ) @@ -1714,7 +1750,7 @@ public class Scanner implements IScanner { if (bracketCount == 0) break; buffer.append((char) c); - c = getChar(); + c = getChar( true ); } String betweenTheBrackets = buffer.toString().trim(); @@ -1724,11 +1760,15 @@ public class Scanner implements IScanner { Token t = null; String str = new String(); boolean space = false; + int nParen = 0; try{ while (true) { t = tokenizer.nextToken(false); - if( t.type == Token.tCOMMA ) - { + if( t.type == Token.tLPAREN ){ + nParen++; + } else if ( t.type == Token.tRPAREN ){ + nParen--; + } else if( t.type == Token.tCOMMA && nParen == 0 ){ parameterValues.add( str ); str = ""; space = false; @@ -1738,11 +1778,13 @@ public class Scanner implements IScanner { if( space ) str += ' '; - if( t.type == Token.tSTRING ) - str += '\"' + t.image + '\"'; - else - str += t.image; - + switch( t.type ) + { + case Token.tSTRING: str += '\"' + t.image + '\"'; break; + case Token.tLSTRING: str += "L\"" + t.image + '\"'; break; + case Token.tCHAR: str += '\'' + t.image + '\''; break; + default: str += t.image; break; + } space = true; } } catch (Parser.EndOfFile e) { @@ -1812,7 +1854,13 @@ public class Scanner implements IScanner { buffer.append('\"'); } } else { - buffer.append(t.image); + switch( t.type ) + { + case Token.tSTRING: buffer.append('\"' + t.image + '\"'); break; + case Token.tLSTRING: buffer.append("L\"" + t.image + '\"'); break; + case Token.tCHAR: buffer.append('\'' + t.image + '\''); break; + default: buffer.append(t.image); break; + } } boolean pastingNext = false; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java index 79fc894e7d6..39a14a984b9 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java @@ -17,6 +17,10 @@ public class Token { image = i; filename = context.getFilename(); offset = context.getOffset() - image.length() - context.undoStackSize(); + + if( type == tLSTRING || type == tSTRING || type == tCHAR ){ + offset--; + } } public Token(int t, String i) { diff --git a/core/org.eclipse.cdt.ui.tests/ChangeLog b/core/org.eclipse.cdt.ui.tests/ChangeLog index 641238c3aef..f24a90b9883 100644 --- a/core/org.eclipse.cdt.ui.tests/ChangeLog +++ b/core/org.eclipse.cdt.ui.tests/ChangeLog @@ -1,3 +1,11 @@ +2003-04-17 Andrew Niefer + Added ScannerTestCase::testBug36695() + Moved ScannerFailedTest::testBug36521 to ScannerTestCase::testBug36521() + Moved ScannerFailedTest::testBug36509 to ScannerTestCase::testBug36509() + Moved ScannerFailedTest::testBug36475 to ScannerTestCase::testBug36475() + Updated ScannerTestCase::testBug36047 + Updated ScannerTestCase::testBug36045 + 2003-04-17 John Camelon Updated DOMTests::testBug36600(). Updated LineNumberTest::testDOMLineNos(). diff --git a/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java b/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java index 3d2e1519895..844a4baf43b 100644 --- a/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java +++ b/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java @@ -36,105 +36,29 @@ public class ScannerFailedTest extends ScannerTestCase { public static Test suite() { TestSuite suite = new TestSuite(); - - suite.addTest( new ScannerFailedTest( "testBug36475" ) ); - suite.addTest( new ScannerFailedTest( "testBug36509" ) ); - suite.addTest( new ScannerFailedTest( "testBug36521" ) ); - + + suite.addTest( new ScannerFailedTest( "testBug36701" ) ); + return suite; } - public void testBug36475() throws Exception + public void testBug36701() throws Exception { boolean testPassed = false; - try - { - StringWriter writer = new StringWriter(); - writer.write( " \"A\" \"B\" \"C\" " ); + try{ + StringWriter writer = new StringWriter(); + writer.write( "#define str(s) # s\n" ); + writer.write( "str( @ \n )\n"); initializeScanner( writer.toString() ); - - validateString( "ABC" ); - validateEOF(); - + validateString( "@ \\\\n" ); + validateEOF(); + testPassed = true; + } catch( AssertionFailedError e ){ + //expected failure + } - } - catch( Throwable e ) - { - if( !(e instanceof AssertionFailedError) ){ - fail( "Unexpected Error: " + e.getMessage() ); - } - } - - if( testPassed ) - fail( "The expected error did not occur." ); - } - public void testBug36509() throws Exception - { - boolean testPassed = false; - - try{ - StringWriter writer = new StringWriter(); - writer.write("#define debug(s, t) printf(\"x\" # s \"= %d, x\" # t \"= %s\", \\\n"); - writer.write(" x ## s, x ## t) \n"); - - initializeScanner( writer.toString() ); - //printf("x" "1" "=%d, x" "2" "= %s", x1, x2); - validateIdentifier( "printf" ); - validateToken( Token.tLPAREN ); - validateString("x"); - validateString("1"); - validateString("= %d, x"); - validateString("2"); - validateString("= %s"); - validateToken(Token.tCOMMA); - validateIdentifier("x1"); - validateToken(Token.tCOMMA); - validateIdentifier("x2"); - validateToken(Token.tRPAREN); - validateToken(Token.tSEMI); - - testPassed = true; - } - catch( Throwable e ) - { - if( !(e instanceof AssertionFailedError) ){ - fail( "Unexpected Error: " + e.getMessage() ); - } - } - - if( testPassed ) - fail( "The expected error did not occur." ); - } - public void testBug36521() throws Exception - { - boolean testPassed = false; - - try{ - StringWriter writer = new StringWriter(); - writer.write("#define str(s) # s\n"); - writer.write("fputs(str(strncmp(\"abc\\0d\", \"abc\", \'\\4\')\n"); - writer.write(" == 0) str(: @\\n), s);\n"); - - initializeScanner( writer.toString() ); - validateIdentifier("fputs"); - validateToken(Token.tLPAREN); - validateString("strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') == 0"); - validateString(": @\\n"); - validateToken(Token.tCOMMA); - validateIdentifier("s"); - validateToken(Token.tRPAREN); - validateToken(Token.tSEMI); - - testPassed = true; - } - catch( ScannerException e ) - { - if( !e.getMessage().equals( "Improper use of macro str" ) ) - fail( "Unexpected Error: " + e.getMessage() ); - } - if( testPassed ) fail( "The expected error did not occur." ); } diff --git a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java index 855d02079e8..517d7022bd0 100644 --- a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java +++ b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java @@ -1072,6 +1072,16 @@ public class ScannerTestCase extends TestCase assertTrue(false); } } + public void validateChar( String expected ) throws ScannerException + { + try { + Token t= scanner.nextToken(); + assertTrue(t.getType() == Token.tCHAR ); + assertEquals( t.getImage(), expected ); + } catch (Parser.EndOfFile e) { + assertTrue(false); + } + } public void validateString( String expectedImage ) throws ScannerException { @@ -1219,8 +1229,7 @@ public class ScannerTestCase extends TestCase buffer.append( '"'); buffer.append( "\n\n"); initializeScanner( buffer.toString()); - validateString( "\\\""); - validateString( "\\\\"); + validateString( "\\\"\\\\"); } public void testConditionalWithBraces() @@ -1343,15 +1352,68 @@ public class ScannerTestCase extends TestCase writer.write( "MAD_VERSION\n" ); initializeScanner( writer.toString() ); - validateString( "2" ); - validateString( "." ); - validateString( "1" ); - validateString( "." ); - validateString( "3" ); - validateString( "." ); - validateString( "boo" ); + validateString( "2.1.3.boo" ); validateEOF(); } - + + public void testBug36475() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write( " \"A\" \"B\" \"C\" " ); + + initializeScanner( writer.toString() ); + + validateString( "ABC" ); + validateEOF(); + } + + public void testBug36509() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write("#define debug(s, t) printf(\"x\" # s \"= %d, x\" # t \"= %s\", \\\n"); + writer.write(" x ## s, x ## t) \n"); + writer.write("debug(1, 2);"); + + initializeScanner( writer.toString() ); + //printf("x1=%d, x2= %s", x1, x2); + validateIdentifier( "printf" ); + validateToken( Token.tLPAREN ); + validateString("x1= %d, x2= %s"); + validateToken(Token.tCOMMA); + validateIdentifier("x1"); + validateToken(Token.tCOMMA); + validateIdentifier("x2"); + validateToken(Token.tRPAREN); + validateToken(Token.tSEMI); + validateEOF(); + } + + public void testBug36695() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write("\'\\4\' \'\\n\'"); + initializeScanner( writer.toString() ); + + validateChar( "\\4" ); + validateChar( "\\n" ); + validateEOF(); + } + + public void testBug36521() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write("#define str(s) # s\n"); + writer.write("fputs(str(strncmp(\"abc\\0d\", \"abc\", \'\\4\')\n"); + writer.write(" == 0), s);\n"); + + initializeScanner( writer.toString() ); + validateIdentifier("fputs"); + validateToken(Token.tLPAREN); + validateString("strncmp ( \\\"abc\\\\0d\\\" , \\\"abc\\\" , '\\\\4' ) == 0"); + validateToken(Token.tCOMMA); + validateIdentifier("s"); + validateToken(Token.tRPAREN); + validateToken(Token.tSEMI); + } }