From e9adc8cbbbdaf99f2b679bbe01db9867da668cfb Mon Sep 17 00:00:00 2001 From: John Camelon Date: Mon, 9 Jun 2003 18:40:17 +0000 Subject: [PATCH] Patch for Victor Mozgin. Fix for PR36701 (Preprocessor stringizing) --- core/org.eclipse.cdt.core/parser/ChangeLog | 3 + .../cdt/internal/core/parser/Scanner.java | 205 ++++++++++++++---- core/org.eclipse.cdt.ui.tests/ChangeLog | 5 + .../parser/failedTests/ScannerFailedTest.java | 64 ------ .../core/parser/tests/ScannerTestCase.java | 22 ++ .../core/suite/AutomatedIntegrationSuite.java | 1 - 6 files changed, 195 insertions(+), 105 deletions(-) delete mode 100644 core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java diff --git a/core/org.eclipse.cdt.core/parser/ChangeLog b/core/org.eclipse.cdt.core/parser/ChangeLog index 2a2b5cf8962..822d97ff014 100644 --- a/core/org.eclipse.cdt.core/parser/ChangeLog +++ b/core/org.eclipse.cdt.core/parser/ChangeLog @@ -2,6 +2,9 @@ Fixes for templated constructor/destructor/operator declarations. This fixed PR 36766, 36767, 36769 (STL parsing problems). +2003-06-09 Victor Mozgin + Fixed Bug 36701 - Scanner looses non-token chars while macro stringizing + 2003-06-06 Victor Mozgin Fixed Bug 38065 - Scanner skipped backslashes inside the code diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java index bb527d5eadd..f3d631dfe9f 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java @@ -1360,6 +1360,120 @@ public class Scanner implements IScanner { throw Parser.endOfFile; } + + protected static class endOfMacroTokenException extends Exception {}; + // the static instance we always use + protected static endOfMacroTokenException endOfMacroToken = new endOfMacroTokenException(); + + protected Token nextTokenForStringizing() throws ScannerException, Parser.EndOfFile + { + int c = getChar(); + StringBuffer tokenImage = new StringBuffer(); + + try { + while (c != NOCHAR) { + + if ((c == ' ') || (c == '\r') || (c == '\t') || (c == '\n')) { + + if (tokenImage.length() > 0) throw endOfMacroToken; + c = getChar(); + continue; + + } else if (c == '"') { + + if (tokenImage.length() > 0) throw endOfMacroToken; + + // string + StringBuffer buff = new StringBuffer(); + int beforePrevious = NOCHAR; + int previous = c; + c = getChar(true); + + for( ; ; ) + { + if ( c =='"' ) break; + if( c == NOCHAR) break; + buff.append((char) c); + beforePrevious = previous; + previous = c; + c = getChar(true); + } + + if (c != NOCHAR ) + { + return newToken( Token.tSTRING, buff.toString(), contextStack.getCurrentContext()); + + } else { + if (throwExceptionOnUnboundedString) + throw new ScannerException( + "Unbounded string" ); + } + + } else { + switch (c) { + case '\'' : + if (tokenImage.length() > 0) throw endOfMacroToken; + c = getChar( true ); + int next = getChar( true ); + if( c == '\\' ){ + c = next; + next = getChar( true ); + if( next == '\'' ) + return newToken( Token.tCHAR, '\\' + new Character( (char)c ).toString(), contextStack.getCurrentContext() ); + else if( throwExceptionOnBadCharacterRead ) + throw new ScannerException( "Invalid character '" + (char)c + "' read @ offset " + contextStack.getCurrentContext().getOffset() + " of file " + contextStack.getCurrentContext().getFilename() ); + } else if( next == '\'' ) + return newToken( Token.tCHAR, new Character( (char)c ).toString(), contextStack.getCurrentContext() ); + else + if( throwExceptionOnBadCharacterRead ) + throw new ScannerException( "Invalid character '" + (char)c + "' read @ offset " + contextStack.getCurrentContext().getOffset() + " of file " + contextStack.getCurrentContext().getFilename() ); + case ',' : + if (tokenImage.length() > 0) throw endOfMacroToken; + return newToken(Token.tCOMMA, ",", contextStack.getCurrentContext()); + case '(' : + if (tokenImage.length() > 0) throw endOfMacroToken; + return newToken(Token.tLPAREN, "(", contextStack.getCurrentContext()); + case ')' : + if (tokenImage.length() > 0) throw endOfMacroToken; + return newToken(Token.tRPAREN, ")", contextStack.getCurrentContext()); + case '/' : + if (tokenImage.length() > 0) throw endOfMacroToken; + c = getChar(); + switch (c) { + case '/' : + c = getChar(); + while (c != '\n' && c != NOCHAR) + c = getChar(); + continue; + case '*' : + skipOverMultilineComment(); + c = getChar(); + continue; + default: + tokenImage.append('/'); + continue; + } + default : + tokenImage.append((char)c); + c = getChar(); + } + } + } + } catch (endOfMacroTokenException e) { + // unget the first character after the end of token + ungetChar(c); + } + + // return completed token + if (tokenImage.length() > 0) { + return newToken(Token.tIDENTIFIER, tokenImage.toString(), contextStack.getCurrentContext()); + } + + // we're done + throw Parser.endOfFile; + } + + static { cppKeywords.put("and", new Integer(Token.t_and)); cppKeywords.put("and_eq", new Integer(Token.t_and_eq)); @@ -1839,7 +1953,49 @@ public class Scanner implements IScanner { callback.macro( key, offset, beginning, contextStack.getCurrentContext().getOffset() ); } } - + + protected Vector getMacroParameters (String params, boolean forStringizing) throws ScannerException { + + Scanner tokenizer = new Scanner(new StringReader(params), TEXT, definitions); + Vector parameterValues = new Vector(); + Token t = null; + String str = new String(); + boolean space = false; + int nParen = 0; + + try { + while (true) { + t = forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false); + if (t.type == Token.tLPAREN) { + nParen++; + } else if (t.type == Token.tRPAREN) { + nParen--; + } else if (t.type == Token.tCOMMA && nParen == 0) { + parameterValues.add(str); + str = ""; + space = false; + continue; + } + + if (space) + str += ' '; + + switch (t.type) { + case Token.tSTRING : str += '\"' + t.image + '\"'; break; + case Token.tLSTRING : str += "L\"" + t.image + '\"'; break; + case Token.tCHAR : str += '\'' + t.image + '\''; break; + default : str += t.image; break; + } + space = true; + } + } catch (Parser.EndOfFile e) { + // Good + parameterValues.add(str); + } + + return parameterValues; + } + protected void expandDefinition(String symbol, Object expansion) throws ScannerException { if (expansion instanceof String ) { @@ -1868,43 +2024,11 @@ public class Scanner implements IScanner { } String betweenTheBrackets = buffer.toString().trim(); - - Scanner tokenizer = new Scanner( new StringReader(betweenTheBrackets), TEXT, definitions ); - Vector parameterValues = new Vector(); - Token t = null; - String str = new String(); - boolean space = false; - int nParen = 0; - try{ - while (true) { - t = tokenizer.nextToken(false); - if( t.type == Token.tLPAREN ){ - nParen++; - } else if ( t.type == Token.tRPAREN ){ - nParen--; - } else if( t.type == Token.tCOMMA && nParen == 0 ){ - parameterValues.add( str ); - str = ""; - space = false; - continue; - } - - if( space ) - str += ' '; - - switch( t.type ) - { - case Token.tSTRING: str += '\"' + t.image + '\"'; break; - case Token.tLSTRING: str += "L\"" + t.image + '\"'; break; - case Token.tCHAR: str += '\'' + t.image + '\''; break; - default: str += t.image; break; - } - space = true; - } - } catch (Parser.EndOfFile e) { - // Good - parameterValues.add( str ); - } + + Vector parameterValues = getMacroParameters(betweenTheBrackets, false); + Vector parameterValuesForStringizing = getMacroParameters(betweenTheBrackets, true); + Token t = null; + // create a string that represents what needs to be tokenized buffer = new StringBuffer(); List tokens = macro.getTokenizedExpansion(); @@ -1944,7 +2068,7 @@ public class Scanner implements IScanner { throw new ScannerException( "Improper use of the # preprocessing token." ); } else { buffer.append('\"'); - String value = (String)parameterValues.elementAt(index); + String value = (String)parameterValuesForStringizing.elementAt(index); char val [] = value.toCharArray(); char ch; int length = value.length(); @@ -1987,7 +2111,8 @@ public class Scanner implements IScanner { } if( t.getType() != tPOUNDPOUND && ! pastingNext ) - buffer.append( " " ); + if (i < (numberOfTokens-1)) // Do not append to the last one + buffer.append( " " ); } String finalString = buffer.toString(); contextStack.updateContext( diff --git a/core/org.eclipse.cdt.ui.tests/ChangeLog b/core/org.eclipse.cdt.ui.tests/ChangeLog index 5796b9a9095..6a2131f7d6b 100644 --- a/core/org.eclipse.cdt.ui.tests/ChangeLog +++ b/core/org.eclipse.cdt.ui.tests/ChangeLog @@ -3,6 +3,11 @@ Renamed them to testBug36766and36769x(), as they cover both PRs. Added testBug36766and36769D() - test for templated destructor. +2003-06-09 Victor Mozgin + Moved testBug36701() from ScannerFailedTests.java to ScannerTestCase.java. + Renamed it to testBug36701A() and fixed it. + Added ScannerTestCase.testBug36701B(). + 2003-06-05 John Camelon Moved testBug23478A() & testBug23478B() from failed tests to TranslationUnitTests.java. Removed TranslationUnitFailedTests.java as it was empty. diff --git a/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java b/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java deleted file mode 100644 index 71c37d96941..00000000000 --- a/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java +++ /dev/null @@ -1,64 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2001 IBM Corporation and others. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Common Public License v0.5 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/cpl-v05.html - * - * Contributors: - * IBM Corp. - Rational Software - initial implementation - ******************************************************************************/ - -package org.eclipse.cdt.core.parser.failedTests; - -import java.io.StringWriter; - -import junit.framework.AssertionFailedError; -import junit.framework.Test; -import junit.framework.TestSuite; - -import org.eclipse.cdt.core.parser.tests.BaseScannerTest; - -/** - * @author aniefer - * - * To change the template for this generated type comment go to - * Window>Preferences>Java>Code Generation>Code and Comments - */ -public class ScannerFailedTest extends BaseScannerTest { - - public ScannerFailedTest(String name){ - super(name); - } - - public static Test suite() - { - TestSuite suite = new TestSuite(); - - suite.addTest( new ScannerFailedTest( "testBug36701" ) ); - - return suite; - } - - public void testBug36701() throws Exception - { - boolean testPassed = false; - try{ - StringWriter writer = new StringWriter(); - writer.write( "#define str(s) # s\n" ); - writer.write( "str( @ \n )\n"); - - initializeScanner( writer.toString() ); - validateString( "@ \\\\n" ); - validateEOF(); - - testPassed = true; - } catch( AssertionFailedError e ){ - //expected failure - } - - if( testPassed ) - fail( "The expected error did not occur." ); - } - -} diff --git a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java index 75ec1cf7b4a..f6f04e10c58 100644 --- a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java +++ b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java @@ -1378,4 +1378,26 @@ public class ScannerTestCase extends BaseScannerTest assertTrue(se.getMessage().equals("Invalid character '\\' read @ offset 5 of file TEXT")); } } + + public void testBug36701A() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write("#define str(s) # s\n"); + writer.write("str( @ \\n )\n"); + + initializeScanner(writer.toString()); + validateString("@ \\\\n"); + validateEOF(); + } + + public void testBug36701B() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write("#define str(s) # s\n"); + writer.write("str( @ /*ff*/ \\n hh \"aa\" )\n"); + + initializeScanner(writer.toString()); + validateString("@ \\\\n hh \\\"aa\\\""); + validateEOF(); + } } diff --git a/core/org.eclipse.cdt.ui.tests/suite/org/eclipse/cdt/core/suite/AutomatedIntegrationSuite.java b/core/org.eclipse.cdt.ui.tests/suite/org/eclipse/cdt/core/suite/AutomatedIntegrationSuite.java index ffdfd540b36..398b4956daa 100644 --- a/core/org.eclipse.cdt.ui.tests/suite/org/eclipse/cdt/core/suite/AutomatedIntegrationSuite.java +++ b/core/org.eclipse.cdt.ui.tests/suite/org/eclipse/cdt/core/suite/AutomatedIntegrationSuite.java @@ -80,7 +80,6 @@ public class AutomatedIntegrationSuite extends TestSuite implements TestListener suite.addTestSuite(ACEFailedTest.class); suite.addTestSuite(DOMFailedTest.class); suite.addTestSuite(LokiFailures.class); - suite.addTestSuite(ScannerFailedTest.class); suite.addTestSuite(STLFailedTests.class); suite.addTestSuite(CModelElementsFailedTests.class);