diff --git a/core/org.eclipse.cdt.core/parser/ChangeLog b/core/org.eclipse.cdt.core/parser/ChangeLog index 7b24739eae1..804314b57fd 100644 --- a/core/org.eclipse.cdt.core/parser/ChangeLog +++ b/core/org.eclipse.cdt.core/parser/ChangeLog @@ -1,3 +1,6 @@ +2003-04-15 Andrew Niefer + Added scanner support to fix Bug36047 + 2003-04-13 John Camelon Minor cleanup of callbacks due to removal of NewModelBuilder. Added parser support to partially fix bug36416 & bug36294. Also added minimal C-Model support for these fixes. diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/IScanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/IScanner.java index 35609664422..f3f089b3b3c 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/IScanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/IScanner.java @@ -15,6 +15,7 @@ import java.util.List; public interface IScanner { public static final int tPOUNDPOUND = -6; + public static final int tPOUND = -7; public IScanner initialize( Reader sourceToBeRead, String fileName ); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java index e6a6c88ed61..1cd0b62bc23 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java @@ -315,6 +315,11 @@ public class Scanner implements IScanner { private boolean throwExceptionOnBadCharacterRead = false; private boolean atEOF = false; + private boolean tokenizingMacroReplacementList = false; + public void setTokenizingMacroReplacementList( boolean mr ){ + tokenizingMacroReplacementList = mr; + } + private boolean quickScan = false; public void setQuickScan(boolean qs) { quickScan = qs; @@ -714,7 +719,10 @@ public class Scanner implements IScanner { throw new ScannerException(BAD_PP + contextStack.getCurrentContext().getOffset()); else return newToken( tPOUNDPOUND, "##" ); - } + } else if( tokenizingMacroReplacementList ) { + ungetChar( c ); + return newToken( tPOUND, "#" ); + } while (((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')) || (c == '_') ) { @@ -730,8 +738,6 @@ public class Scanner implements IScanner { if (throwExceptionOnBadPreprocessorSyntax) throw new ScannerException( BAD_PP + contextStack.getCurrentContext().getOffset()); - - } else { int type = ((Integer) directive).intValue(); switch (type) { @@ -1539,10 +1545,26 @@ public class Scanner implements IScanner { helperScanner.initialize( new StringReader(replacementString), null); + helperScanner.setTokenizingMacroReplacementList( true ); Token t = helperScanner.nextToken(false); try { while (true) { + //each # preprocessing token in the replacement list shall be followed + //by a parameter as the next reprocessing token in the list + if( t.type == tPOUND ){ + macroReplacementTokens.add( t ); + t = helperScanner.nextToken(false); + int index = parameterIdentifiers.indexOf(t.image); + if (index == -1 ) { + //not found + if (throwExceptionOnBadPreprocessorSyntax) + throw new ScannerException( + BAD_PP + contextStack.getCurrentContext().getOffset()); + return; + } + } + macroReplacementTokens.add(t); t = helperScanner.nextToken(false); } @@ -1627,14 +1649,39 @@ public class Scanner implements IScanner { buffer.append((char) c); c = getChar(); } + String betweenTheBrackets = buffer.toString().trim(); - StringTokenizer tokenizer = - new StringTokenizer(betweenTheBrackets, ","); - Vector parameterValues = new Vector(tokenizer.countTokens()); - while (tokenizer.hasMoreTokens()) { - parameterValues.add(tokenizer.nextToken().trim()); - } + Scanner tokenizer = new Scanner( new StringReader(betweenTheBrackets), TEXT, definitions ); + Vector parameterValues = new Vector(); + Token t = null; + String str = new String(); + boolean space = false; + try{ + while (true) { + t = tokenizer.nextToken(false); + if( t.type == Token.tCOMMA ) + { + parameterValues.add( str ); + str = ""; + space = false; + continue; + } + + if( space ) + str += ' '; + + if( t.type == Token.tSTRING ) + str += '\"' + t.image + '\"'; + else + str += t.image; + + space = true; + } + } catch (Parser.EndOfFile e) { + // Good + parameterValues.add( str ); + } // create a string that represents what needs to be tokenized buffer = new StringBuffer(); List tokens = macro.getTokenizedExpansion(); @@ -1649,7 +1696,7 @@ public class Scanner implements IScanner { int numberOfTokens = tokens.size(); for (int i = 0; i < numberOfTokens; ++i) { - Token t = (Token) tokens.get(i); + t = (Token) tokens.get(i); if (t.type == Token.tIDENTIFIER) { String identifierName = t.image; @@ -1664,6 +1711,39 @@ public class Scanner implements IScanner { buffer.append( (String) parameterValues.elementAt(index) ); } + } else if (t.type == tPOUND) { + //next token should be a parameter which needs to be turned into + //a string literal + t = (Token) tokens.get( ++i ); + int index = parameterNames.indexOf(t.image); + if( index == -1 ){ + if (throwExceptionOnBadMacroExpansion) + throw new ScannerException( "Improper use of the # preprocessing token." ); + } else { + buffer.append('\"'); + String value = (String)parameterValues.elementAt(index); + char val [] = value.toCharArray(); + char ch; + int length = value.length(); + for( int j = 0; j < length; j++ ){ + ch = val[j]; + if( ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' ){ + //Each occurance of whitespace becomes a single space character + while( ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' ){ + ch = val[++j]; + } + buffer.append(' '); + } + //a \ character is inserted before each " and \ + if( ch == '\"' || ch == '\\' ){ + buffer.append('\\'); + buffer.append(ch); + } else { + buffer.append(ch); + } + } + buffer.append('\"'); + } } else { buffer.append(t.image); } diff --git a/core/org.eclipse.cdt.ui.tests/ChangeLog b/core/org.eclipse.cdt.ui.tests/ChangeLog index 594537ecfa1..e29642b5acb 100644 --- a/core/org.eclipse.cdt.ui.tests/ChangeLog +++ b/core/org.eclipse.cdt.ui.tests/ChangeLog @@ -1,3 +1,7 @@ +2003-04-15 Andrew Niefer + Moved ScannerFailedTest::testBug36047 to ScannerTestCase::testBug36047 + Added ScannerFailedTest::testBug36475 + 2003-04-13 John Camelon Added DOMTests::testPointersToFunctions. diff --git a/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java b/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java index 535015bb09f..2ea206c07d9 100644 --- a/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java +++ b/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java @@ -37,30 +37,31 @@ public class ScannerFailedTest extends ScannerTestCase { { TestSuite suite = new TestSuite(); - suite.addTest( new ScannerFailedTest( "testBug36047" ) ); + suite.addTest( new ScannerFailedTest( "testBug36475" ) ); return suite; } - public void testBug36047() throws Exception + public void testBug36475() throws Exception { boolean testPassed = false; try { StringWriter writer = new StringWriter(); - writer.write( "# define MAD_VERSION_STRINGIZE(str) #str\n" ); - writer.write( "# define MAD_VERSION_STRING(num) MAD_VERSION_STRINGIZE(num)\n" ); - writer.write( "# define MAD_VERSION MAD_VERSION_STRING(MAD_VERSION_MAJOR) \".\"\n" ); - initializeScanner( writer.toString() ); + writer.write( " \"A\" \"B\" \"C\" " ); + + initializeScanner( writer.toString() ); + + validateString( "ABC" ); validateEOF(); testPassed = true; } - catch( ScannerException se ) + catch( Throwable e ) { - if( !se.getMessage().equals( "Invalid preprocessor directive encountered at offset 5" ) ){ - fail( "Unexpected Error: " + se.getMessage() ); + if( !(e instanceof AssertionFailedError) ){ + fail( "Unexpected Error: " + e.getMessage() ); } } diff --git a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java index d48129ecf5f..69f45694781 100644 --- a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java +++ b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java @@ -1,6 +1,7 @@ package org.eclipse.cdt.core.parser.tests; import java.io.StringReader; +import java.io.StringWriter; import java.util.List; import junit.framework.Test; @@ -1302,4 +1303,31 @@ public class ScannerTestCase extends TestCase validateIdentifier("A"); validateEOF(); } + + public void testBug36047() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write( "# define MAD_VERSION_STRINGIZE(str) #str\n" ); + writer.write( "# define MAD_VERSION_STRING(num) MAD_VERSION_STRINGIZE(num)\n" ); + writer.write( "# define MAD_VERSION MAD_VERSION_STRING(MAD_VERSION_MAJOR) \".\" \\\n" ); + writer.write( " MAD_VERSION_STRING(MAD_VERSION_MINOR) \".\" \\\n" ); + writer.write( " MAD_VERSION_STRING(MAD_VERSION_PATCH) \".\" \\\n" ); + writer.write( " MAD_VERSION_STRING(MAD_VERSION_EXTRA)\n" ); + writer.write( "# define MAD_VERSION_MAJOR 2\n" ); + writer.write( "# define MAD_VERSION_MINOR 1\n" ); + writer.write( "# define MAD_VERSION_PATCH 3\n" ); + writer.write( "# define MAD_VERSION_EXTRA boo\n" ); + writer.write( "MAD_VERSION\n" ); + initializeScanner( writer.toString() ); + + validateString( "2" ); + validateString( "." ); + validateString( "1" ); + validateString( "." ); + validateString( "3" ); + validateString( "." ); + validateString( "boo" ); + + validateEOF(); + } }