mirrors/cdt
1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-04-29 19:45:01 +02:00
Code Issues Wiki Activity

Patch for Victor Mozgin.

Browse source 
Fix for PR36701 (Preprocessor stringizing)
This commit is contained in:
John Camelon 2003-06-09 18:40:17 +00:00
parent de2f7c52bc
commit e9adc8cbbb
6 changed files with 195 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
core/org.eclipse.cdt.core/parser/ChangeLog
View file

@ -2,6 +2,9 @@
Fixes for templated constructor/destructor/operator declarations.
This fixed PR 36766, 36767, 36769 (STL parsing problems).
2003-06-09 Victor Mozgin
Fixed Bug 36701 - Scanner looses non-token chars while macro stringizing
2003-06-06 Victor Mozgin
Fixed Bug 38065 - Scanner skipped backslashes inside the code

195
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java
View file

@ -1360,6 +1360,120 @@ public class Scanner implements IScanner {
throw Parser.endOfFile;
}
protected static class endOfMacroTokenException extends Exception {};
// the static instance we always use
protected static endOfMacroTokenException endOfMacroToken = new endOfMacroTokenException();
protected Token nextTokenForStringizing() throws ScannerException, Parser.EndOfFile
{
int c = getChar();
StringBuffer tokenImage = new StringBuffer();
try {
while (c != NOCHAR) {
if ((c == ' ') || (c == '\r') || (c == '\t') || (c == '\n')) {
if (tokenImage.length() > 0) throw endOfMacroToken;
c = getChar();
continue;
} else if (c == '"') {
if (tokenImage.length() > 0) throw endOfMacroToken;
// string
StringBuffer buff = new StringBuffer();
int beforePrevious = NOCHAR;
int previous = c;
c = getChar(true);
for( ; ; )
{
if ( c =='"' ) break;
if( c == NOCHAR) break;
buff.append((char) c);
beforePrevious = previous;
previous = c;
c = getChar(true);
}
if (c != NOCHAR )
{
return newToken( Token.tSTRING, buff.toString(), contextStack.getCurrentContext());
} else {
if (throwExceptionOnUnboundedString)
throw new ScannerException(
"Unbounded string" );
}
} else {
switch (c) {
case '\'' :
if (tokenImage.length() > 0) throw endOfMacroToken;
c = getChar( true );
int next = getChar( true );
if( c == '\\' ){
c = next;
next = getChar( true );
if( next == '\'' )
return newToken( Token.tCHAR, '\\' + new Character( (char)c ).toString(), contextStack.getCurrentContext() );
else if( throwExceptionOnBadCharacterRead )
throw new ScannerException( "Invalid character '" + (char)c + "' read @ offset " + contextStack.getCurrentContext().getOffset() + " of file " + contextStack.getCurrentContext().getFilename() );
} else if( next == '\'' )
return newToken( Token.tCHAR, new Character( (char)c ).toString(), contextStack.getCurrentContext() );
else
if( throwExceptionOnBadCharacterRead )
throw new ScannerException( "Invalid character '" + (char)c + "' read @ offset " + contextStack.getCurrentContext().getOffset() + " of file " + contextStack.getCurrentContext().getFilename() );
case ',' :
if (tokenImage.length() > 0) throw endOfMacroToken;
return newToken(Token.tCOMMA, ",", contextStack.getCurrentContext());
case '(' :
if (tokenImage.length() > 0) throw endOfMacroToken;
return newToken(Token.tLPAREN, "(", contextStack.getCurrentContext());
case ')' :
if (tokenImage.length() > 0) throw endOfMacroToken;
return newToken(Token.tRPAREN, ")", contextStack.getCurrentContext());
case '/' :
if (tokenImage.length() > 0) throw endOfMacroToken;
c = getChar();
switch (c) {
case '/' :
c = getChar();
while (c != '\n' && c != NOCHAR)
c = getChar();
continue;
case '*' :
skipOverMultilineComment();
c = getChar();
continue;
default:
tokenImage.append('/');
continue;
}
default :
tokenImage.append((char)c);
c = getChar();
}
}
}
} catch (endOfMacroTokenException e) {
// unget the first character after the end of token
ungetChar(c);
}
// return completed token
if (tokenImage.length() > 0) {
return newToken(Token.tIDENTIFIER, tokenImage.toString(), contextStack.getCurrentContext());
}
// we're done
throw Parser.endOfFile;
}
static {
cppKeywords.put("and", new Integer(Token.t_and));
cppKeywords.put("and_eq", new Integer(Token.t_and_eq));
@ -1840,6 +1954,48 @@ public class Scanner implements IScanner {
}
}
protected Vector getMacroParameters (String params, boolean forStringizing) throws ScannerException {
Scanner tokenizer = new Scanner(new StringReader(params), TEXT, definitions);
Vector parameterValues = new Vector();
Token t = null;
String str = new String();
boolean space = false;
int nParen = 0;
try {
while (true) {
t = forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false);
if (t.type == Token.tLPAREN) {
nParen++;
} else if (t.type == Token.tRPAREN) {
nParen--;
} else if (t.type == Token.tCOMMA && nParen == 0) {
parameterValues.add(str);
str = "";
space = false;
continue;
}
if (space)
str += ' ';
switch (t.type) {
case Token.tSTRING : str += '\"' + t.image + '\"'; break;
case Token.tLSTRING : str += "L\"" + t.image + '\"'; break;
case Token.tCHAR : str += '\'' + t.image + '\''; break;
default : str += t.image; break;
}
space = true;
}
} catch (Parser.EndOfFile e) {
// Good
parameterValues.add(str);
}
return parameterValues;
}
protected void expandDefinition(String symbol, Object expansion)
throws ScannerException {
if (expansion instanceof String ) {
@ -1869,42 +2025,10 @@ public class Scanner implements IScanner {
String betweenTheBrackets = buffer.toString().trim();
Scanner tokenizer = new Scanner( new StringReader(betweenTheBrackets), TEXT, definitions );
Vector parameterValues = new Vector();
Vector parameterValues = getMacroParameters(betweenTheBrackets, false);
Vector parameterValuesForStringizing = getMacroParameters(betweenTheBrackets, true);
Token t = null;
String str = new String();
boolean space = false;
int nParen = 0;
try{
while (true) {
t = tokenizer.nextToken(false);
if( t.type == Token.tLPAREN ){
nParen++;
} else if ( t.type == Token.tRPAREN ){
nParen--;
} else if( t.type == Token.tCOMMA && nParen == 0 ){
parameterValues.add( str );
str = "";
space = false;
continue;
}
if( space )
str += ' ';
switch( t.type )
{
case Token.tSTRING: str += '\"' + t.image + '\"'; break;
case Token.tLSTRING: str += "L\"" + t.image + '\"'; break;
case Token.tCHAR: str += '\'' + t.image + '\''; break;
default: str += t.image; break;
}
space = true;
}
} catch (Parser.EndOfFile e) {
// Good
parameterValues.add( str );
}
// create a string that represents what needs to be tokenized
buffer = new StringBuffer();
List tokens = macro.getTokenizedExpansion();
@ -1944,7 +2068,7 @@ public class Scanner implements IScanner {
throw new ScannerException( "Improper use of the # preprocessing token." );
} else {
buffer.append('\"');
String value = (String)parameterValues.elementAt(index);
String value = (String)parameterValuesForStringizing.elementAt(index);
char val [] = value.toCharArray();
char ch;
int length = value.length();
@ -1987,6 +2111,7 @@ public class Scanner implements IScanner {
}
if( t.getType() != tPOUNDPOUND && ! pastingNext )
if (i < (numberOfTokens-1)) // Do not append to the last one
buffer.append( " " );
}
String finalString = buffer.toString();

5
core/org.eclipse.cdt.ui.tests/ChangeLog
View file

@ -3,6 +3,11 @@
Renamed them to testBug36766and36769x(), as they cover both PRs.
Added testBug36766and36769D() - test for templated destructor.
2003-06-09 Victor Mozgin
Moved testBug36701() from ScannerFailedTests.java to ScannerTestCase.java.
Renamed it to testBug36701A() and fixed it.
Added ScannerTestCase.testBug36701B().
2003-06-05 John Camelon
Moved testBug23478A() & testBug23478B() from failed tests to TranslationUnitTests.java.
Removed TranslationUnitFailedTests.java as it was empty.

64
core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java
View file

@ -1,64 +0,0 @@
/*******************************************************************************
* Copyright (c) 2001 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Common Public License v0.5
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/cpl-v05.html
*
* Contributors:
* IBM Corp. - Rational Software - initial implementation
******************************************************************************/
package org.eclipse.cdt.core.parser.failedTests;
import java.io.StringWriter;
import junit.framework.AssertionFailedError;
import junit.framework.Test;
import junit.framework.TestSuite;
import org.eclipse.cdt.core.parser.tests.BaseScannerTest;
/**
* @author aniefer
*
* To change the template for this generated type comment go to
* Window>Preferences>Java>Code Generation>Code and Comments
*/
public class ScannerFailedTest extends BaseScannerTest {
public ScannerFailedTest(String name){
super(name);
}
public static Test suite()
{
TestSuite suite = new TestSuite();
suite.addTest( new ScannerFailedTest( "testBug36701" ) );
return suite;
}
public void testBug36701() throws Exception
{
boolean testPassed = false;
try{
StringWriter writer = new StringWriter();
writer.write( "#define str(s) # s\n" );
writer.write( "str( @ \n )\n");
initializeScanner( writer.toString() );
validateString( "@ \\\\n" );
validateEOF();
testPassed = true;
} catch( AssertionFailedError e ){
//expected failure
}
if( testPassed )
fail( "The expected error did not occur." );
}
}

22
core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java
View file

@ -1378,4 +1378,26 @@ public class ScannerTestCase extends BaseScannerTest
assertTrue(se.getMessage().equals("Invalid character '\\' read @ offset 5 of file TEXT"));
}
}
public void testBug36701A() throws Exception
{
StringWriter writer = new StringWriter();
writer.write("#define str(s) # s\n");
writer.write("str( @ \\n )\n");
initializeScanner(writer.toString());
validateString("@ \\\\n");
validateEOF();
}
public void testBug36701B() throws Exception
{
StringWriter writer = new StringWriter();
writer.write("#define str(s) # s\n");
writer.write("str( @ /*ff*/ \\n hh \"aa\" )\n");
initializeScanner(writer.toString());
validateString("@ \\\\n hh \\\"aa\\\"");
validateEOF();
}
}

1
core/org.eclipse.cdt.ui.tests/suite/org/eclipse/cdt/core/suite/AutomatedIntegrationSuite.java
View file

@ -80,7 +80,6 @@ public class AutomatedIntegrationSuite extends TestSuite implements TestListener
suite.addTestSuite(ACEFailedTest.class);
suite.addTestSuite(DOMFailedTest.class);
suite.addTestSuite(LokiFailures.class);
suite.addTestSuite(ScannerFailedTest.class);
suite.addTestSuite(STLFailedTests.class);
suite.addTestSuite(CModelElementsFailedTests.class);