diff --git a/core/org.eclipse.cdt.core/parser/ChangeLog b/core/org.eclipse.cdt.core/parser/ChangeLog index ddb3174e4b1..91312df8157 100644 --- a/core/org.eclipse.cdt.core/parser/ChangeLog +++ b/core/org.eclipse.cdt.core/parser/ChangeLog @@ -1,3 +1,7 @@ +2003-05-01 Andrew Niefer + Fixed Bug 36287 - Parser failure with new CDT 1.1 parser + Fixed Bug 37011 - Scanner: #define A "//" not properly handled + 2003-04-30 John Camelon Added some Javadoc to IParser.java and Parser.java. diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java index 88b62a244cb..d7e78ca2edd 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java @@ -120,17 +120,28 @@ public class Scanner implements IScanner { StringBuffer buffer = new StringBuffer(); skipOverWhitespace(); int c = getChar(); - + boolean inString = false; + boolean inChar = false; while (true) { while ((c != '\n') && (c != '\r') && (c != '\\') && (c != '/') + && (c != '"' || ( c == '"' && inChar ) ) + && (c != '\'' || ( c == '\'' && inString ) ) && (c != NOCHAR)) { buffer.append((char) c); - c = getChar(); + c = getChar( true ); } + if (c == '/') { + //only care about comments outside of a quote + if( inString || inChar ){ + buffer.append( (char) c ); + c = getChar( true ); + continue; + } + // we need to peek ahead at the next character to see if // this is a comment or not int next = getChar(); @@ -143,7 +154,7 @@ public class Scanner implements IScanner { if (skipOverMultilineComment()) break; else - c = getChar(); + c = getChar( true ); continue; } else { // we are not in a comment @@ -151,12 +162,35 @@ public class Scanner implements IScanner { c = next; continue; } - } else { - if (c != '\\') { - ungetChar(c); + } else if( c == '"' ){ + inString = !inString; + buffer.append((char) c); + c = getChar( true ); + continue; + } else if( c == '\'' ){ + inChar = !inChar; + buffer.append((char) c); + c = getChar( true ); + continue; + } else if( c == '\\' ){ + c = getChar(true); + if( c == '\r' ){ + c = getChar(true); + if( c == '\n' ){ + c = getChar(true); + } + } else if( c == '\n' ){ + c = getChar(true); } else { - c = getChar(); + buffer.append('\\'); + if( c == '"' || c == '\'' ){ + buffer.append((char)c); + c = getChar( true ); + } } + continue; + } else { + ungetChar(c); break; } } @@ -645,13 +679,23 @@ public class Scanner implements IScanner { c = getChar(); - if (c == 'x') { - if( ! firstCharZero && floatingPoint ) - { - ungetChar( c ); - return newToken( Token.tDOT, ".", contextStack.getCurrentContext() ); + if( ! firstCharZero && floatingPoint && !(c >= '0' && c <= '9') ){ + //if pasting, there could actually be a float here instead of just a . + if( buff.toString().equals( "." ) ){ + if( c == '*' ){ + return newToken( Token.tDOTSTAR, ".*", contextStack.getCurrentContext() ); + } else if( c == '.' ){ + if( getChar() == '.' ) + return newToken( Token.tELIPSE, "..." ); + else + throw new ScannerException( "Invalid floating point @ offset " + contextStack.getCurrentContext().getOffset() ); + } else { + ungetChar( c ); + return newToken( Token.tDOT, ".", contextStack.getCurrentContext() ); + } } - else if( ! firstCharZero ) + } else if (c == 'x') { + if( ! firstCharZero ) throw new ScannerException( "Invalid Hexidecimal @ offset " + contextStack.getCurrentContext().getOffset() ); hex = true; @@ -668,11 +712,6 @@ public class Scanner implements IScanner { if( c == '.' ) { buff.append( (char)c); - if( floatingPoint || hex ) { - if( buff.toString().equals( "..") && getChar() == '.' ) - return newToken( Token.tELIPSE, "..." ); - throw new ScannerException( "Invalid floating point @ offset " + contextStack.getCurrentContext().getOffset() ); - } floatingPoint = true; c= getChar(); diff --git a/core/org.eclipse.cdt.ui.tests/ChangeLog b/core/org.eclipse.cdt.ui.tests/ChangeLog index 98135ee3504..0fe1a3c609a 100644 --- a/core/org.eclipse.cdt.ui.tests/ChangeLog +++ b/core/org.eclipse.cdt.ui.tests/ChangeLog @@ -1,3 +1,9 @@ +2003-05-01 Andrew Niefer + Updated FractionalAutomatedTest to use threads + Modified ScannerTestCase::testBug36287 + Added ScannerTestCase::testBug37011 + Added ScannerTestCase::testOtherPreprocessorDefines + 2003-04-28 John Camelon Moved testBug36730() & testBug37019() from DOMFailedTests to DOMTests. diff --git a/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java b/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java index 5052f4bb79d..71c37d96941 100644 --- a/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java +++ b/core/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java @@ -18,8 +18,6 @@ import junit.framework.Test; import junit.framework.TestSuite; import org.eclipse.cdt.core.parser.tests.BaseScannerTest; -import org.eclipse.cdt.internal.core.parser.ScannerException; -import org.eclipse.cdt.internal.core.parser.Token; /** * @author aniefer @@ -38,7 +36,6 @@ public class ScannerFailedTest extends BaseScannerTest { TestSuite suite = new TestSuite(); suite.addTest( new ScannerFailedTest( "testBug36701" ) ); - suite.addTest( new ScannerFailedTest( "testBug37011" ) ); return suite; } @@ -64,27 +61,4 @@ public class ScannerFailedTest extends BaseScannerTest { fail( "The expected error did not occur." ); } - public void testBug37011() throws Exception - { - boolean testPassed = false; - try{ - StringWriter writer = new StringWriter(); - writer.write( "#define A \"//\"\n" ); - writer.write( " { A };\n" ); - - initializeScanner( writer.toString() ); - validateToken( Token.tLBRACE ); - validateString( "//" ); - validateToken( Token.tRBRACE ); - validateToken( Token.tSEMI ); - validateEOF(); - - testPassed = true; - } catch( ScannerException e ){ - //expected failure - } - - if( testPassed ) - fail( "The expected error did not occur." ); - } } diff --git a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/FractionalAutomatedTest.java b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/FractionalAutomatedTest.java index 1686f669b47..d53c7a0711f 100644 --- a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/FractionalAutomatedTest.java +++ b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/FractionalAutomatedTest.java @@ -14,14 +14,13 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; +import java.io.IOException; import java.io.StringWriter; import java.util.NoSuchElementException; import java.util.StringTokenizer; -import junit.framework.AssertionFailedError; import junit.framework.Test; -import org.eclipse.cdt.internal.core.parser.IParser; import org.eclipse.cdt.internal.core.parser.Parser; import org.eclipse.core.runtime.Path; @@ -57,8 +56,8 @@ public class FractionalAutomatedTest extends AutomatedFramework { stepSize = Integer.parseInt( properties.getProperty( "stepSize", "50" ) ); windowSize = Integer.parseInt( properties.getProperty( "windowSize", "200" ) ); - - tempFile = properties.getProperty( "tempFile", "" ); + timeOut = Integer.parseInt( properties.getProperty( "timeOut", "5000" )); + outputDir = properties.getProperty( "outDir", "" ); if( sourceInfo.equals("") ) throw new FileNotFoundException(); @@ -91,95 +90,163 @@ public class FractionalAutomatedTest extends AutomatedFramework { return frame.createSuite(); } - static private void outputTempFile( String code ) { - if( tempFile == null || tempFile.equals("") ) - return; + static private String outputFile( String code ) { + if( outputDir == null || outputDir.equals("") ) + return ""; - File output = new File( tempFile ); + File output = new File( outputDir ); try{ if( output.exists() ){ - output.delete(); + if( output.isFile() ){ + output.delete(); + output.createNewFile(); + FileOutputStream stream = new FileOutputStream( output ); + stream.write( code.getBytes() ); + stream.flush(); + stream.close(); + return outputDir; + } + } else { + output.mkdir(); } - - output.createNewFile(); - FileOutputStream stream = new FileOutputStream( output ); + File file = new File( outputDir + "/" + failures++ + ".tmp" ); + if( file.exists() ) + file.delete(); + file.createNewFile(); + FileOutputStream stream = new FileOutputStream( file ); stream.write( code.getBytes() ); stream.flush(); stream.close(); + + return file.getCanonicalPath(); + } catch ( Exception e ) {} + return ""; + } + + static private void reportHang( String code, String file ){ + String output = outputFile( code.toString() ); + if( output.equals("") ) + output = "Parser hang while parsing " + file + "\n"; + else + output = "Parser hang while parsing " + output + "\n"; + + if( report != null ){ + try{ + report.write( output.getBytes() ); + } catch ( IOException e ) {} + } + + fail( output ); + } + + static private void reportException( String code, String file, String exception ){ + String output = outputFile( code.toString() ); + + if( output.equals("") ) + output = exception.getClass().toString() + " encountered in " + file + "\n"; + else + output = exception.getClass().toString() + " encountered in " + output + "\n"; + + if( report != null ){ + try{ + report.write( output.getBytes() ); + } catch ( IOException e ) {} + } + + fail( output ); } public void doFile() throws Throwable { assertNotNull( fileList ); - File file = null; - IParser parser = null; + File file = (File)fileList.removeFirst(); + FileInputStream stream = new FileInputStream( file ); + + String filePath = file.getCanonicalPath(); + String nature = (String)natures.get( filePath ); + + boolean cppNature = nature.equalsIgnoreCase("cpp"); - try{ - file = (File)fileList.removeFirst(); - FileInputStream stream = new FileInputStream( file ); + StringWriter code = new StringWriter(); + + ParseThread thread = new ParseThread(); + + byte b[] = new byte[stepSize]; + int n = stream.read( b ); + while( n != -1 ){ + code.write( new String( b ) ); - String filePath = file.getCanonicalPath(); - String nature = (String)natures.get( filePath ); - - boolean cppNature = nature.equalsIgnoreCase("cpp"); + thread.code = code.toString(); + thread.cppNature = cppNature; + thread.start(); + thread.join( timeOut ); - StringWriter code = new StringWriter(); - - byte b[] = new byte[stepSize]; - int n = stream.read( b ); - while( n != -1 ){ - code.write( new String( b ) ); - parser = new Parser( code.toString(), nullCallback, true); - parser.setCppNature( cppNature ); - parser.mapLineNumbers(true); - - outputTempFile( code.toString() ); - parser.parse(); - - n = stream.read( b ); + if( thread.isAlive() ){ + //Use deprecated Thread.stop() for now + //alternative is to create a callback which could stop the parse on a flag + //by throwing something, but that has the disadvantage of being unable to + //stop any loops that don't involve callbacks. + thread.stop(); + reportHang( code.toString(), filePath ); + } else if( thread.result != null ) { + reportException( code.toString(), filePath, thread.result ); } - String fullCode = code.toString(); - String windowedCode = null; - int length = fullCode.length(); - int curPos = 0; + n = stream.read( b ); + } + + String fullCode = code.toString(); + String windowedCode = null; + int length = fullCode.length(); + int curPos = 0; + + while( curPos + windowSize < length){ + windowedCode = fullCode.substring( 0, curPos ); + windowedCode += "\n" + fullCode.substring( curPos + windowSize, length ); - while( curPos + windowSize < length){ - windowedCode = fullCode.substring( 0, curPos ); - windowedCode += "\n" + fullCode.substring( curPos + windowSize, length ); - - parser = new Parser( windowedCode, nullCallback, true ); - parser.setCppNature( cppNature ); - parser.mapLineNumbers(true); + thread.code = windowedCode; + thread.cppNature = cppNature; + thread.file = filePath; + thread.start(); + thread.join( timeOut ); - outputTempFile( windowedCode ); - parser.parse(); - - curPos += stepSize; - } - } - catch( Throwable e ) - { - String output = null; - if( e instanceof AssertionFailedError ){ - output = file.getCanonicalPath() + ": Parse failed on line "; - output += parser.getLineNumberForOffset(parser.getLastErrorOffset()) + "\n"; - } else { - output = file.getCanonicalPath() + ": " + e.getClass().toString(); - output += " on line " + parser.getLineNumberForOffset(parser.getLastErrorOffset()) + "\n"; - } - if( report != null ){ - report.write( output.getBytes() ); + if( thread.isAlive() ) + { + thread.stop(); + reportHang( windowedCode, filePath ); + } else if( thread.result != null ) { + reportException( windowedCode, filePath, thread.result ); } - fail( output ); + curPos += stepSize; } } + static class ParseThread extends Thread{ + public String code; + public boolean cppNature; + public String file; + public String result; + + public void run(){ + try{ + result = null; + Parser parser = new Parser( code, nullCallback, true); + parser.setCppNature( cppNature ); + parser.mapLineNumbers(true); + parser.parse(); + } catch ( Exception e ){ + result = e.getClass().toString(); + } + } + } + static private int stepSize = 50; static private int windowSize = 200; - static private String tempFile = null; + static private int timeOut = 5000; + static private String outputDir = null; + static private int failures = 0; } diff --git a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java index 74cdacf1a2f..6151cb91738 100644 --- a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java +++ b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java @@ -995,7 +995,7 @@ public class ScannerTestCase extends BaseScannerTest public void testBug36287() throws Exception { - initializeScanner( "X::X( const X & rtg_arg ) : U( rtg_arg ) , Z( rtg_arg.Z ) , br( rtg_arg.br ){}" ); + initializeScanner( "X::X( const X & rtg_arg ) : U( rtg_arg ) , Z( rtg_arg.Z ) , er( rtg_arg.er ){}" ); validateIdentifier("X"); validateToken( Token.tCOLONCOLON); validateIdentifier("X"); @@ -1018,15 +1018,27 @@ public class ScannerTestCase extends BaseScannerTest validateIdentifier( "Z"); validateToken( Token.tRPAREN ); validateToken( Token.tCOMMA ); - validateIdentifier( "br"); + validateIdentifier( "er"); validateToken( Token.tLPAREN ); validateIdentifier( "rtg_arg"); validateToken( Token.tDOT ); - validateIdentifier( "br"); + validateIdentifier( "er"); validateToken( Token.tRPAREN ); validateToken( Token.tLBRACE); validateToken( Token.tRBRACE); validateEOF(); + + initializeScanner( "foo.*bar"); + validateIdentifier("foo"); + validateToken( Token.tDOTSTAR ); + validateIdentifier("bar"); + validateEOF(); + + initializeScanner( "foo...bar"); + validateIdentifier("foo"); + validateToken( Token.tELIPSE ); + validateIdentifier("bar"); + validateEOF(); } public void testBug35892() @@ -1309,4 +1321,39 @@ public class ScannerTestCase extends BaseScannerTest initializeScanner( writer.toString() ); validateEOF(); } + + public void testBug37011() throws Exception{ + StringWriter writer = new StringWriter(); + writer.write( "#define A \"//\""); + + initializeScanner( writer.toString() ); + + validateEOF(); + validateDefinition("A", "\"//\""); + } + + public void testOtherPreprocessorDefines() throws Exception{ + StringWriter writer = new StringWriter(); + writer.write( "#define A a//boo\n" ); + writer.write( "#define B a /*boo*/ a\n" ); + writer.write( "#define C a \" //boo \"\n" ); + writer.write( "#define D a \\\"//boo\n" ); + writer.write( "#define E a \\n \"\\\"\"\n" ); + writer.write( "#define F a\\\n b\n" ); + writer.write( "#define G a '\"'//boo\n" ); + writer.write( "#define H a '\\'//b'\"/*bo\\o*/\" b\n" ); + + initializeScanner( writer.toString() ); + + validateEOF(); + + validateDefinition("A", "a"); + validateDefinition("B", "a a"); + validateDefinition("C", "a \" //boo \""); + validateDefinition("D", "a \\\""); + validateDefinition("E", "a \\n \"\\\"\""); + validateDefinition("F", "a b"); + validateDefinition("G", "a '\"'"); + validateDefinition("H", "a '\\'//b'\"/*bo\\o*/\" b"); + } }