diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java index 34482b157d0..e5113f9f059 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java @@ -1188,7 +1188,7 @@ public class Scanner2Test extends BaseScanner2Test //TODO as in 36701B, whitespace is not properly replaced inside the string, ok for now. //validateString("strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') == 0"); //$NON-NLS-1$ - validateString("strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4')\n == 0"); //$NON-NLS-1$ + validateString("strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') == 0"); //$NON-NLS-1$ validateToken(IToken.tCOMMA); validateIdentifier("s"); //$NON-NLS-1$ diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java index 41e52561b28..b92d9e1025e 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java @@ -463,9 +463,7 @@ public class Scanner2 implements IScanner, IScannerData { continue; return t; } - //TODO - handle problem invalid char sequence - //error handling - bufferPos[ bufferStackPos ] += 2; + handleProblem( IProblem.SCANNER_BAD_CHARACTER, bufferPos[ bufferStackPos ], new char[] { '\\' } ); continue; case '0': @@ -844,17 +842,24 @@ public class Scanner2 implements IScanner, IScannerData { int stringStart = bufferPos[bufferStackPos] + 1; int stringLen = 0; boolean escaped = false; + boolean foundClosingQuote = false; loop: while (++bufferPos[bufferStackPos] < bufferLimit[bufferStackPos]) { ++stringLen; char c = buffer[bufferPos[bufferStackPos]]; if (c == '"') { - if (!escaped) + if (!escaped){ + foundClosingQuote = true; break; } + } else if (c == '\\') { escaped = !escaped; continue; + } else if(c == '\n'){ + //unescaped end of line before end of string + if( !escaped ) + break; } escaped = false; } @@ -862,8 +867,11 @@ public class Scanner2 implements IScanner, IScannerData { // We should really throw an exception if we didn't get the terminating // quote before the end of buffer - - return newToken(tokenType, CharArrayUtils.extract(buffer, stringStart, stringLen)); + char[] result = CharArrayUtils.extract(buffer, stringStart, stringLen); + if( !foundClosingQuote ){ + handleProblem( IProblem.SCANNER_UNBOUNDED_STRING, stringStart, result ); + } + return newToken(tokenType, result); } private IToken scanCharLiteral(boolean b) { @@ -955,9 +963,11 @@ public class Scanner2 implements IScanner, IScannerData { continue; case '.': - if (isFloat) + if (isFloat){ // second dot + handleProblem( IProblem.SCANNER_BAD_FLOATING_POINT, start, null ); break; + } isFloat = true; continue; @@ -1094,9 +1104,13 @@ public class Scanner2 implements IScanner, IScannerData { --bufferPos[bufferStackPos]; - return newToken( isFloat ? IToken.tFLOATINGPT : IToken.tINTEGER, - CharArrayUtils.extract(buffer, start, - bufferPos[bufferStackPos] - start + 1) ); + char[] result = CharArrayUtils.extract( buffer, start, bufferPos[bufferStackPos] - start + 1); + int tokenType = isFloat ? IToken.tFLOATINGPT : IToken.tINTEGER; + if( tokenType == IToken.tINTEGER && isHex && result.length == 2 ){ + handleProblem( IProblem.SCANNER_BAD_HEX_FORMAT, start, result ); + } + + return newToken( tokenType, result ); } private void handlePPDirective(int pos) throws ScannerException, EndOfFileException { @@ -1114,6 +1128,7 @@ public class Scanner2 implements IScanner, IScannerData { if (start >= limit || buffer[start] == '\n') return; + boolean problem = false; char c = buffer[start]; if ((c >= 'a' && c <= 'z')) { while (++bufferPos[bufferStackPos] < limit) { @@ -1172,10 +1187,19 @@ public class Scanner2 implements IScanner, IScannerData { return; case ppError: throw new ScannerException(null); + case ppEndif: + break; + default: + problem = true; + break; } } - } + } else + problem = true; + if( problem ) + handleProblem( IProblem.PREPROCESSOR_INVALID_DIRECTIVE, start, null ); + // don't know, chew up to the end of line // includes endif which is immatereal at this point skipToNewLine(); @@ -1295,16 +1319,17 @@ public class Scanner2 implements IScanner, IScannerData { if( filename == null || filename == EMPTY_STRING ) { - //TODO IProblem + handleProblem( IProblem.PREPROCESSOR_INVALID_DIRECTIVE, startOffset, null ); return; } + char [] fileNameArray = filename.toCharArray(); // TODO else we need to do macro processing on the rest of the line endLine = bufferLineNums[ bufferStackPos ]; skipToNewLine(); if( parserMode == ParserMode.QUICK_PARSE ) { - IASTInclusion inclusion = getASTFactory().createInclusion( filename.toCharArray(), EMPTY_STRING_CHAR_ARRAY, local, startOffset, startingLineNumber, nameOffset, nameEndOffset, nameLine, endOffset, endLine, getCurrentFilename() ); + IASTInclusion inclusion = getASTFactory().createInclusion( fileNameArray, EMPTY_STRING_CHAR_ARRAY, local, startOffset, startingLineNumber, nameOffset, nameEndOffset, nameLine, endOffset, endLine, getCurrentFilename() ); requestor.enterInclusion( inclusion ); requestor.exitInclusion( inclusion ); } @@ -1327,7 +1352,7 @@ public class Scanner2 implements IScanner, IScannerData { if (reader.filename != null) fileCache.put(reader.filename, reader); if (dlog != null) dlog.println("#include \"" + finalPath + "\""); //$NON-NLS-1$ //$NON-NLS-2$ - IASTInclusion inclusion = getASTFactory().createInclusion( filename.toCharArray(), reader.filename, local, startOffset, startingLineNumber, nameOffset, nameEndOffset, nameLine, endOffset, endLine, getCurrentFilename() ); + IASTInclusion inclusion = getASTFactory().createInclusion( fileNameArray, reader.filename, local, startOffset, startingLineNumber, nameOffset, nameEndOffset, nameLine, endOffset, endLine, getCurrentFilename() ); pushContext(reader.buffer, new InclusionData( reader, inclusion )); return; } @@ -1354,16 +1379,16 @@ public class Scanner2 implements IScanner, IScannerData { if (reader.filename != null) fileCache.put(reader.filename, reader); if (dlog != null) dlog.println("#include <" + finalPath + ">"); //$NON-NLS-1$ //$NON-NLS-2$ - IASTInclusion inclusion = getASTFactory().createInclusion( filename.toCharArray(), reader.filename, local, startOffset, startingLineNumber, nameOffset, nameEndOffset, nameLine, endOffset, endLine, getCurrentFilename() ); + IASTInclusion inclusion = getASTFactory().createInclusion( fileNameArray, reader.filename, local, startOffset, startingLineNumber, nameOffset, nameEndOffset, nameLine, endOffset, endLine, getCurrentFilename() ); pushContext(reader.buffer, new InclusionData( reader, inclusion )); return; } } } - // TODO raise a problem - //if (reader == null) - // handleProblem( IProblem.PREPROCESSOR_INCLUSION_NOT_FOUND, filename, beginOffset, false, true ); + + if (reader == null) + handleProblem( IProblem.PREPROCESSOR_INCLUSION_NOT_FOUND, startOffset, fileNameArray); } } @@ -1383,6 +1408,7 @@ public class Scanner2 implements IScanner, IScannerData { char c = buffer[idstart]; if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_')) { + handleProblem( IProblem.PREPROCESSOR_INVALID_MACRO_DEFN, idstart, null ); skipToNewLine(); return; } @@ -1429,6 +1455,7 @@ public class Scanner2 implements IScanner, IScannerData { arglist[++currarg] = "...".toCharArray(); //$NON-NLS-1$ continue; } else if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + handleProblem( IProblem.PREPROCESSOR_INVALID_MACRO_DEFN, idstart, name ); // yuck skipToNewLine(); return; @@ -1455,7 +1482,22 @@ public class Scanner2 implements IScanner, IScannerData { boolean encounteredMultilineComment = false; while (bufferPos[bufferStackPos] + 1 < limit && buffer[bufferPos[bufferStackPos] + 1] != '\n') { - skipOverNonWhiteSpace(); + if( arglist != null && !skipOverNonWhiteSpace( true ) ){ + ++bufferPos[bufferStackPos]; + if( skipOverWhiteSpace() ) + encounteredMultilineComment = true; + boolean isArg = false; + for( int i = 0; i < arglist.length && arglist[i] != null; i++ ){ + if( CharArrayUtils.equals( buffer, bufferPos[bufferStackPos], arglist[i].length, arglist[i] ) ){ + isArg = true; + break; + } + } + if( !isArg ) + handleProblem( IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, bufferPos[bufferStackPos], null ); + } else { + skipOverNonWhiteSpace(); + } textend = bufferPos[bufferStackPos]; if( skipOverWhiteSpace() ) encounteredMultilineComment = true; @@ -1756,7 +1798,10 @@ public class Scanner2 implements IScanner, IScannerData { return encounteredMultiLineComment; } - private void skipOverNonWhiteSpace() { + private void skipOverNonWhiteSpace(){ + skipOverNonWhiteSpace( false ); + } + private boolean skipOverNonWhiteSpace( boolean stopAtPound ) { char[] buffer = bufferStack[bufferStackPos]; int limit = bufferLimit[bufferStackPos]; @@ -1767,13 +1812,13 @@ public class Scanner2 implements IScanner, IScannerData { case '\r': case '\n': --bufferPos[bufferStackPos]; - return; + return true; case '/': int pos = bufferPos[bufferStackPos]; if( pos +1 < limit && ( buffer[pos+1] == '/' ) || ( buffer[pos+1] == '*') ) { --bufferPos[bufferStackPos]; - return; + return true; } break; @@ -1782,7 +1827,7 @@ public class Scanner2 implements IScanner, IScannerData { if (pos + 1 < limit && buffer[pos + 1] == '\n') { // \n is whitespace --bufferPos[bufferStackPos]; - return; + return true; } break; case '"': @@ -1810,7 +1855,7 @@ public class Scanner2 implements IScanner, IScannerData { buffer[bufferPos[ bufferStackPos ] + 1] == '*' ) ) { --bufferPos[bufferStackPos]; - return; + return true; } default: @@ -1837,9 +1882,16 @@ public class Scanner2 implements IScanner, IScannerData { } } break; + case '#' : + if( stopAtPound ){ + --bufferPos[bufferStackPos]; + return false; + } + break; } } --bufferPos[bufferStackPos]; + return true; } private void skipOverMacroArg() { @@ -1970,9 +2022,11 @@ public class Scanner2 implements IScanner, IScannerData { CharArrayObjectMap argmap = new CharArrayObjectMap(arglist.length); while (bufferPos[bufferStackPos] < limit) { - if (++currarg >= arglist.length || arglist[currarg] == null) + if (++currarg >= arglist.length || arglist[currarg] == null){ // too many args + handleProblem( IProblem.PREPROCESSOR_MACRO_USAGE_ERROR, bufferPos[bufferStackPos], macro.name ); break; + } skipOverWhiteSpace(); @@ -2052,6 +2106,16 @@ public class Scanner2 implements IScanner, IScannerData { if (c == ')') break; } + int numArgs = arglist.length; + for( int i = 0; i < arglist.length; i++ ){ + if( arglist[i] == null ){ + numArgs = i; + break; + } + } + if( argmap.size() < numArgs ){ + handleProblem( IProblem.PREPROCESSOR_MACRO_USAGE_ERROR, bufferPos[bufferStackPos], macro.name ); + } int size = expandFunctionStyleMacro(macro.expansion, argmap, null); char[] result = new char[size]; @@ -2344,7 +2408,10 @@ public class Scanner2 implements IScanner, IScannerData { for( int i = 0; i < argvalue.length; i++ ){ if( argvalue[i] == '"' || argvalue[i] == '\\' ) result[outpos++] = '\\'; - result[outpos++] = argvalue[i]; + if( argvalue[i] == '\r' || argvalue[i] == '\n' ) + result[outpos++] = ' '; + else + result[outpos++] = argvalue[i]; } result[outpos++] = '"'; } else { diff --git a/core/org.eclipse.cdt.core/search/org/eclipse/cdt/internal/core/search/matching/CSearchPattern.java b/core/org.eclipse.cdt.core/search/org/eclipse/cdt/internal/core/search/matching/CSearchPattern.java index 15d9d6ce3e8..1e87f6d5d47 100644 --- a/core/org.eclipse.cdt.core/search/org/eclipse/cdt/internal/core/search/matching/CSearchPattern.java +++ b/core/org.eclipse.cdt.core/search/org/eclipse/cdt/internal/core/search/matching/CSearchPattern.java @@ -65,6 +65,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte protected static class Requestor extends NullSourceElementRequestor { + public int badCharacterOffset = -1; public Requestor( ParserMode mode ) { super( mode ); @@ -72,12 +73,14 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte public boolean acceptProblem( IProblem problem ) { - if( problem.getID() == IProblem.SCANNER_BAD_CHARACTER ) return false; + if( problem.getID() == IProblem.SCANNER_BAD_CHARACTER ){ + badCharacterOffset = problem.getSourceStart(); + return false; + } return super.acceptProblem( problem ); } } - private static Requestor callback = new Requestor( ParserMode.COMPLETE_PARSE ); /** * @param matchMode * @param caseSensitive @@ -176,7 +179,9 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte } char [] patternArray = patternString.toCharArray(); + IScanner scanner = null; + Requestor callback = new Requestor( ParserMode.COMPLETE_PARSE ); try { scanner = ParserFactory.createScanner( @@ -189,7 +194,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte } catch (ParserFactoryError e) { } - LinkedList list = scanForNames( scanner, null, patternArray ); + LinkedList list = scanForNames( scanner, callback, null, patternArray ); char [] name = (char []) list.removeLast(); char [][] qualifications = new char [0][]; @@ -247,6 +252,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte } char [] patternArray = patternString.toCharArray(); + Requestor callback = new Requestor( ParserMode.COMPLETE_PARSE ); IScanner scanner=null; try { scanner = @@ -259,7 +265,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte } catch (ParserFactoryError e) { } - LinkedList list = scanForNames( scanner, null, patternArray ); + LinkedList list = scanForNames( scanner, callback, null, patternArray ); char [] name = (char []) list.removeLast(); char [][] qualifications = new char[0][]; @@ -289,6 +295,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte String nameString = ( index == -1 ) ? patternString : patternString.substring( 0, index ); char [] nameArray = nameString.toCharArray(); IScanner scanner=null; + Requestor callback = new Requestor( ParserMode.COMPLETE_PARSE ); try { scanner = ParserFactory.createScanner( @@ -300,7 +307,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte } catch (ParserFactoryError e) { } - LinkedList names = scanForNames( scanner, null, nameArray ); + LinkedList names = scanForNames( scanner, callback, null, nameArray ); LinkedList params = scanForParameters( paramString ); @@ -340,6 +347,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte char [] patternArray = patternString.toCharArray(); IScanner scanner =null; + Requestor callback = new Requestor( ParserMode.COMPLETE_PARSE ); try { scanner = ParserFactory.createScanner( @@ -380,7 +388,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte } } - LinkedList list = scanForNames( scanner, token, patternArray ); + LinkedList list = scanForNames( scanner, callback, token, patternArray ); char[] name = (char [])list.removeLast(); char [][] qualifications = new char[0][]; @@ -393,6 +401,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte char [] patternArray = patternString.toCharArray(); IScanner scanner =null; + Requestor callback = new Requestor( ParserMode.COMPLETE_PARSE ); try { scanner = ParserFactory.createScanner( @@ -406,7 +415,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte searchFor = DERIVED; - LinkedList list = scanForNames( scanner, null, patternArray ); + LinkedList list = scanForNames( scanner, callback, null, patternArray ); char[] name = (char [])list.removeLast(); char [][] qualifications = new char[0][]; @@ -418,6 +427,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte char [] patternArray = patternString.toCharArray(); IScanner scanner =null; + Requestor callback = new Requestor( ParserMode.COMPLETE_PARSE ); try { scanner = ParserFactory.createScanner( @@ -431,7 +441,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte searchFor = FRIEND; - LinkedList list = scanForNames( scanner, null, patternArray ); + LinkedList list = scanForNames( scanner, callback, null, patternArray ); char[] name = (char [])list.removeLast(); char [][] qualifications = new char[0][]; @@ -452,6 +462,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte String functionString = "void f " + paramString + ";"; //$NON-NLS-1$ //$NON-NLS-2$ IScanner scanner=null; + IQuickParseCallback callback = ParserFactory.createQuickParseCallback(); try { scanner = ParserFactory.createScanner( @@ -462,7 +473,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte callback,new NullLogService(), null); } catch (ParserFactoryError e1) { } - IQuickParseCallback callback = ParserFactory.createQuickParseCallback(); + IParser parser=null; try { parser = @@ -508,7 +519,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte return list; } - static private LinkedList scanForNames( IScanner scanner, IToken unusedToken, char[] pattern ){ + static private LinkedList scanForNames( IScanner scanner, Requestor callback, IToken unusedToken, char[] pattern ){ LinkedList list = new LinkedList(); String name = new String(""); //$NON-NLS-1$ @@ -517,9 +528,7 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte try { IToken token = ( unusedToken != null ) ? unusedToken : scanner.nextToken(); IToken prev = null; - - scanner.setThrowExceptionOnBadCharacterRead( true ); - + boolean encounteredWild = false; boolean lastTokenWasOperator = false; @@ -579,19 +588,17 @@ public abstract class CSearchPattern implements ICSearchConstants, ICSearchPatte token = null; while( token == null ){ - try{ - token = scanner.nextToken(); - } catch ( ScannerException e ){ - if( e.getProblem().getID() == IProblem.SCANNER_BAD_CHARACTER ){ - //TODO : This may not be \\, it could be another bad character - if( !encounteredWild && !lastTokenWasOperator && prev.getType() != IToken.tARROW ) name += " "; //$NON-NLS-1$ - name += "\\"; //$NON-NLS-1$ - idx++; - encounteredWild = true; - lastTokenWasOperator = false; - prev = null; - } - } + token = scanner.nextToken(); + if( callback.badCharacterOffset != -1 && token.getOffset() > callback.badCharacterOffset ){ + //TODO : This may not be \\, it could be another bad character + if( !encounteredWild && !lastTokenWasOperator && prev.getType() != IToken.tARROW ) name += " "; //$NON-NLS-1$ + name += "\\"; //$NON-NLS-1$ + idx++; + encounteredWild = true; + lastTokenWasOperator = false; + prev = null; + callback.badCharacterOffset = -1; + } } } } catch (EndOfFileException e) {