From f8ede5731ba7089e380f1d53f05402d0923c9235 Mon Sep 17 00:00:00 2001 From: John Camelon Date: Fri, 4 Jun 2004 14:27:46 +0000 Subject: [PATCH] Patch for David Daoust - remove all instances of sub-Scanners in the scanner. Performance gains abound. --- .../core/parser/tests/QuickParseASTTests.java | 6 +- .../core/parser/scanner/ContextStack.java | 5 +- .../parser/scanner/GCCScannerExtension.java | 16 +- .../core/parser/scanner/IScannerContext.java | 1 + .../core/parser/scanner/IScannerData.java | 8 + .../internal/core/parser/scanner/Scanner.java | 351 +++++++++++++----- .../scanner/ScannerContextInclusion.java | 2 + .../parser/scanner/ScannerContextMacro.java | 2 + .../scanner/ScannerContextTopString.java | 113 ++++++ .../core/parser/scanner/ScannerUtility.java | 104 +----- 10 files changed, 398 insertions(+), 210 deletions(-) create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextTopString.java diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java index 692ce0fae38..10fd3992aa9 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java @@ -1156,7 +1156,7 @@ public class QuickParseASTTests extends BaseASTTest String code = "#include \n#define DEF VALUE\n"; //$NON-NLS-1$ - IASTCompilationUnit tu = parse( code.toString() ); + IASTCompilationUnit tu = parse( code ); assertFalse( tu.getDeclarations().hasNext()); Iterator inclusions = quickParseCallback.getInclusions(); Iterator macros = quickParseCallback.getMacros(); @@ -1166,8 +1166,8 @@ public class QuickParseASTTests extends BaseASTTest assertEquals( i.getName(), "stdio.h"); //$NON-NLS-1$ assertEquals( i.getStartingOffset(), 0 ); - assertEquals( i.getNameOffset(), 10 ); - assertEquals( i.getEndingOffset(), 19 ); + assertEquals( i.getNameOffset(), code.indexOf("stdio.h") ); //$NON-NLS-1$ + assertEquals( i.getEndingOffset(), code.indexOf(">") + 1); //$NON-NLS-1$ IASTMacro m = (IASTMacro)macros.next(); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ContextStack.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ContextStack.java index a6e9fa1702f..442d88e719e 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ContextStack.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ContextStack.java @@ -33,6 +33,7 @@ public class ContextStack { public String getContextName() { return ""; } //$NON-NLS-1$ public int getOffset() { return 0; } public void ungetChar(int undo) { } + public boolean isFinal() { return false; } public int getKind() { return IScannerContext.ContextKind.SENTINEL; } public void close() { } } @@ -83,7 +84,7 @@ public class ContextStack { private IScanner scanner; - private final void cs_push(IScannerContext c) { + public final void cs_push(IScannerContext c) { try { cs[cs_pos++] = c; } @@ -98,7 +99,7 @@ public class ContextStack { } scanner.setScannerContext(c); } - private final IScannerContext cs_pop() { + public final IScannerContext cs_pop() { IScannerContext context = cs[--cs_pos]; scanner.setScannerContext((cs_pos == 0) ? sentinel : cs[cs_pos -1]); return context; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/GCCScannerExtension.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/GCCScannerExtension.java index 04c5b06db10..1874ec73e2a 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/GCCScannerExtension.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/GCCScannerExtension.java @@ -158,19 +158,19 @@ public class GCCScannerExtension implements IScannerExtension { /* (non-Javadoc) * @see org.eclipse.cdt.core.parser.extension.IScannerExtension#handlePreprocessorDirective(java.lang.String, java.lang.String) */ - public void handlePreprocessorDirective(IScannerData scannerData, String directive, String restOfLine) { + public void handlePreprocessorDirective(IScannerData iscanner, String directive, String restOfLine) { if( directive.equals(POUND_INCLUDE_NEXT) ) { - TraceUtil.outputTrace(scannerData.getLogService(), "GCCScannerExtension handling #include_next directive", null, null, null, null); //$NON-NLS-1$ + TraceUtil.outputTrace(iscanner.getLogService(), "GCCScannerExtension handling #include_next directive", null, null, null, null); //$NON-NLS-1$ // figure out the name of the current file and its path - IScannerContext context = scannerData.getContextStack().getCurrentContext(); + IScannerContext context = iscanner.getContextStack().getCurrentContext(); if( context == null || context.getKind() != IScannerContext.ContextKind.INCLUSION ) return; String fullInclusionPath = context.getContextName(); IASTInclusion inclusion = ((ScannerContextInclusion)context).getExtension(); - Iterator iter = scannerData.getIncludePathNames().iterator(); + Iterator iter = iscanner.getIncludePathNames().iterator(); while (iter.hasNext()) { String path = (String)iter.next(); @@ -181,7 +181,7 @@ public class GCCScannerExtension implements IScannerExtension { ScannerUtility.InclusionDirective parsedDirective = null; try { - parsedDirective = ScannerUtility.parseInclusionDirective( scannerData, this, restOfLine, scannerData.getContextStack().getCurrentContext().getOffset() ); + parsedDirective = iscanner.parseInclusionDirective( restOfLine, iscanner.getContextStack().getCurrentContext().getOffset() ); } catch (InclusionParseException e) { return; } @@ -189,7 +189,7 @@ public class GCCScannerExtension implements IScannerExtension { // search through include paths while (iter.hasNext()) { String path = (String)iter.next(); - duple = ScannerUtility.createReaderDuple( path, parsedDirective.getFilename(), scannerData.getClientRequestor(), scannerData.getWorkingCopies() ); + duple = ScannerUtility.createReaderDuple( path, parsedDirective.getFilename(), iscanner.getClientRequestor(), iscanner.getWorkingCopies() ); if( duple != null ) break; } @@ -198,8 +198,8 @@ public class GCCScannerExtension implements IScannerExtension { { try { - scannerData.getContextStack().updateInclusionContext(duple.getUnderlyingReader(), duple.getFilename(), inclusion, scannerData.getClientRequestor() ); - TraceUtil.outputTrace( scannerData.getLogService(), "GCCScannerExtension handling #include_next directive successfully pushed on new include file" ); //$NON-NLS-1$ + iscanner.getContextStack().updateInclusionContext(duple.getUnderlyingReader(), duple.getFilename(), inclusion, iscanner.getClientRequestor() ); + TraceUtil.outputTrace( iscanner.getLogService(), "GCCScannerExtension handling #include_next directive successfully pushed on new include file" ); //$NON-NLS-1$ } catch (ContextException e1) { diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IScannerContext.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IScannerContext.java index bc3080942c3..b49160d9bae 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IScannerContext.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IScannerContext.java @@ -29,6 +29,7 @@ public interface IScannerContext { public int getChar(); public void ungetChar(int undo); + public boolean isFinal(); public String getContextName(); public int getOffset(); public void close(); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IScannerData.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IScannerData.java index 1a8352fd174..ab4c13ebad6 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IScannerData.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IScannerData.java @@ -23,6 +23,8 @@ import org.eclipse.cdt.core.parser.ParserLanguage; import org.eclipse.cdt.core.parser.ParserMode; import org.eclipse.cdt.core.parser.ast.IASTFactory; import org.eclipse.cdt.internal.core.parser.problem.IProblemFactory; +import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionDirective; +import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionParseException; /** * @author jcamelon @@ -84,4 +86,10 @@ public interface IScannerData { public abstract void setDefinitions(Map map); public Iterator getWorkingCopies(); + /** + * @param restOfLine + * @param offset + * @return + */ + public abstract InclusionDirective parseInclusionDirective(String restOfLine, int offset) throws InclusionParseException; } \ No newline at end of file diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java index d61d2b6fb7b..e3706590b30 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java @@ -43,7 +43,6 @@ import org.eclipse.cdt.core.parser.NullLogService; import org.eclipse.cdt.core.parser.NullSourceElementRequestor; import org.eclipse.cdt.core.parser.OffsetLimitReachedException; import org.eclipse.cdt.core.parser.ParserFactory; -import org.eclipse.cdt.core.parser.ParserFactoryError; import org.eclipse.cdt.core.parser.ParserLanguage; import org.eclipse.cdt.core.parser.ParserMode; import org.eclipse.cdt.core.parser.ScannerException; @@ -60,6 +59,8 @@ import org.eclipse.cdt.internal.core.parser.InternalParserUtil; import org.eclipse.cdt.internal.core.parser.ast.ASTCompletionNode; import org.eclipse.cdt.internal.core.parser.ast.EmptyIterator; import org.eclipse.cdt.internal.core.parser.problem.IProblemFactory; +import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionDirective; +import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionParseException; import org.eclipse.cdt.internal.core.parser.token.KeywordSetKey; import org.eclipse.cdt.internal.core.parser.token.KeywordSets; import org.eclipse.cdt.internal.core.parser.token.SimpleToken; @@ -74,7 +75,7 @@ import org.eclipse.cdt.internal.core.parser.util.TraceUtil; public final class Scanner implements IScanner, IScannerData { protected static final EndOfFileException EOF = new EndOfFileException(); - static ScannerStringBuffer strbuff = new ScannerStringBuffer(100); + private ScannerStringBuffer strbuff = new ScannerStringBuffer(100); protected static final String HEX_PREFIX = "0x"; //$NON-NLS-1$ private static final ObjectMacroDescriptor CPLUSPLUS_MACRO = new ObjectMacroDescriptor( __CPLUSPLUS, "199711L"); //$NON-NLS-1$ private static final ObjectMacroDescriptor STDC_VERSION_MACRO = new ObjectMacroDescriptor( __STDC_VERSION__, "199001L"); //$NON-NLS-1$ @@ -85,12 +86,12 @@ public final class Scanner implements IScanner, IScannerData { private final List workingCopies; protected final ContextStack contextStack; private IASTFactory astFactory = null; - private final ISourceElementRequestor requestor; - private final ParserMode parserMode; + private ISourceElementRequestor requestor; + private ParserMode parserMode; private final String filename; private final Reader reader; private final ParserLanguage language; - protected final IParserLogService log; + protected IParserLogService log; private final IProblemFactory problemFactory = new ScannerProblemFactory(); private Map definitions = new Hashtable(); private BranchTracker branches = new BranchTracker(); @@ -780,10 +781,15 @@ public final class Scanner implements IScanner, IScannerData { if (c != NOCHAR) return c; + if (currentContext.isFinal()) + return c; + while (contextStack.rollbackContext(requestor)) { c = currentContext.getChar(); if (c != NOCHAR) return c; + if (currentContext.isFinal()) + return c; } return NOCHAR; @@ -1571,14 +1577,15 @@ public final class Scanner implements IScanner, IScannerData { return null; } - IMacroDescriptor mapping = getDefinition(ident); - - if (mapping != null && !isLimitReached() && !mapping.isCircular() ) - if( contextStack.shouldExpandDefinition( ident ) ) { - expandDefinition(ident, mapping, baseOffset); - return null; - } - + if (!disableMacroExpansion) { + IMacroDescriptor mapping = getDefinition(ident); + + if (mapping != null && !isLimitReached() && !mapping.isCircular() ) + if( contextStack.shouldExpandDefinition( ident ) ) { + expandDefinition(ident, mapping, baseOffset); + return null; + } + } if( pasting && pasteIntoInputStream(ident)) return null; @@ -2531,9 +2538,51 @@ public final class Scanner implements IScanner, IScannerData { return branches.getDepth(); } - protected boolean evaluateExpression(String expression, int beginningOffset ) + protected boolean evaluateExpressionNew(String expression, int beginningOffset ) throws ScannerException { - + + // TODO John HELP! something has changed. If I turn this to true, My tests finish early (but the JUnits pass!) + IScannerContext context = new ScannerContextTopString(expression, EXPRESSION, ';', true); + contextStack.cs_push(context); + + ISourceElementRequestor savedRequestor = requestor; + IParserLogService savedLog = log; + log = NULL_LOG_SERVICE; + requestor = NULL_REQUESTOR; + + + boolean savedPassOnToClient = passOnToClient; + ParserMode savedParserMode = parserMode; + IASTFactory savedFactory = astFactory; + + + passOnToClient = true; + parserMode = ParserMode.QUICK_PARSE; + + IExpressionParser parser = InternalParserUtil.createExpressionParser(this, language, NULL_LOG_SERVICE); + try { + IASTExpression exp = parser.expression(null, null, null); + return (exp.evaluateExpression() != 0); + } catch( BacktrackException backtrack ) + { + return false; + } catch (ASTExpressionEvaluationException e) { + return false; + } catch (EndOfFileException e) { + return false; + } finally { + contextStack.cs_pop(); + requestor = savedRequestor; + passOnToClient = savedPassOnToClient; + parserMode = savedParserMode; + astFactory = savedFactory; + log = savedLog; + } + } + + protected boolean evaluateExpressionOld(String expression, int beginningOffset ) + throws ScannerException { + IExpressionParser parser = null; strbuff.startString(); strbuff.append(expression); @@ -2553,9 +2602,7 @@ public final class Scanner implements IScanner, IScannerData { parser = InternalParserUtil.createExpressionParser(trial, language, NULL_LOG_SERVICE); try { IASTExpression exp = parser.expression(null, null, null); - if( exp.evaluateExpression() == 0 ) - return false; - return true; + return (exp.evaluateExpression() != 0); } catch( BacktrackException backtrack ) { if( parserMode == ParserMode.QUICK_PARSE ) @@ -2572,6 +2619,21 @@ public final class Scanner implements IScanner, IScannerData { handleProblem( IProblem.PREPROCESSOR_CONDITIONAL_EVAL_ERROR, expression, beginningOffset, false, true ); } return true; + + } + protected boolean evaluateExpression(String expression, int beginningOffset ) + throws ScannerException { + +// boolean old_e = evaluateExpressionOld(expression, beginningOffset); + boolean new_e = evaluateExpressionNew(expression, beginningOffset); + +// if (old_e != new_e) { +// System.out.println("Ouch " + expression + " New: " + new_e + " Old: " + old_e); +// } +// if (true) + return new_e; +// else +// return old_e; } @@ -2632,7 +2694,93 @@ public final class Scanner implements IScanner, IScannerData { return encounteredNewline; } + + private static final InclusionParseException INCLUSION_PARSE_EXCEPTION = new InclusionParseException(); + public InclusionDirective parseInclusionDirective( String includeLine, int baseOffset ) throws InclusionParseException + { + if (includeLine.equals("")) //$NON-NLS-1$ + throw INCLUSION_PARSE_EXCEPTION ; + + ISourceElementRequestor savedRequestor = requestor; + try + { + IScannerContext context = new ScannerContextTopString( includeLine, "INCLUDE", true ); //$NON-NLS-1$ + contextStack.cs_push(context); + requestor = NULL_REQUESTOR; + + boolean useIncludePath = true; + StringBuffer localStringBuff = new StringBuffer(100); + int startOffset = baseOffset, endOffset = baseOffset; + + IToken t = null; + + try { + t = nextToken(false); + } catch (EndOfFileException eof) { + throw INCLUSION_PARSE_EXCEPTION ; + } + + try { + if (t.getType() == IToken.tSTRING) { + localStringBuff.append(t.getImage()); + startOffset = baseOffset + t.getOffset(); + endOffset = baseOffset + t.getEndOffset(); + useIncludePath = false; + + // This should throw EOF + t = nextToken(false); + contextStack.cs_pop(); + requestor = savedRequestor; + throw INCLUSION_PARSE_EXCEPTION ; + } else if (t.getType() == IToken.tLT) { + disableMacroExpansion = true; + try { + + t = nextToken(false); + startOffset = baseOffset + t.getOffset(); + + while (t.getType() != IToken.tGT) { + localStringBuff.append(t.getImage()); + skipOverWhitespace(); + int c = getChar(); + if (c == '\\') + localStringBuff.append('\\'); + else + ungetChar(c); + t = nextToken(false); + } + + endOffset = baseOffset + t.getEndOffset(); + + } catch (EndOfFileException eof) { + throw INCLUSION_PARSE_EXCEPTION ; + } + + // This should throw EOF + t = nextToken(false); + + throw INCLUSION_PARSE_EXCEPTION ; + + } else + throw INCLUSION_PARSE_EXCEPTION ; + } + catch( EndOfFileException eof ) + { + // good + } + + return new InclusionDirective( localStringBuff.toString(), useIncludePath, startOffset, endOffset ); + } + catch( ScannerException se ) + { + throw INCLUSION_PARSE_EXCEPTION ; + } finally { + contextStack.cs_pop(); + requestor = savedRequestor; + disableMacroExpansion = false; + } + } protected void poundInclude( int beginningOffset, int startLine ) throws ScannerException, EndOfFileException { skipOverWhitespace(); int baseOffset = lastContext.getOffset() ; @@ -2646,7 +2794,7 @@ public final class Scanner implements IScanner, IScannerData { ScannerUtility.InclusionDirective directive = null; try { - directive = ScannerUtility.parseInclusionDirective( this, scannerExtension, includeLine, baseOffset ); + directive = parseInclusionDirective( includeLine, baseOffset ); } catch( ScannerUtility.InclusionParseException ipe ) { @@ -2717,40 +2865,69 @@ public final class Scanner implements IScanner, IScannerData { forInclusion = b; } + public boolean disableMacroExpansion = false; + protected IToken[] tokenizeReplacementString( int beginning, String key, String replacementString, String[] parameterIdentifiers ) { if( replacementString.trim().equals( "" ) ) //$NON-NLS-1$ return EMPTY_TOKEN_ARRAY; IToken [] macroReplacementTokens = getTokenBuffer(); int currentIndex = 0; - IScanner helperScanner=null; - try { - helperScanner = new Scanner( - new StringReader(replacementString), - SCRATCH, - getTemporaryHashtable(), Collections.EMPTY_LIST, - NULL_REQUESTOR, - parserMode, - language, - NULL_LOG_SERVICE, scannerExtension); - } catch (ParserFactoryError e1) { - } - helperScanner.setTokenizingMacroReplacementList( true ); - IToken t = null; - try { - t = helperScanner.nextToken(false); - } catch (ScannerException e) { - } catch (EndOfFileException e) { - } + IScannerContext context = new ScannerContextTopString(replacementString, SCRATCH, true); + contextStack.cs_push(context); + ISourceElementRequestor savedRequestor = requestor; + IParserLogService savedLog = log; - if( t == null ) - return EMPTY_TOKEN_ARRAY; + setTokenizingMacroReplacementList( true ); + disableMacroExpansion = true; + requestor = NULL_REQUESTOR; + log = NULL_LOG_SERVICE; try { - while (true) { - //each # preprocessing token in the replacement list shall be followed - //by a parameter as the next reprocessing token in the list - if( t.getType() == tPOUND ){ + IToken t = null; + try { + t = nextToken(false); + } catch (ScannerException e) { + } catch (EndOfFileException e) { + } + + if( t == null ) + return EMPTY_TOKEN_ARRAY; + + try { + while (true) { + //each # preprocessing token in the replacement list shall be followed + //by a parameter as the next reprocessing token in the list + if( t.getType() == tPOUND ){ + if( currentIndex == macroReplacementTokens.length ) + { + IToken [] doubled = new IToken[macroReplacementTokens.length * 2]; + System.arraycopy( macroReplacementTokens, 0, doubled, 0, macroReplacementTokens.length ); + macroReplacementTokens = doubled; + } + macroReplacementTokens[currentIndex++] = t; + t = nextToken(false); + if( parameterIdentifiers != null ) + { + int index = findIndex( parameterIdentifiers, t.getImage()); + if (index == -1 ) { + //not found + + if( beginning != NO_OFFSET_LIMIT ) + { + strbuff.startString(); + strbuff.append( POUND_DEFINE ); + strbuff.append( key ); + strbuff.append( ' ' ); + strbuff.append( replacementString ); + handleProblem( IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, strbuff.toString(), + beginning, false, true ); + return EMPTY_TOKEN_ARRAY; + } + } + } + } + if( currentIndex == macroReplacementTokens.length ) { IToken [] doubled = new IToken[macroReplacementTokens.length * 2]; @@ -2758,48 +2935,27 @@ public final class Scanner implements IScanner, IScannerData { macroReplacementTokens = doubled; } macroReplacementTokens[currentIndex++] = t; - t = helperScanner.nextToken(false); - if( parameterIdentifiers != null ) - { - int index = findIndex( parameterIdentifiers, t.getImage()); - if (index == -1 ) { - //not found - - if( beginning != NO_OFFSET_LIMIT ) - { - strbuff.startString(); - strbuff.append( POUND_DEFINE ); - strbuff.append( key ); - strbuff.append( ' ' ); - strbuff.append( replacementString ); - handleProblem( IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, strbuff.toString(), - beginning, false, true ); - return EMPTY_TOKEN_ARRAY; - } - } - } + t = nextToken(false); } - - if( currentIndex == macroReplacementTokens.length ) - { - IToken [] doubled = new IToken[macroReplacementTokens.length * 2]; - System.arraycopy( macroReplacementTokens, 0, doubled, 0, macroReplacementTokens.length ); - macroReplacementTokens = doubled; - } - macroReplacementTokens[currentIndex++] = t; - t = helperScanner.nextToken(false); } + catch( EndOfFileException eof ) + { + } + catch( ScannerException sc ) + { + } + + IToken [] result = new IToken[ currentIndex ]; + System.arraycopy( macroReplacementTokens, 0, result, 0, currentIndex ); + return result; } - catch( EndOfFileException eof ) - { + finally { + contextStack.cs_pop(); + setTokenizingMacroReplacementList( false ); + requestor = savedRequestor; + log = savedLog; + disableMacroExpansion = false; } - catch( ScannerException sc ) - { - } - - IToken [] result = new IToken[ currentIndex ]; - System.arraycopy( macroReplacementTokens, 0, result, 0, currentIndex ); - return result; } /** @@ -3023,31 +3179,29 @@ public final class Scanner implements IScanner, IScannerData { } parameters.add(strbuff.toString()); + setThrowExceptionOnBadCharacterRead(false); + ISourceElementRequestor savedRequestor = requestor; + IParserLogService savedLog = log; + log = NULL_LOG_SERVICE; + requestor = NULL_REQUESTOR; + + Vector parameterValues = new Vector(); for (int i = 0; i < parameters.size(); i++) { - Scanner tokenizer = new Scanner( - new StringReader((String)parameters.elementAt(i)), - TEXT, - definitions, - Collections.EMPTY_LIST, - NULL_REQUESTOR, - parserMode, - language, - NULL_LOG_SERVICE, - scannerExtension ); - tokenizer.setThrowExceptionOnBadCharacterRead(false); + IScannerContext context = new ScannerContextTopString((String)parameters.elementAt(i), TEXT, true); + contextStack.cs_push(context); IToken t = null; StringBuffer strBuff2 = new StringBuffer(); boolean space = false; try { while (true) { - int c = tokenizer.getCharacter(); + int c = getCharacter(); if ((c != ' ') && (c != '\t') && (c != '\r') && (c != '\n')) { space = false; } - if (c != NOCHAR) tokenizer.ungetChar(c); - t = (forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false)); + if (c != NOCHAR) ungetChar(c); + t = (forStringizing ? nextTokenForStringizing() : nextToken(false)); if (space) strBuff2.append( ' ' ); @@ -3077,10 +3231,13 @@ public final class Scanner implements IScanner, IScannerData { } catch (EndOfFileException e) { // Good + contextStack.cs_pop(); parameterValues.add(strBuff2.toString()); } } - + setThrowExceptionOnBadCharacterRead(true); + requestor = savedRequestor; + log = savedLog; return parameterValues; } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextInclusion.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextInclusion.java index b9a9ab3533d..cde6e014702 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextInclusion.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextInclusion.java @@ -37,6 +37,8 @@ public class ScannerContextInclusion implements IScannerContext this.index = index; } + public boolean isFinal() { return false; } + public final String getContextName() { return filename; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextMacro.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextMacro.java index cf9bbbcd2d7..a5e8d76d2c4 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextMacro.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextMacro.java @@ -32,6 +32,8 @@ public class ScannerContextMacro implements IScannerContext macroLength = mL; } + public boolean isFinal() { return false; } + public final String getContextName() { return macroName; } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextTopString.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextTopString.java new file mode 100644 index 00000000000..c4e470485a4 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextTopString.java @@ -0,0 +1,113 @@ +/******************************************************************************* + * Copyright (c) 2001, 2004 IBM - Rational Software and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM - Rational Software - initial implementation + * + */ +package org.eclipse.cdt.internal.core.parser.scanner; + +/** + * @author ddaoust + * + */ +public class ScannerContextTopString implements IScannerContext { + int position=0; + int line = 1; + int length; + String reader; + String textName; + int extra = -1; + boolean stop = false; + + public ScannerContextTopString(String in, String contextName, boolean stop) + { + textName = contextName; + reader = in; + length = in.length(); + this.stop = stop; + } + public ScannerContextTopString(String in, String contextName, char e, boolean stop) + { + textName = contextName; + reader = in; + length = in.length(); + this.stop = stop; + extra = e; + } + + + public boolean isFinal() { return stop;} + public final String getContextName() { + return textName; + } + public int getChar() { + int c; + if (position < length) + c = reader.charAt(position++); + else if (position++ == length) + c = extra; + else c = -1; + if (c == '\n') line++; + return c; + } + + public void close() { + + } + + public int getFilenameIndex() { return 0; } + + public final void ungetChar(int c) + { + position--; + if (c == '\n') line--; + // may want to assert that reader.charAt(position) == c + } + + + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.IScannerContext#getOffset() + */ + public int getOffset() + { + // All the tokens generated by the macro expansion + // will have dimensions (offset and length) equal to the expanding symbol. + return position; + } + public int getLine() + { + return line; + } + + /** + * Returns the kind. + * @return int + */ + public int getKind() { + return IScannerContext.ContextKind.TOP; + } + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + public String toString() { + StringBuffer buffer = new StringBuffer(); + buffer.append( getContextName() ); + if (position <= length) { + buffer.append( reader.substring(0, position )); + buffer.append( '*' ); + buffer.append( reader.substring(position, length-position)); + if (extra != -1 ) buffer.append( (char)extra ); + } + else { + buffer.append( reader.substring(0, length )); + if ( extra != -1 ) buffer.append( (char)extra ); + buffer.append( '*' ); + } + return buffer.toString(); + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java index 8e766d0bc23..ed62ffca4f2 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java @@ -11,27 +11,17 @@ import java.io.File; import java.io.Reader; -import java.io.StringReader; - import java.util.Iterator; import java.util.Vector; import org.eclipse.cdt.core.parser.CodeReader; -import org.eclipse.cdt.core.parser.EndOfFileException; -import org.eclipse.cdt.core.parser.IParserLogService; import org.eclipse.cdt.core.parser.ISourceElementRequestor; -import org.eclipse.cdt.core.parser.IToken; -import org.eclipse.cdt.core.parser.NullLogService; -import org.eclipse.cdt.core.parser.NullSourceElementRequestor; -import org.eclipse.cdt.core.parser.ScannerException; -import org.eclipse.cdt.core.parser.extension.IScannerExtension; /** * @author jcamelon */ public class ScannerUtility { - - static ScannerStringBuffer strbuff = new ScannerStringBuffer(100); + static String reconcilePath(String originalPath ) { if( originalPath == null ) return null; originalPath = removeQuotes( originalPath ); @@ -51,7 +41,7 @@ public class ScannerUtility { else results.add( segment ); } - strbuff.startString(); + StringBuffer strbuff = new StringBuffer(128); for( int i = 0; i < results.size(); ++i ) { strbuff.append( (String)results.elementAt(i) ); @@ -69,7 +59,7 @@ public class ScannerUtility { private static String removeQuotes(String originalPath) { String [] segments = originalPath.split( "\""); //$NON-NLS-1$ if( segments.length == 1 ) return originalPath; - strbuff.startString(); + StringBuffer strbuff = new StringBuffer(); for( int i = 0; i < segments.length; ++ i ) if( segments[i] != null ) strbuff.append( segments[i]); @@ -139,91 +129,5 @@ public class ScannerUtility { static class InclusionParseException extends Exception { } - - private static final ISourceElementRequestor NULL_REQUESTOR = new NullSourceElementRequestor(); - private static final IParserLogService NULL_LOG_SERVICE = new NullLogService(); - private static final InclusionParseException INCLUSION_PARSE_EXCEPTION = new InclusionParseException(); - - static InclusionDirective parseInclusionDirective( IScannerData scannerData, IScannerExtension extension, String includeLine, int baseOffset ) throws InclusionParseException - { - try - { - boolean useIncludePath = true; - strbuff.startString(); - int startOffset = baseOffset, endOffset = baseOffset; - - if (! includeLine.equals("")) { //$NON-NLS-1$ - Scanner helperScanner = new Scanner( - new StringReader(includeLine), - null, - scannerData.getPublicDefinitions(), scannerData.getIncludePathNames(), - NULL_REQUESTOR, - scannerData.getParserMode(), - scannerData.getLanguage(), NULL_LOG_SERVICE, extension ); - helperScanner.setForInclusion( true ); - IToken t = null; - - try { - t = helperScanner.nextToken(false); - } catch (EndOfFileException eof) { - throw INCLUSION_PARSE_EXCEPTION ; - } - - try { - if (t.getType() == IToken.tSTRING) { - strbuff.append(t.getImage()); - startOffset = baseOffset + t.getOffset(); - endOffset = baseOffset + t.getEndOffset(); - useIncludePath = false; - - // This should throw EOF - t = helperScanner.nextToken(false); - throw INCLUSION_PARSE_EXCEPTION ; - } else if (t.getType() == IToken.tLT) { - - try { - - t = helperScanner.nextToken(false); - startOffset = baseOffset + t.getOffset(); - - while (t.getType() != IToken.tGT) { - strbuff.append(t.getImage()); - helperScanner.skipOverWhitespace(); - int c = helperScanner.getChar(); - if (c == '\\') - strbuff.append('\\'); - else - helperScanner.ungetChar(c); - t = helperScanner.nextToken(false); - } - - endOffset = baseOffset + t.getEndOffset(); - - } catch (EndOfFileException eof) { - throw INCLUSION_PARSE_EXCEPTION ; - } - - // This should throw EOF - t = helperScanner.nextToken(false); - throw INCLUSION_PARSE_EXCEPTION ; - - } else - throw INCLUSION_PARSE_EXCEPTION ; - } - catch( EndOfFileException eof ) - { - // good - } - - } else - throw INCLUSION_PARSE_EXCEPTION ; - - return new InclusionDirective( strbuff.toString(), useIncludePath, startOffset, endOffset ); - } - catch( ScannerException se ) - { - throw INCLUSION_PARSE_EXCEPTION ; - } - - } + }