Patch for David Daoust - remove all instances of sub-Scanners in the scanner. Performance gains abound.

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java

@@ -1156,7 +1156,7 @@ public class QuickParseASTTests extends BaseASTTest
 		
 		String code = "#include <stdio.h>\n#define DEF VALUE\n"; //$NON-NLS-1$
 		
-		IASTCompilationUnit tu = parse( code.toString()  );
+		IASTCompilationUnit tu = parse( code  );
 		assertFalse( tu.getDeclarations().hasNext());
 		Iterator inclusions = quickParseCallback.getInclusions();
 		Iterator macros = quickParseCallback.getMacros();
@@ -1166,8 +1166,8 @@ public class QuickParseASTTests extends BaseASTTest
 		
 		assertEquals( i.getName(), "stdio.h"); //$NON-NLS-1$
 		assertEquals( i.getStartingOffset(), 0 ); 
-		assertEquals( i.getNameOffset(), 10 ); 
+		assertEquals( i.getNameOffset(), code.indexOf("stdio.h") );  //$NON-NLS-1$
-		assertEquals( i.getEndingOffset(), 19 );
+		assertEquals( i.getEndingOffset(), code.indexOf(">") + 1); //$NON-NLS-1$
 		
 		
 		IASTMacro m = (IASTMacro)macros.next();

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ContextStack.java

@@ -33,6 +33,7 @@ public class ContextStack {
 		public String getContextName() { return ""; } //$NON-NLS-1$
 		public int getOffset() { return 0; }
 		public void ungetChar(int undo) { }
+		public boolean isFinal() { return false; }
 		public int getKind() { return IScannerContext.ContextKind.SENTINEL; }
 		public void close() { }
 	}
@@ -83,7 +84,7 @@ public class ContextStack {
 	 
 	private IScanner scanner;
 
-	private final void cs_push(IScannerContext c) {
+	public final void cs_push(IScannerContext c) {
 		try {
 			cs[cs_pos++] = c;
 		}
@@ -98,7 +99,7 @@ public class ContextStack {
 		}
 		scanner.setScannerContext(c);
 	}
-	private final IScannerContext cs_pop() {
+	public final IScannerContext cs_pop() {
 		IScannerContext context = cs[--cs_pos];
 		scanner.setScannerContext((cs_pos == 0) ? sentinel : cs[cs_pos -1]);
 		return context;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/GCCScannerExtension.java

@@ -158,19 +158,19 @@ public class GCCScannerExtension implements IScannerExtension {
 	/* (non-Javadoc)
 	 * @see org.eclipse.cdt.core.parser.extension.IScannerExtension#handlePreprocessorDirective(java.lang.String, java.lang.String)
 	 */
-	public void handlePreprocessorDirective(IScannerData scannerData, String directive, String restOfLine) {
+	public void handlePreprocessorDirective(IScannerData iscanner, String directive, String restOfLine) {
 		if( directive.equals(POUND_INCLUDE_NEXT) ) 
 		{
-			TraceUtil.outputTrace(scannerData.getLogService(), "GCCScannerExtension handling #include_next directive", null, null, null, null); //$NON-NLS-1$
+			TraceUtil.outputTrace(iscanner.getLogService(), "GCCScannerExtension handling #include_next directive", null, null, null, null); //$NON-NLS-1$
 			// figure out the name of the current file and its path
-			IScannerContext context = scannerData.getContextStack().getCurrentContext();
+			IScannerContext context = iscanner.getContextStack().getCurrentContext();
 			if( context == null || context.getKind() != IScannerContext.ContextKind.INCLUSION ) 
 				return;
 			
 			String fullInclusionPath = context.getContextName();
 			IASTInclusion inclusion = ((ScannerContextInclusion)context).getExtension();
 			
-			Iterator iter = scannerData.getIncludePathNames().iterator();
+			Iterator iter = iscanner.getIncludePathNames().iterator();
 			
 			while (iter.hasNext()) {
 				String path = (String)iter.next();
@@ -181,7 +181,7 @@ public class GCCScannerExtension implements IScannerExtension {
 			
 			ScannerUtility.InclusionDirective parsedDirective = null;
 			try {
-				parsedDirective = ScannerUtility.parseInclusionDirective( scannerData, this, restOfLine, scannerData.getContextStack().getCurrentContext().getOffset() );
+				parsedDirective = iscanner.parseInclusionDirective( restOfLine, iscanner.getContextStack().getCurrentContext().getOffset() );
 			} catch (InclusionParseException e) {
 				return;
 			}
@@ -189,7 +189,7 @@ public class GCCScannerExtension implements IScannerExtension {
 			// search through include paths
 			while (iter.hasNext()) {	
 				String path = (String)iter.next();
-				duple = ScannerUtility.createReaderDuple( path, parsedDirective.getFilename(), scannerData.getClientRequestor(), scannerData.getWorkingCopies() );
+				duple = ScannerUtility.createReaderDuple( path, parsedDirective.getFilename(), iscanner.getClientRequestor(), iscanner.getWorkingCopies() );
 				if( duple != null )
 					break;
 			}
@@ -198,8 +198,8 @@ public class GCCScannerExtension implements IScannerExtension {
 			{
 				try			
 				{
-					scannerData.getContextStack().updateInclusionContext(duple.getUnderlyingReader(), duple.getFilename(), inclusion, scannerData.getClientRequestor() );
+					iscanner.getContextStack().updateInclusionContext(duple.getUnderlyingReader(), duple.getFilename(), inclusion, iscanner.getClientRequestor() );
-					TraceUtil.outputTrace( scannerData.getLogService(), "GCCScannerExtension handling #include_next directive successfully pushed on new include file" ); //$NON-NLS-1$
+					TraceUtil.outputTrace( iscanner.getLogService(), "GCCScannerExtension handling #include_next directive successfully pushed on new include file" ); //$NON-NLS-1$
 				}
 				catch (ContextException e1)
 				{

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IScannerContext.java

@@ -29,6 +29,7 @@ public interface IScannerContext {
 	public int getChar();
 	public void ungetChar(int undo);
 	
+	public boolean isFinal();
 	public String getContextName();
 	public int getOffset();   
 	public void close();

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IScannerData.java

@@ -23,6 +23,8 @@ import org.eclipse.cdt.core.parser.ParserLanguage;
 import org.eclipse.cdt.core.parser.ParserMode;
 import org.eclipse.cdt.core.parser.ast.IASTFactory;
 import org.eclipse.cdt.internal.core.parser.problem.IProblemFactory;
+import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionDirective;
+import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionParseException;
 
 /**
  * @author jcamelon
@@ -84,4 +86,10 @@ public interface IScannerData {
 	public abstract void setDefinitions(Map map);
 	
 	public Iterator getWorkingCopies();
+	/**
+	 * @param restOfLine
+	 * @param offset
+	 * @return
+	 */
+	public abstract InclusionDirective parseInclusionDirective(String restOfLine, int offset) throws InclusionParseException;
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java

@@ -43,7 +43,6 @@ import org.eclipse.cdt.core.parser.NullLogService;
 import org.eclipse.cdt.core.parser.NullSourceElementRequestor;
 import org.eclipse.cdt.core.parser.OffsetLimitReachedException;
 import org.eclipse.cdt.core.parser.ParserFactory;
-import org.eclipse.cdt.core.parser.ParserFactoryError;
 import org.eclipse.cdt.core.parser.ParserLanguage;
 import org.eclipse.cdt.core.parser.ParserMode;
 import org.eclipse.cdt.core.parser.ScannerException;
@@ -60,6 +59,8 @@ import org.eclipse.cdt.internal.core.parser.InternalParserUtil;
 import org.eclipse.cdt.internal.core.parser.ast.ASTCompletionNode;
 import org.eclipse.cdt.internal.core.parser.ast.EmptyIterator;
 import org.eclipse.cdt.internal.core.parser.problem.IProblemFactory;
+import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionDirective;
+import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionParseException;
 import org.eclipse.cdt.internal.core.parser.token.KeywordSetKey;
 import org.eclipse.cdt.internal.core.parser.token.KeywordSets;
 import org.eclipse.cdt.internal.core.parser.token.SimpleToken;
@@ -74,7 +75,7 @@ import org.eclipse.cdt.internal.core.parser.util.TraceUtil;
 public final class Scanner implements IScanner, IScannerData {
 
 	protected static final EndOfFileException EOF = new EndOfFileException();
-	static ScannerStringBuffer strbuff = new ScannerStringBuffer(100);
+	private ScannerStringBuffer strbuff = new ScannerStringBuffer(100);
 	protected static final String HEX_PREFIX = "0x"; //$NON-NLS-1$
 	private static final ObjectMacroDescriptor CPLUSPLUS_MACRO = new ObjectMacroDescriptor( __CPLUSPLUS, "199711L"); //$NON-NLS-1$
 	private static final ObjectMacroDescriptor STDC_VERSION_MACRO = new ObjectMacroDescriptor( __STDC_VERSION__, "199001L"); //$NON-NLS-1$
@@ -85,12 +86,12 @@ public final class Scanner implements IScanner, IScannerData {
 	private final List workingCopies;
 	protected final ContextStack contextStack;
 	private IASTFactory astFactory = null;
-	private final ISourceElementRequestor requestor;
+	private ISourceElementRequestor requestor;
-	private final ParserMode parserMode;
+	private ParserMode parserMode;
 	private final String filename;
 	private final Reader reader;
 	private final ParserLanguage language;
-	protected final IParserLogService log;
+	protected IParserLogService log;
 	private final IProblemFactory problemFactory = new ScannerProblemFactory();
 	private Map definitions = new Hashtable();
 	private BranchTracker branches = new BranchTracker();
@@ -780,10 +781,15 @@ public final class Scanner implements IScanner, IScannerData {
     	if (c != NOCHAR)
     		return c;
     	
+    	if (currentContext.isFinal())
+    		return c;
+    	
     	while (contextStack.rollbackContext(requestor)) {
     		c = currentContext.getChar();
     		if (c != NOCHAR)
     			return c;
+    		if (currentContext.isFinal())
+        		return c;
     	}
 
     	return NOCHAR;
@@ -1571,6 +1577,7 @@ public final class Scanner implements IScanner, IScannerData {
 			return null;
 		}
 			
+		if (!disableMacroExpansion) {
 			IMacroDescriptor mapping = getDefinition(ident);
 	
 			if (mapping != null && !isLimitReached() && !mapping.isCircular() )
@@ -1578,7 +1585,7 @@ public final class Scanner implements IScanner, IScannerData {
 					expandDefinition(ident, mapping, baseOffset);
 					return null;
 				}
-
+		}
 		if( pasting && pasteIntoInputStream(ident))
 			return null;
 		
@@ -2531,7 +2538,49 @@ public final class Scanner implements IScanner, IScannerData {
 		return branches.getDepth(); 
 	}
 
-	protected boolean evaluateExpression(String expression, int beginningOffset )
+	protected boolean evaluateExpressionNew(String expression, int beginningOffset ) 
+		throws ScannerException {
+		  
+		// TODO John HELP!  something has changed.   If I turn this to true, My tests finish early (but the JUnits pass!)
+		IScannerContext context = new ScannerContextTopString(expression, EXPRESSION, ';', true);
+		contextStack.cs_push(context);
+
+		ISourceElementRequestor savedRequestor =  requestor;
+		IParserLogService savedLog = log;
+		log = NULL_LOG_SERVICE;
+		requestor = NULL_REQUESTOR;
+		
+		
+		boolean savedPassOnToClient = passOnToClient;
+		ParserMode savedParserMode = parserMode;
+		IASTFactory savedFactory = astFactory;
+
+		
+		passOnToClient = true;
+		parserMode = ParserMode.QUICK_PARSE;
+		
+		IExpressionParser parser = InternalParserUtil.createExpressionParser(this, language, NULL_LOG_SERVICE);
+		try {
+			IASTExpression exp = parser.expression(null, null, null);
+			return (exp.evaluateExpression() != 0);
+		} catch( BacktrackException backtrack  )
+		{
+				return false;
+		} catch (ASTExpressionEvaluationException e) {
+				return false;			
+		} catch (EndOfFileException e) {
+				return false;
+		} finally {
+			contextStack.cs_pop();
+			requestor = savedRequestor;
+			passOnToClient = savedPassOnToClient;
+			parserMode = savedParserMode;
+			astFactory = savedFactory;
+			log = savedLog;
+		}
+	}
+	
+	protected boolean evaluateExpressionOld(String expression, int beginningOffset )
 		throws ScannerException {
 
 		IExpressionParser parser = null;
@@ -2553,9 +2602,7 @@ public final class Scanner implements IScanner, IScannerData {
         parser = InternalParserUtil.createExpressionParser(trial, language, NULL_LOG_SERVICE);
 		try {
 			IASTExpression exp = parser.expression(null, null, null);
-			if( exp.evaluateExpression() == 0 )
+			return (exp.evaluateExpression() != 0);
-				return false;
-			return true;
 		} catch( BacktrackException backtrack  )
 		{
 			if( parserMode == ParserMode.QUICK_PARSE )
@@ -2572,6 +2619,21 @@ public final class Scanner implements IScanner, IScannerData {
 			handleProblem( IProblem.PREPROCESSOR_CONDITIONAL_EVAL_ERROR, expression, beginningOffset, false, true );
 		}
 		return true; 
+	
+	}
+	protected boolean evaluateExpression(String expression, int beginningOffset )
+		throws ScannerException {
+
+//		boolean old_e = evaluateExpressionOld(expression, beginningOffset);
+		boolean new_e = evaluateExpressionNew(expression, beginningOffset);
+
+//		if (old_e != new_e) {
+//			System.out.println("Ouch " + expression + " New: " + new_e + " Old: " + old_e);
+//		}
+//		if (true)
+			return new_e;
+//		else 
+//			return old_e;
 	}
 
 	
@@ -2633,6 +2695,92 @@ public final class Scanner implements IScanner, IScannerData {
 		return encounteredNewline;
 	}
 	
+	private static final InclusionParseException INCLUSION_PARSE_EXCEPTION  = new InclusionParseException(); 
+
+	public InclusionDirective parseInclusionDirective( String includeLine, int baseOffset ) throws InclusionParseException 
+	{
+		if (includeLine.equals(""))  //$NON-NLS-1$
+			throw INCLUSION_PARSE_EXCEPTION ;
+		
+		ISourceElementRequestor savedRequestor = requestor;
+		try
+		{
+			IScannerContext context = new ScannerContextTopString( includeLine, "INCLUDE", true );  //$NON-NLS-1$
+			contextStack.cs_push(context);
+			requestor = NULL_REQUESTOR;
+			
+			boolean useIncludePath = true;
+			StringBuffer localStringBuff = new StringBuffer(100);
+			int startOffset = baseOffset, endOffset = baseOffset;
+
+			IToken t = null;
+			
+			try {
+				t = nextToken(false);
+			} catch (EndOfFileException eof) {
+				throw INCLUSION_PARSE_EXCEPTION ;
+			} 
+
+			try {
+				if (t.getType() == IToken.tSTRING) {
+					localStringBuff.append(t.getImage());
+					startOffset = baseOffset + t.getOffset();
+					endOffset = baseOffset + t.getEndOffset();
+					useIncludePath = false;
+					
+					// This should throw EOF
+					t = nextToken(false);
+					contextStack.cs_pop();
+					requestor = savedRequestor;
+					throw INCLUSION_PARSE_EXCEPTION ;
+				} else if (t.getType() == IToken.tLT) {
+					disableMacroExpansion = true;
+					try {
+											
+						t = nextToken(false);
+						startOffset = baseOffset + t.getOffset();
+						
+						while (t.getType() != IToken.tGT) {
+							localStringBuff.append(t.getImage());
+							skipOverWhitespace();
+							int c = getChar();
+							if (c == '\\') 
+								localStringBuff.append('\\'); 
+							else 
+								ungetChar(c);
+							t = nextToken(false);
+						}
+						
+						endOffset = baseOffset + t.getEndOffset();
+						
+					} catch (EndOfFileException eof) {
+						throw INCLUSION_PARSE_EXCEPTION ;
+					}
+					
+					// This should throw EOF
+					t = nextToken(false);
+
+					throw INCLUSION_PARSE_EXCEPTION ;
+					
+				} else 
+					throw INCLUSION_PARSE_EXCEPTION ;
+			}
+			catch( EndOfFileException eof )
+			{
+				// good
+			} 
+			
+			return new InclusionDirective( localStringBuff.toString(), useIncludePath, startOffset, endOffset );
+		}
+		catch( ScannerException se )
+		{
+			throw INCLUSION_PARSE_EXCEPTION ;
+		} finally {
+			contextStack.cs_pop();
+			requestor = savedRequestor;
+			disableMacroExpansion = false;
+		}
+	}
 	protected void poundInclude( int beginningOffset, int startLine ) throws ScannerException, EndOfFileException {
 		skipOverWhitespace();				
 		int baseOffset = lastContext.getOffset() ;
@@ -2646,7 +2794,7 @@ public final class Scanner implements IScanner, IScannerData {
 		ScannerUtility.InclusionDirective directive = null;
 		try
 		{
-			directive = ScannerUtility.parseInclusionDirective( this, scannerExtension, includeLine, baseOffset );
+			directive = parseInclusionDirective( includeLine, baseOffset );
 		}
 		catch( ScannerUtility.InclusionParseException ipe )
 		{
@@ -2717,28 +2865,28 @@ public final class Scanner implements IScanner, IScannerData {
 		forInclusion = b;
 	}
 
+	public boolean disableMacroExpansion = false;
+	
 	protected IToken[] tokenizeReplacementString( int beginning, String key, String replacementString, String[] parameterIdentifiers ) 
 	{
 		if( replacementString.trim().equals( "" ) )  //$NON-NLS-1$
 			return EMPTY_TOKEN_ARRAY;
 		IToken [] macroReplacementTokens = getTokenBuffer();
 		int currentIndex = 0;
-		IScanner helperScanner=null;
+		IScannerContext context = new ScannerContextTopString(replacementString, SCRATCH, true);
+		contextStack.cs_push(context);
+		ISourceElementRequestor savedRequestor =  requestor;
+		IParserLogService savedLog = log;
+		
+		setTokenizingMacroReplacementList( true );
+		disableMacroExpansion = true;
+		requestor = NULL_REQUESTOR;
+		log = NULL_LOG_SERVICE;
+		
 		try {
-			helperScanner = new Scanner( 
-						new StringReader(replacementString),
-						SCRATCH,
-						getTemporaryHashtable(), Collections.EMPTY_LIST, 
-						NULL_REQUESTOR, 
-						parserMode,
-						language,
-						NULL_LOG_SERVICE, scannerExtension);
-		} catch (ParserFactoryError e1) {
-		}
-		helperScanner.setTokenizingMacroReplacementList( true );
 			IToken t = null;
 			try {
-			t = helperScanner.nextToken(false);
+				t = nextToken(false);
 			} catch (ScannerException e) {
 			} catch (EndOfFileException e) {
 			}
@@ -2758,7 +2906,7 @@ public final class Scanner implements IScanner, IScannerData {
 							macroReplacementTokens = doubled;
 						}
 						macroReplacementTokens[currentIndex++] = t;
-					t = helperScanner.nextToken(false);
+						t = nextToken(false);
 						if( parameterIdentifiers != null )
 						{	
 							int index = findIndex( parameterIdentifiers, t.getImage());
@@ -2787,7 +2935,7 @@ public final class Scanner implements IScanner, IScannerData {
 						macroReplacementTokens = doubled;
 					}
 					macroReplacementTokens[currentIndex++] = t;
-				t = helperScanner.nextToken(false);
+					t = nextToken(false);
 				}
 			}
 			catch( EndOfFileException eof )
@@ -2801,6 +2949,14 @@ public final class Scanner implements IScanner, IScannerData {
 			System.arraycopy( macroReplacementTokens, 0, result, 0, currentIndex );
 			return result;
 		}
+		finally {
+			contextStack.cs_pop();
+			setTokenizingMacroReplacementList( false );
+			requestor = savedRequestor;
+			log = savedLog;
+			disableMacroExpansion = false;
+		}
+	}
 	
 	/**
 	 * @return
@@ -3023,31 +3179,29 @@ public final class Scanner implements IScanner, IScannerData {
 		}
 		parameters.add(strbuff.toString());
 		
+		setThrowExceptionOnBadCharacterRead(false);
+		ISourceElementRequestor savedRequestor =  requestor;
+		IParserLogService savedLog = log;
+		log = NULL_LOG_SERVICE;
+		requestor = NULL_REQUESTOR;
+		
+		
         Vector parameterValues = new Vector();
 		for (int i = 0; i < parameters.size(); i++) {
-	        Scanner tokenizer  = new Scanner(
+			IScannerContext context = new ScannerContextTopString((String)parameters.elementAt(i), TEXT, true);
-	        		new StringReader((String)parameters.elementAt(i)), 
+	        contextStack.cs_push(context);     
-					TEXT, 
-					definitions,
-					Collections.EMPTY_LIST, 
-					NULL_REQUESTOR, 
-					parserMode,
-					language,
-					NULL_LOG_SERVICE, 
-					scannerExtension );
-	        tokenizer.setThrowExceptionOnBadCharacterRead(false);
 	        IToken t = null;
 	        StringBuffer strBuff2 = new StringBuffer();
 	        boolean space = false;
 	       
 	        try {
 	            while (true) {
-					int c = tokenizer.getCharacter();
+					int c = getCharacter();
 					if ((c != ' ') && (c != '\t') && (c != '\r') && (c != '\n')) {
 						space = false;
 					}
-					if (c != NOCHAR) tokenizer.ungetChar(c);
+					if (c != NOCHAR) ungetChar(c);
-					t = (forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false));
+					t = (forStringizing ? nextTokenForStringizing() : nextToken(false));
 	
 	                if (space)
 	                    strBuff2.append( ' ' );
@@ -3077,10 +3231,13 @@ public final class Scanner implements IScanner, IScannerData {
 	        }
 	        catch (EndOfFileException e) {
 	            // Good
+	        	contextStack.cs_pop();     	
 	            parameterValues.add(strBuff2.toString());
 	        }
 		}
-        
+		setThrowExceptionOnBadCharacterRead(true);
+		requestor = savedRequestor;
+		log = savedLog;
         return parameterValues;
     }
     

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextInclusion.java

@@ -37,6 +37,8 @@ public class ScannerContextInclusion implements IScannerContext
         this.index = index;
     }
 		
+    public boolean isFinal() { return false; }
+    
 	public final String getContextName()
 	{
 		return filename;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextMacro.java

@@ -32,6 +32,8 @@ public class ScannerContextMacro implements IScannerContext
         macroLength = mL;
 	}
     
+	public boolean isFinal() { return false; }
+
 	public final String getContextName() {
 		return macroName;
 	}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextTopString.java

@@ -0,0 +1,113 @@
+/*******************************************************************************
+ * Copyright (c) 2001, 2004 IBM - Rational Software and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors:
+ *     IBM - Rational Software - initial implementation
+ * 
+ */
+package org.eclipse.cdt.internal.core.parser.scanner;
+
+/**
+ * @author ddaoust
+ *
+ */
+public class ScannerContextTopString  implements IScannerContext {
+    int position=0;
+    int line = 1;
+    int length;
+    String reader;
+    String textName;
+    int extra = -1;
+    boolean stop = false;
+    
+	public ScannerContextTopString(String in, String contextName, boolean stop)
+	{
+		textName = contextName;
+		reader = in;
+		length = in.length();
+		this.stop = stop;
+	}
+	public ScannerContextTopString(String in, String contextName, char e, boolean stop)
+	{
+		textName = contextName;
+		reader = in;
+		length = in.length();
+		this.stop = stop;
+		extra = e;
+	}
+	
+    
+	public boolean isFinal() { return stop;}
+	public final String getContextName() {
+		return textName;
+	}
+	public int getChar() {
+		int c;
+		if (position < length)
+			c = reader.charAt(position++);
+		else if (position++ == length)
+			c = extra;
+		else c = -1;
+		if (c == '\n') line++;
+		return c;
+	}
+	
+	public void close() {
+		
+	}
+	
+	public int getFilenameIndex() { return 0; }
+	
+	public final void ungetChar(int c)
+	{
+		position--;
+		if (c == '\n') line--;
+		// may want to assert that reader.charAt(position) == c
+	}
+
+
+	/* (non-Javadoc)
+     * @see org.eclipse.cdt.internal.core.parser.IScannerContext#getOffset()
+     */
+	public int getOffset()
+	{
+        // All the tokens generated by the macro expansion 
+        // will have dimensions (offset and length) equal to the expanding symbol.
+		return position;
+	}
+	public int getLine()
+	{
+		return line;
+	}
+
+	/**
+	 * Returns the kind.
+	 * @return int
+	 */
+	public int getKind() {
+		return IScannerContext.ContextKind.TOP;
+	}
+	/* (non-Javadoc)
+	 * @see java.lang.Object#toString()
+	 */
+	public String toString() {
+		StringBuffer buffer = new StringBuffer();
+		buffer.append( getContextName() );
+		if (position <= length) {
+			buffer.append( reader.substring(0, position ));
+			buffer.append( '*' );
+			buffer.append( reader.substring(position, length-position));
+			if (extra != -1 ) buffer.append( (char)extra );
+		}
+		else {
+			buffer.append( reader.substring(0, length ));
+			if ( extra != -1 ) buffer.append( (char)extra );
+			buffer.append( '*' );
+		}
+		return buffer.toString();
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java

@@ -11,27 +11,17 @@
 
 import java.io.File;
 import java.io.Reader;
-import java.io.StringReader;
-
 import java.util.Iterator;
 import java.util.Vector;
 
 import org.eclipse.cdt.core.parser.CodeReader;
-import org.eclipse.cdt.core.parser.EndOfFileException;
-import org.eclipse.cdt.core.parser.IParserLogService;
 import org.eclipse.cdt.core.parser.ISourceElementRequestor;
-import org.eclipse.cdt.core.parser.IToken;
-import org.eclipse.cdt.core.parser.NullLogService;
-import org.eclipse.cdt.core.parser.NullSourceElementRequestor;
-import org.eclipse.cdt.core.parser.ScannerException;
-import org.eclipse.cdt.core.parser.extension.IScannerExtension;
 
 /**
  * @author jcamelon
  */
 public class ScannerUtility {
 	
-	static ScannerStringBuffer strbuff = new ScannerStringBuffer(100);
 	static String reconcilePath(String originalPath ) {
 		if( originalPath == null ) return null;
 		originalPath = removeQuotes( originalPath );
@@ -51,7 +41,7 @@ public class ScannerUtility {
 			else
 				results.add( segment );
 		}
-		strbuff.startString(); 
+		StringBuffer strbuff = new StringBuffer(128); 
 		for( int i = 0; i < results.size(); ++i )
 		{
 			strbuff.append( (String)results.elementAt(i) );
@@ -69,7 +59,7 @@ public class ScannerUtility {
 	private static String removeQuotes(String originalPath) {
 		String [] segments = originalPath.split( "\""); //$NON-NLS-1$
 		if( segments.length == 1 ) return originalPath;
-		strbuff.startString();
+		StringBuffer strbuff = new StringBuffer();
 		for( int i = 0; i < segments.length; ++ i )
 			if( segments[i] != null )
 				strbuff.append( segments[i]);
@@ -140,90 +130,4 @@ public class ScannerUtility {
 	{
 	}
 		
-	private static final ISourceElementRequestor NULL_REQUESTOR = new NullSourceElementRequestor();
-	private static final IParserLogService NULL_LOG_SERVICE = new NullLogService();
-	private static final InclusionParseException INCLUSION_PARSE_EXCEPTION  = new InclusionParseException(); 
-	
-	static InclusionDirective parseInclusionDirective( IScannerData scannerData, IScannerExtension extension, String includeLine, int baseOffset ) throws InclusionParseException 
-	{
-		try
-		{
-			boolean useIncludePath = true;
-			strbuff.startString();
-			int startOffset = baseOffset, endOffset = baseOffset;
-			
-			if (! includeLine.equals("")) { //$NON-NLS-1$
-				Scanner helperScanner = new Scanner(
-											new StringReader(includeLine), 
-											null, 
-											scannerData.getPublicDefinitions(), scannerData.getIncludePathNames(),
-											NULL_REQUESTOR,
-											scannerData.getParserMode(),
-											scannerData.getLanguage(), NULL_LOG_SERVICE, extension );
-				helperScanner.setForInclusion( true );
-				IToken t = null;
-				
-				try {
-					t = helperScanner.nextToken(false);
-				} catch (EndOfFileException eof) {
-					throw INCLUSION_PARSE_EXCEPTION ;
-				} 
-	
-				try {
-					if (t.getType() == IToken.tSTRING) {
-						strbuff.append(t.getImage());
-						startOffset = baseOffset + t.getOffset();
-						endOffset = baseOffset + t.getEndOffset();
-						useIncludePath = false;
-						
-						// This should throw EOF
-						t = helperScanner.nextToken(false);
-						throw INCLUSION_PARSE_EXCEPTION ;
-					} else if (t.getType() == IToken.tLT) {
-						
-						try {
-												
-							t = helperScanner.nextToken(false);
-							startOffset = baseOffset + t.getOffset();
-							
-							while (t.getType() != IToken.tGT) {
-								strbuff.append(t.getImage());
-								helperScanner.skipOverWhitespace();
-								int c = helperScanner.getChar();
-								if (c == '\\') 
-									strbuff.append('\\'); 
-								else 
-									helperScanner.ungetChar(c);
-								t = helperScanner.nextToken(false);
-							}
-							
-							endOffset = baseOffset + t.getEndOffset();
-							
-						} catch (EndOfFileException eof) {
-							throw INCLUSION_PARSE_EXCEPTION ;
-						}
-						
-						// This should throw EOF
-						t = helperScanner.nextToken(false);
-						throw INCLUSION_PARSE_EXCEPTION ;
-						
-					} else 
-						throw INCLUSION_PARSE_EXCEPTION ;
-				}
-				catch( EndOfFileException eof )
-				{
-					// good
-				} 
-				
-			} else
-				throw INCLUSION_PARSE_EXCEPTION ;
-	
-			return new InclusionDirective( strbuff.toString(), useIncludePath, startOffset, endOffset );
-		}
-		catch( ScannerException se )
-		{
-			throw INCLUSION_PARSE_EXCEPTION ;
-		}
-
-	}
 }