Patch for John Camelon:

- added in support for detecting and reporting circular inclusions
- added optimization by caching inclusion directories
- added macro pasting capabilities and tests
- updated inclusion searching algorithm for local inclusions

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/IScannerContext.java

@@ -1,6 +1,6 @@
 package org.eclipse.cdt.internal.core.parser;
-import java.io.Reader;
 import java.io.IOException;
+import java.io.Reader;
 /**
  * @author jcamelon
  *
@@ -11,11 +11,24 @@ import java.io.IOException;
  * Window>Preferences>Java>Code Generation.
  */
 public interface IScannerContext {
-	IScannerContext initialize(Reader r, String f, int u);
+	
+	public static int SENTINEL = 0;
+	public static int TOP = 1; 
+	public static int INCLUSION = 2; 
+	public static int MACROEXPANSION = 3; 
+
+	
+	IScannerContext initialize(Reader r, String f, int k);
 	int read() throws IOException;
 	String getFilename();
 	int getOffset();
 	Reader getReader();
-	int getUndo();
+	
-	void setUndo(int undo);
+	int undoStackSize();  
+	int popUndo();
+	void pushUndo(int undo);
+	
+	
+	int getKind(); 
+	void setKind( int kind ); 
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java

@@ -10,6 +10,7 @@
  ******************************************************************************/
 package org.eclipse.cdt.internal.core.parser;
 
+import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
@@ -17,9 +18,11 @@ import java.io.Reader;
 import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Hashtable;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Set;
 import java.util.Stack;
 import java.util.StringTokenizer;
 import java.util.Vector;
@@ -46,13 +49,13 @@ public class Scanner implements IScanner {
 			new ScannerContext().initialize(
 				new StringReader("\n"),
 				START,
-				NOCHAR));
+				ScannerContext.SENTINEL));
 		if (filename == null)
 			currentContext =
-				new ScannerContext().initialize(reader, TEXT, NOCHAR);
+				new ScannerContext().initialize(reader, TEXT, ScannerContext.TOP ); 
 		else
 			currentContext =
-				new ScannerContext().initialize(reader, filename, NOCHAR);
+				new ScannerContext().initialize(reader, filename, ScannerContext.TOP ); 
 	}
 
 	public Scanner() {
@@ -63,13 +66,19 @@ public class Scanner implements IScanner {
 		definitions = defns;
 	}
 
-	protected void updateContext(Reader reader, String filename) {
+	protected void updateContext(Reader reader, String filename, int type) throws ScannerException {
-//		if (callback != null)
+		if( type == IScannerContext.INCLUSION )
-//			callback.inclusionBegin(filename); // not quite right ... fix me!!!
+		{
+			if( !inclusions.add( filename ) )
+				throw new ScannerException( "Inclusion " + filename + " already encountered." ); 	
+				
+			System.out.println( "Handle inclusion - " + filename );	
+		}
 
 		contextStack.push(currentContext);
 		currentContext =
-			new ScannerContext().initialize(reader, filename, NOCHAR);
+			new ScannerContext().initialize(reader, filename, type );
+			
 	}
 
 	protected boolean rollbackContext() {
@@ -79,26 +88,41 @@ public class Scanner implements IScanner {
 			System.out.println("Error closing reader");
 		}
 
+		if( currentContext.getKind() == IScannerContext.INCLUSION )
+		{
+			inclusions.remove( currentContext.getFilename() );
+			System.out.println( "Completed inclusion - " + currentContext.getFilename() );
+		}
+
 		if (contextStack.isEmpty()) {
 			currentContext = null;
 			return false;
 		}
 
-		//if (callback != null)
-		//	callback.inclusionEnd();
-
 		currentContext = (ScannerContext) contextStack.pop();
 		return true;
 	}
 
 	public void addIncludePath(String includePath) {
-		includePaths.add(includePath);
+		includePathNames.add(includePath);
+		includePaths.add( new File( includePath ) );
 	}
 
 	public void overwriteIncludePath(List newIncludePaths) {
+		includePathNames = null;
 		includePaths = null; 
+		includePathNames = new ArrayList();
 		includePaths = new ArrayList(); 
-		includePaths.addAll(newIncludePaths);
+		includePathNames.addAll(newIncludePaths);
+		
+		Iterator i = includePathNames.iterator(); 
+		while( i.hasNext() )
+		{
+			String path = (String) i.next();
+			includePaths.add( new File( path )); 
+		}
+		
+		
 	}
 
 	public void addDefinition(String key, IMacroDescriptor macro) {
@@ -114,15 +138,20 @@ public class Scanner implements IScanner {
 	}
 
 	public final Object[] getIncludePaths() {
-		return includePaths.toArray();
+		return includePathNames.toArray();
 	}
 
-	protected void skipOverWhitespace() {
+	protected boolean skipOverWhitespace() {
 		int c = getChar();
+		boolean result = false; 
 		while ((c != NOCHAR) && ((c == ' ') || (c == '\t')))
+		{
 			c = getChar();
+			result = true; 
+		}
 		if (c != NOCHAR)
 			ungetChar(c);
+		return result; 
 
 	}
 
@@ -192,6 +221,12 @@ public class Scanner implements IScanner {
 		currentToken = t;
 	}
 
+	protected void resetStorageBuffer()
+	{
+		if( storageBuffer != null ) 
+			storageBuffer = null; 
+	}
+
 	protected Token newToken(int t, String i, IScannerContext c) {
 		setCurrentToken(new Token(t, i, c));
 		return currentToken;
@@ -225,56 +260,81 @@ public class Scanner implements IScanner {
 		return buffer.toString();
 	}
 
-	protected void handleInclusion(String fileName) throws ScannerException {
+	protected void handleInclusion(String fileName, boolean useIncludePaths ) throws ScannerException {
-		// Skip over inclusions in quickScan mode
-		if (quickScan)
-			return;
 
-		// iterate through the include paths 
+		if( useIncludePaths ) // search include paths for this file
-		Iterator iter = includePaths.iterator();
+		{
+			// iterate through the include paths 
+			Iterator iter = includePaths.iterator();
 	
-		while (iter.hasNext()) {
+			while (iter.hasNext()) {
-			String path = (String) iter.next();
 	
-			java.io.File pathFile = new java.io.File(path);
+				File pathFile = (File)iter.next();
-			if (pathFile.isDirectory()) {
+				if (pathFile.isDirectory()) {
-				String newPath = pathFile + "\\" + fileName;
+					String newPath = pathFile.getPath() + File.separatorChar + fileName;
 	
-				java.io.File includeFile = new java.io.File(newPath);
+					File includeFile = new File(newPath);
 	
-				if (includeFile.exists() && includeFile.isFile()) {
+					if (includeFile.exists() && includeFile.isFile()) {
-					try {
+						try {
-						FileReader inclusionReader =
+							FileReader inclusionReader =
-							new FileReader(includeFile);
+								new FileReader(includeFile);
-						//System.out.println( "Parsing inclusion file " + newPath );
+							updateContext(inclusionReader, newPath, ScannerContext.INCLUSION );
-						updateContext(inclusionReader, newPath);
+							return;
-						return;
+						} catch (FileNotFoundException fnf) {
-					} catch (FileNotFoundException fnf) {
+							// do nothing - check the next directory
-						// do nothing	
+						}
 					}
 				}
 			}
 		}
+		else // local inclusion
+		{
+			String currentFilename = currentContext.getFilename(); 
+			File currentIncludeFile = new File( currentFilename );
+			String parentDirectory = currentIncludeFile.getParent();
+			currentIncludeFile = null; 
+			String fullPath = parentDirectory + File.separatorChar + fileName;
+			File includeFile = new File( fullPath );
+			if (includeFile.exists() && includeFile.isFile()) {
+				try {
+					FileReader inclusionReader =
+						new FileReader(includeFile);
+					updateContext(inclusionReader, fullPath, ScannerContext.INCLUSION );
+					return;
+				} catch (FileNotFoundException fnf) {
+					if (throwExceptionOnInclusionNotFound)
+						throw new ScannerException("Cannot find inclusion " + fileName);	
+				}
+			}
+		}
+		
 		if (throwExceptionOnInclusionNotFound)
 			throw new ScannerException("Cannot find inclusion " + fileName);
 	}
 
 	// constants
-	public static final int NOCHAR = -1;
+	private static final int NOCHAR = -1;
 
 	private static final String TEXT = "<text>";
 	private static final String START = "<initial reader>";
 	private static final String EXPRESSION = "<expression>";
+	private static final String PASTING = "<pasting>";
 	private static final String BAD_PP =
 		"Invalid preprocessor directive encountered at offset ";
 	private static final String DEFINED = "defined";
 	private static final String POUND_DEFINE = "#define ";
 
+	public static final int tPOUNDPOUND = -6;
+
 	private IScannerContext currentContext;
 	private Stack contextStack = new Stack();
 
+	private List includePathNames = new ArrayList();
 	private List includePaths = new ArrayList();
 	private Hashtable definitions = new Hashtable();
+	private StringBuffer storageBuffer = null; 
+	private Set inclusions = new HashSet(); 
 	private int count = 0;
 	private static HashMap keywords = new HashMap();
 	private static HashMap ppDirectives = new HashMap();
@@ -295,7 +355,6 @@ public class Scanner implements IScanner {
 	private boolean throwExceptionOnUnboundedString = true;
 	private boolean throwExceptionOnEOFWithinMultilineComment = true;
 	private boolean throwExceptionOnEOFWithoutBalancedEndifs = true;
-	private boolean providedDefinedMacro = true;
 
 	private boolean quickScan = false;
 	public void setQuickScan(boolean qs) {
@@ -314,9 +373,8 @@ public class Scanner implements IScanner {
 		do {
 			done = true;
 
-			if (currentContext.getUndo() != NOCHAR) {
+			if (currentContext.undoStackSize() != 0 ) {
-				c = currentContext.getUndo();
+				c = currentContext.popUndo();
-				currentContext.setUndo(NOCHAR);
 			} else {
 				try {
 					c = currentContext.read();
@@ -354,10 +412,39 @@ public class Scanner implements IScanner {
 	private void ungetChar(int c) {
 		// Should really check whether there already is a char there
 		// If so, we should be using a buffer, instead of a single char
-		currentContext.setUndo(c);
+		currentContext.pushUndo(c);
 	}
 
+	protected boolean lookAheadForTokenPasting()
+	{
+		int c = getChar(); 
+		if( c == '#' )
+		{
+			c = getChar(); 
+			if( c == '#' )
+			{
+				return true; 
+			}
+			else
+			{
+				ungetChar( c );
+			}
+		}
+
+		ungetChar( c );
+		return false; 
+
+	}
+
+	
+
 	public Token nextToken() throws ScannerException {
+		return nextToken( true ); 
+	}
+
+
+	protected Token nextToken( boolean pasting ) throws ScannerException
+	{
 	
 		count++;
 
@@ -393,11 +480,8 @@ public class Scanner implements IScanner {
 
 				String ident = buff.toString();
 
-				if (providedDefinedMacro) {
+				if (ident.equals(DEFINED)) 
-					if (ident.equals(DEFINED)) {
+					return newToken(Token.tINTEGER, handleDefinedMacro());
-						return newToken(Token.tINTEGER, handleDefinedMacro());
-					}
-				}
 				
 				Object mapping = definitions.get(ident);
 
@@ -412,6 +496,31 @@ public class Scanner implements IScanner {
 				if (tokenTypeObject != null)
 					tokenType = ((Integer) tokenTypeObject).intValue();
 
+				if( pasting )
+				{
+					if( lookAheadForTokenPasting() )
+					{
+						if( storageBuffer == null )
+							storageBuffer = buff;
+						else
+							storageBuffer.append( ident ); 
+							 
+						c = getChar(); 
+						continue; 	
+					}
+					else
+					{
+						if( storageBuffer != null )
+						{
+							storageBuffer.append( ident );
+							updateContext( new StringReader( storageBuffer.toString()), PASTING, IScannerContext.MACROEXPANSION );
+							storageBuffer = null;  
+							c = getChar(); 
+							continue;
+						}
+					}
+				}
+
 				return newToken(tokenType, ident, currentContext);
 			} else if (c == '"') {
 				// string
@@ -436,7 +545,18 @@ public class Scanner implements IScanner {
 				}
 
 			} else if ((c >= '0') && (c <= '9')) {
-				StringBuffer buff = new StringBuffer();
+				StringBuffer buff;
+				
+				if( pasting )
+				{
+					if( storageBuffer != null )
+						buff = storageBuffer;
+					else
+						buff = new StringBuffer();
+				}
+				else
+					buff = new StringBuffer();
+					
 				buff.append((char) c);
 
 				c = getChar();
@@ -454,21 +574,52 @@ public class Scanner implements IScanner {
 				}
 
 				ungetChar(c);
+				
+				if( pasting )
+				{
+					if( lookAheadForTokenPasting() )
+					{
+						storageBuffer = buff; 
+						c = getChar(); 
+						continue; 	
+					}
+					else
+					{
+						if( storageBuffer != null )
+						{
+							updateContext( new StringReader( buff.toString()), PASTING, IScannerContext.MACROEXPANSION );
+							storageBuffer = null;  
+							c = getChar(); 
+							continue; 
+						}
+					}
+				}
+				
 				return newToken(
 					Token.tINTEGER,
 					buff.toString(),
 					currentContext);
+				
 			} else if (c == '#') {
 				// lets prepare for a preprocessor statement
 				StringBuffer buff = new StringBuffer();
 				buff.append((char) c);
 
 				// we are allowed arbitrary whitespace after the '#' and before the rest of the text
-				skipOverWhitespace();
+				boolean skipped = skipOverWhitespace();
 
 				c = getChar();
+				
+				if( c == '#' )
+				{
+					if( skipped )
+						throw new ScannerException(BAD_PP + currentContext.getOffset()); 
+					else 
+						return newToken( tPOUNDPOUND, "##" );
+				} 
+				
 				while (((c >= 'a') && (c <= 'z'))
-					|| ((c >= 'A') && (c <= 'Z')) | (c == '_')) {
+					|| ((c >= 'A') && (c <= 'Z')) || (c == '_') ) {
 					buff.append((char) c);
 					c = getChar();
 				}
@@ -1128,14 +1279,14 @@ public class Scanner implements IScanner {
 			}
 	
 			
-			if (expressionEvalResult.getClass() == java.lang.Integer.class) {
+			if (expressionEvalResult instanceof Integer ) {
 				int i = ((Integer) expressionEvalResult).intValue();
 				if (i == 0) {
 					return false;
 				}
 				return true;
 			} else if (
-				expressionEvalResult.getClass() == java.lang.Boolean.class) {
+				expressionEvalResult instanceof Boolean ) {
 				return ((Boolean) expressionEvalResult).booleanValue();
 			} else {
 				throw new ScannerException(
@@ -1189,6 +1340,7 @@ public class Scanner implements IScanner {
 		int offset;
 
 		StringBuffer fileName = new StringBuffer();
+		boolean useIncludePath = true;
 		if (c == '<') {
 			c = getChar();
 			while ((c != '>')) {
@@ -1202,7 +1354,7 @@ public class Scanner implements IScanner {
 				fileName.append((char) c);
 				c = getChar();
 			}
-
+			useIncludePath = false; 
 			// TO DO: Make sure the directory of the current file is in the
 			// inclusion paths.
 		}
@@ -1220,16 +1372,14 @@ public class Scanner implements IScanner {
 			}
 		}
 		else
-			handleInclusion(f.trim());
+			handleInclusion(f.trim(), useIncludePath );
 	}
 
 	protected void poundDefine() throws ScannerException {
 		skipOverWhitespace();
 		// definition 
 		String key = getNextIdentifier();
-		int offset = currentContext.getOffset() - key.length();
+		int offset = currentContext.getOffset() - key.length() - currentContext.undoStackSize();
-		if( currentContext.getUndo() != Scanner.NOCHAR )
-			offset -= 1;
 
 		if (throwExceptionOnRedefinition) {
 			String checkForRedefinition = (String) definitions.get(key);
@@ -1274,11 +1424,11 @@ public class Scanner implements IScanner {
 			helperScanner.initialize(
 				new StringReader(replacementString),
 				null);
-			Token t = helperScanner.nextToken();
+			Token t = helperScanner.nextToken(false);
 
 			while (t.type != Token.tEOF) {
 				macroReplacementTokens.add(t);
-				t = helperScanner.nextToken();
+				t = helperScanner.nextToken(false);
 			}
 
 			IMacroDescriptor descriptor = new MacroDescriptor();
@@ -1295,7 +1445,8 @@ public class Scanner implements IScanner {
 
 			// get what we are to map the name to and add it to the definitions list
 			String value = getRestOfPreprocessorLine();
-			addDefinition(key, value);
+			addDefinition( key, value ); 				
+		
 		} else if (c == '/') {
 			// this could be a comment	
 			c = getChar();
@@ -1333,12 +1484,10 @@ public class Scanner implements IScanner {
 
 	protected void expandDefinition(String symbol, Object expansion)
 		throws ScannerException {
-		if (expansion.getClass() == String.class) {
+		if (expansion instanceof String ) {
 			String replacementValue = (String) expansion;
-			updateContext(
+			updateContext( new StringReader(replacementValue), (POUND_DEFINE + symbol ), ScannerContext.MACROEXPANSION );
-				new StringReader(replacementValue),
+		} else if (expansion instanceof IMacroDescriptor ) {
-				POUND_DEFINE + symbol);
-		} else if (expansion.getClass() == MacroDescriptor.class) {
 			IMacroDescriptor macro = (IMacroDescriptor) expansion;
 			skipOverWhitespace();
 			int c = getChar();
@@ -1359,7 +1508,7 @@ public class Scanner implements IScanner {
 					buffer.append((char) c);
 					c = getChar();
 				}
-				String betweenTheBrackets = buffer.toString();
+				String betweenTheBrackets = buffer.toString().trim();
 				StringTokenizer tokenizer =
 					new StringTokenizer(betweenTheBrackets, ",");
 				Vector parameterValues = new Vector(tokenizer.countTokens());
@@ -1387,30 +1536,40 @@ public class Scanner implements IScanner {
 
 						// is this identifier in the parameterNames
 						// list? 
-
 						int index = parameterNames.indexOf(t.image);
 						if (index == -1 ) {
 							// not found
 							// just add image to buffer
-							buffer.append(t.image);
+							buffer.append(t.image );
 						} else {
 							buffer.append(
-								(String) parameterValues.elementAt(index));
+								(String) parameterValues.elementAt(index) );
 						}
 					} else {
 						buffer.append(t.image);
 					}
+					
+					boolean pastingNext = false;
+					
+					if( i != numberOfTokens - 1)
+					{
+						Token t2 = (Token) tokens.get(i+1);
+						if( t2.getType() == tPOUNDPOUND )
+							pastingNext = true;  
+					}
+					
+					if( t.getType() != tPOUNDPOUND && ! pastingNext )
+						buffer.append( " " ); 
 				}
+				String finalString = buffer.toString(); 
 				updateContext(
-					new StringReader(buffer.toString()),
+					new StringReader(finalString),
-					POUND_DEFINE + macro.getSignature());
+					POUND_DEFINE + macro.getSignature(), ScannerContext.MACROEXPANSION );
-			} else {
+			} else 
 				if (throwExceptionOnBadMacroExpansion)
 					throw new ScannerException(
 						"Improper use of macro " + symbol);
 
-			}
-
 		} else {
 			System.out.println(
 				"Unexpected class stored in definitions table. "

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ScannerContext.java

@@ -12,21 +12,23 @@ package org.eclipse.cdt.internal.core.parser;
 
 import java.io.IOException;
 import java.io.Reader;
+import java.util.Stack;
 
 public class ScannerContext implements IScannerContext
 {
 	private Reader reader;
 	private String filename;
 	private int offset;
-	private int undo; 
+	private Stack undo = new Stack(); 
+	private int kind; 
 				
 	public ScannerContext(){}
-	public IScannerContext initialize(Reader r, String f, int u )
+	public IScannerContext initialize(Reader r, String f, int k)
 	{
 		reader = r;
 		filename = f;
 		offset = 0;
-		undo = u;
+		kind = k; 
 		return this;
 	}
 		
@@ -62,22 +64,44 @@ public class ScannerContext implements IScannerContext
 		return reader;
 	}
 
+	public final int undoStackSize()
+	{
+		return undo.size();
+	}
+
 	/**
 	 * Returns the undo.
 	 * @return int
 	 */
-	public final int getUndo()
+	public final int popUndo()
 	{
-		return undo;
+		return ((Integer)undo.pop()).intValue();
 	}
 
 	/**
 	 * Sets the undo.
 	 * @param undo The undo to set
 	 */
-	public void setUndo(int undo)
+	public void pushUndo(int undo)
 	{
-		this.undo= undo;
+		this.undo.push( new Integer( undo )); 
+	}
+
+
+	/**
+	 * Returns the kind.
+	 * @return int
+	 */
+	public int getKind() {
+		return kind;
+	}
+
+	/**
+	 * Sets the kind.
+	 * @param kind The kind to set
+	 */
+	public void setKind(int kind) {
+		this.kind = kind;
 	}
 
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java

@@ -16,10 +16,7 @@ public class Token {
 		type = t;
 		image = i;
 		filename = context.getFilename();
-		offset = context.getOffset() - image.length();
+		offset = context.getOffset() - image.length() - context.undoStackSize();
-		
-		if( context.getUndo() != Scanner.NOCHAR )
-			offset -= 1;
 	}
 	
 	public Token(int t, String i) {

core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java

@@ -154,7 +154,6 @@ public class ScannerTestCase extends TestCase
 		"This statement should not be reached "
 			+ "as we sent in bad preprocessor input to the scanner";
 	public final static boolean verbose= false;
-	public final static boolean doConcatenation= false;
 	public final static boolean doIncludeStdio= false;
 	public final static boolean doIncludeWindowsH= false;
 	public final static boolean doIncludeWinUserH= false;
@@ -349,7 +348,7 @@ public class ScannerTestCase extends TestCase
 	public void prepareForWindowsRH()
 	{
 		scanner.addIncludePath(
-			"C:\\Program Files\\Microsoft Visual Studio .NET\\Vc7\\PlatformSDK\\include");
+			"C:\\Program Files\\Microsoft Visual Studio\\VC98\\Include");
 		scanner.addDefinition("_WIN32_WINNT", "0x0300");
 		scanner.addDefinition("WINVER", "0x0400");
 		scanner.addDefinition("_WIN32_WINDOWS", "0x0300");
@@ -359,51 +358,74 @@ public class ScannerTestCase extends TestCase
 	public void prepareForWindowsH()
 	{
 		scanner.addIncludePath(
-			"C:\\Program Files\\Microsoft Visual Studio .NET\\Vc7\\PlatformSDK\\include");
+			"C:\\Program Files\\Microsoft Visual Studio\\VC98\\Include");
-		scanner.addIncludePath(
-			"C:\\Program Files\\Microsoft Visual Studio .NET\\Vc7\\include");
 		scanner.addDefinition("_MSC_VER", "1200");
 		scanner.addDefinition("__cplusplus", "1");
 		scanner.addDefinition("__STDC__", "1");
 		scanner.addDefinition("_WIN32", "");		
 		scanner.addDefinition( "__midl", "1000" );
+		scanner.addDefinition("_WIN32_WINNT", "0x0300");
+		scanner.addDefinition("WINVER", "0x0400");
+		scanner.addDefinition( "_M_IX86", "300");
+		scanner.addDefinition( "_INTEGRAL_MAX_BITS", "64");
 	}
 
 	public void prepareForStdio()
 	{
 		scanner.addIncludePath(
-			"C:\\Program Files\\Microsoft Visual Studio .NET\\Vc7\\include");
+			"C:\\Program Files\\Microsoft Visual Studio\\VC98\\Include");
 		scanner.addDefinition("_MSC_VER", "1100");
 		scanner.addDefinition("__STDC__", "1");
 		scanner.addDefinition("_INTEGRAL_MAX_BITS", "64");
 		scanner.addDefinition("_WIN32", "");
+		scanner.addDefinition( "_M_IX86", "300");
 	}
 
 	public void testConcatenation()
 	{
-		if (doConcatenation)
+		try
 		{
-			try
+			initializeScanner("#define F1 3\n#define F2 F1##F1\nint x=F2;");
-			{
+			validateToken(Token.t_int);
-				initializeScanner("#define F1 3\n#define F2 F1##F1\nint x=F2;");
+			validateDefinition("F1", "3");
-				validateToken(Token.t_int);
+			validateDefinition( "F2", "F1##F1");
-				validateDefinition("F1", "3");
+			validateIdentifier("x");
-				validateDefinition("F2", "F1##F1");
+			validateToken(Token.tASSIGN);
-				validateIdentifier("x");
+			validateInteger("33");
-				validateToken(Token.tASSIGN);
+			validateToken(Token.tSEMI);
-				validateInteger("33");
+			validateEOF();
-				validateToken(Token.tSEMI);
-				validateEOF();
 			
-				initializeScanner("#define PREFIX RT_\n#define RUN PREFIX##Run"); 
+			initializeScanner("#define PREFIX RT_\n#define RUN PREFIX##Run"); 
-				validateEOF(); 
+			validateEOF(); 
-				validateDefinition( "PREFIX", "RT_" ); 
+			validateDefinition( "PREFIX", "RT_" ); 
-				validateDefinition( "RUN", "RT_Run" );
+			validateDefinition( "RUN", "PREFIX##Run" );
-			}
+		}
-			catch (Exception e)
+		catch (Exception e)
-			{
+		{
-				fail(EXCEPTION_THROWN + e.toString());
+			fail(EXCEPTION_THROWN + e.toString());
-			}
+		}
+		
+		try
+		{
+			initializeScanner( "#define DECLARE_HANDLE(name) struct name##__ { int unused; }; typedef struct name##__ *name\n DECLARE_HANDLE( joe )" );
+			validateToken( Token.t_struct );
+			validateIdentifier( "joe__"); 
+			validateToken( Token.tLBRACE);  
+			validateToken( Token.t_int ); 
+			validateIdentifier( "unused"); 
+			validateToken( Token.tSEMI ); 
+			validateToken( Token.tRBRACE );
+			validateToken( Token.tSEMI ); 
+			validateToken( Token.t_typedef ); 
+			validateToken( Token.t_struct ); 
+			validateIdentifier( "joe__" ); 
+			validateToken( Token.tSTAR ); 
+			validateIdentifier( "joe");  
+			validateEOF();
+		}
+		catch( Exception e )
+		{ 
+			fail(EXCEPTION_THROWN + e.toString());			
 		}
 	}