Footprint optimizations for ITokenDuple implementations.

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/SelectionParseBaseTest.java

@@ -18,7 +18,6 @@ import org.eclipse.cdt.core.parser.ParserLanguage;
 import org.eclipse.cdt.core.parser.ParserMode;
 import org.eclipse.cdt.core.parser.ScannerInfo;
 import org.eclipse.cdt.core.parser.ast.IASTNode;
-import org.eclipse.cdt.internal.core.parser.InternalParserUtil;
 
 /**
  * @author johnc
@@ -54,7 +53,7 @@ public class SelectionParseBaseTest extends CompleteParseBaseTest {
 							callback,
 							ParserMode.SELECTION_PARSE,
 							ParserLanguage.CPP,
-							InternalParserUtil.createDefaultLogService());
+							ParserFactory.createDefaultLogService());
 		
 		IParser.ISelectionParseResult result =parser.parse( offset1, offset2 );
 		if( expectedToPass )

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/SelectionParseTest.java

@@ -272,7 +272,6 @@ public class SelectionParseTest extends SelectionParseBaseTest {
 		writer.write( "foo(1); \n }" ); //$NON-NLS-1$
 		String code = writer.toString();
 		int startIndex = code.indexOf( "foo(1)"); //$NON-NLS-1$
-		IASTNode node = parse( code, startIndex, startIndex + 3 );
+		parse( code, startIndex, startIndex + 3 );
-		
 	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/ITokenDuple.java

@@ -28,7 +28,6 @@ public interface ITokenDuple {
 	public abstract IToken getLastToken();
 	
 	public List [] getTemplateIdArgLists();
-	public IToken consumeTemplateIdArguments( IToken name, Iterator iter );
 	
 	public ITokenDuple getLastSegment();
 	public ITokenDuple getLeadingSegments();

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java

@@ -41,7 +41,6 @@ import org.eclipse.cdt.core.parser.ast.IASTCompletionNode.CompletionKind;
 import org.eclipse.cdt.core.parser.ast.IASTExpression.Kind;
 import org.eclipse.cdt.core.parser.extension.IParserExtension;
 import org.eclipse.cdt.internal.core.parser.token.KeywordSets;
-import org.eclipse.cdt.internal.core.parser.token.TokenDuple;
 import org.eclipse.cdt.internal.core.parser.token.TokenFactory;
 import org.eclipse.cdt.internal.core.parser.token.KeywordSets.Key;
 import org.eclipse.cdt.internal.core.parser.util.TraceUtil;
@@ -1711,7 +1710,7 @@ public class ExpressionParser implements IExpressionParser, IParserData {
 	    {
 			String signature = "";//$NON-NLS-1$
 			if( lastToken != null )
-				signature = TokenDuple.createStringRepresentation(mark, lastToken);
+				signature = TokenFactory.createStringRepresentation(mark, lastToken);
 	        return astFactory.createTypeId( scope, kind, isConst, isVolatile, isShort, isLong, isSigned, isUnsigned, isTypename, name, id.getPointerOperators(), id.getArrayModifiers(), signature);
 	    }
 	    catch (ASTSemanticException e)

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ast/complete/CompleteParseASTFactory.java

@@ -98,6 +98,7 @@ import org.eclipse.cdt.internal.core.parser.pst.TemplateSymbolExtension;
 import org.eclipse.cdt.internal.core.parser.pst.TypeInfo;
 import org.eclipse.cdt.internal.core.parser.pst.ISymbolASTExtension.ExtensionException;
 import org.eclipse.cdt.internal.core.parser.pst.TypeInfo.PtrOp;
+import org.eclipse.cdt.internal.core.parser.token.TokenFactory;
 import org.eclipse.cdt.internal.core.parser.util.TraceUtil;
 
 
@@ -364,7 +365,7 @@ public class CompleteParseASTFactory extends BaseASTFactory implements IASTFacto
 					
 					if( templateArgLists != null && templateArgLists[ idx ] != null ){
 						if( iter.hasNext() && t.getNext().getType() == IToken.tLT )
-							t = name.consumeTemplateIdArguments( (IToken) iter.next(), iter );
+							t = TokenFactory.consumeTemplateIdArguments( (IToken) iter.next(), iter );
 					}
 					
 					try
@@ -1936,7 +1937,7 @@ public class CompleteParseASTFactory extends BaseASTFactory implements IASTFacto
 				
 				if( argLists != null && argLists[ idx ] != null ){
 					if( i.hasNext() && current.getNext().getType() == IToken.tLT )
-						current = typeName.consumeTemplateIdArguments( (IToken) i.next(), i );
+						current = TokenFactory.consumeTemplateIdArguments( (IToken) i.next(), i );
 				}
 				
 				if( typeSymbol instanceof IDeferredTemplateInstance ){

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/AbstractToken.java

@@ -56,7 +56,6 @@ public abstract class AbstractToken implements IToken, ITokenDuple {
 	protected int type;
 	protected int lineNumber = 1;
 	protected IToken next = null;
-	private String [] qualifiedName = null;
 
 	/* (non-Javadoc)
 	 * @see java.lang.Object#equals(java.lang.Object)
@@ -172,12 +171,6 @@ public abstract class AbstractToken implements IToken, ITokenDuple {
 	public final IToken getNext() { return next; }
 	public void setNext(IToken t) { next = t; }
 	
-	/* (non-Javadoc)
-	 * @see org.eclipse.cdt.core.parser.ITokenDuple#consumeTemplateIdArguments(org.eclipse.cdt.core.parser.IToken, java.util.Iterator)
-	 */
-	public IToken consumeTemplateIdArguments(IToken name, Iterator iter) {
-		return this;
-	}
 	/* (non-Javadoc)
 	 * @see org.eclipse.cdt.core.parser.ITokenDuple#contains(org.eclipse.cdt.core.parser.ITokenDuple)
 	 */
@@ -310,11 +303,8 @@ public abstract class AbstractToken implements IToken, ITokenDuple {
 	 * @see org.eclipse.cdt.core.parser.ITokenDuple#toQualifiedName()
 	 */
 	public String[] toQualifiedName() {
-		if( qualifiedName == null )
+		String [] qualifiedName = new String[1];
-		{
-			qualifiedName = new String[1];
 		qualifiedName[0] = getImage();
-		}
 		return qualifiedName;
 	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/BasicTokenDuple.java

@@ -12,7 +12,6 @@ package org.eclipse.cdt.internal.core.parser.token;
 
 import java.util.ArrayList;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.NoSuchElementException;
 
@@ -23,50 +22,24 @@ import org.eclipse.cdt.core.parser.ITokenDuple;
  * @author jcamelon
  *
  */
-public class TokenDuple implements ITokenDuple {
+public class BasicTokenDuple implements ITokenDuple {
 
-	TokenDuple( IToken first, IToken last )
+	BasicTokenDuple( IToken first, IToken last )
 	{
 //		assert ( first != null && last != null ) : this; 
 		firstToken = first; 
 		lastToken = last; 
-		argLists = null;
 	}
 	
-	TokenDuple( IToken first, IToken last, List templateArgLists ){
+	//TODO - move numSegments to a subclass
-		firstToken = first;
+	private int numSegments = -1;
-		lastToken = last;
-		if( templateArgLists != null && !templateArgLists.isEmpty() ){
-			argLists = (List[]) templateArgLists.toArray( new List [templateArgLists.size()] );
-		} else {
-			argLists = null;
-		}
-	}
 	
-	TokenDuple( ITokenDuple firstDuple, ITokenDuple secondDuple ){
+	BasicTokenDuple( ITokenDuple firstDuple, ITokenDuple secondDuple ){
-		firstToken = firstDuple.getFirstToken();
+		this( firstDuple.getFirstToken(), secondDuple.getLastToken() );
-		lastToken = secondDuple.getLastToken();
-		
-		List [] a1 = firstDuple.getTemplateIdArgLists();
-		List [] a2 = secondDuple.getTemplateIdArgLists();
-		
-		if( a1 == null && a2 == null ){
-			argLists = null;
-		} else {
-			int l1 = ( a1 != null ) ? a1.length : firstDuple.getSegmentCount();
-			int l2 = ( a2 != null ) ? a2.length : firstDuple.getSegmentCount();
-
-			argLists = new List[ l1 + l2 ];
-			if( a1 != null )
-				System.arraycopy( a1, 0, argLists, 0, l1 );
-			if( a2 != null )
-				System.arraycopy( a2, 0, argLists, l1, l2 );
-		}
 	}
 	
 	protected final IToken firstToken, lastToken;
-	protected final List [] argLists;
+
-	private int numSegments = -1;
 	
 	/**
 	 * @return
@@ -87,10 +60,10 @@ public class TokenDuple implements ITokenDuple {
 		return new TokenIterator(); 
 	}
 	
-	public ITokenDuple getLastSegment() {
+	
+	public ITokenDuple getLastSegment()
+	{
 		Iterator iter = iterator();
-		if( !iter.hasNext() )
-			return null;
 		
 		IToken first = null, last = null, token = null;
 		while( iter.hasNext() ){
@@ -98,7 +71,7 @@ public class TokenDuple implements ITokenDuple {
 			if( first == null )
 				first = token;
 			if( token.getType() == IToken.tLT )
-				token = consumeTemplateIdArguments( token, iter );
+				token = TokenFactory.consumeTemplateIdArguments( token, iter );
 			else if( token.getType() == IToken.tCOLONCOLON ){
 				first = null;
 				continue;
@@ -116,6 +89,8 @@ public class TokenDuple implements ITokenDuple {
 		
 	}
 
+	
+	
 	public ITokenDuple getLeadingSegments(){
 		Iterator iter = iterator();
 		if( !iter.hasNext() )
@@ -134,12 +109,11 @@ public class TokenDuple implements ITokenDuple {
 			if( first == null )
 				first = token;
 			if( token.getType() == IToken.tLT )
-				token = consumeTemplateIdArguments( token, iter );
+				token = TokenFactory.consumeTemplateIdArguments( token, iter );
 			else if( token.getType() == IToken.tCOLONCOLON ){
 				last = previous;
 				continue;
 			}
-			
 			previous = token;
 		}
 		
@@ -162,82 +136,21 @@ public class TokenDuple implements ITokenDuple {
 		return TokenFactory.createTokenDuple( first, last );
 	}
 	
-	public int getSegmentCount()
+	/* (non-Javadoc)
-	{
+	 * @see org.eclipse.cdt.core.parser.ITokenDuple#getSegmentCount()
-		if( numSegments > -1 )
+	 */
-			return numSegments;
+	public int getSegmentCount() {
-		
+		if( numSegments == -1 )
-		numSegments = 1;
+			numSegments = calculateSegmentCount();
-		
-		if( firstToken == lastToken )
-			return numSegments;
-		
-		Iterator iter = iterator();
-		
-		IToken token = null;
-		while( iter.hasNext() ){
-			token = (IToken) iter.next();
-			if( token.getType() == IToken.tLT )
-				token = consumeTemplateIdArguments( token, iter );
-			if( token.getType() == IToken.tCOLONCOLON ){
-				numSegments++;
-				continue;
-			}
-		}
 		return numSegments;
 	}	
 	
-	private static final Integer LT = new Integer( IToken.tLT );
-	private static final Integer LBRACKET = new Integer( IToken.tLBRACKET );
-	private static final Integer LPAREN = new Integer( IToken.tLPAREN );
-	private String [] qualifiedName = null;
 	private static final String EMPTY_STRING = ""; //$NON-NLS-1$
 	private String stringRepresentation = null;
 	
-	public IToken consumeTemplateIdArguments( IToken name, Iterator iter ){
-	    IToken token = name;
-	    
-	    if( token.getType() == IToken.tLT )
-	    {
-	    	if( ! iter.hasNext() )
-	    		return token;
-	    	
-	    	LinkedList scopes = new LinkedList();
-	        scopes.add( LT );
-	        
-	        while (!scopes.isEmpty() && iter.hasNext() )
-	        {
-	        	Integer top;
-	        	
-	        	token = (IToken) iter.next();
-	        	switch( token.getType() ){
-	        		case IToken.tGT:
-	        			if( scopes.getLast() == LT ) {
-							scopes.removeLast();
-						}
-	                    break;
-	        		case IToken.tRBRACKET :
-						do {
-							top = (Integer)scopes.removeLast();
-						} while (!scopes.isEmpty() && top == LT);
-						break;
-	        		case IToken.tRPAREN :
-						do {
-							top = (Integer)scopes.removeLast();
-						} while (!scopes.isEmpty() && top == LT);
-						break;
-	                case IToken.tLT:		scopes.add( LT );		break;
-					case IToken.tLBRACKET:	scopes.add( LBRACKET );	break;
-					case IToken.tLPAREN:	scopes.add( LPAREN );   break;
-	        	}
-	        }
-	    }
-	   
-	    return token;
-	}
 	private class TokenIterator implements Iterator
 	{
-		private IToken iter = TokenDuple.this.firstToken;
+		private IToken iter = firstToken;
 
 		/* (non-Javadoc)
 		 * @see java.util.Iterator#hasNext()
@@ -315,12 +228,12 @@ public class TokenDuple implements ITokenDuple {
      */
     public int length()
     {
-        int count = 0; 
+        int count = 1; 
-        Iterator i = iterator();
+        IToken i = firstToken;
-        while( i.hasNext() )
+        while( i != lastToken )
         {
         	++count;
-        	i.next();
+        	i = i.getNext();
         }
         return count;
     }
@@ -338,14 +251,15 @@ public class TokenDuple implements ITokenDuple {
      */
     public IToken getToken(int index)
     {
-        if( index < 0 || index >= length() ) return null;
+        if( index < 0 ) return null;
-        Iterator i = iterator();
+        
+        IToken iter = firstToken;
         int count = 0;
-        while( i.hasNext() )
+        while( iter != lastToken )
         {
-        	IToken r = (IToken)i.next();
+        	iter = iter.getNext();
         	if( count == index )
-        		return r;
+        		return iter;
         	++count; 
         }
         return null;
@@ -358,13 +272,13 @@ public class TokenDuple implements ITokenDuple {
     {
 		int count = 0; 
 		int lastFound = -1;
-        Iterator i = iterator();
+        IToken i = firstToken;
-        while( i.hasNext() )
+        while( i != lastToken )
         {
-        	IToken token = (IToken)i.next();
+        	if( i.getType() == type )
-        	if( token.getType() == type )
         		lastFound = count; 
         	++count;
+        	i = i.getNext();
         }
         
         return lastFound;
@@ -393,25 +307,28 @@ public class TokenDuple implements ITokenDuple {
 	 * @see org.eclipse.cdt.core.parser.ITokenDuple#getTemplateIdArgLists()
 	 */
 	public List[] getTemplateIdArgLists() {
-		return argLists;
+		return null;
 	}
 
 	/* (non-Javadoc)
 	 * @see org.eclipse.cdt.core.parser.ITokenDuple#syntaxOfName()
 	 */
 	public boolean syntaxOfName() {
-		Iterator iter = iterator();
+		IToken iter = firstToken;
-		if( ! iter.hasNext() ) return false; // empty is not good
+		while( iter != lastToken)
-		while( iter.hasNext() )
 		{
-			IToken token = (IToken) iter.next();
+			if( iter.isOperator() )
-			if( token.isOperator() ) continue;
+			{
-			switch( token.getType() )
+				iter = iter.getNext();
+				continue;
+			}
+			switch( iter.getType() )
 			{
 				case IToken.tCOMPL:
 				case IToken.tIDENTIFIER:
 				case IToken.tCOLONCOLON:
 				case IToken.t_operator:
+					iter = iter.getNext();
 					continue;
 				default:
 					return false;
@@ -438,7 +355,7 @@ public class TokenDuple implements ITokenDuple {
     	Iterator i = nameDuple.iterator();
     	
     	if( !i.hasNext() )
-    		return "";//$NON-NLS-1$
+    		return EMPTY_STRING;
     	
     	StringBuffer nameBuffer = new StringBuffer();
     	IToken token = (IToken) i.next();
@@ -510,39 +427,63 @@ public class TokenDuple implements ITokenDuple {
 	 * @see org.eclipse.cdt.core.parser.ITokenDuple#toQualifiedName()
 	 */
 	public String[] toQualifiedName() {
-		if( qualifiedName == null )
+		return generateQualifiedName();
-			generateQualifiedName();
-		return qualifiedName;
 	}
 
 	/**
 	 * 
 	 */
-	private void generateQualifiedName() {
+	private String [] generateQualifiedName() {
 		List qn = new ArrayList();
-		Iterator i = iterator();
+		IToken i = firstToken;
-		while( i.hasNext() )
+		while( i != lastToken )
 		{
-			IToken t = (IToken) i.next();
 			boolean compl = false;
-			if( t.getType() == IToken.tCOLONCOLON ) continue;
+			if( i.getType() == IToken.tCOLONCOLON )
-			if( t.getType() == IToken.tCOMPL )
+			{
+				i = i.getNext();
+				continue;
+			}
+			if( i.getType() == IToken.tCOMPL )
 			{
 				compl = true;
-				if( !i.hasNext() ) break;
+				i = i.getNext();
-				t = (IToken) i.next();
 			}
-			if( t.getType() == IToken.tIDENTIFIER )
+			if( i.getType() == IToken.tIDENTIFIER )
 			{
 				if( compl )
-					qn.add( "~" + t.getImage() ); //$NON-NLS-1$
+				{
+					StringBuffer buff = new StringBuffer( "~" ); //$NON-NLS-1$
+					buff.append( i.getImage() );
+					qn.add(  buff.toString() ); 
+				}
 				else
-					qn.add( t.getImage() );
+					qn.add( i.getImage() );
 			}
+			i = i.getNext();
+		}
+		String [] qualifiedName = new String[ qn.size() ];
+		return (String[]) qn.toArray( qualifiedName );
 	}
-		qualifiedName = new String[ qn.size() ];
-		qualifiedName = (String[]) qn.toArray( qualifiedName );
 
+	/**
+	 * 
+	 */
+	protected int calculateSegmentCount() {
+		int n = 1;
+		Iterator iter = iterator();
+		
+		IToken token = null;
+		while( iter.hasNext() ){
+			token = (IToken) iter.next();
+			if( token.getType() == IToken.tLT )
+				token = TokenFactory.consumeTemplateIdArguments( token, iter );
+			if( token.getType() == IToken.tCOLONCOLON  ){
+				n++;
+				continue;
+			}
+		}
+		return n;
 	}
 
 	

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SegmentedTokenDuple.java

@@ -0,0 +1,39 @@
+/**********************************************************************
+ * Copyright (c) 2002-2004 Rational Software Corporation and others.
+ * All rights reserved.   This program and the accompanying materials
+ * are made available under the terms of the Common Public License v0.5
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors: 
+ * IBM Rational Software - Initial API and implementation
+***********************************************************************/
+package org.eclipse.cdt.internal.core.parser.token;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.eclipse.cdt.core.parser.IToken;
+import org.eclipse.cdt.core.parser.ITokenDuple;
+
+/**
+ * @author jcamelon
+ *
+ */
+public class SegmentedTokenDuple extends BasicTokenDuple {
+
+	/**
+	 * @param first
+	 * @param last
+	 */
+	public SegmentedTokenDuple(IToken first, IToken last) {
+		super(first, last);
+		numSegments = calculateSegmentCount();
+	}
+	
+	private final int numSegments;
+	
+
+
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TemplateTokenDuple.java

@@ -0,0 +1,98 @@
+/**********************************************************************
+ * Copyright (c) 2002-2004 Rational Software Corporation and others.
+ * All rights reserved.   This program and the accompanying materials
+ * are made available under the terms of the Common Public License v0.5
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors: 
+ * IBM Rational Software - Initial API and implementation
+***********************************************************************/
+package org.eclipse.cdt.internal.core.parser.token;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.eclipse.cdt.core.parser.IToken;
+import org.eclipse.cdt.core.parser.ITokenDuple;
+
+/**
+ * @author jcamelon
+ *
+ */
+public class TemplateTokenDuple extends BasicTokenDuple {
+
+	protected final List [] argLists;
+	private final int numSegments;
+
+	/**
+	 * @param first
+	 * @param last
+	 * @param templateArgLists
+	 */
+	public TemplateTokenDuple(IToken first, IToken last, List templateArgLists) {
+		super(first, last);
+		argLists = (List[]) templateArgLists.toArray( new List [templateArgLists.size()] );
+		numSegments = calculateSegmentCount();
+	}
+	
+	
+	
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.core.parser.ITokenDuple#getSegmentCount()
+	 */
+	public int getSegmentCount() {
+		return numSegments;
+	}
+	public ITokenDuple getLastSegment()
+	{
+		Iterator iter = iterator();
+		
+		IToken first = null, last = null, token = null;
+		while( iter.hasNext() ){
+			token = (IToken) iter.next();
+			if( first == null )
+				first = token;
+			if( token.getType() == IToken.tLT )
+				token = TokenFactory.consumeTemplateIdArguments( token, iter );
+			else if( token.getType() == IToken.tCOLONCOLON ){
+				first = null;
+				continue;
+			}
+			last = token;
+		}
+		
+		List [] args = getTemplateIdArgLists();
+		if( args != null && args[ args.length - 1 ] != null ){
+			List newArgs = new ArrayList( 1 );
+			newArgs.add( args[ args.length - 1 ] );
+			return TokenFactory.createTokenDuple( first, last, newArgs );
+		} 
+		return TokenFactory.createTokenDuple( first, last );
+		
+	}
+
+	public TemplateTokenDuple( ITokenDuple first, ITokenDuple last )
+	{
+		super( first, last );
+		List [] a1 = first.getTemplateIdArgLists();
+		List [] a2 = last.getTemplateIdArgLists();
+		
+		int l1 = ( a1 != null ) ? a1.length : first.getSegmentCount();
+		int l2 = ( a2 != null ) ? a2.length : first.getSegmentCount();
+		argLists = new List[ l1 + l2 ];
+		if( a1 != null )
+			System.arraycopy( a1, 0, argLists, 0, l1 );
+		if( a2 != null )
+			System.arraycopy( a2, 0, argLists, l1, l2 );
+		numSegments = calculateSegmentCount();
+	}
+
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.core.parser.ITokenDuple#getTemplateIdArgLists()
+	 */
+	public List[] getTemplateIdArgLists() {
+		return argLists;
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenFactory.java

@@ -10,6 +10,8 @@
 ***********************************************************************/
 package org.eclipse.cdt.internal.core.parser.token;
 
+import java.util.Arrays;
+import java.util.Iterator;
 import java.util.List;
 
 import org.eclipse.cdt.core.parser.IToken;
@@ -22,11 +24,6 @@ import org.eclipse.cdt.internal.core.parser.scanner.IScannerData;
  */
 public class TokenFactory {
 		
-	static final String MAX_LONG_STRING = Long.toString( Long.MAX_VALUE );
-	static final int MAX_LONG_STRING_LENGTH = MAX_LONG_STRING.length();
-	static final String MAX_HEX_LONG_STRING = "0x" + Long.toString( Long.MAX_VALUE, 16 ); //$NON-NLS-1$
-	static final int MAX_HEX_LONG_STRING_LENGTH = MAX_HEX_LONG_STRING.length();	
-	
 	public static IToken createToken( int tokenType, IScannerData scannerData )
 	{
 		if( scannerData.getContextStack().getCurrentContext().getKind() == IScannerContext.ContextKind.MACROEXPANSION )
@@ -56,16 +53,187 @@ public class TokenFactory {
 	public static ITokenDuple createTokenDuple( IToken first, IToken last )
 	{
 		if( (first == last) && ( first instanceof ITokenDuple )) return (ITokenDuple) first;
-		return new TokenDuple( first, last );
+		return new BasicTokenDuple( first, last );
 	}
 
 	public static ITokenDuple createTokenDuple( IToken first, IToken last, List templateArgLists )
 	{
 		if( (first == last) && ( templateArgLists == null ) && ( first instanceof ITokenDuple )) return (ITokenDuple) first;
-		return new TokenDuple( first, last, templateArgLists );
+		if( templateArgLists != null && !templateArgLists.isEmpty())
+			return new TemplateTokenDuple( first, last, templateArgLists );
+		return new BasicTokenDuple( first, last );
 	}
 
 	public static ITokenDuple createTokenDuple( ITokenDuple firstDuple, ITokenDuple secondDuple ){
-		return new TokenDuple( firstDuple, secondDuple );
+		List [] f1 = firstDuple.getTemplateIdArgLists();
+		List [] f2 = secondDuple.getTemplateIdArgLists();
+		if( f1 == null && f2 == null )
+			return new BasicTokenDuple( firstDuple, secondDuple );
+		return new TemplateTokenDuple( firstDuple, secondDuple );
+	}
+	
+	public static IToken consumeTemplateIdArguments( IToken name, Iterator iter ){
+	    IToken token = name;
+	    
+	    if( token.getType() == IToken.tLT )
+	    {
+	    	if( ! iter.hasNext() )
+	    		return token;
+	    		    	
+	    	BraceCounter scopes = BraceCounter.getCounter();
+	    	try
+			{		    	
+		        scopes.addValue( IToken.tLT );
+		        
+		        while (!scopes.isEmpty() && iter.hasNext() )
+		        {
+		        	int top;
+		        	
+		        	token = (IToken) iter.next();
+		        	switch( token.getType() ){
+		        		case IToken.tGT:
+		        			if( scopes.getLast() == IToken.tLT ) {
+								scopes.removeValue();
+							}
+		                    break;
+		        		case IToken.tRBRACKET :
+							do {
+								top = scopes.removeValue();
+							} while (!scopes.isEmpty() && top == IToken.tLT);
+							break;
+		        		case IToken.tRPAREN :
+							do {
+								top = scopes.removeValue();
+							} while (!scopes.isEmpty() && top == IToken.tLT);
+							break;
+		                case IToken.tLT:		scopes.addValue( IToken.tLT );		break;
+						case IToken.tLBRACKET:	scopes.addValue( IToken.tLBRACKET );	break;
+						case IToken.tLPAREN:	scopes.addValue( IToken.tLPAREN );   break;
+		        	}
+		        }
+			}
+	    	finally
+			{
+	    		BraceCounter.returnCounter(scopes);
+			}
+	    }
+	   
+	    return token;
+	}
+	
+	protected static class BraceCounter
+	{
+		private static final int POOLSIZE = 8;
+		private static final BraceCounter [] pool;
+		private static final boolean [] free;
+		private static int newObjectCount = POOLSIZE;
+
+		static
+		{
+			pool = new BraceCounter[ POOLSIZE ];
+			free = new boolean[8];
+			for( int i = 0; i < POOLSIZE; ++i )
+			{
+				pool[i] = new BraceCounter(i);
+				free[i] = true;
+			}
+		}	
+
+		public static synchronized BraceCounter getCounter()
+		{
+			for( int i = 0; i < POOLSIZE; ++i )
+			{
+				if( free[i] )
+				{
+					free[i] = false;
+					return pool[i];
+				}
+			}
+			//if there is nothing free, allocate a new one and return it
+			return new BraceCounter(newObjectCount++);
+		}
+		
+		public static synchronized void returnCounter( BraceCounter c )
+		{
+			if( c.getKey() > 0 && c.getKey() < POOLSIZE )
+			{
+				free[ c.getKey() ] = true;
+				c.clear();
+			}
+			// otherwise, the object shall get garbage collected
+		}
+
+		/**
+		 * 
+		 */
+		private void clear() {
+			currentIndex = 0;
+			Arrays.fill( array, 0, array.length, -1 );
+		}
+
+		private final int key;
+		private int [] array = new int[ 8 ];
+		int currentIndex = 0;
+		
+		private void resizeArray()
+		{
+			int [] newArray = new int[ (array.length * 2) ];
+			System.arraycopy( array, 0, newArray, 0, array.length );
+			array = newArray;
+		}
+		
+		public void addValue( int value )
+		{
+			if( currentIndex == array.length )
+				resizeArray();
+			array[currentIndex] = value;
+			++currentIndex;
+		}
+		
+		public int removeValue()
+		{
+			int result = array[currentIndex];
+			array[currentIndex] = -1;
+			--currentIndex;
+			return result;
+		}
+		
+		public int getLast()
+		{
+			if( isEmpty() ) return -1;
+			return array[ currentIndex - 1 ];
+		}
+		
+		/**
+		 * @return
+		 */
+		public boolean isEmpty() {
+			return (currentIndex == 0 );
+		}
+
+		/**
+		 * @param i
+		 */
+		public BraceCounter(int i) {
+			key = i; 
+			clear();
+		}
+		/**
+		 * @return Returns the key.
+		 */
+		public int getKey() {
+			return key;
+		}
+		
+		
+	}
+
+	/**
+	 * @param first
+	 * @param last
+	 * @return
+	 */
+	public static String createStringRepresentation(IToken first, IToken last) {
+		return BasicTokenDuple.createStringRepresentation(first, last);
 	}
 }